activate new script
diff --git a/buildkite/pipelines/bazelci.py b/buildkite/pipelines/bazelci.py
index 34fb04b..9005167 100644
--- a/buildkite/pipelines/bazelci.py
+++ b/buildkite/pipelines/bazelci.py
@@ -1,3 +1,4 @@
+from __future__ import print_function
import argparse
import codecs
import json
@@ -5,95 +6,251 @@
import shutil
import subprocess
import sys
+import stat
import tempfile
import urllib.request
from shutil import copyfile
from urllib.parse import urlparse
+def downstream_projects():
+ return {
+ "BUILD_file_generator" : {
+ "git_repository" : "https://github.com/bazelbuild/BUILD_file_generator.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/BUILD_file_generator-postsubmit.json"
+ },
+ "buildtools" : {
+ "git_repository" : "https://github.com/bazelbuild/buildtools.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/buildtools-postsubmit.json"
+ },
+ "examples" : {
+ "git_repository" : "https://github.com/bazelbuild/examples.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/examples-postsubmit.json"
+ },
+ "migration-tooling" : {
+ "git_repository" : "https://github.com/bazelbuild/migration-tooling.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/migration-tooling-postsubmit.json"
+ },
+ "protobuf" : {
+ "git_repository" : "https://github.com/google/protobuf.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/protobuf-postsubmit.json"
+ },
+ "re2" : {
+ "git_repository" : "https://github.com/google/re2.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/re2-postsubmit.json"
+ },
+ "rules_appengine" : {
+ "git_repository" : "https://github.com/bazelbuild/rules_appengine.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_appengine-postsubmit.json"
+ },
+ "rules_closure" : {
+ "git_repository" : "https://github.com/bazelbuild/rules_closure.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_closure-postsubmit.json"
+ },
+ "rules_go" : {
+ "git_repository" : "https://github.com/bazelbuild/rules_go.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_go-postsubmit.json"
+ },
+ "rules_groovy" : {
+ "git_repository" : "https://github.com/bazelbuild/rules_groovy.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_groovy-postsubmit.json"
+ },
+ "rules_gwt" : {
+ "git_repository" : "https://github.com/bazelbuild/rules_gwt.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_gwt-postsubmit.json"
+ },
+ "rules_jsonnet" : {
+ "git_repository" : "https://github.com/bazelbuild/rules_jsonnet.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_jsonnet-postsubmit.json"
+ },
+ "rules_k8s" : {
+ "git_repository" : "https://github.com/bazelbuild/rules_k8s.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_k8s-postsubmit.json"
+ },
+ "rules_nodejs" : {
+ "git_repository" : "https://github.com/bazelbuild/rules_nodejs.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_nodejs-postsubmit.json"
+ },
+ "rules_perl" : {
+ "git_repository" : "https://github.com/bazelbuild/rules_perl.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_perl-postsubmit.json"
+ },
+ "rules_python" : {
+ "git_repository" : "https://github.com/bazelbuild/rules_python.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_python-postsubmit.json"
+ },
+ "rules_scala" : {
+ "git_repository" : "https://github.com/bazelbuild/rules_scala.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_scala-postsubmit.json"
+ },
+ "rules_typescript" : {
+ "git_repository" : "https://github.com/bazelbuild/rules_typescript.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_typescript-postsubmit.json"
+ },
+ "skydoc" : {
+ "git_repository" : "https://github.com/bazelbuild/skydoc.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/skydoc-postsubmit.json"
+ },
+ "subpar" : {
+ "git_repository" : "https://github.com/google/subpar.git",
+ "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/subpar-postsubmit.json"
+ }
+ }
+
+def python_binary():
+ return "python3.6"
+
+def bazelcipy_url():
+ '''
+ URL to the latest version of this script.
+ '''
+ return "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/bazelci1.py"
+
+def eprint(*args, **kwargs):
+ '''
+ Print to stderr and exit the process.
+ '''
+ print(*args, file=sys.stderr, **kwargs)
+ exit(1)
+
def supported_platforms():
+ '''
+ Returns a map containing all supported platform names as keys, with the
+ values being the platform name in a human readable format.
+ '''
return {
"ubuntu1404" : "Ubuntu 14.04",
"ubuntu1604" : "Ubuntu 16.04",
"macos" : "macOS"
}
-def fetch_configs(http_config):
- if http_config is None:
+def git_clone_path():
+ return "project-under-test"
+
+def fetch_configs(http_url):
+ '''
+ If specified fetches the build configuration from http_url, else tries to
+ read it from .bazelci/config.json.
+ Returns the json configuration as a python data structure.
+ '''
+ if http_url is None:
with open(".bazelci/config.json", "r") as fd:
return json.load(fd)
- with urllib.request.urlopen(http_config) as resp:
+ with urllib.request.urlopen(http_url) as resp:
reader = codecs.getreader("utf-8")
return json.load(reader(resp))
-def run(config, platform, bazel_binary, git_repository):
- exit_code = 0
+def execute_commands(config, platform, git_repository, use_but, save_but,
+ build_only, test_only):
+ exit_code = -1
tmpdir = None
+ bazel_binary = "bazel"
try:
tmpdir = tempfile.mkdtemp()
+ if use_but:
+ source_step = create_label(supported_platforms()[platform], "Bazel",
+ build_only=True, test_only=False)
+ bazel_binary = download_bazel_binary(tmpdir, source_step)
if git_repository:
- if os.path.exists("downstream-repo"):
- shutil.rmtree("downstream-repo")
- clone_repository(git_repository)
+ clone_git_repository(git_repository)
+ os.chdir(git_clone_path())
cleanup(bazel_binary)
else:
cleanup(bazel_binary)
- shell_commands(config.get("shell_commands", None))
- bazel_run(bazel_binary, config.get("run_targets", None))
- bazel_build(bazel_binary, config.get("build_flags", []), config.get("build_targets", None))
- bep_file = os.path.join(tmpdir, "build_event_json_file.json")
- exit_code = bazel_test(bazel_binary, config.get("test_flags", []), config.get("test_targets", None), bep_file)
- upload_failed_test_logs(bep_file, tmpdir)
- if git_repository:
- delete_repository(git_repository)
+ execute_shell_commands(config.get("shell_commands", None))
+ execute_bazel_run(bazel_binary, config.get("run_targets", None))
+ if not test_only:
+ execute_bazel_build(bazel_binary, config.get("build_flags", []),
+ config.get("build_targets", None))
+ if save_but:
+ upload_bazel_binary()
+ if not build_only:
+ bep_file = os.path.join(tmpdir, "build_event_json_file.json")
+ exit_code = execute_bazel_test(bazel_binary, config.get("test_flags", []),
+ config.get("test_targets", None), bep_file)
+ upload_failed_test_logs(bep_file, tmpdir)
finally:
+ if git_repository:
+ os.chdir("..")
+ delete_git_checkout()
if tmpdir:
shutil.rmtree(tmpdir)
- cleanup(bazel_binary)
- exit(exit_code)
+ if exit_code > -1:
+ exit(exit_code)
-def clone_repository(git_repository):
- fail_if_nonzero(run_command(["git", "clone", git_repository, "downstream-repo"]))
- os.chdir("downstream-repo")
+def upload_bazel_binary():
+ print("\n--- Uploading Bazel under test")
+ fail_if_nonzero(execute_command(["buildkite-agent", "artifact", "upload",
+ "bazel-bin/src/bazel"]))
-def delete_repository(git_repository):
+def download_bazel_binary(dest_dir, source_step):
+ print("\n--- Downloading Bazel under test")
+ fail_if_nonzero(execute_command(["buildkite-agent", "artifact", "download",
+ "bazel-bin/src/bazel", dest_dir, "--step", source_step]))
+ bazel_binary_path = os.path.join(dest_dir, "bazel-bin/src/bazel")
+ st = os.stat(bazel_binary_path)
+ os.chmod(bazel_binary_path, st.st_mode | stat.S_IEXEC)
+ return bazel_binary_path
+
+def clone_git_repository(git_repository):
+ delete_git_checkout()
+ fail_if_nonzero(execute_command(["git", "clone", git_repository,
+ git_clone_path()]))
+
+def delete_git_checkout():
+ if os.path.exists(git_clone_path()):
+ shutil.rmtree(git_clone_path())
+
+def cleanup(bazel_binary):
+ print("\n--- Cleanup")
+ if os.path.exists("WORKSPACE"):
+ fail_if_nonzero(execute_command([bazel_binary, "clean", "--expunge"]))
+ delete_git_checkout()
+
+def delete_git_repository(git_repository):
os.chdir("..")
- shutil.rmtree("downstream-repo")
+ shutil.rmtree(git_clone_path())
-def shell_commands(commands):
+def execute_shell_commands(commands):
if not commands:
return
print("\n--- Shell Commands")
shell_command = "\n".join(commands)
- fail_if_nonzero(run_command([shell_command], shell=True))
+ fail_if_nonzero(execute_command([shell_command], shell=True))
-def bazel_run(bazel_binary, targets):
+def execute_bazel_run(bazel_binary, targets):
if not targets:
return
print("\n--- Run Targets")
for target in targets:
- fail_if_nonzero(run_command([bazel_binary, "run", target]))
+ fail_if_nonzero(execute_command([bazel_binary, "run", target]))
-def bazel_build(bazel_binary, flags, targets):
+def execute_bazel_build(bazel_binary, flags, targets):
if not targets:
return
print("\n+++ Build")
- fail_if_nonzero(run_command([bazel_binary, "build", "--color=yes", "--keep_going"] + flags + targets))
+ fail_if_nonzero(execute_command([bazel_binary, "build", "--color=yes",
+ "--keep_going"] + flags + targets))
-def bazel_test(bazel_binary, flags, targets, bep_file):
+def execute_bazel_test(bazel_binary, flags, targets, bep_file):
if not targets:
- return
+ return 0
print("\n+++ Test")
- return run_command([bazel_binary, "test", "--color=yes", "--keep_going", "--build_tests_only", "--build_event_json_file=" + bep_file] + flags + targets)
+ return execute_command([bazel_binary, "test", "--color=yes", "--keep_going",
+ "--build_tests_only", "--build_event_json_file=" + bep_file] + flags +
+ targets)
def fail_if_nonzero(exitcode):
if exitcode is not 0:
exit(exitcode)
def upload_failed_test_logs(bep_file, tmpdir):
- for logfile in failed_test_logs(bep_file, tmpdir):
- fail_if_nonzero(run_command(["buildkite-agent", "artifact", "upload", logfile]))
+ if not os.path.exists(bep_file):
+ return
+ for logfile in failed_logs_from_bep(bep_file, tmpdir):
+ fail_if_nonzero(execute_command(["buildkite-agent", "artifact", "upload",
+ logfile]))
-def failed_test_logs(bep_file, tmpdir):
+def failed_logs_from_bep(bep_file, tmpdir):
test_logs = []
raw_data = ""
with open(bep_file) as f:
@@ -122,74 +279,173 @@
path = path.replace(":", "/")
return os.path.join(tmpdir, path + ".log")
-def cleanup(bazel_binary):
- print("\n--- Cleanup")
- if os.path.exists("WORKSPACE"):
- fail_if_nonzero(run_command([bazel_binary, "clean", "--expunge"]))
- if os.path.exists("downstream-repo"):
- shutil.rmtree("downstream-repo")
-
-def run_command(args, shell=False):
+def execute_command(args, shell=False):
print(" ".join(args))
res = subprocess.run(args, shell=shell)
return res.returncode
-def generate_pipeline(configs, http_config):
- if not configs:
- print("The CI config is empty.")
- exit(1)
+def print_project_pipeline(platform_configs, project_name, http_config,
+ git_repository, use_but):
pipeline_steps = []
- for platform, config in configs.items():
- if platform not in supported_platforms():
- print("'{0}' is not a supported platform on Bazel CI".format(platform))
- exit(1)
- pipeline_steps.append(command_step(supported_platforms()[platform], platform, http_config))
- if not pipeline_steps:
- print("The CI config is empty.")
- exit(1)
- write_pipeline_file(pipeline_steps)
+ for platform, config in platform_configs.items():
+ step = runner_step(platform, project_name, http_config, git_repository,
+ use_but)
+ pipeline_steps.append(step)
-def write_pipeline_file(steps):
+ print_pipeline(pipeline_steps)
+
+def runner_step(platform, project_name=None, http_config=None,
+ git_repository=None, use_but=False, save_but=False, build_only=False,
+ test_only=False):
+ command = python_binary() + " bazelci.py runner --platform=" + platform
+ if http_config:
+ command = command + " --http_config=" + http_config
+ if git_repository:
+ command = command + " --git_repository=" + git_repository
+ if use_but:
+ command = command + " --use_but"
+ if save_but:
+ command = command + " --save_but"
+ if build_only:
+ command = command + " --build_only"
+ if test_only:
+ command = command + " --test_only"
+ platform_name = supported_platforms()[platform]
+ label = create_label(platform_name, project_name, build_only, test_only)
+ return """
+ - label: \"{0}\"
+ command: \"{1}\\n{2}\"
+ agents:
+ - \"os={3}\"""".format(label, fetch_bazelcipy_command(), command, platform)
+
+def print_pipeline(steps):
print("steps:")
for step in steps:
print(step)
-def command_step(label, platform, http_config):
+def wait_step():
return """
- - label: \"{0}\"
- command: \"curl -s https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/bazelci.py > bazelci.py\\n{1}\"
- agents:
- - \"os={2}\"""".format(label, "{0} --platform={1} {2} ".format(runner_command(platform), platform, http_config_flag(http_config)), platform)
-
-def runner_command(platform):
- return "python3.6 bazelci.py --runner=true"
+ - wait"""
def http_config_flag(http_config):
if http_config is not None:
return "--http_config=" + http_config
return ""
+def fetch_bazelcipy_command():
+ return "curl -s {0} > bazelci.py".format(bazelcipy_url())
+
+def upload_project_pipeline_step(project_name, git_repository, http_config):
+ pipeline_command = ("{0} bazelci.py project_pipeline --project_name=\\\"{1}\\\" " +
+ "--use_but --git_repository={2}").format(python_binary(), project_name,
+ git_repository)
+ if http_config:
+ pipeline_command = pipeline_command + " --http_config=" + http_config
+ pipeline_command = pipeline_command + " | buildkite-agent pipeline upload"
+
+ return """
+ - label: \"Setup {0}\"
+ command: \"{1}\\n{2}\"
+ agents:
+ - \"pipeline=true\"""".format(project_name, fetch_bazelcipy_command(),
+ pipeline_command)
+
+def create_label(platform_name, project_name=None, build_only=False,
+ test_only=False):
+ label = ""
+ if build_only:
+ label = "Build "
+ if test_only:
+ label = "Test "
+ if project_name:
+ label = label + "{0} ({1})".format(project_name, platform_name)
+ else:
+ label = label + platform_name
+ return label
+
+def bazel_build_step(platform_name, platform, project_name, http_config=None,
+ build_only=False, test_only=False):
+ pipeline_command = python_binary() + " bazelci.py runner"
+ if build_only:
+ pipeline_command = pipeline_command + " --build_only --save_but"
+ if test_only:
+ pipeline_command = pipeline_command + " --test_only"
+ if http_config:
+ pipeline_command = pipeline_command + " --http_config=" + http_config
+ label = create_label(platform_name, project_name, build_only=build_only,
+ test_only=test_only)
+ pipeline_command = pipeline_command + " --platform=" + platform
+
+ return """
+ - label: \"{0}\"
+ command: \"{1}\\n{2}\"
+ agents:
+ - \"os={3}\"""".format(label, fetch_bazelcipy_command(),
+ pipeline_command, platform)
+
+def print_bazel_postsubmit_pipeline(configs, http_config):
+ if not configs:
+ eprint("Bazel postsubmit pipeline configuration is empty.")
+ if set(configs.keys()) != set(supported_platforms().keys()):
+ eprint("Bazel postsubmit pipeline needs to build Bazel on all " +
+ "supported platforms.")
+
+ pipeline_steps = []
+ for platform, config in configs.items():
+ platform_name = supported_platforms()[platform]
+ pipeline_steps.append(bazel_build_step(platform_name, platform, "Bazel",
+ http_config, build_only=True))
+ pipeline_steps.append(wait_step())
+ for platform, config in configs.items():
+ platform_name = supported_platforms()[platform]
+ pipeline_steps.append(bazel_build_step(platform_name, platform, "Bazel",
+ http_config, test_only=True))
+ for project, config in downstream_projects().items():
+ git_repository = config["git_repository"]
+ http_config = config.get("http_config", None)
+ pipeline_steps.append(upload_project_pipeline_step(project,
+ git_repository, http_config))
+
+ print_pipeline(pipeline_steps)
+
if __name__ == "__main__":
- parser = argparse.ArgumentParser(description='Bazel Continous Integration Runner')
- parser.add_argument("--generate_pipeline", type=bool)
- parser.add_argument("--runner", type=bool)
- parser.add_argument("--platform", type=str, help="The platform the script is running on. Required.")
- parser.add_argument("--bazel_binary", type=str, help="The path to the Bazel binary. Optional.")
- parser.add_argument("--http_config", type=str, help="The URL of the config file. Optional.")
+ parser = argparse.ArgumentParser(description='Bazel Continuous Integration Script')
+
+ subparsers = parser.add_subparsers(dest="subparsers_name")
+ bazel_postsubmit_pipeline = subparsers.add_parser("bazel_postsubmit_pipeline")
+ bazel_postsubmit_pipeline.add_argument("--http_config", type=str)
+ bazel_postsubmit_pipeline.add_argument("--git_repository", type=str)
+
+ project_pipeline = subparsers.add_parser("project_pipeline")
+ project_pipeline.add_argument("--project_name", type=str)
+ project_pipeline.add_argument("--http_config", type=str)
+ project_pipeline.add_argument("--git_repository", type=str)
+ project_pipeline.add_argument("--use_but", type=bool, nargs="?", const=True)
+
+ runner = subparsers.add_parser("runner")
+ runner.add_argument("--platform", action="store",
+ choices=list(supported_platforms()))
+ runner.add_argument("--http_config", type=str)
+ runner.add_argument("--git_repository", type=str)
+ runner.add_argument("--use_but", type=bool, nargs="?", const=True)
+ runner.add_argument("--save_but", type=bool, nargs="?", const=True)
+ runner.add_argument("--build_only", type=bool, nargs="?", const=True)
+ runner.add_argument("--test_only", type=bool, nargs="?", const=True)
+
args = parser.parse_args()
- if args.generate_pipeline:
+ if args.subparsers_name == "bazel_postsubmit_pipeline":
configs = fetch_configs(args.http_config)
- generate_pipeline(configs.get("platforms", None), args.http_config)
- elif args.runner:
+ print_bazel_postsubmit_pipeline(configs.get("platforms", None),
+ args.http_config)
+ elif args.subparsers_name == "project_pipeline":
configs = fetch_configs(args.http_config)
- bazel_binary = "bazel"
- if args.bazel_binary is not None:
- bazel_binary = args.bazel_binary
- git_repository = configs.get("git_repository", None)
- if args.platform not in configs["platforms"]:
- print("No configuration exists for '{0}'".format(args.platform))
- run(configs["platforms"][args.platform], args.platform, bazel_binary, git_repository)
+ print_project_pipeline(configs.get("platforms", None), args.project_name,
+ args.http_config, args.git_repository, args.use_but)
+ elif args.subparsers_name == "runner":
+ configs = fetch_configs(args.http_config)
+ execute_commands(configs.get("platforms", None)[args.platform],
+ args.platform, args.git_repository, args.use_but, args.save_but,
+ args.build_only, args.test_only)
else:
- print("Need to specify either --runner or --generate_pipeline")
- exit(1)
+ parser.print_help()
\ No newline at end of file
diff --git a/buildkite/pipelines/bazelci1.py b/buildkite/pipelines/bazelci1.py
deleted file mode 100644
index 9005167..0000000
--- a/buildkite/pipelines/bazelci1.py
+++ /dev/null
@@ -1,451 +0,0 @@
-from __future__ import print_function
-import argparse
-import codecs
-import json
-import os.path
-import shutil
-import subprocess
-import sys
-import stat
-import tempfile
-import urllib.request
-from shutil import copyfile
-from urllib.parse import urlparse
-
-def downstream_projects():
- return {
- "BUILD_file_generator" : {
- "git_repository" : "https://github.com/bazelbuild/BUILD_file_generator.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/BUILD_file_generator-postsubmit.json"
- },
- "buildtools" : {
- "git_repository" : "https://github.com/bazelbuild/buildtools.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/buildtools-postsubmit.json"
- },
- "examples" : {
- "git_repository" : "https://github.com/bazelbuild/examples.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/examples-postsubmit.json"
- },
- "migration-tooling" : {
- "git_repository" : "https://github.com/bazelbuild/migration-tooling.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/migration-tooling-postsubmit.json"
- },
- "protobuf" : {
- "git_repository" : "https://github.com/google/protobuf.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/protobuf-postsubmit.json"
- },
- "re2" : {
- "git_repository" : "https://github.com/google/re2.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/re2-postsubmit.json"
- },
- "rules_appengine" : {
- "git_repository" : "https://github.com/bazelbuild/rules_appengine.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_appengine-postsubmit.json"
- },
- "rules_closure" : {
- "git_repository" : "https://github.com/bazelbuild/rules_closure.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_closure-postsubmit.json"
- },
- "rules_go" : {
- "git_repository" : "https://github.com/bazelbuild/rules_go.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_go-postsubmit.json"
- },
- "rules_groovy" : {
- "git_repository" : "https://github.com/bazelbuild/rules_groovy.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_groovy-postsubmit.json"
- },
- "rules_gwt" : {
- "git_repository" : "https://github.com/bazelbuild/rules_gwt.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_gwt-postsubmit.json"
- },
- "rules_jsonnet" : {
- "git_repository" : "https://github.com/bazelbuild/rules_jsonnet.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_jsonnet-postsubmit.json"
- },
- "rules_k8s" : {
- "git_repository" : "https://github.com/bazelbuild/rules_k8s.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_k8s-postsubmit.json"
- },
- "rules_nodejs" : {
- "git_repository" : "https://github.com/bazelbuild/rules_nodejs.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_nodejs-postsubmit.json"
- },
- "rules_perl" : {
- "git_repository" : "https://github.com/bazelbuild/rules_perl.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_perl-postsubmit.json"
- },
- "rules_python" : {
- "git_repository" : "https://github.com/bazelbuild/rules_python.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_python-postsubmit.json"
- },
- "rules_scala" : {
- "git_repository" : "https://github.com/bazelbuild/rules_scala.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_scala-postsubmit.json"
- },
- "rules_typescript" : {
- "git_repository" : "https://github.com/bazelbuild/rules_typescript.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_typescript-postsubmit.json"
- },
- "skydoc" : {
- "git_repository" : "https://github.com/bazelbuild/skydoc.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/skydoc-postsubmit.json"
- },
- "subpar" : {
- "git_repository" : "https://github.com/google/subpar.git",
- "http_config" : "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/subpar-postsubmit.json"
- }
- }
-
-def python_binary():
- return "python3.6"
-
-def bazelcipy_url():
- '''
- URL to the latest version of this script.
- '''
- return "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/bazelci1.py"
-
-def eprint(*args, **kwargs):
- '''
- Print to stderr and exit the process.
- '''
- print(*args, file=sys.stderr, **kwargs)
- exit(1)
-
-def supported_platforms():
- '''
- Returns a map containing all supported platform names as keys, with the
- values being the platform name in a human readable format.
- '''
- return {
- "ubuntu1404" : "Ubuntu 14.04",
- "ubuntu1604" : "Ubuntu 16.04",
- "macos" : "macOS"
- }
-
-def git_clone_path():
- return "project-under-test"
-
-def fetch_configs(http_url):
- '''
- If specified fetches the build configuration from http_url, else tries to
- read it from .bazelci/config.json.
- Returns the json configuration as a python data structure.
- '''
- if http_url is None:
- with open(".bazelci/config.json", "r") as fd:
- return json.load(fd)
- with urllib.request.urlopen(http_url) as resp:
- reader = codecs.getreader("utf-8")
- return json.load(reader(resp))
-
-def execute_commands(config, platform, git_repository, use_but, save_but,
- build_only, test_only):
- exit_code = -1
- tmpdir = None
- bazel_binary = "bazel"
- try:
- tmpdir = tempfile.mkdtemp()
- if use_but:
- source_step = create_label(supported_platforms()[platform], "Bazel",
- build_only=True, test_only=False)
- bazel_binary = download_bazel_binary(tmpdir, source_step)
- if git_repository:
- clone_git_repository(git_repository)
- os.chdir(git_clone_path())
- cleanup(bazel_binary)
- else:
- cleanup(bazel_binary)
- execute_shell_commands(config.get("shell_commands", None))
- execute_bazel_run(bazel_binary, config.get("run_targets", None))
- if not test_only:
- execute_bazel_build(bazel_binary, config.get("build_flags", []),
- config.get("build_targets", None))
- if save_but:
- upload_bazel_binary()
- if not build_only:
- bep_file = os.path.join(tmpdir, "build_event_json_file.json")
- exit_code = execute_bazel_test(bazel_binary, config.get("test_flags", []),
- config.get("test_targets", None), bep_file)
- upload_failed_test_logs(bep_file, tmpdir)
- finally:
- if git_repository:
- os.chdir("..")
- delete_git_checkout()
- if tmpdir:
- shutil.rmtree(tmpdir)
- if exit_code > -1:
- exit(exit_code)
-
-def upload_bazel_binary():
- print("\n--- Uploading Bazel under test")
- fail_if_nonzero(execute_command(["buildkite-agent", "artifact", "upload",
- "bazel-bin/src/bazel"]))
-
-def download_bazel_binary(dest_dir, source_step):
- print("\n--- Downloading Bazel under test")
- fail_if_nonzero(execute_command(["buildkite-agent", "artifact", "download",
- "bazel-bin/src/bazel", dest_dir, "--step", source_step]))
- bazel_binary_path = os.path.join(dest_dir, "bazel-bin/src/bazel")
- st = os.stat(bazel_binary_path)
- os.chmod(bazel_binary_path, st.st_mode | stat.S_IEXEC)
- return bazel_binary_path
-
-def clone_git_repository(git_repository):
- delete_git_checkout()
- fail_if_nonzero(execute_command(["git", "clone", git_repository,
- git_clone_path()]))
-
-def delete_git_checkout():
- if os.path.exists(git_clone_path()):
- shutil.rmtree(git_clone_path())
-
-def cleanup(bazel_binary):
- print("\n--- Cleanup")
- if os.path.exists("WORKSPACE"):
- fail_if_nonzero(execute_command([bazel_binary, "clean", "--expunge"]))
- delete_git_checkout()
-
-def delete_git_repository(git_repository):
- os.chdir("..")
- shutil.rmtree(git_clone_path())
-
-def execute_shell_commands(commands):
- if not commands:
- return
- print("\n--- Shell Commands")
- shell_command = "\n".join(commands)
- fail_if_nonzero(execute_command([shell_command], shell=True))
-
-def execute_bazel_run(bazel_binary, targets):
- if not targets:
- return
- print("\n--- Run Targets")
- for target in targets:
- fail_if_nonzero(execute_command([bazel_binary, "run", target]))
-
-def execute_bazel_build(bazel_binary, flags, targets):
- if not targets:
- return
- print("\n+++ Build")
- fail_if_nonzero(execute_command([bazel_binary, "build", "--color=yes",
- "--keep_going"] + flags + targets))
-
-def execute_bazel_test(bazel_binary, flags, targets, bep_file):
- if not targets:
- return 0
- print("\n+++ Test")
- return execute_command([bazel_binary, "test", "--color=yes", "--keep_going",
- "--build_tests_only", "--build_event_json_file=" + bep_file] + flags +
- targets)
-
-def fail_if_nonzero(exitcode):
- if exitcode is not 0:
- exit(exitcode)
-
-def upload_failed_test_logs(bep_file, tmpdir):
- if not os.path.exists(bep_file):
- return
- for logfile in failed_logs_from_bep(bep_file, tmpdir):
- fail_if_nonzero(execute_command(["buildkite-agent", "artifact", "upload",
- logfile]))
-
-def failed_logs_from_bep(bep_file, tmpdir):
- test_logs = []
- raw_data = ""
- with open(bep_file) as f:
- raw_data = f.read()
- decoder = json.JSONDecoder()
-
- pos = 0
- while pos < len(raw_data):
- json_dict, size = decoder.raw_decode(raw_data[pos:])
- if "testResult" in json_dict:
- test_result = json_dict["testResult"]
- if test_result["status"] != "PASSED":
- outputs = test_result["testActionOutput"]
- for output in outputs:
- if output["name"] == "test.log":
- new_path = label_to_path(tmpdir, json_dict["id"]["testResult"]["label"])
- os.makedirs(os.path.dirname(new_path), exist_ok=True)
- copyfile(urlparse(output["uri"]).path, new_path)
- test_logs.append(new_path)
- pos += size + 1
- return test_logs
-
-def label_to_path(tmpdir, label):
- # remove leading //
- path = label[2:]
- path = path.replace(":", "/")
- return os.path.join(tmpdir, path + ".log")
-
-def execute_command(args, shell=False):
- print(" ".join(args))
- res = subprocess.run(args, shell=shell)
- return res.returncode
-
-def print_project_pipeline(platform_configs, project_name, http_config,
- git_repository, use_but):
- pipeline_steps = []
- for platform, config in platform_configs.items():
- step = runner_step(platform, project_name, http_config, git_repository,
- use_but)
- pipeline_steps.append(step)
-
- print_pipeline(pipeline_steps)
-
-def runner_step(platform, project_name=None, http_config=None,
- git_repository=None, use_but=False, save_but=False, build_only=False,
- test_only=False):
- command = python_binary() + " bazelci.py runner --platform=" + platform
- if http_config:
- command = command + " --http_config=" + http_config
- if git_repository:
- command = command + " --git_repository=" + git_repository
- if use_but:
- command = command + " --use_but"
- if save_but:
- command = command + " --save_but"
- if build_only:
- command = command + " --build_only"
- if test_only:
- command = command + " --test_only"
- platform_name = supported_platforms()[platform]
- label = create_label(platform_name, project_name, build_only, test_only)
- return """
- - label: \"{0}\"
- command: \"{1}\\n{2}\"
- agents:
- - \"os={3}\"""".format(label, fetch_bazelcipy_command(), command, platform)
-
-def print_pipeline(steps):
- print("steps:")
- for step in steps:
- print(step)
-
-def wait_step():
- return """
- - wait"""
-
-def http_config_flag(http_config):
- if http_config is not None:
- return "--http_config=" + http_config
- return ""
-
-def fetch_bazelcipy_command():
- return "curl -s {0} > bazelci.py".format(bazelcipy_url())
-
-def upload_project_pipeline_step(project_name, git_repository, http_config):
- pipeline_command = ("{0} bazelci.py project_pipeline --project_name=\\\"{1}\\\" " +
- "--use_but --git_repository={2}").format(python_binary(), project_name,
- git_repository)
- if http_config:
- pipeline_command = pipeline_command + " --http_config=" + http_config
- pipeline_command = pipeline_command + " | buildkite-agent pipeline upload"
-
- return """
- - label: \"Setup {0}\"
- command: \"{1}\\n{2}\"
- agents:
- - \"pipeline=true\"""".format(project_name, fetch_bazelcipy_command(),
- pipeline_command)
-
-def create_label(platform_name, project_name=None, build_only=False,
- test_only=False):
- label = ""
- if build_only:
- label = "Build "
- if test_only:
- label = "Test "
- if project_name:
- label = label + "{0} ({1})".format(project_name, platform_name)
- else:
- label = label + platform_name
- return label
-
-def bazel_build_step(platform_name, platform, project_name, http_config=None,
- build_only=False, test_only=False):
- pipeline_command = python_binary() + " bazelci.py runner"
- if build_only:
- pipeline_command = pipeline_command + " --build_only --save_but"
- if test_only:
- pipeline_command = pipeline_command + " --test_only"
- if http_config:
- pipeline_command = pipeline_command + " --http_config=" + http_config
- label = create_label(platform_name, project_name, build_only=build_only,
- test_only=test_only)
- pipeline_command = pipeline_command + " --platform=" + platform
-
- return """
- - label: \"{0}\"
- command: \"{1}\\n{2}\"
- agents:
- - \"os={3}\"""".format(label, fetch_bazelcipy_command(),
- pipeline_command, platform)
-
-def print_bazel_postsubmit_pipeline(configs, http_config):
- if not configs:
- eprint("Bazel postsubmit pipeline configuration is empty.")
- if set(configs.keys()) != set(supported_platforms().keys()):
- eprint("Bazel postsubmit pipeline needs to build Bazel on all " +
- "supported platforms.")
-
- pipeline_steps = []
- for platform, config in configs.items():
- platform_name = supported_platforms()[platform]
- pipeline_steps.append(bazel_build_step(platform_name, platform, "Bazel",
- http_config, build_only=True))
- pipeline_steps.append(wait_step())
- for platform, config in configs.items():
- platform_name = supported_platforms()[platform]
- pipeline_steps.append(bazel_build_step(platform_name, platform, "Bazel",
- http_config, test_only=True))
- for project, config in downstream_projects().items():
- git_repository = config["git_repository"]
- http_config = config.get("http_config", None)
- pipeline_steps.append(upload_project_pipeline_step(project,
- git_repository, http_config))
-
- print_pipeline(pipeline_steps)
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser(description='Bazel Continuous Integration Script')
-
- subparsers = parser.add_subparsers(dest="subparsers_name")
- bazel_postsubmit_pipeline = subparsers.add_parser("bazel_postsubmit_pipeline")
- bazel_postsubmit_pipeline.add_argument("--http_config", type=str)
- bazel_postsubmit_pipeline.add_argument("--git_repository", type=str)
-
- project_pipeline = subparsers.add_parser("project_pipeline")
- project_pipeline.add_argument("--project_name", type=str)
- project_pipeline.add_argument("--http_config", type=str)
- project_pipeline.add_argument("--git_repository", type=str)
- project_pipeline.add_argument("--use_but", type=bool, nargs="?", const=True)
-
- runner = subparsers.add_parser("runner")
- runner.add_argument("--platform", action="store",
- choices=list(supported_platforms()))
- runner.add_argument("--http_config", type=str)
- runner.add_argument("--git_repository", type=str)
- runner.add_argument("--use_but", type=bool, nargs="?", const=True)
- runner.add_argument("--save_but", type=bool, nargs="?", const=True)
- runner.add_argument("--build_only", type=bool, nargs="?", const=True)
- runner.add_argument("--test_only", type=bool, nargs="?", const=True)
-
- args = parser.parse_args()
-
- if args.subparsers_name == "bazel_postsubmit_pipeline":
- configs = fetch_configs(args.http_config)
- print_bazel_postsubmit_pipeline(configs.get("platforms", None),
- args.http_config)
- elif args.subparsers_name == "project_pipeline":
- configs = fetch_configs(args.http_config)
- print_project_pipeline(configs.get("platforms", None), args.project_name,
- args.http_config, args.git_repository, args.use_but)
- elif args.subparsers_name == "runner":
- configs = fetch_configs(args.http_config)
- execute_commands(configs.get("platforms", None)[args.platform],
- args.platform, args.git_repository, args.use_but, args.save_but,
- args.build_only, args.test_only)
- else:
- parser.print_help()
\ No newline at end of file
