Indent bazelci.py with 4 spaces
diff --git a/buildkite/bazelci.py b/buildkite/bazelci.py
index a23bf1a..efd1531 100644
--- a/buildkite/bazelci.py
+++ b/buildkite/bazelci.py
@@ -32,482 +32,479 @@
def downstream_projects():
- return {
- "BUILD_file_generator": {
- "git_repository": "https://github.com/bazelbuild/BUILD_file_generator.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/BUILD_file_generator-postsubmit.json"
- },
- "bazel-toolchains": {
- "git_repository": "https://github.com/bazelbuild/bazel-toolchains.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/bazel-toolchains-postsubmit.json"
- },
- "buildtools": {
- "git_repository": "https://github.com/bazelbuild/buildtools.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/buildtools-postsubmit.json"
- },
- "CLion Plugin": {
- "git_repository": "https://github.com/bazelbuild/intellij.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/clion-postsubmit.json"
- },
- "Eclipse Plugin": {
- "git_repository": "https://github.com/bazelbuild/eclipse.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/eclipse-postsubmit.json"
- },
- "Gerrit": {
- "git_repository": "https://gerrit.googlesource.com/gerrit.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/gerrit-postsubmit.json"
- },
- "Google Logging": {
- "git_repository": "https://github.com/google/glog.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/glog-postsubmit.json"
- },
- "IntelliJ Plugin": {
- "git_repository": "https://github.com/bazelbuild/intellij.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/intellij-postsubmit.json"
- },
- "migration-tooling": {
- "git_repository": "https://github.com/bazelbuild/migration-tooling.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/migration-tooling-postsubmit.json"
- },
- "protobuf": {
- "git_repository": "https://github.com/google/protobuf.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/protobuf-postsubmit.json"
- },
- "re2": {
- "git_repository": "https://github.com/google/re2.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/re2-postsubmit.json"
- },
- "rules_appengine": {
- "git_repository": "https://github.com/bazelbuild/rules_appengine.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_appengine-postsubmit.json"
- },
- "rules_closure": {
- "git_repository": "https://github.com/bazelbuild/rules_closure.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_closure-postsubmit.json"
- },
- "rules_d": {
- "git_repository": "https://github.com/bazelbuild/rules_d.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_d-postsubmit.json"
- },
- "rules_go": {
- "git_repository": "https://github.com/bazelbuild/rules_go.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_go-postsubmit.json"
- },
- "rules_groovy": {
- "git_repository": "https://github.com/bazelbuild/rules_groovy.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_groovy-postsubmit.json"
- },
- "rules_gwt": {
- "git_repository": "https://github.com/bazelbuild/rules_gwt.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_gwt-postsubmit.json"
- },
- "rules_jsonnet": {
- "git_repository": "https://github.com/bazelbuild/rules_jsonnet.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_jsonnet-postsubmit.json"
- },
- "rules_k8s": {
- "git_repository": "https://github.com/bazelbuild/rules_k8s.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_k8s-postsubmit.json"
- },
- "rules_nodejs": {
- "git_repository": "https://github.com/bazelbuild/rules_nodejs.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_nodejs-postsubmit.json"
- },
- "rules_perl": {
- "git_repository": "https://github.com/bazelbuild/rules_perl.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_perl-postsubmit.json"
- },
- "rules_python": {
- "git_repository": "https://github.com/bazelbuild/rules_python.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_python-postsubmit.json"
- },
- "rules_rust": {
- "git_repository": "https://github.com/bazelbuild/rules_rust.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_rust-postsubmit.json"
- },
- "rules_sass": {
- "git_repository": "https://github.com/bazelbuild/rules_sass.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_sass-postsubmit.json"
- },
- "rules_scala": {
- "git_repository": "https://github.com/bazelbuild/rules_scala.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_scala-postsubmit.json"
- },
- "rules_typescript": {
- "git_repository": "https://github.com/bazelbuild/rules_typescript.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_typescript-postsubmit.json"
- },
- # Enable once is resolved: https://github.com/bazelbuild/continuous-integration/issues/191
- # "rules_webtesting": {
- # "git_repository": "https://github.com/bazelbuild/rules_webtesting.git",
- # "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_webtesting-postsubmit.json"
- # },
- "skydoc": {
- "git_repository": "https://github.com/bazelbuild/skydoc.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/skydoc-postsubmit.json"
- },
- "subpar": {
- "git_repository": "https://github.com/google/subpar.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/subpar-postsubmit.json"
- },
- "TensorFlow": {
- "git_repository": "https://github.com/tensorflow/tensorflow.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/tensorflow-postsubmit.json"
- },
- "TensorFlow Serving": {
- "git_repository": "https://github.com/tensorflow/serving.git",
- "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/tensorflow-serving-postsubmit.json"
- }
- }
+ return {
+ "BUILD_file_generator": {
+ "git_repository": "https://github.com/bazelbuild/BUILD_file_generator.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/BUILD_file_generator-postsubmit.json"
+ },
+ "bazel-toolchains": {
+ "git_repository": "https://github.com/bazelbuild/bazel-toolchains.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/bazel-toolchains-postsubmit.json"
+ },
+ "buildtools": {
+ "git_repository": "https://github.com/bazelbuild/buildtools.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/buildtools-postsubmit.json"
+ },
+ "CLion Plugin": {
+ "git_repository": "https://github.com/bazelbuild/intellij.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/clion-postsubmit.json"
+ },
+ "Eclipse Plugin": {
+ "git_repository": "https://github.com/bazelbuild/eclipse.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/eclipse-postsubmit.json"
+ },
+ "Gerrit": {
+ "git_repository": "https://gerrit.googlesource.com/gerrit.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/gerrit-postsubmit.json"
+ },
+ "Google Logging": {
+ "git_repository": "https://github.com/google/glog.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/glog-postsubmit.json"
+ },
+ "IntelliJ Plugin": {
+ "git_repository": "https://github.com/bazelbuild/intellij.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/intellij-postsubmit.json"
+ },
+ "migration-tooling": {
+ "git_repository": "https://github.com/bazelbuild/migration-tooling.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/migration-tooling-postsubmit.json"
+ },
+ "protobuf": {
+ "git_repository": "https://github.com/google/protobuf.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/protobuf-postsubmit.json"
+ },
+ "re2": {
+ "git_repository": "https://github.com/google/re2.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/re2-postsubmit.json"
+ },
+ "rules_appengine": {
+ "git_repository": "https://github.com/bazelbuild/rules_appengine.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_appengine-postsubmit.json"
+ },
+ "rules_closure": {
+ "git_repository": "https://github.com/bazelbuild/rules_closure.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_closure-postsubmit.json"
+ },
+ "rules_d": {
+ "git_repository": "https://github.com/bazelbuild/rules_d.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_d-postsubmit.json"
+ },
+ "rules_go": {
+ "git_repository": "https://github.com/bazelbuild/rules_go.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_go-postsubmit.json"
+ },
+ "rules_groovy": {
+ "git_repository": "https://github.com/bazelbuild/rules_groovy.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_groovy-postsubmit.json"
+ },
+ "rules_gwt": {
+ "git_repository": "https://github.com/bazelbuild/rules_gwt.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_gwt-postsubmit.json"
+ },
+ "rules_jsonnet": {
+ "git_repository": "https://github.com/bazelbuild/rules_jsonnet.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_jsonnet-postsubmit.json"
+ },
+ "rules_k8s": {
+ "git_repository": "https://github.com/bazelbuild/rules_k8s.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_k8s-postsubmit.json"
+ },
+ "rules_nodejs": {
+ "git_repository": "https://github.com/bazelbuild/rules_nodejs.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_nodejs-postsubmit.json"
+ },
+ "rules_perl": {
+ "git_repository": "https://github.com/bazelbuild/rules_perl.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_perl-postsubmit.json"
+ },
+ "rules_python": {
+ "git_repository": "https://github.com/bazelbuild/rules_python.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_python-postsubmit.json"
+ },
+ "rules_rust": {
+ "git_repository": "https://github.com/bazelbuild/rules_rust.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_rust-postsubmit.json"
+ },
+ "rules_sass": {
+ "git_repository": "https://github.com/bazelbuild/rules_sass.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_sass-postsubmit.json"
+ },
+ "rules_scala": {
+ "git_repository": "https://github.com/bazelbuild/rules_scala.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_scala-postsubmit.json"
+ },
+ "rules_typescript": {
+ "git_repository": "https://github.com/bazelbuild/rules_typescript.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_typescript-postsubmit.json"
+ },
+ # Enable once is resolved: https://github.com/bazelbuild/continuous-integration/issues/191
+ # "rules_webtesting": {
+ # "git_repository": "https://github.com/bazelbuild/rules_webtesting.git",
+ # "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/rules_webtesting-postsubmit.json"
+ # },
+ "skydoc": {
+ "git_repository": "https://github.com/bazelbuild/skydoc.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/skydoc-postsubmit.json"
+ },
+ "subpar": {
+ "git_repository": "https://github.com/google/subpar.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/subpar-postsubmit.json"
+ },
+ "TensorFlow": {
+ "git_repository": "https://github.com/tensorflow/tensorflow.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/tensorflow-postsubmit.json"
+ },
+ "TensorFlow Serving": {
+ "git_repository": "https://github.com/tensorflow/serving.git",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/tensorflow-serving-postsubmit.json"
+ }
+ }
def python_binary():
- return "python3.6"
+ return "python3.6"
def bazelcipy_url():
- '''
- URL to the latest version of this script.
- '''
- return "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/bazelci.py"
+ '''
+ URL to the latest version of this script.
+ '''
+ return "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/bazelci.py"
def eprint(*args, **kwargs):
- '''
- Print to stderr and exit the process.
- '''
- print(*args, file=sys.stderr, **kwargs)
- exit(1)
+ '''
+ Print to stderr and exit the process.
+ '''
+ print(*args, file=sys.stderr, **kwargs)
+ exit(1)
def platforms_info():
- '''
- Returns a map containing all supported platform names as keys, with the
- values being the platform name in a human readable format, and a the
- buildkite-agent's working directory.
- '''
- return {
- "ubuntu1404":
- {
- "name": "Ubuntu 14.04",
- "agent-directory": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/"
- },
- "ubuntu1604":
- {
- "name": "Ubuntu 16.04",
- "agent-directory": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/"
- },
- "macos":
- {
- "name": "macOS",
- "agent-directory": "/usr/local/var/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/"
- }
- }
+ '''
+ Returns a map containing all supported platform names as keys, with the
+ values being the platform name in a human readable format, and a the
+ buildkite-agent's working directory.
+ '''
+ return {
+ "ubuntu1404": {
+ "name": "Ubuntu 14.04",
+ "agent-directory": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/"
+ },
+ "ubuntu1604": {
+ "name": "Ubuntu 16.04",
+ "agent-directory": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/"
+ },
+ "macos": {
+ "name": "macOS",
+ "agent-directory": "/usr/local/var/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/"
+ }
+ }
def downstream_projects_root(platform):
- downstream_projects_dir = os.path.expandvars(
- "${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects")
- path = os.path.join(agent_directory(platform), downstream_projects_dir)
- if not os.path.exists(path):
- os.makedirs(path)
- return path
+ downstream_projects_dir = os.path.expandvars(
+ "${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects")
+ path = os.path.join(agent_directory(platform), downstream_projects_dir)
+ if not os.path.exists(path):
+ os.makedirs(path)
+ return path
def agent_directory(platform):
- return os.path.expandvars(platforms_info()[platform]["agent-directory"])
+ return os.path.expandvars(platforms_info()[platform]["agent-directory"])
def supported_platforms():
- return set(platforms_info().keys())
+ return set(platforms_info().keys())
def platform_name(platform):
- return platforms_info()[platform]["name"]
+ return platforms_info()[platform]["name"]
def fetch_configs(http_url):
- '''
- If specified fetches the build configuration from http_url, else tries to
- read it from .bazelci/config.json.
- Returns the json configuration as a python data structure.
- '''
- if http_url is None:
- with open(".bazelci/config.json", "r") as fd:
- return json.load(fd)
- with urllib.request.urlopen(http_url) as resp:
- reader = codecs.getreader("utf-8")
- return json.load(reader(resp))
+ '''
+ If specified fetches the build configuration from http_url, else tries to
+ read it from .bazelci/config.json.
+ Returns the json configuration as a python data structure.
+ '''
+ if http_url is None:
+ with open(".bazelci/config.json", "r") as fd:
+ return json.load(fd)
+ with urllib.request.urlopen(http_url) as resp:
+ reader = codecs.getreader("utf-8")
+ return json.load(reader(resp))
def print_collapsed_group(name):
- print("\n--- {0}\n".format(name))
+ print("\n--- {0}\n".format(name))
def print_expanded_group(name):
- print("\n+++ {0}\n".format(name))
+ print("\n+++ {0}\n".format(name))
def execute_commands(config, platform, git_repository, use_but, save_but,
build_only, test_only):
- exit_code = -1
- tmpdir = None
- bazel_binary = "bazel"
- try:
- if git_repository:
- clone_git_repository(git_repository, platform)
- cleanup(platform)
- tmpdir = tempfile.mkdtemp()
- if use_but:
- print_collapsed_group("Downloading Bazel under test")
- bazel_binary = download_bazel_binary(tmpdir, platform)
- print_bazel_version_info(bazel_binary)
- execute_shell_commands(config.get("shell_commands", None))
- execute_bazel_run(bazel_binary, config.get("run_targets", None))
- if not test_only:
- execute_bazel_build(bazel_binary, config.get("build_flags", []),
- config.get("build_targets", None))
- if save_but:
- upload_bazel_binary()
- if not build_only:
- bep_file = os.path.join(tmpdir, "build_event_json_file.json")
- exit_code = execute_bazel_test(bazel_binary, config.get("test_flags", []),
- config.get("test_targets", None), bep_file)
- # Fail the pipeline if there were any flaky tests.
- if has_flaky_tests() and exit_code == 0:
- exit_code = 1
- upload_test_logs(bep_file, tmpdir)
- finally:
- if tmpdir:
- shutil.rmtree(tmpdir)
- cleanup(platform)
- if exit_code > -1:
- exit(exit_code)
+ exit_code = -1
+ tmpdir = None
+ bazel_binary = "bazel"
+ try:
+ if git_repository:
+ clone_git_repository(git_repository, platform)
+ cleanup(platform)
+ tmpdir = tempfile.mkdtemp()
+ if use_but:
+ print_collapsed_group("Downloading Bazel under test")
+ bazel_binary = download_bazel_binary(tmpdir, platform)
+ print_bazel_version_info(bazel_binary)
+ execute_shell_commands(config.get("shell_commands", None))
+ execute_bazel_run(bazel_binary, config.get("run_targets", None))
+ if not test_only:
+ execute_bazel_build(bazel_binary, config.get("build_flags", []),
+ config.get("build_targets", None))
+ if save_but:
+ upload_bazel_binary()
+ if not build_only:
+ bep_file = os.path.join(tmpdir, "build_event_json_file.json")
+ exit_code = execute_bazel_test(bazel_binary, config.get("test_flags", []),
+ config.get("test_targets", None), bep_file)
+ # Fail the pipeline if there were any flaky tests.
+ if has_flaky_tests() and exit_code == 0:
+ exit_code = 1
+ upload_test_logs(bep_file, tmpdir)
+ finally:
+ if tmpdir:
+ shutil.rmtree(tmpdir)
+ cleanup(platform)
+ if exit_code > -1:
+ exit(exit_code)
def has_flaky_tests(bep_file):
- return len(test_logs_for_status(bep_file, status="FLAKY")) > 0
+ return len(test_logs_for_status(bep_file, status="FLAKY")) > 0
def print_bazel_version_info(bazel_binary):
- print_collapsed_group("Bazel Info")
- fail_if_nonzero(execute_command([bazel_binary, "version"]))
- fail_if_nonzero(execute_command([bazel_binary, "info"]))
+ print_collapsed_group("Bazel Info")
+ fail_if_nonzero(execute_command([bazel_binary, "version"]))
+ fail_if_nonzero(execute_command([bazel_binary, "info"]))
def upload_bazel_binary():
- print_collapsed_group("Uploading Bazel under test")
- fail_if_nonzero(execute_command(["buildkite-agent", "artifact", "upload",
- "bazel-bin/src/bazel"]))
+ print_collapsed_group("Uploading Bazel under test")
+ fail_if_nonzero(execute_command(["buildkite-agent", "artifact", "upload",
+ "bazel-bin/src/bazel"]))
def download_bazel_binary(dest_dir, platform):
- source_step = create_label(platform_name(platform), "Bazel", build_only=True,
- test_only=False)
- fail_if_nonzero(execute_command(["buildkite-agent", "artifact", "download",
- "bazel-bin/src/bazel", dest_dir, "--step", source_step]))
- bazel_binary_path = os.path.join(dest_dir, "bazel-bin/src/bazel")
- st = os.stat(bazel_binary_path)
- os.chmod(bazel_binary_path, st.st_mode | stat.S_IEXEC)
- return bazel_binary_path
+ source_step = create_label(platform_name(platform), "Bazel", build_only=True,
+ test_only=False)
+ fail_if_nonzero(execute_command(["buildkite-agent", "artifact", "download",
+ "bazel-bin/src/bazel", dest_dir, "--step", source_step]))
+ bazel_binary_path = os.path.join(dest_dir, "bazel-bin/src/bazel")
+ st = os.stat(bazel_binary_path)
+ os.chmod(bazel_binary_path, st.st_mode | stat.S_IEXEC)
+ return bazel_binary_path
def clone_git_repository(git_repository, platform):
- root = downstream_projects_root(platform)
- project_name = re.search("/([^/]+)\.git$", git_repository).group(1)
- clone_path = os.path.join(root, project_name)
- print_collapsed_group("Fetching " + project_name + " sources")
- if os.path.exists(clone_path):
- os.chdir(clone_path)
- fail_if_nonzero(execute_command(
- ["git", "remote", "set-url", "origin", git_repository]))
- fail_if_nonzero(execute_command(["git", "clean", "-fdqx"]))
- fail_if_nonzero(execute_command(
- ["git", "submodule", "foreach", "--recursive", "git", "clean", "-fdqx"]))
- # sync to the latest commit of HEAD. Unlikely git pull this also works after
- # a force push.
- fail_if_nonzero(execute_command(["git", "fetch", "origin"]))
- remote_head = subprocess.check_output(
- ["git", "symbolic-ref", "refs/remotes/origin/HEAD"])
- remote_head = remote_head.decode("utf-8")
- remote_head = remote_head.rstrip()
- fail_if_nonzero(execute_command(
- ["git", "reset", remote_head, "--hard"]))
- fail_if_nonzero(execute_command(
- ["git", "submodule", "sync", "--recursive"]))
- fail_if_nonzero(execute_command(
- ["git", "submodule", "update", "--init", "--recursive", "--force"]))
- fail_if_nonzero(execute_command(
- ["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"]))
- fail_if_nonzero(execute_command(["git", "clean", "-fdqx"]))
- fail_if_nonzero(execute_command(
- ["git", "submodule", "foreach", "--recursive", "git", "clean", "-fdqx"]))
- else:
- fail_if_nonzero(execute_command(
- ["git", "clone", "--recurse-submodules", git_repository, clone_path]))
- os.chdir(clone_path)
+ root = downstream_projects_root(platform)
+ project_name = re.search("/([^/]+)\.git$", git_repository).group(1)
+ clone_path = os.path.join(root, project_name)
+ print_collapsed_group("Fetching " + project_name + " sources")
+ if os.path.exists(clone_path):
+ os.chdir(clone_path)
+ fail_if_nonzero(execute_command(
+ ["git", "remote", "set-url", "origin", git_repository]))
+ fail_if_nonzero(execute_command(["git", "clean", "-fdqx"]))
+ fail_if_nonzero(execute_command(
+ ["git", "submodule", "foreach", "--recursive", "git", "clean", "-fdqx"]))
+ # sync to the latest commit of HEAD. Unlikely git pull this also works after
+ # a force push.
+ fail_if_nonzero(execute_command(["git", "fetch", "origin"]))
+ remote_head = subprocess.check_output(
+ ["git", "symbolic-ref", "refs/remotes/origin/HEAD"])
+ remote_head = remote_head.decode("utf-8")
+ remote_head = remote_head.rstrip()
+ fail_if_nonzero(execute_command(
+ ["git", "reset", remote_head, "--hard"]))
+ fail_if_nonzero(execute_command(
+ ["git", "submodule", "sync", "--recursive"]))
+ fail_if_nonzero(execute_command(
+ ["git", "submodule", "update", "--init", "--recursive", "--force"]))
+ fail_if_nonzero(execute_command(
+ ["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"]))
+ fail_if_nonzero(execute_command(["git", "clean", "-fdqx"]))
+ fail_if_nonzero(execute_command(
+ ["git", "submodule", "foreach", "--recursive", "git", "clean", "-fdqx"]))
+ else:
+ fail_if_nonzero(execute_command(
+ ["git", "clone", "--recurse-submodules", git_repository, clone_path]))
+ os.chdir(clone_path)
def cleanup(platform):
- print_collapsed_group("Cleanup")
- if os.path.exists("WORKSPACE"):
- fail_if_nonzero(execute_command(["bazel", "clean", "--expunge"]))
+ print_collapsed_group("Cleanup")
+ if os.path.exists("WORKSPACE"):
+ fail_if_nonzero(execute_command(["bazel", "clean", "--expunge"]))
def execute_shell_commands(commands):
- if not commands:
- return
- print_collapsed_group("Setup (Shell Commands)")
- shell_command = "\n".join(commands)
- fail_if_nonzero(execute_command([shell_command], shell=True))
+ if not commands:
+ return
+ print_collapsed_group("Setup (Shell Commands)")
+ shell_command = "\n".join(commands)
+ fail_if_nonzero(execute_command([shell_command], shell=True))
def execute_bazel_run(bazel_binary, targets):
- if not targets:
- return
- print_collapsed_group("Setup (Run Targets)")
- for target in targets:
- fail_if_nonzero(execute_command([bazel_binary, "run", "--curses=yes",
- "--color=yes", "--verbose_failurs", target]))
+ if not targets:
+ return
+ print_collapsed_group("Setup (Run Targets)")
+ for target in targets:
+ fail_if_nonzero(execute_command([bazel_binary, "run", "--curses=yes",
+ "--color=yes", "--verbose_failurs", target]))
def execute_bazel_build(bazel_binary, flags, targets):
- if not targets:
- return
- print_expanded_group("Build")
- num_jobs = str(multiprocessing.cpu_count())
- common_flags = ["--curses=yes", "--color=yes", "--keep_going",
- "--verbose_failures", "--jobs=" + num_jobs]
- fail_if_nonzero(execute_command(
- [bazel_binary, "build"] + common_flags + flags + targets))
+ if not targets:
+ return
+ print_expanded_group("Build")
+ num_jobs = str(multiprocessing.cpu_count())
+ common_flags = ["--curses=yes", "--color=yes", "--keep_going",
+ "--verbose_failures", "--jobs=" + num_jobs]
+ fail_if_nonzero(execute_command(
+ [bazel_binary, "build"] + common_flags + flags + targets))
def execute_bazel_test(bazel_binary, flags, targets, bep_file):
- if not targets:
- return 0
- print_expanded_group("Test")
- num_jobs = str(multiprocessing.cpu_count())
- common_flags = ["--curses=yes", "--color=yes", "--keep_going", "--verbose_failures",
- "--flaky_test_attempts=3", "--build_tests_only",
- "--jobs=" + num_jobs, "--local_test_jobs=" + num_jobs,
- "--build_event_json_file=" + bep_file]
- return execute_command([bazel_binary, "test"] + common_flags + flags + targets)
+ if not targets:
+ return 0
+ print_expanded_group("Test")
+ num_jobs = str(multiprocessing.cpu_count())
+ common_flags = ["--curses=yes", "--color=yes", "--keep_going", "--verbose_failures",
+ "--flaky_test_attempts=3", "--build_tests_only",
+ "--jobs=" + num_jobs, "--local_test_jobs=" + num_jobs,
+ "--build_event_json_file=" + bep_file]
+ return execute_command([bazel_binary, "test"] + common_flags + flags + targets)
def fail_if_nonzero(exitcode):
- if exitcode is not 0:
- exit(exitcode)
+ if exitcode is not 0:
+ exit(exitcode)
def upload_test_logs(bep_file, tmpdir):
- if not os.path.exists(bep_file):
- return
- test_logs = test_logs_to_upload(bep_file, tmpdir)
- if test_logs:
- cwd = os.getcwd()
- try:
- os.chdir(tmpdir)
- print_collapsed_group("Uploading test logs")
- for logfile in test_logs:
- relative_path = os.path.relpath(logfile, tmpdir)
- fail_if_nonzero(execute_command(["buildkite-agent", "artifact", "upload",
- relative_path]))
- finally:
- os.chdir(cwd)
+ if not os.path.exists(bep_file):
+ return
+ test_logs = test_logs_to_upload(bep_file, tmpdir)
+ if test_logs:
+ cwd = os.getcwd()
+ try:
+ os.chdir(tmpdir)
+ print_collapsed_group("Uploading test logs")
+ for logfile in test_logs:
+ relative_path = os.path.relpath(logfile, tmpdir)
+ fail_if_nonzero(execute_command(["buildkite-agent", "artifact", "upload",
+ relative_path]))
+ finally:
+ os.chdir(cwd)
def test_logs_to_upload(bep_file, tmpdir):
- failed = test_logs_for_status(bep_file, status="FAILED")
- timed_out = test_logs_for_status(bep_file, status="TIMEOUT")
- flaky = test_logs_for_status(bep_file, status="FLAKY")
- # Rename the test.log files to the target that created them
- # so that it's easy to associate test.log and target.
- new_paths = []
- for label, test_logs in (failed + timed_out + flaky):
- attempt = 0
- if len(test_logs) > 1:
- attempt = 1
- for test_log in test_logs:
- new_path = test_label_to_path(tmpdir, label, attempt)
- os.makedirs(os.path.dirname(new_path), exist_ok=True)
- copyfile(test_logs, new_path)
- new_paths.append(new_path)
- attempt = attempt + 1
- return new_paths
+ failed = test_logs_for_status(bep_file, status="FAILED")
+ timed_out = test_logs_for_status(bep_file, status="TIMEOUT")
+ flaky = test_logs_for_status(bep_file, status="FLAKY")
+ # Rename the test.log files to the target that created them
+ # so that it's easy to associate test.log and target.
+ new_paths = []
+ for label, test_logs in (failed + timed_out + flaky):
+ attempt = 0
+ if len(test_logs) > 1:
+ attempt = 1
+ for test_log in test_logs:
+ new_path = test_label_to_path(tmpdir, label, attempt)
+ os.makedirs(os.path.dirname(new_path), exist_ok=True)
+ copyfile(test_logs, new_path)
+ new_paths.append(new_path)
+ attempt = attempt + 1
+ return new_paths
def test_label_to_path(tmpdir, label, attempt):
- # remove leading //
- path = label[2:]
- path = path.replace(":", "/")
- if attempt == 0:
- path = os.path.join(path, "test.log")
- else:
- path = os.path.join(path, "attempt_" + str(attempt) + ".log")
- return os.path.join(tmpdir, path)
+ # remove leading //
+ path = label[2:]
+ path = path.replace(":", "/")
+ if attempt == 0:
+ path = os.path.join(path, "test.log")
+ else:
+ path = os.path.join(path, "attempt_" + str(attempt) + ".log")
+ return os.path.join(tmpdir, path)
def test_logs_for_status(bep_file, status):
- targets = []
- raw_data = ""
- with open(bep_file) as f:
- raw_data = f.read()
- decoder = json.JSONDecoder()
+ targets = []
+ raw_data = ""
+ with open(bep_file) as f:
+ raw_data = f.read()
+ decoder = json.JSONDecoder()
- pos = 0
- while pos < len(raw_data):
- bep_obj, size = decoder.raw_decode(raw_data[pos:])
- if "testResult" in bep_obj:
- test_target = bep_obj["id"]["testResult"]["label"]
- test_result = bep_obj["testResult"]
- if test_result["status"] == status:
- outputs = test_result["testActionOutput"]
- test_logs = []
- for output in outputs:
- if output["name"] == "test.log":
- test_logs.append(urlparse(output["uri"]).path)
- if test_logs:
- targets.append((test_target, test_logs))
- pos += size + 1
- return targets
+ pos = 0
+ while pos < len(raw_data):
+ bep_obj, size = decoder.raw_decode(raw_data[pos:])
+ if "testResult" in bep_obj:
+ test_target = bep_obj["id"]["testResult"]["label"]
+ test_result = bep_obj["testResult"]
+ if test_result["status"] == status:
+ outputs = test_result["testActionOutput"]
+ test_logs = []
+ for output in outputs:
+ if output["name"] == "test.log":
+ test_logs.append(urlparse(output["uri"]).path)
+ if test_logs:
+ targets.append((test_target, test_logs))
+ pos += size + 1
+ return targets
def execute_command(args, shell=False):
- print(" ".join(args))
- res = subprocess.run(args, shell=shell)
- return res.returncode
+ print(" ".join(args))
+ res = subprocess.run(args, shell=shell)
+ return res.returncode
def print_project_pipeline(platform_configs, project_name, http_config,
git_repository, use_but):
- pipeline_steps = []
- for platform, config in platform_configs.items():
- step = runner_step(platform, project_name, http_config, git_repository,
- use_but)
- pipeline_steps.append(step)
+ pipeline_steps = []
+ for platform, config in platform_configs.items():
+ step = runner_step(platform, project_name, http_config, git_repository,
+ use_but)
+ pipeline_steps.append(step)
- print_pipeline(pipeline_steps)
+ print_pipeline(pipeline_steps)
def runner_step(platform, project_name=None, http_config=None,
git_repository=None, use_but=False, save_but=False, build_only=False,
test_only=False):
- command = python_binary() + " bazelci.py runner --platform=" + platform
- if http_config:
- command = command + " --http_config=" + http_config
- if git_repository:
- command = command + " --git_repository=" + git_repository
- if use_but:
- command = command + " --use_but"
- if save_but:
- command = command + " --save_but"
- if build_only:
- command = command + " --build_only"
- if test_only:
- command = command + " --test_only"
- label = create_label(platform_name(platform),
- project_name, build_only, test_only)
- return """
+ command = python_binary() + " bazelci.py runner --platform=" + platform
+ if http_config:
+ command = command + " --http_config=" + http_config
+ if git_repository:
+ command = command + " --git_repository=" + git_repository
+ if use_but:
+ command = command + " --use_but"
+ if save_but:
+ command = command + " --save_but"
+ if build_only:
+ command = command + " --build_only"
+ if test_only:
+ command = command + " --test_only"
+ label = create_label(platform_name(platform),
+ project_name, build_only, test_only)
+ return """
- label: \"{0}\"
command: \"{1}\\n{2}\"
agents:
@@ -515,35 +512,35 @@
def print_pipeline(steps):
- print("steps:")
- for step in steps:
- print(step)
+ print("steps:")
+ for step in steps:
+ print(step)
def wait_step():
- return """
+ return """
- wait"""
def http_config_flag(http_config):
- if http_config is not None:
- return "--http_config=" + http_config
- return ""
+ if http_config is not None:
+ return "--http_config=" + http_config
+ return ""
def fetch_bazelcipy_command():
- return "curl -s {0} -o bazelci.py".format(bazelcipy_url())
+ return "curl -s {0} -o bazelci.py".format(bazelcipy_url())
def upload_project_pipeline_step(project_name, git_repository, http_config):
- pipeline_command = ("{0} bazelci.py project_pipeline --project_name=\\\"{1}\\\" " +
- "--use_but --git_repository={2}").format(python_binary(), project_name,
- git_repository)
- if http_config:
- pipeline_command = pipeline_command + " --http_config=" + http_config
- pipeline_command = pipeline_command + " | buildkite-agent pipeline upload"
+ pipeline_command = ("{0} bazelci.py project_pipeline --project_name=\\\"{1}\\\" " +
+ "--use_but --git_repository={2}").format(python_binary(), project_name,
+ git_repository)
+ if http_config:
+ pipeline_command = pipeline_command + " --http_config=" + http_config
+ pipeline_command = pipeline_command + " | buildkite-agent pipeline upload"
- return """
+ return """
- label: \"Setup {0}\"
command: \"{1}\\n{2}\"
agents:
@@ -553,32 +550,32 @@
def create_label(platform_name, project_name=None, build_only=False,
test_only=False):
- label = ""
- if build_only:
- label = "Build "
- if test_only:
- label = "Test "
- if project_name:
- label = label + "{0} ({1})".format(project_name, platform_name)
- else:
- label = label + platform_name
- return label
+ label = ""
+ if build_only:
+ label = "Build "
+ if test_only:
+ label = "Test "
+ if project_name:
+ label = label + "{0} ({1})".format(project_name, platform_name)
+ else:
+ label = label + platform_name
+ return label
def bazel_build_step(platform, project_name, http_config=None,
build_only=False, test_only=False):
- pipeline_command = python_binary() + " bazelci.py runner"
- if build_only:
- pipeline_command = pipeline_command + " --build_only --save_but"
- if test_only:
- pipeline_command = pipeline_command + " --test_only"
- if http_config:
- pipeline_command = pipeline_command + " --http_config=" + http_config
- label = create_label(platform_name(platform), project_name, build_only=build_only,
- test_only=test_only)
- pipeline_command = pipeline_command + " --platform=" + platform
+ pipeline_command = python_binary() + " bazelci.py runner"
+ if build_only:
+ pipeline_command = pipeline_command + " --build_only --save_but"
+ if test_only:
+ pipeline_command = pipeline_command + " --test_only"
+ if http_config:
+ pipeline_command = pipeline_command + " --http_config=" + http_config
+ label = create_label(platform_name(platform), project_name, build_only=build_only,
+ test_only=test_only)
+ pipeline_command = pipeline_command + " --platform=" + platform
- return """
+ return """
- label: \"{0}\"
command: \"{1}\\n{2}\"
agents:
@@ -587,8 +584,8 @@
def publish_bazel_binaries_step():
- command = python_binary() + " bazelci.py publish_binaries"
- return """
+ command = python_binary() + " bazelci.py publish_binaries"
+ return """
- label: \"Publish Bazel Binaries\"
command: \"{0}\\n{1}\"
agents:
@@ -596,182 +593,182 @@
def print_bazel_postsubmit_pipeline(configs, http_config):
- if not configs:
- eprint("Bazel postsubmit pipeline configuration is empty.")
- if set(configs.keys()) != set(supported_platforms()):
- eprint("Bazel postsubmit pipeline needs to build Bazel on all " +
- "supported platforms.")
+ if not configs:
+ eprint("Bazel postsubmit pipeline configuration is empty.")
+ if set(configs.keys()) != set(supported_platforms()):
+ eprint("Bazel postsubmit pipeline needs to build Bazel on all " +
+ "supported platforms.")
- pipeline_steps = []
- for platform, config in configs.items():
- pipeline_steps.append(bazel_build_step(platform, "Bazel",
- http_config, build_only=True))
- pipeline_steps.append(wait_step())
+ pipeline_steps = []
+ for platform, config in configs.items():
+ pipeline_steps.append(bazel_build_step(platform, "Bazel",
+ http_config, build_only=True))
+ pipeline_steps.append(wait_step())
- # todo move this to the end with a wait step.
- pipeline_steps.append(publish_bazel_binaries_step())
+ # todo move this to the end with a wait step.
+ pipeline_steps.append(publish_bazel_binaries_step())
- for platform, config in configs.items():
- pipeline_steps.append(bazel_build_step(platform, "Bazel",
- http_config, test_only=True))
- for project, config in downstream_projects().items():
- git_repository = config["git_repository"]
- http_config = config.get("http_config", None)
- pipeline_steps.append(upload_project_pipeline_step(project,
- git_repository, http_config))
+ for platform, config in configs.items():
+ pipeline_steps.append(bazel_build_step(platform, "Bazel",
+ http_config, test_only=True))
+ for project, config in downstream_projects().items():
+ git_repository = config["git_repository"]
+ http_config = config.get("http_config", None)
+ pipeline_steps.append(upload_project_pipeline_step(project,
+ git_repository, http_config))
- print_pipeline(pipeline_steps)
+ print_pipeline(pipeline_steps)
def bazelci_builds_download_url(platform, build_number):
- return "https://storage.googleapis.com/bazel-builds/artifacts/{0}/{1}/bazel".format(platform, build_number)
+ return "https://storage.googleapis.com/bazel-builds/artifacts/{0}/{1}/bazel".format(platform, build_number)
def bazelci_builds_upload_url(platform, build_number):
- return "gs://bazel-builds/artifacts/{0}/{1}/bazel".format(build_number, platform)
+ return "gs://bazel-builds/artifacts/{0}/{1}/bazel".format(build_number, platform)
def bazelci_builds_metadata_url():
- return "gs://bazel-builds/metadata/latest_fully_tested.json"
+ return "gs://bazel-builds/metadata/latest_fully_tested.json"
def latest_generation_and_build_number():
- output = None
- attempt = 0
- while attempt < 5:
- output = subprocess.check_output(
- ["gsutil", "stat", bazelci_builds_metadata_url()])
- match = re.search("Generation:[ ]*([0-9]+)", output.decode("utf-8"))
- if not match:
- eprint("Couldn't parse generation. gsutil output format changed?")
- generation = match.group(1)
+ output = None
+ attempt = 0
+ while attempt < 5:
+ output = subprocess.check_output(
+ ["gsutil", "stat", bazelci_builds_metadata_url()])
+ match = re.search("Generation:[ ]*([0-9]+)", output.decode("utf-8"))
+ if not match:
+ eprint("Couldn't parse generation. gsutil output format changed?")
+ generation = match.group(1)
- match = re.search("Hash \(md5\):[ ]*([^\s]+)", output.decode("utf-8"))
- if not match:
- eprint("Couldn't parse md5 hash. gsutil output format changed?")
- expected_md5hash = base64.b64decode(match.group(1))
+ match = re.search("Hash \(md5\):[ ]*([^\s]+)", output.decode("utf-8"))
+ if not match:
+ eprint("Couldn't parse md5 hash. gsutil output format changed?")
+ expected_md5hash = base64.b64decode(match.group(1))
- output = subprocess.check_output(
- ["gsutil", "cat", bazelci_builds_metadata_url()])
- hasher = hashlib.md5()
- hasher.update(output)
- actual_md5hash = hasher.digest()
+ output = subprocess.check_output(
+ ["gsutil", "cat", bazelci_builds_metadata_url()])
+ hasher = hashlib.md5()
+ hasher.update(output)
+ actual_md5hash = hasher.digest()
- if expected_md5hash == actual_md5hash:
- break
- attempt = attempt + 1
- info = json.loads(output.decode("utf-8"))
- return (generation, info["build_number"])
+ if expected_md5hash == actual_md5hash:
+ break
+ attempt = attempt + 1
+ info = json.loads(output.decode("utf-8"))
+ return (generation, info["build_number"])
def sha256_hexdigest(filename):
- sha256 = hashlib.sha256()
- with open(filename, 'rb') as f:
- for block in iter(lambda: f.read(65536), b''):
- sha256.update(block)
- return sha256.hexdigest()
+ sha256 = hashlib.sha256()
+ with open(filename, 'rb') as f:
+ for block in iter(lambda: f.read(65536), b''):
+ sha256.update(block)
+ return sha256.hexdigest()
def try_publish_binaries(build_number, expected_generation):
- tmpdir = None
- try:
- tmpdir = tempfile.mkdtemp()
- info = {
- "build_number": build_number,
- "git_commit": os.environ["BUILDKITE_COMMIT"],
- "platforms": {}
- }
- for platform in supported_platforms():
- bazel_binary_path = download_bazel_binary(tmpdir, platform)
- fail_if_nonzero(execute_command(["gsutil", "cp", "-a", "public-read", bazel_binary_path,
- bazelci_builds_upload_url(platform, build_number)]))
- info["platforms"][platform] = {
- "url": bazelci_builds_download_url(platform, build_number),
- "sha256": sha256_hexdigest(bazel_binary_path),
- }
+ tmpdir = None
+ try:
+ tmpdir = tempfile.mkdtemp()
+ info = {
+ "build_number": build_number,
+ "git_commit": os.environ["BUILDKITE_COMMIT"],
+ "platforms": {}
+ }
+ for platform in supported_platforms():
+ bazel_binary_path = download_bazel_binary(tmpdir, platform)
+ fail_if_nonzero(execute_command(["gsutil", "cp", "-a", "public-read", bazel_binary_path,
+ bazelci_builds_upload_url(platform, build_number)]))
+ info["platforms"][platform] = {
+ "url": bazelci_builds_download_url(platform, build_number),
+ "sha256": sha256_hexdigest(bazel_binary_path),
+ }
- info_file = os.path.join(tmpdir, "info.json")
- with open(info_file, mode="w", encoding="utf-8") as fp:
- json.dump(info, fp)
- exitcode = execute_command(["gsutil", "-h", "x-goog-if-generation-match:" + expected_generation,
- "-h", "Content-Type:application/json", "cp", "-a",
- "public-read", info_file, bazelci_builds_metadata_url(platform)])
- return exitcode == 0
- finally:
- if tmpdir:
- shutil.rmtree(tmpdir)
+ info_file = os.path.join(tmpdir, "info.json")
+ with open(info_file, mode="w", encoding="utf-8") as fp:
+ json.dump(info, fp)
+ exitcode = execute_command(["gsutil", "-h", "x-goog-if-generation-match:" + expected_generation,
+ "-h", "Content-Type:application/json", "cp", "-a",
+ "public-read", info_file, bazelci_builds_metadata_url(platform)])
+ return exitcode == 0
+ finally:
+ if tmpdir:
+ shutil.rmtree(tmpdir)
def publish_binaries():
- '''
- Publish Bazel binaries to GCS.
- '''
- attempt = 0
- while attempt < 5:
- latest_generation, latest_build_number = latest_generation_and_build_number()
+ '''
+ Publish Bazel binaries to GCS.
+ '''
+ attempt = 0
+ while attempt < 5:
+ latest_generation, latest_build_number = latest_generation_and_build_number()
- current_build_number = os.environ.get("BUILDKITE_BUILD_NUMBER", None)
- if not current_build_number:
- eprint("Not running inside Buildkite")
- current_build_number = int(current_build_number)
- if current_build_number <= latest_build_number:
- print(("Current build '{0}' is not newer than latest published '{1}'. " +
- "Skipping publishing of binaries.").format(current_build_number,
- latest_build_number))
- break
+ current_build_number = os.environ.get("BUILDKITE_BUILD_NUMBER", None)
+ if not current_build_number:
+ eprint("Not running inside Buildkite")
+ current_build_number = int(current_build_number)
+ if current_build_number <= latest_build_number:
+ print(("Current build '{0}' is not newer than latest published '{1}'. " +
+ "Skipping publishing of binaries.").format(current_build_number,
+ latest_build_number))
+ break
- if try_publish_binaries(current_build_number, latest_generation):
- print("Successfully updated '{0}' to binaries from build {1}."
- .format(bazelci_builds_metadata_url(), current_build_number))
- break
- attempt = attempt + 1
+ if try_publish_binaries(current_build_number, latest_generation):
+ print("Successfully updated '{0}' to binaries from build {1}."
+ .format(bazelci_builds_metadata_url(), current_build_number))
+ break
+ attempt = attempt + 1
if __name__ == "__main__":
- parser = argparse.ArgumentParser(
- description='Bazel Continuous Integration Script')
+ parser = argparse.ArgumentParser(
+ description='Bazel Continuous Integration Script')
- subparsers = parser.add_subparsers(dest="subparsers_name")
- bazel_postsubmit_pipeline = subparsers.add_parser(
- "bazel_postsubmit_pipeline")
- bazel_postsubmit_pipeline.add_argument("--http_config", type=str)
- bazel_postsubmit_pipeline.add_argument("--git_repository", type=str)
+ subparsers = parser.add_subparsers(dest="subparsers_name")
+ bazel_postsubmit_pipeline = subparsers.add_parser(
+ "bazel_postsubmit_pipeline")
+ bazel_postsubmit_pipeline.add_argument("--http_config", type=str)
+ bazel_postsubmit_pipeline.add_argument("--git_repository", type=str)
- project_pipeline = subparsers.add_parser("project_pipeline")
- project_pipeline.add_argument("--project_name", type=str)
- project_pipeline.add_argument("--http_config", type=str)
- project_pipeline.add_argument("--git_repository", type=str)
- project_pipeline.add_argument(
- "--use_but", type=bool, nargs="?", const=True)
+ project_pipeline = subparsers.add_parser("project_pipeline")
+ project_pipeline.add_argument("--project_name", type=str)
+ project_pipeline.add_argument("--http_config", type=str)
+ project_pipeline.add_argument("--git_repository", type=str)
+ project_pipeline.add_argument(
+ "--use_but", type=bool, nargs="?", const=True)
- runner = subparsers.add_parser("runner")
- runner.add_argument("--platform", action="store",
- choices=list(supported_platforms()))
- runner.add_argument("--http_config", type=str)
- runner.add_argument("--git_repository", type=str)
- runner.add_argument("--use_but", type=bool, nargs="?", const=True)
- runner.add_argument("--save_but", type=bool, nargs="?", const=True)
- runner.add_argument("--build_only", type=bool, nargs="?", const=True)
- runner.add_argument("--test_only", type=bool, nargs="?", const=True)
+ runner = subparsers.add_parser("runner")
+ runner.add_argument("--platform", action="store",
+ choices=list(supported_platforms()))
+ runner.add_argument("--http_config", type=str)
+ runner.add_argument("--git_repository", type=str)
+ runner.add_argument("--use_but", type=bool, nargs="?", const=True)
+ runner.add_argument("--save_but", type=bool, nargs="?", const=True)
+ runner.add_argument("--build_only", type=bool, nargs="?", const=True)
+ runner.add_argument("--test_only", type=bool, nargs="?", const=True)
- runner = subparsers.add_parser("publish_binaries")
+ runner = subparsers.add_parser("publish_binaries")
- args = parser.parse_args()
+ args = parser.parse_args()
- if args.subparsers_name == "bazel_postsubmit_pipeline":
- configs = fetch_configs(args.http_config)
- print_bazel_postsubmit_pipeline(configs.get("platforms", None),
- args.http_config)
- elif args.subparsers_name == "project_pipeline":
- configs = fetch_configs(args.http_config)
- print_project_pipeline(configs.get("platforms", None), args.project_name,
- args.http_config, args.git_repository, args.use_but)
- elif args.subparsers_name == "runner":
- configs = fetch_configs(args.http_config)
- execute_commands(configs.get("platforms", None)[args.platform],
- args.platform, args.git_repository, args.use_but, args.save_but,
- args.build_only, args.test_only)
- elif args.subparsers_name == "publish_binaries":
- publish_binaries()
- else:
- parser.print_help()
+ if args.subparsers_name == "bazel_postsubmit_pipeline":
+ configs = fetch_configs(args.http_config)
+ print_bazel_postsubmit_pipeline(configs.get("platforms", None),
+ args.http_config)
+ elif args.subparsers_name == "project_pipeline":
+ configs = fetch_configs(args.http_config)
+ print_project_pipeline(configs.get("platforms", None), args.project_name,
+ args.http_config, args.git_repository, args.use_but)
+ elif args.subparsers_name == "runner":
+ configs = fetch_configs(args.http_config)
+ execute_commands(configs.get("platforms", None)[args.platform],
+ args.platform, args.git_repository, args.use_but, args.save_but,
+ args.build_only, args.test_only)
+ elif args.subparsers_name == "publish_binaries":
+ publish_binaries()
+ else:
+ parser.print_help()
diff --git a/buildkite/setup.cfg b/buildkite/setup.cfg
deleted file mode 100644
index 204c84c..0000000
--- a/buildkite/setup.cfg
+++ /dev/null
@@ -1,3 +0,0 @@
-[pep8]
-max-line-length = 80
-indent-size = 2
