Refactor Buildkite API calls. (#544)
This commit merges the almost-identical code for calling the Buildkite API from incompatible_flag_verbose_failures and aggregate_incompatible_flags_test_result into a single class in bazelci.py.
While this seems like a good idea from a clean-code-point of view,
it also helps us with https://github.com/bazelbuild/continuous-integration/issues/543 since we need this functionality in bazelci anyway [*].
Moreover, this commit fixes some lint warnings.
[*] The last green commit for a pipeline is only updated if all previous steps for a given build have finished successfully. However, this means that a failing Buildifier step prevents us from updating the last green commit, even if all tests have passed.
Buildkite doesn't ofter an easy solution since there is no way to say "only execute this step until *some* of the previous ones have succeeded".
Putting the Buildifier step to the end of the pipeline doesn't make sense either since that step would never run if there was a test failure.
Consequently, the solution is to wait for all jobs to finish, then always run the "try update last green commit" step (even if previous steps failed), use the newly-added Buildkite client class to check whether all steps except for Buildifier have succeeded, and then update the commit (or not, depending on the step results)
diff --git a/buildkite/bazelci.py b/buildkite/bazelci.py
index 435241d..226c3ec 100644
--- a/buildkite/bazelci.py
+++ b/buildkite/bazelci.py
@@ -418,6 +418,61 @@
pass
+class BuildkiteClient(object):
+
+ _ENCRYPTED_BUILDKITE_API_TOKEN = """
+CiQA4DEB9ldzC+E39KomywtqXfaQ86hhulgeDsicds2BuvbCYzsSUAAqwcvXZPh9IMWlwWh94J2F
+exosKKaWB0tSRJiPKnv2NPDfEqGul0ZwVjtWeASpugwxxKeLhFhPMcgHMPfndH6j2GEIY6nkKRbP
+uwoRMCwe
+""".strip()
+
+ _BUILD_STATUS_URL_TEMPLATE = ("https://api.buildkite.com/v2/"
+ "organizations/{}/pipelines/{}/builds/{}")
+
+ def __init__(self, org, pipeline):
+ self._org = org
+ self._pipeline = pipeline
+ self._token = self._get_buildkite_token()
+
+ def _get_buildkite_token(self):
+ return (
+ subprocess.check_output(
+ [
+ gcloud_command(),
+ "kms",
+ "decrypt",
+ "--project",
+ "bazel-untrusted",
+ "--location",
+ "global",
+ "--keyring",
+ "buildkite",
+ "--key",
+ "buildkite-untrusted-api-token",
+ "--ciphertext-file",
+ "-",
+ "--plaintext-file",
+ "-",
+ ],
+ input=base64.b64decode(self._ENCRYPTED_BUILDKITE_API_TOKEN),
+ env=os.environ,
+ )
+ .decode("utf-8")
+ .strip()
+ )
+
+ def _open_url(self, url):
+ return urllib.request.urlopen("{}?access_token={}".format(url, self._token)).read().decode("utf-8")
+
+ def get_build_info(self, build_number):
+ url = self._BUILD_STATUS_URL_TEMPLATE.format(self._org, self._pipeline, build_number)
+ output = self._open_url(url)
+ return json.loads(output)
+
+ def get_build_log(self, job):
+ return self._open_url(job["raw_log_url"])
+
+
def eprint(*args, **kwargs):
"""
Print to stderr and flush (just in case).
