BCR bazel compatibility test: Add generate_report.py (#2091)
This will be run in the last step for a
https://buildkite.com/bazel/bcr-bazel-compatibility-test build without
`USE_BAZELISK_MIGRATE` and generate a report in markdown that can be
used for filing a GitHub issue report for broken modules.
e.g.:
<img width="685" alt="image"
src="https://github.com/user-attachments/assets/3e888dd0-601c-4b38-bce0-047eb80be9bb">
diff --git a/buildkite/bazel-central-registry/bcr_compatibility.py b/buildkite/bazel-central-registry/bcr_compatibility.py
index 2c8253d..3f8c559 100644
--- a/buildkite/bazel-central-registry/bcr_compatibility.py
+++ b/buildkite/bazel-central-registry/bcr_compatibility.py
@@ -23,6 +23,7 @@
import os
import sys
import subprocess
+import time
import bazelci
import bcr_presubmit
@@ -47,6 +48,14 @@
# Default to use only 30% of CI resources for each type of machines.
CI_RESOURCE_PERCENTAGE = int(os.environ.get('CI_RESOURCE_PERCENTAGE', 30))
+SCRIPT_URL = "https://raw.githubusercontent.com/bazelbuild/continuous-integration/{}/buildkite/bazel-central-registry/generate_report.py?{}".format(
+ bazelci.GITHUB_BRANCH, int(time.time())
+)
+
+
+def fetch_generate_report_py_command():
+ return "curl -s {0} -o generate_report.py".format(SCRIPT_URL)
+
def select_modules_from_env_vars():
"""
@@ -108,6 +117,26 @@
),
]
+def create_step_for_generate_report():
+ parts = [
+ bazelci.PLATFORMS[bazelci.DEFAULT_PLATFORM]["python"],
+ "generate_report.py",
+ "--build_number=%s" % os.getenv("BUILDKITE_BUILD_NUMBER"),
+ ]
+ return [
+ {"wait": "~", "continue_on_failure": "true"},
+ bazelci.create_step(
+ label="Generate report in markdown",
+ commands=[
+ bazelci.fetch_bazelcipy_command(),
+ bcr_presubmit.fetch_bcr_presubmit_py_command(),
+ fetch_generate_report_py_command(),
+ " ".join(parts),
+ ],
+ platform=bazelci.DEFAULT_PLATFORM,
+ ),
+ ]
+
def main():
modules = get_target_modules()
pipeline_steps = []
@@ -127,6 +156,8 @@
pipeline_steps.insert(0, {"block": "Please review generated jobs before proceeding", "blocked_state": "running"})
if bazelci.use_bazelisk_migrate():
pipeline_steps += create_step_for_report_flags_results()
+ else:
+ pipeline_steps += create_step_for_generate_report()
bcr_presubmit.upload_jobs_to_pipeline(pipeline_steps)
diff --git a/buildkite/bazel-central-registry/generate_report.py b/buildkite/bazel-central-registry/generate_report.py
new file mode 100755
index 0000000..f9176ff
--- /dev/null
+++ b/buildkite/bazel-central-registry/generate_report.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+#
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# pylint: disable=line-too-long
+# pylint: disable=missing-function-docstring
+# pylint: disable=unspecified-encoding
+# pylint: disable=invalid-name
+"""The CI script for generate report for BCR Bazel Compatibility Test pipeline."""
+
+
+import argparse
+import collections
+import os
+import json
+import re
+import sys
+
+import bazelci
+import bcr_presubmit
+
+BUILDKITE_ORG = os.environ["BUILDKITE_ORGANIZATION_SLUG"]
+
+PIPELINE = os.environ["BUILDKITE_PIPELINE_SLUG"]
+
+MODULE_VERSION_PATTERN = re.compile(r'(?P<module_version>[a-z](?:[a-z0-9._-]*[a-z0-9])?@[^\s]+)')
+
+def extract_module_version(line):
+ match = MODULE_VERSION_PATTERN.search(line)
+ if match:
+ return match.group("module_version")
+
+
+def get_github_maintainer(module_name):
+ metadata = json.load(open(bcr_presubmit.get_metadata_json(module_name), "r"))
+ github_maintainers = []
+ for maintainer in metadata["maintainers"]:
+ if "github" in maintainer:
+ github_maintainers.append(maintainer["github"])
+
+ if not github_maintainers:
+ github_maintainers.append("bazelbuild/bcr-maintainers")
+ return github_maintainers
+
+
+def print_report_in_markdown(failed_jobs_per_module, pipeline_url):
+ bazel_version = os.environ.get("USE_BAZEL_VERSION")
+ print("\n")
+ print("## The following modules are broken%s:" % (f" with Bazel@{bazel_version}" if bazel_version else ""))
+ print("BCR Bazel Compatibility Test: ", pipeline_url)
+ for module, jobs in failed_jobs_per_module.items():
+ module_name = module.strip().split("@")[0]
+ github_maintainers = get_github_maintainer(module_name)
+ print(f"### {module}")
+ print("Maintainers: ", ", ".join(f"@{maintainer}" for maintainer in github_maintainers))
+ for job in jobs:
+ print(f"- [{job['name']}]({job['web_url']})")
+ print("\n")
+
+
+def main(argv=None):
+ if argv is None:
+ argv = sys.argv[1:]
+
+ parser = argparse.ArgumentParser(description="Script to report BCR Bazel Compatibility Test result.")
+ parser.add_argument("--build_number", type=str)
+
+ args = parser.parse_args(argv)
+ if not args.build_number:
+ parser.print_help()
+ return 2
+
+ client = bazelci.BuildkiteClient(org=BUILDKITE_ORG, pipeline=PIPELINE)
+ build_info = client.get_build_info(args.build_number)
+ failed_jobs_per_module = collections.defaultdict(list)
+ for job in build_info["jobs"]:
+ if job.get("state") == "failed" and "name" in job:
+ module = extract_module_version(job["name"])
+ if not module:
+ continue
+ failed_jobs_per_module[module].append(job)
+
+ print_report_in_markdown(failed_jobs_per_module, build_info["web_url"])
+
+if __name__ == "__main__":
+ sys.exit(main())
