Add bcr_compatibility.py (#2079)
Stack on https://github.com/bazelbuild/continuous-integration/pull/2077
- Added `bcr_compatibility.py` for
https://buildkite.com/bazel/bcr-bazel-compatibility-test
- Added documentation for BCR scripts.
diff --git a/buildkite/bazel-central-registry/README.md b/buildkite/bazel-central-registry/README.md
new file mode 100644
index 0000000..f002499
--- /dev/null
+++ b/buildkite/bazel-central-registry/README.md
@@ -0,0 +1,39 @@
+## BCR Postsubmit
+
+`bcr_postsubmit.py` is a script used for Bazel Central Registry (BCR) postsubmit operations. It synchronizes the `bazel_registry.json` and the `modules/` directory from the main branch of the Bazel Central Registry to the BCR's public cloud storage bucket.
+
+## BCR Presubmit
+
+`bcr_presubmit.py` is a script used for Bazel Central Registry (BCR) [presubmit operations](https://github.com/bazelbuild/bazel-central-registry/blob/main/docs/README.md#presubmit). This script primarily handles the preparation and execution of tests for new modules or updated versions of modules being added to the Bazel Central Registry.
+
+This script powers the [BCR Presubmit](https://buildkite.com/bazel/bcr-presubmit) pipeline.
+
+## BCR Bazel Compatibility Test
+
+`bcr_compatibility.py` is a script used for testing compatibility between any versions of Bazel and BCR modules, and optionally with given incompatible flags.
+
+A new build can be triggered via the [BCR Bazel Compatibility Test](https://buildkite.com/bazel/bcr-bazel-compatibility-test) pipeline with the following environment variables:
+
+* `MODULE_SELECTIONS`: (Mandatory) A comma-separated list of module patterns to be tested in the format `<module_pattern>@<version_pattern>`. A module is selected if it matches any of the given patterns.
+
+ The `<module_pattern>` can include wildcards (*) to match multiple modules (e.g. `rules_*`).
+
+ The `<version_pattern>` can be:
+
+ - A specific version (e.g. `1.2.3`)
+ - `latest` to select the latest version
+ - A comparison operator followed by a version (e.g. `>=1.0.0`, `<2.0.0`)
+
+ Examples: `rules_cc@0.0.13,rules_java@latest`, `rules_*@latest`, `protobuf@<29.0-rc1`
+
+* `SMOKE_TEST_PERCENTAGE`: (Optional) Specifies a percentage of selected modules to be randomly sampled for smoke testing.
+
+ For example, if `MODULE_SELECTIONS=rules_*@latest` and `SMOKE_TEST_PERCENTAGE=10`, then 10% of modules with name starting with `rules_` will be randomly selected.
+
+* `USE_BAZEL_VERSION`: (Optional) Specifies the Bazel version to be used. The script will override Bazel version for all task configs.
+
+* `USE_BAZELISK_MIGRATE`: (Optional) Set this env var to `1` to enable testing incompatible flags with Bazelisk's [`--migrate`](https://github.com/bazelbuild/bazelisk?tab=readme-ov-file#--migrate) feature. A report will be generated for the pipeline if this feature is enabled.
+
+* `INCOMPATIBLE_FLAGS`: (Optional) Specifies the list of incompatible flags to be tested with Bazelisk. By default incompatible flags are fetched by parsing titles of [open Bazel Github issues](https://github.com/bazelbuild/bazel/issues?q=is%3Aopen+is%3Aissue+label%3Aincompatible-change+label%3Amigration-ready) with `incompatible-change` and `migration-ready` labels. Make sure the Bazel version you select support those flags.
+
+* `CI_RESOURCE_PERCENTAGE`: (Optional) Specifies the percentage of CI machine resources to use for running tests. Default is 30%. **ATTENTION**: please do NOT overwhelm CI during busy hours.
diff --git a/buildkite/bazel-central-registry/bcr_compatibility.py b/buildkite/bazel-central-registry/bcr_compatibility.py
new file mode 100644
index 0000000..a3397d3
--- /dev/null
+++ b/buildkite/bazel-central-registry/bcr_compatibility.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+#
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# pylint: disable=line-too-long
+# pylint: disable=missing-function-docstring
+# pylint: disable=unspecified-encoding
+# pylint: disable=invalid-name
+"""The CI script for BCR Bazel Compatibility Test pipeline."""
+
+
+import os
+import sys
+import subprocess
+
+import bazelci
+import bcr_presubmit
+
+CI_MACHINE_NUM = {
+ "bazel": {
+ "default": 100,
+ "windows": 30,
+ "macos_arm64": 45,
+ "macos": 110,
+ "arm64": 1,
+ },
+ "bazel-testing": {
+ "default": 30,
+ "windows": 4,
+ "macos_arm64": 1,
+ "macos": 10,
+ "arm64": 1,
+ },
+}[bazelci.BUILDKITE_ORG]
+
+# Default to use only 30% of CI resources for each type of machines.
+CI_RESOURCE_PERCENTAGE = int(os.environ.get('CI_RESOURCE_PERCENTAGE', 30))
+
+
+def select_modules_from_env_vars():
+ """
+ Parses MODULE_SELECTIONS and SMOKE_TEST_PERCENTAGE environment variables
+ and returns a list of selected module versions.
+ """
+ MODULE_SELECTIONS = os.environ.get('MODULE_SELECTIONS', '')
+ SMOKE_TEST_PERCENTAGE = os.environ.get('SMOKE_TEST_PERCENTAGE', None)
+
+ if not MODULE_SELECTIONS:
+ return []
+
+ selections = [s.strip() for s in MODULE_SELECTIONS.split(',') if s.strip()]
+ args = [f"--select={s}" for s in selections]
+ if SMOKE_TEST_PERCENTAGE:
+ args += [f"--random-percentage={SMOKE_TEST_PERCENTAGE}"]
+ output = subprocess.check_output(
+ ["python3", "./tools/module_selector.py"] + args,
+ )
+ modules = []
+ for line in output.decode("utf-8").split():
+ name, version = line.strip().split("@")
+ modules.append((name, version))
+ return modules
+
+
+def get_target_modules():
+ """
+ If the `MODULE_SELECTIONS` and `SMOKE_TEST_PERCENTAGE(S)` are specified, calculate the target modules from those env vars.
+ Otherwise, calculate target modules based on changed files from the main branch.
+ """
+ if "MODULE_SELECTIONS" not in os.environ:
+ raise ValueError("Please set MODULE_SELECTIONS env var to select modules for testing!")
+
+ modules = select_modules_from_env_vars()
+ if modules:
+ bazelci.print_expanded_group("The following modules are selected:\n\n%s" % "\n".join([f"{name}@{version}" for name, version in modules]))
+ return sorted(list(set(modules)))
+ else:
+ raise ValueError("MODULE_SELECTIONS env var didn't select any modules!")
+
+
+def create_step_for_report_flags_results():
+ parts = [
+ bazelci.PLATFORMS[bazelci.DEFAULT_PLATFORM]["python"],
+ "aggregate_incompatible_flags_test_result.py",
+ "--build_number=%s" % os.getenv("BUILDKITE_BUILD_NUMBER"),
+ ]
+ return [
+ {"wait": "~", "continue_on_failure": "true"},
+ bazelci.create_step(
+ label="Aggregate incompatible flags test result",
+ commands=[
+ bazelci.fetch_bazelcipy_command(),
+ bazelci.fetch_aggregate_incompatible_flags_test_result_command(),
+ " ".join(parts),
+ ],
+ platform=bazelci.DEFAULT_PLATFORM,
+ ),
+ ]
+
+def main():
+ modules = get_target_modules()
+ pipeline_steps = []
+ # A function to calculate concurrency number for each BuildKite queue
+ calc_concurrency = lambda queue : max(1, (CI_RESOURCE_PERCENTAGE * CI_MACHINE_NUM[queue]) // 100)
+ # Respect USE_BAZEL_VERSION to override bazel version in presubmit.yml files.
+ bazel_version = os.environ.get("USE_BAZEL_VERSION")
+ for module_name, module_version in modules:
+ previous_size = len(pipeline_steps)
+
+ configs = bcr_presubmit.get_anonymous_module_task_config(module_name, module_version, bazel_version)
+ bcr_presubmit.add_presubmit_jobs(module_name, module_version, configs.get("tasks", {}), pipeline_steps, overwrite_bazel_version=bazel_version, calc_concurrency=calc_concurrency)
+ configs = bcr_presubmit.get_test_module_task_config(module_name, module_version, bazel_version)
+ bcr_presubmit.add_presubmit_jobs(module_name, module_version, configs.get("tasks", {}), pipeline_steps, is_test_module=True, overwrite_bazel_version=bazel_version, calc_concurrency=calc_concurrency)
+
+ if len(pipeline_steps) == previous_size:
+ bcr_presubmit.error("No pipeline steps generated for %s@%s. Please check the configuration." % (module_name, module_version))
+
+ if pipeline_steps:
+ # Always wait for approval to proceed
+ pipeline_steps = [{"block": "Please review generated jobs before proceeding", "blocked_state": "running"}] + pipeline_steps
+ if bazelci.use_bazelisk_migrate():
+ pipeline_steps += create_step_for_report_flags_results()
+
+ bcr_presubmit.upload_jobs_to_pipeline(pipeline_steps)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/buildkite/bazel-central-registry/bcr_presubmit.py b/buildkite/bazel-central-registry/bcr_presubmit.py
index b48f459..ef4d99d 100755
--- a/buildkite/bazel-central-registry/bcr_presubmit.py
+++ b/buildkite/bazel-central-registry/bcr_presubmit.py
@@ -499,7 +499,7 @@
error("No pipeline steps generated for %s@%s. Please check the configuration." % (module_name, module_version))
if should_wait_bcr_maintainer_review(modules) and pipeline_steps:
- pipeline_steps = [{"block": "Wait on BCR maintainer review", "blocked_state": "running"}] + pipeline_steps
+ pipeline_steps.insert(0, {"block": "Wait on BCR maintainer review", "blocked_state": "running"})
upload_jobs_to_pipeline(pipeline_steps)
elif args.subparsers_name == "anonymous_module_runner":
