Add Protobuf (main, 33.x, 35.x) to downstream pipeline (#2808)
This PR configures Protobuf in the downstream pipeline:
- Adds Protobuf (main), Protobuf 33.x, and Protobuf 35.x to
`DOWNSTREAM_PROJECTS_PRODUCTION` using `.bazelci/presubmit.yml` from
`https://github.com/protocolbuffers/protobuf.git`.
- Removes obsolete `pipelines/protobuf.yml`.
- Supports `git_branch` in `DOWNSTREAM_PROJECTS` by passing
`git_commit="origin/<branch>"` to the downstream `project_pipeline` step
and skipping `get_last_green_commit` when a branch is configured.
- Adds unit tests in `buildkite/bazelci_test.py`.
diff --git a/buildkite/bazelci.py b/buildkite/bazelci.py
index 858ba12..01ab9bc 100755
--- a/buildkite/bazelci.py
+++ b/buildkite/bazelci.py
@@ -242,6 +242,23 @@
"pipeline_slug": "intellij-plugin-aspect-google",
"disabled_reason": "https://github.com/bazelbuild/intellij/issues/7232#issuecomment-2617127267",
},
+ "Protobuf": {
+ "git_repository": "https://github.com/protocolbuffers/protobuf.git",
+ "file_config": ".bazelci/presubmit.yml",
+ "pipeline_slug": "protobuf",
+ },
+ "Protobuf 33.x": {
+ "git_repository": "https://github.com/protocolbuffers/protobuf.git",
+ "git_branch": "33.x",
+ "file_config": ".bazelci/presubmit.yml",
+ "pipeline_slug": "protobuf",
+ },
+ "Protobuf 35.x": {
+ "git_repository": "https://github.com/protocolbuffers/protobuf.git",
+ "git_branch": "35.x",
+ "file_config": ".bazelci/presubmit.yml",
+ "pipeline_slug": "protobuf",
+ },
"Stardoc": {
"git_repository": "https://github.com/bazelbuild/stardoc.git",
"pipeline_slug": "stardoc",
@@ -3268,10 +3285,11 @@
http_config,
file_config,
git_repository,
- monitor_flaky_tests,
- use_but,
- notify,
- print_shard_summary,
+ git_commit=None,
+ monitor_flaky_tests=False,
+ use_but=False,
+ notify=False,
+ print_shard_summary=False,
):
task_configs = configs.get("tasks", None)
if not task_configs:
@@ -3299,7 +3317,8 @@
)
# In Bazel Downstream Project pipelines, we should test the project at the last green commit.
- git_commit = get_last_green_commit(project_name) if is_downstream_pipeline() else None
+ if is_downstream_pipeline() and not git_commit:
+ git_commit = get_last_green_commit(project_name)
config_hashes = set()
skipped_downstream_tasks = []
@@ -3729,7 +3748,9 @@
)
-def upload_project_pipeline_step(project_name, git_repository, http_config, file_config):
+def upload_project_pipeline_step(
+ project_name, git_repository, http_config, file_config, git_commit=None
+):
pipeline_command = (
'{0} bazelci.py project_pipeline --project_name="{1}" ' + "--git_repository={2}"
).format(PLATFORMS[DEFAULT_PLATFORM]["python"], project_name, git_repository)
@@ -3738,6 +3759,8 @@
pipeline_command += " --http_config=" + http_config
if file_config:
pipeline_command += " --file_config=" + file_config
+ if git_commit:
+ pipeline_command += " --git_commit=" + git_commit
pipeline_command += " | tee /dev/tty | buildkite-agent pipeline upload"
return create_step(
@@ -4075,12 +4098,16 @@
if (test_disabled_projects and disabled_reason) or (
not test_disabled_projects and not disabled_reason
):
+ git_commit = (
+ "origin/" + config["git_branch"] if "git_branch" in config else None
+ )
pipeline_steps.append(
upload_project_pipeline_step(
project_name=project,
git_repository=config["git_repository"],
http_config=config.get("http_config", None),
file_config=config.get("file_config", None),
+ git_commit=git_commit,
)
)
@@ -4201,6 +4228,9 @@
def get_last_green_commit(project_name):
+ # If a specific branch is configured for downstream testing, don't use the main branch's last green commit
+ if DOWNSTREAM_PROJECTS.get(project_name, {}).get("git_branch"):
+ return None
last_green_commit_url = bazelci_last_green_commit_url(
DOWNSTREAM_PROJECTS[project_name]["git_repository"],
DOWNSTREAM_PROJECTS[project_name]["pipeline_slug"],
@@ -4979,6 +5009,9 @@
project_pipeline.add_argument("--file_config", type=str)
project_pipeline.add_argument("--http_config", type=str)
project_pipeline.add_argument("--git_repository", type=str)
+ project_pipeline.add_argument(
+ "--git_commit", type=str, help="Reset the git repository to this commit after cloning it"
+ )
project_pipeline.add_argument("--monitor_flaky_tests", type=bool, nargs="?", const=True)
project_pipeline.add_argument("--use_but", type=bool, nargs="?", const=True)
project_pipeline.add_argument("--notify", type=bool, nargs="?", const=True)
@@ -5048,7 +5081,13 @@
# Fetch the repo in case we need to use file_config.
if args.git_repository:
git_commit = (
- get_last_green_commit(args.project_name) if is_downstream_pipeline() else None
+ args.git_commit
+ if args.git_commit
+ else (
+ get_last_green_commit(args.project_name)
+ if is_downstream_pipeline()
+ else None
+ )
)
clone_git_repository(args.git_repository, git_commit, suppress_stdout=True)
@@ -5059,6 +5098,7 @@
http_config=args.http_config,
file_config=args.file_config,
git_repository=args.git_repository,
+ git_commit=args.git_commit,
monitor_flaky_tests=args.monitor_flaky_tests,
use_but=args.use_but,
notify=args.notify,
diff --git a/buildkite/bazelci_test.py b/buildkite/bazelci_test.py
index 5437884..141d055 100755
--- a/buildkite/bazelci_test.py
+++ b/buildkite/bazelci_test.py
@@ -673,6 +673,83 @@
self.assertEqual(bazelci.get_ci_script_ref(), "testing")
+class GitBranchHandlingTest(unittest.TestCase):
+
+ def test_upload_project_pipeline_step_with_git_commit(self):
+ step = bazelci.upload_project_pipeline_step(
+ project_name="Protobuf 35.x",
+ git_repository="https://github.com/protocolbuffers/protobuf.git",
+ http_config=None,
+ file_config=".bazelci/presubmit.yml",
+ git_commit="origin/35.x",
+ )
+ commands = step["command"]
+ project_pipeline_cmd = [c for c in commands if "project_pipeline" in c][0]
+ self.assertIn("--git_commit=origin/35.x", project_pipeline_cmd)
+ self.assertIn('--project_name="Protobuf 35.x"', project_pipeline_cmd)
+ self.assertIn('--file_config=.bazelci/presubmit.yml', project_pipeline_cmd)
+
+ def test_print_bazel_downstream_pipeline_passes_origin_branch_as_git_commit(self):
+ test_projects = {
+ "Protobuf 35.x": {
+ "git_repository": "https://github.com/protocolbuffers/protobuf.git",
+ "git_branch": "35.x",
+ "file_config": ".bazelci/presubmit.yml",
+ "pipeline_slug": "protobuf",
+ }
+ }
+ with mock.patch.dict(bazelci.DOWNSTREAM_PROJECTS, test_projects, clear=True), \
+ mock.patch.object(bazelci, "upload_project_pipeline_step") as mock_upload, \
+ mock.patch.object(bazelci, "print_pipeline_steps"), \
+ mock.patch.object(bazelci, "bazel_build_step", return_value={"label": "Bazel"}), \
+ mock.patch.object(bazelci, "get_platform_for_task", return_value="ubuntu2004"):
+ bazelci.print_bazel_downstream_pipeline(
+ task_configs={"basic": {}},
+ http_config=None,
+ file_config=None,
+ test_disabled_projects=False,
+ notify=False,
+ )
+ mock_upload.assert_called_once_with(
+ project_name="Protobuf 35.x",
+ git_repository="https://github.com/protocolbuffers/protobuf.git",
+ http_config=None,
+ file_config=".bazelci/presubmit.yml",
+ git_commit="origin/35.x",
+ )
+
+ def test_get_last_green_commit_ignores_projects_with_git_branch(self):
+ with mock.patch.dict(
+ bazelci.DOWNSTREAM_PROJECTS,
+ {
+ "TestProject": {
+ "git_repository": "https://github.com/foo/bar.git",
+ "pipeline_slug": "bar",
+ "git_branch": "35.x",
+ }
+ },
+ clear=True,
+ ):
+ self.assertIsNone(bazelci.get_last_green_commit("TestProject"))
+
+ def test_clone_git_repository_resets_to_git_commit(self):
+ executed_commands = []
+
+ def fake_execute(args, print_output=True, suppress_stdout=False):
+ executed_commands.append(args)
+
+ with mock.patch.object(bazelci, "execute_command", side_effect=fake_execute), \
+ mock.patch("os.path.exists", return_value=True), \
+ mock.patch("os.chdir"):
+ bazelci.clone_git_repository(
+ "https://github.com/protocolbuffers/protobuf.git",
+ git_commit="origin/35.x",
+ suppress_stdout=True,
+ )
+
+ self.assertIn(["git", "reset", "origin/35.x", "--hard"], executed_commands)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/pipelines/protobuf.yml b/pipelines/protobuf.yml
deleted file mode 100644
index fa61e8d..0000000
--- a/pipelines/protobuf.yml
+++ /dev/null
@@ -1,64 +0,0 @@
-matrix:
- platform: ["debian10", "macos", "ubuntu2004"]
- bazel: [7.x, 8.x]
-
-tasks:
- verify_targets:
- name: "Verify build targets"
- platform: ${{ platform }}
- bazel: ${{ bazel }}
- build_flags:
- - '--host_cxxopt=-std=c++17'
- - '--cxxopt=-std=c++17'
- build_targets:
- - '//:protobuf'
- - '//:protobuf_lite'
- - '//:protobuf_python'
- - '//:protobuf_java'
- - '//:protoc'
- - '//:test_messages_proto2_cc_proto'
- - '//:test_messages_proto3_cc_proto'
-
- # Separate windows for different c++ build flags.
- verify_targets_windows:
- name: "Verify windows build targets"
- platform: windows
- bazel: ${{ bazel }}
- build_flags:
- - '--cxxopt=/std:c++17'
- - '--host_cxxopt=/std:c++17'
- # Run using clang-cl
- - '--extra_execution_platforms=//build_defs:x64_windows-clang-cl'
- - '--host_platform=//build_defs:x64_windows-clang-cl'
- - '--extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl'
- build_targets:
- - '//:protobuf'
- - '//:protobuf_lite'
- - '//:protobuf_python'
- - '//:protobuf_java'
- - '//:protoc'
- - '//:test_messages_proto2_cc_proto'
- - '//:test_messages_proto3_cc_proto'
-
- run_test_module:
- name: "Run test module"
- working_directory: "examples"
- platform: ${{ platform }}
- bazel: ${{ bazel }}
- build_flags:
- - '--host_cxxopt=-std=c++17'
- - '--cxxopt=-std=c++17'
- build_targets:
- - "//..."
-
- # Separate windows for different c++ build flags.
- run_test_module_windows:
- name: "Run windows test module"
- working_directory: "examples"
- platform: windows
- bazel: ${{ bazel }}
- build_flags:
- - '--cxxopt=/std:c++17'
- - '--host_cxxopt=/std:c++17'
- build_targets:
- - "//..."
