Use a custom postsubmit.yml for Bazel in downstream (#2165) Addresses https://buildkite.com/bazel/bazel-at-head-plus-downstream/builds/4431#0194b040-d16f-44ea-abd5-f11ed5e82f95 We can revert to the official one after upgrading Bazel to 8.1.0 which would contain https://github.com/bazelbuild/bazel/commit/59cb6fe990933eb764fd22ef3115c0d7f51baded
diff --git a/buildkite/bazelci.py b/buildkite/bazelci.py index 000dff0..4606a64 100755 --- a/buildkite/bazelci.py +++ b/buildkite/bazelci.py
@@ -105,7 +105,8 @@ }, "Bazel": { "git_repository": "https://github.com/bazelbuild/bazel.git", - "file_config": ".bazelci/postsubmit.yml", + # "file_config": ".bazelci/postsubmit.yml", + "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/pipelines/bazel-postsubmit.yml", "pipeline_slug": "bazel-bazel", }, "Bazel Bench": {
diff --git a/pipelines/bazel-postsubmit.patch b/pipelines/bazel-postsubmit.patch index 8d66997..665095a 100644 --- a/pipelines/bazel-postsubmit.patch +++ b/pipelines/bazel-postsubmit.patch
@@ -1,38 +1,51 @@ -diff --git bazel-postsubmit.yml bazel-postsubmit.yml -index 1ee30a9a94..7995b62de5 100644 ---- bazel-postsubmit.yml -+++ bazel-postsubmit.yml -@@ -1,4 +1,5 @@ +--- bazel-postsubmit-orig.yml 2025-01-29 14:04:50 ++++ bazel-postsubmit.yml 2025-01-29 14:07:11 +@@ -1,10 +1,12 @@ --- +# Update this file by running ./update-bazel-postsubmit.sh under the same directory - + tasks: - centos7_java11_devtoolset10: -@@ -8,6 +9,7 @@ tasks: - - rm -f WORKSPACE.bzlmod.bak + centos7: + shell_commands: - rm -rf $HOME/bazeltest - mkdir $HOME/bazeltest + - bazel mod deps --lockfile_mode=update build_flags: - "--config=ci-linux" build_targets: -@@ -73,6 +75,7 @@ tasks: - - rm -f WORKSPACE.bzlmod.bak +@@ -47,6 +49,7 @@ + shell_commands: - rm -rf $HOME/bazeltest - mkdir $HOME/bazeltest + - bazel mod deps --lockfile_mode=update build_flags: - "--config=ci-linux" build_targets: -@@ -132,6 +135,7 @@ tasks: - - rm -f WORKSPACE.bzlmod.bak +@@ -60,6 +63,7 @@ + shell_commands: - rm -rf $HOME/bazeltest - mkdir $HOME/bazeltest + - bazel mod deps --lockfile_mode=update build_flags: - "--config=ci-linux" build_targets: -@@ -165,6 +169,7 @@ tasks: +@@ -94,6 +98,7 @@ + shell_commands: + - rm -rf $HOME/bazeltest + - mkdir $HOME/bazeltest ++ - bazel mod deps --lockfile_mode=update + build_flags: + - "--config=ci-linux" + build_targets: +@@ -112,6 +117,7 @@ + shell_commands: + - rm -rf $HOME/bazeltest + - mkdir $HOME/bazeltest ++ - bazel mod deps --lockfile_mode=update + build_flags: + - "--config=ci-linux" + build_targets: +@@ -143,6 +149,7 @@ - rm -rf $HOME/bazeltest - mkdir $HOME/bazeltest - ln -sf $OUTPUT_BASE/external $HOME/bazeltest/external @@ -40,7 +53,7 @@ build_flags: - "--config=ci-macos" build_targets: -@@ -228,6 +233,7 @@ tasks: +@@ -186,6 +193,7 @@ - rm -rf $HOME/bazeltest - mkdir $HOME/bazeltest - ln -sf $OUTPUT_BASE/external $HOME/bazeltest/external @@ -48,19 +61,28 @@ build_flags: - "--config=ci-macos" build_targets: -@@ -289,6 +295,7 @@ tasks: +@@ -219,6 +227,7 @@ + setup: + - mkdir C:\b - mklink /J C:\b\bazeltest_external %OUTPUT_BASE:/=\%\external - batch_commands: - - powershell -Command "(Get-Content WORKSPACE.bzlmod) -Replace '# android_', 'android_' | Set-Content WORKSPACE.bzlmod" + - bazel mod deps --lockfile_mode=update build_flags: - "--config=ci-windows" build_targets: -@@ -359,6 +366,7 @@ tasks: - -e 's/^# android_sdk_repository/android_sdk_repository/' - -e 's/^# android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod - - rm -f WORKSPACE.bzlmod.bak +@@ -268,6 +277,7 @@ + setup: + - mkdir C:\b + - mklink /J C:\b\bazeltest_external %OUTPUT_BASE:/=\%\external + - bazel mod deps --lockfile_mode=update build_flags: - - "--config=ubuntu2004_java11" + - "--config=ci-windows" + - "--config=windows_arm64" +@@ -279,6 +289,8 @@ + rbe_ubuntu2004: + platform: ubuntu2004 + name: "RBE" ++ shell_commands: ++ - bazel mod deps --lockfile_mode=update + build_flags: + - "--config=remote" - "--remote_executor=grpcs://remotebuildexecution.googleapis.com"
diff --git a/pipelines/bazel-postsubmit.yml b/pipelines/bazel-postsubmit.yml index 1be57a9..0803c3b 100644 --- a/pipelines/bazel-postsubmit.yml +++ b/pipelines/bazel-postsubmit.yml
@@ -4,9 +4,6 @@ tasks: centos7: shell_commands: - - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^# - android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod - - rm -f WORKSPACE.bzlmod.bak - rm -rf $HOME/bazeltest - mkdir $HOME/bazeltest - bazel mod deps --lockfile_mode=update @@ -20,6 +17,7 @@ - "//src:bazel_jdk_minimal" - "//src:test_repos" - "//src/main/java/..." + - "//src/tools/diskcache/..." - "//src/tools/execlog/..." test_flags: - "--config=ci-linux" @@ -31,10 +29,10 @@ - "//src/main/starlark/tests/builtins_bzl/..." - "//src/test/..." - "//src/tools/execlog/..." + - "//src/tools/one_version/..." - "//src/tools/singlejar/..." - "//src/tools/workspacelog/..." - "//third_party/ijar/..." - - "//tools/android/..." - "//tools/aquery_differ/..." - "//tools/compliance/..." - "//tools/python/..." @@ -44,21 +42,14 @@ - "-//src/test/shell/bazel:bazel_coverage_cc_released_test_gcc" - "-//src/test/shell/bazel:bazel_coverage_cc_head_test_gcc" - "-//src/test/shell/bazel:bazel_coverage_sh_test" - # Centos7 uses python 2 by default, so these fail: https://github.com/bazelbuild/bazel/issues/18776 - - "-//src/test/shell/bazel/android:android_instrumentation_test_integration_test" - - "-//src/test/shell/bazel/android:android_instrumentation_test_integration_test_with_head_android_tools" - - "-//src/test/shell/bazel/android:aapt_integration_test" - - "-//src/test/shell/bazel/android:aapt_integration_test_with_head_android_tools" include_json_profile: - build - test fedora39: shell_commands: - - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^# - android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod - - rm -f WORKSPACE.bzlmod.bak - rm -rf $HOME/bazeltest - mkdir $HOME/bazeltest + - bazel mod deps --lockfile_mode=update build_flags: - "--config=ci-linux" build_targets: @@ -70,9 +61,6 @@ - test ubuntu2204: shell_commands: - - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^# - android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod - - rm -f WORKSPACE.bzlmod.bak - rm -rf $HOME/bazeltest - mkdir $HOME/bazeltest - bazel mod deps --lockfile_mode=update @@ -91,10 +79,10 @@ - "//src/main/starlark/tests/builtins_bzl/..." - "//src/test/..." - "//src/tools/execlog/..." + - "//src/tools/one_version/..." - "//src/tools/singlejar/..." - "//src/tools/workspacelog/..." - "//third_party/ijar/..." - - "//tools/android/..." - "//tools/aquery_differ/..." - "//tools/python/..." - "//tools/bash/..." @@ -108,11 +96,9 @@ CC_CONFIGURE_DEBUG: 1 name: "Clang" shell_commands: - - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^# - android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod - - rm -f WORKSPACE.bzlmod.bak - rm -rf $HOME/bazeltest - mkdir $HOME/bazeltest + - bazel mod deps --lockfile_mode=update build_flags: - "--config=ci-linux" build_targets: @@ -129,9 +115,6 @@ - test ubuntu2004: shell_commands: - - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^# - android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod - - rm -f WORKSPACE.bzlmod.bak - rm -rf $HOME/bazeltest - mkdir $HOME/bazeltest - bazel mod deps --lockfile_mode=update @@ -150,10 +133,10 @@ - "//src/main/starlark/tests/builtins_bzl/..." - "//src/test/..." - "//src/tools/execlog/..." + - "//src/tools/one_version/..." - "//src/tools/singlejar/..." - "//src/tools/workspacelog/..." - "//third_party/ijar/..." - - "//tools/android/..." - "//tools/aquery_differ/..." - "//tools/python/..." - "//tools/bash/..." @@ -161,10 +144,8 @@ - build - test macos: + shards: 5 shell_commands: - - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^# - android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod - - rm -f WORKSPACE.bzlmod.bak - rm -rf $HOME/bazeltest - mkdir $HOME/bazeltest - ln -sf $OUTPUT_BASE/external $HOME/bazeltest/external @@ -175,50 +156,40 @@ - "//src:bazel" - "//src:bazel_jdk_minimal" - "//src:test_repos" - - "//src/main/java/..." test_flags: - "--config=ci-macos" # Fine tune the number of test jobs running in parallel to avoid timeout - - "--local_test_jobs=8" + - "--local_test_jobs=2" test_targets: - "//scripts/..." - "//src/main/starlark/tests/builtins_bzl/..." - "//src/test/..." - "//src/tools/execlog/..." + - "//src/tools/one_version/..." - "//src/tools/singlejar/..." - "//src/tools/workspacelog/..." - "//third_party/ijar/..." - - "//tools/android/..." - "//tools/aquery_differ/..." - "//tools/python/..." - "//tools/bash/..." - # https://github.com/bazelbuild/bazel/issues/16526 - - "-//src/test/shell/bazel:starlark_repository_test" - # https://github.com/bazelbuild/bazel/issues/17407 - - "-//src/test/shell/bazel/apple:bazel_apple_test" - # https://github.com/bazelbuild/bazel/issues/17408 - - "-//src/test/shell/bazel/apple:bazel_objc_test" + # Disable python and shell integration tests on Intel Macs + - "-//src/test/shell/..." + - "-//src/test/py/..." # https://github.com/bazelbuild/bazel/issues/17410 - "-//src/test/java/com/google/devtools/build/lib/platform:SystemMemoryPressureEventTest" - # https://github.com/bazelbuild/bazel/issues/17456 - - "-//src/test/shell/bazel:bazel_determinism_test" - # https://github.com/bazelbuild/bazel/issues/17457 - - "-//src/test/shell/bazel:jdeps_test" - # https://github.com/bazelbuild/bazel/issues/21495 - - "-//src/test/shell/bazel:srcs_test" - # Macs can't find python, so these fail: https://github.com/bazelbuild/bazel/issues/18776 - - "-//src/test/shell/bazel/android:android_instrumentation_test_integration_test" - - "-//src/test/shell/bazel/android:android_instrumentation_test_integration_test_with_head_android_tools" - - "-//src/test/shell/bazel/android:aapt_integration_test" - - "-//src/test/shell/bazel/android:aapt_integration_test_with_head_android_tools" + # Disable android tests since we are moving Android rules out of the Bazel repo. + # ServerTests frequently runs into deadlocks on Intel Macs + - "-//src/test/java/com/google/devtools/build/lib/server:ServerTests" + # Add back a few Apple specific tests + - "+//src/test/shell/bazel/apple/..." + # Add back one bootstrap test for Intel Mac + - "+//src/test/shell/bazel:bazel_bootstrap_distfile_test" include_json_profile: - build - test macos_arm64: + shards: 5 shell_commands: - - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^# - android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod - - rm -f WORKSPACE.bzlmod.bak - rm -rf $HOME/bazeltest - mkdir $HOME/bazeltest - ln -sf $OUTPUT_BASE/external $HOME/bazeltest/external @@ -229,27 +200,26 @@ - "//src:bazel" - "//src:bazel_jdk_minimal" - "//src:test_repos" - - "//src/main/java/..." test_flags: - "--config=ci-macos" + # Fine tune the number of test jobs running in parallel to avoid timeout + - "--local_test_jobs=2" + # Increase the test timeout by 20% to avoid flaky test failures due to timeout + - "--test_timeout=72,360,1080,4320" test_targets: - "//scripts/..." - "//src/main/starlark/tests/builtins_bzl/..." - "//src/test/..." - "//src/tools/execlog/..." + - "//src/tools/one_version/..." - "//src/tools/singlejar/..." - "//src/tools/workspacelog/..." - "//third_party/ijar/..." - - "//tools/android/..." - "//tools/aquery_differ/..." - "//tools/python/..." - "//tools/bash/..." # https://github.com/bazelbuild/bazel/issues/17410 - "-//src/test/java/com/google/devtools/build/lib/platform:SystemMemoryPressureEventTest" - # https://github.com/bazelbuild/bazel/issues/16521 & https://github.com/bazelbuild/bazel/issues/18776 - - "-//src/test/shell/bazel/android/..." - - "-//src/tools/android/java/com/google/devtools/build/android/..." - - "-//src/test/java/com/google/devtools/build/android/dexer:AllTests" include_json_profile: - build - test @@ -257,8 +227,6 @@ setup: - mkdir C:\b - mklink /J C:\b\bazeltest_external %OUTPUT_BASE:/=\%\external - batch_commands: - - powershell -Command "(Get-Content WORKSPACE.bzlmod) -Replace '# android_', 'android_' | Set-Content WORKSPACE.bzlmod" - bazel mod deps --lockfile_mode=update build_flags: - "--config=ci-windows" @@ -273,7 +241,6 @@ - "//src:embedded_tools_size_test" - "//src/main/starlark/tests/builtins_bzl/..." - "//src/test/cpp/..." - - "//src/test/java/com/google/devtools/build/android/..." - "//src/test/java/com/google/devtools/build/lib/..." - "//src/test/java/com/google/devtools/build/skyframe/..." - "//src/test/java/com/google/devtools/common/options/..." @@ -282,20 +249,18 @@ - "//src/test/res/..." - "//src/test/shell/..." - "//src/tools/launcher/..." + - "//src/tools/one_version/..." - "//src/tools/singlejar/..." - "//third_party/def_parser/..." - - "//tools/android/..." - "//tools/aquery_differ/..." - "//tools/bash/..." - "//tools/build_defs/..." - - "//tools/cpp/runfiles/..." - "//tools/java/..." - "//tools/jdk/..." - "//tools/python/..." - "//tools/test/..." # Re-enable the following tests on Windows: # https://github.com/bazelbuild/bazel/issues/4292 - - "-//src/test/java/com/google/devtools/build/android/r8/..." - "-//src/test/java/com/google/devtools/build/lib/query2/cquery/..." - "-//src/test/java/com/google/devtools/build/lib/query2/engine/..." - "-//src/test/java/com/google/devtools/build/lib/versioning/..." @@ -303,8 +268,6 @@ - "-//src/test/java/com/google/devtools/build/lib/remote:RemoteTests" - "-//src/test/shell/bazel/remote/..." - "-//tools/python:pywrapper_test" - # https://github.com/bazelbuild/bazel/issues/21593 - - "-//src/test/java/com/google/devtools/build/lib/bazel/repository/downloader:DownloaderTestSuite" include_json_profile: - build - test @@ -314,8 +277,7 @@ setup: - mkdir C:\b - mklink /J C:\b\bazeltest_external %OUTPUT_BASE:/=\%\external - batch_commands: - - powershell -Command "(Get-Content WORKSPACE.bzlmod) -Replace '# android_', 'android_' | Set-Content WORKSPACE.bzlmod" + - bazel mod deps --lockfile_mode=update build_flags: - "--config=ci-windows" - "--config=windows_arm64" @@ -327,15 +289,7 @@ rbe_ubuntu2004: platform: ubuntu2004 name: "RBE" - environment: - # Use last_green to pickup the fix for https://github.com/bazelbuild/bazel/issues/21604. - # TODO(chiwang): Remove this after Bazel 7.2.0 is released. - USE_BAZEL_VERSION: last_green shell_commands: - - sed -i.bak - -e 's/^# android_sdk_repository/android_sdk_repository/' - -e 's/^# android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod - - rm -f WORKSPACE.bzlmod.bak - bazel mod deps --lockfile_mode=update build_flags: - "--config=remote" @@ -344,6 +298,7 @@ - "--experimental_remote_cache_async" - "--experimental_remote_merkle_tree_cache" - "--remote_download_minimal" + - "--experimental_output_paths=strip" build_targets: - "//src:bazel" - "//src:bazel_jdk_minimal" @@ -355,19 +310,20 @@ - "--experimental_remote_cache_async" - "--experimental_remote_merkle_tree_cache" - "--remote_download_minimal" + - "--experimental_output_paths=strip" test_targets: - "//scripts/..." - "//src/java_tools/..." - "//src/main/starlark/tests/builtins_bzl/..." - "//src/test/..." - "//src/tools/execlog/..." + - "//src/tools/one_version/..." - "//src/tools/singlejar/..." - "//src/tools/workspacelog/..." - "//third_party/ijar/..." - "//tools/aquery_differ/..." - "//tools/python/..." - "//tools/bash/..." - - "//tools/android/..." # See https://github.com/bazelbuild/bazel/issues/8033 - "-//src/tools/singlejar:output_jar_simple_test" - "-//src/test/shell/bazel:external_integration_test" @@ -382,18 +338,12 @@ - "-//src/test/py/bazel:bazel_yanked_versions_test" - "-//src/test/py/bazel:bzlmod_query_test" - "-//src/test/py/bazel:mod_command_test" + - "-//src/test/shell/bazel:starlark_repository_test" - "-//src/test/shell/bazel:verify_workspace" include_json_profile: - build - test - kythe_ubuntu2204: - shell_commands: - - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' - -e 's/^# android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod - - rm -f WORKSPACE.bzlmod.bak - # Remove _nowkt suffix because it's only available in newer protobuf releases. See b/329055020. - # TODO(chiwang): Remove this once we have upgraded to protobuf 25+. - - sed -i -e 's/protobuf_nowkt/protobuf/' /usr/local/kythe/BUILD + kythe_ubuntu2404: index_flags: - "--define=kythe_corpus=github.com/bazelbuild/bazel" index_targets_query: "kind(\"cc_(binary|library|test|proto_library) rule\", ...) union kind(\"java_(binary|import|library|plugin|test|proto_library) rule\", ...) union kind(\"proto_library rule\", ...)"