Use a custom postsubmit.yml for Bazel in downstream (#2165)
Addresses
https://buildkite.com/bazel/bazel-at-head-plus-downstream/builds/4431#0194b040-d16f-44ea-abd5-f11ed5e82f95
We can revert to the official one after upgrading Bazel to 8.1.0 which
would contain
https://github.com/bazelbuild/bazel/commit/59cb6fe990933eb764fd22ef3115c0d7f51baded
diff --git a/buildkite/bazelci.py b/buildkite/bazelci.py
index 000dff0..4606a64 100755
--- a/buildkite/bazelci.py
+++ b/buildkite/bazelci.py
@@ -105,7 +105,8 @@
},
"Bazel": {
"git_repository": "https://github.com/bazelbuild/bazel.git",
- "file_config": ".bazelci/postsubmit.yml",
+ # "file_config": ".bazelci/postsubmit.yml",
+ "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/pipelines/bazel-postsubmit.yml",
"pipeline_slug": "bazel-bazel",
},
"Bazel Bench": {
diff --git a/pipelines/bazel-postsubmit.patch b/pipelines/bazel-postsubmit.patch
index 8d66997..665095a 100644
--- a/pipelines/bazel-postsubmit.patch
+++ b/pipelines/bazel-postsubmit.patch
@@ -1,38 +1,51 @@
-diff --git bazel-postsubmit.yml bazel-postsubmit.yml
-index 1ee30a9a94..7995b62de5 100644
---- bazel-postsubmit.yml
-+++ bazel-postsubmit.yml
-@@ -1,4 +1,5 @@
+--- bazel-postsubmit-orig.yml 2025-01-29 14:04:50
++++ bazel-postsubmit.yml 2025-01-29 14:07:11
+@@ -1,10 +1,12 @@
---
+# Update this file by running ./update-bazel-postsubmit.sh under the same directory
-
+
tasks:
- centos7_java11_devtoolset10:
-@@ -8,6 +9,7 @@ tasks:
- - rm -f WORKSPACE.bzlmod.bak
+ centos7:
+ shell_commands:
- rm -rf $HOME/bazeltest
- mkdir $HOME/bazeltest
+ - bazel mod deps --lockfile_mode=update
build_flags:
- "--config=ci-linux"
build_targets:
-@@ -73,6 +75,7 @@ tasks:
- - rm -f WORKSPACE.bzlmod.bak
+@@ -47,6 +49,7 @@
+ shell_commands:
- rm -rf $HOME/bazeltest
- mkdir $HOME/bazeltest
+ - bazel mod deps --lockfile_mode=update
build_flags:
- "--config=ci-linux"
build_targets:
-@@ -132,6 +135,7 @@ tasks:
- - rm -f WORKSPACE.bzlmod.bak
+@@ -60,6 +63,7 @@
+ shell_commands:
- rm -rf $HOME/bazeltest
- mkdir $HOME/bazeltest
+ - bazel mod deps --lockfile_mode=update
build_flags:
- "--config=ci-linux"
build_targets:
-@@ -165,6 +169,7 @@ tasks:
+@@ -94,6 +98,7 @@
+ shell_commands:
+ - rm -rf $HOME/bazeltest
+ - mkdir $HOME/bazeltest
++ - bazel mod deps --lockfile_mode=update
+ build_flags:
+ - "--config=ci-linux"
+ build_targets:
+@@ -112,6 +117,7 @@
+ shell_commands:
+ - rm -rf $HOME/bazeltest
+ - mkdir $HOME/bazeltest
++ - bazel mod deps --lockfile_mode=update
+ build_flags:
+ - "--config=ci-linux"
+ build_targets:
+@@ -143,6 +149,7 @@
- rm -rf $HOME/bazeltest
- mkdir $HOME/bazeltest
- ln -sf $OUTPUT_BASE/external $HOME/bazeltest/external
@@ -40,7 +53,7 @@
build_flags:
- "--config=ci-macos"
build_targets:
-@@ -228,6 +233,7 @@ tasks:
+@@ -186,6 +193,7 @@
- rm -rf $HOME/bazeltest
- mkdir $HOME/bazeltest
- ln -sf $OUTPUT_BASE/external $HOME/bazeltest/external
@@ -48,19 +61,28 @@
build_flags:
- "--config=ci-macos"
build_targets:
-@@ -289,6 +295,7 @@ tasks:
+@@ -219,6 +227,7 @@
+ setup:
+ - mkdir C:\b
- mklink /J C:\b\bazeltest_external %OUTPUT_BASE:/=\%\external
- batch_commands:
- - powershell -Command "(Get-Content WORKSPACE.bzlmod) -Replace '# android_', 'android_' | Set-Content WORKSPACE.bzlmod"
+ - bazel mod deps --lockfile_mode=update
build_flags:
- "--config=ci-windows"
build_targets:
-@@ -359,6 +366,7 @@ tasks:
- -e 's/^# android_sdk_repository/android_sdk_repository/'
- -e 's/^# android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod
- - rm -f WORKSPACE.bzlmod.bak
+@@ -268,6 +277,7 @@
+ setup:
+ - mkdir C:\b
+ - mklink /J C:\b\bazeltest_external %OUTPUT_BASE:/=\%\external
+ - bazel mod deps --lockfile_mode=update
build_flags:
- - "--config=ubuntu2004_java11"
+ - "--config=ci-windows"
+ - "--config=windows_arm64"
+@@ -279,6 +289,8 @@
+ rbe_ubuntu2004:
+ platform: ubuntu2004
+ name: "RBE"
++ shell_commands:
++ - bazel mod deps --lockfile_mode=update
+ build_flags:
+ - "--config=remote"
- "--remote_executor=grpcs://remotebuildexecution.googleapis.com"
diff --git a/pipelines/bazel-postsubmit.yml b/pipelines/bazel-postsubmit.yml
index 1be57a9..0803c3b 100644
--- a/pipelines/bazel-postsubmit.yml
+++ b/pipelines/bazel-postsubmit.yml
@@ -4,9 +4,6 @@
tasks:
centos7:
shell_commands:
- - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^#
- android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod
- - rm -f WORKSPACE.bzlmod.bak
- rm -rf $HOME/bazeltest
- mkdir $HOME/bazeltest
- bazel mod deps --lockfile_mode=update
@@ -20,6 +17,7 @@
- "//src:bazel_jdk_minimal"
- "//src:test_repos"
- "//src/main/java/..."
+ - "//src/tools/diskcache/..."
- "//src/tools/execlog/..."
test_flags:
- "--config=ci-linux"
@@ -31,10 +29,10 @@
- "//src/main/starlark/tests/builtins_bzl/..."
- "//src/test/..."
- "//src/tools/execlog/..."
+ - "//src/tools/one_version/..."
- "//src/tools/singlejar/..."
- "//src/tools/workspacelog/..."
- "//third_party/ijar/..."
- - "//tools/android/..."
- "//tools/aquery_differ/..."
- "//tools/compliance/..."
- "//tools/python/..."
@@ -44,21 +42,14 @@
- "-//src/test/shell/bazel:bazel_coverage_cc_released_test_gcc"
- "-//src/test/shell/bazel:bazel_coverage_cc_head_test_gcc"
- "-//src/test/shell/bazel:bazel_coverage_sh_test"
- # Centos7 uses python 2 by default, so these fail: https://github.com/bazelbuild/bazel/issues/18776
- - "-//src/test/shell/bazel/android:android_instrumentation_test_integration_test"
- - "-//src/test/shell/bazel/android:android_instrumentation_test_integration_test_with_head_android_tools"
- - "-//src/test/shell/bazel/android:aapt_integration_test"
- - "-//src/test/shell/bazel/android:aapt_integration_test_with_head_android_tools"
include_json_profile:
- build
- test
fedora39:
shell_commands:
- - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^#
- android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod
- - rm -f WORKSPACE.bzlmod.bak
- rm -rf $HOME/bazeltest
- mkdir $HOME/bazeltest
+ - bazel mod deps --lockfile_mode=update
build_flags:
- "--config=ci-linux"
build_targets:
@@ -70,9 +61,6 @@
- test
ubuntu2204:
shell_commands:
- - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^#
- android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod
- - rm -f WORKSPACE.bzlmod.bak
- rm -rf $HOME/bazeltest
- mkdir $HOME/bazeltest
- bazel mod deps --lockfile_mode=update
@@ -91,10 +79,10 @@
- "//src/main/starlark/tests/builtins_bzl/..."
- "//src/test/..."
- "//src/tools/execlog/..."
+ - "//src/tools/one_version/..."
- "//src/tools/singlejar/..."
- "//src/tools/workspacelog/..."
- "//third_party/ijar/..."
- - "//tools/android/..."
- "//tools/aquery_differ/..."
- "//tools/python/..."
- "//tools/bash/..."
@@ -108,11 +96,9 @@
CC_CONFIGURE_DEBUG: 1
name: "Clang"
shell_commands:
- - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^#
- android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod
- - rm -f WORKSPACE.bzlmod.bak
- rm -rf $HOME/bazeltest
- mkdir $HOME/bazeltest
+ - bazel mod deps --lockfile_mode=update
build_flags:
- "--config=ci-linux"
build_targets:
@@ -129,9 +115,6 @@
- test
ubuntu2004:
shell_commands:
- - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^#
- android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod
- - rm -f WORKSPACE.bzlmod.bak
- rm -rf $HOME/bazeltest
- mkdir $HOME/bazeltest
- bazel mod deps --lockfile_mode=update
@@ -150,10 +133,10 @@
- "//src/main/starlark/tests/builtins_bzl/..."
- "//src/test/..."
- "//src/tools/execlog/..."
+ - "//src/tools/one_version/..."
- "//src/tools/singlejar/..."
- "//src/tools/workspacelog/..."
- "//third_party/ijar/..."
- - "//tools/android/..."
- "//tools/aquery_differ/..."
- "//tools/python/..."
- "//tools/bash/..."
@@ -161,10 +144,8 @@
- build
- test
macos:
+ shards: 5
shell_commands:
- - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^#
- android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod
- - rm -f WORKSPACE.bzlmod.bak
- rm -rf $HOME/bazeltest
- mkdir $HOME/bazeltest
- ln -sf $OUTPUT_BASE/external $HOME/bazeltest/external
@@ -175,50 +156,40 @@
- "//src:bazel"
- "//src:bazel_jdk_minimal"
- "//src:test_repos"
- - "//src/main/java/..."
test_flags:
- "--config=ci-macos"
# Fine tune the number of test jobs running in parallel to avoid timeout
- - "--local_test_jobs=8"
+ - "--local_test_jobs=2"
test_targets:
- "//scripts/..."
- "//src/main/starlark/tests/builtins_bzl/..."
- "//src/test/..."
- "//src/tools/execlog/..."
+ - "//src/tools/one_version/..."
- "//src/tools/singlejar/..."
- "//src/tools/workspacelog/..."
- "//third_party/ijar/..."
- - "//tools/android/..."
- "//tools/aquery_differ/..."
- "//tools/python/..."
- "//tools/bash/..."
- # https://github.com/bazelbuild/bazel/issues/16526
- - "-//src/test/shell/bazel:starlark_repository_test"
- # https://github.com/bazelbuild/bazel/issues/17407
- - "-//src/test/shell/bazel/apple:bazel_apple_test"
- # https://github.com/bazelbuild/bazel/issues/17408
- - "-//src/test/shell/bazel/apple:bazel_objc_test"
+ # Disable python and shell integration tests on Intel Macs
+ - "-//src/test/shell/..."
+ - "-//src/test/py/..."
# https://github.com/bazelbuild/bazel/issues/17410
- "-//src/test/java/com/google/devtools/build/lib/platform:SystemMemoryPressureEventTest"
- # https://github.com/bazelbuild/bazel/issues/17456
- - "-//src/test/shell/bazel:bazel_determinism_test"
- # https://github.com/bazelbuild/bazel/issues/17457
- - "-//src/test/shell/bazel:jdeps_test"
- # https://github.com/bazelbuild/bazel/issues/21495
- - "-//src/test/shell/bazel:srcs_test"
- # Macs can't find python, so these fail: https://github.com/bazelbuild/bazel/issues/18776
- - "-//src/test/shell/bazel/android:android_instrumentation_test_integration_test"
- - "-//src/test/shell/bazel/android:android_instrumentation_test_integration_test_with_head_android_tools"
- - "-//src/test/shell/bazel/android:aapt_integration_test"
- - "-//src/test/shell/bazel/android:aapt_integration_test_with_head_android_tools"
+ # Disable android tests since we are moving Android rules out of the Bazel repo.
+ # ServerTests frequently runs into deadlocks on Intel Macs
+ - "-//src/test/java/com/google/devtools/build/lib/server:ServerTests"
+ # Add back a few Apple specific tests
+ - "+//src/test/shell/bazel/apple/..."
+ # Add back one bootstrap test for Intel Mac
+ - "+//src/test/shell/bazel:bazel_bootstrap_distfile_test"
include_json_profile:
- build
- test
macos_arm64:
+ shards: 5
shell_commands:
- - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^#
- android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod
- - rm -f WORKSPACE.bzlmod.bak
- rm -rf $HOME/bazeltest
- mkdir $HOME/bazeltest
- ln -sf $OUTPUT_BASE/external $HOME/bazeltest/external
@@ -229,27 +200,26 @@
- "//src:bazel"
- "//src:bazel_jdk_minimal"
- "//src:test_repos"
- - "//src/main/java/..."
test_flags:
- "--config=ci-macos"
+ # Fine tune the number of test jobs running in parallel to avoid timeout
+ - "--local_test_jobs=2"
+ # Increase the test timeout by 20% to avoid flaky test failures due to timeout
+ - "--test_timeout=72,360,1080,4320"
test_targets:
- "//scripts/..."
- "//src/main/starlark/tests/builtins_bzl/..."
- "//src/test/..."
- "//src/tools/execlog/..."
+ - "//src/tools/one_version/..."
- "//src/tools/singlejar/..."
- "//src/tools/workspacelog/..."
- "//third_party/ijar/..."
- - "//tools/android/..."
- "//tools/aquery_differ/..."
- "//tools/python/..."
- "//tools/bash/..."
# https://github.com/bazelbuild/bazel/issues/17410
- "-//src/test/java/com/google/devtools/build/lib/platform:SystemMemoryPressureEventTest"
- # https://github.com/bazelbuild/bazel/issues/16521 & https://github.com/bazelbuild/bazel/issues/18776
- - "-//src/test/shell/bazel/android/..."
- - "-//src/tools/android/java/com/google/devtools/build/android/..."
- - "-//src/test/java/com/google/devtools/build/android/dexer:AllTests"
include_json_profile:
- build
- test
@@ -257,8 +227,6 @@
setup:
- mkdir C:\b
- mklink /J C:\b\bazeltest_external %OUTPUT_BASE:/=\%\external
- batch_commands:
- - powershell -Command "(Get-Content WORKSPACE.bzlmod) -Replace '# android_', 'android_' | Set-Content WORKSPACE.bzlmod"
- bazel mod deps --lockfile_mode=update
build_flags:
- "--config=ci-windows"
@@ -273,7 +241,6 @@
- "//src:embedded_tools_size_test"
- "//src/main/starlark/tests/builtins_bzl/..."
- "//src/test/cpp/..."
- - "//src/test/java/com/google/devtools/build/android/..."
- "//src/test/java/com/google/devtools/build/lib/..."
- "//src/test/java/com/google/devtools/build/skyframe/..."
- "//src/test/java/com/google/devtools/common/options/..."
@@ -282,20 +249,18 @@
- "//src/test/res/..."
- "//src/test/shell/..."
- "//src/tools/launcher/..."
+ - "//src/tools/one_version/..."
- "//src/tools/singlejar/..."
- "//third_party/def_parser/..."
- - "//tools/android/..."
- "//tools/aquery_differ/..."
- "//tools/bash/..."
- "//tools/build_defs/..."
- - "//tools/cpp/runfiles/..."
- "//tools/java/..."
- "//tools/jdk/..."
- "//tools/python/..."
- "//tools/test/..."
# Re-enable the following tests on Windows:
# https://github.com/bazelbuild/bazel/issues/4292
- - "-//src/test/java/com/google/devtools/build/android/r8/..."
- "-//src/test/java/com/google/devtools/build/lib/query2/cquery/..."
- "-//src/test/java/com/google/devtools/build/lib/query2/engine/..."
- "-//src/test/java/com/google/devtools/build/lib/versioning/..."
@@ -303,8 +268,6 @@
- "-//src/test/java/com/google/devtools/build/lib/remote:RemoteTests"
- "-//src/test/shell/bazel/remote/..."
- "-//tools/python:pywrapper_test"
- # https://github.com/bazelbuild/bazel/issues/21593
- - "-//src/test/java/com/google/devtools/build/lib/bazel/repository/downloader:DownloaderTestSuite"
include_json_profile:
- build
- test
@@ -314,8 +277,7 @@
setup:
- mkdir C:\b
- mklink /J C:\b\bazeltest_external %OUTPUT_BASE:/=\%\external
- batch_commands:
- - powershell -Command "(Get-Content WORKSPACE.bzlmod) -Replace '# android_', 'android_' | Set-Content WORKSPACE.bzlmod"
+ - bazel mod deps --lockfile_mode=update
build_flags:
- "--config=ci-windows"
- "--config=windows_arm64"
@@ -327,15 +289,7 @@
rbe_ubuntu2004:
platform: ubuntu2004
name: "RBE"
- environment:
- # Use last_green to pickup the fix for https://github.com/bazelbuild/bazel/issues/21604.
- # TODO(chiwang): Remove this after Bazel 7.2.0 is released.
- USE_BAZEL_VERSION: last_green
shell_commands:
- - sed -i.bak
- -e 's/^# android_sdk_repository/android_sdk_repository/'
- -e 's/^# android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod
- - rm -f WORKSPACE.bzlmod.bak
- bazel mod deps --lockfile_mode=update
build_flags:
- "--config=remote"
@@ -344,6 +298,7 @@
- "--experimental_remote_cache_async"
- "--experimental_remote_merkle_tree_cache"
- "--remote_download_minimal"
+ - "--experimental_output_paths=strip"
build_targets:
- "//src:bazel"
- "//src:bazel_jdk_minimal"
@@ -355,19 +310,20 @@
- "--experimental_remote_cache_async"
- "--experimental_remote_merkle_tree_cache"
- "--remote_download_minimal"
+ - "--experimental_output_paths=strip"
test_targets:
- "//scripts/..."
- "//src/java_tools/..."
- "//src/main/starlark/tests/builtins_bzl/..."
- "//src/test/..."
- "//src/tools/execlog/..."
+ - "//src/tools/one_version/..."
- "//src/tools/singlejar/..."
- "//src/tools/workspacelog/..."
- "//third_party/ijar/..."
- "//tools/aquery_differ/..."
- "//tools/python/..."
- "//tools/bash/..."
- - "//tools/android/..."
# See https://github.com/bazelbuild/bazel/issues/8033
- "-//src/tools/singlejar:output_jar_simple_test"
- "-//src/test/shell/bazel:external_integration_test"
@@ -382,18 +338,12 @@
- "-//src/test/py/bazel:bazel_yanked_versions_test"
- "-//src/test/py/bazel:bzlmod_query_test"
- "-//src/test/py/bazel:mod_command_test"
+ - "-//src/test/shell/bazel:starlark_repository_test"
- "-//src/test/shell/bazel:verify_workspace"
include_json_profile:
- build
- test
- kythe_ubuntu2204:
- shell_commands:
- - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/'
- -e 's/^# android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod
- - rm -f WORKSPACE.bzlmod.bak
- # Remove _nowkt suffix because it's only available in newer protobuf releases. See b/329055020.
- # TODO(chiwang): Remove this once we have upgraded to protobuf 25+.
- - sed -i -e 's/protobuf_nowkt/protobuf/' /usr/local/kythe/BUILD
+ kythe_ubuntu2404:
index_flags:
- "--define=kythe_corpus=github.com/bazelbuild/bazel"
index_targets_query: "kind(\"cc_(binary|library|test|proto_library) rule\", ...) union kind(\"java_(binary|import|library|plugin|test|proto_library) rule\", ...) union kind(\"proto_library rule\", ...)"
