Use RBE for remote caching on GCE (#659)
* Use RBE for remote caching on GCE.
This also changes the anti-cache-poisoning strategy back to using
--remote_default_platform_properties with a cache-silo-key instead of
appending the platform key to the URL of the cache, because the latter
is not supported by gRPC caching.
* Seems like grpcs:// is not yet supported by Bazel 0.25, use --tls_enabled instead
diff --git a/buildkite/bazelci.py b/buildkite/bazelci.py
index e23a42d..77f98a0 100755
--- a/buildkite/bazelci.py
+++ b/buildkite/bazelci.py
@@ -1083,30 +1083,33 @@
subprocess.check_output(["/usr/bin/xcodebuild", "-version"]),
]
# Use a local cache server for our macOS machines.
- cache_url = "http://100.107.73.186"
+ flags = ["--remote_cache=http://100.107.73.186"]
else:
platform_cache_key = [
# Platform name:
platform.encode("utf-8")
]
- # Use GCS for caching builds running on GCE.
- cache_url = "https://storage.googleapis.com/bazel-untrusted-buildkite-cache"
+ # Use RBE for caching builds running on GCE.
+ flags = [
+ "--google_default_credentials",
+ "--remote_cache=remotebuildexecution.googleapis.com",
+ "--remote_instance_name=projects/{}/instances/default_instance".format(CLOUD_PROJECT),
+ "--tls_enabled=true",
+ ]
platform_cache_digest = hashlib.sha256()
for key in platform_cache_key:
+ eprint("Adding to platform cache key: {}".format(key))
platform_cache_digest.update(key)
platform_cache_digest.update(b":")
- flags = [
+ flags += [
"--remote_timeout=60",
"--remote_max_connections=200",
- "--remote_http_cache={}/{}".format(cache_url, platform_cache_digest.hexdigest()),
+ '--remote_default_platform_properties=properties:{name:"cache-silo-key" value:"%s"}'
+ % platform_cache_digest.hexdigest(),
]
- # Need to use the correct credentials when running on GCE.
- if platform != "macos":
- flags += ["--google_default_credentials"]
-
return flags
@@ -1265,8 +1268,7 @@
def execute_bazel_build(
bazel_version, bazel_binary, platform, flags, targets, bep_file, incompatible_flags
):
- print_expanded_group(":bazel: Build ({})".format(bazel_version))
-
+ print_collapsed_group(":bazel: Computing flags for build step")
aggregated_flags = compute_flags(
platform,
flags,
@@ -1276,6 +1278,8 @@
bep_file,
enable_remote_cache=True,
)
+
+ print_expanded_group(":bazel: Build ({})".format(bazel_version))
try:
execute_command(
[bazel_binary]
@@ -1366,8 +1370,6 @@
monitor_flaky_tests,
incompatible_flags,
):
- print_expanded_group(":bazel: Test ({})".format(bazel_version))
-
aggregated_flags = [
"--flaky_test_attempts=3",
"--build_tests_only",
@@ -1376,6 +1378,7 @@
# Don't enable remote caching if the user enabled remote execution / caching themselves
# or flaky test monitoring is enabled, as remote caching makes tests look less flaky than
# they are.
+ print_collapsed_group(":bazel: Computing flags for test step")
aggregated_flags += compute_flags(
platform,
flags,
@@ -1386,6 +1389,7 @@
enable_remote_cache=not monitor_flaky_tests,
)
+ print_expanded_group(":bazel: Test ({})".format(bazel_version))
try:
execute_command(
[bazel_binary]
