Embed @platforms into the Bazel binary
This PR implements the change discussed at https://docs.google.com/document/d/1EArrWYUDugqJzBcb0-OxY5BH1FFPYV3jxLIXbJpD9RY/edit?pli=1#heading=h.5mcn15i0e1ch.
Closes https://github.com/bazelbuild/bazel/pull/8600.
https://github.com/bazelbuild/bazel/issues/6516
This is encore of https://github.com/bazelbuild/bazel/commit/324dc44e6bafb487331724ae83d67bc18ed8a8aa with these changes:
* removing WORKSPACE file from the platforms_archive - repository rules always create new WORKSPACE file and that messes up the timestamps after re-archiving. This fixes the determinism test.
* after @aehlig kindly explained the semantics of external repositories, I discovered that overriding @platforms doesn't work when there is a load statement somewhere above the override, which is the common case. Therefore I had to move bundled platforms to the workspace suffix and had to use maybe pattern.
* because of maybe pattern I have to mock the bzl file in Bazel unit tests.
RELNOTES:
PiperOrigin-RevId: 253193463
diff --git a/src/BUILD b/src/BUILD
index c0e70b8..e7c6827 100644
--- a/src/BUILD
+++ b/src/BUILD
@@ -328,10 +328,11 @@
name = "package-zip" + suffix,
srcs = ([":embedded_tools" + suffix + ".zip"] if embed else []) + [
# The script assumes that the embedded tools zip (if exists) is the
- # first item here, the deploy jar the second and install base key is the
- # third
+ # first item here, the deploy jar the second, install base key is the
+ # third, and platforms archive is the fourth.
"//src/main/java/com/google/devtools/build/lib:bazel/BazelServer_deploy.jar",
"install_base_key" + suffix,
+ ":platforms_archive",
":libunix",
"//src/main/tools:build-runfiles",
"//src/main/tools:process-wrapper",
@@ -355,6 +356,15 @@
("_nojdk", True),
]]
+genrule(
+ name = "platforms_archive",
+ srcs = ["@platforms//:srcs"],
+ outs = ["platforms.zip"],
+ # Removing the WORKSPACE file since local_repository creates it no matter what and leaving
+ # it there would make the zip nondeterministic.
+ cmd = "zip -qX $@ $$(echo $(SRCS) | xargs -n 1 | grep -v '.*/WORKSPACE$$' | sort | xargs)",
+)
+
[genrule(
name = "bazel-bin" + suffix,
srcs = [
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/local_config_platform.WORKSPACE b/src/main/java/com/google/devtools/build/lib/bazel/repository/local_config_platform.WORKSPACE
index d0ce5b7..6793501 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/local_config_platform.WORKSPACE
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/local_config_platform.WORKSPACE
@@ -1 +1,12 @@
-local_config_platform(name = "local_config_platform")
+load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+
+maybe(
+ local_repository,
+ "platforms",
+ path = __embedded_dir__ + "/platforms",
+)
+
+maybe(
+ local_config_platform,
+ "local_config_platform",
+)
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
index d1de9db..1d5a322 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
@@ -406,7 +406,7 @@
builder.addRuleDefinition(new LocalConfigPlatformRule());
try {
- builder.addWorkspaceFilePrefix(
+ builder.addWorkspaceFileSuffix(
ResourceFileLoader.loadResource(
LocalConfigPlatformRule.class, "local_config_platform.WORKSPACE"));
} catch (IOException e) {
diff --git a/src/package-bazel.sh b/src/package-bazel.sh
index 632b133..0ebefed 100755
--- a/src/package-bazel.sh
+++ b/src/package-bazel.sh
@@ -16,16 +16,15 @@
set -euo pipefail
-# This script bootstraps building a Bazel binary without Bazel then
-# use this compiled Bazel to bootstrap Bazel itself. It can also
-# be provided with a previous version of Bazel to bootstrap Bazel
-# itself.
+# This script creates the Bazel archive that Bazel client unpacks and then
+# starts the server from.
WORKDIR=$(pwd)
OUT=$1
EMBEDDED_TOOLS=$2
DEPLOY_JAR=$3
INSTALL_BASE_KEY=$4
+PLATFORMS_ARCHIVE=$5
shift 4
TMP_DIR=${TMPDIR:-/tmp}
@@ -66,4 +65,16 @@
(cd ${PACKAGE_DIR}/embedded_tools && unzip -q "${WORKDIR}/${EMBEDDED_TOOLS}")
fi
+# Unzip platforms.zip into platforms/, move files up from external/platforms
+# subdirectory, and cleanup after itself.
+( \
+ cd ${PACKAGE_DIR} && \
+ unzip -q -d platforms platforms.zip && \
+ rm platforms.zip && \
+ cd platforms && \
+ mv external/platforms/* . && \
+ rmdir -p external/platforms \
+)
+touch -t 198001010000.00 ${PACKAGE_DIR}/platforms/WORKSPACE
+
(cd ${PACKAGE_DIR} && find . -type f | sort | zip -q9DX@ "${WORKDIR}/${OUT}")
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/mock/BazelAnalysisMock.java b/src/test/java/com/google/devtools/build/lib/analysis/mock/BazelAnalysisMock.java
index f6cae4e..edf34a1 100644
--- a/src/test/java/com/google/devtools/build/lib/analysis/mock/BazelAnalysisMock.java
+++ b/src/test/java/com/google/devtools/build/lib/analysis/mock/BazelAnalysisMock.java
@@ -383,6 +383,11 @@
config.create("/bazel_tools_workspace/WORKSPACE", "workspace(name = 'bazel_tools')");
config.create("/bazel_tools_workspace/tools/build_defs/repo/BUILD");
config.create(
+ "/bazel_tools_workspace/tools/build_defs/repo/utils.bzl",
+ "def maybe(repo_rule, name, **kwargs):",
+ " if name not in native.existing_rules():",
+ " repo_rule(name = name, **kwargs)");
+ config.create(
"/bazel_tools_workspace/tools/build_defs/repo/http.bzl",
"def http_archive(**kwargs):",
" pass",
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java
index 834d8ff..aa7d269 100644
--- a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java
+++ b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java
@@ -238,6 +238,11 @@
mockToolsConfig.create("/bazel_tools_workspace/WORKSPACE", "workspace(name = 'bazel_tools')");
mockToolsConfig.create("/bazel_tools_workspace/tools/build_defs/repo/BUILD");
mockToolsConfig.create(
+ "/bazel_tools_workspace/tools/build_defs/repo/utils.bzl",
+ "def maybe(repo_rule, name, **kwargs):",
+ " if name not in native.existing_rules():",
+ " repo_rule(name = name, **kwargs)");
+ mockToolsConfig.create(
"/bazel_tools_workspace/tools/build_defs/repo/http.bzl",
"def http_archive(**kwargs):",
" pass",
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/ConfigurationTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/ConfigurationTestCase.java
index 8122469..ea96da0 100644
--- a/src/test/java/com/google/devtools/build/lib/analysis/util/ConfigurationTestCase.java
+++ b/src/test/java/com/google/devtools/build/lib/analysis/util/ConfigurationTestCase.java
@@ -114,6 +114,11 @@
mockToolsConfig.create("/bazel_tools_workspace/WORKSPACE", "workspace(name = 'bazel_tools')");
mockToolsConfig.create("/bazel_tools_workspace/tools/build_defs/repo/BUILD");
mockToolsConfig.create(
+ "/bazel_tools_workspace/tools/build_defs/repo/utils.bzl",
+ "def maybe(repo_rule, name, **kwargs):",
+ " if name not in native.existing_rules():",
+ " repo_rule(name = name, **kwargs)");
+ mockToolsConfig.create(
"/bazel_tools_workspace/tools/build_defs/repo/http.bzl",
"def http_archive(**kwargs):",
" pass",
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoaderTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoaderTest.java
index a3370c1..49b4148 100644
--- a/src/test/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoaderTest.java
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoaderTest.java
@@ -86,6 +86,11 @@
"",
"def http_file(**kwargs):",
" pass");
+ FileSystemUtils.writeIsoLatin1(
+ tools.getRelative("tools/build_defs/repo/utils.bzl"),
+ "def maybe(repo_rule, name, **kwargs):",
+ " if name not in native.existing_rules():",
+ " repo_rule(name = name, **kwargs)");
}
private void fetchExternalRepo(RepositoryName externalRepo) {
diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD
index 1a773ce..e085eee 100644
--- a/src/test/shell/bazel/BUILD
+++ b/src/test/shell/bazel/BUILD
@@ -1104,6 +1104,13 @@
)
sh_test(
+ name = "platforms_test",
+ srcs = ["platforms_test.sh"],
+ data = [":test-deps"],
+ deps = ["@bazel_tools//tools/bash/runfiles"],
+)
+
+sh_test(
name = "platform_mapping_test",
srcs = ["platform_mapping_test.sh"],
data = [":test-deps"],
diff --git a/src/test/shell/bazel/platforms_test.sh b/src/test/shell/bazel/platforms_test.sh
new file mode 100755
index 0000000..6db625a
--- /dev/null
+++ b/src/test/shell/bazel/platforms_test.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+#
+# Copyright 2019 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Test related to @platforms embedded repository
+#
+
+# --- begin runfiles.bash initialization ---
+set -euo pipefail
+if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+ if [[ -f "$0.runfiles_manifest" ]]; then
+ export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest"
+ elif [[ -f "$0.runfiles/MANIFEST" ]]; then
+ export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST"
+ elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+ export RUNFILES_DIR="$0.runfiles"
+ fi
+fi
+if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+ source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash"
+elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+ source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \
+ "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)"
+else
+ echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash"
+ exit 1
+fi
+# --- end runfiles.bash initialization ---
+
+source "$(rlocation "io_bazel/src/test/shell/integration_test_setup.sh")" \
+ || { echo "integration_test_setup.sh not found!" >&2; exit 1; }
+
+function test_platforms_repository_builds_itself() {
+ # We test that a built-in @platforms repository is buildable.
+ bazel build @platforms//:all &> $TEST_log \
+ || fail "Build failed unexpectedly"
+}
+
+function test_platforms_can_be_overridden() {
+ # We test that a custom repository can override @platforms in their
+ # WORKSPACE file.
+ mkdir -p platforms_can_be_overridden || fail "couldn't create directory"
+ touch platforms_can_be_overridden/BUILD || \ fail "couldn't touch BUILD file"
+ cat > platforms_can_be_overridden/WORKSPACE <<EOF
+local_repository(
+ name = 'platforms',
+ path = '../override',
+)
+EOF
+
+ mkdir -p override || fail "couldn't create override directory"
+ touch override/WORKSPACE || fail "couldn't touch override/WORKSPACE"
+ cat > override/BUILD <<EOF
+filegroup(name = 'yolo')
+EOF
+
+ cd platforms_can_be_overridden || fail "couldn't cd into workspace"
+ bazel build @platforms//:yolo &> $TEST_log || \
+ fail "Bazel failed to build @platforms"
+}
+
+run_suite "platform mapping test"
+
diff --git a/src/test/shell/integration/discard_graph_edges_test.sh b/src/test/shell/integration/discard_graph_edges_test.sh
index aab82d9..926dc79 100755
--- a/src/test/shell/integration/discard_graph_edges_test.sh
+++ b/src/test/shell/integration/discard_graph_edges_test.sh
@@ -271,7 +271,7 @@
package_count="$(extract_histogram_count "$histo_file" \
'devtools\.build\.lib\..*\.Package$')"
# A few packages aren't cleared.
- [[ "$package_count" -le 16 ]] \
+ [[ "$package_count" -le 17 ]] \
|| fail "package count $package_count too high"
glob_count="$(extract_histogram_count "$histo_file" "GlobValue$")"
[[ "$glob_count" -le 1 ]] \
