Embed @platforms into the Bazel binary
This PR implements the change discussed at https://docs.google.com/document/d/1EArrWYUDugqJzBcb0-OxY5BH1FFPYV3jxLIXbJpD9RY/edit?pli=1#heading=h.5mcn15i0e1ch.
Closes https://github.com/bazelbuild/bazel/pull/8600.
https://github.com/bazelbuild/bazel/issues/6516
This is encore of https://github.com/bazelbuild/bazel/commit/324dc44e6bafb487331724ae83d67bc18ed8a8aa with these changes:
* removing WORKSPACE file from the platforms_archive - repository rules always create new WORKSPACE file and that messes up the timestamps after re-archiving. This fixes the determinism test.
* after @aehlig kindly explained the semantics of external repositories, I discovered that overriding @platforms doesn't work when there is a load statement somewhere above the override, which is the common case. Therefore I had to move bundled platforms to the workspace suffix and had to use maybe pattern.
* because of maybe pattern I have to mock the bzl file in Bazel unit tests.
RELNOTES:
PiperOrigin-RevId: 253193463
diff --git a/BUILD b/BUILD
index 4aef01e..d68f3ca 100644
--- a/BUILD
+++ b/BUILD
@@ -78,6 +78,14 @@
visibility = ["//:__subpackages__"],
)
+pkg_tar(
+ name = "platforms-srcs",
+ srcs = ["@platforms//:srcs"],
+ package_dir = "platforms",
+ strip_prefix = ".",
+ visibility = ["//:__subpackages__"],
+)
+
py_binary(
name = "combine_distfiles",
srcs = ["combine_distfiles.py"],
@@ -89,6 +97,7 @@
name = "bazel-distfile",
srcs = [
":bazel-srcs",
+ ":platforms-srcs",
"//src:derived_java_srcs",
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec:bootstrap_autocodec.tar",
"@additional_distfiles//:archives.tar",
@@ -104,6 +113,7 @@
name = "bazel-distfile-tar",
srcs = [
":bazel-srcs",
+ ":platforms-srcs",
"//src:derived_java_srcs",
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec:bootstrap_autocodec.tar",
"@additional_distfiles//:archives.tar",
diff --git a/WORKSPACE b/WORKSPACE
index 549e95d..d4766bc 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -117,6 +117,8 @@
"8ccf4f1c351928b55d5dddf3672e3667f6978d60.tar.gz",
"0.16.2.zip",
"android_tools_pkg-0.6.tar.gz",
+ # bazelbuild/platforms
+ "441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip",
],
dirname = "derived/distdir",
sha256 = {
@@ -130,6 +132,8 @@
"8ccf4f1c351928b55d5dddf3672e3667f6978d60.tar.gz": "d868ce50d592ef4aad7dec4dd32ae68d2151261913450fac8390b3fd474bb898",
"0.16.2.zip": "9b72bb0aea72d7cbcfc82a01b1e25bf3d85f791e790ddec16c65e2d906382ee0",
"android_tools_pkg-0.6.tar.gz": "8231a3305ba5f75daed851930af3d7c2fb0f00cd5dfd397e26e44fdf295b237e", # built at ae148c75dd0e122b01ef5ce9dd90259e616d3b76
+ # bazelbuild/platforms
+ "441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip": "a07fe5e75964361885db725039c2ba673f0ee0313d971ae4f50c9b18cd28b0b5",
},
urls = {
"e0b0291b2c51fbe5a7cfa14473a1ae850f94f021.zip": [
@@ -167,6 +171,11 @@
"android_tools_pkg-0.6.tar.gz": [
"https://mirror.bazel.build/bazel_android_tools/android_tools_pkg-0.6.tar.gz",
],
+ # bazelbuild/platforms
+ "441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip": [
+ "https://mirror.bazel.build/github.com/bazelbuild/platforms/archive/441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip",
+ "https://github.com/bazelbuild/platforms/archive/441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip",
+ ],
},
)
@@ -391,6 +400,8 @@
"zulu11.29.3-ca-jdk11.0.2-macosx_x64.zip",
"zulu11.29.3-ca-jdk11.0.2-win_x64.zip",
"android_tools_pkg-0.6.tar.gz",
+ # bazelbuild/platforms
+ "441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip",
],
dirname = "test_WORKSPACE/distdir",
sha256 = {
@@ -413,6 +424,8 @@
"zulu11.29.3-ca-jdk11.0.2-macosx_x64.zip": "059f8e3484bf07b63a8f2820d5f528f473eff1befdb1896ee4f8ff06be3b8d8f",
"zulu11.29.3-ca-jdk11.0.2-win_x64.zip": "e1f5b4ce1b9148140fae2fcfb8a96d1c9b7eac5b8df0e13fbcad9b8561284880",
"android_tools_pkg-0.6.tar.gz": "8231a3305ba5f75daed851930af3d7c2fb0f00cd5dfd397e26e44fdf295b237e", # built at ae148c75dd0e122b01ef5ce9dd90259e616d3b76
+ # bazelbuild/platforms
+ "441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip": "a07fe5e75964361885db725039c2ba673f0ee0313d971ae4f50c9b18cd28b0b5",
},
urls = {
"zulu9.0.7.1-jdk9.0.7-linux_x64-allmodules.tar.gz": ["https://mirror.bazel.build/openjdk/azul-zulu-9.0.7.1-jdk9.0.7/zulu9.0.7.1-jdk9.0.7-linux_x64-allmodules.tar.gz"],
@@ -436,6 +449,11 @@
"android_tools_pkg-0.6.tar.gz": [
"https://mirror.bazel.build/bazel_android_tools/android_tools_pkg-0.6.tar.gz",
],
+ # bazelbuild/platforms
+ "441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip": [
+ "https://mirror.bazel.build/github.com/bazelbuild/platforms/archive/441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip",
+ "https://github.com/bazelbuild/platforms/archive/441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip",
+ ],
},
)
@@ -497,6 +515,16 @@
)
http_archive(
+ name = "platforms",
+ sha256 = "a07fe5e75964361885db725039c2ba673f0ee0313d971ae4f50c9b18cd28b0b5",
+ urls = [
+ "https://mirror.bazel.build/github.com/bazelbuild/platforms/archive/441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip",
+ "https://github.com/bazelbuild/platforms/archive/441afe1bfdadd6236988e9cac159df6b5a9f5a98.zip",
+ ],
+ strip_prefix = "platforms-441afe1bfdadd6236988e9cac159df6b5a9f5a98"
+)
+
+http_archive(
name = "java_tools_langtools_javac12",
sha256 = "99b107105165a91df82cd7cf82a8efb930d803fb7de1663cf7f780142104cd14",
urls = [
diff --git a/scripts/bootstrap/compile.sh b/scripts/bootstrap/compile.sh
index 069ee91..90782f9 100755
--- a/scripts/bootstrap/compile.sh
+++ b/scripts/bootstrap/compile.sh
@@ -289,6 +289,9 @@
ARCHIVE_DIR=${OUTPUT_DIR}/archive
mkdir -p ${ARCHIVE_DIR}/_embedded_binaries
+# Prepare @platforms local repository
+link_dir ${PWD}/platforms ${ARCHIVE_DIR}/_embedded_binaries/platforms
+
# Dummy build-runfiles (we can't compile C++ yet, so we can't have the real one)
if [ "${PLATFORM}" = "windows" ]; then
# We don't rely on runfiles trees on Windows
diff --git a/src/BUILD b/src/BUILD
index c0e70b8..e7c6827 100644
--- a/src/BUILD
+++ b/src/BUILD
@@ -328,10 +328,11 @@
name = "package-zip" + suffix,
srcs = ([":embedded_tools" + suffix + ".zip"] if embed else []) + [
# The script assumes that the embedded tools zip (if exists) is the
- # first item here, the deploy jar the second and install base key is the
- # third
+ # first item here, the deploy jar the second, install base key is the
+ # third, and platforms archive is the fourth.
"//src/main/java/com/google/devtools/build/lib:bazel/BazelServer_deploy.jar",
"install_base_key" + suffix,
+ ":platforms_archive",
":libunix",
"//src/main/tools:build-runfiles",
"//src/main/tools:process-wrapper",
@@ -355,6 +356,15 @@
("_nojdk", True),
]]
+genrule(
+ name = "platforms_archive",
+ srcs = ["@platforms//:srcs"],
+ outs = ["platforms.zip"],
+ # Removing the WORKSPACE file since local_repository creates it no matter what and leaving
+ # it there would make the zip nondeterministic.
+ cmd = "zip -qX $@ $$(echo $(SRCS) | xargs -n 1 | grep -v '.*/WORKSPACE$$' | sort | xargs)",
+)
+
[genrule(
name = "bazel-bin" + suffix,
srcs = [
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/local_config_platform.WORKSPACE b/src/main/java/com/google/devtools/build/lib/bazel/repository/local_config_platform.WORKSPACE
index d0ce5b7..6793501 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/local_config_platform.WORKSPACE
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/local_config_platform.WORKSPACE
@@ -1 +1,12 @@
-local_config_platform(name = "local_config_platform")
+load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+
+maybe(
+ local_repository,
+ "platforms",
+ path = __embedded_dir__ + "/platforms",
+)
+
+maybe(
+ local_config_platform,
+ "local_config_platform",
+)
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
index d1de9db..1d5a322 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
@@ -406,7 +406,7 @@
builder.addRuleDefinition(new LocalConfigPlatformRule());
try {
- builder.addWorkspaceFilePrefix(
+ builder.addWorkspaceFileSuffix(
ResourceFileLoader.loadResource(
LocalConfigPlatformRule.class, "local_config_platform.WORKSPACE"));
} catch (IOException e) {
diff --git a/src/package-bazel.sh b/src/package-bazel.sh
index 632b133..0ebefed 100755
--- a/src/package-bazel.sh
+++ b/src/package-bazel.sh
@@ -16,16 +16,15 @@
set -euo pipefail
-# This script bootstraps building a Bazel binary without Bazel then
-# use this compiled Bazel to bootstrap Bazel itself. It can also
-# be provided with a previous version of Bazel to bootstrap Bazel
-# itself.
+# This script creates the Bazel archive that Bazel client unpacks and then
+# starts the server from.
WORKDIR=$(pwd)
OUT=$1
EMBEDDED_TOOLS=$2
DEPLOY_JAR=$3
INSTALL_BASE_KEY=$4
+PLATFORMS_ARCHIVE=$5
shift 4
TMP_DIR=${TMPDIR:-/tmp}
@@ -66,4 +65,16 @@
(cd ${PACKAGE_DIR}/embedded_tools && unzip -q "${WORKDIR}/${EMBEDDED_TOOLS}")
fi
+# Unzip platforms.zip into platforms/, move files up from external/platforms
+# subdirectory, and cleanup after itself.
+( \
+ cd ${PACKAGE_DIR} && \
+ unzip -q -d platforms platforms.zip && \
+ rm platforms.zip && \
+ cd platforms && \
+ mv external/platforms/* . && \
+ rmdir -p external/platforms \
+)
+touch -t 198001010000.00 ${PACKAGE_DIR}/platforms/WORKSPACE
+
(cd ${PACKAGE_DIR} && find . -type f | sort | zip -q9DX@ "${WORKDIR}/${OUT}")
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/mock/BazelAnalysisMock.java b/src/test/java/com/google/devtools/build/lib/analysis/mock/BazelAnalysisMock.java
index f6cae4e..edf34a1 100644
--- a/src/test/java/com/google/devtools/build/lib/analysis/mock/BazelAnalysisMock.java
+++ b/src/test/java/com/google/devtools/build/lib/analysis/mock/BazelAnalysisMock.java
@@ -383,6 +383,11 @@
config.create("/bazel_tools_workspace/WORKSPACE", "workspace(name = 'bazel_tools')");
config.create("/bazel_tools_workspace/tools/build_defs/repo/BUILD");
config.create(
+ "/bazel_tools_workspace/tools/build_defs/repo/utils.bzl",
+ "def maybe(repo_rule, name, **kwargs):",
+ " if name not in native.existing_rules():",
+ " repo_rule(name = name, **kwargs)");
+ config.create(
"/bazel_tools_workspace/tools/build_defs/repo/http.bzl",
"def http_archive(**kwargs):",
" pass",
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java
index 834d8ff..aa7d269 100644
--- a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java
+++ b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java
@@ -238,6 +238,11 @@
mockToolsConfig.create("/bazel_tools_workspace/WORKSPACE", "workspace(name = 'bazel_tools')");
mockToolsConfig.create("/bazel_tools_workspace/tools/build_defs/repo/BUILD");
mockToolsConfig.create(
+ "/bazel_tools_workspace/tools/build_defs/repo/utils.bzl",
+ "def maybe(repo_rule, name, **kwargs):",
+ " if name not in native.existing_rules():",
+ " repo_rule(name = name, **kwargs)");
+ mockToolsConfig.create(
"/bazel_tools_workspace/tools/build_defs/repo/http.bzl",
"def http_archive(**kwargs):",
" pass",
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/ConfigurationTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/ConfigurationTestCase.java
index 8122469..ea96da0 100644
--- a/src/test/java/com/google/devtools/build/lib/analysis/util/ConfigurationTestCase.java
+++ b/src/test/java/com/google/devtools/build/lib/analysis/util/ConfigurationTestCase.java
@@ -114,6 +114,11 @@
mockToolsConfig.create("/bazel_tools_workspace/WORKSPACE", "workspace(name = 'bazel_tools')");
mockToolsConfig.create("/bazel_tools_workspace/tools/build_defs/repo/BUILD");
mockToolsConfig.create(
+ "/bazel_tools_workspace/tools/build_defs/repo/utils.bzl",
+ "def maybe(repo_rule, name, **kwargs):",
+ " if name not in native.existing_rules():",
+ " repo_rule(name = name, **kwargs)");
+ mockToolsConfig.create(
"/bazel_tools_workspace/tools/build_defs/repo/http.bzl",
"def http_archive(**kwargs):",
" pass",
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoaderTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoaderTest.java
index a3370c1..49b4148 100644
--- a/src/test/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoaderTest.java
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoaderTest.java
@@ -86,6 +86,11 @@
"",
"def http_file(**kwargs):",
" pass");
+ FileSystemUtils.writeIsoLatin1(
+ tools.getRelative("tools/build_defs/repo/utils.bzl"),
+ "def maybe(repo_rule, name, **kwargs):",
+ " if name not in native.existing_rules():",
+ " repo_rule(name = name, **kwargs)");
}
private void fetchExternalRepo(RepositoryName externalRepo) {
diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD
index 1a773ce..e085eee 100644
--- a/src/test/shell/bazel/BUILD
+++ b/src/test/shell/bazel/BUILD
@@ -1104,6 +1104,13 @@
)
sh_test(
+ name = "platforms_test",
+ srcs = ["platforms_test.sh"],
+ data = [":test-deps"],
+ deps = ["@bazel_tools//tools/bash/runfiles"],
+)
+
+sh_test(
name = "platform_mapping_test",
srcs = ["platform_mapping_test.sh"],
data = [":test-deps"],
diff --git a/src/test/shell/bazel/platforms_test.sh b/src/test/shell/bazel/platforms_test.sh
new file mode 100755
index 0000000..6db625a
--- /dev/null
+++ b/src/test/shell/bazel/platforms_test.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+#
+# Copyright 2019 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Test related to @platforms embedded repository
+#
+
+# --- begin runfiles.bash initialization ---
+set -euo pipefail
+if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+ if [[ -f "$0.runfiles_manifest" ]]; then
+ export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest"
+ elif [[ -f "$0.runfiles/MANIFEST" ]]; then
+ export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST"
+ elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+ export RUNFILES_DIR="$0.runfiles"
+ fi
+fi
+if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+ source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash"
+elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+ source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \
+ "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)"
+else
+ echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash"
+ exit 1
+fi
+# --- end runfiles.bash initialization ---
+
+source "$(rlocation "io_bazel/src/test/shell/integration_test_setup.sh")" \
+ || { echo "integration_test_setup.sh not found!" >&2; exit 1; }
+
+function test_platforms_repository_builds_itself() {
+ # We test that a built-in @platforms repository is buildable.
+ bazel build @platforms//:all &> $TEST_log \
+ || fail "Build failed unexpectedly"
+}
+
+function test_platforms_can_be_overridden() {
+ # We test that a custom repository can override @platforms in their
+ # WORKSPACE file.
+ mkdir -p platforms_can_be_overridden || fail "couldn't create directory"
+ touch platforms_can_be_overridden/BUILD || \ fail "couldn't touch BUILD file"
+ cat > platforms_can_be_overridden/WORKSPACE <<EOF
+local_repository(
+ name = 'platforms',
+ path = '../override',
+)
+EOF
+
+ mkdir -p override || fail "couldn't create override directory"
+ touch override/WORKSPACE || fail "couldn't touch override/WORKSPACE"
+ cat > override/BUILD <<EOF
+filegroup(name = 'yolo')
+EOF
+
+ cd platforms_can_be_overridden || fail "couldn't cd into workspace"
+ bazel build @platforms//:yolo &> $TEST_log || \
+ fail "Bazel failed to build @platforms"
+}
+
+run_suite "platform mapping test"
+
diff --git a/src/test/shell/integration/discard_graph_edges_test.sh b/src/test/shell/integration/discard_graph_edges_test.sh
index aab82d9..926dc79 100755
--- a/src/test/shell/integration/discard_graph_edges_test.sh
+++ b/src/test/shell/integration/discard_graph_edges_test.sh
@@ -271,7 +271,7 @@
package_count="$(extract_histogram_count "$histo_file" \
'devtools\.build\.lib\..*\.Package$')"
# A few packages aren't cleared.
- [[ "$package_count" -le 16 ]] \
+ [[ "$package_count" -le 17 ]] \
|| fail "package count $package_count too high"
glob_count="$(extract_histogram_count "$histo_file" "GlobValue$")"
[[ "$glob_count" -le 1 ]] \
diff --git a/tools/build_defs/repo/utils.bzl b/tools/build_defs/repo/utils.bzl
index 5521137..c38b045 100644
--- a/tools/build_defs/repo/utils.bzl
+++ b/tools/build_defs/repo/utils.bzl
@@ -117,3 +117,20 @@
result["name"] = orig.name
result.update(override)
return result
+
+def maybe(repo_rule, name, **kwargs):
+ """Utility function for only adding a repository if it's not already present.
+
+ This is to implement safe repositories.bzl macro documented in
+ https://docs.bazel.build/versions/master/skylark/deploying.html#dependencies.
+
+ Args:
+ repo_rule: repository rule function.
+ name: name of the repository to create.
+ **kwargs: remaining arguments that are passed to the repo_rule function.
+
+ Returns:
+ Nothing, defines the repository when needed as a side-effect.
+ """
+ if name not in native.existing_rules():
+ repo_rule(name = name, **kwargs)
