Add builtins bzls to the install base
This creates //src/main/starlark/builtins_bzl, which is the source root of the @_builtins tree, a valid target of --experimental_builtins_bzl_path. Furthermore, the bzls underneath this directory are now packaged into the install base within a "builtins_bzl" subdir.
Future CLs will complete the proof of concept for builtins injection by adding something trivial for builtins to override, and ensuring that our Java unit tests patch in builtins_bzl correctly.
Work toward #11437.
PiperOrigin-RevId: 342944924
diff --git a/src/BUILD b/src/BUILD
index e621335..d6713d5 100644
--- a/src/BUILD
+++ b/src/BUILD
@@ -12,6 +12,7 @@
name = "install_base_key-file" + suffix,
srcs = [
"//src/main/java/com/google/devtools/build/lib/bazel:BazelServer_deploy.jar",
+ "//src/main/starlark/builtins_bzl:builtins_bzl.zip",
"//src/main/java/net/starlark/java/eval:cpu_profiler",
# TODO(brandjon): ensure we haven't forgotten any package-zip items,
# otherwise bazel won't correctly reextract modified files.
@@ -96,6 +97,14 @@
deps = [":create_embedded_tools_lib"],
)
+# Script for turning builtins_bzl sources into a zip file for unioning into the
+# package zip / install base.
+sh_binary(
+ name = "zip_builtins",
+ srcs = ["zip_builtins.sh"],
+ visibility = ["//src/main/starlark/builtins_bzl:__pkg__"],
+)
+
# The tools Bazel uses to compile Java.
# TODO(#6316): Gradually remove the targets here.
JAVA_TOOLS = [
@@ -315,14 +324,15 @@
[genrule(
name = "package-zip" + suffix,
+ # This script assumes the following arg order: 1) embedded tools zip (if it
+ # exists), 2) the deploy jar, 3) the install base key, 4) the builtins bzl
+ # zip, 5) the platforms archive, 6) everything else to be bundled.
srcs = ([":embedded_tools" + suffix + ".zip"] if embed else []) + [
- # The script assumes that the embedded tools zip (if exists) is the
- # first item here, the deploy jar the second, install base key is the
- # third, and platforms archive is the fourth.
"//src/main/java/com/google/devtools/build/lib/bazel:BazelServer_deploy.jar",
"install_base_key" + suffix,
+ "//src/main/starlark/builtins_bzl:builtins_bzl.zip",
":platforms_archive",
- # Ordinary items follow:
+ # Non-ordered items follow:
"//src/main/java/net/starlark/java/eval:cpu_profiler",
"//src/main/tools:build-runfiles",
"//src/main/tools:process-wrapper",
@@ -439,6 +449,7 @@
"//src/main/java/net/starlark/java/lib/json:srcs",
"//src/main/native:srcs",
"//src/main/protobuf:srcs",
+ "//src/main/starlark/builtins_bzl:srcs",
"//src/main/tools:srcs",
"//src/test/cpp:srcs",
"//src/test/gen:srcs",
diff --git a/src/main/starlark/builtins_bzl/BUILD b/src/main/starlark/builtins_bzl/BUILD
new file mode 100644
index 0000000..ee13edc
--- /dev/null
+++ b/src/main/starlark/builtins_bzl/BUILD
@@ -0,0 +1,26 @@
+# This directory serves as the root of the builtins_bzl tree for Bazel, both in
+# source form and as the value of --experimental_builtins_bzl_path.
+#
+# Because we use globs to gather builtins sources, there should be no
+# subpackages here.
+
+filegroup(
+ name = "srcs",
+ srcs = glob(["**"]),
+ visibility = ["//src:__pkg__"],
+)
+
+# A zipfile containing the builtins_bzl/ directory as it should appear in the
+# package zip / install base.
+genrule(
+ name = "builtins_bzl_zip",
+ srcs = [f for f in glob(["**"]) if f.endswith(".bzl")] + ["BUILD.builtins"],
+ outs = ["builtins_bzl.zip"],
+ # builtins_zip.sh zip output builtins_root files...
+ cmd = "$(location //src:zip_builtins)" +
+ " ''" + # system zip
+ " $@ src/main/starlark/builtins_bzl $(SRCS)",
+ output_to_bindir = 1,
+ tools = ["//src:zip_builtins"],
+ visibility = ["//src:__pkg__"],
+)
diff --git a/src/main/starlark/builtins_bzl/BUILD.builtins b/src/main/starlark/builtins_bzl/BUILD.builtins
new file mode 100644
index 0000000..bdb8e52
--- /dev/null
+++ b/src/main/starlark/builtins_bzl/BUILD.builtins
@@ -0,0 +1,8 @@
+# The .bzl files in this directory are used by Bazel itself, and should not be
+# modified or copied into a Bazel workspace. Aside from not being a stable
+# interface, they use a modified dialect of Bazel's Build Language and cannot be
+# loaded as regular .bzl files.
+#
+# Bazel developers: This directory is used whenever "%install_base%" is passed
+# to --experimental_builtins_bzl_path. See StarlarkBuiltinsFunction.java and
+# BzlLoadFunction.java for more information.
diff --git a/src/main/starlark/builtins_bzl/bazel/exports.bzl b/src/main/starlark/builtins_bzl/bazel/exports.bzl
new file mode 100644
index 0000000..3d5df8f
--- /dev/null
+++ b/src/main/starlark/builtins_bzl/bazel/exports.bzl
@@ -0,0 +1,19 @@
+# Copyright 2020 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Exported builtins symbols that are specific to OSS Bazel."""
+
+exported_toplevels = {}
+exported_rules = {}
+exported_to_java = {}
diff --git a/src/main/starlark/builtins_bzl/common/exports.bzl b/src/main/starlark/builtins_bzl/common/exports.bzl
new file mode 100644
index 0000000..7597b94
--- /dev/null
+++ b/src/main/starlark/builtins_bzl/common/exports.bzl
@@ -0,0 +1,19 @@
+# Copyright 2020 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Exported builtins symbols that are not specific to OSS Bazel."""
+
+exported_toplevels = {}
+exported_rules = {}
+exported_to_java = {}
diff --git a/src/main/starlark/builtins_bzl/common/util.bzl b/src/main/starlark/builtins_bzl/common/util.bzl
new file mode 100644
index 0000000..ce1b085
--- /dev/null
+++ b/src/main/starlark/builtins_bzl/common/util.bzl
@@ -0,0 +1,25 @@
+# Copyright 2020 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Used by top-level exports.bzl."""
+
+def dict_union(*dicts):
+ """Returns the strict union of zero or more dicts."""
+ result = {}
+ for d in dicts:
+ for k, v in d.items():
+ if k in result:
+ fail("Duplicate key: '%s'" % k)
+ result[k] = v
+ return result
diff --git a/src/main/starlark/builtins_bzl/exports.bzl b/src/main/starlark/builtins_bzl/exports.bzl
new file mode 100644
index 0000000..2622e79
--- /dev/null
+++ b/src/main/starlark/builtins_bzl/exports.bzl
@@ -0,0 +1,34 @@
+# Copyright 2020 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Top-level exports file read by Bazel."""
+# (See StarlarkBuiltinsFunction.java)
+
+load(
+ "@_builtins//:bazel/exports.bzl",
+ bazel_exported_rules = "exported_rules",
+ bazel_exported_to_java = "exported_to_java",
+ bazel_exported_toplevels = "exported_toplevels",
+)
+load(
+ "@_builtins//:common/exports.bzl",
+ common_exported_rules = "exported_rules",
+ common_exported_to_java = "exported_to_java",
+ common_exported_toplevels = "exported_toplevels",
+)
+load("@_builtins//:common/util.bzl", "dict_union")
+
+exported_rules = dict_union(common_exported_rules, bazel_exported_rules)
+exported_toplevels = dict_union(common_exported_toplevels, bazel_exported_toplevels)
+exported_to_java = dict_union(common_exported_to_java, bazel_exported_to_java)
diff --git a/src/package-bazel.sh b/src/package-bazel.sh
index 7cc1d33..9a00625 100755
--- a/src/package-bazel.sh
+++ b/src/package-bazel.sh
@@ -20,12 +20,12 @@
# starts the server from.
WORKDIR=$(pwd)
-OUT=$1
-EMBEDDED_TOOLS=$2
-DEPLOY_JAR=$3
-INSTALL_BASE_KEY=$4
-PLATFORMS_ARCHIVE=$5
-shift 5
+OUT=$1; shift
+EMBEDDED_TOOLS=$1; shift
+DEPLOY_JAR=$1; shift
+INSTALL_BASE_KEY=$1; shift
+BUILTINS_ZIP=$1; shift
+PLATFORMS_ARCHIVE=$1; shift
if [[ "$OUT" == *jdk_allmodules.zip ]]; then
DEV_BUILD=1
@@ -71,6 +71,12 @@
(cd ${PACKAGE_DIR}/embedded_tools && unzip -q "${WORKDIR}/${EMBEDDED_TOOLS}")
fi
+# Add the builtins bzls.
+(
+ cd $PACKAGE_DIR
+ unzip -q $WORKDIR/$BUILTINS_ZIP
+)
+
# Unzip platforms.zip into platforms/, move files up from external/platforms
# subdirectory, and create WORKSPACE if it doesn't exist.
(
diff --git a/src/zip_builtins.sh b/src/zip_builtins.sh
new file mode 100755
index 0000000..11fcf94
--- /dev/null
+++ b/src/zip_builtins.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# Copyright 2020 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Packages the builtins bzl files as a zip, while preserving their directory
+# structure and expanding any symlinks.
+#
+# Usage:
+# zip_builtins.sh zip output builtins_root files...
+
+set -euo pipefail
+
+origdir="$(pwd)"
+
+# the "zip" command to use; system "zip" if empty
+if [ -z "$1" ]; then
+ zip="zip"
+else
+ zip=$origdir/$1;
+fi; shift
+output=$origdir/$1; shift # zip file to write
+builtins_root=$1; shift # root-relative path to builtins_bzl source dir
+# "$@" contains the paths of the files to archive. They must all be under the
+# source root (no generated files as inputs).
+
+TMPDIR="$(mktemp -d "${TMPDIR:-/tmp}/tmp.XXXXXXXXXX")"
+trap "rm -rf $TMPDIR" EXIT
+
+# Do the cp in origdir so $@ resolves. We'd like to use cp --parents, but macOS
+# doesn't have that flag, so break it out into a loop.
+mkdir -p "$TMPDIR/staging"
+for src in "$@"; do
+ dst="$TMPDIR/staging/$src"
+ mkdir -p $(dirname "$dst")
+ # Make certain to expand any symlinked files (-L).
+ cp -L "$src" "$dst"
+done
+
+cd "$TMPDIR"
+
+# Strip the dir prefix leading up to the builtins root.
+mv "staging/$builtins_root" builtins_bzl
+mv builtins_bzl/BUILD.builtins builtins_bzl/BUILD
+
+# The zip step must take place while cwd is tmpdir, so the paths in the zipfile
+# are relative to tmpdir.
+#
+# For determinism, sort the files and zero out their timestamps before zipping.
+find builtins_bzl -type f -print0 | xargs -0 touch -t 198001010000.00
+find builtins_bzl -type f | sort | "$zip" -qDX0r@ "$output" builtins_bzl
