Add builtins bzls to the install base This creates //src/main/starlark/builtins_bzl, which is the source root of the @_builtins tree, a valid target of --experimental_builtins_bzl_path. Furthermore, the bzls underneath this directory are now packaged into the install base within a "builtins_bzl" subdir. Future CLs will complete the proof of concept for builtins injection by adding something trivial for builtins to override, and ensuring that our Java unit tests patch in builtins_bzl correctly. Work toward #11437. PiperOrigin-RevId: 342944924
diff --git a/src/BUILD b/src/BUILD index e621335..d6713d5 100644 --- a/src/BUILD +++ b/src/BUILD
@@ -12,6 +12,7 @@ name = "install_base_key-file" + suffix, srcs = [ "//src/main/java/com/google/devtools/build/lib/bazel:BazelServer_deploy.jar", + "//src/main/starlark/builtins_bzl:builtins_bzl.zip", "//src/main/java/net/starlark/java/eval:cpu_profiler", # TODO(brandjon): ensure we haven't forgotten any package-zip items, # otherwise bazel won't correctly reextract modified files. @@ -96,6 +97,14 @@ deps = [":create_embedded_tools_lib"], ) +# Script for turning builtins_bzl sources into a zip file for unioning into the +# package zip / install base. +sh_binary( + name = "zip_builtins", + srcs = ["zip_builtins.sh"], + visibility = ["//src/main/starlark/builtins_bzl:__pkg__"], +) + # The tools Bazel uses to compile Java. # TODO(#6316): Gradually remove the targets here. JAVA_TOOLS = [ @@ -315,14 +324,15 @@ [genrule( name = "package-zip" + suffix, + # This script assumes the following arg order: 1) embedded tools zip (if it + # exists), 2) the deploy jar, 3) the install base key, 4) the builtins bzl + # zip, 5) the platforms archive, 6) everything else to be bundled. srcs = ([":embedded_tools" + suffix + ".zip"] if embed else []) + [ - # The script assumes that the embedded tools zip (if exists) is the - # first item here, the deploy jar the second, install base key is the - # third, and platforms archive is the fourth. "//src/main/java/com/google/devtools/build/lib/bazel:BazelServer_deploy.jar", "install_base_key" + suffix, + "//src/main/starlark/builtins_bzl:builtins_bzl.zip", ":platforms_archive", - # Ordinary items follow: + # Non-ordered items follow: "//src/main/java/net/starlark/java/eval:cpu_profiler", "//src/main/tools:build-runfiles", "//src/main/tools:process-wrapper", @@ -439,6 +449,7 @@ "//src/main/java/net/starlark/java/lib/json:srcs", "//src/main/native:srcs", "//src/main/protobuf:srcs", + "//src/main/starlark/builtins_bzl:srcs", "//src/main/tools:srcs", "//src/test/cpp:srcs", "//src/test/gen:srcs",
diff --git a/src/main/starlark/builtins_bzl/BUILD b/src/main/starlark/builtins_bzl/BUILD new file mode 100644 index 0000000..ee13edc --- /dev/null +++ b/src/main/starlark/builtins_bzl/BUILD
@@ -0,0 +1,26 @@ +# This directory serves as the root of the builtins_bzl tree for Bazel, both in +# source form and as the value of --experimental_builtins_bzl_path. +# +# Because we use globs to gather builtins sources, there should be no +# subpackages here. + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//src:__pkg__"], +) + +# A zipfile containing the builtins_bzl/ directory as it should appear in the +# package zip / install base. +genrule( + name = "builtins_bzl_zip", + srcs = [f for f in glob(["**"]) if f.endswith(".bzl")] + ["BUILD.builtins"], + outs = ["builtins_bzl.zip"], + # builtins_zip.sh zip output builtins_root files... + cmd = "$(location //src:zip_builtins)" + + " ''" + # system zip + " $@ src/main/starlark/builtins_bzl $(SRCS)", + output_to_bindir = 1, + tools = ["//src:zip_builtins"], + visibility = ["//src:__pkg__"], +)
diff --git a/src/main/starlark/builtins_bzl/BUILD.builtins b/src/main/starlark/builtins_bzl/BUILD.builtins new file mode 100644 index 0000000..bdb8e52 --- /dev/null +++ b/src/main/starlark/builtins_bzl/BUILD.builtins
@@ -0,0 +1,8 @@ +# The .bzl files in this directory are used by Bazel itself, and should not be +# modified or copied into a Bazel workspace. Aside from not being a stable +# interface, they use a modified dialect of Bazel's Build Language and cannot be +# loaded as regular .bzl files. +# +# Bazel developers: This directory is used whenever "%install_base%" is passed +# to --experimental_builtins_bzl_path. See StarlarkBuiltinsFunction.java and +# BzlLoadFunction.java for more information.
diff --git a/src/main/starlark/builtins_bzl/bazel/exports.bzl b/src/main/starlark/builtins_bzl/bazel/exports.bzl new file mode 100644 index 0000000..3d5df8f --- /dev/null +++ b/src/main/starlark/builtins_bzl/bazel/exports.bzl
@@ -0,0 +1,19 @@ +# Copyright 2020 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exported builtins symbols that are specific to OSS Bazel.""" + +exported_toplevels = {} +exported_rules = {} +exported_to_java = {}
diff --git a/src/main/starlark/builtins_bzl/common/exports.bzl b/src/main/starlark/builtins_bzl/common/exports.bzl new file mode 100644 index 0000000..7597b94 --- /dev/null +++ b/src/main/starlark/builtins_bzl/common/exports.bzl
@@ -0,0 +1,19 @@ +# Copyright 2020 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exported builtins symbols that are not specific to OSS Bazel.""" + +exported_toplevels = {} +exported_rules = {} +exported_to_java = {}
diff --git a/src/main/starlark/builtins_bzl/common/util.bzl b/src/main/starlark/builtins_bzl/common/util.bzl new file mode 100644 index 0000000..ce1b085 --- /dev/null +++ b/src/main/starlark/builtins_bzl/common/util.bzl
@@ -0,0 +1,25 @@ +# Copyright 2020 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Used by top-level exports.bzl.""" + +def dict_union(*dicts): + """Returns the strict union of zero or more dicts.""" + result = {} + for d in dicts: + for k, v in d.items(): + if k in result: + fail("Duplicate key: '%s'" % k) + result[k] = v + return result
diff --git a/src/main/starlark/builtins_bzl/exports.bzl b/src/main/starlark/builtins_bzl/exports.bzl new file mode 100644 index 0000000..2622e79 --- /dev/null +++ b/src/main/starlark/builtins_bzl/exports.bzl
@@ -0,0 +1,34 @@ +# Copyright 2020 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Top-level exports file read by Bazel.""" +# (See StarlarkBuiltinsFunction.java) + +load( + "@_builtins//:bazel/exports.bzl", + bazel_exported_rules = "exported_rules", + bazel_exported_to_java = "exported_to_java", + bazel_exported_toplevels = "exported_toplevels", +) +load( + "@_builtins//:common/exports.bzl", + common_exported_rules = "exported_rules", + common_exported_to_java = "exported_to_java", + common_exported_toplevels = "exported_toplevels", +) +load("@_builtins//:common/util.bzl", "dict_union") + +exported_rules = dict_union(common_exported_rules, bazel_exported_rules) +exported_toplevels = dict_union(common_exported_toplevels, bazel_exported_toplevels) +exported_to_java = dict_union(common_exported_to_java, bazel_exported_to_java)
diff --git a/src/package-bazel.sh b/src/package-bazel.sh index 7cc1d33..9a00625 100755 --- a/src/package-bazel.sh +++ b/src/package-bazel.sh
@@ -20,12 +20,12 @@ # starts the server from. WORKDIR=$(pwd) -OUT=$1 -EMBEDDED_TOOLS=$2 -DEPLOY_JAR=$3 -INSTALL_BASE_KEY=$4 -PLATFORMS_ARCHIVE=$5 -shift 5 +OUT=$1; shift +EMBEDDED_TOOLS=$1; shift +DEPLOY_JAR=$1; shift +INSTALL_BASE_KEY=$1; shift +BUILTINS_ZIP=$1; shift +PLATFORMS_ARCHIVE=$1; shift if [[ "$OUT" == *jdk_allmodules.zip ]]; then DEV_BUILD=1 @@ -71,6 +71,12 @@ (cd ${PACKAGE_DIR}/embedded_tools && unzip -q "${WORKDIR}/${EMBEDDED_TOOLS}") fi +# Add the builtins bzls. +( + cd $PACKAGE_DIR + unzip -q $WORKDIR/$BUILTINS_ZIP +) + # Unzip platforms.zip into platforms/, move files up from external/platforms # subdirectory, and create WORKSPACE if it doesn't exist. (
diff --git a/src/zip_builtins.sh b/src/zip_builtins.sh new file mode 100755 index 0000000..11fcf94 --- /dev/null +++ b/src/zip_builtins.sh
@@ -0,0 +1,62 @@ +#!/bin/bash + +# Copyright 2020 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Packages the builtins bzl files as a zip, while preserving their directory +# structure and expanding any symlinks. +# +# Usage: +# zip_builtins.sh zip output builtins_root files... + +set -euo pipefail + +origdir="$(pwd)" + +# the "zip" command to use; system "zip" if empty +if [ -z "$1" ]; then + zip="zip" +else + zip=$origdir/$1; +fi; shift +output=$origdir/$1; shift # zip file to write +builtins_root=$1; shift # root-relative path to builtins_bzl source dir +# "$@" contains the paths of the files to archive. They must all be under the +# source root (no generated files as inputs). + +TMPDIR="$(mktemp -d "${TMPDIR:-/tmp}/tmp.XXXXXXXXXX")" +trap "rm -rf $TMPDIR" EXIT + +# Do the cp in origdir so $@ resolves. We'd like to use cp --parents, but macOS +# doesn't have that flag, so break it out into a loop. +mkdir -p "$TMPDIR/staging" +for src in "$@"; do + dst="$TMPDIR/staging/$src" + mkdir -p $(dirname "$dst") + # Make certain to expand any symlinked files (-L). + cp -L "$src" "$dst" +done + +cd "$TMPDIR" + +# Strip the dir prefix leading up to the builtins root. +mv "staging/$builtins_root" builtins_bzl +mv builtins_bzl/BUILD.builtins builtins_bzl/BUILD + +# The zip step must take place while cwd is tmpdir, so the paths in the zipfile +# are relative to tmpdir. +# +# For determinism, sort the files and zero out their timestamps before zipping. +find builtins_bzl -type f -print0 | xargs -0 touch -t 198001010000.00 +find builtins_bzl -type f | sort | "$zip" -qDX0r@ "$output" builtins_bzl