Start making distribution dependencies easier to update.
This creates a mechanism to address #12081, but does not convert all use cases yet.
The basic idea is to change distdir_tar so that each dependency is defined in a block rather than split into distinct dictionaries for archive, sha256, and urls. This means that updating a dependency can be isolated to 3 or 4 lines which appear together, rather than being sprinkled throughout WORKSPACE.
The next steps would be
- do the same for the other dependencies. I want to see this land and stay that way before doing the rest. Those PRs should have trivial reviews.
- determine how to use DISTDIR_DEPS in the other embedded workspace files, such as src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/cc_configure.WORKSPACE
- we need to splice distdir_deps.bzl into the dynamically created WORKSPACE files
- or, export distdir_deps.bzl to the workspace and load it
- or, since cc_configure.WORKSPACE is data, we generate it at build time.
These could be done in either order.
*other potential changes*
- streamline the http_archive and maybe instances to read directly from DISTDIR. This would eliminate the tedious pattern of
```
sha256 = DIST_DEPS["rules_pkg"]["sha256"],
urls = DIST_DEPS["rules_pkg"]["urls"],
```
- hide DISTDIR_DEPS from WORKSPACE.
- pass just a list of package names (not archive names) to distfile_tar().
- distdir.bzl loads the deps list directly and looks up data by package name
RELNOTES: None
PiperOrigin-RevId: 346580497
diff --git a/BUILD b/BUILD
index c4f89bb..5a271bc 100644
--- a/BUILD
+++ b/BUILD
@@ -49,6 +49,7 @@
srcs = [
":WORKSPACE",
":distdir.bzl",
+ ":distdir_deps.bzl",
],
visibility = [
"//src/test/shell/bazel:__subpackages__",
diff --git a/WORKSPACE b/WORKSPACE
index 42a507b..a5f019c 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -140,6 +140,7 @@
)
load("//:distdir.bzl", "distdir_tar")
+load("//:distdir_deps.bzl", "DIST_DEPS")
distdir_tar(
name = "additional_distfiles",
@@ -164,12 +165,8 @@
"platforms-0.0.2.tar.gz",
# bazelbuild/rules_java
"7cf3cefd652008d0a64a419c34c13bdca6c8f178.zip",
- # bazelbuild/rules_cc
- "b1c40e1de81913a3c40e5948f78719c28152486d.zip",
# bazelbuild/bazel-toolchains
"bazel-toolchains-3.1.0.tar.gz",
- # bazelbuild/rules_pkg
- "rules_pkg-0.2.4.tar.gz",
# bazelbuild/rules_proto
"7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz",
# protocolbuffers/protobuf
@@ -186,6 +183,7 @@
"df3ea785d8c30a9503321a3d35ee7d35808f190d.tar.gz",
],
dirname = "derived/distdir",
+ dist_deps = DIST_DEPS,
sha256 = {
"e0b0291b2c51fbe5a7cfa14473a1ae850f94f021.zip": "fe2e04f91ce8c59d49d91b8102edc6627c6fa2906c1b0e7346f01419ec4f419d",
"java_tools-v11.0.zip": "09ecd438f1a10aa36bf0a6a2f24ead884ef7e8e8a46d086f8af6db33556b76a8",
@@ -206,12 +204,8 @@
"platforms-0.0.2.tar.gz": "48a2d8d343863989c232843e01afc8a986eb8738766bfd8611420a7db8f6f0c3",
# bazelbuild/rules_java
"7cf3cefd652008d0a64a419c34c13bdca6c8f178.zip": "bc81f1ba47ef5cc68ad32225c3d0e70b8c6f6077663835438da8d5733f917598",
- # bazelbuild/rules_cc
- "b1c40e1de81913a3c40e5948f78719c28152486d.zip": "d0c573b94a6ef20ef6ff20154a23d0efcb409fb0e1ff0979cec318dfe42f0cdd",
# bazelbuild/bazel-toolchains
"bazel-toolchains-3.1.0.tar.gz": "726b5423e1c7a3866a3a6d68e7123b4a955e9fcbe912a51e0f737e6dab1d0af2",
- # bazelbuild/rules_pkg
- "rules_pkg-0.2.4.tar.gz": "4ba8f4ab0ff85f2484287ab06c0d871dcb31cc54d439457d28fd4ae14b18450a",
# bazelbuild/rules_proto
"7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz": "8e7d59a5b12b233be5652e3d29f42fba01c7cbab09f6b3a8d0a57ed6d1e9a0da",
# protocolbuffers/protobuf
@@ -277,21 +271,11 @@
"https://mirror.bazel.build/github.com/bazelbuild/rules_java/archive/7cf3cefd652008d0a64a419c34c13bdca6c8f178.zip",
"https://github.com/bazelbuild/rules_java/archive/7cf3cefd652008d0a64a419c34c13bdca6c8f178.zip",
],
- # bazelbuild/rules_cc
- "b1c40e1de81913a3c40e5948f78719c28152486d.zip": [
- "https://mirror.bazel.build/github.com/bazelbuild/rules_cc/archive/b1c40e1de81913a3c40e5948f78719c28152486d.zip",
- "https://github.com/bazelbuild/rules_cc/archive/b1c40e1de81913a3c40e5948f78719c28152486d.zip",
- ],
# bazelbuild/bazel-toolchains
"bazel-toolchains-3.1.0.tar.gz": [
"https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/releases/download/3.1.0/bazel-toolchains-3.1.0.tar.gz",
"https://github.com/bazelbuild/bazel-toolchains/releases/download/3.1.0/bazel-toolchains-3.1.0.tar.gz",
],
- # bazelbuild/rules_pkg
- "rules_pkg-0.2.4.tar.gz": [
- "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.2.4/rules_pkg-0.2.4.tar.gz",
- "https://github.com/bazelbuild/rules_pkg/releases/download/0.2.4/rules_pkg-0.2.4.tar.gz",
- ],
# bazelbuild/rules_proto
"7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz": [
"https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz",
@@ -512,12 +496,9 @@
name = "rules_cc",
patch_cmds = EXPORT_WORKSPACE_IN_BUILD_FILE,
patch_cmds_win = EXPORT_WORKSPACE_IN_BUILD_FILE_WIN,
- sha256 = "d0c573b94a6ef20ef6ff20154a23d0efcb409fb0e1ff0979cec318dfe42f0cdd",
- strip_prefix = "rules_cc-b1c40e1de81913a3c40e5948f78719c28152486d",
- urls = [
- "https://mirror.bazel.build/github.com/bazelbuild/rules_cc/archive/b1c40e1de81913a3c40e5948f78719c28152486d.zip",
- "https://github.com/bazelbuild/rules_cc/archive/b1c40e1de81913a3c40e5948f78719c28152486d.zip",
- ],
+ sha256 = DIST_DEPS["rules_cc"]["sha256"],
+ strip_prefix = DIST_DEPS["rules_cc"]["strip_prefix"],
+ urls = DIST_DEPS["rules_cc"]["urls"],
)
http_archive(
@@ -566,8 +547,6 @@
"platforms-0.0.2.tar.gz",
# bazelbuild/rules_java
"7cf3cefd652008d0a64a419c34c13bdca6c8f178.zip",
- # bazelbuild/rules_cc
- "b1c40e1de81913a3c40e5948f78719c28152486d.zip",
# bazelbuild/rules_proto
"7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz",
# protocolbuffers/protobuf
@@ -584,6 +563,7 @@
"df3ea785d8c30a9503321a3d35ee7d35808f190d.tar.gz",
],
dirname = "test_WORKSPACE/distdir",
+ dist_deps = {dep: attrs for dep, attrs in DIST_DEPS.items() if attrs.get("need_in_test_WORKSPACE")},
sha256 = {
"java_tools-v11.0.zip": "09ecd438f1a10aa36bf0a6a2f24ead884ef7e8e8a46d086f8af6db33556b76a8",
"java_tools_linux-v11.0.zip": "b66d5b97b90cb20787cfa61565672b0538912d230f120a03f38020052f25c4bc",
@@ -601,8 +581,6 @@
"platforms-0.0.2.tar.gz": "48a2d8d343863989c232843e01afc8a986eb8738766bfd8611420a7db8f6f0c3",
# bazelbuild/rules_java
"7cf3cefd652008d0a64a419c34c13bdca6c8f178.zip": "bc81f1ba47ef5cc68ad32225c3d0e70b8c6f6077663835438da8d5733f917598",
- # bazelbuild/rules_cc
- "b1c40e1de81913a3c40e5948f78719c28152486d.zip": "d0c573b94a6ef20ef6ff20154a23d0efcb409fb0e1ff0979cec318dfe42f0cdd",
# bazelbuild/rules_proto
"7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz": "8e7d59a5b12b233be5652e3d29f42fba01c7cbab09f6b3a8d0a57ed6d1e9a0da",
# protocolbuffers/protobuf
@@ -646,11 +624,6 @@
"https://mirror.bazel.build/github.com/bazelbuild/rules_java/archive/7cf3cefd652008d0a64a419c34c13bdca6c8f178.zip",
"https://github.com/bazelbuild/rules_java/archive/7cf3cefd652008d0a64a419c34c13bdca6c8f178.zip",
],
- # bazelbuild/rules_cc
- "b1c40e1de81913a3c40e5948f78719c28152486d.zip": [
- "https://mirror.bazel.build/github.com/bazelbuild/rules_cc/archive/b1c40e1de81913a3c40e5948f78719c28152486d.zip",
- "https://github.com/bazelbuild/rules_cc/archive/b1c40e1de81913a3c40e5948f78719c28152486d.zip",
- ],
# bazelbuild/rules_proto
"7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz": [
"https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz",
@@ -1153,11 +1126,8 @@
name = "rules_pkg",
patch_cmds = EXPORT_WORKSPACE_IN_BUILD_FILE,
patch_cmds_win = EXPORT_WORKSPACE_IN_BUILD_FILE_WIN,
- sha256 = "4ba8f4ab0ff85f2484287ab06c0d871dcb31cc54d439457d28fd4ae14b18450a",
- urls = [
- "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.2.4/rules_pkg-0.2.4.tar.gz",
- "https://github.com/bazelbuild/rules_pkg/releases/download/0.2.4/rules_pkg-0.2.4.tar.gz",
- ],
+ sha256 = DIST_DEPS["rules_pkg"]["sha256"],
+ urls = DIST_DEPS["rules_pkg"]["urls"],
)
load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
diff --git a/distdir.bzl b/distdir.bzl
index a4d9265..60b8678 100644
--- a/distdir.bzl
+++ b/distdir.bzl
@@ -41,7 +41,32 @@
"dirname": attr.string(default = "distdir"),
}
-distdir_tar = repository_rule(
+_distdir_tar = repository_rule(
implementation = _distdir_tar_impl,
attrs = _distdir_tar_attrs,
)
+
+def distdir_tar(name, archives, sha256, urls, dirname, dist_deps = None):
+ """Creates a repository whose content is a set of tar files.
+
+ Args:
+ name: repo name.
+ archives: list of tar file names.
+ sha256: map of tar file names to SHAs.
+ urls: map of tar file names to URL lists.
+ dirname: output directory in repo.
+ dist_deps: map of repo names to dict of archive, sha256, and urls.
+ """
+ if dist_deps:
+ for dep, info in dist_deps.items():
+ archive_file = info["archive"]
+ archives.append(archive_file)
+ sha256[archive_file] = info["sha256"]
+ urls[archive_file] = info["urls"]
+ _distdir_tar(
+ name = name,
+ archives = archives,
+ sha256 = sha256,
+ urls = urls,
+ dirname = dirname,
+ )
diff --git a/distdir_deps.bzl b/distdir_deps.bzl
new file mode 100644
index 0000000..c1a84f1
--- /dev/null
+++ b/distdir_deps.bzl
@@ -0,0 +1,37 @@
+# Copyright 2020 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""List the distribution dependencies we need to build Bazel."""
+
+DIST_DEPS = {
+ # This must be kept in sync with src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/cc_configure.WORKSPACE.
+ # This must be kept in sync with src/main/java/com/google/devtools/build/lib/bazel/rules/java/jdk.WORKSPACE.
+ "rules_cc": {
+ "archive": "b1c40e1de81913a3c40e5948f78719c28152486d.zip",
+ "sha256": "d0c573b94a6ef20ef6ff20154a23d0efcb409fb0e1ff0979cec318dfe42f0cdd",
+ "strip_prefix": "rules_cc-b1c40e1de81913a3c40e5948f78719c28152486d",
+ "urls": [
+ "https://mirror.bazel.build/github.com/bazelbuild/rules_cc/archive/b1c40e1de81913a3c40e5948f78719c28152486d.zip",
+ "https://github.com/bazelbuild/rules_cc/archive/b1c40e1de81913a3c40e5948f78719c28152486d.zip",
+ ],
+ "need_in_test_WORKSPACE": True,
+ },
+ "rules_pkg": {
+ "archive": "rules_pkg-0.2.4.tar.gz",
+ "sha256": "4ba8f4ab0ff85f2484287ab06c0d871dcb31cc54d439457d28fd4ae14b18450a",
+ "urls": [
+ "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.2.4/rules_pkg-0.2.4.tar.gz",
+ "https://github.com/bazelbuild/rules_pkg/releases/download/0.2.4/rules_pkg-0.2.4.tar.gz",
+ ],
+ },
+}
