Add modules to create and download debian packages (#154)
* Add modules to get debian packages
1. Extract a new rule to download debian packages and create a tarball
2. Create a new file for the new rule aggregate_debian_pkgs
3. Move the use of new rule into language_tool_layer
diff --git a/container/rules/debian_pkg_tar.bzl b/container/rules/debian_pkg_tar.bzl
new file mode 100644
index 0000000..d76fb38
--- /dev/null
+++ b/container/rules/debian_pkg_tar.bzl
@@ -0,0 +1,188 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@base_images_docker//package_managers:download_pkgs.bzl", _download_deb_pkgs = "download")
+load("@base_images_docker//package_managers:apt_key.bzl", _apt_key = "key")
+
+def _input_validation(kwargs):
+ allowed_attribues = ["name", "base", "language_layers"]
+
+ for key in kwargs.keys():
+ if key not in allowed_attribues:
+ fail("Attribute " + key + " is not supported.")
+
+container = [
+ ".tar.gz",
+ ".tgz",
+ ".tar",
+ ".tar.xz",
+]
+
+generate_deb_tar_attrs = _download_deb_pkgs.attrs + {
+ "base": attr.label(allow_files = container),
+ "packages": attr.string_list(),
+ "keys": attr.label_list(
+ allow_files = True,
+ ),
+}
+
+aggregate_debian_pkgs_attrs = {
+ "base": attr.label(allow_files = container),
+ "language_layers": attr.label_list(),
+
+ # Declare the following attributes since _download_deb_pkgs.implementation
+ # need access those attribute if their overrides are None
+ "additional_repos": attr.string_list(),
+ "_image_id_extractor": attr.label(
+ default = "@io_bazel_rules_docker//contrib:extract_image_id.py",
+ allow_files = True,
+ single_file = True,
+ ),
+}
+
+InstallableTarInfo = provider(fields = [
+ "installables_tar",
+])
+
+def _generate_deb_tar(
+ ctx,
+ packages = None,
+ additional_repos = None,
+ keys = None,
+ download_pkgs_output_tar = None,
+ download_pkgs_output_script = None):
+ """A function for producing a tarball for a set of debian packages.
+
+ Args:
+ ctx: ctx has either generate_deb_tar_attrs or aggregate_debian_pkgs_attrs,
+ this depends on the rule that uses this function
+ packages: list of packages to fetch and install in the base image.
+ additional_repos: list of additional debian package repos to use,
+ in sources.list format.
+ keys: list of labels of additional gpg keys to use while downloading
+ packages.
+ download_pkgs_output_tar: output tar file generated by download_pkgs rule to
+ override default output_tar
+ download_pkgs_output_script: output script generated by download_pkgs rule to
+ override default output_script
+ """
+
+ # Prepare base image for the download_pkgs rule.
+ download_base = ctx.files.base[0]
+
+ # Create an intermediate image with additional gpg keys used to download packages.
+ if keys != []:
+ image_with_keys = "%s_with_keys" % ctx.attr.name
+
+ # Declare intermediate output file generated by add_apt_key rule.
+ image_with_keys_output_executable = ctx.actions.declare_file(image_with_keys)
+ image_with_keys_output_tarball = ctx.actions.declare_file(image_with_keys + ".tar")
+ image_with_keys_output_layer = ctx.actions.declare_file(image_with_keys + "-layer.tar")
+
+ _apt_key.implementation(
+ ctx,
+ name = image_with_keys,
+ image_tar = ctx.files.base[0],
+ keys = keys,
+ output_executable = image_with_keys_output_executable,
+ output_tarball = image_with_keys_output_tarball,
+ output_layer = image_with_keys_output_layer,
+ )
+ download_base = image_with_keys_output_tarball
+
+ # Declare intermediate output file generated by download_pkgs rule.
+ output_executable = ctx.actions.declare_file(ctx.attr.name + "-output_executable.sh")
+ download_pkgs_output_tar = download_pkgs_output_tar or ctx.attr.name + ".tar"
+ download_pkgs_output_script = download_pkgs_output_script or ctx.attr.name + ".sh"
+ output_tar = ctx.actions.declare_file(download_pkgs_output_tar)
+ output_script = ctx.actions.declare_file(download_pkgs_output_script)
+
+ # download_pkgs rule consumes 'packages' and 'additional_repos'.
+ _download_deb_pkgs.implementation(
+ ctx,
+ image_tar = download_base,
+ packages = packages,
+ additional_repos = additional_repos,
+ output_executable = output_executable,
+ output_tar = output_tar,
+ output_script = output_script,
+ )
+
+ return struct(
+ providers = [
+ InstallableTarInfo(
+ installables_tar = output_tar,
+ ),
+ ],
+ )
+
+def _aggregate_debian_pkgs_impl(ctx):
+ """Implementation for the aggregate_debian_pkgs rule.
+
+ aggregate_debian_pkgs rule produces a tarball with all debian packages declared
+ in the language_tool_layer(s) this rule depends on.
+
+ Args:
+ ctx: ctx only has name, base, and language_layers attributes
+ """
+
+ packages = []
+ additional_repos = []
+ keys = []
+
+ for layer in ctx.attr.language_layers:
+ packages.extend(layer.packages)
+ additional_repos.extend(layer.additional_repos)
+ keys.extend(layer.keys)
+
+ packages = depset(packages).to_list()
+ additional_repos = depset(additional_repos).to_list()
+ keys = depset(keys).to_list()
+
+ return _generate_deb_tar(
+ ctx,
+ packages = packages,
+ additional_repos = additional_repos,
+ keys = keys,
+ )
+
+# Export _generate_deb_tar function for other bazel rules to use.
+generate = struct(
+ attrs = generate_deb_tar_attrs,
+ outputs = _download_deb_pkgs.outputs,
+ implementation = _generate_deb_tar,
+)
+
+aggregate_debian_pkgs_ = rule(
+ attrs = aggregate_debian_pkgs_attrs,
+ outputs = _download_deb_pkgs.outputs,
+ implementation = _aggregate_debian_pkgs_impl,
+)
+
+def aggregate_debian_pkgs(**kwargs):
+ """Aggregate debian packages from multiple language_tool_layers into a tarball.
+
+ Args:
+ name: a unique name for this rule.
+ base: base os image used for this rule.
+ language_layers: a list of language_tool_layer targets.
+
+ Only name, base, and language_layers attributes are allowed in this rule.
+
+ Experimental rule.
+ """
+
+ _input_validation(kwargs)
+
+ aggregate_debian_pkgs_(**kwargs)
diff --git a/container/rules/docker_toolchains.bzl b/container/rules/docker_toolchains.bzl
index 2fba04e..b6e8964 100644
--- a/container/rules/docker_toolchains.bzl
+++ b/container/rules/docker_toolchains.bzl
@@ -18,6 +18,7 @@
load("@base_images_docker//package_managers:download_pkgs.bzl", _download = "download")
load("@base_images_docker//package_managers:install_pkgs.bzl", _install = "install")
load("@base_images_docker//package_managers:apt_key.bzl", _key = "key")
+load(":debian_pkg_tar.bzl", _generate_deb_tar = "generate")
def _input_validation(kwargs):
if "debs" in kwargs:
@@ -91,45 +92,19 @@
# debs to installables_tars.
elif packages != []:
- # Prepare base image for the download_pkgs rule.
- download_base = ctx.files.base[0]
+ download_pkgs_output_tar = ctx.attr.name + "-download_pkgs_output_tar.tar"
+ download_pkgs_output_script = ctx.attr.name + "-download_pkgs_output_script.sh"
- # Create an intermediate image with additional gpg keys used to download packages.
- if keys != []:
- image_with_keys = "%s_with_keys" % ctx.attr.name
-
- # Declare intermediate output file generated by add_apt_key rule.
- image_with_keys_output_executable = ctx.actions.declare_file(image_with_keys)
- image_with_keys_output_tarball = ctx.actions.declare_file(image_with_keys + ".tar")
- image_with_keys_output_layer = ctx.actions.declare_file(image_with_keys + "-layer.tar")
-
- _key.implementation(
- ctx,
- name = image_with_keys,
- image_tar = ctx.files.base[0],
- keys = keys,
- output_executable = image_with_keys_output_executable,
- output_tarball = image_with_keys_output_tarball,
- output_layer = image_with_keys_output_layer,
- )
- download_base = image_with_keys_output_tarball
-
- # Declare intermediate output file generated by download_pkgs rule.
- download_pkgs_output_executable = ctx.actions.declare_file(ctx.attr.name + "-download_pkgs_output_executable.sh")
- download_pkgs_output_tar = ctx.actions.declare_file(ctx.attr.name + "-download_pkgs_output_tar.tar")
- download_pkgs_output_script = ctx.actions.declare_file(ctx.attr.name + "-download_pkgs_output_script.sh")
-
- # download_pkgs rule consumes 'packages' and 'additional_repos'.
- _download.implementation(
+ aggregated_debian_tar = _generate_deb_tar.implementation(
ctx,
- image_tar = download_base,
packages = packages,
additional_repos = additional_repos,
- output_executable = download_pkgs_output_executable,
- output_tar = download_pkgs_output_tar,
- output_script = download_pkgs_output_script,
+ keys = keys,
+ download_pkgs_output_tar = download_pkgs_output_tar,
+ download_pkgs_output_script = download_pkgs_output_script,
)
- installables_tars.append(download_pkgs_output_tar)
+
+ installables_tars.append(aggregated_debian_tar.providers[0].installables_tar)
# Prepare new base image for the container_image rule.
new_base = ctx.files.base[0]
diff --git a/container/ubuntu16_04/debian_pkgs/BUILD b/container/ubuntu16_04/debian_pkgs/BUILD
new file mode 100644
index 0000000..5ae66cc
--- /dev/null
+++ b/container/ubuntu16_04/debian_pkgs/BUILD
@@ -0,0 +1,35 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"]) # Apache 2.0
+
+package(default_visibility = ["//visibility:public"])
+
+load(
+ "//container/rules:debian_pkg_tar.bzl",
+ "aggregate_debian_pkgs",
+)
+
+aggregate_debian_pkgs(
+ name = "ubuntu16-04-debian_pkgs",
+ base = "@ubuntu16_04//image",
+ language_layers = [
+ "//container/ubuntu16_04/builders/rbe-ubuntu16_04:base-ltl",
+ "//container/ubuntu16_04/layers/clang:clang-ltl",
+ "//container/ubuntu16_04/layers/go:go-ltl",
+ "//container/ubuntu16_04/layers/java:java-ltl",
+ "//container/ubuntu16_04/layers/java:java10-ltl",
+ "//container/ubuntu16_04/layers/python:python-ltl",
+ ],
+)
