Add modules to create and download debian packages (#154) * Add modules to get debian packages 1. Extract a new rule to download debian packages and create a tarball 2. Create a new file for the new rule aggregate_debian_pkgs 3. Move the use of new rule into language_tool_layer
diff --git a/container/rules/debian_pkg_tar.bzl b/container/rules/debian_pkg_tar.bzl new file mode 100644 index 0000000..d76fb38 --- /dev/null +++ b/container/rules/debian_pkg_tar.bzl
@@ -0,0 +1,188 @@ +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@base_images_docker//package_managers:download_pkgs.bzl", _download_deb_pkgs = "download") +load("@base_images_docker//package_managers:apt_key.bzl", _apt_key = "key") + +def _input_validation(kwargs): + allowed_attribues = ["name", "base", "language_layers"] + + for key in kwargs.keys(): + if key not in allowed_attribues: + fail("Attribute " + key + " is not supported.") + +container = [ + ".tar.gz", + ".tgz", + ".tar", + ".tar.xz", +] + +generate_deb_tar_attrs = _download_deb_pkgs.attrs + { + "base": attr.label(allow_files = container), + "packages": attr.string_list(), + "keys": attr.label_list( + allow_files = True, + ), +} + +aggregate_debian_pkgs_attrs = { + "base": attr.label(allow_files = container), + "language_layers": attr.label_list(), + + # Declare the following attributes since _download_deb_pkgs.implementation + # need access those attribute if their overrides are None + "additional_repos": attr.string_list(), + "_image_id_extractor": attr.label( + default = "@io_bazel_rules_docker//contrib:extract_image_id.py", + allow_files = True, + single_file = True, + ), +} + +InstallableTarInfo = provider(fields = [ + "installables_tar", +]) + +def _generate_deb_tar( + ctx, + packages = None, + additional_repos = None, + keys = None, + download_pkgs_output_tar = None, + download_pkgs_output_script = None): + """A function for producing a tarball for a set of debian packages. + + Args: + ctx: ctx has either generate_deb_tar_attrs or aggregate_debian_pkgs_attrs, + this depends on the rule that uses this function + packages: list of packages to fetch and install in the base image. + additional_repos: list of additional debian package repos to use, + in sources.list format. + keys: list of labels of additional gpg keys to use while downloading + packages. + download_pkgs_output_tar: output tar file generated by download_pkgs rule to + override default output_tar + download_pkgs_output_script: output script generated by download_pkgs rule to + override default output_script + """ + + # Prepare base image for the download_pkgs rule. + download_base = ctx.files.base[0] + + # Create an intermediate image with additional gpg keys used to download packages. + if keys != []: + image_with_keys = "%s_with_keys" % ctx.attr.name + + # Declare intermediate output file generated by add_apt_key rule. + image_with_keys_output_executable = ctx.actions.declare_file(image_with_keys) + image_with_keys_output_tarball = ctx.actions.declare_file(image_with_keys + ".tar") + image_with_keys_output_layer = ctx.actions.declare_file(image_with_keys + "-layer.tar") + + _apt_key.implementation( + ctx, + name = image_with_keys, + image_tar = ctx.files.base[0], + keys = keys, + output_executable = image_with_keys_output_executable, + output_tarball = image_with_keys_output_tarball, + output_layer = image_with_keys_output_layer, + ) + download_base = image_with_keys_output_tarball + + # Declare intermediate output file generated by download_pkgs rule. + output_executable = ctx.actions.declare_file(ctx.attr.name + "-output_executable.sh") + download_pkgs_output_tar = download_pkgs_output_tar or ctx.attr.name + ".tar" + download_pkgs_output_script = download_pkgs_output_script or ctx.attr.name + ".sh" + output_tar = ctx.actions.declare_file(download_pkgs_output_tar) + output_script = ctx.actions.declare_file(download_pkgs_output_script) + + # download_pkgs rule consumes 'packages' and 'additional_repos'. + _download_deb_pkgs.implementation( + ctx, + image_tar = download_base, + packages = packages, + additional_repos = additional_repos, + output_executable = output_executable, + output_tar = output_tar, + output_script = output_script, + ) + + return struct( + providers = [ + InstallableTarInfo( + installables_tar = output_tar, + ), + ], + ) + +def _aggregate_debian_pkgs_impl(ctx): + """Implementation for the aggregate_debian_pkgs rule. + + aggregate_debian_pkgs rule produces a tarball with all debian packages declared + in the language_tool_layer(s) this rule depends on. + + Args: + ctx: ctx only has name, base, and language_layers attributes + """ + + packages = [] + additional_repos = [] + keys = [] + + for layer in ctx.attr.language_layers: + packages.extend(layer.packages) + additional_repos.extend(layer.additional_repos) + keys.extend(layer.keys) + + packages = depset(packages).to_list() + additional_repos = depset(additional_repos).to_list() + keys = depset(keys).to_list() + + return _generate_deb_tar( + ctx, + packages = packages, + additional_repos = additional_repos, + keys = keys, + ) + +# Export _generate_deb_tar function for other bazel rules to use. +generate = struct( + attrs = generate_deb_tar_attrs, + outputs = _download_deb_pkgs.outputs, + implementation = _generate_deb_tar, +) + +aggregate_debian_pkgs_ = rule( + attrs = aggregate_debian_pkgs_attrs, + outputs = _download_deb_pkgs.outputs, + implementation = _aggregate_debian_pkgs_impl, +) + +def aggregate_debian_pkgs(**kwargs): + """Aggregate debian packages from multiple language_tool_layers into a tarball. + + Args: + name: a unique name for this rule. + base: base os image used for this rule. + language_layers: a list of language_tool_layer targets. + + Only name, base, and language_layers attributes are allowed in this rule. + + Experimental rule. + """ + + _input_validation(kwargs) + + aggregate_debian_pkgs_(**kwargs)
diff --git a/container/rules/docker_toolchains.bzl b/container/rules/docker_toolchains.bzl index 2fba04e..b6e8964 100644 --- a/container/rules/docker_toolchains.bzl +++ b/container/rules/docker_toolchains.bzl
@@ -18,6 +18,7 @@ load("@base_images_docker//package_managers:download_pkgs.bzl", _download = "download") load("@base_images_docker//package_managers:install_pkgs.bzl", _install = "install") load("@base_images_docker//package_managers:apt_key.bzl", _key = "key") +load(":debian_pkg_tar.bzl", _generate_deb_tar = "generate") def _input_validation(kwargs): if "debs" in kwargs: @@ -91,45 +92,19 @@ # debs to installables_tars. elif packages != []: - # Prepare base image for the download_pkgs rule. - download_base = ctx.files.base[0] + download_pkgs_output_tar = ctx.attr.name + "-download_pkgs_output_tar.tar" + download_pkgs_output_script = ctx.attr.name + "-download_pkgs_output_script.sh" - # Create an intermediate image with additional gpg keys used to download packages. - if keys != []: - image_with_keys = "%s_with_keys" % ctx.attr.name - - # Declare intermediate output file generated by add_apt_key rule. - image_with_keys_output_executable = ctx.actions.declare_file(image_with_keys) - image_with_keys_output_tarball = ctx.actions.declare_file(image_with_keys + ".tar") - image_with_keys_output_layer = ctx.actions.declare_file(image_with_keys + "-layer.tar") - - _key.implementation( - ctx, - name = image_with_keys, - image_tar = ctx.files.base[0], - keys = keys, - output_executable = image_with_keys_output_executable, - output_tarball = image_with_keys_output_tarball, - output_layer = image_with_keys_output_layer, - ) - download_base = image_with_keys_output_tarball - - # Declare intermediate output file generated by download_pkgs rule. - download_pkgs_output_executable = ctx.actions.declare_file(ctx.attr.name + "-download_pkgs_output_executable.sh") - download_pkgs_output_tar = ctx.actions.declare_file(ctx.attr.name + "-download_pkgs_output_tar.tar") - download_pkgs_output_script = ctx.actions.declare_file(ctx.attr.name + "-download_pkgs_output_script.sh") - - # download_pkgs rule consumes 'packages' and 'additional_repos'. - _download.implementation( + aggregated_debian_tar = _generate_deb_tar.implementation( ctx, - image_tar = download_base, packages = packages, additional_repos = additional_repos, - output_executable = download_pkgs_output_executable, - output_tar = download_pkgs_output_tar, - output_script = download_pkgs_output_script, + keys = keys, + download_pkgs_output_tar = download_pkgs_output_tar, + download_pkgs_output_script = download_pkgs_output_script, ) - installables_tars.append(download_pkgs_output_tar) + + installables_tars.append(aggregated_debian_tar.providers[0].installables_tar) # Prepare new base image for the container_image rule. new_base = ctx.files.base[0]
diff --git a/container/ubuntu16_04/debian_pkgs/BUILD b/container/ubuntu16_04/debian_pkgs/BUILD new file mode 100644 index 0000000..5ae66cc --- /dev/null +++ b/container/ubuntu16_04/debian_pkgs/BUILD
@@ -0,0 +1,35 @@ +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//visibility:public"]) + +load( + "//container/rules:debian_pkg_tar.bzl", + "aggregate_debian_pkgs", +) + +aggregate_debian_pkgs( + name = "ubuntu16-04-debian_pkgs", + base = "@ubuntu16_04//image", + language_layers = [ + "//container/ubuntu16_04/builders/rbe-ubuntu16_04:base-ltl", + "//container/ubuntu16_04/layers/clang:clang-ltl", + "//container/ubuntu16_04/layers/go:go-ltl", + "//container/ubuntu16_04/layers/java:java-ltl", + "//container/ubuntu16_04/layers/java:java10-ltl", + "//container/ubuntu16_04/layers/python:python-ltl", + ], +)