rules/docker_config.bzl

# Copyright 2017 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Rules for generating toolchain configs for a Docker container.

Exposes the docker_autoconfigure rule that does the following:
- Receive a base container as main input. Base container could have a desired
  set of toolchains (i.e., a C compiler, C libraries, java, python, zip, and
  other tools) installed.
- Optionally, install more debian packages in the base container (any packages
  that might be needed by Bazel not installed in your container).
- Optionally, install a given Bazel version on the container.
- Extend the container to install sources for a project.
- Run a bazel command to build one or more targets from
  remote repositories, inside the container.
- Copy toolchain configs (outputs of remote repo targets) produced
  from the execution of Bazel inside the container to the host.
- Optionally copy outputs to a folder defined via build variables.

Example:

  docker_toolchain_autoconfig(
      name = "my-autoconfig-rule",
      base = "@my_image//image:image.tar",
      bazel_version = "0.10.0",
      config_repos = ["local_config_cc", "<some_other_skylark_repo>"],
      git_repo = "https://github.com/some_git_repo",
      env = {
          ... Dictionary of env variables to configure Bazel properly
              for the container, see environments.bzl for examples.
      },
      packages = [
          "package_1",
          "package_2=version",
      ],
      # Any additional debian repos and keys needed to install packages above,
      # not needed if no packages are installed.
      additional_repos = [
          "deb http://deb.debian.org/debian jessie-backports main",
      ],
      keys = [
          "@some_gpg//file",
      ],
  )

Add to your WORKSPACE file the following:

  http_archive(
    name = "bazel_toolchains",
    urls = [
      "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/<latest_release>.tar.gz",
      "https://github.com/bazelbuild/bazel-toolchains/archive/<latest_release>.tar.gz",
    ],
    strip_prefix = "bazel-toolchains-<latest_commit>",
    sha256 = "<sha256>",
  )

  load(
    "@bazel_toolchains//repositories:repositories.bzl",
    bazel_toolchains_repositories = "repositories",
  )

  bazel_toolchains_repositories()

  load(
      "@io_bazel_rules_docker//container:container.bzl",
      container_repositories = "repositories",
      "container_pull",
  )

  container_repositories()

  # Pulls the my_image used as base for example above
  container_pull(
      name = "my_image",
      digest = "sha256:<sha256>",
      registry = "<registry>",
      repository = "<repo>",
  )

  # GPG file used by example above
  http_file(
    name = "some_gpg",
    sha256 = "<sha256>",
    url = "<URL>",
  )

For values of <latest_release> and other placeholders above, please see
the WORKSPACE file in this repo.

To use the rule run:

  bazel run --define=DOCKER_AUTOCONF_OUTPUT=<some_output_dir> //<location_of_rule>:my-autoconfig-rule

Once rule finishes running the file <some_output_dir>/my-autoconfig-rule.tar
will be created with all toolchain configs generated by
"local_config_cc" and "<some_other_skylark_repo>". If no value for
DOCKER_AUTOCONF_OUTPUT is passed, the resulting tar file is left in /tmp.

Known issues:

 - 'name' of rule must conform to docker image naming standards
 - Rule cannot be placed in the BUILD file at the root of a project
"""

load(
    "@io_bazel_rules_docker//container:container.bzl",
    _container = "container",
)
load("//container/rules:docker_toolchains.bzl", "toolchain_container")

# External folder is set to be deprecated, lets keep it here for easy
# refactoring
# https://github.com/bazelbuild/bazel/issues/1262
_EXTERNAL_FOLDER_PREFIX = "external/"

# Name of the current workspace
_WORKSPACE_NAME = "bazel_toolchains"

_WORKSPACE_PREFIX = "@" + _WORKSPACE_NAME + "//"

# Default cc project to use if no git_repo is provided.
_DEFAULT_AUTOCONFIG_PROJECT_PKG_TAR = _WORKSPACE_PREFIX + "rules:cc-sample-project-tar"

# Build variable to define the location of the output tar produced by
# docker_toolchain_autoconfig
_DOCKER_AUTOCONF_OUTPUT = "DOCKER_AUTOCONF_OUTPUT"

# Filetype to restrict inputs
tar_filetype = [
    ".tar",
    ".tar.xz",
]

def _docker_toolchain_autoconfig_impl(ctx):
    """Implementation for the docker_toolchain_autoconfig rule.

    Args:
      ctx: context. See docker_toolchain_autoconfig below for details
          of what this ctx must include
    Returns:
      null
    """
    bazel_config_dir = "/bazel-config"
    project_repo_dir = "project_src"

    # Command to retrieve the project from github if requested.
    clone_repo_cmd = "cd ."
    if ctx.attr.git_repo:
        clone_repo_cmd = ("cd " + bazel_config_dir + " && git clone " +
                          ctx.attr.git_repo + " " + project_repo_dir)

    # Command to install custom Bazel version (if requested)
    install_bazel_cmd = "cd ."
    if ctx.attr.use_bazel_head:
        # If use_bazel_head was requested, we clone the source code from github and compile
        # it using the release version with "bazel build //src:bazel".
        install_bazel_cmd = "/install_bazel_head.sh"
    elif ctx.attr.bazel_version:
        # If a specific Bazel and Bazel RC version is specified, install that version.
        bazel_url = "https://releases.bazel.build/" + ctx.attr.bazel_version
        if ctx.attr.bazel_rc_version:
            bazel_url += ("/rc" + ctx.attr.bazel_rc_version +
                          "/bazel-" + ctx.attr.bazel_version + "rc" +
                          ctx.attr.bazel_rc_version)
        else:
            bazel_url += "/release/bazel-" + ctx.attr.bazel_version
        bazel_url += "-installer-linux-x86_64.sh"
        install_bazel_cmd = "/install_bazel_version.sh " + bazel_url

    # Command to recursively convert soft links to hard links in the config_repos
    deref_symlinks_cmd = []
    for config_repo in ctx.attr.config_repos:
        symlinks_cmd = ("find $(bazel info output_base)/" +
                        _EXTERNAL_FOLDER_PREFIX + config_repo +
                        " -type l -exec bash -c 'ln -f \"$(readlink -m \"$0\")\" \"$0\"' {} \;")
        deref_symlinks_cmd.append(symlinks_cmd)
    deref_symlinks_cmd = " && ".join(deref_symlinks_cmd)

    # Command to copy produced toolchain configs to outside of
    # the containter (to the mount_point).
    copy_cmd = []
    for config_repo in ctx.attr.config_repos:
        src_dir = "$(bazel info output_base)/" + _EXTERNAL_FOLDER_PREFIX + config_repo
        copy_cmd.append("cp -dr " + src_dir + " " + bazel_config_dir + "/")

        # We need to change the owner of the files we copied, so that they can
        # be manipulated from outside the container.
        copy_cmd.append("chown -R $USER_ID " + bazel_config_dir + "/" + config_repo)
    output_copy_cmd = " && ".join(copy_cmd)

    # Command to run autoconfigure targets.
    bazel_cmd = "cd " + bazel_config_dir + "/" + project_repo_dir
    if ctx.attr.use_default_project:
        bazel_cmd += " && touch WORKSPACE && mv BUILD.sample BUILD"

    # For each config repo we run the target @<config_repo>//...
    bazel_targets = "@" + "//... @".join(ctx.attr.config_repos) + "//..."
    bazel_flags = ""
    if not ctx.attr.incompatible_changes_off:
        bazel_flags += " --all_incompatible_changes"
    bazel_cmd += " && bazel build " + bazel_flags + " " + bazel_targets

    # Command to run to clean up after autoconfiguration.
    # we start with "cd ." to make sure in case of failure everything after the
    # ";" will be executed
    clean_cmd = "cd . ; bazel clean"
    if ctx.attr.use_default_project:
        clean_cmd += " && rm WORKSPACE"
    if ctx.attr.git_repo:
        clean_cmd += " && cd " + bazel_config_dir + " && rm -drf " + project_repo_dir

    # Full command to use for docker container
    # TODO(xingao): Make sure the command exits with error right away if a sub
    # command fails.
    docker_cmd = [
        "/bin/sh",
        "-c",
        " && ".join([
            "set -ex",
            ctx.attr.setup_cmd,
            install_bazel_cmd,
            "echo === Cloning project repo ===",
            clone_repo_cmd,
            "echo === Running Bazel autoconfigure command ===",
            bazel_cmd,
            "echo === Copying outputs ===",
            deref_symlinks_cmd,
            output_copy_cmd,
            "echo === Cleaning up ===",
            clean_cmd,
        ]),
    ]

    # Expand contents of repo_pkg_tar
    # (and remove them after we're done running the docker command).
    # A dummy command that does nothing in case git_repo was used
    expand_repo_cmd = "cd ."
    remove_repo_cmd = "cd ."
    if ctx.attr.repo_pkg_tar:
        repo_pkg_tar = str(ctx.attr.repo_pkg_tar.label.name)
        package_name = str(ctx.attr.repo_pkg_tar.label.package)

        # Expand the tar file pointed by repo_pkg_tar
        expand_repo_cmd = ("mkdir ./%s ; tar -xf %s/%s.tar -C ./%s" %
                           (project_repo_dir, package_name, repo_pkg_tar, project_repo_dir))
        remove_repo_cmd = ("rm -drf ./%s" % project_repo_dir)

    result = _container.image.implementation(ctx, cmd = docker_cmd, output_executable = ctx.outputs.load_image)

    # By default we copy the produced tar file to /tmp/
    output_location = "/tmp/" + ctx.attr.name + ".tar"
    if _DOCKER_AUTOCONF_OUTPUT in ctx.var:
        output_location = ctx.var[_DOCKER_AUTOCONF_OUTPUT] + "/" + ctx.attr.name + ".tar"

    # Create the script to load image and run it
    ctx.actions.expand_template(
        template = ctx.files.run_tpl[0],
        substitutions = {
            "%{EXPAND_REPO_CMD}": expand_repo_cmd,
            "%{LOAD_IMAGE_SH}": ctx.outputs.load_image.short_path,
            "%{IMAGE_NAME}": "bazel/" + ctx.label.package + ":" + ctx.label.name,
            "%{RM_REPO_CMD}": remove_repo_cmd,
            "%{CONFIG_REPOS}": " ".join(ctx.attr.config_repos),
            "%{OUTPUT}": output_location,
        },
        output = ctx.outputs.executable,
        is_executable = True,
    )

    # add to the runfiles the script to load image and (if needed) the repo_pkg_tar file
    if hasattr(result.providers[1], "runfiles"):
        result_runfiles = result.providers[1].runfiles
    else:
        result_runfiles = result.providers[1].default_runfiles
    runfiles_list = result_runfiles.files.to_list() + [ctx.outputs.load_image]
    if ctx.attr.repo_pkg_tar:
        runfiles_list += ctx.files.repo_pkg_tar

    runfiles = ctx.runfiles(files = runfiles_list)

    return struct(
        runfiles = runfiles,
        files = result.providers[1].files,
        container_parts = result.container_parts,
    )

docker_toolchain_autoconfig_ = rule(
    attrs = _container.image.attrs + {
        "config_repos": attr.string_list(default = ["local_config_cc"]),
        "use_default_project": attr.bool(default = False),
        "git_repo": attr.string(),
        "repo_pkg_tar": attr.label(allow_files = tar_filetype),
        "bazel_version": attr.string(),
        "bazel_rc_version": attr.string(),
        "use_bazel_head": attr.bool(default = False),
        "run_tpl": attr.label(allow_files = True),
        "setup_cmd": attr.string(default = "cd ."),
        "packages": attr.string_list(),
        "additional_repos": attr.string_list(),
        "keys": attr.string_list(),
        "incompatible_changes_off": attr.bool(default = False),
        "test": attr.bool(default = True),
    },
    executable = True,
    outputs = _container.image.outputs + {
        "load_image": "%{name}_load_image.sh",
    },
    implementation = _docker_toolchain_autoconfig_impl,
)

# Attributes below are expected in ctx, but should not be provided
# in the BUILD file.
reserved_attrs = [
    "use_default_project",
    "files",
    "debs",
    "repo_pkg_tar",
    "run_tpl",
    # all the attrs from docker_build we dont want users to set
    "directory",
    "tars",
    "legacy_repository_naming",
    "legacy_run_behavior",
    "docker_run_flags",
    "mode",
    "symlinks",
    "entrypoint",
    "cmd",
    "user",
    "labels",
    "ports",
    "volumes",
    "workdir",
    "repository",
    "label_files",
    "label_file_strings",
    "empty_files",
    "build_layer",
    "create_image_config",
    "sha256",
    "incremental_load_template",
    "join_layers",
    "extract_config",
]

# Attrs expected in the BUILD rule
required_attrs = [
    "base",
]

def docker_toolchain_autoconfig(**kwargs):
    """Generate toolchain configs for a docker container.

    This rule produces a tar file with toolchain configs produced from the
    execution of targets in skylark remote repositories. Typically, this rule is
    used to produce toolchain configs for the local_config_cc repository.
    This repo (as well as others, depending on the project) contains generated
    toolchain configs that Bazel uses to properly use a toolchain. For instance,
    the local_config_cc repo generates a cc_toolchain rule.

    The toolchain configs that this rule produces, can be used to, for
    instance, use a remote execution service that runs actions inside docker
    containers.

    All the toolchain configs published in the bazel-toolchains
    repo (https://github.com/bazelbuild/bazel-toolchains/) have been produced
    using this rule.

    This rule is implemented by extending the container_image rule in
    https://github.com/bazelbuild/rules_docker. The rule installs debs packages
    to run bazel (using the package manager rules offered by
    https://github.com/GoogleCloudPlatform/distroless/tree/master/package_manager).
    The rule creates the container with a command that pulls a repo from github,
    and runs bazel build for a series of remote repos. Files generated in these
    repos are copied to a mount point inside the Bazel output tree, and finally
    copied to the /tmp directory or to the DOCKER_AUTOCONF_OUTPUT directory
    if passed as build variable.

    Args:
      **kwargs:
    Required Args
      name: A unique name for this rule.
      base: Docker image base - optionally with all tools pre-installed for
          which a configuration will be generated. Packages can also be installed
          by listing them in the 'packages' attriute.
    Default Args:
      config_repos: a list of remote repositories. Autoconfig will run targets in
          each of these remote repositories and copy all contents to the mount
          point.
      env: Dictionary of env variables for Bazel / project specific autoconfigure
      git_repo: A git repo with the sources for the project to be used for
          autoconfigure. If no git_repo is passed, autoconfig will run with a
          sample c++ project.
      bazel_version: a specific version of Bazel used to generate toolchain
          configs. Format: x.x.x
      bazel_rc_version: a specific version of Bazel release candidate used to
          generate toolchain configs. Input "2" if you would like to use rc2.
      use_bazel_head = Download bazel head from github, compile it and use it
          to run autoconfigure targets.
      setup_cmd: a customized command that will run as the very first command
          inside the docker container.
      packages: list of packages to fetch and install in the base image.
      additional_repos: list of additional debian package repos to use,
          in sources.list format.
      keys: list of additional gpg keys to use while downloading packages.
      incompatible_changes_off: If True Bazel will run without the
          all_incompatible_changes flag. Default False.
      test: a boolean which specifies whether a test target for this
          docker_toolchain_autoconfig will be added.
          If True, a test target with name {name}_test will be added.
          The test will build this docker_toolchain_autoconfig target, run the
          output script, and check the toolchain configs for the c++ auto
          generated config exist.
    """
    for reserved in reserved_attrs:
        if reserved in kwargs:
            fail("reserved for internal use by docker_toolchain_autoconfig macro", attr = reserved)

    for required in required_attrs:
        if required not in kwargs:
            fail("required for docker_toolchain_autoconfig", attr = required)

    # Input validations
    if "use_bazel_head" in kwargs and ("bazel_version" in kwargs or "bazel_rc_version" in kwargs):
        fail("Only one of use_bazel_head or a combination of bazel_version and" +
             "bazel_rc_version can be set at a time.")

    packages_is_empty = "packages" not in kwargs or kwargs["packages"] == []

    if packages_is_empty and "additional_repos" in kwargs:
        fail("'additional_repos' can only be specified when 'packages' is not empty.")
    if packages_is_empty and "keys" in kwargs:
        fail("'keys' can only be specified when 'packages' is not empty.")

    # If the git_repo was not provided, use the default autoconfig project
    if "git_repo" not in kwargs:
        kwargs["repo_pkg_tar"] = _DEFAULT_AUTOCONFIG_PROJECT_PKG_TAR
        kwargs["use_default_project"] = True
    kwargs["files"] = [
        _WORKSPACE_PREFIX + "rules:install_bazel_head.sh",
        _WORKSPACE_PREFIX + "rules:install_bazel_version.sh",
    ]

    # The template for the main script to execute for this rule, which produces
    # the toolchain configs
    kwargs["run_tpl"] = _WORKSPACE_PREFIX + "rules:docker_config.sh.tpl"

    # Do not install packags if 'packages' is not specified or is an ampty list.
    if not packages_is_empty:
        # "additional_repos" and "keys" are optional for docker_toolchain_autoconfig,
        # but required for toolchain_container". Use empty lists as placeholder.
        if "additional_repos" not in kwargs:
            kwargs["additional_repos"] = []
        if "keys" not in kwargs:
            kwargs["keys"] = []

        # Install packages in the base image.
        toolchain_container(
            name = kwargs["name"] + "_image",
            base = kwargs["base"],
            packages = kwargs["packages"],
            additional_repos = kwargs["additional_repos"],
            keys = kwargs["keys"],
        )

        # Use the image with packages installed as the new base for autoconfiguring.
        kwargs["base"] = ":" + kwargs["name"] + "_image.tar"

    if "test" in kwargs and kwargs["test"] == True:
        # Create a test target for the current docker_toolchain_autoconfig target,
        # which builds this docker_toolchain_autoconfig target, runs the output
        # script, and checks the toolchain configs for the c++ auto generated config
        # exist.
        native.sh_test(
            name = kwargs["name"] + "_test",
            size = "medium",
            timeout = "long",
            srcs = ["//test/configs:autoconfig_test.sh"],
            data = [":" + kwargs["name"]],
        )

    docker_toolchain_autoconfig_(**kwargs)