blob: 85d272c60125253d19309faef7ad080d91514a03 [file] [log] [blame]
# Copyright 2017 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Rules for generating toolchain configs for a Docker container.
Exposes the docker_autoconfigure rule that does the following:
- Receive a base container as main input. Base container could have a desired
set of toolchains (i.e., a C compiler, C libraries, java, python, zip, and
other tools) installed.
- Optionally, install more debian packages in the base container (any packages
that might be needed by Bazel not installed in your container).
- Optionally, install a given Bazel version on the container.
- Extend the container to install sources for a project.
- Run a bazel command to build one or more targets from
remote repositories, inside the container.
- Copy toolchain configs (outputs of remote repo targets) produced
from the execution of Bazel inside the container to the host.
Example:
docker_toolchain_autoconfig(
name = "my-autoconfig-rule",
base = "@my_image//image:image.tar",
bazel_version = "0.10.0",
config_repos = ["local_config_cc", "<some_other_skylark_repo>"],
git_repo = "https://github.com/some_git_repo",
env = {
... Dictionary of env variables to configure Bazel properly
for the container, see environments.bzl for examples.
},
packages = [
"package_1",
"package_2=version",
],
# Any additional debian repos and keys needed to install packages above,
# not needed if no packages are installed.
additional_repos = [
"deb http://deb.debian.org/debian jessie-backports main",
],
keys = [
"@some_gpg//file",
],
)
Add to your WORKSPACE file the following:
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
http_archive(
name = "bazel_toolchains",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/<latest_release>.tar.gz",
"https://github.com/bazelbuild/bazel-toolchains/archive/<latest_release>.tar.gz",
],
strip_prefix = "bazel-toolchains-<latest_commit>",
sha256 = "<sha256>",
)
load(
"@bazel_toolchains//repositories:repositories.bzl",
bazel_toolchains_repositories = "repositories",
)
bazel_toolchains_repositories()
load(
"@io_bazel_rules_docker//repositories:repositories.bzl",
container_repositories = "repositories",
)
container_repositories()
load(
"@io_bazel_rules_docker//container:container.bzl",
"container_pull",
)
# Pulls the my_image used as base for example above
container_pull(
name = "my_image",
digest = "sha256:<sha256>",
registry = "<registry>",
repository = "<repo>",
)
# GPG file used by example above
http_file(
name = "some_gpg",
sha256 = "<sha256>",
url = "<URL>",
)
For values of <latest_release> and other placeholders above, please see
the WORKSPACE file in this repo.
To use the rule run:
bazel build //<location_of_rule>:my-autoconfig-rule
Once rule finishes running the file my-autoconfig-rule_output.tar
will be created with all toolchain configs generated by
"local_config_cc" and "<some_other_skylark_repo>".
Known issues:
- 'name' of rule must conform to docker image naming standards
- Rule cannot be placed in the BUILD file at the root of a project
"""
load(
"@io_bazel_rules_docker//container:container.bzl",
_container = "container",
)
load("@base_images_docker//util:run.bzl", _extract = "extract")
load("@bazel_toolchains//rules/container:docker_toolchains.bzl", "toolchain_container")
load("@bazel_skylib//lib:dicts.bzl", "dicts")
# External folder is set to be deprecated, lets keep it here for easy
# refactoring
# https://github.com/bazelbuild/bazel/issues/1262
_EXTERNAL_FOLDER_PREFIX = "external/"
# Name of the current workspace
_WORKSPACE_NAME = "bazel_toolchains"
_WORKSPACE_PREFIX = "@" + _WORKSPACE_NAME + "//"
# Default cc project to use if no git_repo is provided.
_DEFAULT_AUTOCONFIG_PROJECT_PKG_TAR = _WORKSPACE_PREFIX + "rules:cc-sample-project-tar"
# Filetype to restrict inputs
tar_filetype = [
".tar",
".tar.xz",
]
def _docker_toolchain_autoconfig_impl(ctx):
"""Implementation for the docker_toolchain_autoconfig rule.
Args:
ctx: context. See docker_toolchain_autoconfig below for details
of what this ctx must include
Returns:
null
"""
bazel_config_dir = "/bazel-config"
project_repo_dir = "project_src"
output_dir = bazel_config_dir + "/autoconf_out"
name = ctx.attr.name
outputs_tar = ctx.outputs.output_tar.basename
# Command to retrieve the project from github if requested.
clone_repo_cmd = "cd ."
if ctx.attr.git_repo:
clone_repo_cmd = ("cd " + bazel_config_dir + " && git clone " +
ctx.attr.git_repo + " " + project_repo_dir)
repo_dir = bazel_config_dir + "/" + project_repo_dir
if ctx.attr.repo_pkg_tar:
# if package tar was used then the command should expand it
clone_repo_cmd = ("mkdir %s && tar -xf /%s -C %s " %
(repo_dir, ctx.file.repo_pkg_tar.basename, repo_dir))
# if mount_project was selected, we'll mount it using docker_run_flags
docker_run_flags = [""]
if ctx.attr.mount_project:
mount_project = ctx.attr.mount_project
mount_project = ctx.expand_make_variables("mount_project", mount_project, {})
target = mount_project + ":" + repo_dir + ":ro"
docker_run_flags = ["-v", target]
# Command to install custom Bazel version (if requested)
install_bazel_cmd = "cd ."
if ctx.attr.use_bazel_head:
# If use_bazel_head was requested, we clone the source code from github and compile
# it using the release version with "bazel build //src:bazel".
install_bazel_cmd = "/install_bazel_head.sh"
elif ctx.attr.bazel_version:
# If a specific Bazel and Bazel RC version is specified, install that version.
bazel_url = "https://releases.bazel.build/" + ctx.attr.bazel_version
if ctx.attr.bazel_rc_version:
bazel_url += ("/rc" + ctx.attr.bazel_rc_version +
"/bazel-" + ctx.attr.bazel_version + "rc" +
ctx.attr.bazel_rc_version)
else:
bazel_url += "/release/bazel-" + ctx.attr.bazel_version
bazel_url += "-installer-linux-x86_64.sh"
install_bazel_cmd = "/install_bazel_version.sh " + bazel_url
# Command to recursively convert soft links to hard links in the config_repos
deref_symlinks_cmd = []
for config_repo in ctx.attr.config_repos:
symlinks_cmd = ("find $(bazel info output_base)/" +
_EXTERNAL_FOLDER_PREFIX + config_repo +
" -type l -exec bash -c 'ln -f \"$(readlink -m \"$0\")\" \"$0\"' {} \;")
deref_symlinks_cmd.append(symlinks_cmd)
deref_symlinks_cmd = " && ".join(deref_symlinks_cmd)
# Command to copy produced toolchain configs to a tar at the root
# of the container.
copy_cmd = ["mkdir " + output_dir]
for config_repo in ctx.attr.config_repos:
src_dir = "$(bazel info output_base)/" + _EXTERNAL_FOLDER_PREFIX + config_repo
copy_cmd.append("cp -dr " + src_dir + " " + output_dir)
copy_cmd.append("tar -cf /" + outputs_tar + " -C " + output_dir + "/ . ")
output_copy_cmd = " && ".join(copy_cmd)
# Command to run autoconfigure targets.
bazel_cmd = "cd " + bazel_config_dir + "/" + project_repo_dir
if ctx.attr.use_default_project:
bazel_cmd += " && touch WORKSPACE && mv BUILD.sample BUILD"
# For each config repo we run the target @<config_repo>//...
bazel_targets = "@" + "//... @".join(ctx.attr.config_repos) + "//..."
bazel_flags = ""
if not ctx.attr.incompatible_changes_off:
bazel_flags += " --all_incompatible_changes"
bazel_cmd += " && bazel build " + bazel_flags + " " + bazel_targets
# Command to run to clean up after autoconfiguration.
# we start with "cd ." to make sure in case of failure everything after the
# ";" will be executed
clean_cmd = "cd . ; bazel clean"
if ctx.attr.use_default_project:
clean_cmd += " && rm WORKSPACE"
if ctx.attr.git_repo:
clean_cmd += " && cd " + bazel_config_dir + " && rm -drf " + project_repo_dir
install_sh = ctx.actions.declare_file(name + "_install.sh")
ctx.actions.write(
output = install_sh,
content = "\n ".join([
"set -ex",
"echo === Starting docker autoconfig ===",
ctx.attr.setup_cmd,
install_bazel_cmd,
"echo === Cloning / expand project repo ===",
clone_repo_cmd,
"echo === Running Bazel autoconfigure command ===",
bazel_cmd,
"echo === Copying outputs ===",
deref_symlinks_cmd,
output_copy_cmd,
"echo === Cleaning up ===",
clean_cmd,
]),
)
# Include the repo_pkg_tar if needed
files = [install_sh] + ctx.files._installers
if ctx.attr.repo_pkg_tar:
files += [ctx.file.repo_pkg_tar]
image_tar = ctx.actions.declare_file(name + ".tar")
# TODO(nlopezgi): fix upstream issue that output_executable is required
load_image_sh_file = ctx.actions.declare_file(name + "load.sh")
_container.image.implementation(
ctx,
files = files,
output_executable = load_image_sh_file,
output_tarball = image_tar,
workdir = bazel_config_dir,
)
# Commands to run script to create autoconf results, output stderr to log file
# add the log file to a tar file and append the output.tar to that same tar file
commands = []
commands += ["/" + ctx.attr.name + "_install.sh 2> /" + ctx.attr.name + ".log"]
commands += ["tar -cf /extract.tar /" + ctx.attr.name + ".log"]
commands += [
("if [ -f /" + outputs_tar + " ]; " +
"then tar -rf /extract.tar /" + outputs_tar + "; fi"),
]
print(("\n== Docker autoconfig will run. ==\n" +
"To debug any errors run:\n" +
"> docker run -it {mount_flags} <image_id> bash\n" +
"Where <image_id> is the image id printed out by the " +
"{name}_extract.tar rule.\n" +
"Then run:\n>/ {run_cmd}\n" +
"from inside the container.").format(
mount_flags = " ".join(docker_run_flags),
name = ctx.attr.name,
run_cmd = install_sh.basename,
))
extract_tar_file = ctx.actions.declare_file(name + "_extract.tar")
_extract.implementation(
ctx,
name = ctx.attr.name + "_extract",
image = image_tar,
docker_run_flags = docker_run_flags,
commands = commands,
extract_file = "/extract.tar",
script_file = ctx.actions.declare_file(ctx.attr.name + ".build"),
output_file = extract_tar_file,
)
# Extracts the two outputs produced by this rule (outputs.tar + log file)
# from the tar file extracted from the container in the rule above
ctx.actions.run_shell(
inputs = [extract_tar_file],
outputs = [ctx.outputs.output_tar, ctx.outputs.log],
command = ("tar -C %s -xf %s" % (ctx.outputs.output_tar.dirname, extract_tar_file.path)),
)
docker_toolchain_autoconfig_ = rule(
attrs = dicts.add(_container.image.attrs, {
"config_repos": attr.string_list(default = ["local_config_cc"]),
"mount_project": attr.string(),
"use_default_project": attr.bool(default = False),
"git_repo": attr.string(),
"repo_pkg_tar": attr.label(allow_single_file = tar_filetype),
"bazel_version": attr.string(),
"bazel_rc_version": attr.string(),
"use_bazel_head": attr.bool(default = False),
"setup_cmd": attr.string(default = "cd ."),
"packages": attr.string_list(),
"additional_repos": attr.string_list(),
"keys": attr.string_list(),
"incompatible_changes_off": attr.bool(default = False),
"test": attr.bool(default = True),
"_installers": attr.label(default = ":bazel_installers", allow_files = True),
# TODO(nlopezgi): fix upstream attr declaration that is missing repo name
"_extract_tpl": attr.label(
default = Label("@base_images_docker//util:extract.sh.tpl"),
allow_single_file = True,
),
"_image_id_extractor": attr.label(
default = "@io_bazel_rules_docker//contrib:extract_image_id.py",
allow_single_file = True,
),
}),
outputs = dicts.add(_container.image.outputs, {
"log": "%{name}.log",
"output_tar": "%{name}_outputs.tar",
}),
toolchains = ["@io_bazel_rules_docker//toolchains/docker:toolchain_type"],
implementation = _docker_toolchain_autoconfig_impl,
)
# Attributes below are expected in ctx, but should not be provided
# in the BUILD file.
reserved_attrs = [
"use_default_project",
"files",
"debs",
"repo_pkg_tar",
# all the attrs from docker_build we dont want users to set
"directory",
"tars",
"legacy_repository_naming",
"legacy_run_behavior",
"docker_run_flags",
"mode",
"symlinks",
"entrypoint",
"cmd",
"user",
"labels",
"ports",
"volumes",
"workdir",
"repository",
"label_files",
"label_file_strings",
"empty_files",
"build_layer",
"create_image_config",
"sha256",
"incremental_load_template",
"join_layers",
"extract_config",
]
# Attrs expected in the BUILD rule
required_attrs = [
"base",
]
def docker_toolchain_autoconfig(**kwargs):
"""Generate toolchain configs for a docker container.
This rule produces a tar file with toolchain configs produced from the
execution of targets in skylark remote repositories. Typically, this rule is
used to produce toolchain configs for the local_config_cc repository.
This repo (as well as others, depending on the project) contains generated
toolchain configs that Bazel uses to properly use a toolchain. For instance,
the local_config_cc repo generates a cc_toolchain rule.
The toolchain configs that this rule produces, can be used to, for
instance, use a remote execution service that runs actions inside docker
containers.
All the toolchain configs published in the bazel-toolchains
repo (https://github.com/bazelbuild/bazel-toolchains/) have been produced
using this rule.
This rule is implemented by extending the container_image rule in
https://github.com/bazelbuild/rules_docker. The rule installs debs packages
to run bazel (using the package manager rules offered by
https://github.com/GoogleContainerTools/base-images-docker).
The rule creates the container with a command that pulls a repo from github,
and runs bazel build for a series of remote repos. Files generated in these
repos are copied to a mount point inside the Bazel output tree.
Args:
**kwargs:
Required Args
name: A unique name for this rule.
base: Docker image base - optionally with all tools pre-installed for
which a configuration will be generated. Packages can also be installed
by listing them in the 'packages' attriute.
Default Args:
config_repos: a list of remote repositories. Autoconfig will run targets in
each of these remote repositories and copy all contents to the mount
point.
env: Dictionary of env variables for Bazel / project specific autoconfigure
git_repo: A git repo with the sources for the project to be used for
autoconfigure. If no git_repo is passed, autoconfig will run with a
sample c++ project.
mount_project: mounts a directory passed in an absolute path as the project
to use for autoconfig. Cannot be used if git_repo is passed.
Make variable substitution is enabled, so use:
mount_project = "$(mount_project)",
and then run:
bazel build <autoconf target> --define mount_project=$(realpath .)
from the root of the project to mount it as the project to use for
autoconfig.
bazel_version: a specific version of Bazel used to generate toolchain
configs. Format: x.x.x
bazel_rc_version: a specific version of Bazel release candidate used to
generate toolchain configs. Input "2" if you would like to use rc2.
use_bazel_head = Download bazel head from github, compile it and use it
to run autoconfigure targets.
setup_cmd: a customized command that will run as the very first command
inside the docker container.
packages: list of packages to fetch and install in the base image.
additional_repos: list of additional debian package repos to use,
in sources.list format.
keys: list of additional gpg keys to use while downloading packages.
incompatible_changes_off: If True Bazel will run without the
all_incompatible_changes flag. Default False.
test: a boolean which specifies whether a test target for this
docker_toolchain_autoconfig will be added.
If True, a test target with name {name}_test will be added.
The test will build this docker_toolchain_autoconfig target, run the
output script, and check the toolchain configs for the c++ auto
generated config exist.
"""
for reserved in reserved_attrs:
if reserved in kwargs:
fail("reserved for internal use by docker_toolchain_autoconfig macro", attr = reserved)
for required in required_attrs:
if required not in kwargs:
fail("required for docker_toolchain_autoconfig", attr = required)
# Input validations
if "use_bazel_head" in kwargs and ("bazel_version" in kwargs or "bazel_rc_version" in kwargs):
fail("Only one of use_bazel_head or a combination of bazel_version and" +
"bazel_rc_version can be set at a time.")
packages_is_empty = "packages" not in kwargs or kwargs["packages"] == []
if packages_is_empty and "additional_repos" in kwargs:
fail("'additional_repos' can only be specified when 'packages' is not empty.")
if packages_is_empty and "keys" in kwargs:
fail("'keys' can only be specified when 'packages' is not empty.")
if "git_repo" in kwargs and "mount_project" in kwargs:
fail("'git_repo' cannot be used with 'mount_project'.")
# If a git_repo or mount_project was not provided
# use the default autoconfig project
if "git_repo" not in kwargs and "mount_project" not in kwargs:
kwargs["repo_pkg_tar"] = _DEFAULT_AUTOCONFIG_PROJECT_PKG_TAR
kwargs["use_default_project"] = True
kwargs["files"] = [
_WORKSPACE_PREFIX + "rules:install_bazel_head.sh",
_WORKSPACE_PREFIX + "rules:install_bazel_version.sh",
]
# Do not install packags if 'packages' is not specified or is an empty list.
if not packages_is_empty:
# "additional_repos" and "keys" are optional for docker_toolchain_autoconfig,
# but required for toolchain_container". Use empty lists as placeholder.
if "additional_repos" not in kwargs:
kwargs["additional_repos"] = []
if "keys" not in kwargs:
kwargs["keys"] = []
# Install packages in the base image.
toolchain_container(
name = kwargs["name"] + "_image",
base = kwargs["base"],
packages = kwargs["packages"],
additional_repos = kwargs["additional_repos"],
keys = kwargs["keys"],
)
# Use the image with packages installed as the new base for autoconfiguring.
kwargs["base"] = ":" + kwargs["name"] + "_image.tar"
if "test" in kwargs and kwargs["test"] == True:
# Create a test target for the current docker_toolchain_autoconfig target,
# which builds this docker_toolchain_autoconfig target, runs the output
# script, and checks the toolchain configs for the c++ auto generated config
# exist.
native.sh_test(
name = kwargs["name"] + "_test",
size = "medium",
timeout = "long",
srcs = ["@bazel_toolchains//tests/config:autoconfig_test.sh"],
data = [":" + kwargs["name"] + "_outputs.tar"],
)
docker_toolchain_autoconfig_(**kwargs)