Automate more parts of config release. (#126)
* Automate more parts of config release.
diff --git a/.gitignore b/.gitignore
index 62c8d69..f4f2b91 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,6 @@
/bazel-*
# Ignore outputs generated during Bazel bootstrapping.
/output/
+# Python runtime generated from .bzl files.
+*.bzlc
+*.pyc
diff --git a/WORKSPACE b/WORKSPACE
index 4f70e76..514424d 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -99,7 +99,7 @@
# l.gcr.io/google/clang-debian8:r328903
container_pull(
name = "debian8-clang",
- digest = "sha256:8bb65bf0a0da8be48bbac07ebe743805f3dc5259203e19517098162bd23a768f",
+ digest = toolchain_container_sha256s()["debian8_clang"],
registry = "l.gcr.io",
repository = "google/clang-debian8",
)
@@ -108,7 +108,7 @@
# l.gcr.io/google/clang-ubuntu:r328903
container_pull(
name = "ubuntu16_04-clang",
- digest = "sha256:d553634f23f7c437ca35bbc4b6f1f38bb81be32b9ef2df4329dcd36762277bf7",
+ digest = toolchain_container_sha256s()["ubuntu16_04_clang"],
registry = "l.gcr.io",
repository = "google/clang-ubuntu",
)
diff --git a/release/bazelrc.py b/release/bazelrc.py
new file mode 100644
index 0000000..c4bddda
--- /dev/null
+++ b/release/bazelrc.py
@@ -0,0 +1,63 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module to generate sample bazelrc file."""
+
+import os
+from string import Template
+
+from util import get_git_root
+
+GIT_ROOT = get_git_root()
+BAZELRC_DIR = os.path.join(GIT_ROOT, "bazelrc")
+LATEST_BAZELRC_LINK = BAZELRC = os.path.join(BAZELRC_DIR, "latest.bazelrc")
+LICENCE_TPL = os.path.join(GIT_ROOT, "release", "license.tpl")
+BAZELRC_TPL = os.path.join(GIT_ROOT, "release", "bazelrc.tpl")
+
+
+def create_bazelrc_and_update_link(bazel_version):
+ """Creates new sample .bazelrc file and update latest.bazelrc symlink.
+
+ Example bazelrc files can be found in directory bazelrc/.
+
+ There is one sample bazelrc file Bazel version. bazelrc/latest.bazelrc should
+ always be symlinked to the .bazelrc file for the latest version of Bazel.
+
+ If the file already exists in this repo, the script will delete it and
+ generate new one.
+
+ Args:
+ bazel_version: string, the version of Bazel used to generate the configs.
+
+ """
+ bazelrc_path = os.path.join(
+ BAZELRC_DIR, "bazel-{version}.bazelrc".format(version=bazel_version))
+
+ # Remove old version of this .bazelrc file.
+ if os.path.exists(bazelrc_path):
+ os.remove(bazelrc_path)
+
+ with open(bazelrc_path, "w") as bazelrc_file:
+ # Write license header.
+ with open(LICENCE_TPL, "r") as license_header:
+ bazelrc_file.write(license_header.read())
+
+ # Write sample .bazelrc body.
+ with open(BAZELRC_TPL, "r") as tpl_file:
+ tpl = Template(tpl_file.read()).substitute(BAZEL_VERSION=bazel_version)
+ bazelrc_file.write(tpl)
+
+ # Update latest.bazelrc link
+ if os.path.exists(LATEST_BAZELRC_LINK):
+ os.remove(LATEST_BAZELRC_LINK)
+ os.symlink(os.path.basename(bazelrc_path), LATEST_BAZELRC_LINK)
diff --git a/release/bazelrc.tpl b/release/bazelrc.tpl
new file mode 100644
index 0000000..43dbd95
--- /dev/null
+++ b/release/bazelrc.tpl
@@ -0,0 +1,126 @@
+# This file is auto-generated from release/bazelrc.tpl and should not be
+# modified directly.
+
+# This .bazelrc file contains all of the flags required for the toolchain,
+# Remote Build Execution, and the Bazel Build Results UI. Specific flags in
+# your Bazel command allow you to use only the remote build, to use only the
+# results UI, or to use them both together.
+#
+# This .bazelrc file also contains all of the flags required for the local
+# docker sandboxing.
+
+# Remote Build Execution requires a strong hash function, such as SHA256.
+startup --host_jvm_args=-Dbazel.DigestFunction=SHA256
+
+# Depending on how many machines are in the remote execution instance, setting
+# this higher can make builds faster by allowing more jobs to run in parallel.
+# Setting it too high can result in jobs that timeout, however, while waiting
+# for a remote machine to execute them.
+build:remote --jobs=50
+
+# Set several flags related to specifying the platform, toolchain and java
+# properties.
+# These flags are duplicated rather than imported from (for example)
+# %workspace%/configs/ubuntu16_04_clang/1.0/toolchain.bazelrc to make this
+# bazelrc a standalone file that can be copied more easily.
+# These flags should only be used as is for the rbe-ubuntu16-04 container
+# and need to be adapted to work with other toolchain containers.
+build:remote --host_javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.0:jdk8
+build:remote --javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.0:jdk8
+build:remote --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
+build:remote --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
+build:remote --crosstool_top=@bazel_toolchains//configs/ubuntu16_04_clang/1.0/bazel_${BAZEL_VERSION}/default:toolchain
+build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+# Platform flags:
+# The toolchain container used for execution is defined in the target indicated
+# by "extra_execution_platforms", "host_platform" and "platforms".
+# If you are using your own toolchain container, you need to create a platform
+# target with "constraint_values" that allow for the toolchain specified with
+# "extra_toolchains" to be selected (given constraints defined in
+# "exec_compatible_with").
+# More about platforms: https://docs.bazel.build/versions/master/platforms.html
+build:remote --extra_toolchains=@bazel_toolchains//configs/ubuntu16_04_clang/1.0/bazel_${BAZEL_VERSION}/cpp:cc-toolchain-clang-x86_64-default
+build:remote --extra_execution_platforms=@bazel_toolchains//configs/ubuntu16_04_clang/1.0:rbe_ubuntu1604
+build:remote --host_platform=@bazel_toolchains//configs/ubuntu16_04_clang/1.0:rbe_ubuntu1604
+build:remote --platforms=@bazel_toolchains//configs/ubuntu16_04_clang/1.0:rbe_ubuntu1604
+
+# Set various strategies so that all actions execute remotely. Mixing remote
+# and local execution will lead to errors unless the toolchain and remote
+# machine exactly match the host machine.
+build:remote --spawn_strategy=remote
+build:remote --strategy=Javac=remote
+build:remote --strategy=Closure=remote
+build:remote --genrule_strategy=remote
+build:remote --define=EXECUTOR=remote
+
+# Enable the remote cache so action results can be shared across machines,
+# developers, and workspaces.
+build:remote --remote_cache=remotebuildexecution.googleapis.com
+
+# Enable remote execution so actions are performed on the remote systems.
+build:remote --remote_executor=remotebuildexecution.googleapis.com
+
+# Enable encryption.
+build:remote --tls_enabled=true
+
+# Enforce stricter environment rules, which eliminates some non-hermetic
+# behavior and therefore improves both the remote cache hit rate and the
+# correctness and repeatability of the build.
+build:remote --experimental_strict_action_env=true
+
+# Set a higher timeout value, just in case.
+build:remote --remote_timeout=3600
+
+# Enable authentication. This will pick up application default credentials by
+# default. You can use --auth_credentials=some_file.json to use a service
+# account credential instead.
+build:remote --auth_enabled=true
+
+# Set flags for uploading to BES in order to view results in the Bazel Build
+# Results UI.
+build:results --bes_backend="buildeventservice.googleapis.com"
+build:results --bes_timeout=60s
+
+# If the upload to BES fails, the build will fail.
+build:results --bes_best_effort=false
+
+# Output BES results url
+build:results --bes_results_url="https://source.cloud.google.com/results/invocations/"
+
+# Set flags for uploading to BES without Remote Build Execution.
+build:results-local --bes_backend="buildeventservice.googleapis.com"
+build:results-local --bes_timeout=60s
+build:results-local --bes_best_effort=false
+build:results-local --tls_enabled=true
+build:results-local --auth_enabled=true
+build:results-local --spawn_strategy=local
+build:results-local --experimental_remote_spawn_cache
+build:results-local --remote_cache=remotebuildexecution.googleapis.com
+build:results-local --remote_timeout=3600
+build:results-local --bes_results_url="https://source.cloud.google.com/results/invocations/"
+
+# The following flags are only necessary for local docker sandboxing
+# with the rbe-ubuntu16-04 container. Use of these flags is still experimental.
+build:docker-sandbox --host_javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.0:jdk8
+build:docker-sandbox --javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.0:jdk8
+build:docker-sandbox --crosstool_top=@bazel_toolchains//configs/ubuntu16_04_clang/1.0/bazel_${BAZEL_VERSION}/default:toolchain
+build:docker-sandbox --experimental_docker_image=gcr.io/cloud-marketplace/google/rbe-ubuntu16-04@sha256:59bf0e191a6b5cc1ab62c2224c810681d1326bad5a27b1d36c9f40113e79da7f
+build:docker-sandbox --spawn_strategy=docker
+build:docker-sandbox --strategy=Javac=docker
+build:docker-sandbox --strategy=Closure=docker
+build:docker-sandbox --genrule_strategy=docker
+build:docker-sandbox --define=EXECUTOR=remote
+build:docker-sandbox --experimental_docker_verbose
+build:docker-sandbox --experimental_enable_docker_sandbox
+
+# The following flags enable the remote cache so action results can be shared
+# across machines, developers, and workspaces.
+build:remote-cache --remote_cache=remotebuildexecution.googleapis.com
+build:remote-cache --tls_enabled=true
+build:remote-cache --experimental_strict_action_env=true
+build:remote-cache --remote_timeout=3600
+build:remote-cache --auth_enabled=true
+build:remote-cache --spawn_strategy=standalone
+build:remote-cache --strategy=Javac=standalone
+build:remote-cache --strategy=Closure=standalone
+build:remote-cache --genrule_strategy=standalone
diff --git a/release/cc/__init__.py b/release/cc/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/release/cc/__init__.py
diff --git a/release/cc/cpp.tpl b/release/cc/cpp.tpl
new file mode 100644
index 0000000..d2cd6d9
--- /dev/null
+++ b/release/cc/cpp.tpl
@@ -0,0 +1,17 @@
+# This target is auto-generated from release/cpp.tpl and should not be
+# modified directly.
+toolchain(
+ name = "cc-toolchain-clang-x86_64-${TYPE}",
+ exec_compatible_with = [
+ "@bazel_tools//platforms:linux",
+ "@bazel_tools//platforms:x86_64",
+ "@bazel_tools//tools/cpp:clang",
+ ${EXTRA_CONSTRAINTS}
+ ],
+ target_compatible_with = [
+ "@bazel_tools//platforms:linux",
+ "@bazel_tools//platforms:x86_64",
+ ],
+ toolchain = "//${PACKAGE}/${CONFIG_VERSION}/bazel_${BAZEL_VERSION}/${TYPE}:cc-compiler-k8",
+ toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
+)
diff --git a/release/cc/create_artifacts.py b/release/cc/create_artifacts.py
new file mode 100644
index 0000000..b43313b
--- /dev/null
+++ b/release/cc/create_artifacts.py
@@ -0,0 +1,181 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module to add METADATA and cc toolchain targets to BUILD for all configs."""
+
+import imp
+import os
+import shlex
+import shutil
+from string import Template
+import subprocess
+
+from util import get_autoconfig_target_name
+from util import get_date
+from util import get_git_root
+
+BUILD_EXTRA_LICENCE = """
+licenses(["notice"]) # Apache 2.0
+
+package(default_visibility = ["//visibility:public"])
+"""
+
+GIT_ROOT = get_git_root()
+LICENCE_TPL = os.path.join(GIT_ROOT, "release", "license.tpl")
+CPP_TPL = os.path.join(GIT_ROOT, "release", "cc", "cpp.tpl")
+SHA_MAP_FILE = os.path.join(GIT_ROOT, "rules/toolchain_containers.bzl")
+
+
+def create_targets(container_configs_list, bazel_version):
+ """Creates the new docker_toolchain_autoconfig target if not exists.
+
+ An example target located in configs/ubuntu16_04_clang/BUILD is:
+ //configs/ubuntu16_04_clang:msan-ubuntu16_04-clang-1.0-bazel_0.15.0-autoconfig
+
+ There is one target per container per Bazel version per config type.
+
+ The script only creates new targets in the BUILD file if they do not exist,
+ i.e. if a target for the given version of Bazel, type and config version
+ already exists, then the script does not re-create it.
+
+ Args:
+ container_configs_list: list of ContainerConfigs, the list of
+ ContainerConfigs to generate configs for.
+ bazel_version: string, the version of Bazel used to generate the configs.
+
+ """
+
+ container_sha_map = imp.load_source("toolchain_containers", SHA_MAP_FILE)
+
+ for container_configs in container_configs_list:
+
+ # Get the sha256 value of the container used to generate the configs.
+ sha = container_sha_map.toolchain_container_sha256s()[
+ "%s_clang" % container_configs.distro]
+
+ for config in container_configs.configs:
+
+ # Get target basename from config definitions.
+ target = get_autoconfig_target_name(
+ config_type=config.config_type,
+ distro=container_configs.distro,
+ config_version=container_configs.version,
+ bazel_version=bazel_version)
+
+ with open(container_configs.get_target_build_path(), "a+") as build_file:
+ if target not in build_file.read():
+ tpl_file_path = os.path.join(GIT_ROOT, "release", "cc",
+ "%s.tpl" % config.config_type)
+ with open(tpl_file_path, "r") as tpl_file:
+ tpl = Template(tpl_file.read()).substitute(
+ DATE=get_date(),
+ DISTRO=container_configs.distro,
+ CONFIG_VERSION=container_configs.version,
+ BAZEL_VERSION=bazel_version,
+ NAME=container_configs.image,
+ SHA=sha)
+
+ build_file.write(tpl)
+
+
+def generate_toolchain_definition(container_configs_list, bazel_version):
+ """Generates new cpp toolchain definitions.
+
+ Example cpp toolchain definitions for clang-ubuntu container are located in
+ configs/ubuntu16_04_clang/1.0/bazel_0.15.0/cpp/.
+
+ There is one BUILD file to contain all cpp toolchain definitions for each
+ config type (e.g. default, msan) per container per Bazel version.
+
+ If the file already exists in this repo, the script will delete it and
+ generate new one.
+
+ Args:
+ container_configs_list: list of ContainerConfigs, the list of
+ ContainerConfigs to generate configs for.
+ bazel_version: string, the version of Bazel used to generate the configs.
+
+ """
+
+ for container_configs in container_configs_list:
+
+ cpp_dir = os.path.dirname(container_configs.get_cpp_build_path())
+
+ # Remove old cpp directory if exists.
+ if os.path.exists(cpp_dir):
+ print("\nOld version of cpp toolchain definition already exists. "
+ "Deleting and generating again.")
+ shutil.rmtree(cpp_dir)
+ os.makedirs(cpp_dir)
+
+ with open(container_configs.get_cpp_build_path(), "w") as build_file:
+ # Write license header.
+ with open(LICENCE_TPL, "r") as license_header:
+ build_file.write(license_header.read())
+
+ # Write extra license string required for BUILD file.
+ build_file.write(BUILD_EXTRA_LICENCE)
+
+ for config in container_configs.configs:
+
+ with open(container_configs.get_cpp_build_path(), "a") as build_file:
+ with open(CPP_TPL, "r") as tpl_file:
+
+ # Merge constraint lists. Remove duplicates while perserving order.
+ constraints = container_configs.constraints
+ for constraint in config.constraints:
+ if constraint not in constraints:
+ constraints.append(constraint)
+
+ tpl = Template(tpl_file.read()).substitute(
+ TYPE=config.config_type,
+ CONFIG_VERSION=container_configs.version,
+ BAZEL_VERSION=bazel_version,
+ PACKAGE=container_configs.package,
+ EXTRA_CONSTRAINTS="\n".join(
+ [("\"%s\"," % constraint) for constraint in constraints]))
+
+ build_file.write(tpl)
+
+ subprocess.call(
+ shlex.split("buildifier %s" % container_configs.get_cpp_build_path()))
+
+
+def generate_metadata(container_configs_list):
+ """Creates the METADATA file with the container register path.
+
+ Example METADATA file can be found at
+ configs/ubuntu16_04_clang/1.0/bazel_0.15.0/default/METADATA.
+
+ There is one METADATA file per container per Bazel version per config type.
+
+ If the file already exists in this repo, the script will delete it and
+ generate new one.
+
+ Args:
+ container_configs_list: list of ContainerConfigs, the list of
+ ContainerConfigs to generate configs for.
+
+ """
+
+ container_sha_map = imp.load_source("toolchain_containers", SHA_MAP_FILE)
+
+ for container_configs in container_configs_list:
+ # Get the sha256 value of the container used to generate the configs.
+ sha = container_sha_map.toolchain_container_sha256s()[
+ "%s_clang" % container_configs.distro]
+
+ for config in container_configs.configs:
+ with open(config.get_metadata_path(), "w") as metadata_file:
+ metadata_file.write("{image}@{sha}\n".format(
+ image=container_configs.image, sha=sha))
diff --git a/release/cc/default.tpl b/release/cc/default.tpl
new file mode 100644
index 0000000..0a9bb35
--- /dev/null
+++ b/release/cc/default.tpl
@@ -0,0 +1,15 @@
+# This target is auto-generated from release/default.tpl and should not be
+# modified directly.
+# Created on ${DATE}
+# Container: ${NAME}@${SHA}
+docker_toolchain_autoconfig(
+ name = "default-${DISTRO}-clang-${CONFIG_VERSION}-bazel_${BAZEL_VERSION}-autoconfig",
+ additional_repos = ${DISTRO}_clang_default_repos(),
+ base = "@${DISTRO}-clang//image",
+ bazel_version = "${BAZEL_VERSION}",
+ env = clang_env(),
+ keys = ${DISTRO}_clang_default_keys(),
+ packages = ${DISTRO}_clang_default_packages(),
+ tags = ["manual"],
+ test = True,
+)
diff --git a/release/cc/execute_targets.py b/release/cc/execute_targets.py
new file mode 100644
index 0000000..b533500
--- /dev/null
+++ b/release/cc/execute_targets.py
@@ -0,0 +1,102 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module to exec cc toolchain targets and extract contents for all configs."""
+
+from __future__ import print_function
+
+import atexit
+import os
+import shlex
+import shutil
+import subprocess
+import tarfile
+
+from util import get_autoconfig_target_name
+from util import get_git_root
+
+GIT_ROOT = get_git_root()
+CONFIG_REPO = "local_config_cc"
+CONFIG_FILES = ["CROSSTOOL", "BUILD", "cc_wrapper.sh", "dummy_toolchain.bzl"]
+TMP_DIR = os.path.join(GIT_ROOT, "release", "tmp")
+
+
+def _cleanup():
+ """Cleanup generated files."""
+ if os.path.exists(TMP_DIR):
+ shutil.rmtree(TMP_DIR)
+
+
+def execute_and_extract_configs(container_configs_list, bazel_version):
+ """Executes the docker_toolchain_autoconfig targets and extract configs.
+
+ If configs already exist in this repo, the script will delete them and
+ generate new ones.
+
+ It generate cc configs which currently includes: CROSSTOOL, BUILD,
+ cc_wrapper.sh and dummy_toolchain.bzl. Examples can be found in
+ configs/ubuntu16_04_clang/1.0/bazel_0.15.0/default/.
+
+ There is one such set of cc configs per container per Bazel version per config
+ type.
+
+ Args:
+ container_configs_list: list of ContainerConfigs, the list of
+ ContainerConfigs to generate configs for.
+ bazel_version: string, the version of Bazel used to generate the configs.
+
+ """
+
+ atexit.register(_cleanup)
+
+ # Create temporary directory to store generated tarballs of configs.
+ if os.path.exists(TMP_DIR):
+ shutil.rmtree(TMP_DIR)
+ os.makedirs(TMP_DIR)
+
+ for container_configs in container_configs_list:
+ for config in container_configs.configs:
+
+ # Get target basename from config definitions.
+ target = get_autoconfig_target_name(
+ config_type=config.config_type,
+ distro=container_configs.distro,
+ config_version=container_configs.version,
+ bazel_version=bazel_version)
+
+ # Remove old config dir if exists.
+ if os.path.exists(config.get_config_dir()):
+ print("\nOld version of toolchain configs for {target} already exists. "
+ "Deleting and generating again.".format(target=target))
+ shutil.rmtree(config.get_config_dir())
+
+ # Generate config directory.
+ os.makedirs(config.get_config_dir())
+
+ command = ("bazel run --define=DOCKER_AUTOCONF_OUTPUT={OUTPUT_DIR} "
+ "//{PACKAGE}:{TARGET}").format(
+ OUTPUT_DIR=TMP_DIR,
+ PACKAGE=container_configs.package,
+ TARGET=target)
+ print("\nExecuting command: %s\n" % command)
+ subprocess.call(shlex.split(command))
+
+ # Extract toolchain configs.
+ tar_path = os.path.join(TMP_DIR, "%s.tar" % target)
+ tar = tarfile.open(tar_path)
+
+ for config_file in CONFIG_FILES:
+ # Extract toolchain config without the CONFIG_REPO name.
+ member = tar.getmember(os.path.join(CONFIG_REPO, config_file))
+ member.name = os.path.basename(member.name)
+ tar.extract(member, config.get_config_dir())
diff --git a/release/cc/msan.tpl b/release/cc/msan.tpl
new file mode 100644
index 0000000..dc65fa6
--- /dev/null
+++ b/release/cc/msan.tpl
@@ -0,0 +1,17 @@
+# This target is auto-generated from release/msan.tpl and should not be
+# modified directly.
+# Created on ${DATE}
+# Container: ${NAME}@${SHA}
+docker_toolchain_autoconfig(
+ name = "msan-${DISTRO}-clang-${CONFIG_VERSION}-bazel_${BAZEL_VERSION}-autoconfig",
+ additional_repos = ${DISTRO}_clang_default_repos(),
+ base = "@${DISTRO}-clang//image",
+ bazel_version = "${BAZEL_VERSION}",
+ env = clang_env() + {
+ "BAZEL_LINKOPTS": "-lc++:-lc++abi:-lm",
+ },
+ keys = ${DISTRO}_clang_default_keys(),
+ packages = ${DISTRO}_clang_default_packages(),
+ tags = ["manual"],
+ test = True,
+)
diff --git a/release/config.py b/release/config.py
new file mode 100644
index 0000000..ec0ca90
--- /dev/null
+++ b/release/config.py
@@ -0,0 +1,186 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Data structures of toolchain configs."""
+
+import os
+
+
+class ContainerConfigs(object):
+ """Group of configs that are generated using the same container.
+
+ Attributes:
+ distro: string, base distro of container used to generate configs.
+ version: string, version of the configs.
+ image: string, the container registry entry of the image used to
+ generated the configs, e.g. gcr.io/cloud-marketplace/google/clang-ubuntu.
+ package: string, the Bazel package in which we will generate the target to
+ build configs.
+ platform_target: string, the platform target name of the corresponding RBE
+ container these configs will be used together with, e.g. rbe_ubuntu1604.
+ This is required to set value for flags:
+ --extra_execution_platforms, --host_platform and --platforms.
+ (Platform: https://docs.bazel.build/versions/master/platforms.html)
+ constraints: string, additional toolchain constraints needed for the cpp
+ toolchain definition.
+ These must be valid targets in this repo or @bazel_tools.
+ (Toolchain: https://docs.bazel.build/versions/master/toolchains.html)
+ configs: list of Config object, a list of configs for each supported types
+ generated using the container specified in the current ContainerConfig.
+ """
+
+ # Currently supported distros.
+ _SUPPORTED_DISTROS = ["debian8", "ubuntu16_04"]
+
+ # Map from the container distro to additional pre-defined cpp toolchain
+ # definition constraints.
+ _DISTRO_CONSTRAINTS_MAP = {
+ "debian8": ["//constraints:jessie"],
+ "ubuntu16_04": ["//constraints:xenial"]
+ }
+
+ def __init__(self, distro, version, image, package, config_types,
+ platform_target, git_root, bazel_version):
+ """Inits ContainerConfigs.
+
+ Args:
+ distro: string, base distro of container used to generate configs.
+ version: string, version of the configs.
+ image: string, the container registry entry of the image used to
+ generated the configs, e.g.
+ gcr.io/cloud-marketplace/google/clang-ubuntu.
+ package: string, the Bazel package in which we will generate the target to
+ build configs.
+ config_types: types of config to generated with this container, e.g.
+ default, msan.
+ platform_target: string, the platform target name of the corresponding RBE
+ container these configs will be used together with e.g. rbe_ubuntu1604.
+ This is required to set value for flags:
+ --extra_execution_platforms, --host_platform and --platforms.
+ (Platform: https://docs.bazel.build/versions/master/platforms.html)
+ git_root: the absolute path of the root directory of the current
+ repository.
+ bazel_version: the version of Bazel used to generated the configs.
+
+ Returns:
+ A ContainerConfigs object.
+ Raises:
+ ValueError: An error occurred when input distro is not supported.
+ """
+
+ # Validate distro is supported.
+ if distro not in ContainerConfigs._SUPPORTED_DISTROS:
+ raise ValueError(
+ "Input distro: %s is not supported. Supported distros are %s" %
+ (distro, " ".join(ContainerConfigs._SUPPORTED_DISTROS)))
+
+ self._git_root = git_root
+ self._bazel_version = bazel_version
+
+ self.distro = distro
+ self.version = version
+ self.image = image
+ self.package = package
+ self.platform_target = platform_target
+ self.constraints = ContainerConfigs._DISTRO_CONSTRAINTS_MAP[distro]
+
+ self.configs = [
+ Config(root=self._get_config_base_dir(), config_type=config_type)
+ for config_type in config_types
+ ]
+
+ def get_target_build_path(self):
+ """Returns the absolute path of the target BUILD file."""
+ return os.path.join(self._git_root, self.package, "BUILD")
+
+ def get_toolchain_bazelrc_path(self):
+ """Returns the absolute path of the toolchain.bazelrc file."""
+ return os.path.join(self._git_root, self.package, self.version,
+ "toolchain.bazelrc")
+
+ def get_platform_build_path(self):
+ """Returns the absolute path of BUILD file with platform definition."""
+ return os.path.join(self._git_root, self.package, self.version, "BUILD")
+
+ def get_java_runtime_build_path(self):
+ """Returns the absolute path of BUILD file with java runtime."""
+ return self.get_platform_build_path()
+
+ def get_cpp_build_path(self):
+ """Returns the absolute path of BUILD file with cpp toolchain definition."""
+ return os.path.join(self._get_config_base_dir(), "cpp", "BUILD")
+
+ def _get_config_base_dir(self):
+ """Returns the absolute path of bazel_{version} directory.
+
+ This returns the absolute path of bazel_{version} directory where configs
+ are stored in the bazel-toolchains repo.
+
+ For example: /.../configs/ubuntu16_04_clang/1.0/bazel_0.15.0/
+ """
+ return os.path.join(self._git_root, self.package, self.version,
+ "bazel_%s" % self._bazel_version)
+
+
+class Config(object):
+ """Configs of a single type that are generated using a container.
+
+ Attributes:
+ config_type: string, type of the configs, e.g. default, msan.
+ constraints: string, additional toolchain constraints needed for the cpp
+ toolchain definition.
+ These must be valid targets in this repo or @bazel_tools.
+ (Toolchain: https://docs.bazel.build/versions/master/toolchains.html)
+ """
+
+ # Currently supported config types.
+ _SUPPORTED_CONFIG_TYPES = ["default", "msan"]
+
+ # Map from the config type to additional pre-defined cpp toolchain
+ # definition constraints.
+ _TYPE_CONSTRAINTS_MAP = {
+ "default": [],
+ "msan": ["//constraints/sanitizers:support_msan"]
+ }
+
+ def __init__(self, root, config_type):
+ """Inits Config.
+
+ Args:
+ root: string, absolute path to the bazel_{version} directory, e.g.
+ /.../configs/ubuntu16_04_clang/1.0/bazel_0.15.0/.
+ config_type: string, type of the configs.
+
+ Returns:
+ A Config object.
+ Raises:
+ ValueError: An error occurred when input config type is not supported.
+ """
+
+ # Validate config_type is supported.
+ if config_type not in Config._SUPPORTED_CONFIG_TYPES:
+ raise ValueError(
+ "Input config type: %s is not supported. Supported types are %s" %
+ (config_type, " ".join(Config._SUPPORTED_CONFIG_TYPES)))
+
+ self._root = root
+ self.config_type = config_type
+ self.constraints = Config._TYPE_CONSTRAINTS_MAP[config_type]
+
+ def get_config_dir(self):
+ """Returns the absolute path of the cc toolchain configs directory."""
+ return os.path.join(self._root, self.config_type)
+
+ def get_metadata_path(self):
+ """Returns the absolute path of the METADATA file."""
+ return os.path.join(self.get_config_dir(), "METADATA")
diff --git a/release/config_release.py b/release/config_release.py
new file mode 100644
index 0000000..f8e991b
--- /dev/null
+++ b/release/config_release.py
@@ -0,0 +1,121 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Script to generate toolchain configs for all config types and containers."""
+
+from __future__ import print_function
+
+import argparse
+import sys
+
+# Do not generate .pyc files.
+sys.dont_write_bytecode = True
+
+import bazelrc
+import cc.create_artifacts as cc_create
+import cc.execute_targets as cc_execute
+from config import ContainerConfigs
+import toolchain_flags
+from util import get_git_root
+
+CONFIG_TYPES = ["default", "msan"]
+
+# Define path constants.
+GIT_ROOT = get_git_root()
+
+
+def _get_container_configs_list(bazel_version):
+ """Gets the list of container configs to generate.
+
+ Args:
+ bazel_version: string, the version of Bazel used to generate configs.
+
+ Returns:
+ A list of ContainerConfigs objects corresponding to the configs to generate.
+ """
+ debian8_clang_configs = ContainerConfigs(
+ distro="debian8",
+ version="0.3.0",
+ image="gcr.io/cloud-marketplace/google/clang-debian8",
+ package="configs/debian8_clang",
+ config_types=CONFIG_TYPES,
+ platform_target="rbe_debian8",
+ git_root=GIT_ROOT,
+ bazel_version=bazel_version)
+
+ ubuntu16_04_clang_configs = ContainerConfigs(
+ distro="ubuntu16_04",
+ version="1.0",
+ image="gcr.io/cloud-marketplace/google/clang-ubuntu",
+ package="configs/ubuntu16_04_clang",
+ config_types=CONFIG_TYPES,
+ platform_target="rbe_ubuntu1604",
+ git_root=GIT_ROOT,
+ bazel_version=bazel_version)
+
+ return [debian8_clang_configs, ubuntu16_04_clang_configs]
+
+
+def _parse_arguments():
+ """Parses command line arguments for the script.
+
+ Returns:
+ args object containing the arguments
+ """
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "-b",
+ "--bazel_version",
+ required=True,
+ help="the version of Bazel used to generate toolchain configs")
+ return parser.parse_args()
+
+
+def main(bazel_version):
+ """Main function.
+
+ Examples of usage:
+ python release/config_release.py -b 0.15.0
+
+ Args:
+ bazel_version: string, the version of Bazel used to generate the configs.
+
+ """
+
+ # Get current supported list of container configs to generate.
+ container_configs_list = _get_container_configs_list(bazel_version)
+
+ # Only create the new target in the BUILD file if it does not exist.
+ cc_create.create_targets(container_configs_list, bazel_version)
+
+ # Execute the target and extract toolchain configs.
+ cc_execute.execute_and_extract_configs(container_configs_list, bazel_version)
+
+ # Generate METADATA file.
+ cc_create.generate_metadata(container_configs_list)
+
+ # Generate new cpp toolchain definition targets.
+ cc_create.generate_toolchain_definition(container_configs_list, bazel_version)
+
+ # Update toolchain.bazelrc file.
+ toolchain_flags.update_toolchain_bazelrc_file(container_configs_list,
+ bazel_version)
+
+ # Create sample .bazelrc file and update latest.bazelrc symlink.
+ bazelrc.create_bazelrc_and_update_link(bazel_version)
+
+
+if __name__ == "__main__":
+
+ args = _parse_arguments()
+ main(args.bazel_version)
diff --git a/release/license.tpl b/release/license.tpl
new file mode 100644
index 0000000..18a2a14
--- /dev/null
+++ b/release/license.tpl
@@ -0,0 +1,14 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/release/toolchain.bazelrc.tpl b/release/toolchain.bazelrc.tpl
new file mode 100644
index 0000000..f055c77
--- /dev/null
+++ b/release/toolchain.bazelrc.tpl
@@ -0,0 +1,40 @@
+# This file is auto-generated from release/toolchain.bazelrc.tpl and should not
+# be modified directly.
+
+# Toolchain related flags to append at the end of your .bazelrc file.
+build:remote --host_javabase=@bazel_toolchains//${PACKAGE}/${CONFIG_VERSION}:jdk8
+build:remote --javabase=@bazel_toolchains//${PACKAGE}/${CONFIG_VERSION}:jdk8
+build:remote --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
+build:remote --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
+build:remote --crosstool_top=@bazel_toolchains//${PACKAGE}/${CONFIG_VERSION}/bazel_${BAZEL_VERSION}/default:toolchain
+build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+# Platform flags:
+# The toolchain container used for execution is defined in the target indicated
+# by "extra_execution_platforms", "host_platform" and "platforms".
+# If you are using your own toolchain container, you need to create a platform
+# target with "constraint_values" that allow for the toolchain specified with
+# "extra_toolchains" to be selected (given constraints defined in
+# "exec_compatible_with").
+# More about platforms: https://docs.bazel.build/versions/master/platforms.html
+build:remote --extra_toolchains=@bazel_toolchains//${PACKAGE}/${CONFIG_VERSION}/bazel_${BAZEL_VERSION}/cpp:cc-toolchain-clang-x86_64-default
+build:remote --extra_execution_platforms=@bazel_toolchains//${PACKAGE}/${CONFIG_VERSION}:${PLATFORM}
+build:remote --host_platform=@bazel_toolchains//${PACKAGE}/${CONFIG_VERSION}:${PLATFORM}
+build:remote --platforms=@bazel_toolchains//${PACKAGE}/${CONFIG_VERSION}:${PLATFORM}
+
+# Experimental configs for sanitizers, use --config=remote,remote-xxsan,remote-<asan/tsan/msan> (in that order)
+# See https://github.com/bazelbuild/bazel/issues/5291.
+build:remote-xxsan --copt=-gmlt
+build:remote-xxsan --strip=never
+
+build:remote-asan --copt=-fsanitize=address
+build:remote-asan --linkopt=-fsanitize=address
+
+build:remote-tsan --copt=-fsanitize=thread
+build:remote-tsan --linkopt=-fsanitize=thread
+
+build:remote-msan --copt=-fsanitize=memory
+build:remote-msan --linkopt=-fsanitize=memory
+build:remote-msan --cxxopt=--stdlib=libc++
+build:remote-msan --copt=-fsanitize-memory-track-origins
+build:remote-msan --host_crosstool_top=@bazel_toolchains//${PACKAGE}/${CONFIG_VERSION}/bazel_${BAZEL_VERSION}/default:toolchain
+build:remote-msan --crosstool_top=@bazel_toolchains//${PACKAGE}/${CONFIG_VERSION}/bazel_${BAZEL_VERSION}/msan:toolchain
diff --git a/release/toolchain_flags.py b/release/toolchain_flags.py
new file mode 100644
index 0000000..5dec5fd
--- /dev/null
+++ b/release/toolchain_flags.py
@@ -0,0 +1,54 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module to create or update sample toolchain.bazelrc file."""
+
+import os
+from string import Template
+
+from util import get_git_root
+
+TPL = os.path.join(get_git_root(), "release", "toolchain.bazelrc.tpl")
+
+
+def update_toolchain_bazelrc_file(container_configs_list, bazel_version):
+ """Creates/updates toolchain.bazelrc file.
+
+ Example toolchain.bazelrc file can be found at
+ configs/ubuntu16_04_clang/1.0/toolchain.bazelrc.
+
+ There is one toolchain.bazelrc file per container per config version.
+
+ If the file already exists in this repo, the script will delete it and
+ generate new one.
+
+ Args:
+ container_configs_list: list of ContainerConfigs, the list of
+ ContainerConfigs to generate configs for.
+ bazel_version: string, the version of Bazel used to generate the configs.
+
+ """
+
+ for container_configs in container_configs_list:
+ with open(container_configs.get_toolchain_bazelrc_path(),
+ "w") as toolchain_bazelrc_file:
+ # Create or update toolchain.bazelrc file.
+ with open(TPL, "r") as tpl_file:
+ tpl = Template(tpl_file.read()).substitute(
+ CONFIG_VERSION=container_configs.version,
+ BAZEL_VERSION=bazel_version,
+ PACKAGE=container_configs.package,
+ PLATFORM=container_configs.platform_target,
+ )
+
+ toolchain_bazelrc_file.write(tpl)
diff --git a/release/util.py b/release/util.py
new file mode 100644
index 0000000..74eac14
--- /dev/null
+++ b/release/util.py
@@ -0,0 +1,50 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A set of utility functions."""
+
+import datetime
+import subprocess
+
+
+def get_git_root():
+ """Returns the root directory of current git repository."""
+ return subprocess.check_output(["git", "rev-parse",
+ "--show-toplevel"]).strip()
+
+
+def get_date():
+ """Returns the current date in YYYY.MM.DD format."""
+ now = datetime.datetime.now()
+ return "%s.%s.%s" % (now.year, now.month, now.day)
+
+
+def get_autoconfig_target_name(config_type, distro, config_version,
+ bazel_version):
+ """Generates the docker_toolchain_autoconfig target name.
+
+ Args:
+ config_type: string, the type of the configs, e.g. default, msan.
+ distro: string, base distro of container used to generate configs.
+ config_version: string, the version of the configs.
+ bazel_version: string, the version of Bazel used to generate the configs.
+ Returns:
+ The docker_toolchain_autoconfig target to generate the configs.
+ """
+
+ return ("{type}-{distro}-clang-{config_version}-"
+ "bazel_{bazel_version}-autoconfig").format(
+ type=config_type,
+ distro=distro,
+ config_version=config_version,
+ bazel_version=bazel_version)
diff --git a/rules/toolchain_containers.bzl b/rules/toolchain_containers.bzl
index 8fbc658..924fb2f 100644
--- a/rules/toolchain_containers.bzl
+++ b/rules/toolchain_containers.bzl
@@ -17,4 +17,12 @@
"debian8_python3": "sha256:ace668f0f01e5e562ad09c3f128488ec33fa9126313f16505a86ae77865d1696",
# gcr.io/google-appengine/python:latest
"ubuntu16_04_python3": "sha256:67fd35064a812fd0ba0a6e9485410f9f2710ebf7b0787a7b350ce6a20f166bfe",
+
+ ###########################################################
+ # Clang images #
+ ###########################################################
+ # gcr.io/cloud-marketplace/google/clang-debian8:r328903
+ "debian8_clang": "sha256:8bb65bf0a0da8be48bbac07ebe743805f3dc5259203e19517098162bd23a768f",
+ # gcr.io/cloud-marketplace/google/clang-ubuntu:r328903
+ "ubuntu16_04_clang": "sha256:d553634f23f7c437ca35bbc4b6f1f38bb81be32b9ef2df4329dcd36762277bf7",
}
