Update latest aliases as part of release script. (#226) * Update latest aliases as part of release script. * Address code review comments.
diff --git a/release/bazelrc.py b/release/bazelrc.py index c4bddda..1a5c0a4 100644 --- a/release/bazelrc.py +++ b/release/bazelrc.py
@@ -38,7 +38,6 @@ Args: bazel_version: string, the version of Bazel used to generate the configs. - """ bazelrc_path = os.path.join( BAZELRC_DIR, "bazel-{version}.bazelrc".format(version=bazel_version))
diff --git a/release/cc/create_artifacts.py b/release/cc/create_artifacts.py index d56382d..ddf0e2f 100644 --- a/release/cc/create_artifacts.py +++ b/release/cc/create_artifacts.py
@@ -33,6 +33,7 @@ GIT_ROOT = get_git_root() LICENCE_TPL = os.path.join(GIT_ROOT, "release", "license.tpl") CPP_TPL = os.path.join(GIT_ROOT, "release", "cc", "cpp.tpl") +LATEST_TPL = os.path.join(GIT_ROOT, "release", "cc", "latest.tpl") SHA_MAP_FILE = os.path.join(GIT_ROOT, "rules/toolchain_containers.bzl") CLANG_REVISION_FILE = os.path.join(GIT_ROOT, "third_party/clang/revision.bzl") @@ -53,7 +54,6 @@ container_configs_list: list of ContainerConfigs, the list of ContainerConfigs to generate configs for. bazel_version: string, the version of Bazel used to generate the configs. - """ container_sha_map = imp.load_source("toolchain_containers", SHA_MAP_FILE) @@ -92,6 +92,38 @@ build_file.write(tpl) +def update_latest_target_aliases(container_configs_list, bazel_version): + """Updates the alias targets pointing to latest toolchain targets. + + Example latest aliases clang-ubuntu container are located in + configs/ubuntu16_04_clang/latest/BUILD. + + There is one BUILD file to contain all aliases for a container_config. + + Args: + container_configs_list: list of ContainerConfigs, the list of + ContainerConfigs to generate configs for. + bazel_version: string, the version of Bazel used to generate the configs. + """ + + for container_configs in container_configs_list: + with open(container_configs.get_latest_aliases_build_path(), + "w") as build_file: + # Update the BUILD file with aliases for latest toolchain targets. + with open(LATEST_TPL, "r") as tpl_file: + tpl = Template(tpl_file.read()).substitute( + CONFIG_VERSION=container_configs.version, + BAZEL_VERSION=bazel_version, + PACKAGE=container_configs.package, + PLATFORM=container_configs.platform_target, + CONFIG_TYPES=", ".join( + [("\"%s\"" % config_type) + for config_type in container_configs.config_types]), + ) + + build_file.write(tpl) + + def generate_toolchain_definition(container_configs_list, bazel_version): """Generates new cpp toolchain definitions. @@ -108,7 +140,6 @@ container_configs_list: list of ContainerConfigs, the list of ContainerConfigs to generate configs for. bazel_version: string, the version of Bazel used to generate the configs. - """ for container_configs in container_configs_list: @@ -169,7 +200,6 @@ Args: container_configs_list: list of ContainerConfigs, the list of ContainerConfigs to generate configs for. - """ container_sha_map = imp.load_source("toolchain_containers", SHA_MAP_FILE)
diff --git a/release/cc/execute_targets.py b/release/cc/execute_targets.py index d255b88..e0700f6 100644 --- a/release/cc/execute_targets.py +++ b/release/cc/execute_targets.py
@@ -54,7 +54,6 @@ container_configs_list: list of ContainerConfigs, the list of ContainerConfigs to generate configs for. bazel_version: string, the version of Bazel used to generate the configs. - """ atexit.register(_cleanup) @@ -84,16 +83,18 @@ os.makedirs(config.get_config_dir()) command = ("bazel build //{PACKAGE}:{TARGET}").format( - PACKAGE=container_configs.package, - TARGET=target) + PACKAGE=container_configs.package, TARGET=target) print("\nExecuting command: %s\n" % command) subprocess.check_call(shlex.split(command)) - command = ("cp ./bazel-out/k8-fastbuild/bin/{PACKAGE}/{TARGET}_outputs.tar " - "{OUTPUT_DIR}/").format( - OUTPUT_DIR=TMP_DIR, - PACKAGE=container_configs.package, - TARGET=target) + command = ( + "cp " + "{GIT_ROOT}/bazel-out/k8-fastbuild/bin/{PACKAGE}/{TARGET}_outputs.tar" + " {OUTPUT_DIR}/").format( + GIT_ROOT=GIT_ROOT, + OUTPUT_DIR=TMP_DIR, + PACKAGE=container_configs.package, + TARGET=target) print("\nExecuting command: %s\n" % command) subprocess.check_call(shlex.split(command))
diff --git a/release/cc/latest.tpl b/release/cc/latest.tpl new file mode 100644 index 0000000..c3c0e87 --- /dev/null +++ b/release/cc/latest.tpl
@@ -0,0 +1,51 @@ +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//visibility:public"]) + +# This file is auto-generated from release/cc/latest.tpl and should not +# be modified directly. + +PACKAGE = "//${PACKAGE}/" + +LATEST_CONFIG_VERSION = "${CONFIG_VERSION}" + +LATEST_BAZEL_VERSION = "${BAZEL_VERSION}" + +CONFIG_TYPES = [${CONFIG_TYPES}] + +# DO NOT depend on the following latest alias in your production jobs. +# These are for internal and our CI use only. We DO NOT guarantee that they +# will always work. +[alias( + name = "crosstool_top_" + config_type, + actual = PACKAGE + LATEST_CONFIG_VERSION + "/bazel_" + LATEST_BAZEL_VERSION + "/" + config_type + ":toolchain", +) for config_type in CONFIG_TYPES] + +[alias( + name = "toolchain_" + config_type, + actual = PACKAGE + LATEST_CONFIG_VERSION + "/bazel_" + LATEST_BAZEL_VERSION + "/cpp:cc-toolchain-clang-x86_64-" + config_type, +) for config_type in CONFIG_TYPES] + +alias( + name = "platform", + actual = PACKAGE + LATEST_CONFIG_VERSION + ":${PLATFORM}_jdk8", +) + +alias( + name = "javabase", + actual = PACKAGE + LATEST_CONFIG_VERSION + ":jdk8", +)
diff --git a/release/config.py b/release/config.py index ec0ca90..bd5382c 100644 --- a/release/config.py +++ b/release/config.py
@@ -22,18 +22,18 @@ Attributes: distro: string, base distro of container used to generate configs. version: string, version of the configs. - image: string, the container registry entry of the image used to - generated the configs, e.g. gcr.io/cloud-marketplace/google/clang-ubuntu. + image: string, the container registry entry of the image used to generated + the configs, e.g. gcr.io/cloud-marketplace/google/clang-ubuntu. package: string, the Bazel package in which we will generate the target to build configs. platform_target: string, the platform target name of the corresponding RBE container these configs will be used together with, e.g. rbe_ubuntu1604. - This is required to set value for flags: - --extra_execution_platforms, --host_platform and --platforms. + This is required to set value for flags: --extra_execution_platforms, + --host_platform and --platforms. (Platform: https://docs.bazel.build/versions/master/platforms.html) constraints: string, additional toolchain constraints needed for the cpp - toolchain definition. - These must be valid targets in this repo or @bazel_tools. + toolchain definition. These must be valid targets in this repo or + @bazel_tools. (Toolchain: https://docs.bazel.build/versions/master/toolchains.html) configs: list of Config object, a list of configs for each supported types generated using the container specified in the current ContainerConfig. @@ -56,17 +56,16 @@ Args: distro: string, base distro of container used to generate configs. version: string, version of the configs. - image: string, the container registry entry of the image used to - generated the configs, e.g. - gcr.io/cloud-marketplace/google/clang-ubuntu. + image: string, the container registry entry of the image used to generated + the configs, e.g. gcr.io/cloud-marketplace/google/clang-ubuntu. package: string, the Bazel package in which we will generate the target to - build configs. + build configs. config_types: types of config to generated with this container, e.g. default, msan. platform_target: string, the platform target name of the corresponding RBE container these configs will be used together with e.g. rbe_ubuntu1604. - This is required to set value for flags: - --extra_execution_platforms, --host_platform and --platforms. + This is required to set value for flags: --extra_execution_platforms, + --host_platform and --platforms. (Platform: https://docs.bazel.build/versions/master/platforms.html) git_root: the absolute path of the root directory of the current repository. @@ -93,6 +92,7 @@ self.package = package self.platform_target = platform_target self.constraints = ContainerConfigs._DISTRO_CONSTRAINTS_MAP[distro] + self.config_types = config_types self.configs = [ Config(root=self._get_config_base_dir(), config_type=config_type) @@ -103,6 +103,10 @@ """Returns the absolute path of the target BUILD file.""" return os.path.join(self._git_root, self.package, "BUILD") + def get_latest_aliases_build_path(self): + """Returns the absolute path of BUILD file with latest target aliases.""" + return os.path.join(self._git_root, self.package, "latest", "BUILD") + def get_toolchain_bazelrc_path(self): """Returns the absolute path of the toolchain.bazelrc file.""" return os.path.join(self._git_root, self.package, self.version, @@ -138,8 +142,8 @@ Attributes: config_type: string, type of the configs, e.g. default, msan. constraints: string, additional toolchain constraints needed for the cpp - toolchain definition. - These must be valid targets in this repo or @bazel_tools. + toolchain definition. These must be valid targets in this repo or + @bazel_tools. (Toolchain: https://docs.bazel.build/versions/master/toolchains.html) """
diff --git a/release/config_release.py b/release/config_release.py index 6976495..6326d25 100644 --- a/release/config_release.py +++ b/release/config_release.py
@@ -89,7 +89,6 @@ Args: bazel_version: string, the version of Bazel used to generate the configs. - """ # Get current supported list of container configs to generate. @@ -107,6 +106,9 @@ # Generate new cpp toolchain definition targets. cc_create.generate_toolchain_definition(container_configs_list, bazel_version) + # Update aliases to latest toolchain configs. + cc_create.update_latest_target_aliases(container_configs_list, bazel_version) + # Update toolchain.bazelrc file. toolchain_flags.update_toolchain_bazelrc_file(container_configs_list, bazel_version)
diff --git a/release/toolchain_flags.py b/release/toolchain_flags.py index 5dec5fd..f9b0bf4 100644 --- a/release/toolchain_flags.py +++ b/release/toolchain_flags.py
@@ -36,7 +36,6 @@ container_configs_list: list of ContainerConfigs, the list of ContainerConfigs to generate configs for. bazel_version: string, the version of Bazel used to generate the configs. - """ for container_configs in container_configs_list:
diff --git a/release/util.py b/release/util.py index 74eac14..e396523 100644 --- a/release/util.py +++ b/release/util.py
@@ -38,6 +38,7 @@ distro: string, base distro of container used to generate configs. config_version: string, the version of the configs. bazel_version: string, the version of Bazel used to generate the configs. + Returns: The docker_toolchain_autoconfig target to generate the configs. """