release/cc/create_artifacts.py - bazel-toolchains - Git at Google

 # Copyright 2016 The Bazel Authors. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #    http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Module to add METADATA and cc toolchain targets to BUILD for all configs."""

 import imp
 import os
 import shlex
 import shutil
 from string import Template
 import subprocess

 from util import get_autoconfig_target_name
 from util import get_date
 from util import get_git_root

 BUILD_EXTRA_LICENCE = """
 licenses(["notice"])  # Apache 2.0

 package(default_visibility = ["//visibility:public"])
 """

 GIT_ROOT = get_git_root()
 LICENCE_TPL = os.path.join(GIT_ROOT, "release", "license.tpl")
 CPP_TPL = os.path.join(GIT_ROOT, "release", "cc", "cpp.tpl")
 LATEST_TPL = os.path.join(GIT_ROOT, "release", "cc", "latest.tpl")
 SHA_MAP_FILE = os.path.join(GIT_ROOT, "rules/toolchain_containers.bzl")
 CLANG_REVISION_FILE = os.path.join(GIT_ROOT, "third_party/clang/revision.bzl")


 def create_targets(container_configs_list, bazel_version):
   """Creates the new docker_toolchain_autoconfig target if not exists.

   An example target located in configs/ubuntu16_04_clang/BUILD is:
   //configs/ubuntu16_04_clang:msan-ubuntu16_04-clang-1.0-bazel_0.15.0-autoconfig

   There is one target per container per Bazel version per config type.

   The script only creates new targets in the BUILD file if they do not exist,
   i.e. if a target for the given version of Bazel, type and config version
   already exists, then the script does not re-create it.

   Args:
     container_configs_list: list of ContainerConfigs, the list of
       ContainerConfigs to generate configs for.
     bazel_version: string, the version of Bazel used to generate the configs.
   """

   container_sha_map = imp.load_source("toolchain_containers", SHA_MAP_FILE)
   clang_revision_map = imp.load_source("clang_revision", CLANG_REVISION_FILE)
   clang_revision = clang_revision_map.CLANG_REVISION

   for container_configs in container_configs_list:

     # Get the sha256 value of the container used to generate the configs.
     sha = container_sha_map.toolchain_container_sha256s()[
         "%s_clang" % container_configs.distro]

     for config in container_configs.configs:

       # Get target basename from config definitions.
       target = get_autoconfig_target_name(
           config_type=config.config_type,
           distro=container_configs.distro,
           config_version=container_configs.version,
           bazel_version=bazel_version)

       with open(container_configs.get_target_build_path(), "a+") as build_file:
         if target not in build_file.read():
           tpl_file_path = os.path.join(GIT_ROOT, "release", "cc",
                                        "%s.tpl" % config.config_type)
           with open(tpl_file_path, "r") as tpl_file:
             tpl = Template(tpl_file.read()).substitute(
                 DATE=get_date(),
                 DISTRO=container_configs.distro,
                 CONFIG_VERSION=container_configs.version,
                 BAZEL_VERSION=bazel_version,
                 NAME=container_configs.image,
                 SHA=sha,
                 CLANG_REVISION=clang_revision)

             build_file.write(tpl)


 def update_latest_target_aliases(container_configs_list, bazel_version):
   """Updates the alias targets pointing to latest toolchain targets.

   Example latest aliases clang-ubuntu container are located in
   configs/ubuntu16_04_clang/latest/BUILD.

   There is one BUILD file to contain all aliases for a container_config.

   Args:
     container_configs_list: list of ContainerConfigs, the list of
       ContainerConfigs to generate configs for.
     bazel_version: string, the version of Bazel used to generate the configs.
   """

   for container_configs in container_configs_list:

     constraints = list(container_configs.constraints)

     with open(container_configs.get_latest_aliases_build_path(),
               "w") as build_file:
       # Update the BUILD file with aliases for latest toolchain targets.
       with open(LATEST_TPL, "r") as tpl_file:
         tpl = Template(tpl_file.read()).substitute(
             CONFIG_VERSION=container_configs.version,
             BAZEL_VERSION=bazel_version,
             PACKAGE=container_configs.package,
             PLATFORM=container_configs.platform_target,
             CONFIG_TYPES=", ".join(
                 [("\"%s\"" % config_type)
                  for config_type in container_configs.config_types]),
             EXTRA_CONSTRAINTS="\n".join(
                 [("\"%s\"," % constraint) for constraint in constraints])
         )

         build_file.write(tpl)


 def generate_toolchain_definition(container_configs_list, bazel_version):
   """Generates new cpp toolchain definitions.

   Example cpp toolchain definitions for clang-ubuntu container are located in
   configs/ubuntu16_04_clang/1.0/bazel_0.15.0/cpp/.

   There is one BUILD file to contain all cpp toolchain definitions for each
   config type (e.g. default, msan) per container per Bazel version.

   If the file already exists in this repo, the script will delete it and
   generate new one.

   Args:
     container_configs_list: list of ContainerConfigs, the list of
       ContainerConfigs to generate configs for.
     bazel_version: string, the version of Bazel used to generate the configs.
   """

   for container_configs in container_configs_list:

     cpp_dir = os.path.dirname(container_configs.get_cpp_build_path())

     # Remove old cpp directory if exists.
     if os.path.exists(cpp_dir):
       print("\nOld version of cpp toolchain definition already exists. "
             "Deleting and generating again.")
       shutil.rmtree(cpp_dir)
     os.makedirs(cpp_dir)

     with open(container_configs.get_cpp_build_path(), "w") as build_file:
       # Write license header.
       with open(LICENCE_TPL, "r") as license_header:
         build_file.write(license_header.read())

       # Write extra license string required for BUILD file.
       build_file.write(BUILD_EXTRA_LICENCE)

     for config in container_configs.configs:

       with open(container_configs.get_cpp_build_path(), "a") as build_file:
         with open(CPP_TPL, "r") as tpl_file:

           # Merge constraint lists. Remove duplicates while perserving order.
           constraints = list(container_configs.constraints)
           for constraint in config.constraints:
             if constraint not in constraints:
               constraints.append(constraint)

           tpl = Template(tpl_file.read()).substitute(
               TYPE=config.config_type,
               CONFIG_VERSION=container_configs.version,
               BAZEL_VERSION=bazel_version,
               PACKAGE=container_configs.package,
               EXTRA_CONSTRAINTS="\n".join(
                   [("\"%s\"," % constraint) for constraint in constraints]))

           build_file.write(tpl)

       subprocess.check_call(
           shlex.split("buildifier %s" % container_configs.get_cpp_build_path()))


 def generate_metadata(container_configs_list):
   """Creates the METADATA file with the container register path.

   Example METADATA file can be found at
   configs/ubuntu16_04_clang/1.0/bazel_0.15.0/default/METADATA.

   There is one METADATA file per container per Bazel version per config type.

   If the file already exists in this repo, the script will delete it and
   generate new one.

   Args:
     container_configs_list: list of ContainerConfigs, the list of
       ContainerConfigs to generate configs for.
   """

   container_sha_map = imp.load_source("toolchain_containers", SHA_MAP_FILE)

   for container_configs in container_configs_list:
     # Get the sha256 value of the container used to generate the configs.
     sha = container_sha_map.toolchain_container_sha256s()[
         "%s_clang" % container_configs.distro]

     for config in container_configs.configs:
       with open(config.get_metadata_path(), "w") as metadata_file:
         metadata_file.write("{image}@{sha}\n".format(
             image=container_configs.image, sha=sha))
	# Copyright 2016 The Bazel Authors. All rights reserved.
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	"""Module to add METADATA and cc toolchain targets to BUILD for all configs."""

	import imp
	import os
	import shlex
	import shutil
	from string import Template
	import subprocess

	from util import get_autoconfig_target_name
	from util import get_date
	from util import get_git_root

	BUILD_EXTRA_LICENCE = """
	licenses(["notice"]) # Apache 2.0

	package(default_visibility = ["//visibility:public"])
	"""

	GIT_ROOT = get_git_root()
	LICENCE_TPL = os.path.join(GIT_ROOT, "release", "license.tpl")
	CPP_TPL = os.path.join(GIT_ROOT, "release", "cc", "cpp.tpl")
	LATEST_TPL = os.path.join(GIT_ROOT, "release", "cc", "latest.tpl")
	SHA_MAP_FILE = os.path.join(GIT_ROOT, "rules/toolchain_containers.bzl")
	CLANG_REVISION_FILE = os.path.join(GIT_ROOT, "third_party/clang/revision.bzl")


	def create_targets(container_configs_list, bazel_version):
	"""Creates the new docker_toolchain_autoconfig target if not exists.

	An example target located in configs/ubuntu16_04_clang/BUILD is:
	//configs/ubuntu16_04_clang:msan-ubuntu16_04-clang-1.0-bazel_0.15.0-autoconfig

	There is one target per container per Bazel version per config type.

	The script only creates new targets in the BUILD file if they do not exist,
	i.e. if a target for the given version of Bazel, type and config version
	already exists, then the script does not re-create it.

	Args:
	container_configs_list: list of ContainerConfigs, the list of
	ContainerConfigs to generate configs for.
	bazel_version: string, the version of Bazel used to generate the configs.
	"""

	container_sha_map = imp.load_source("toolchain_containers", SHA_MAP_FILE)
	clang_revision_map = imp.load_source("clang_revision", CLANG_REVISION_FILE)
	clang_revision = clang_revision_map.CLANG_REVISION

	for container_configs in container_configs_list:

	# Get the sha256 value of the container used to generate the configs.
	sha = container_sha_map.toolchain_container_sha256s()[
	"%s_clang" % container_configs.distro]

	for config in container_configs.configs:

	# Get target basename from config definitions.
	target = get_autoconfig_target_name(
	config_type=config.config_type,
	distro=container_configs.distro,
	config_version=container_configs.version,
	bazel_version=bazel_version)

	with open(container_configs.get_target_build_path(), "a+") as build_file:
	if target not in build_file.read():
	tpl_file_path = os.path.join(GIT_ROOT, "release", "cc",
	"%s.tpl" % config.config_type)
	with open(tpl_file_path, "r") as tpl_file:
	tpl = Template(tpl_file.read()).substitute(
	DATE=get_date(),
	DISTRO=container_configs.distro,
	CONFIG_VERSION=container_configs.version,
	BAZEL_VERSION=bazel_version,
	NAME=container_configs.image,
	SHA=sha,
	CLANG_REVISION=clang_revision)

	build_file.write(tpl)


	def update_latest_target_aliases(container_configs_list, bazel_version):
	"""Updates the alias targets pointing to latest toolchain targets.

	Example latest aliases clang-ubuntu container are located in
	configs/ubuntu16_04_clang/latest/BUILD.

	There is one BUILD file to contain all aliases for a container_config.

	Args:
	container_configs_list: list of ContainerConfigs, the list of
	ContainerConfigs to generate configs for.
	bazel_version: string, the version of Bazel used to generate the configs.
	"""

	for container_configs in container_configs_list:

	constraints = list(container_configs.constraints)

	with open(container_configs.get_latest_aliases_build_path(),
	"w") as build_file:
	# Update the BUILD file with aliases for latest toolchain targets.
	with open(LATEST_TPL, "r") as tpl_file:
	tpl = Template(tpl_file.read()).substitute(
	CONFIG_VERSION=container_configs.version,
	BAZEL_VERSION=bazel_version,
	PACKAGE=container_configs.package,
	PLATFORM=container_configs.platform_target,
	CONFIG_TYPES=", ".join(
	[("\"%s\"" % config_type)
	for config_type in container_configs.config_types]),
	EXTRA_CONSTRAINTS="\n".join(
	[("\"%s\"," % constraint) for constraint in constraints])
	)

	build_file.write(tpl)


	def generate_toolchain_definition(container_configs_list, bazel_version):
	"""Generates new cpp toolchain definitions.

	Example cpp toolchain definitions for clang-ubuntu container are located in
	configs/ubuntu16_04_clang/1.0/bazel_0.15.0/cpp/.

	There is one BUILD file to contain all cpp toolchain definitions for each
	config type (e.g. default, msan) per container per Bazel version.

	If the file already exists in this repo, the script will delete it and
	generate new one.

	Args:
	container_configs_list: list of ContainerConfigs, the list of
	ContainerConfigs to generate configs for.
	bazel_version: string, the version of Bazel used to generate the configs.
	"""

	for container_configs in container_configs_list:

	cpp_dir = os.path.dirname(container_configs.get_cpp_build_path())

	# Remove old cpp directory if exists.
	if os.path.exists(cpp_dir):
	print("\nOld version of cpp toolchain definition already exists. "
	"Deleting and generating again.")
	shutil.rmtree(cpp_dir)
	os.makedirs(cpp_dir)

	with open(container_configs.get_cpp_build_path(), "w") as build_file:
	# Write license header.
	with open(LICENCE_TPL, "r") as license_header:
	build_file.write(license_header.read())

	# Write extra license string required for BUILD file.
	build_file.write(BUILD_EXTRA_LICENCE)

	for config in container_configs.configs:

	with open(container_configs.get_cpp_build_path(), "a") as build_file:
	with open(CPP_TPL, "r") as tpl_file:

	# Merge constraint lists. Remove duplicates while perserving order.
	constraints = list(container_configs.constraints)
	for constraint in config.constraints:
	if constraint not in constraints:
	constraints.append(constraint)

	tpl = Template(tpl_file.read()).substitute(
	TYPE=config.config_type,
	CONFIG_VERSION=container_configs.version,
	BAZEL_VERSION=bazel_version,
	PACKAGE=container_configs.package,
	EXTRA_CONSTRAINTS="\n".join(
	[("\"%s\"," % constraint) for constraint in constraints]))

	build_file.write(tpl)

	subprocess.check_call(
	shlex.split("buildifier %s" % container_configs.get_cpp_build_path()))


	def generate_metadata(container_configs_list):
	"""Creates the METADATA file with the container register path.

	Example METADATA file can be found at
	configs/ubuntu16_04_clang/1.0/bazel_0.15.0/default/METADATA.

	There is one METADATA file per container per Bazel version per config type.

	If the file already exists in this repo, the script will delete it and
	generate new one.

	Args:
	container_configs_list: list of ContainerConfigs, the list of
	ContainerConfigs to generate configs for.
	"""

	container_sha_map = imp.load_source("toolchain_containers", SHA_MAP_FILE)

	for container_configs in container_configs_list:
	# Get the sha256 value of the container used to generate the configs.
	sha = container_sha_map.toolchain_container_sha256s()[
	"%s_clang" % container_configs.distro]

	for config in container_configs.configs:
	with open(config.get_metadata_path(), "w") as metadata_file:
	metadata_file.write("{image}@{sha}\n".format(
	image=container_configs.image, sha=sha))