Migrate to new package manager.
Change-Id: I82c65a7734eb0118b8a95309434eaded86d5c284
diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml
index b4e6134..9b6d97e 100644
--- a/.bazelci/presubmit.yml
+++ b/.bazelci/presubmit.yml
@@ -2,29 +2,29 @@
platforms:
ubuntu1404:
test_targets:
- - "//test:debian-jessie-autoconfig_test"
- - "//test:debian-jessie-bazel-head-autoconfig_test"
- - "//test:debian-jessie-custom-bazel-version-autoconfig_test"
- - "//test:debian-jessie-custom-bazel-rc-version-autoconfig_test"
- - "//test:ubuntu-xenial-autoconfig_test"
- - "//test:ubuntu-xenial-bazel-head-autoconfig_test"
- - "//test:ubuntu-xenial-custom-bazel-version-autoconfig_test"
- - "//test:ubuntu-xenial-custom-bazel-rc-version-autoconfig_test"
- - "//test:debian8_clang_autoconfig_test"
+ - "//test/configs:debian-jessie-autoconfig_test"
+ - "//test/configs:debian-jessie-bazel-head-autoconfig_test"
+ - "//test/configs:debian-jessie-custom-bazel-version-autoconfig_test"
+ - "//test/configs:debian-jessie-custom-bazel-rc-version-autoconfig_test"
+ - "//test/configs:ubuntu-xenial-autoconfig_test"
+ - "//test/configs:ubuntu-xenial-bazel-head-autoconfig_test"
+ - "//test/configs:ubuntu-xenial-custom-bazel-version-autoconfig_test"
+ - "//test/configs:ubuntu-xenial-custom-bazel-rc-version-autoconfig_test"
+ - "//test/configs:debian8_clang_autoconfig_test"
- "//rules:debian8-clang-0.2.0-bazel_0.9.0-autoconfig_test"
- "//rules:debian8-clang-0.3.0-bazel_0.10.0-autoconfig_test"
- "//container/debian8-clang-fully-loaded:fl-toolchain-test"
ubuntu1604:
test_targets:
- - "//test:debian-jessie-autoconfig_test"
- - "//test:debian-jessie-bazel-head-autoconfig_test"
- - "//test:debian-jessie-custom-bazel-version-autoconfig_test"
- - "//test:debian-jessie-custom-bazel-rc-version-autoconfig_test"
- - "//test:ubuntu-xenial-autoconfig_test"
- - "//test:ubuntu-xenial-bazel-head-autoconfig_test"
- - "//test:ubuntu-xenial-custom-bazel-version-autoconfig_test"
- - "//test:ubuntu-xenial-custom-bazel-rc-version-autoconfig_test"
- - "//test:debian8_clang_autoconfig_test"
+ - "//test/configs:debian-jessie-autoconfig_test"
+ - "//test/configs:debian-jessie-bazel-head-autoconfig_test"
+ - "//test/configs:debian-jessie-custom-bazel-version-autoconfig_test"
+ - "//test/configs:debian-jessie-custom-bazel-rc-version-autoconfig_test"
+ - "//test/configs:ubuntu-xenial-autoconfig_test"
+ - "//test/configs:ubuntu-xenial-bazel-head-autoconfig_test"
+ - "//test/configs:ubuntu-xenial-custom-bazel-version-autoconfig_test"
+ - "//test/configs:ubuntu-xenial-custom-bazel-rc-version-autoconfig_test"
+ - "//test/configs:debian8_clang_autoconfig_test"
- "//rules:debian8-clang-0.2.0-bazel_0.9.0-autoconfig_test"
- "//rules:debian8-clang-0.3.0-bazel_0.10.0-autoconfig_test"
- "//container/debian8-clang-fully-loaded:fl-toolchain-test"
diff --git a/WORKSPACE b/WORKSPACE
index 56e00cb..01e3bde 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -22,9 +22,9 @@
# https://docs.bazel.build/versions/master/be/workspace.html#git_repository
http_archive(
name = "io_bazel_rules_docker",
- sha256 = "d2e6408b8f8b03ad1f2172acadf979204dedd0423c5194a82a1301cc6d1c224b",
- strip_prefix = "rules_docker-17f2587dc58642cd494b9d56890e6210f406380d",
- urls = ["https://github.com/bazelbuild/rules_docker/archive/17f2587dc58642cd494b9d56890e6210f406380d.tar.gz"],
+ sha256 = "3af41275396fba9fcbb88741d946ac37e51c48888cbdaba2a58a2e75a492da91",
+ strip_prefix = "rules_docker-4d8ec6570a5313fb0128e2354f2bc4323685282a",
+ urls = ["https://github.com/bazelbuild/rules_docker/archive/4d8ec6570a5313fb0128e2354f2bc4323685282a.tar.gz"],
)
load(
@@ -78,9 +78,9 @@
http_archive(
name = "debian_docker",
- sha256 = "15068de8576474c1852f48f8bedc547247d121266eae537b033d9864669b1294",
- strip_prefix = "base-images-docker-9a938f030b6eb1068ed9842bc43e2ceb601fb753",
- urls = ["https://github.com/GoogleCloudPlatform/base-images-docker/archive/9a938f030b6eb1068ed9842bc43e2ceb601fb753.tar.gz"],
+ sha256 = "36a80b159ce0e1095dd0af662255318fcf1d47190cd132d16eb829cae7a211d5",
+ strip_prefix = "base-images-docker-33624db0aaa8bc98192a6ba126afa6296b9ce952",
+ urls = ["https://github.com/GoogleCloudPlatform/base-images-docker/archive/33624db0aaa8bc98192a6ba126afa6296b9ce952.tar.gz"],
)
http_file(
@@ -93,9 +93,9 @@
# https://docs.bazel.build/versions/master/be/workspace.html#git_repository
http_archive(
name = "distroless",
- sha256 = "5401f820fddcc65fae34b5dc025ed522731aa55d89507078e63da85f420a0d63",
- strip_prefix = "distroless-886114394dfed219001ec3b068b139a3456e49d4",
- urls = ["https://github.com/GoogleCloudPlatform/distroless/archive/886114394dfed219001ec3b068b139a3456e49d4.tar.gz"],
+ sha256 = "44c5d3370df6983ef53cfc2347447c6595fea2d1951a1645660baf67657b8e23",
+ strip_prefix = "distroless-94b5126dbe06c2cb4dc74f7c9bfe6394b8e6e44c",
+ urls = ["https://github.com/GoogleCloudPlatform/distroless/archive/94b5126dbe06c2cb4dc74f7c9bfe6394b8e6e44c.tar.gz"],
)
load(
diff --git a/container/debian8-clang-fully-loaded/BUILD b/container/debian8-clang-fully-loaded/BUILD
index b96c194..2bd3b57 100644
--- a/container/debian8-clang-fully-loaded/BUILD
+++ b/container/debian8-clang-fully-loaded/BUILD
@@ -16,27 +16,14 @@
package(default_visibility = ["//visibility:public"])
-load("@distroless//cacerts:cacerts.bzl", "cacerts")
-load("@jessie_package_bundle//file:packages.bzl", "packages")
load(
"//container/rules:docker_toolchains.bzl",
"language_tool_layer",
"toolchain_container",
-)
-load(
- "//skylib:packages.bzl",
- "base_layer_packages",
- "clang_layer_packages",
- "java_layer_packages",
- "python_layer_packages",
+ "cleanup_commands",
)
load("@io_bazel_rules_docker//contrib:test.bzl", "container_test")
-cacerts(
- name = "cacerts",
- deb = packages["ca-certificates"],
-)
-
toolchain_container(
name = "fl-toolchain",
base = "@debian8//image",
@@ -52,29 +39,36 @@
"java-ltl",
"python-ltl",
],
- tags = ["manual"],
)
language_tool_layer(
name = "base-ltl",
base = "@debian8//image",
- debs = base_layer_packages(),
- tags = ["manual"],
- tars = [
- ":cacerts.tar",
+ packages = [
+ "binutils",
+ "ca-certificates",
+ "curl",
+ "ed",
+ "file",
+ "git",
+ "openssh-client",
+ "wget",
+ "zip",
+ "unzip",
],
)
language_tool_layer(
name = "clang-ltl",
base = "@debian8//image",
- debs = clang_layer_packages(),
env = {
"CC": "/usr/local/bin/clang",
"ASAN_SYMBOLIZER_PATH": "/usr/local/bin/llvm-symbolizer",
},
- tags = ["manual"],
tars = ["//third_party/clang:tar"],
+ packages = [
+ "libstdc++-4.9-dev",
+ ],
)
language_tool_layer(
@@ -84,27 +78,32 @@
"GOPATH": "/go",
"PATH": "$PATH:/usr/local/go/bin",
},
- tags = ["manual"],
tars = ["//third_party/golang:tar"],
)
language_tool_layer(
name = "java-ltl",
base = "@debian8//image",
- debs = java_layer_packages(),
env = {
"JAVA_HOME": "/usr/lib/jvm/java-8-openjdk-amd64",
},
symlinks = {
"/usr/bin/java": "/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java",
},
- tags = ["manual"],
+ packages = [
+ "openjdk-8-jdk-headless",
+ "openjdk-8-jre-headless",
+ "ca-certificates-java=20161107'*'",
+ ],
+ additional_repos = [
+ "deb http://deb.debian.org/debian jessie-backports main",
+ ],
+ installation_cleanup_commands = cleanup_commands()["java"],
)
language_tool_layer(
name = "python-ltl",
base = "@debian8//image",
- debs = python_layer_packages(),
env = {
"PATH": "$PATH:/opt/python3.6/bin",
},
@@ -113,8 +112,16 @@
"/opt/python3.6/bin/python3": "/opt/python3.6/bin/python3.6",
"/opt/python3.6/bin/pip3": "/opt/python3.6/bin/pip3.6",
},
- tags = ["manual"],
tars = ["//third_party/python:tar"],
+ packages = [
+ "python-dev",
+ "python-numpy",
+ "python-pip",
+ "python3-dev",
+ "python3-numpy",
+ "python3-pip",
+ ],
+ installation_cleanup_commands = cleanup_commands()["python"],
)
container_test(
diff --git a/container/debian8-clang-fully-loaded/build.sh b/container/debian8-clang-fully-loaded/build.sh
index d7e3e57..b8f2398 100755
--- a/container/debian8-clang-fully-loaded/build.sh
+++ b/container/debian8-clang-fully-loaded/build.sh
@@ -26,6 +26,7 @@
-p|--project GCP project ID
-c|--container docker container name
-t|--tag docker tag for the image
+ -b|--bucket GCS bucket to store the tarball of debian packages
Optional parameters (when build with Google Cloud Container Builder):
-a|--async asynchronous execute Cloud Container Builder
@@ -34,9 +35,11 @@
-l|--local build container locally
To build with Google Cloud Container Builder:
-$ ./build.sh -p my-gcp-project -c debian8-clang-fully-loaded -t latest
+$ ./build.sh -p my-gcp-project -c debian8-clang-fully-loaded -t latest -b my_bucket
will produce docker images in Google Container Registry:
gcr.io/my-gcp-project/debian8-clang-fully-loaded:{latest, clang_revision}
+and the debian packages installed will be packed as a tarball and stored in
+gs://my_bucket for future reference.
To build locally:
$ ./build.sh -l
@@ -69,6 +72,11 @@
TAG=$1
shift
;;
+ -b|--bucket)
+ shift
+ BUCKET=$1
+ shift
+ ;;
-a|--async)
ASYNC=" --async "
shift
@@ -85,7 +93,7 @@
esac
done
- if [[ ("$PROJECT" == "" || "$CONTAINER" == "" || "$TAG" == "") && "$LOCAL" == "" ]]; then
+ if [[ ("$PROJECT" == "" || "$CONTAINER" == "" || "$TAG" == "" || "$BUCKET" == "" ) && "$LOCAL" == "" ]]; then
echo "Please specify all required options for building in Google Cloud Container Builder"
show_usage
exit 1
@@ -125,7 +133,7 @@
# Start Google Cloud Container Builder
gcloud container builds submit . \
--config=${PROJECT_ROOT}/container/debian8-clang-fully-loaded/cloudbuild.yaml \
- --substitutions _PROJECT=${PROJECT},_CONTAINER=${CONTAINER},_TAG=${TAG},_DIR=${DIR} \
+ --substitutions _PROJECT=${PROJECT},_CONTAINER=${CONTAINER},_TAG=${TAG},_DIR=${DIR},_BUCKET=${BUCKET} \
${ASYNC}
fi
}
diff --git a/container/debian8-clang-fully-loaded/cloudbuild.yaml b/container/debian8-clang-fully-loaded/cloudbuild.yaml
index 7d93cb5..535162a 100644
--- a/container/debian8-clang-fully-loaded/cloudbuild.yaml
+++ b/container/debian8-clang-fully-loaded/cloudbuild.yaml
@@ -17,7 +17,9 @@
steps:
# Step: build the fully loaded container using rules_docker
- name: 'gcr.io/cloud-builders/bazel'
- args: ['run' , '--verbose_failures', '--spawn_strategy=standalone', '--genrule_strategy=standalone', '//${_DIR}:fl-toolchain']
+ # Set Bazel output_base to /workspace, which is a mounted directory on Google Cloud Builder.
+ # This is to make sure Bazel generated files can be accessed by multiple containers.
+ args: ['--output_base=/workspace', 'run' , '--verbose_failures', '--spawn_strategy=standalone', '--genrule_strategy=standalone', '//${_DIR}:fl-toolchain']
id: 'fl-container'
waitFor:
- "-" # wait for nothing - start immediately
@@ -35,6 +37,13 @@
'--image', 'gcr.io/${_PROJECT}/${_CONTAINER}:${_TAG}',
'--config', '/workspace/container/test/rbe-debian8.yaml']
+# Step: store the tarball of debian packages in GCS
+ - name: gcr.io/cloud-builders/gsutil
+ args: ['cp',
+ '/workspace/bazel-out/k8-fastbuild/bin/container/debian8-clang-fully-loaded/fl-toolchain-packages.tar',
+ 'gs://${_BUCKET}/fl-toolchain-packages-${_TAG}.tar'
+ ]
+
# Build the release-container
images:
- 'gcr.io/${_PROJECT}/${_CONTAINER}:${_TAG}'
diff --git a/container/rules/docker_toolchains.bzl b/container/rules/docker_toolchains.bzl
index 468430b..c999240 100644
--- a/container/rules/docker_toolchains.bzl
+++ b/container/rules/docker_toolchains.bzl
@@ -14,50 +14,184 @@
"""Definitions of language_tool_layer and toolchain_container rules."""
+load("@io_bazel_rules_docker//container:container.bzl", _container = "container")
+load("@debian_docker//package_managers:download_pkgs.bzl", _download = "download")
+load("@debian_docker//package_managers:install_pkgs.bzl", _install = "install")
-load(
- "@io_bazel_rules_docker//container:container.bzl",
- _container = "container",
-)
+def cleanup_commands():
+ return {
+ "java":"rm -rf /etc/ssl/certs/java/cacerts /tmp/hsperfdata_root/* /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/classes.jsa",
+ "python":"rm -rf /usr/lib/python3.4/__pycache__/ /usr/lib/python3.4/idlelib/__pycache__/ /usr/lib/python3.4/asyncio/__pycache__/",
+ }
-
-def _language_tool_layer_impl(ctx):
+def _language_tool_layer_impl(ctx, symlinks=None, env=None, tars=None,
+ files=None, packages=None, additional_repos=None,
+ installables_tars=None, installation_cleanup_commands=""):
"""Implementation for the language_tool_layer rule.
Args:
- ctx: ctx as the same for container_image rule.
- https://github.com/bazelbuild/rules_docker#container_image
+ ctx: ctx of container_image rule
+ (https://github.com/bazelbuild/rules_docker#container_image-1) +
+ ctx of download_pkgs rule
+ (https://github.com/GoogleCloudPlatform/base-images-docker/blob/master/package_managers/download_pkgs.bzl) +
+ ctx of install_pkgs rule
+ (https://github.com/GoogleCloudPlatform/base-images-docker/blob/master/package_managers/install_pkgs.bzl) +
+ some overrides.
+ symlinks: str Dict, overrides ctx.attr.symlinks
+ env: str Dict, overrides ctx.attr.env
+ tars: File list, overrides ctx.files.tars
+ files: File list, overrides ctx.files.files
+ packages: str List, overrides ctx.attr.packages
+ additional_repos: str List, overrides ctx.attr.additional_repos
+ installables_tars: File list, overrides [ctx.file.installables_tar]
+ installation_cleanup_commands: str, overrides ctx.attr.installation_cleanup_commands
TODO(ngiraldo): add validations to restrict use of any other attrs.
"""
- result = _container.image.implementation(ctx)
+
+ symlinks = symlinks or ctx.attr.symlinks
+ env = env or ctx.attr.env
+ tars = tars or ctx.files.tars
+ files = files or ctx.files.files
+ packages = packages or ctx.attr.packages
+ additional_repos = additional_repos or ctx.attr.additional_repos
+ installables_tars = installables_tars or []
+ if installables_tars == [] and ctx.file.installables_tar:
+ installables_tars = [ctx.file.installables_tar]
+ installation_cleanup_commands = installation_cleanup_commands or ctx.attr.installation_cleanup_commands
+
+ # Download packages if packages list is not empty.
+ if packages != []:
+ # Declare intermediate output file generated by download_pkgs rule.
+ download_pkgs_output_executable = ctx.actions.declare_file(ctx.attr.name + "-download_pkgs_output_executable.sh")
+ download_pkgs_output_tar = ctx.actions.declare_file(ctx.attr.name + "-download_pkgs_output_tar.tar")
+ download_pkgs_output_script = ctx.actions.declare_file(ctx.attr.name + "-download_pkgs_output_script.sh")
+
+ # download_pkgs rule consumes 'packages' and 'additional_repos'.
+ _download.implementation(
+ ctx,
+ image_tar=ctx.files.base[0],
+ packages=packages,
+ additional_repos=additional_repos,
+ output_executable=download_pkgs_output_executable,
+ output_tar=download_pkgs_output_tar,
+ output_script=download_pkgs_output_script
+ )
+ installables_tars.append(download_pkgs_output_tar)
+
+ # Prepare new base image for the container_image rule.
+ new_base = ctx.files.base[0]
+
+ # Install debian packages in the base image.
+ if installables_tars != []:
+ # Create a list of paths of installables_tars.
+ installables_tars_paths = []
+ for tar in installables_tars:
+ if tar:
+ installables_tars_paths.append(tar.path)
+
+ # Declare intermediate output file generated by install_pkgs rule.
+ combined_installables_tar = ctx.actions.declare_file(ctx.attr.name + "-packages.tar")
+
+ # Combine all installables_tars into one tar. install_pkgs only takes a
+ # single installables_tar as input.
+ ctx.actions.run_shell(
+ inputs=installables_tars,
+ outputs=[combined_installables_tar],
+ command="tar cvf {output_tar} --files-from /dev/null && tar --concatenate -v --file={output_tar} {input_tars}".format(
+ output_tar=combined_installables_tar.path,
+ input_tars=' '.join(installables_tars_paths)
+ ),
+ )
+
+ # Declare intermediate output file generated by install_pkgs rule.
+ install_pkgs_out = ctx.actions.declare_file(ctx.attr.name + "-with-packages.tar")
+ # install_pkgs rule consumes 'combined_installables_tar' and 'installation_cleanup_commands'.
+ _install.implementation(
+ ctx,
+ image_tar=ctx.files.base[0],
+ installables_tar= combined_installables_tar,
+ installation_cleanup_commands = installation_cleanup_commands,
+ output_image_name = ctx.attr.name + "-with-packages",
+ output_tar=install_pkgs_out
+ )
+ # Set the image with packages installed to be the new base.
+ new_base = install_pkgs_out
+
+ # Install tars and configure env, symlinks using the container_image rule.
+ result = _container.image.implementation(ctx, base=new_base, symlinks=symlinks, env=env, tars=tars, files=files)
+
return struct(runfiles = result.runfiles,
files = result.files,
container_parts = result.container_parts,
- base = ctx.attr.base,
- debs = ctx.files.debs,
- tars = ctx.files.tars,
- input_files = ctx.files.files,
- env = ctx.attr.env,
- symlinks = ctx.attr.symlinks)
+ tars = tars,
+ input_files = files,
+ env = env,
+ symlinks = symlinks,
+ packages = packages,
+ additional_repos = additional_repos,
+ installables_tar = ctx.file.installables_tar,
+ installation_cleanup_commands = installation_cleanup_commands
+ )
+
+language_tool_layer_attrs = _container.image.attrs + _install.attrs + _download.attrs + {
+ # Redeclare following attributes as non-mandatory.
+ "image_tar": attr.label(
+ allow_files = True,
+ single_file = True,
+ ),
+ "packages": attr.string_list(),
+ "installables_tar": attr.label(
+ allow_files = True,
+ single_file = True,
+ ),
+ "output_image_name": attr.string(),
+}
language_tool_layer_ = rule(
- attrs = _container.image.attrs,
+ attrs = language_tool_layer_attrs,
executable = True,
outputs = _container.image.outputs,
implementation = _language_tool_layer_impl,
)
def language_tool_layer(**kwargs):
- """A thin wrapper around attrs in container_image rule.
+ """A wrapper around attrs in container_image, download_pkgs and install_pkgs rules.
- All attrs in language_tool_layer will be passed into
- toolchain_container rule.
+ Downloads and installs debian packages using
+ https://github.com/GoogleCloudPlatform/base-images-docker/tree/master/package_managers,
+ and configures the rest using https://github.com/bazelbuild/rules_docker#container_image-1.
+
+ Args:
+ Same args as https://github.com/bazelbuild/rules_docker#container_image-1
+ minus:
+ debs: debian packages should be listed in 'packages', or be included in
+ 'installables_tar' as .deb files.
+ plus:
+ packages: list of packages to fetch and install in the base image.
+ additional_repos: list of additional debian package repos to use,
+ in sources.list format.
+ installables_tar: a tar of debian packages to install in the base image.
+ installation_cleanup_commands: cleanup commands to run after package
+ installation.
+
+ Note:
+ - 'additional_repos' can only be specified when 'packages' is speficified.
+ - 'installation_cleanup_commands' should also only be specified when
+ 'packages' or 'installables_tar' is specified.
Experimental rule.
"""
- language_tool_layer_(**kwargs)
+ # Input validations
+ if "debs" in kwargs:
+ fail("debs is not supported in language_tool_layer.")
+
+ has_packages = "packages" not in kwargs or kwargs["packages"] == []
+ if has_packages and "additional_repos" in kwargs:
+ fail("'additional_repos' can only be specified when 'packages' is not empty.")
+
+ language_tool_layer_(**kwargs)
def _toolchain_container_impl(ctx):
"""Implementation for the toolchain_container rule.
@@ -69,29 +203,57 @@
ctx: ctx as the same as for container_image + list of language_tool_layer(s)
https://github.com/bazelbuild/rules_docker#container_image
"""
- debs = []
+
tars = []
files = []
env = {}
symlinks = {}
+ packages = []
+ additional_repos = []
+ installables_tars = []
+ installation_cleanup_commands = "cd ."
+
# TODO(ngiraldo): we rewrite env and symlinks if there are conficts,
# warn the user of conflicts or error out.
for layer in ctx.attr.language_layers:
- debs.extend(layer.debs)
tars.extend(layer.tars)
files.extend(layer.input_files)
env.update(layer.env)
symlinks.update(layer.symlinks)
- debs.extend(ctx.attr.debs)
+ packages.extend(layer.packages)
+ additional_repos.extend(layer.additional_repos)
+ if layer.installables_tar:
+ installables_tars.append(layer.installables_tar)
+ if layer.installation_cleanup_commands:
+ installation_cleanup_commands += (" && " + layer.installation_cleanup_commands)
tars.extend(ctx.attr.tars)
env.update(ctx.attr.env)
symlinks.update(ctx.attr.symlinks)
- debs = depset(debs).to_list()
+ packages.extend(ctx.attr.packages)
+ additional_repos.extend(ctx.attr.additional_repos)
+ if ctx.file.installables_tar:
+ installables_tars.append(ctx.file.installables_tar)
+ if ctx.attr.installation_cleanup_commands:
+ installation_cleanup_commands += (" && " + ctx.attr.installation_cleanup_commands)
+
files = depset(files).to_list()
- return _container.image.implementation(ctx, symlinks=symlinks, env=env, debs=debs, tars=tars, files=files)
+ packages = depset(packages).to_list()
+ additional_repos = depset(additional_repos).to_list()
+ installables_tars = depset(installables_tars).to_list()
+
+ return _language_tool_layer_impl(ctx,
+ symlinks=symlinks,
+ env=env,
+ tars=tars,
+ files=files,
+ packages=packages,
+ additional_repos=additional_repos,
+ installables_tars=installables_tars,
+ installation_cleanup_commands=installation_cleanup_commands
+ )
toolchain_container_ = rule(
- attrs = _container.image.attrs + {
+ attrs = language_tool_layer_attrs + {
"language_layers": attr.label_list(),
},
executable = True,
@@ -102,6 +264,29 @@
def toolchain_container(**kwargs):
"""Composes multiple language_tool_layers into a single resulting image.
+ Args:
+ Same args as https://github.com/bazelbuild/rules_docker#container_image-1
+ minus:
+ debs: debian packages should be listed in 'packages', or be included in
+ 'installables_tar' as .deb files.
+ plus:
+ language_layers: a list of language_tool_layer.
+ installables_tar: a tar of debian packages to install in the base image.
+ packages: list of packages to fetch and install in the base image.
+ additional_repos: list of additional debian package repos to use,
+ in sources.list format.
+ installation_cleanup_commands: cleanup commands to run after package
+ installation.
+
Experimental rule.
"""
+
+ # Input validations
+ if "debs" in kwargs:
+ fail("debs is not supported in toolchain_container.")
+
+ has_packages = "packages" not in kwargs or kwargs["packages"] == []
+ if has_packages and "additional_repos" in kwargs:
+ fail("'additional_repos' can only be specified when 'packages' is not empty.")
+
toolchain_container_(**kwargs)
diff --git a/rules/docker_config.bzl b/rules/docker_config.bzl
index f4e430e..499c43f 100644
--- a/rules/docker_config.bzl
+++ b/rules/docker_config.bzl
@@ -179,7 +179,7 @@
download_pkgs(
name = name + "_pkgs",
additional_repos = additional_repos,
- image_tar = ":" + name + "_with_keys",
+ image_tar = ":" + name + "_with_keys.tar",
packages = packages,
)
@@ -301,7 +301,7 @@
(project_repo_dir, package_name, repo_pkg_tar, project_repo_dir))
remove_repo_cmd = ("rm -drf ./%s" % project_repo_dir)
- result = _container.image.implementation(ctx, cmd=docker_cmd, output=ctx.outputs.load_image)
+ result = _container.image.implementation(ctx, cmd=docker_cmd, output_executable=ctx.outputs.load_image)
# By default we copy the produced tar file to /tmp/
output_location = "/tmp/" + ctx.attr.name + ".tar"
@@ -521,7 +521,7 @@
name = kwargs["name"] + "_test",
size = "medium",
timeout = "long",
- srcs = ["//test:autoconfig_test.sh"],
+ srcs = ["//test/configs:autoconfig_test.sh"],
data = [":" + kwargs["name"]],
)
diff --git a/test/BUILD b/test/configs/BUILD
similarity index 98%
rename from test/BUILD
rename to test/configs/BUILD
index c2b2ece..3a3f129 100644
--- a/test/BUILD
+++ b/test/configs/BUILD
@@ -66,7 +66,7 @@
docker_toolchain_autoconfig(
name = "debian-jessie-custom-bazel-version-autoconfig",
base = ":jessie-with-pkgs.tar",
- bazel_version = "0.9.0",
+ bazel_version = "0.11.0",
env = gcc_env(),
tags = ["manual"],
test = True,
@@ -125,7 +125,7 @@
docker_toolchain_autoconfig(
name = "ubuntu-xenial-custom-bazel-version-autoconfig",
base = ":xenial-with-pkgs.tar",
- bazel_version = "0.9.0",
+ bazel_version = "0.11.0",
env = gcc_env(),
tags = ["manual"],
test = True,
diff --git a/test/autoconfig_test.sh b/test/configs/autoconfig_test.sh
similarity index 100%
rename from test/autoconfig_test.sh
rename to test/configs/autoconfig_test.sh
diff --git a/test/debian8_clang_autoconfig_test.sh b/test/configs/debian8_clang_autoconfig_test.sh
similarity index 100%
rename from test/debian8_clang_autoconfig_test.sh
rename to test/configs/debian8_clang_autoconfig_test.sh
diff --git a/test/container/BUILD b/test/container/BUILD
new file mode 100644
index 0000000..9c1604a
--- /dev/null
+++ b/test/container/BUILD
@@ -0,0 +1,17 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"]) # Apache 2.0
+
+package(default_visibility = ["//visibility:public"])
diff --git a/test/container/container_determinism_test.bzl b/test/container/container_determinism_test.bzl
new file mode 100644
index 0000000..698396a
--- /dev/null
+++ b/test/container/container_determinism_test.bzl
@@ -0,0 +1,18 @@
+def container_determinism_test(name, container_name, container_sha):
+ """Macro to invoke a sh_test rule with right parameters.
+
+ Args:
+ name: name of this rule.
+ container_name: name of the container
+ container_sha: value of the expected sha to check against
+ """
+
+ return native.sh_test(
+ name = name,
+ size = "medium",
+ timeout = "long",
+ srcs = ["container_determinism_test.sh"],
+ data = [container_name],
+ tags = ["manual"],
+ args = ["-c %s -s %s" % (container_name, container_sha)],
+ )
diff --git a/test/container/container_determinism_test.sh b/test/container/container_determinism_test.sh
new file mode 100755
index 0000000..8af78b1
--- /dev/null
+++ b/test/container/container_determinism_test.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Test each language layer and the fully-loaded container can be fully
+# reproduced from our saved tar of debian packages.
+
+set -e
+
+function print_usage {
+ echo "Usage:"
+ echo " -c name of the container"
+ echo " -s value of the expected sha to check against"
+ if [[ -n $1 ]]; then
+ echo $1
+ fi
+ exit 1
+}
+
+# Define constants.
+WORKSPACE_ROOT=$(pwd)
+NAME=${TEST_BINARY##*/}
+DIR=${TEST_BINARY%${NAME}}
+
+container=""
+valid_sha=""
+
+OPTIND=1 # Reset for getopts, just in case.
+ while getopts "c:s:" opt; do
+ case "$opt" in
+ c)
+ [[ -z "$container" ]] || print_usage "ERROR: Flag specified twice"
+ container=$OPTARG
+ ;;
+ s)
+ [[ -z "$valid_sha" ]] || print_usage "ERROR: Flag specified twice"
+ valid_sha=$OPTARG
+ ;;
+ *)
+ print_usage "ERROR: unknown option"
+ ;;
+ esac
+ done
+
+[[ "$container" != "" ]] || print_usage "ERROR: must specify the container name"
+[[ "$valid_sha" != "" ]] || print_usage "ERROR: must specify the value of valid valid_sha"
+
+# Execute the script to build the container.
+${WORKSPACE_ROOT}/${DIR}${container}
+current_sha=$(docker inspect --format="{{.Id}}" bazel/${DIR%/}:${container})
+
+if [ "${current_sha}" != "${valid_sha}" ]; then
+ echo "Image valid_sha of bazel/${DIR%/}:${container} is changed."
+ exit -1
+fi
+
+echo "PASS"
+
+# TODO(xingao): clean up test images when test finishes.
