better python3 installation for rbe-{debian9,ubuntu16_04} (#32)
- for rbe-debian9, we use the python3.6 from debian testing repo
- for rbe-ubuntu16_04, we use the python3.6 built from ubuntu by
appengine python image
- add setuptools to make python package install more stable
- also simplify the cleanup command by searching and deleting all
__pycache__ folders
- fix container tests problems
Tests:
- container/build.sh -p my-project -d debian8 -c test-rbe-debian8 -t test-r327695 -b my-gcs-bucket -a
- container/build.sh -p my-project -d debian9 -c test-rbe-debian9 -t test-r327695 -b my-gcs-bucket -a
- container/build.sh -p my-project -d ubuntu16_04 -c test-rbe-ubuntu16_04 -t test-r327695 -b my-gcs-bucket -a
diff --git a/container/experimental/rbe-debian8/BUILD b/container/experimental/rbe-debian8/BUILD
index b5c5431..d106805 100644
--- a/container/experimental/rbe-debian8/BUILD
+++ b/container/experimental/rbe-debian8/BUILD
@@ -101,9 +101,10 @@
"/usr/bin/python": "/usr/bin/python2.7",
"/opt/python3.6/bin/python3": "/opt/python3.6/bin/python3.6",
"/opt/python3.6/bin/pip3": "/opt/python3.6/bin/pip3.6",
+ "/opt/python3.6/bin/easy_install3": "/opt/python3.6/bin/easy_install-3.6",
},
tags = ["manual"],
- tars = ["//third_party/python:tar"],
+ tars = ["//third_party/python:debian8_tar"],
)
container_test(
diff --git a/container/experimental/rbe-debian9/BUILD b/container/experimental/rbe-debian9/BUILD
index 26a836d..5d0fd85 100644
--- a/container/experimental/rbe-debian9/BUILD
+++ b/container/experimental/rbe-debian9/BUILD
@@ -23,19 +23,15 @@
)
load("@io_bazel_rules_docker//contrib:test.bzl", "container_test")
-JAVA_CLEANUP_COMMANDS=(
+JAVA_CLEANUP_COMMANDS = (
"rm -rf " +
"/etc/ssl/certs/java/cacerts " +
"/tmp/hsperfdata_root/* " +
"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/classes.jsa"
)
-PYTHON_CLEANUP_COMMANDS=(
- "rm -rf " +
- "/usr/lib/python3.5/__pycache__/ " +
- "/usr/lib/python3.5/idlelib/__pycache__/ " +
- "/usr/lib/python3.5/asyncio/__pycache__/ " +
- "/usr/lib/python3.5/json/__pycache__/ " +
- "/usr/lib/python3/dist-packages/numpy/testing/__pycache__/"
+
+PYTHON_CLEANUP_COMMANDS = (
+ "find /usr/lib/python* -name __pycache__ -exec rm -rf {} +"
)
toolchain_container(
@@ -117,33 +113,31 @@
language_tool_layer(
name = "python-ltl",
+ additional_repos = [
+ "deb http://httpredir.debian.org/debian testing main",
+ ],
base = "@debian9//image",
- env = {
- "PATH": "$PATH:/opt/python3.6/bin",
- },
installation_cleanup_commands = PYTHON_CLEANUP_COMMANDS,
packages = [
"python-dev",
"python-numpy",
"python-pip",
- "python3-dev",
+ "python-setuptools",
+ "python3.6",
+ "python3.6-dev",
"python3-numpy",
"python3-pip",
+ "python3-setuptools",
],
symlinks = {
"/usr/bin/python": "/usr/bin/python2.7",
- "/opt/python3.6/bin/python3": "/opt/python3.6/bin/python3.6",
- "/opt/python3.6/bin/pip3": "/opt/python3.6/bin/pip3.6",
},
- tars = ["//third_party/python:tar"],
)
container_test(
name = "toolchain-test",
configs = [
- "//container/test:common.yaml",
- # TODO: find a better way for testing distro version
- # "//container/test:rbe-debian9.yaml",
+ "//container/test:rbe-debian9.yaml",
],
image = ":toolchain",
)
diff --git a/container/experimental/rbe-ubuntu16_04/BUILD b/container/experimental/rbe-ubuntu16_04/BUILD
index 1a9f6ab..ca70491 100644
--- a/container/experimental/rbe-ubuntu16_04/BUILD
+++ b/container/experimental/rbe-ubuntu16_04/BUILD
@@ -23,17 +23,15 @@
)
load("@io_bazel_rules_docker//contrib:test.bzl", "container_test")
-JAVA_CLEANUP_COMMANDS=(
+JAVA_CLEANUP_COMMANDS = (
"rm -rf " +
"/etc/ssl/certs/java/cacerts " +
"/tmp/hsperfdata_root/* " +
"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/classes.jsa"
)
-PYTHON_CLEANUP_COMMANDS=(
- "rm -rf " +
- "/usr/lib/python3.5/__pycache__/ " +
- "/usr/lib/python3.5/idlelib/__pycache__/ " +
- "/usr/lib/python3.5/asyncio/__pycache__/"
+
+PYTHON_CLEANUP_COMMANDS = (
+ "find /usr/lib/python* -name __pycache__ -exec rm -rf {} +"
)
toolchain_container(
@@ -121,24 +119,25 @@
"python-dev",
"python-numpy",
"python-pip",
+ "python-setuptools",
"python3-dev",
"python3-numpy",
"python3-pip",
+ "python3-setuptools",
],
symlinks = {
"/usr/bin/python": "/usr/bin/python2.7",
"/opt/python3.6/bin/python3": "/opt/python3.6/bin/python3.6",
"/opt/python3.6/bin/pip3": "/opt/python3.6/bin/pip3.6",
+ "/opt/python3.6/bin/easy_install3": "/opt/python3.6/bin/easy_install-3.6",
},
- tars = ["//third_party/python:tar"],
+ tars = ["//third_party/python:ubuntu16_04_tar"],
)
container_test(
name = "toolchain-test",
configs = [
- "//container/test:common.yaml",
- # TODO: find a better way for testing distro version
- # "//container/test:rbe-ubuntu16_04.yaml",
+ "//container/test:rbe-ubuntu16_04.yaml",
],
image = ":toolchain",
)
diff --git a/container/rbe-debian8/BUILD b/container/rbe-debian8/BUILD
index 644c8f6..79a2efc 100644
--- a/container/rbe-debian8/BUILD
+++ b/container/rbe-debian8/BUILD
@@ -23,17 +23,15 @@
)
load("@io_bazel_rules_docker//contrib:test.bzl", "container_test")
-JAVA_CLEANUP_COMMANDS=(
+JAVA_CLEANUP_COMMANDS = (
"rm -rf " +
"/etc/ssl/certs/java/cacerts " +
"/tmp/hsperfdata_root/* " +
"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/classes.jsa"
)
-PYTHON_CLEANUP_COMMANDS=(
- "rm -rf " +
- "/usr/lib/python3.4/__pycache__/ " +
- "/usr/lib/python3.4/idlelib/__pycache__/ " +
- "/usr/lib/python3.4/asyncio/__pycache__/"
+
+PYTHON_CLEANUP_COMMANDS = (
+ "find /usr/lib/python* -name __pycache__ -exec rm -rf {} +"
)
toolchain_container(
@@ -124,24 +122,25 @@
"python-dev",
"python-numpy",
"python-pip",
+ "python-setuptools",
"python3-dev",
"python3-numpy",
"python3-pip",
+ "python3-setuptools",
],
symlinks = {
"/usr/bin/python": "/usr/bin/python2.7",
"/opt/python3.6/bin/python3": "/opt/python3.6/bin/python3.6",
"/opt/python3.6/bin/pip3": "/opt/python3.6/bin/pip3.6",
+ "/opt/python3.6/bin/easy_install3": "/opt/python3.6/bin/easy_install-3.6",
},
- tars = ["//third_party/python:tar"],
+ tars = ["//third_party/python:debian8_tar"],
)
container_test(
name = "toolchain-test",
configs = [
- "//container/test:common.yaml",
- # TODO: find a better way for testing distro version
- # "//container/test:rbe-debian8.yaml",
+ "//container/test:rbe-debian8.yaml",
],
image = ":toolchain",
)
diff --git a/container/test/common.yaml b/container/test/common.yaml
deleted file mode 100644
index eb7943c..0000000
--- a/container/test/common.yaml
+++ /dev/null
@@ -1,125 +0,0 @@
-schemaVersion: "1.0.0"
-
-commandTests:
-- name: 'path-envvar'
- command: ['sh', '-c', 'echo $PATH']
- expectedOutput: ['/opt/python3.6/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin']
-- name: 'cc-envvar'
- command: ['sh', '-c', 'echo $CC']
- expectedOutput: ['/usr/local/bin/clang']
-- name: 'javahome-envvar'
- command: ['sh', '-c', 'echo $JAVA_HOME']
- expectedOutput: ['/usr/lib/jvm/java-8-openjdk-amd64']
-- name: 'gopath-envvar'
- command: ['sh', '-c', 'echo $GOPATH']
- expectedOutput: ['/go']
-- name: 'clang-version'
- command: ['bash', '-c', 'clang --version']
- expectedOutput: ['clang version 7.0.0.*']
-- name: 'java-version'
- command: ['bash', '-c', 'java -version 2>&1']
- expectedOutput: ['openjdk version \"1.8.*']
-- name: 'python2-version'
- command: ['bash', '-c', 'python -V 2>&1']
- expectedOutput: ['Python 2.7.*']
-- name: 'python3-version'
- command: ['bash', '-c', 'python3 -V']
- expectedOutput: ['Python 3.6.2']
-- name: 'go-version'
- command: ['bash', '-c', 'go version']
- expectedOutput: ['go version go1.10 linux/amd64']
-- name: 'check-curl'
- command: ['bash', '-c', 'curl --version']
- expectedOutput: ['curl .* \(x86_64-pc-linux-gnu\).*']
-- name: 'check-ed'
- command: ['bash', '-c', 'ed --version']
- expectedOutput: ['GNU Ed .*']
-- name: 'check-file'
- command: ['bash', '-c', 'file --version']
- expectedOutput: ['file-.*']
-- name: 'check-git'
- command: ['bash', '-c', 'git --version']
- expectedOutput: ['git version .*']
-- name: 'check-openssl'
- command: ['bash', '-c', 'openssl version']
- expectedOutput: ['OpenSSL .*']
-- name: 'check-wget'
- command: ['bash', '-c', 'wget --version']
- expectedOutput: ['GNU Wget.* built on linux-gnu.*']
-- name: 'check-zip'
- command: ['bash', '-c', 'zip --version']
- expectedOutput: ['.*This is Zip.*']
-
-fileExistenceTests:
-- name: 'Root'
- isDirectory: true
- path: '/'
- shouldExist: true
-- name: 'Clang'
- isDirectory: false
- path: '/usr/local/bin/clang'
- shouldExist: true
-- name: 'OpenJDK'
- isDirectory: true
- path: '/usr/lib/jvm/java-8-openjdk-amd64'
- shouldExist: true
-- name: 'Python2'
- isDirectory: false
- path: '/usr/bin/python2.7'
- shouldExist: true
-- name: 'Python3'
- isDirectory: false
- path: '/opt/python3.6/bin/python3'
- shouldExist: true
-- name: 'Golang'
- isDirectory: false
- path: '/usr/local/go/bin/go'
- shouldExist: true
-- name: 'sanitizer'
- isDirectory: true
- path: '/usr/local/lib/clang/7.0.0/include/sanitizer'
- shouldExist: true
-- name: 'sanitizer-allocator'
- isDirectory: false
- path: '/usr/local/lib/clang/7.0.0/include/sanitizer/allocator_interface.h'
- shouldExist: true
-- name: 'sanitizer-asan'
- isDirectory: false
- path: '/usr/local/lib/clang/7.0.0/include/sanitizer/asan_interface.h'
- shouldExist: true
-- name: 'sanitizer-common_defs'
- isDirectory: false
- path: '/usr/local/lib/clang/7.0.0/include/sanitizer/common_interface_defs.h'
- shouldExist: true
-- name: 'sanitizer-coverage'
- isDirectory: false
- path: '/usr/local/lib/clang/7.0.0/include/sanitizer/coverage_interface.h'
- shouldExist: true
-- name: 'sanitizer-dfsan'
- isDirectory: false
- path: '/usr/local/lib/clang/7.0.0/include/sanitizer/dfsan_interface.h'
- shouldExist: true
-- name: 'sanitizer-esan'
- isDirectory: false
- path: '/usr/local/lib/clang/7.0.0/include/sanitizer/esan_interface.h'
- shouldExist: true
-- name: 'sanitizer-linux_syscall_hooks'
- isDirectory: false
- path: '/usr/local/lib/clang/7.0.0/include/sanitizer/linux_syscall_hooks.h'
- shouldExist: true
-- name: 'sanitizer-lsan'
- isDirectory: false
- path: '/usr/local/lib/clang/7.0.0/include/sanitizer/lsan_interface.h'
- shouldExist: true
-- name: 'sanitizer-msan'
- isDirectory: false
- path: '/usr/local/lib/clang/7.0.0/include/sanitizer/msan_interface.h'
- shouldExist: true
-- name: 'sanitizer-tsan_atomic'
- isDirectory: false
- path: '/usr/local/lib/clang/7.0.0/include/sanitizer/tsan_interface_atomic.h'
- shouldExist: true
-- name: 'sanitizer-tsan'
- isDirectory: false
- path: '/usr/local/lib/clang/7.0.0/include/sanitizer/tsan_interface.h'
- shouldExist: true
diff --git a/container/test/rbe-debian8.yaml b/container/test/rbe-debian8.yaml
index 5184d98..82a9181 100644
--- a/container/test/rbe-debian8.yaml
+++ b/container/test/rbe-debian8.yaml
@@ -1,6 +1,135 @@
schemaVersion: "1.0.0"
+# TODO: split out the common tests once resolve the CI issues
+
+# Distro-specific tests
commandTests:
- name: 'os-version'
command: ['sh', '-c', 'cat /etc/issue.net']
expectedOutput: ['Debian GNU/Linux 8']
+fileExistenceTests:
+- name: 'Python3'
+ isDirectory: false
+ path: '/opt/python3.6/bin/python3'
+ shouldExist: true
+
+# Common tests
+commandTests:
+- name: 'path-envvar'
+ command: ['sh', '-c', 'echo $PATH']
+ expectedOutput: ['/opt/python3.6/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin']
+- name: 'cc-envvar'
+ command: ['sh', '-c', 'echo $CC']
+ expectedOutput: ['/usr/local/bin/clang']
+- name: 'javahome-envvar'
+ command: ['sh', '-c', 'echo $JAVA_HOME']
+ expectedOutput: ['/usr/lib/jvm/java-8-openjdk-amd64']
+- name: 'gopath-envvar'
+ command: ['sh', '-c', 'echo $GOPATH']
+ expectedOutput: ['/go']
+- name: 'clang-version'
+ command: ['bash', '-c', 'clang --version']
+ expectedOutput: ['clang version 7.0.0.*']
+- name: 'java-version'
+ command: ['bash', '-c', 'java -version 2>&1']
+ expectedOutput: ['openjdk version \"1.8.*']
+- name: 'python2-version'
+ command: ['bash', '-c', 'python -V 2>&1']
+ expectedOutput: ['Python 2.7.*']
+- name: 'python3-version'
+ command: ['bash', '-c', 'python3 -V']
+ expectedOutput: ['Python 3.6.*']
+- name: 'go-version'
+ command: ['bash', '-c', 'go version']
+ expectedOutput: ['go version go1.10 linux/amd64']
+- name: 'check-curl'
+ command: ['bash', '-c', 'curl --version']
+ expectedOutput: ['curl .* \(x86_64-pc-linux-gnu\).*']
+- name: 'check-ed'
+ command: ['bash', '-c', 'ed --version']
+ expectedOutput: ['GNU [eE]d .*']
+- name: 'check-file'
+ command: ['bash', '-c', 'file --version']
+ expectedOutput: ['file-.*']
+- name: 'check-git'
+ command: ['bash', '-c', 'git --version']
+ expectedOutput: ['git version .*']
+- name: 'check-openssl'
+ command: ['bash', '-c', 'openssl version']
+ expectedOutput: ['OpenSSL .*']
+- name: 'check-wget'
+ command: ['bash', '-c', 'wget --version']
+ expectedOutput: ['GNU Wget.* built on linux-gnu.*']
+- name: 'check-zip'
+ command: ['bash', '-c', 'zip --version']
+ expectedOutput: ['.*This is Zip.*']
+
+fileExistenceTests:
+- name: 'Root'
+ isDirectory: true
+ path: '/'
+ shouldExist: true
+- name: 'Clang'
+ isDirectory: false
+ path: '/usr/local/bin/clang'
+ shouldExist: true
+- name: 'OpenJDK'
+ isDirectory: true
+ path: '/usr/lib/jvm/java-8-openjdk-amd64'
+ shouldExist: true
+- name: 'Python2'
+ isDirectory: false
+ path: '/usr/bin/python2.7'
+ shouldExist: true
+- name: 'Golang'
+ isDirectory: false
+ path: '/usr/local/go/bin/go'
+ shouldExist: true
+- name: 'sanitizer'
+ isDirectory: true
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer'
+ shouldExist: true
+- name: 'sanitizer-allocator'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/allocator_interface.h'
+ shouldExist: true
+- name: 'sanitizer-asan'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/asan_interface.h'
+ shouldExist: true
+- name: 'sanitizer-common_defs'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/common_interface_defs.h'
+ shouldExist: true
+- name: 'sanitizer-coverage'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/coverage_interface.h'
+ shouldExist: true
+- name: 'sanitizer-dfsan'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/dfsan_interface.h'
+ shouldExist: true
+- name: 'sanitizer-esan'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/esan_interface.h'
+ shouldExist: true
+- name: 'sanitizer-linux_syscall_hooks'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/linux_syscall_hooks.h'
+ shouldExist: true
+- name: 'sanitizer-lsan'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/lsan_interface.h'
+ shouldExist: true
+- name: 'sanitizer-msan'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/msan_interface.h'
+ shouldExist: true
+- name: 'sanitizer-tsan_atomic'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/tsan_interface_atomic.h'
+ shouldExist: true
+- name: 'sanitizer-tsan'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/tsan_interface.h'
+ shouldExist: true
diff --git a/container/test/rbe-debian9.yaml b/container/test/rbe-debian9.yaml
index 3f97c6b..2a04453 100644
--- a/container/test/rbe-debian9.yaml
+++ b/container/test/rbe-debian9.yaml
@@ -1,6 +1,135 @@
schemaVersion: "1.0.0"
+# TODO: split out the common tests once resolve the CI issues
+
+# Distro-specific tests
commandTests:
- name: 'os-version'
command: ['sh', '-c', 'cat /etc/issue.net']
expectedOutput: ['Debian GNU/Linux 9']
+fileExistenceTests:
+- name: 'Python3'
+ isDirectory: false
+ path: '/usr/bin/python3'
+ shouldExist: true
+
+# Common tests
+commandTests:
+- name: 'path-envvar'
+ command: ['sh', '-c', 'echo $PATH']
+ expectedOutput: ['/opt/python3.6/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin']
+- name: 'cc-envvar'
+ command: ['sh', '-c', 'echo $CC']
+ expectedOutput: ['/usr/local/bin/clang']
+- name: 'javahome-envvar'
+ command: ['sh', '-c', 'echo $JAVA_HOME']
+ expectedOutput: ['/usr/lib/jvm/java-8-openjdk-amd64']
+- name: 'gopath-envvar'
+ command: ['sh', '-c', 'echo $GOPATH']
+ expectedOutput: ['/go']
+- name: 'clang-version'
+ command: ['bash', '-c', 'clang --version']
+ expectedOutput: ['clang version 7.0.0.*']
+- name: 'java-version'
+ command: ['bash', '-c', 'java -version 2>&1']
+ expectedOutput: ['openjdk version \"1.8.*']
+- name: 'python2-version'
+ command: ['bash', '-c', 'python -V 2>&1']
+ expectedOutput: ['Python 2.7.*']
+- name: 'python3-version'
+ command: ['bash', '-c', 'python3 -V']
+ expectedOutput: ['Python 3.6.*']
+- name: 'go-version'
+ command: ['bash', '-c', 'go version']
+ expectedOutput: ['go version go1.10 linux/amd64']
+- name: 'check-curl'
+ command: ['bash', '-c', 'curl --version']
+ expectedOutput: ['curl .* \(x86_64-pc-linux-gnu\).*']
+- name: 'check-ed'
+ command: ['bash', '-c', 'ed --version']
+ expectedOutput: ['GNU [eE]d .*']
+- name: 'check-file'
+ command: ['bash', '-c', 'file --version']
+ expectedOutput: ['file-.*']
+- name: 'check-git'
+ command: ['bash', '-c', 'git --version']
+ expectedOutput: ['git version .*']
+- name: 'check-openssl'
+ command: ['bash', '-c', 'openssl version']
+ expectedOutput: ['OpenSSL .*']
+- name: 'check-wget'
+ command: ['bash', '-c', 'wget --version']
+ expectedOutput: ['GNU Wget.* built on linux-gnu.*']
+- name: 'check-zip'
+ command: ['bash', '-c', 'zip --version']
+ expectedOutput: ['.*This is Zip.*']
+
+fileExistenceTests:
+- name: 'Root'
+ isDirectory: true
+ path: '/'
+ shouldExist: true
+- name: 'Clang'
+ isDirectory: false
+ path: '/usr/local/bin/clang'
+ shouldExist: true
+- name: 'OpenJDK'
+ isDirectory: true
+ path: '/usr/lib/jvm/java-8-openjdk-amd64'
+ shouldExist: true
+- name: 'Python2'
+ isDirectory: false
+ path: '/usr/bin/python2.7'
+ shouldExist: true
+- name: 'Golang'
+ isDirectory: false
+ path: '/usr/local/go/bin/go'
+ shouldExist: true
+- name: 'sanitizer'
+ isDirectory: true
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer'
+ shouldExist: true
+- name: 'sanitizer-allocator'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/allocator_interface.h'
+ shouldExist: true
+- name: 'sanitizer-asan'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/asan_interface.h'
+ shouldExist: true
+- name: 'sanitizer-common_defs'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/common_interface_defs.h'
+ shouldExist: true
+- name: 'sanitizer-coverage'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/coverage_interface.h'
+ shouldExist: true
+- name: 'sanitizer-dfsan'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/dfsan_interface.h'
+ shouldExist: true
+- name: 'sanitizer-esan'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/esan_interface.h'
+ shouldExist: true
+- name: 'sanitizer-linux_syscall_hooks'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/linux_syscall_hooks.h'
+ shouldExist: true
+- name: 'sanitizer-lsan'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/lsan_interface.h'
+ shouldExist: true
+- name: 'sanitizer-msan'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/msan_interface.h'
+ shouldExist: true
+- name: 'sanitizer-tsan_atomic'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/tsan_interface_atomic.h'
+ shouldExist: true
+- name: 'sanitizer-tsan'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/tsan_interface.h'
+ shouldExist: true
diff --git a/container/test/rbe-ubuntu16_04.yaml b/container/test/rbe-ubuntu16_04.yaml
index 41eb4d2..62c1379 100644
--- a/container/test/rbe-ubuntu16_04.yaml
+++ b/container/test/rbe-ubuntu16_04.yaml
@@ -1,6 +1,135 @@
schemaVersion: "1.0.0"
+# TODO: split out the common tests once resolve the CI issues
+
+# Distro-specific tests
commandTests:
- name: 'os-version'
command: ['sh', '-c', 'cat /etc/issue.net']
expectedOutput: ['Ubuntu 16.04.3 LTS']
+fileExistenceTests:
+- name: 'Python3'
+ isDirectory: false
+ path: '/opt/python3.6/bin/python3'
+ shouldExist: true
+
+# Common tests
+commandTests:
+- name: 'path-envvar'
+ command: ['sh', '-c', 'echo $PATH']
+ expectedOutput: ['/opt/python3.6/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin']
+- name: 'cc-envvar'
+ command: ['sh', '-c', 'echo $CC']
+ expectedOutput: ['/usr/local/bin/clang']
+- name: 'javahome-envvar'
+ command: ['sh', '-c', 'echo $JAVA_HOME']
+ expectedOutput: ['/usr/lib/jvm/java-8-openjdk-amd64']
+- name: 'gopath-envvar'
+ command: ['sh', '-c', 'echo $GOPATH']
+ expectedOutput: ['/go']
+- name: 'clang-version'
+ command: ['bash', '-c', 'clang --version']
+ expectedOutput: ['clang version 7.0.0.*']
+- name: 'java-version'
+ command: ['bash', '-c', 'java -version 2>&1']
+ expectedOutput: ['openjdk version \"1.8.*']
+- name: 'python2-version'
+ command: ['bash', '-c', 'python -V 2>&1']
+ expectedOutput: ['Python 2.7.*']
+- name: 'python3-version'
+ command: ['bash', '-c', 'python3 -V']
+ expectedOutput: ['Python 3.6.*']
+- name: 'go-version'
+ command: ['bash', '-c', 'go version']
+ expectedOutput: ['go version go1.10 linux/amd64']
+- name: 'check-curl'
+ command: ['bash', '-c', 'curl --version']
+ expectedOutput: ['curl .* \(x86_64-pc-linux-gnu\).*']
+- name: 'check-ed'
+ command: ['bash', '-c', 'ed --version']
+ expectedOutput: ['GNU [eE]d .*']
+- name: 'check-file'
+ command: ['bash', '-c', 'file --version']
+ expectedOutput: ['file-.*']
+- name: 'check-git'
+ command: ['bash', '-c', 'git --version']
+ expectedOutput: ['git version .*']
+- name: 'check-openssl'
+ command: ['bash', '-c', 'openssl version']
+ expectedOutput: ['OpenSSL .*']
+- name: 'check-wget'
+ command: ['bash', '-c', 'wget --version']
+ expectedOutput: ['GNU Wget.* built on linux-gnu.*']
+- name: 'check-zip'
+ command: ['bash', '-c', 'zip --version']
+ expectedOutput: ['.*This is Zip.*']
+
+fileExistenceTests:
+- name: 'Root'
+ isDirectory: true
+ path: '/'
+ shouldExist: true
+- name: 'Clang'
+ isDirectory: false
+ path: '/usr/local/bin/clang'
+ shouldExist: true
+- name: 'OpenJDK'
+ isDirectory: true
+ path: '/usr/lib/jvm/java-8-openjdk-amd64'
+ shouldExist: true
+- name: 'Python2'
+ isDirectory: false
+ path: '/usr/bin/python2.7'
+ shouldExist: true
+- name: 'Golang'
+ isDirectory: false
+ path: '/usr/local/go/bin/go'
+ shouldExist: true
+- name: 'sanitizer'
+ isDirectory: true
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer'
+ shouldExist: true
+- name: 'sanitizer-allocator'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/allocator_interface.h'
+ shouldExist: true
+- name: 'sanitizer-asan'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/asan_interface.h'
+ shouldExist: true
+- name: 'sanitizer-common_defs'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/common_interface_defs.h'
+ shouldExist: true
+- name: 'sanitizer-coverage'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/coverage_interface.h'
+ shouldExist: true
+- name: 'sanitizer-dfsan'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/dfsan_interface.h'
+ shouldExist: true
+- name: 'sanitizer-esan'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/esan_interface.h'
+ shouldExist: true
+- name: 'sanitizer-linux_syscall_hooks'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/linux_syscall_hooks.h'
+ shouldExist: true
+- name: 'sanitizer-lsan'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/lsan_interface.h'
+ shouldExist: true
+- name: 'sanitizer-msan'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/msan_interface.h'
+ shouldExist: true
+- name: 'sanitizer-tsan_atomic'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/tsan_interface_atomic.h'
+ shouldExist: true
+- name: 'sanitizer-tsan'
+ isDirectory: false
+ path: '/usr/local/lib/clang/7.0.0/include/sanitizer/tsan_interface.h'
+ shouldExist: true
diff --git a/skylib/container_file_export.sh b/skylib/container_file_export.sh
index b2b0b1a..53053b5 100755
--- a/skylib/container_file_export.sh
+++ b/skylib/container_file_export.sh
@@ -23,7 +23,7 @@
#!/usr/bin/env bash
-set -e
+set -ex
main() {
if [ "$#" -ne 3 ]; then
@@ -34,7 +34,10 @@
SOURCE=$2
TARGET=$3
- container_name="data-container-$RANDOM"
+ # On Bazel CI $RANDOM somehow doesn't work. Use another way for generating a random number
+ # TODO: change back to use $RANDOM once it is available on Bazel CI.
+ random_number=$(python -c "import random; print random.randint(1, 1024)")
+ container_name="data-container-${random_number}"
docker run -t -d --name ${container_name} $IMAGE sleep infinity
docker exec -e GZIP=-n ${container_name} tar -czf /tmp/data.tar.gz --mtime='1970-01-01' $SOURCE
docker cp ${container_name}:/tmp/data.tar.gz $TARGET
diff --git a/third_party/python/BUILD b/third_party/python/BUILD
index ba33b51..2cdb940 100644
--- a/third_party/python/BUILD
+++ b/third_party/python/BUILD
@@ -26,7 +26,7 @@
load("//skylib:container_file_export.bzl", "container_file_export")
pkg_tar(
- name = "tar",
+ name = "debian8_tar",
srcs = glob(
["**/*"],
exclude = ["**/BUILD"],
@@ -34,14 +34,37 @@
package_dir = "/",
strip_prefix = ".",
tags = ["manual"],
- deps = [":python3"],
+ deps = [":debian8_python3"],
)
-# Get python3 interpreter from the latest l.gcr.io/google/python container
-# https://github.com/GoogleCloudPlatform/python-runtime/tree/master/python-interpreter-builder
+pkg_tar(
+ name = "ubuntu16_04_tar",
+ srcs = glob(
+ ["**/*"],
+ exclude = ["**/BUILD"],
+ ),
+ package_dir = "/",
+ strip_prefix = ".",
+ tags = ["manual"],
+ deps = [":ubuntu16_04_python3"],
+)
+
+# Get debian8-built python3 interpreter from l.gcr.io/google/python container
+# Base image: gcr.io/google-appengine/debian8:latest
+# https://github.com/GoogleCloudPlatform/python-runtime/blob/a8a3e8b2d3239c184843db818e34a06f12dc1190/build.sh#L155
container_file_export(
- name = "python3",
+ name = "debian8_python3",
image = "l.gcr.io/google/python@sha256:de9ab4346c65a611b1721aeae0633d897f56861a0cda0b40bc032771007ac914",
src_path = "/opt/python3.6",
tags = ["manual"],
)
+
+# Get ubuntu16_04-built python3 interpreter from gcr.io/google-appengine/python:staging
+# Base image: gcr.io/gcp-runtimes/ubuntu_16_0_4:latest
+# https://github.com/GoogleCloudPlatform/python-runtime/blob/a8a3e8b2d3239c184843db818e34a06f12dc1190/build.sh#L153
+container_file_export(
+ name = "ubuntu16_04_python3",
+ image = "gcr.io/google-appengine/python@sha256:e720c0e853697373efe5e3eed6f4db113f9291327a6f169feb54271fe364b40b",
+ src_path = "/opt/python3.6",
+ tags = ["manual"],
+)
