Add BazelConfiguration groovy class
This is just a convenience class to read configuration from
hierarchical json descriptor and return a list of configuration
to execute. Each configuration contains a descriptor (key=value pair,
e.g. the node label to run on) and a set of configuration parameters.
This change also add a BUILD file and some test to the groovy library
so we need to change the way we ship the library inside the docker
container.
Change-Id: I5d6a2090b4360a0497b96e466df65ee52d19bbdb
diff --git a/WORKSPACE b/WORKSPACE
index a868c1f..8c704fd 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -103,3 +103,29 @@
remote = "https://github.com/google/python-gflags",
tag = "python-gflags-2.0",
)
+
+# Testing Jenkins pipeline library
+# TODO(dmarting): the groovy support is really rudimentary we should fix it:
+# - Need for adding more dependency
+# - Groovy test absolutely want you to declare a specific structure
+# - The release is not working with latest bazel
+# - Repository overrely on bind() and does not respect naming conventions
+http_archive(
+ name = "io_bazel_rules_groovy",
+ url = "https://github.com/bazelbuild/rules_groovy/archive/1256063915da0e46c229ce93489175f0d084f0cb.zip",
+ sha256 = "0e7a425e3ebcc649c4fac9876ebbfdb3948bf97c47954ffda3e15b32a5bf4b6e",
+ strip_prefix = "rules_groovy-1256063915da0e46c229ce93489175f0d084f0cb",
+)
+load("@io_bazel_rules_groovy//groovy:groovy.bzl", "groovy_repositories")
+groovy_repositories()
+
+# For groovy tests
+maven_jar(
+ name = "org_codehaus_groovy_all",
+ artifact = "org.codehaus.groovy:groovy-all:jar:2.4.4",
+)
+
+maven_jar(
+ name = "org_hamcrest",
+ artifact = "org.hamcrest:hamcrest-all:jar:1.3",
+)
diff --git a/jenkins/BUILD b/jenkins/BUILD
index 727fea9..4124a2e 100644
--- a/jenkins/BUILD
+++ b/jenkins/BUILD
@@ -301,6 +301,7 @@
"SECURITY_CONFIG": SECURITY_CONFIG,
"PUBLIC_JENKINS_URL": "http://ci.bazel.io/",
} + JOBS_SUBSTITUTIONS,
+ tars = ["//jenkins/lib"],
visibility = ["//visibility:public"],
)
@@ -325,6 +326,7 @@
"SECURITY_CONFIG": SECURITY_CONFIG,
"PUBLIC_JENKINS_URL": "http://ci-staging.bazel.io/",
} + STAGING_JOBS_SUBSTITUTIONS,
+ tars = ["//jenkins/lib"],
visibility = ["//gcr:__pkg__"],
)
@@ -345,6 +347,7 @@
"SECURITY_CONFIG": "<useSecurity>false</useSecurity>",
"PUBLIC_JENKINS_URL": "##ENV:JENKINS_SERVER##",
} + JOBS_SUBSTITUTIONS,
+ tars = ["//jenkins/lib"],
)
filegroup(
diff --git a/jenkins/jenkins.bzl b/jenkins/jenkins.bzl
index 30ec2c8..6410d31 100644
--- a/jenkins/jenkins.bzl
+++ b/jenkins/jenkins.bzl
@@ -400,7 +400,7 @@
)
def jenkins_build(name, plugins = None, base = "//jenkins/base", configs = [],
- jobs = [], substitutions = {}, visibility = None):
+ jobs = [], substitutions = {}, visibility = None, tars = []):
"""Build the docker image for the Jenkins instance."""
substitutions = substitutions + MAILS_SUBSTITUTIONS
# Expands config files in a tar ball
@@ -422,22 +422,13 @@
directory = "/usr/share/jenkins/ref",
)
- # Create a tar of the library files
- pkg_tar(
- name = "%s-lib" % name,
- files = native.glob(["lib/**"]),
- strip_prefix = "lib",
- package_dir = "/opt/lib",
- )
-
### FINAL IMAGE ###
docker_build(
name = name,
tars = [
":%s-jobs" % name,
":%s-configs" % name,
- ":%s-lib" % name,
- ],
+ ] + tars,
# Workaround no way to specify owner in pkg_tar
# TODO(dmarting): use https://cr.bazel.build/10255 when it hits a release.
user = "root",
diff --git a/jenkins/lib/BUILD b/jenkins/lib/BUILD
new file mode 100644
index 0000000..b15b2f1
--- /dev/null
+++ b/jenkins/lib/BUILD
@@ -0,0 +1,33 @@
+package(default_visibility = ["//jenkins:__subpackages__"])
+
+load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
+load("@io_bazel_rules_groovy//groovy:groovy.bzl", "groovy_library")
+
+pkg_tar(
+ name = "lib",
+ files = glob([
+ "src/build/**",
+ "vars/**",
+ ]),
+ package_dir = "/opt/lib",
+ strip_prefix = ".",
+)
+
+groovy_library(
+ name = "BazelConfigurationTests",
+ srcs = [
+ "src/build/bazel/ci/BazelConfiguration.groovy",
+ "tests/build/bazel/ci/BazelConfigurationTests.groovy",
+ ],
+)
+
+java_test(
+ name = "BazelConfigurationTests-test",
+ size = "small",
+ runtime_deps = [
+ "@org_codehaus_groovy_all//jar",
+ "@org_hamcrest//jar",
+ ":BazelConfigurationTests",
+ ],
+ test_class = "build.bazel.ci.BazelConfigurationTests",
+)
diff --git a/jenkins/lib/src/build/bazel/ci/BazelConfiguration.groovy b/jenkins/lib/src/build/bazel/ci/BazelConfiguration.groovy
new file mode 100644
index 0000000..148dadb
--- /dev/null
+++ b/jenkins/lib/src/build/bazel/ci/BazelConfiguration.groovy
@@ -0,0 +1,187 @@
+// Copyright (C) 2017 The Bazel Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build.bazel.ci
+
+import groovy.json.JsonSlurper
+import groovy.json.JsonParserType
+
+/**
+ * A class that stores configuration for a Bazel job.
+ *
+ * A configuration is composed of a descriptor and a list of parameters.
+ * A descriptor is a key-value map describing the configuration itself and
+ * parameters are parameters for the job common to all configurations but for
+ * which the value is configuration specific. E.g. test options or targets to
+ * build.
+ *
+ * A configuration can contains several other configurations. In which case the
+ * child configurations get factored with the parent configuration to create N
+ * configurations that inherit the parameters and descriptor of the parent
+ * configuration. If a child configuration specify a value already present in the
+ * parent configuration, the parent configuration value will be ignored and the child
+ * configuration value will be used.
+ *
+ * Example:
+ * BazelConfiguration(["descriptor": "yeah"],
+ * ["params1": false],
+ * [BazelConfiguration(["descriptor2": "a"], [], [params2: true]),
+ * BazelConfiguration(["descriptor2": "b"], [], [params1: true, params2: false])])
+ *
+ * would expand to the following configurations:
+ *
+ * BazelConfiguration(["descriptor": "yeah", "descriptor2": "a"], [params1: false, params2: true])])
+ * BazelConfiguration(["descriptor": "yeah", "descriptor2": "b"], [params1: true, params2: false])])
+ */
+class BazelConfiguration implements java.io.Serializable {
+ private Map<String, String> descriptor
+ private Map<String, Object> parameters
+ private List<BazelConfiguration> configurations
+
+ static public interface ConfigurationContainer {
+ def addConfiguration(BazelConfiguration)
+ }
+
+ private static List<BazelConfiguration> parseJson(Object json) {
+ List<BazelConfiguration> result = []
+ for (Object o in json) {
+ result.add(new BazelConfiguration(o))
+ }
+ return result
+ }
+
+ private static Object toSerializable(Object jsonObject) {
+ // JsonSlurper map and list are not serializable, which make them unsuitable for
+ // usage inside a Jenkins pipeline, convert them to HashMap and ArrayList
+ if (jsonObject instanceof Map) {
+ def result = [:]
+ for (e in jsonObject) {
+ result[e.key] = toSerializable(e.value)
+ }
+ return result
+ } else if (jsonObject instanceof List) {
+ def result = []
+ for (it in jsonObject) {
+ result.add(toSerializable(it))
+ }
+ return result
+ } else {
+ return jsonObject
+ }
+ }
+
+ /**
+ * Parse a list of configurations for a JSON string.
+ * A JSON configuration is an object whose key -> values are the entries
+ * of the configuration descriptor, except for "configurations" and "parameters"
+ * keys which correspond respectively to the child configurations and the parameters.
+ */
+ public static List<BazelConfiguration> parse(String jsonString) {
+ // There is a subtle bug in the parser so skip the first comment line
+ while (jsonString.trim().startsWith("//")) {
+ def pos = jsonString.indexOf("\n")
+ jsonString = pos < 0 ? "" : jsonString.substring(pos + 1)
+ }
+ def jsonObject = new JsonSlurper().setType(JsonParserType.LAX).parseText(jsonString);
+ return parseJson(toSerializable(jsonObject))
+ }
+
+ /** Parse a list of configurations from a JSON file. */
+ public static List<BazelConfiguration> parse(File jsonFile) {
+ return parse(jsonFile.text)
+ }
+
+ /**
+ * Flatten a list of configurations into a map of descriptor -> parameters.
+ *
+ * Restrictions can be applied to configuration to select only configuration compatible with
+ * the environment we run on (e.g. testing environment inside docker has only linux slave available).
+ * The restrictions are specified by a map of descriptor key-values. A configuration will be selected
+ * only if, for any key k in the descriptor, either the k is not a key of configurationRestrictions
+ * or the value of the descriptor is in configurationRestrictions[k].
+ *
+ * Examples:
+ * - configurationRestrictions = ["a": ["a", "b"], "b": []] would match descriptor
+ * ["a": "a"], ["a": "b"], ["c": "whatever"] but not ["a": "c"] nor ["b": "whatever"]
+ * - configurationRestrictions = ["node": ["linux-x86_64"]] would match only the descriptors that
+ * point to an execution on a linux node.
+ */
+ public static Map<Map<String, String>, Map<String, Object>> flattenConfigurations(
+ List<BazelConfiguration> configurations,
+ Map<String, List<String>> configurationRestrictions = [:]) {
+ def result = [:]
+ for (conf in configurations) {
+ result += conf.flatten(configurationRestrictions)
+ }
+ return result
+ }
+
+ private BazelConfiguration(Object json) {
+ this.parameters = ("parameters" in json) ? json["parameters"] : [:]
+ this.configurations = ("configurations" in json) ?
+ json["configurations"].collect { it -> new BazelConfiguration(it) } : []
+ this.descriptor = json.findAll { k, v -> k != "configurations" && k != "parameters" }
+ }
+
+ public BazelConfiguration(Map<String, String> descriptor,
+ Map<String, Object> parameters,
+ List<BazelConfiguration> configurations = []) {
+ this.descriptor = descriptor
+ this.parameters = parameters
+ this.configurations = configurations
+ }
+
+ def getDescriptor() {
+ return descriptor
+ }
+
+ def getParameters() {
+ return parameters
+ }
+
+ def getConfigurations() {
+ return configurations
+ }
+
+ private Map<Map<String, String>, Map<String, Object>> flatten(Map<String, List<String>> configurationRestrictions = [:]) {
+ Map<Map<String, String>, Map<String, Object>> result = [:]
+ if (descriptor.any {
+ k, v -> (k in configurationRestrictions) && !(v in configurationRestrictions[k]) }) {
+ return result
+ }
+
+ if (configurations.isEmpty()) {
+ if (!descriptor.isEmpty()) {
+ result[descriptor] = parameters
+ }
+ return result
+ } else {
+ for (conf in configurations) {
+ def configs = conf.flatten(configurationRestrictions)
+ for (e in configs) {
+ def descr2 = [:]
+ descr2.putAll(descriptor)
+ descr2.putAll(e.key)
+ result[descr2] = [:]
+ result[descr2].putAll(parameters)
+ result[descr2].putAll(e.value)
+ }
+ }
+ if (result.isEmpty() && !descriptor.isEmpty()) {
+ result[descriptor] = parameters
+ }
+ return result
+ }
+ }
+}
diff --git a/jenkins/lib/tests/build/bazel/ci/BazelConfigurationTests.groovy b/jenkins/lib/tests/build/bazel/ci/BazelConfigurationTests.groovy
new file mode 100644
index 0000000..0a670a7
--- /dev/null
+++ b/jenkins/lib/tests/build/bazel/ci/BazelConfigurationTests.groovy
@@ -0,0 +1,211 @@
+// Copyright (C) 2017 The Bazel Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build.bazel.ci
+
+import org.junit.Test
+
+/** Tests for {@link BazelConguration} */
+class BazelConfigurationTests {
+ static final String JSON_TEST = '''
+
+// This is a test
+// Double comment to test the workaround the parser issue
+
+// more comment
+// And now the initial bazel tests
+[
+ {
+ // This is a configuration that have 3 subconfiguration: linux, ubuntu and darwin
+ // Each of those configuration have 4 bazel variations: HEAD, HEAD-jdk7, latest,
+ // and latest-jdk7
+ "configurations": [
+ {
+ "node": "linux-x86_64",
+ "configurations": [
+ // XXX(dmarting): Remove HEAD from here.
+ {"variation": "HEAD"},
+ {"variation": "HEAD-jdk7"},
+ {"variation": "latest"},
+ {"variation": "latest-jdk7"}
+ ]
+ },
+ {
+ "node": "ubuntu_16.04-x86_64",
+ "configurations": [
+ {"variation": "HEAD"},
+ {"variation": "HEAD-jdk7"},
+ {"variation": "latest"},
+ {"variation": "latest-jdk7"}
+ ]
+ },
+ {
+ "node": "darwin-x86_64",
+ "configurations": [
+ {"variation": "HEAD"},
+ {"variation": "HEAD-jdk7"},
+ {"variation": "latest"},
+ {"variation": "latest-jdk7"}
+ ]
+ }
+ ],
+ // And specify the parameters for these configurations.
+ "parameters": {
+ "configure": [
+ "source scripts/ci/build.sh",
+ "setup_android_repositories"
+ ],
+ "test_opts": ["-k", "--build_tests_only"],
+ "tests": [
+ "//scripts/...",
+ "//src/...",
+ "//third_party/ijar/..."
+ ],
+ "targets": []
+ }
+ }, {
+ "toolchain": "msvc",
+ "configurations": [{
+ // XXX(dmarting): MSVC is a misnommer, it should have been called win32
+ // (for win32 native binary).
+ // XXX(dmarting): really MSVC/Win32 should be a bazel variation, not part of
+ // the node.
+ "node": "windows-msvc-x86_64",
+ "configurations": [{"variation": "HEAD"},{"variation": "latest"}]
+ }, {
+ "node": "windows-x86_64",
+ "configurations": [{"variation": "HEAD"},{"variation": "latest"}]
+ }],
+ "parameters": {
+ "test_opts": ["-k", "--build_tests_only"],
+ "tests": [
+ "//src/test/java/...",
+ "//src/test/cpp/...",
+ "//src/test/naive:all_tests"
+ ],
+ "targets": ["//src:bazel"]
+ }
+ }, {
+ "toolchain": "msys",
+ "configurations": [{
+ "node": "windows-msvc-x86_64",
+ "configurations": [{"variation": "HEAD"},{"variation": "latest"}]
+ }, {
+ "node": "windows-x86_64",
+ "configurations": [{"variation": "HEAD"},{"variation": "latest"}]
+ }],
+ "parameters": {
+ "test_opts": ["-k", "--build_tests_only"],
+ "tests": ["//src/tst/shell/bazel:bazel_windows_example_test"],
+ "targets": []
+ }
+ }
+]
+
+// Ending comment
+'''
+
+ private void assertConfigurationCorrect(confs) {
+ assert confs.size() == 3
+ assert confs[0].descriptor.size() == 0
+ assert confs[0].parameters.size() == 4
+ assert confs[0].parameters["configure"] == [
+ "source scripts/ci/build.sh",
+ "setup_android_repositories"
+ ]
+ assert confs[0].configurations.size() == 3
+ assert confs[0].configurations.every {
+ v -> v.configurations.size() == 4 && v.parameters.size() == 0 && v.descriptor.size() == 1 }
+ assert confs[1].descriptor.size() == 1
+ assert confs[1].parameters.size() == 3
+ assert confs[1].configurations.size() == 2
+ assert confs[1].configurations.every {
+ v -> v.configurations.size() == 2 && v.parameters.size() == 0 && v.descriptor.size() == 1 }
+ assert confs[2].descriptor.size() == 1
+ assert confs[2].parameters.size() == 3
+ assert confs[2].configurations.size() == 2
+ assert confs[2].configurations.every {
+ v -> v.configurations.size() == 2 && v.parameters.size() == 0 && v.descriptor.size() == 1 }
+ }
+
+ @Test
+ void testParseJsonString() {
+ assertConfigurationCorrect(BazelConfiguration.parse(JSON_TEST))
+ }
+
+ @Test
+ void testSerialization() {
+ // Just test that the object from parsing JSON is serializable
+ new ObjectOutputStream(new ByteArrayOutputStream()).writeObject(BazelConfiguration.parse(JSON_TEST));
+ }
+
+ @Test
+ void testParseJsonFile() {
+ def tempDir = System.getenv("TEST_TMPDIR")
+ def tempDirFile = null
+ if (tempDir != null) {
+ tempDirFile = new File(tempDir)
+ }
+ def testFile = File.createTempFile('temp', '.json', tempDirFile)
+ try {
+ testFile.write(JSON_TEST)
+ assertConfigurationCorrect(BazelConfiguration.parse(testFile))
+ } finally {
+ testFile.delete()
+ }
+ }
+
+ @Test
+ void testFlatten() {
+ def result = BazelConfiguration.flattenConfigurations(BazelConfiguration.parse(JSON_TEST))
+ def allKeys = result.collect {
+ k, v -> k.collect { k1, v1 -> "${k1}=${v1}" }.toSorted().join(",") }.toSorted()
+ // Once flatten there are 20 configurations
+ assert allKeys.size() == 20
+ assert allKeys.join("\n") == '''node=darwin-x86_64,variation=HEAD
+node=darwin-x86_64,variation=HEAD-jdk7
+node=darwin-x86_64,variation=latest
+node=darwin-x86_64,variation=latest-jdk7
+node=linux-x86_64,variation=HEAD
+node=linux-x86_64,variation=HEAD-jdk7
+node=linux-x86_64,variation=latest
+node=linux-x86_64,variation=latest-jdk7
+node=ubuntu_16.04-x86_64,variation=HEAD
+node=ubuntu_16.04-x86_64,variation=HEAD-jdk7
+node=ubuntu_16.04-x86_64,variation=latest
+node=ubuntu_16.04-x86_64,variation=latest-jdk7
+node=windows-msvc-x86_64,toolchain=msvc,variation=HEAD
+node=windows-msvc-x86_64,toolchain=msvc,variation=latest
+node=windows-msvc-x86_64,toolchain=msys,variation=HEAD
+node=windows-msvc-x86_64,toolchain=msys,variation=latest
+node=windows-x86_64,toolchain=msvc,variation=HEAD
+node=windows-x86_64,toolchain=msvc,variation=latest
+node=windows-x86_64,toolchain=msys,variation=HEAD
+node=windows-x86_64,toolchain=msys,variation=latest'''
+ }
+
+ @Test
+ void testFlattenWithRestriction() {
+ def result = BazelConfiguration.flattenConfigurations(
+ BazelConfiguration.parse(JSON_TEST),
+ [node: ["linux-x86_64", "windows-x86_64"], variation: ["HEAD", "HEAD-jdk7"]])
+ def allKeys = result.collect {
+ k, v -> k.collect { k1, v1 -> "${k1}=${v1}" }.toSorted().join(",") }.toSorted()
+ assert allKeys.size() == 4
+ assert allKeys.join("\n") == '''node=linux-x86_64,variation=HEAD
+node=linux-x86_64,variation=HEAD-jdk7
+node=windows-x86_64,toolchain=msvc,variation=HEAD
+node=windows-x86_64,toolchain=msys,variation=HEAD'''
+ }
+}
