Add a global pipeline job
This job will be used to trigger a global test suite, suitable
to run a global pre/post-submit of Bazel paving the way for
global pre submit from Gerrit.
This job trigger all jobs in the "Global" folder but that folder
is still empty.
Missing for #55:
- Add job in the Global folder
- Trigger from Gerrit
- Trigger the new Global/pipeline job instead of the legacy pipeline
- Installation of Bazel releases automatically
Change-Id: I69937b1531061ca1575564777c539bd97edd90db
diff --git a/jenkins/jobs/BUILD b/jenkins/jobs/BUILD
index cae77c6..5039ac9 100644
--- a/jenkins/jobs/BUILD
+++ b/jenkins/jobs/BUILD
@@ -27,7 +27,16 @@
project_url = "http://bazel.io",
substitutions = JOBS_SUBSTITUTIONS,
deps = glob(["%s.*.tpl" % job]),
-) for job in BAZEL_JOBS.keys()]
+) for job in BAZEL_JOBS.keys() if job != "Global/pipeline"]
+
+jenkins_job(
+ name = "Global/pipeline",
+ config = "global.xml.tpl",
+ deps = [
+ ":global.groovy",
+ ":configs/bootstrap.json",
+ ],
+)
# TODO(dmarting): activate Tensorflow on mac (missing dependencies)
bazel_github_job(
diff --git a/jenkins/jobs/global.groovy b/jenkins/jobs/global.groovy
new file mode 100644
index 0000000..5517dd8
--- /dev/null
+++ b/jenkins/jobs/global.groovy
@@ -0,0 +1,134 @@
+// Copyright (C) 2017 The Bazel Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This pipeline control a global test for Bazel:
+// - Bootstrap bazel and do some basic tests
+// - Deploy the artifacts (site, releases) and send mails
+// - Run all downstream job
+
+stage("Startup global test") {
+ echo "Running global test for branch ${params.BRANCH} (refspec: ${params.REFSPEC})"
+}
+
+notifyStatus(mail_recipient) {
+ // First we bootstrap bazel on all platform
+ stage("Bootstrap on all platforms") {
+ bootstrapBazelAll(branch: params.BRANCH,
+ refspec: params.REFSPEC,
+ email: mail_recipient,
+ configuration: json_config,
+ restrict_configuration: restrict_configuration)
+ }
+
+ // Some basic tests
+ // TODO(dmarting): maybe we want to run it in parallel of other jobs?
+ stage("Test that all sources are in the //:srcs filegroup") {
+ node("linux-x86_64") {
+ recursiveGit(repository: "https://bazel.googlesource.com/bazel",
+ refspec: params.REFSPEC,
+ branch: params.BRANCH)
+ def bazel = fetchBazel("latest", "linux-x86_64")
+ sh(script: "./compile.sh srcs ${bazel}")
+ }
+ }
+}
+
+
+// Deployment steps
+if(params.BRANCH.matches('^(.*/)master$')) {
+ stage("Push website") {
+ node("deploy") {
+ recursiveGit(repository: "https://bazel.googlesource.com/bazel",
+ refspec: params.REFSPEC,
+ branch: params.BRANCH)
+ unstash "bazel--node=linux-x86_64--variation="
+ sh script: '''#!/bin/bash
+. scripts/ci/build.sh
+for i in $(find input -name \'*.bazel.build.tar\'); do
+ build_and_publish_site "$i" "$(basename $i .tar)" "build"
+done
+for i in $(find input -name \'*.bazel.build.tar.nobuild\'); do
+ build_and_publish_site "$i" "$(basename $i .tar.nobuild)" "nobuild"
+done
+'''
+ }
+ }
+} else if(params.BRANCH.matches('^refs/((heads/release-)|(tags/)).*$')) {
+ def r_name = ""
+ node("deploy") {
+ r_name = sh(script: "bash -c 'source scripts/release/common.sh; get_full_release_name'",
+ returnStdout: true)
+ if (!r_name.isEmpty()) {
+ stage("Push release") {
+ // unstash all the things
+ def conf = BazelConfiguration.flattenConfigurations(
+ BazelConfiguration.parse(json_config), restrict_configuration).keySet().toArray()
+ for (int k = 0; k < entrySet.size; k++) {
+ unstash "bazel--node=${conf[k].node}--variation=${conf[k].variation}"
+ }
+ // Delete files we do not need
+ sh "rm -f node=*/variation=*/bazel node=*/variation=*/*.bazel.build.tar*"
+ // Now the actual release
+ withEnv(["GCS_BUCKET=bazel",
+ "GIT_REPOSITORY_URL=https://github.com/bazelbuild/bazel"]) {
+ dir("output/ci") { -> }
+ sh '''#!/bin/bash
+# Credentials should not be displayed on the command line
+export GITHUB_TOKEN="$(cat "$GITHUB_TOKEN_FILE")"
+export APT_GPG_KEY_ID="$(cat "${APT_GPG_KEY_ID_FILE}")"
+source scripts/ci/build.sh
+
+args=()
+for i in node=*; do
+ for j in $i/variation=*/*; do
+ args+=("$(echo $i | cut -d = -f 2)" "$j")
+done
+
+bazel_release "${args[@]}"
+echo "${RELEASE_EMAIL_RECIPIENT}" | tee output/ci/recipient
+echo "${RELEASE_EMAIL_SUBJECT}" | tee output/ci/subject
+echo "${RELEASE_EMAIL_CONTENT}" | tee output/ci/content
+'''
+ if (r_name.contains("test")) {
+ echo "Test release, skipping announcement mail"
+ } else {
+ stage("Announcement mail") {
+ mail(subject: readFile("output/ci/subject"),
+ to: readFile("output/ci/recipient"),
+ replyTo: "bazel-ci@googlegroups.com",
+ body: readFile("output/ci/content"))
+ }
+ }
+ }
+ }
+ }
+ }
+ // TODO(dmarting): trigger bazel install everywhere in case of release.
+}
+
+// Then we run all the job in the Global folder but myself
+stage("Test downstream jobs") {
+ runAll(folder: "Global",
+ parameters: [
+ [$class: 'TextParameterValue',
+ name: 'EXTRA_BAZELRC',
+ value: "${params.EXTRA_BAZELRC}"],
+ [$class: 'StringParameterValue',
+ name: 'BRANCH',
+ value: "${params.BRANCH}"],
+ [$class: 'StringParameterValue',
+ name: 'REFSPEC',
+ value: "${params.REFSPEC}"]
+ ])
+}
diff --git a/jenkins/jobs/global.xml.tpl b/jenkins/jobs/global.xml.tpl
new file mode 100644
index 0000000..dcd7e5c
--- /dev/null
+++ b/jenkins/jobs/global.xml.tpl
@@ -0,0 +1,44 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<flow-definition>
+ <actions/>
+ <description>Global pipeline to bootstrap bazel and runs all downstream jobs</description>
+ <keepDependencies>false</keepDependencies>
+ <properties>
+ <org.jenkinsci.plugins.workflow.job.properties.DisableConcurrentBuildsJobProperty/>
+ <com.coravy.hudson.plugins.github.GithubProjectProperty>
+ <projectUrl>https://github.com/bazelbuild/bazel/</projectUrl>
+ <displayName></displayName>
+ </com.coravy.hudson.plugins.github.GithubProjectProperty>
+ <hudson.model.ParametersDefinitionProperty>
+ <parameterDefinitions>
+ <hudson.model.StringParameterDefinition>
+ <name>BRANCH</name>
+ <description>The branch to build</description>
+ <defaultValue>master</defaultValue>
+ </hudson.model.StringParameterDefinition>
+ <hudson.model.StringParameterDefinition>
+ <name>REFSPEC</name>
+ <description>The refspec to fetch</description>
+ <defaultValue>+refs/heads/*:refs/remotes/origin/*</defaultValue>
+ </hudson.model.StringParameterDefinition>
+ <hudson.model.TextParameterDefinition>
+ <name>EXTRA_BAZELRC</name>
+ <description>To inject new option to the .bazelrc file in downstream projects.</description>
+ <defaultValue></defaultValue>
+ </hudson.model.TextParameterDefinition>
+ </parameterDefinitions>
+ </hudson.model.ParametersDefinitionProperty>
+ <org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+ <triggers/>
+ </org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+ </properties>
+ <definition class="org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition">
+ <script>
+json_config = '''{{ raw_imports['//jenkins/jobs:configs/bootstrap.json'].replace('\\', '\\\\').replace("'", "\\'") }}'''
+restrict_configuration = {{ variables.RESTRICT_CONFIGURATION }}
+mail_recipient = "{{ variables.BAZEL_BUILD_RECIPIENT }}"
+{{ imports['//jenkins/jobs:global.groovy'] }}</script>
+ <sandbox>true</sandbox>
+ </definition>
+ <triggers/>
+</flow-definition>
diff --git a/jenkins/jobs/jobs.bzl b/jenkins/jobs/jobs.bzl
index 0b30083..8b8c431 100644
--- a/jenkins/jobs/jobs.bzl
+++ b/jenkins/jobs/jobs.bzl
@@ -65,6 +65,7 @@
BAZEL_STAGING_JOBS = {
"Bazel": ALL_PLATFORMS + BSD_PLATFORMS,
"Github-Trigger": UNIX_PLATFORMS,
+ "Global/pipeline": [],
"Bazel-Install": [],
"Bazel-Install-Trigger": [],
}
diff --git a/templating/main.py b/templating/main.py
index e8b960a..c3da715 100644
--- a/templating/main.py
+++ b/templating/main.py
@@ -68,11 +68,13 @@
return source, self.path, lambda: mtime == os.path.getmtime(self.path)
-def expand_template(template, variables, imports):
+def expand_template(template, variables, imports, raw_imports=None):
"""Expand a template."""
+ if raw_imports is None:
+ raw_imports = imports
env = jinja2.Environment(loader=OneFileLoader(template))
template = env.get_template(template)
- return template.render(imports=imports, variables=variables)
+ return template.render(imports=imports, variables=variables, raw_imports=raw_imports)
def quote_xml(d):
"""Returns a copy of d where all values where escaped for XML."""
@@ -99,10 +101,11 @@
sys.exit(-1)
variables[kv[0]] = kv[1]
imports = construct_imports(variables, flags.imports)
+ raw_imports = imports
if flags.escape_xml:
imports = quote_xml(imports)
variables = quote_xml(variables)
- result = expand_template(flags.template, variables, imports)
+ result = expand_template(flags.template, variables, imports, raw_imports)
with open(flags.output, "w") as f:
f.write(result)
if flags.executable:
