Generate a build log to push to GCS with the release
To do so, save the log from the Jenkins master, stop saving
the README.md file. Moved the pushing release to a specific steps
so it can actually access those special JenkinsUtils functions.
Change-Id: I6ef891d302270c59efbf038dfac0ab364f134833
diff --git a/jenkins/jobs/configs/bootstrap.json b/jenkins/jobs/configs/bootstrap.json
index 0f3d81b..4528d8b 100644
--- a/jenkins/jobs/configs/bootstrap.json
+++ b/jenkins/jobs/configs/bootstrap.json
@@ -17,7 +17,6 @@
"bazel-genfiles/bazel-distfile.zip": "bazel-%{release_name}-dist.zip"
},
"stash": {
- "bazel-genfiles/scripts/packages/README.md": "README.md",
"bazel-genfiles/scripts/packages/debian/bazel.dsc": "bazel.dsc",
"bazel-genfiles/scripts/packages/debian/bazel.tar.gz": "bazel.tar.gz",
"bazel-genfiles/site/jekyll-tree.tar": "docs.bazel.build.tar",
diff --git a/jenkins/jobs/global.groovy b/jenkins/jobs/global.groovy
index 2514531..ade08a5 100644
--- a/jenkins/jobs/global.groovy
+++ b/jenkins/jobs/global.groovy
@@ -25,7 +25,7 @@
// First we bootstrap bazel on all platform
stage("Bootstrap on all platforms") {
bootstrapBazelAll(repository: params.REPOSITORY,
- branch: params.BRANCH,
+ branch: params.BRANCH,
refspec: params.REFSPEC,
email: mail_recipient,
configuration: json_config,
@@ -47,14 +47,18 @@
// Deployment steps
-if(params.BRANCH.matches('^(.*/)master$')) {
- stage("Push website") {
+def is_master = params.BRANCH.matches('^(.*/)?master$')
+def is_rc = params.BRANCH.matches('^(refs/heads/)?release-.*$')
+def is_release = params.BRANCH.matches('^refs/tags/.*$')
+if(is_master || is_rc | is_release) {
+ stage(is_master ? "Push website" : "Push release") {
machine("deploy") {
recursiveGit(repository: params.REPOSITORY,
refspec: params.REFSPEC,
branch: params.BRANCH)
- unstash "bazel--node=linux-x86_64--variation="
- sh script: '''#!/bin/bash
+ if (is_master) {
+ unstash "bazel--node=linux-x86_64--variation="
+ sh script: '''#!/bin/bash
. scripts/ci/build.sh
for i in $(find input -name \'*.bazel.build.tar\'); do
build_and_publish_site "$i" "$(basename $i .tar)" "build"
@@ -63,62 +67,22 @@
build_and_publish_site "$i" "$(basename $i .tar.nobuild)" "nobuild"
done
'''
- }
- }
-} else if(params.BRANCH.matches('^refs/((heads/release-)|(tags/)).*$')) {
- def r_name = ""
- machine("deploy") {
- r_name = sh(script: "bash -c 'source scripts/release/common.sh; get_full_release_name'",
- returnStdout: true)
- if (!r_name.isEmpty()) {
- stage("Push release") {
- // unstash all the things
- def conf = BazelConfiguration.flattenConfigurations(
- BazelConfiguration.parse(json_config), restrict_configuration).keySet().toArray()
- for (int k = 0; k < entrySet.size; k++) {
- unstash "bazel--node=${conf[k].node}--variation=${conf[k].variation}"
- }
- // Delete files we do not need
- sh "rm -f node=*/variation=*/bazel node=*/variation=*/*.bazel.build.tar*"
- // Now the actual release
- withEnv(["GCS_BUCKET=bazel",
- "GIT_REPOSITORY_URL=https://github.com/bazelbuild/bazel"]) {
- dir("output/ci") { -> }
- sh '''#!/bin/bash
-# Credentials should not be displayed on the command line
-export GITHUB_TOKEN="$(cat "$GITHUB_TOKEN_FILE")"
-export APT_GPG_KEY_ID="$(cat "${APT_GPG_KEY_ID_FILE}")"
-source scripts/ci/build.sh
-
-args=()
-for i in node=*; do
- for j in $i/variation=*/*; do
- args+=("$(echo $i | cut -d = -f 2)" "$j")
-done
-
-bazel_release "${args[@]}"
-echo "${RELEASE_EMAIL_RECIPIENT}" | tee output/ci/recipient
-echo "${RELEASE_EMAIL_SUBJECT}" | tee output/ci/subject
-echo "${RELEASE_EMAIL_CONTENT}" | tee output/ci/content
-'''
- if (r_name.contains("test")) {
- echo "Test release, skipping announcement mail"
- } else {
- stage("Announcement mail") {
- mail(subject: readFile("output/ci/subject"),
- to: readFile("output/ci/recipient"),
- replyTo: "bazel-ci@googlegroups.com",
- body: readFile("output/ci/content"))
- }
- }
+ } else {
+ def r_name = sh(script: "bash -c 'source scripts/release/common.sh; get_full_release_name'",
+ returnStdout: true)
+ if (!r_name.isEmpty()) {
+ pushRelease(name: r_name,
+ configuration: json_config,
+ restrict_configuration: restrict_configuration,
+ excludes: "node=*/variation=*/bazel node=*/variation=*/*.bazel.build.tar*")
+ // TODO(dmarting): trigger bazel install everywhere in case of release.
}
}
}
}
- // TODO(dmarting): trigger bazel install everywhere in case of release.
}
-// Then we run all jobs in the Global folder except the global pipeline job (the current job).
+// Then we run all jobs in the Global folder except the global pipeline job (the current job).
report = null
stage("Test downstream jobs") {
report = runAll(folder: "Global",
diff --git a/jenkins/lib/src/build/bazel/ci/JenkinsUtils.groovy b/jenkins/lib/src/build/bazel/ci/JenkinsUtils.groovy
index 2a2f820..6ef6dc1 100644
--- a/jenkins/lib/src/build/bazel/ci/JenkinsUtils.groovy
+++ b/jenkins/lib/src/build/bazel/ci/JenkinsUtils.groovy
@@ -187,16 +187,25 @@
return false
}
+ @NonCPS
+ private static def createFilePath(env, path) {
+ if (env['NODE_NAME'].equals("master")) {
+ return new FilePath(path);
+ } else {
+ return new FilePath(Jenkins.getInstance().getComputer(env['NODE_NAME']).getChannel(), path);
+ }
+ }
+
/** Prune file that are older than timestamp on the current node. */
@NonCPS
- public static def pruneIfOlderThan(path, timestamp) {
- return _pruneIfOlderThan(new FilePath(Channel.current(), path), timestamp)
+ public static def pruneIfOlderThan(env, path, timestamp) {
+ return _pruneIfOlderThan(createFilePath(env, path), timestamp)
}
/** Touch a file anywhere on the FS on the current node. */
@NonCPS
- public static def touchFileIfExists(path) {
- FilePath f = new FilePath(Channel.current(), path)
+ public static def touchFileIfExists(env, path) {
+ FilePath f = createFilePath(env, path)
def r = f.exists()
if (r) {
try {
@@ -210,7 +219,13 @@
/** Read a file from the node, but without reporting anything in the Jenkins UI. */
@NonCPS
- public static def readFile(path) {
- return new FilePath(Channel.current(), path).readToString()
+ public static def readFile(env, path) {
+ return createFilePath(env, path).readToString()
+ }
+
+ /** Save the current log to a file on the current node. */
+ @NonCPS
+ public static void saveLog(env, RunWrapper run, path) {
+ createFilePath(env, path).copyFrom(run.getRawBuild().getLogInputStream())
}
}
diff --git a/jenkins/lib/vars/bazelPath.groovy b/jenkins/lib/vars/bazelPath.groovy
index 4a62e8f..647d168 100644
--- a/jenkins/lib/vars/bazelPath.groovy
+++ b/jenkins/lib/vars/bazelPath.groovy
@@ -20,7 +20,8 @@
// but using FilePath.act needs a class that can be shiped to the client, so
// needs to be in the client classpath. If the number of RPC became a problem,
// maybe we can use a Jenkins plugins.
- JenkinsUtils.pruneIfOlderThan(getBazelInstallBase(node_label) + "custom",
+ JenkinsUtils.pruneIfOlderThan(env,
+ getBazelInstallBase(node_label) + "custom",
System.currentTimeMillis() - 172800000 /* 2 days */)
} catch(IOException ex) {
// Several error can occurs, we ignore them all as this step
@@ -68,7 +69,7 @@
cause.upstreamProject.toString().replaceAll("/", "_"),
cause.upstreamBuild.toString(),
"variation_${variation}")
- if (!JenkinsUtils.touchFileIfExists(bazel)) {
+ if (!JenkinsUtils.touchFileIfExists(env, bazel)) {
dir(".bazel") { deleteDir() }
step([$class: 'CopyArtifact',
filter: cause.artifactPath,
diff --git a/jenkins/lib/vars/bootstrapBazel.groovy b/jenkins/lib/vars/bootstrapBazel.groovy
index fe5950a..084436c 100644
--- a/jenkins/lib/vars/bootstrapBazel.groovy
+++ b/jenkins/lib/vars/bootstrapBazel.groovy
@@ -62,7 +62,6 @@
stage("[${config.node},${variation}] bootstrap") {
def envs = ["BUILD_BY=Jenkins",
- "BUILD_LOG=${currentBuild.absoluteUrl}",
"GIT_REPOSITORY_URL=${env.GIT_URL}"]
utils.build(["//src:bazel"] + targets)
}
diff --git a/jenkins/lib/vars/pushRelease.groovy b/jenkins/lib/vars/pushRelease.groovy
new file mode 100644
index 0000000..d2e7cf1
--- /dev/null
+++ b/jenkins/lib/vars/pushRelease.groovy
@@ -0,0 +1,75 @@
+// Copyright (C) 2017 The Bazel Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import build.bazel.ci.JenkinsUtils
+import build.bazel.ci.BazelConfiguration
+
+// A step that push a release for Bazel.
+def call(params = [:]) {
+ def r_name = params.name
+ def stashes = params.get("stashes",
+ { conf -> "bazel--node=${conf.node}--variation=${conf.variation}" })
+ def bucket = params.get("bucket", "bazel")
+ def release_script = params.get("script", "source scripts/ci/build.sh; bazel_release")
+ def repository = params.get("repository", "https://github.com/bazelbuild/bazel")
+ def replyTo = params.get("replyTo", "bazel-ci@googlegroups.com")
+ // unstash all the things
+ def conf = BazelConfiguration.flattenConfigurations(
+ BazelConfiguration.parse(params.configuration),
+ params.restrict_configuration).keySet().toArray()
+ for (int k = 0; k < conf.length; k++) {
+ def stashName = stashes(conf[k])
+ if (stashName) {
+ unstash stashName
+ }
+ }
+ // Delete files we do not need
+ if ("excludes" in params) {
+ sh "rm -f ${params.excludes}"
+ }
+ // Now the actual release
+ withEnv(["GCS_BUCKET=${bucket}",
+ "GIT_REPOSITORY_URL=${repository}"]) {
+ JenkinsUtils.saveLog(env, currentBuild, "${pwd()}/build.log")
+ sh '''#!/bin/bash
+# Credentials should not be displayed on the command line
+export GITHUB_TOKEN="$(cat "$GITHUB_TOKEN_FILE")"
+export APT_GPG_KEY_ID="$(cat "${APT_GPG_KEY_ID_FILE}")"
+
+args=()
+# TODO(dmarting): Add build.log to the list of artifacts to deploy
+for i in node=*; do
+ for j in $i/variation=*; do
+ args+=("$(echo $i | cut -d = -f 2)" "$j")
+ done
+done
+
+set -x
+''' + release_script + ''' "${args[@]}"
+echo "${RELEASE_EMAIL_RECIPIENT}" | tee output/ci/recipient
+echo "${RELEASE_EMAIL_SUBJECT}" | tee output/ci/subject
+echo "${RELEASE_EMAIL_CONTENT}" | tee output/ci/content
+'''
+ if (r_name.contains("test")) {
+ echo "Test release, skipping announcement mail"
+ } else {
+ stage("Announcement mail") {
+ mail(subject: JenkinsUtils.readFile(env, "output/ci/subject"),
+ to: JenkinsUtils.readFile(env, "output/ci/recipient"),
+ replyTo: replyTo,
+ body: JenkinsUtils.readFile(env, "output/ci/content"))
+ }
+ }
+ }
+}
