New pipeline: make release pipeline works
The old pipeline was relying on some very peculiar layout of the files
and that prevented to provide a build.log artifact. This change
transfer the reponsibility of the layout from script/ci/build.sh (function
bazel_release) to pushRelease.
Change-Id: I59a3c33966bee769313fb8babe1bad3799f5b830
diff --git a/jenkins/jobs/configs/bootstrap.json b/jenkins/jobs/configs/bootstrap.json
index 4528d8b..7636ca8 100644
--- a/jenkins/jobs/configs/bootstrap.json
+++ b/jenkins/jobs/configs/bootstrap.json
@@ -11,9 +11,9 @@
"parameters": {
"archive": {
"bazel-bin/src/bazel": "bazel",
- "bazel-bin/scripts/packages/with-jdk/install.sh": "bazel-%{release_name}-installer.sh",
- "bazel-bin/scripts/packages/without-jdk/install.sh": "bazel-%{release_name}-without-jdk-installer.sh",
- "bazel-bin/scripts/packages/debian/bazel-debian.deb": "bazel_%{release_name}.deb",
+ "bazel-bin/scripts/packages/with-jdk/install.sh": "bazel-%{release_name}-installer-linux-x86_64.sh",
+ "bazel-bin/scripts/packages/without-jdk/install.sh": "bazel-%{release_name}-without-jdk-installer-linux-x86_64.sh",
+ "bazel-bin/scripts/packages/debian/bazel-debian.deb": "bazel_%{release_name}-linux-x86_64.deb",
"bazel-genfiles/bazel-distfile.zip": "bazel-%{release_name}-dist.zip"
},
"stash": {
@@ -46,8 +46,8 @@
"parameters": {
"archive": {
"bazel-bin/src/bazel": "bazel",
- "bazel-bin/scripts/packages/with-jdk/install.sh": "bazel-%{release_name}-installer.sh",
- "bazel-bin/scripts/packages/without-jdk/install.sh": "bazel-%{release_name}-without-jdk-installer.sh"
+ "bazel-bin/scripts/packages/with-jdk/install.sh": "bazel-%{release_name}-darwin-x86_64-installer.sh",
+ "bazel-bin/scripts/packages/without-jdk/install.sh": "bazel-%{release_name}-darwin-x86_64-without-jdk-installer.sh"
},
"targets": [
"//scripts/packages"
@@ -63,7 +63,7 @@
"node": "windows-x86_64",
"parameters": {
"archive": {
- "bazel-bin/src/bazel": ["bazel.exe", "bazel-%{release_name}.exe"]
+ "bazel-bin/src/bazel": ["bazel.exe", "bazel-%{release_name}-windows-x86_64.exe"]
},
"opts": [
"--cpu=x64_windows_msys",
@@ -77,7 +77,7 @@
"node": "windows-msvc-x86_64",
"parameters": {
"archive": {
- "bazel-bin/src/bazel": ["bazel.exe", "bazel-msvc-%{release_name}.exe"]
+ "bazel-bin/src/bazel": ["bazel.exe", "bazel-msvc-%{release_name}-windows-x86_64.exe"]
},
"opts": [
"--copt=-w",
diff --git a/jenkins/jobs/global.groovy b/jenkins/jobs/global.groovy
index ade08a5..7466e69 100644
--- a/jenkins/jobs/global.groovy
+++ b/jenkins/jobs/global.groovy
@@ -69,12 +69,12 @@
'''
} else {
def r_name = sh(script: "bash -c 'source scripts/release/common.sh; get_full_release_name'",
- returnStdout: true)
+ returnStdout: true).trim()
if (!r_name.isEmpty()) {
pushRelease(name: r_name,
configuration: json_config,
restrict_configuration: restrict_configuration,
- excludes: "node=*/variation=*/bazel node=*/variation=*/*.bazel.build.tar*")
+ excludes: ["**/*.bazel.build.tar*", "**/bazel", "**/bazel.exe"])
// TODO(dmarting): trigger bazel install everywhere in case of release.
}
}
diff --git a/jenkins/lib/src/build/bazel/ci/JenkinsUtils.groovy b/jenkins/lib/src/build/bazel/ci/JenkinsUtils.groovy
index 6ef6dc1..9451176 100644
--- a/jenkins/lib/src/build/bazel/ci/JenkinsUtils.groovy
+++ b/jenkins/lib/src/build/bazel/ci/JenkinsUtils.groovy
@@ -228,4 +228,13 @@
public static void saveLog(env, RunWrapper run, path) {
createFilePath(env, path).copyFrom(run.getRawBuild().getLogInputStream())
}
+
+ /** Returns the recursive list of files of a folder, ignoring some files. */
+ @NonCPS
+ public static def list(env, dir, excludes) {
+ def directory = createFilePath(env, dir)
+ def results = directory.list("**", excludes.join(","))
+ def directoryUri = directory.toURI()
+ return results.collect { it -> directoryUri.relativize(it.toURI()) }
+ }
}
diff --git a/jenkins/lib/vars/bootstrapBazel.groovy b/jenkins/lib/vars/bootstrapBazel.groovy
index 084436c..e799cce 100644
--- a/jenkins/lib/vars/bootstrapBazel.groovy
+++ b/jenkins/lib/vars/bootstrapBazel.groovy
@@ -47,7 +47,6 @@
release_name =
sh(script: "bash -c 'source scripts/release/common.sh; get_full_release_name'",
returnStdout: true).trim()
-
def opts = config.get("opts", [])
opts <<= "--stamp"
if (!isWindows) {
diff --git a/jenkins/lib/vars/pushRelease.groovy b/jenkins/lib/vars/pushRelease.groovy
index d2e7cf1..8db9a34 100644
--- a/jenkins/lib/vars/pushRelease.groovy
+++ b/jenkins/lib/vars/pushRelease.groovy
@@ -15,60 +15,104 @@
import build.bazel.ci.JenkinsUtils
import build.bazel.ci.BazelConfiguration
+private def ensureGpgSecretKeyImported() {
+ sh '''#!/bin/bash
+echo "Import GPG Secret key"
+(gpg --list-secret-keys | grep "${APT_GPG_KEY_ID}" > /dev/null) || \\
+ gpg --allow-secret-key-import --import "${APT_GPG_KEY_PATH}"
+# Make sure we use stronger digest algorithm.
+# We use reprepro to generate the debian repository,
+# but there is no way to pass flags to gpg using reprepro, so writting it into
+# ~/.gnupg/gpg.conf
+(grep "digest-algo sha256" ~/.gnupg/gpg.conf > /dev/null) || \\
+ echo "digest-algo sha256" >> ~/.gnupg/gpg.conf
+'''
+}
+
+// Generate the SHA-256 checksum and the GPG signature for a list of artifacts.
+// Returns a new list of artifacts that include the generated checksum and signature files.
+private def signArtifacts(files) {
+ def result = []
+ def script = []
+ for (def file : files) {
+ script <<= "echo 'Signing ${file}'"
+ script <<= "(cd \"\$(dirname '${file}')\" && sha256sum \"\$(basename '${file}')\") > '${file}.sha256'"
+ script <<= "gpg --no-tty --detach-sign -u \"\${APT_GPG_KEY_ID}\" '${file}'"
+ result <<= file
+ result <<= "${file}.sha256"
+ result <<= "${file}.sig"
+ }
+ sh "#!/bin/bash \n${script.join '\n'}"
+ return result
+}
+
+@NonCPS
+private def listArtifacts(ws, dir, excludes) {
+ return JenkinsUtils.list(env, "${ws}/${dir}", excludes).collect { "${dir}/${it}" }
+}
+
+private def listStashes(configuration, restrict_configuration) {
+ def result = []
+ def conf = BazelConfiguration.flattenConfigurations(
+ BazelConfiguration.parse(configuration), restrict_configuration)
+ for (k in conf.keySet()) {
+ if ("stash" in conf[k] || "archive" in conf[k]) {
+ result.add("bazel--node=${k.node}--variation=${k.variation}")
+ }
+ }
+ return result
+}
+
// A step that push a release for Bazel.
def call(params = [:]) {
+ // Parameters
def r_name = params.name
- def stashes = params.get("stashes",
- { conf -> "bazel--node=${conf.node}--variation=${conf.variation}" })
def bucket = params.get("bucket", "bazel")
- def release_script = params.get("script", "source scripts/ci/build.sh; bazel_release")
+ def release_script = params.get("release_script", "source scripts/ci/build.sh; deploy_release")
+ def email_script = params.get("email_script", "source scripts/ci/build.sh; generate_email")
def repository = params.get("repository", "https://github.com/bazelbuild/bazel")
def replyTo = params.get("replyTo", "bazel-ci@googlegroups.com")
+
+ def ws = pwd()
+
+ // Save the build log
+ JenkinsUtils.saveLog(env, currentBuild, "${ws}/build.log")
+
// unstash all the things
- def conf = BazelConfiguration.flattenConfigurations(
- BazelConfiguration.parse(params.configuration),
- params.restrict_configuration).keySet().toArray()
- for (int k = 0; k < conf.length; k++) {
- def stashName = stashes(conf[k])
- if (stashName) {
+ dir("artifacts") {
+ def stashNames = listStashes(params.configuration, params.restrict_configuration)
+ for (def stashName : stashNames) {
unstash stashName
}
}
- // Delete files we do not need
- if ("excludes" in params) {
- sh "rm -f ${params.excludes}"
- }
+ def artifacts = listArtifacts(ws, "artifacts", params.get("excludes", []))
+
// Now the actual release
withEnv(["GCS_BUCKET=${bucket}",
- "GIT_REPOSITORY_URL=${repository}"]) {
- JenkinsUtils.saveLog(env, currentBuild, "${pwd()}/build.log")
- sh '''#!/bin/bash
-# Credentials should not be displayed on the command line
-export GITHUB_TOKEN="$(cat "$GITHUB_TOKEN_FILE")"
-export APT_GPG_KEY_ID="$(cat "${APT_GPG_KEY_ID_FILE}")"
+ "GIT_REPOSITORY_URL=${repository}",
+ "GITHUB_TOKEN=${readFile(env.GITHUB_TOKEN_FILE).trim()}",
+ "APT_GPG_KEY_ID=${readFile(env.APT_GPG_KEY_ID_FILE).trim()}",
+ // TODO(dmarting): hack to work with release_to_apt(), we should get rid of it.
+ "tmpdir=${ws}/artifacts/node=linux-x86_64/variation="]) {
+ // Sign artifacts
+ ensureGpgSecretKeyImported()
+ def artifact_list = signArtifacts(artifacts)
-args=()
-# TODO(dmarting): Add build.log to the list of artifacts to deploy
-for i in node=*; do
- for j in $i/variation=*; do
- args+=("$(echo $i | cut -d = -f 2)" "$j")
- done
-done
+ // Release
+ sh "#!/bin/bash\n${release_script} build.log ${artifact_list.join ' '}"
-set -x
-''' + release_script + ''' "${args[@]}"
-echo "${RELEASE_EMAIL_RECIPIENT}" | tee output/ci/recipient
-echo "${RELEASE_EMAIL_SUBJECT}" | tee output/ci/subject
-echo "${RELEASE_EMAIL_CONTENT}" | tee output/ci/content
-'''
- if (r_name.contains("test")) {
- echo "Test release, skipping announcement mail"
- } else {
- stage("Announcement mail") {
- mail(subject: JenkinsUtils.readFile(env, "output/ci/subject"),
- to: JenkinsUtils.readFile(env, "output/ci/recipient"),
+ // Send email announcement
+ stage("Announcement mail") {
+ def email = sh(script: "bash -c '${email_script}'", returnStdout: true)
+ echo "Mail to: ${email}"
+ if (r_name.contains("test")) {
+ echo "Test release, skipping announcement mail."
+ } else {
+ def splittedEmail = email.split("\n", 3)
+ mail(subject: splittedEmail[1],
+ to: splittedEmail[0],
replyTo: replyTo,
- body: JenkinsUtils.readFile(env, "output/ci/content"))
+ body: splittedEmail[2])
}
}
}
