blob: e31add08832699038cc34f609c6f130eed0e5db8 [file] [log] [blame]
#!/bin/bash
# Copyright 2015 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eu
# Main deploy functions for the continous build system
# Just source this file and use the various method:
# bazel_build build bazel and run all its test
# bazel_release use the artifact generated by bazel_build and push
# them to github for a release and to GCS for a release candidate.
# Also prepare an email for announcing the release.
# Load common.sh
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$(dirname ${SCRIPT_DIR})/release/common.sh"
if ! command -v gsutil &>/dev/null; then
echo "Required tool 'gsutil' not found. Please install it:"
echo "See https://cloud.google.com/sdk/downloads for instructions."
exit 1
fi
if ! command -v github-release &>/dev/null; then
echo "Required tool 'github-release' not found. Download it from here:"
echo "https://github.com/c4milo/github-release/releases"
echo "Just extract the archive and put the binary on your PATH."
exit 1
fi
if ! command -v debsign &>/dev/null; then
echo "Required tool 'debsign' not found. Please install it via apt-get:"
echo "apt-get install devscripts"
exit 1
fi
if ! command -v reprepro &>/dev/null; then
echo "Required tool 'reprepro' not found. Please install it via apt-get:"
echo "apt-get install reprepro"
exit 1
fi
if ! command -v gpg &>/dev/null; then
echo "Required tool 'gpg' not found. Please install it via apt-get:"
echo "apt-get install gnupg"
exit 1
fi
if ! command -v pandoc &>/dev/null; then
echo "Required tool 'pandoc' not found. Please install it via apt-get:"
echo "apt-get install pandoc"
exit 1
fi
# if ! command -v ssmtp &>/dev/null; then
# echo "Required tool 'ssmtp' not found. Please install it via apt-get:"
# echo "apt-get install ssmtp"
# exit 1
# fi
export APT_GPG_KEY_ID=$(gsutil cat gs://bazel-trusted-encrypted-secrets/release-key.gpg.id)
# Generate a string from a template and a list of substitutions.
# The first parameter is the template name and each subsequent parameter
# is taken as a couple: first is the string the substitute and the second
# is the result of the substitution.
function generate_from_template() {
local value="$1"
shift
while (( $# >= 2 )); do
value="${value//$1/$2}"
shift 2
done
echo "${value}"
}
# Generate the email for the release.
# The first line of the output will be the recipient, the second line
# the mail subjects and the subsequent lines the mail, its content.
# If no planed release, then this function output will be empty.
function generate_email() {
RELEASE_CANDIDATE_URL="https://releases.bazel.build/%release_name%/rc%rc%/index.html"
RELEASE_URL="https://github.com/bazelbuild/bazel/releases/tag/%release_name%"
local release_name=$(get_release_name)
local rc=$(get_release_candidate)
local args=(
"%release_name%" "${release_name}"
"%rc%" "${rc}"
"%relnotes%" "# $(get_full_release_notes)"
)
if [ -n "${rc}" ]; then
args+=(
"%url%" "$(generate_from_template "${RELEASE_CANDIDATE_URL}" "${args[@]}")"
)
generate_from_template \
"$(cat "${SCRIPT_DIR}/rc_email.txt")" \
"${args[@]}"
elif [ -n "${release_name}" ]; then
args+=(
"%url%" "$(generate_from_template "${RELEASE_URL}" "${args[@]}")"
)
generate_from_template \
"$(cat "${SCRIPT_DIR}/release_email.txt")" "${args[@]}"
fi
}
function get_release_page() {
echo "# $(get_full_release_notes)"'
_Notice_: Bazel installers contain binaries licensed under the GPLv2 with
Classpath exception. Those installers should always be redistributed along with
the source code.
Some versions of Bazel contain a bundled version of OpenJDK. The license of the
bundled OpenJDK and other open-source components can be displayed by running
the command `bazel license`. The vendor and version information of the bundled
OpenJDK can be displayed by running the command `bazel info java-runtime`.
The binaries and source-code of the bundled OpenJDK can be
[downloaded from our mirror server](https://mirror.bazel.build/openjdk/index.html).
_Security_: All our binaries are signed with our
[public key](https://bazel.build/bazel-release.pub.gpg) 48457EE0.
'
}
# Deploy a github release using a third party tool:
# https://github.com/c4milo/github-release
# This methods expects the following arguments:
# $1..$n files generated by package_build (should not contains the README file)
# Please set GITHUB_TOKEN to talk to the Github API.
function release_to_github() {
local artifact_dir="$1"
local release_name=$(get_release_name)
local rc=$(get_release_candidate)
if [ -n "${release_name}" ] && [ -z "${rc}" ]; then
local github_token="$(gsutil cat gs://bazel-trusted-encrypted-secrets/github-trusted-token.enc | \
gcloud kms decrypt --project bazel-public --location global --keyring buildkite --key github-trusted-token --ciphertext-file - --plaintext-file -)"
GITHUB_TOKEN="${github_token}" github-release "bazelbuild/bazel" "${release_name}" "" "$(get_release_page)" "${artifact_dir}/*"
fi
}
# Creates an index of the files contained in folder $1 in Markdown format.
function create_index_md() {
# First, add the release notes
get_release_page
# Then, add the list of files
echo
echo "## Index of files"
echo
for f in $1/*.sha256; do # just list the sha256 ones
local filename=$(basename $f .sha256);
echo " - [${filename}](${filename}) [[SHA-256](${filename}.sha256)] [[SIG](${filename}.sig)]"
done
}
# Creates an index of the files contained in folder $1 in HTML format.
function create_index_html() {
create_index_md "${@}" | pandoc -f markdown -t html
}
# Deploy a release candidate to Google Cloud Storage.
# It requires to have gsutil installed. You can force the path to gsutil
# by setting the GSUTIL environment variable.
# This methods expects the following arguments:
# $1..$n files generated by package_build
function release_to_gcs() {
local artifact_dir="$1"
local release_name="$(get_release_name)"
local rc="$(get_release_candidate)"
if [ -n "${release_name}" ]; then
local release_path="${release_name}/release"
if [ -n "${rc}" ]; then
release_path="${release_name}/rc${rc}"
fi
create_index_html "${artifact_dir}" > "${artifact_dir}/index.html"
gsutil -m cp "${artifact_dir}/**" "gs://bazel/${release_path}"
fi
}
function ensure_gpg_secret_key_imported() {
if ! gpg --list-secret-keys | grep "${APT_GPG_KEY_ID}" > /dev/null; then
keyfile=$(mktemp --tmpdir)
chmod 0600 "${keyfile}"
gsutil cat "gs://bazel-trusted-encrypted-secrets/release-key.gpg.enc" | \
gcloud kms decrypt --location "global" --keyring "buildkite" --key "bazel-release-key" --ciphertext-file "-" --plaintext-file "${keyfile}"
gpg --allow-secret-key-import --import "${keyfile}"
rm -f "${keyfile}"
fi
# Make sure we use stronger digest algorithm。
# We use reprepro to generate the debian repository,
# but there's no way to pass flags to gpg using reprepro, so writting it into
# ~/.gnupg/gpg.conf
if ! grep "digest-algo sha256" ~/.gnupg/gpg.conf > /dev/null; then
echo "digest-algo sha256" >> ~/.gnupg/gpg.conf
fi
}
function create_apt_repository() {
mkdir conf
cat > conf/distributions <<EOF
Origin: Bazel Authors
Label: Bazel
Codename: stable
Architectures: amd64 source
Components: jdk1.8
Description: Bazel APT Repository
DebOverride: override.stable
DscOverride: override.stable
SignWith: ${APT_GPG_KEY_ID}
Origin: Bazel Authors
Label: Bazel
Codename: testing
Architectures: amd64 source
Components: jdk1.8
Description: Bazel APT Repository
DebOverride: override.testing
DscOverride: override.testing
SignWith: ${APT_GPG_KEY_ID}
EOF
cat > conf/options <<EOF
verbose
ask-passphrase
basedir .
EOF
# TODO(#2264): this is a quick workaround #2256, figure out a correct fix.
cat > conf/override.stable <<EOF
bazel Section contrib/devel
bazel Priority optional
EOF
cat > conf/override.testing <<EOF
bazel Section contrib/devel
bazel Priority optional
EOF
ensure_gpg_secret_key_imported
local distribution="$1"
local deb_pkg_name="$2"
local deb_dsc_name="$3"
debsign -k "${APT_GPG_KEY_ID}" "${deb_dsc_name}"
reprepro -C jdk1.8 includedeb "${distribution}" "${deb_pkg_name}"
reprepro -C jdk1.8 includedsc "${distribution}" "${deb_dsc_name}"
gsutil -m cp -r dists pool "gs://bazel-apt"
}
function release_to_apt() {
local artifact_dir="$1"
local release_name="$(get_release_name)"
local rc="$(get_release_candidate)"
if [ -n "${release_name}" ]; then
local release_label="$(get_full_release_name)"
local deb_pkg_name="${release_name}/bazel_${release_label}-linux-x86_64.deb"
local deb_dsc_name="${release_name}/bazel_${release_label}.dsc"
local deb_tar_name="${release_name}/bazel_${release_label}.tar.gz"
pushd "${artifact_dir}"
if [ -n "${rc}" ]; then
create_apt_repository testing "${deb_pkg_name}" "${deb_dsc_name}"
else
create_apt_repository stable "${deb_pkg_name}" "${deb_dsc_name}"
fi
popd
fi
}
# A wrapper around the release deployment methods.
function deploy_release() {
local release_label="$(get_full_release_name)"
local release_name="$(get_release_name)"
if [[ ! -d $1 ]]; then
echo "Usage: deploy_release ARTIFACT_DIR"
exit 1
fi
artifact_dir="$1"
if [[ -z $release_name ]]; then
echo "Could not get the release name - are you in a release branch directory?"
exit 1
fi
ensure_gpg_secret_key_imported
rm -f "${artifact_dir}"/*.{sha256,sig}
for file in "${artifact_dir}"/*; do
(cd "${artifact_dir}" && sha256sum "$(basename "${file}")" > "${file}.sha256")
gpg --no-tty --detach-sign -u "${APT_GPG_KEY_ID}" "${file}"
done
apt_working_dir="$(mktemp -d --tmpdir)"
echo "apt_working_dir = ${apt_working_dir}"
mkdir "${apt_working_dir}/${release_name}"
cp "${artifact_dir}/bazel_${release_label}-linux-x86_64.deb" "${apt_working_dir}/${release_name}"
cp "${artifact_dir}/bazel_${release_label}.dsc" "${apt_working_dir}/${release_name}"
cp "${artifact_dir}/bazel_${release_label}.tar.gz" "${apt_working_dir}/${release_name}"
release_to_apt "${apt_working_dir}"
gcs_working_dir="$(mktemp -d --tmpdir)"
echo "gcs_working_dir = ${gcs_working_dir}"
cp "${artifact_dir}"/* "${gcs_working_dir}"
release_to_gcs "${gcs_working_dir}"
github_working_dir="$(mktemp -d --tmpdir)"
echo "github_working_dir = ${github_working_dir}"
cp "${artifact_dir}"/* "${github_working_dir}"
rm -f "${github_working_dir}/bazel_${release_label}"*.{dsc,tar.gz}{,.sha256,.sig}
release_to_github "${github_working_dir}"
}