scripts/ci/build.sh

#!/bin/bash -eu

# Copyright 2015 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Main deploy functions for the continous build system
# Just source this file and use the various method:
#   bazel_build build bazel and run all its test
#   bazel_release use the artifact generated by bazel_build and push
#     them to github for a release and to GCS for a release candidate.
#     Also prepare an email for announcing the release.

# Load common.sh
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
source $(dirname ${SCRIPT_DIR})/release/common.sh

: ${GIT_REPOSITORY_URL:=https://github.com/bazelbuild/bazel}

: ${GCS_BASE_URL:=https://storage.googleapis.com}
: ${GCS_BUCKET:=bucket-o-bazel}

: ${EMAIL_TEMPLATE_RC:=${SCRIPT_DIR}/rc_email.txt}
: ${EMAIL_TEMPLATE_RELEASE:=${SCRIPT_DIR}/release_email.txt}

: ${RELEASE_CANDIDATE_URL:="${GCS_BASE_URL}/${GCS_BUCKET}/%release_name%/rc%rc%/index.html"}
: ${RELEASE_URL="${GIT_REPOSITORY_URL}/releases/tag/%release_name%"}

set -eu

PLATFORM="$(uname -s | tr 'A-Z' 'a-z')"
if [[ ${PLATFORM} == "darwin" ]]; then
  function checksum() {
    (cd "$(dirname "$1")" && shasum -a 256 "$(basename "$1")")
  }
else
  function checksum() {
    (cd "$(dirname "$1")" && sha256sum "$(basename "$1")")
  }
fi

GIT_ROOT="$(git rev-parse --show-toplevel)"
BUILD_SCRIPT_PATH="${GIT_ROOT}/compile.sh"

# Returns the full release name in the form NAME(rcRC)?
function get_full_release_name() {
  local rc=$(get_release_candidate)
  local name=$(get_release_name)
  if [ -n "${rc}" ]; then
    echo "${name}rc${rc}"
  else
    echo "${name}"
  fi
}

function setup_android_repositories() {
  if [ ! -f WORKSPACE.bak ] && [ -n "${ANDROID_SDK_PATH-}" ]; then
    cp WORKSPACE WORKSPACE.bak
    trap '[ -f WORKSPACE.bak ] && rm WORKSPACE && mv WORKSPACE.bak WORKSPACE' \
      EXIT
    cat >>WORKSPACE <<EOF
android_sdk_repository(
    name = "androidsdk",
    path = "${ANDROID_SDK_PATH}",
    build_tools_version = "${ANDROID_SDK_BUILD_TOOLS_VERSION:-22.0.1}",
    api_level = ${ANDROID_SDK_API_LEVEL:-21},
)

bind(
    name = "android_sdk_for_testing",
    actual = "@androidsdk//:files",
)
EOF
    if [ -n "${ANDROID_NDK_PATH-}" ]; then
      cat >>WORKSPACE <<EOF
android_ndk_repository(
    name = "androidndk",
    path = "${ANDROID_NDK_PATH}",
    api_level = ${ANDROID_NDK_API_LEVEL:-21},
)

bind(
    name = "android_ndk_for_testing",
    actual = "@androidndk//:files",
)
EOF
    fi
  fi
}

# Set the various arguments when JDK 7 is required (deprecated).
# This method is here to continue to build binary release of Bazel
# for JDK 7. We will drop this method and JDK 7 support when our
# ci system turn red on this one.
function setup_jdk7() {
  # This is a JDK 7 JavaBuilder from release 0.1.0.
  local javabuilder_url="https://storage.googleapis.com/bazel/0.1.0/JavaBuilder_deploy.jar"
  local javac_url="https://github.com/bazelbuild/bazel/blob/0.1.0/third_party/java/jdk/langtools/javac.jar?raw=true"
  sed -i.bak 's/_version = "8"/_version = "7"/' tools/jdk/BUILD
  rm -f tools/jdk/BUILD.bak
  rm -f third_party/java/jdk/langtools/javac.jar
  curl -Ls -o tools/jdk/JavaBuilder_deploy.jar "${javabuilder_url}"
  curl -Ls -o third_party/java/jdk/langtools/javac.jar "${javac_url}"
  # Do not use the skylark bootstrapped version of JavaBuilder
  export BAZEL_ARGS="--singlejar_top=//src/java_tools/singlejar:bootstrap_deploy.jar \
      --genclass_top=//src/java_tools/buildjar:bootstrap_genclass_deploy.jar \
      --ijar_top=//third_party/ijar"
  # Skip building JavaBuilder
  export BAZEL_SKIP_TOOL_COMPILATION=tools/jdk/JavaBuilder_deploy.jar
  # Ignore JDK8 tests
  export BAZEL_TEST_FILTERS="-jdk8"
  if ! grep -Fq 'RealJavaBuilder' src/java_tools/buildjar/BUILD; then
    # And more ugly hack. Overwrite the BUILD file of JavaBuilder
    # so we use the pre-built version in integration tests.
    sed -i.bak 's/name = \"JavaBuilder\"/name = \"RealJavaBuilder\"/' \
        src/java_tools/buildjar/BUILD
    rm -f src/java_tools/buildjar/BUILD.bak
    cat >>src/java_tools/buildjar/BUILD <<'EOF'
genrule(
    name = "JavaBuilder",
    outs = ["JavaBuilder_deploy.jar"],
    srcs = ["//tools/jdk:JavaBuilder_deploy.jar"],
    cmd = "cp $< $@",
    visibility = ["//visibility:public"],
)
EOF
  fi
}

# Main entry point for building bazel.
# It sets the embed label to the release name if any, calls the whole
# test suite, compile the various packages, then copy the artifacts
# to the folder in $1
function bazel_build() {
  local release_label="$(get_full_release_name)"
  local embed_label_opts=

  if [ -n "${release_label}" ]; then
    export EMBED_LABEL="${release_label}"
  fi

  if [[ "${JAVA_VERSION-}" =~ ^(1\.)?7$ ]]; then
    JAVA_VERSION=1.7
    setup_jdk7
    release_label="${release_label}-jdk7"
  else
    JAVA_VERSION=1.8
  fi

  setup_android_repositories
  retCode=0
  ${BUILD_SCRIPT_PATH} ${BAZEL_COMPILE_TARGET:-all} || retCode=$?

  # Exit for failure except for test failures (exit code 3).
  if (( $retCode != 0 && $retCode != 3 )); then
    exit $retCode
  fi

  # Build the packages
  ./output/bazel --bazelrc=${BAZELRC:-/dev/null} --nomaster_bazelrc build \
      --embed_label=${release_label} --stamp \
      --workspace_status_command=scripts/ci/build_status_command.sh \
      --define JAVA_VERSION=${JAVA_VERSION} \
      //scripts/packages/... || exit $?

  if [ -n "${1-}" ]; then
    # Copy the results to the output directory
    mkdir -p $1/packages
    cp output/bazel $1/bazel
    cp bazel-bin/scripts/packages/install.sh $1/bazel-${release_label}-installer.sh
    if [ "$PLATFORM" = "linux" ]; then
      cp bazel-bin/scripts/packages/bazel-debian.deb $1/bazel_${release_label}.deb
    fi
    cp bazel-genfiles/scripts/packages/README.md $1/README.md
  fi

  if (( $retCode )); then
    export BUILD_UNSTABLE=1
  fi
}

# Generate a string from a template and a list of substitutions.
# The first parameter is the template name and each subsequent parameter
# is taken as a couple: first is the string the substitute and the second
# is the result of the substitution.
function generate_from_template() {
  local value="$1"
  shift
  while (( $# >= 2 )); do
    value="${value//$1/$2}"
    shift 2
  done
  echo "${value}"
}

# Generate the email for the release.
# The first line of the output will be the recipient, the second line
# the mail subjects and the subsequent lines the mail, its content.
# If no planed release, then this function output will be empty.
function generate_email() {
  local release_name=$(get_release_name)
  local rc=$(get_release_candidate)
  local args=(
      "%release_name%" "${release_name}"
      "%rc%" "${rc}"
      "%relnotes%" "# $(git_commit_msg)"
  )
  if [ -n "${rc}" ]; then
    args+=(
        "%url%"
        "$(generate_from_template "${RELEASE_CANDIDATE_URL}" "${args[@]}")"
    )
    generate_from_template "$(cat ${EMAIL_TEMPLATE_RC})" "${args[@]}"
  elif [ -n "${release_name}" ]; then
    args+=(
        "%url%"
        "$(generate_from_template "${RELEASE_URL}" "${args[@]}")"
    )
    generate_from_template "$(cat ${EMAIL_TEMPLATE_RELEASE})" "${args[@]}"
  fi
}

# Deploy a github release using a third party tool:
#   https://github.com/c4milo/github-release
# This methods expects the following arguments:
#   $1..$n files generated by package_build (should not contains the README file)
# Please set GITHUB_TOKEN to talk to the Github API and GITHUB_RELEASE
# for the path to the https://github.com/c4milo/github-release tool.
# This method is also affected by GIT_REPOSITORY_URL which should be the
# URL to the github repository (defaulted to https://github.com/bazelbuild/bazel).
function release_to_github() {
  local url="${GIT_REPOSITORY_URL}"
  local release_name=$(get_release_name)
  local rc=$(get_release_candidate)
  local release_tool="${GITHUB_RELEASE:-$(which github-release 2>/dev/null || true)}"
  if [ ! -x "${release_tool}" ]; then
    echo "Please set GITHUB_RELEASE to the path to the github-release binary." >&2
    echo "This probably means you haven't installed https://github.com/c4milo/github-release " >&2
    echo "on this machine." >&2
    return 1
  fi
  local github_repo="$(echo "$url" | sed -E 's|https?://github.com/([^/]*/[^/]*).*$|\1|')"
  if [ -n "${release_name}" ] && [ -z "${rc}" ]; then
    mkdir -p "${tmpdir}/to-github"
    cp "${@}" "${tmpdir}/to-github"
    "${GITHUB_RELEASE}" "${github_repo}" "${release_name}" "" "# $(git_commit_msg)" "${tmpdir}/to-github/"'*'
  fi
}

# Creates an index of the files contained in folder $1 in mardown format
function create_index_md() {
  # First, add the README.md
  local file=$1/__temp.md
  if [ -f $1/README.md ]; then
    cat $1/README.md
  fi
  # Then, add the list of files
  echo
  echo "## Index of files"
  echo
  for f in $1/*.sha256; do  # just list the sha256 ones
    local filename=$(basename $f .sha256);
    echo " - [${filename}](${filename}) [[SHA-256](${filename}.sha256)]"
  done
}

# Creates an index of the files contained in folder $1 in HTML format
# It supposes hoedown (https://github.com/hoedown/hoedown) is on the path,
# if not, set the HOEDOWN environment variable to the good path.
function create_index_html() {
  local hoedown="${HOEDOWN:-$(which hoedown 2>/dev/null || true)}"
  # Second line is to trick hoedown to behave as Github
  create_index_md "${@}" \
      | sed -E 's/^(Baseline.*)$/\1\
/' | sed 's/^   + / - /' | sed 's/_/\\_/g' \
      | "${hoedown}"
}

# Deploy a release candidate to Google Cloud Storage.
# It requires to have gsutil installed. You can force the path to gsutil
# by setting the GSUTIL environment variable. The GCS_BUCKET should be the
# name of the Google cloud bucket to deploy to.
# This methods expects the following arguments:
#   $1..$n files generated by package_build
function release_to_gcs() {
  local gs="${GSUTIL:-$(which gsutil 2>/dev/null || true) -m}"
  local release_name=$(get_release_name)
  local rc=$(get_release_candidate)
  if [ ! -x "${gs}" ]; then
    echo "Please set GSUTIL to the path the gsutil binary." >&2
    echo "gsutil (https://cloud.google.com/storage/docs/gsutil/) is the" >&2
    echo "command-line interface to google cloud." >&2
    return 1
  fi
  if [ -z "${GCS_BUCKET-}" ]; then
    echo "Please set GCS_BUCKET to the name of your Google Cloud Storage bucket." >&2
    return 1
  fi
  if [ -n "${release_name}" ] && [ -n "${rc}" ]; then
    # Make a temporary folder with the desired structure
    local dir="$(mktemp -d ${TMPDIR:-/tmp}/tmp.XXXXXXXX)"
    local prev_dir="$PWD"
    trap "{ cd ${prev_dir}; rm -fr ${dir}; }" EXIT
    mkdir -p "${dir}/${release_name}/rc${rc}"
    cp "${@}" "${dir}/${release_name}/rc${rc}"
    # Add a index.html file:
    create_index_html "${dir}/${release_name}/rc${rc}" \
        >"${dir}/${release_name}/rc${rc}"/index.html
    cd ${dir}
    "${gs}" cp -a public-read -r . "gs://${GCS_BUCKET}"
    cd ${prev_dir}
    rm -fr ${dir}
    trap - EXIT
  fi
}

# A wrapper around the release deployment methods.
function deploy_release() {
  local github_args=()
  # Filters out README.md for github releases
  for i in "$@"; do
    if ! [[ "$i" =~ README.md$ ]]; then
      github_args+=("$i")
    fi
  done
  release_to_github "${github_args[@]}"
  release_to_gcs "$@"
}

# A wrapper for the whole release phase:
#   Compute the SHA-256, and arrange the input
#   Deploy the release
#   Generate the email
# Input: $1 $2 [$3 $4 [$5 $6 ...]]
#    Each pair denotes a couple (platform, folder) where the platform
#    is the platform built for and the folder is the folder where the
#    artifacts for this platform are.
# Ouputs:
#   RELEASE_EMAIL_RECIPIENT: who to send a mail to
#   RELEASE_EMAIL_SUBJECT: the subject of the email to be sent
#   RELEASE_EMAIL_CONTENT: the content of the email to be sent
function bazel_release() {
  local README=$2/README.md
  tmpdir=$(mktemp -d ${TMPDIR:-/tmp}/tmp.XXXXXXXX)
  trap 'rm -fr ${tmpdir}' EXIT
  while (( $# > 1 )); do
    local platform=$1
    local folder=$2
    shift 2
    for file in $folder/*; do
      if [ $(basename $file) != README.md ]; then
        if [[ "$file" =~ /([^/]*)(\.[^\./]+)$ ]]; then
          local destfile=${tmpdir}/${BASH_REMATCH[1]}-${platform}${BASH_REMATCH[2]}
        else
          local destfile=${tmpdir}/$(basename $file)-${platform}
        fi
        mv $file $destfile
        checksum $destfile > $destfile.sha256
      fi
    done
  done
  deploy_release $README $(find ${tmpdir} -type f)

  export RELEASE_EMAIL="$(generate_email)"

  export RELEASE_EMAIL_RECIPIENT="$(echo "${RELEASE_EMAIL}" | head -1)"
  export RELEASE_EMAIL_SUBJECT="$(echo "${RELEASE_EMAIL}" | head -2 | tail -1)"
  export RELEASE_EMAIL_CONTENT="$(echo "${RELEASE_EMAIL}" | tail -n +3)"
}