pipelines/bazel-release.yml

---
steps:
  - label: "Get release name"
    agents:
      - "queue=default"
    plugins:
      docker#v3.8.0:
        always-pull: true
        environment:
          - ANDROID_HOME
          - ANDROID_NDK_HOME
          - BUILDKITE_ARTIFACT_UPLOAD_DESTINATION
        image: gcr.io/bazel-public/ubuntu1804-java11
        network: host
        privileged: true
        propagate-environment: true
        propagate-uid-gid: true
        shell: ["/bin/bash", "-e", "-c"]
        volumes:
          - "/etc/group:/etc/group:ro"
          - "/etc/passwd:/etc/passwd:ro"
          - "/etc/shadow:/etc/shadow:ro"
          - "/opt/android-ndk-r15c:/opt/android-ndk-r15c:ro"
          - "/opt/android-sdk-linux:/opt/android-sdk-linux:ro"
          - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
          - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
          - "/var/run/docker.sock:/var/run/docker.sock"
    command: |
      git fetch origin ${BUILDKITE_BRANCH}
      git checkout ${BUILDKITE_BRANCH}

      release_name=\$(source scripts/release/common.sh; get_full_release_name)
      echo "release_name = \"\$release_name\""
      buildkite-agent meta-data set "release_name" "\$release_name"

  - wait

  - label: "CentOS 7"
    agents:
      - "queue=default"
    plugins:
      docker#v3.8.0:
        always-pull: true
        environment:
          - ANDROID_HOME
          - ANDROID_NDK_HOME
          - BUILDKITE_ARTIFACT_UPLOAD_DESTINATION
        image: gcr.io/bazel-public/centos7-releaser
        network: host
        privileged: true
        propagate-environment: true
        propagate-uid-gid: true
        shell: ["/bin/bash", "-e", "-c"]
        volumes:
          - "/etc/group:/etc/group:ro"
          - "/etc/passwd:/etc/passwd:ro"
          - "/etc/shadow:/etc/shadow:ro"
          - "/opt/android-ndk-r15c:/opt/android-ndk-r15c:ro"
          - "/opt/android-sdk-linux:/opt/android-sdk-linux:ro"
          - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
          - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
          - "/var/run/docker.sock:/var/run/docker.sock"
    command: |
      git fetch origin ${BUILDKITE_BRANCH}
      git checkout ${BUILDKITE_BRANCH}

      release_name=$(buildkite-agent meta-data get "release_name")
      echo "release_name = \"\$release_name\""

      bazel build --sandbox_tmpfs_path=/tmp //src:bazel
      mkdir output
      cp bazel-bin/src/bazel output/bazel

      output/bazel build \
          -c opt \
          --stamp \
          --sandbox_tmpfs_path=/tmp \
          --embed_label "\${release_name}" \
          --workspace_status_command=scripts/ci/build_status_command.sh \
          src/bazel \
          src/bazel_nojdk \
          scripts/packages/with-jdk/install.sh \
          scripts/packages/debian/bazel-debian.deb \
          scripts/packages/debian/bazel.dsc \
          scripts/packages/debian/bazel.tar.gz \
          bazel-distfile.zip

      mkdir artifacts
      cp "bazel-bin/src/bazel" "artifacts/bazel-\${release_name}-linux-x86_64"
      cp "bazel-bin/src/bazel_nojdk" "artifacts/bazel_nojdk-\${release_name}-linux-x86_64"
      cp "bazel-bin/scripts/packages/with-jdk/install.sh" "artifacts/bazel-\${release_name}-installer-linux-x86_64.sh"
      cp "bazel-bin/scripts/packages/debian/bazel-debian.deb" "artifacts/bazel_\${release_name}-linux-x86_64.deb"
      cp "bazel-bin/scripts/packages/debian/bazel.dsc" "artifacts/bazel_\${release_name}.dsc"
      cp "bazel-bin/scripts/packages/debian/bazel.tar.gz" "artifacts/bazel_\${release_name}.tar.gz"
      cp "bazel-bin/bazel-distfile.zip" "artifacts/bazel-\${release_name}-dist.zip"

      cd artifacts
      buildkite-agent artifact upload "*"

  - label: "CentOS 7 (arm64)"
    agents:
      - "queue=arm64"
    plugins:
      docker#v3.8.0:
        environment:
          - BUILDKITE_ARTIFACT_UPLOAD_DESTINATION
          - GOOGLE_APPLICATION_CREDENTIALS
        image: gcr.io/bazel-public/centos7-releaser
        network: host
        privileged: true
        propagate-environment: true
        propagate-uid-gid: true
        shell: ["/bin/bash", "-e", "-c"]
        volumes:
          - "/etc/group:/etc/group:ro"
          - "/etc/passwd:/etc/passwd:ro"
          - "/etc/shadow:/etc/shadow:ro"
          - "/opt/android-ndk-r15c:/opt/android-ndk-r15c:ro"
          - "/opt/android-sdk-linux:/opt/android-sdk-linux:ro"
          - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
          - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
          - "/var/run/docker.sock:/var/run/docker.sock"
    command: |
      git fetch origin ${BUILDKITE_BRANCH}
      git checkout FETCH_HEAD

      release_name=$(buildkite-agent meta-data get "release_name")
      echo "release_name = \"\$release_name\""

      # Our arm64 machine keeps state between runs, but we want to also do a clean build.
      bazel clean --expunge

      bazel build --sandbox_tmpfs_path=/tmp //src:bazel
      mkdir output
      cp bazel-bin/src/bazel output/bazel

      output/bazel build \
          -c opt \
          --stamp \
          --sandbox_tmpfs_path=/tmp \
          --embed_label "\${release_name}" \
          --workspace_status_command=scripts/ci/build_status_command.sh \
          src/bazel

      mkdir artifacts
      cp "bazel-bin/src/bazel" "artifacts/bazel-\${release_name}-linux-arm64"

      cd artifacts
      buildkite-agent artifact upload "*"

  - label: macOS
    agents:
      - queue=macos
    command: |
      git fetch origin ${BUILDKITE_BRANCH}
      git checkout ${BUILDKITE_BRANCH}

      release_name=$(buildkite-agent meta-data get "release_name")
      echo "release_name = \"\$release_name\""

      bazel build //src:bazel
      mkdir output
      cp bazel-bin/src/bazel output/bazel

      output/bazel build \
          --define IPHONE_SDK=1 \
          -c opt \
          --stamp \
          --embed_label "\${release_name}" \
          --workspace_status_command=scripts/ci/build_status_command.sh \
          src/bazel \
          src/bazel_nojdk \
          scripts/packages/with-jdk/install.sh

      mkdir artifacts
      cp "bazel-bin/src/bazel" "artifacts/bazel-\${release_name}-darwin-x86_64"
      cp "bazel-bin/src/bazel_nojdk" "artifacts/bazel_nojdk-\${release_name}-darwin-x86_64"
      cp "bazel-bin/scripts/packages/with-jdk/install.sh" "artifacts/bazel-\${release_name}-installer-darwin-x86_64.sh"

      cd artifacts
      buildkite-agent artifact upload "*"

  - label: macOS (arm64)
    agents:
      # TODO(pcloudy): Switch to macos_arm64 queue when Apple Silicon machines are available,
      # current we just use x86_64 machines to do cross compile.
      - queue=macos
    command: |
      git fetch origin ${BUILDKITE_BRANCH}
      git checkout ${BUILDKITE_BRANCH}

      release_name=$(buildkite-agent meta-data get "release_name")
      echo "release_name = \"\$release_name\""

      bazel build //src:bazel
      mkdir output
      cp bazel-bin/src/bazel output/bazel

      output/bazel build \
          --define IPHONE_SDK=1 \
          -c opt \
          --stamp \
          --embed_label "\${release_name}" \
          --workspace_status_command=scripts/ci/build_status_command.sh \
          --cpu=darwin_arm64 \
          src/bazel \
          src/bazel_nojdk \
          scripts/packages/with-jdk/install.sh

      mkdir artifacts
      cp "bazel-bin/src/bazel" "artifacts/bazel-\${release_name}-darwin-arm64"
      cp "bazel-bin/src/bazel_nojdk" "artifacts/bazel_nojdk-\${release_name}-darwin-arm64"
      cp "bazel-bin/scripts/packages/with-jdk/install.sh" "artifacts/bazel-\${release_name}-installer-darwin-arm64.sh"

      cd artifacts
      buildkite-agent artifact upload "*"

  - label: Windows
    agents:
      - queue=windows
    command: |
      set BUILDKITE_MESSAGE=%BUILDKITE_MESSAGE:~0,1000%

      git fetch origin ${BUILDKITE_BRANCH}
      git checkout ${BUILDKITE_BRANCH}

      buildkite-agent meta-data get "release_name" > release_name.txt
      SET /p RELEASE_NAME=<release_name.txt
      DEL /q release_name.txt

      echo Release: %RELEASE_NAME%

      bazel build //src:bazel.exe
      mkdir output
      copy bazel-bin\src\bazel.exe output\bazel.exe

      output\bazel build -c opt --copt=-w --host_copt=-w --stamp --embed_label %RELEASE_NAME% src/bazel src/bazel_nojdk scripts/packages/bazel.zip

      mkdir artifacts
      move bazel-bin\src\bazel artifacts\bazel-%RELEASE_NAME%-windows-x86_64.exe
      move bazel-bin\src\bazel_nojdk artifacts\bazel_nojdk-%RELEASE_NAME%-windows-x86_64.exe
      move bazel-bin\scripts\packages\bazel.zip artifacts\bazel-%RELEASE_NAME%-windows-x86_64.zip

      cd artifacts
      buildkite-agent artifact upload "*"

  - wait

  - label: "Test on CentOS 7"
    agents:
      - "queue=default"
    plugins:
      docker#v3.8.0:
        always-pull: true
        environment:
          - ANDROID_HOME
          - ANDROID_NDK_HOME
          - BUILDKITE_ARTIFACT_UPLOAD_DESTINATION
        image: gcr.io/bazel-public/centos7-java8
        network: host
        privileged: true
        propagate-environment: true
        propagate-uid-gid: true
        shell: ["/bin/bash", "-e", "-c"]
        volumes:
          - "/etc/group:/etc/group:ro"
          - "/etc/passwd:/etc/passwd:ro"
          - "/etc/shadow:/etc/shadow:ro"
          - "/opt/android-ndk-r15c:/opt/android-ndk-r15c:ro"
          - "/opt/android-sdk-linux:/opt/android-sdk-linux:ro"
          - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
          - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
          - "/var/run/docker.sock:/var/run/docker.sock"
    command: |
      git fetch origin ${BUILDKITE_BRANCH}
      git checkout ${BUILDKITE_BRANCH}

      release_name=$(buildkite-agent meta-data get "release_name")
      echo "release_name = \"\$release_name\""

      buildkite-agent artifact download "bazel-\${release_name}-linux-x86_64" .

      chmod +x "bazel-\${release_name}-linux-x86_64"

      "./bazel-\${release_name}-linux-x86_64" info

  - label: "Test on CentOS 7 (arm64)"
    agents:
      - "queue=arm64"
    plugins:
      docker#v3.8.0:
        environment:
          - BUILDKITE_ARTIFACT_UPLOAD_DESTINATION
          - GOOGLE_APPLICATION_CREDENTIALS
        image: gcr.io/bazel-public/centos7-java8
        network: host
        privileged: true
        propagate-environment: true
        propagate-uid-gid: true
        shell: ["/bin/bash", "-e", "-c"]
        volumes:
          - "/etc/group:/etc/group:ro"
          - "/etc/passwd:/etc/passwd:ro"
          - "/etc/shadow:/etc/shadow:ro"
          - "/opt/android-ndk-r15c:/opt/android-ndk-r15c:ro"
          - "/opt/android-sdk-linux:/opt/android-sdk-linux:ro"
          - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
          - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
          - "/var/run/docker.sock:/var/run/docker.sock"
    command: |
      git fetch origin ${BUILDKITE_BRANCH}
      git checkout FETCH_HEAD

      release_name=$(buildkite-agent meta-data get "release_name")
      echo "release_name = \"\$release_name\""

      buildkite-agent artifact download "bazel-\${release_name}-linux-arm64" .

      chmod +x "bazel-\${release_name}-linux-arm64"

      "./bazel-\${release_name}-linux-arm64" info

  - label: "Test on macOS"
    agents:
      - queue=macos
    command: |
      git fetch origin ${BUILDKITE_BRANCH}
      git checkout ${BUILDKITE_BRANCH}

      release_name=$(buildkite-agent meta-data get "release_name")
      echo "release_name = \"\$release_name\""

      buildkite-agent artifact download "bazel-\${release_name}-darwin-x86_64" .

      chmod +x "bazel-\${release_name}-darwin-x86_64"

      "./bazel-\${release_name}-darwin-x86_64" info

  - label: "Test on macOS (arm64)"
    agents:
      # TODO(pcloudy): Switch to macos_arm64 queue and write a proper test step
      # when Apple Silicon machines are available,
      - queue=macos
    command: |
      git fetch origin ${BUILDKITE_BRANCH}
      git checkout ${BUILDKITE_BRANCH}

      release_name=$(buildkite-agent meta-data get "release_name")
      echo "release_name = \"\$release_name\""

      buildkite-agent artifact download "bazel-\${release_name}-darwin-arm64" .

      output=$(lipo -info "bazel-\${release_name}-darwin-arm64")
      [[ \${output} == *"architecture: arm64"* ]] || (echo "architecture arm64 expected, but got: \"\${output}\"" && exit 1)

  - label: "Test on Windows"
    agents:
      - queue=windows
    command: |
      set BUILDKITE_MESSAGE=%BUILDKITE_MESSAGE:~0,1000%

      git fetch origin ${BUILDKITE_BRANCH}
      git checkout ${BUILDKITE_BRANCH}

      buildkite-agent meta-data get "release_name" > release_name.txt
      SET /p RELEASE_NAME=<release_name.txt
      DEL /q release_name.txt

      echo Release: %RELEASE_NAME%

      buildkite-agent artifact download bazel-%RELEASE_NAME%-windows-x86_64.exe .

      bazel-%RELEASE_NAME%-windows-x86_64.exe info

  - wait

  - block: ":rocket: Deploy release artifacts"

  - label: "Deploy release artifacts"
    agents:
      - "queue=default"
    plugins:
      docker#v3.8.0:
        always-pull: true
        environment:
          - ANDROID_HOME
          - ANDROID_NDK_HOME
          - BUILDKITE_ARTIFACT_UPLOAD_DESTINATION
        image: gcr.io/bazel-public/ubuntu1804-java11
        network: host
        privileged: true
        propagate-environment: true
        propagate-uid-gid: true
        shell: ["/bin/bash", "-e", "-c"]
        volumes:
          - "/etc/group:/etc/group:ro"
          - "/etc/passwd:/etc/passwd:ro"
          - "/etc/shadow:/etc/shadow:ro"
          - "/opt/android-ndk-r15c:/opt/android-ndk-r15c:ro"
          - "/opt/android-sdk-linux:/opt/android-sdk-linux:ro"
          - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
          - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
          - "/var/run/docker.sock:/var/run/docker.sock"
    command: |
      echo "+++ Installing required packages"
      sudo apt -y update && sudo apt -y install devscripts pandoc reprepro

      echo "+++ Downloading github-release"
      curl -L https://mirror.bazel.build/github.com/c4milo/github-release/releases/download/v1.1.0/github-release_v1.1.0_linux_amd64.tar.gz | sudo tar xz -C /usr/local/bin
      sudo chown root:root /usr/local/bin/github-release
      sudo chmod 0755 /usr/local/bin/github-release

      echo "+++ Checking out Git branch"
      git fetch origin ${BUILDKITE_BRANCH}
      git checkout ${BUILDKITE_BRANCH}

      echo "+++ Downloading release artifacts"
      ARTIFACTS="$(mktemp -d)"
      buildkite-agent artifact download "*" "\${ARTIFACTS}/"

      echo "+++ Importing GPG release key"
      keyfile="$(mktemp --tmpdir)"
      chmod 0600 "\${keyfile}"
      gsutil cat "gs://bazel-trusted-encrypted-secrets/release-key.gpg.enc" | gcloud kms decrypt --project "bazel-public" --location "global" --keyring "buildkite" --key "bazel-release-key" --ciphertext-file "-" --plaintext-file "\${keyfile}"
      gpg --allow-secret-key-import --import "\${keyfile}"
      rm -f "\${keyfile}"

      echo "+++ Deploying release"
      source scripts/ci/build.sh
      deploy_release "\${ARTIFACTS}"

  - block: ":e-mail: Generate announcement mail text"

  - label: "Generate announcement mail text"
    agents:
      - "queue=default"
    plugins:
      docker#v3.8.0:
        always-pull: true
        environment:
          - ANDROID_HOME
          - ANDROID_NDK_HOME
          - BUILDKITE_ARTIFACT_UPLOAD_DESTINATION
        image: gcr.io/bazel-public/ubuntu1804-java11
        network: host
        privileged: true
        propagate-environment: true
        propagate-uid-gid: true
        shell: ["/bin/bash", "-e", "-c"]
        volumes:
          - "/etc/group:/etc/group:ro"
          - "/etc/passwd:/etc/passwd:ro"
          - "/etc/shadow:/etc/shadow:ro"
          - "/opt/android-ndk-r15c:/opt/android-ndk-r15c:ro"
          - "/opt/android-sdk-linux:/opt/android-sdk-linux:ro"
          - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
          - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
          - "/var/run/docker.sock:/var/run/docker.sock"
    command: |
      echo "+++ Installing required packages"
      sudo apt -y update && sudo apt -y install devscripts pandoc reprepro

      echo "+++ Downloading github-release"
      curl -L https://mirror.bazel.build/github.com/c4milo/github-release/releases/download/v1.1.0/github-release_v1.1.0_linux_amd64.tar.gz | sudo tar xz -C /usr/local/bin
      sudo chown root:root /usr/local/bin/github-release
      sudo chmod 0755 /usr/local/bin/github-release

      echo "+++ Checking out Git branch"
      git fetch origin ${BUILDKITE_BRANCH}
      git checkout ${BUILDKITE_BRANCH}

      echo "+++ Generating e-mail text"
      source scripts/ci/build.sh
      generate_email