[7.7.2] Cherry-pick release script fixes (gh CLI, gcloud storage, Secret Manager) (#31974) Cherry-picks release script updates required by the `bazel-release` pipeline onto `release-7.7.2`: - a49b777ccfc1d69bee090471ed95e76f35e1ddb2 (`Fix: Prevent GitHub 'Latest' badge regression on maintenance releases (#29452)` — migrates `scripts/ci/build.sh` from `github-release` to `gh` CLI and prevents maintenance releases from hijacking the GitHub Latest badge) - 8ff7ba386c819af4000fc71b573894428322a0bc (`Migrate from gsutil to gcloud storage in Bazel release/CI scripts. (#29771)`) - 6aa589e8935d9d40c02c69c472d8dc2583d27e0f (`Read tokens from secret manager (#31149)`) - 27ac55328ac3dab00715be231c4ab83a2f13305b (`[8.8.0] Fix the release.sh (#30419)`) --------- Co-authored-by: deepalak56 <deepalakshmib@google.com> Co-authored-by: Salma Samy <16399431+SalmaSamy@users.noreply.github.com>
diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index 2628c66..e1f8850 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh
@@ -28,15 +28,14 @@ source "$(dirname ${BUILD_SCRIPT_DIR})/release/common.sh" source "$(dirname ${BUILD_SCRIPT_DIR})/release/relnotes.sh" -if ! command -v gsutil &>/dev/null; then - echo "Required tool 'gsutil' not found. Please install it:" +if ! command -v gcloud &>/dev/null; then + echo "Required tool 'gcloud' not found. Please install it:" echo "See https://cloud.google.com/sdk/downloads for instructions." exit 1 fi -if ! command -v github-release &>/dev/null; then - echo "Required tool 'github-release' not found. Download it from here:" - echo "https://github.com/c4milo/github-release/releases" - echo "Just extract the archive and put the binary on your PATH." +if ! command -v gh &>/dev/null; then + echo "Required tool 'gh' (GitHub CLI) not found. Please install it:" + echo "See https://cli.github.com/ for instructions." exit 1 fi if ! command -v debsign &>/dev/null; then @@ -65,7 +64,7 @@ # exit 1 # fi -export APT_GPG_KEY_ID=$(gsutil cat gs://bazel-trusted-encrypted-secrets/release-key.gpg.id) +export APT_GPG_KEY_ID=$(gcloud secrets versions access latest --secret="release-key-gpg-id" --project="bazel-public") # Generate a string from a template and a list of substitutions. # The first parameter is the template name and each subsequent parameter @@ -136,8 +135,8 @@ ' } -# Deploy a github release using a third party tool: -# https://github.com/c4milo/github-release +# Deploy a github release using the official GitHub CLI (gh): +# https://cli.github.com/ # This methods expects the following arguments: # $1..$n files generated by package_build (should not contains the README file) # Please set GITHUB_TOKEN to talk to the Github API. @@ -150,13 +149,54 @@ local release_branch=$(get_release_branch) if [ -n "${release_name}" ]; then - local github_token="$(gsutil cat gs://bazel-trusted-encrypted-secrets/github-trusted-token.enc | \ - gcloud kms decrypt --project bazel-public --location global --keyring buildkite --key github-trusted-token --ciphertext-file - --plaintext-file -)" - if [ -z "${rc}" ]; then - GITHUB_TOKEN="${github_token}" github-release "bazelbuild/bazel" "${release_name}" "" "$(get_release_page)" "${artifact_dir}/*" + local github_token + github_token="$(gcloud secrets versions access latest --secret="github-trusted-token" --project="bazel-public")" + + local latest_flag="true" + local prerelease_flag="" + local tag_to_deploy="${release_name}" + + if [ -n "${rc}" ]; then + tag_to_deploy="${full_release_name}" + prerelease_flag="--prerelease" + latest_flag="false" else - GITHUB_TOKEN="${github_token}" github-release -prerelease "bazelbuild/bazel" "${full_release_name}" "${release_branch}" "$(get_release_page)" "${artifact_dir}/*" + echo "+++ Checking if ${release_name} should be marked as Latest" + # Query the current GitHub "latest" release to compare versions. + local current_latest_tag + if current_latest_tag=$(GH_TOKEN="${github_token}" gh release view --repo "bazelbuild/bazel" --json tagName --jq .tagName 2>/dev/null); then + if [[ -n "${current_latest_tag}" ]]; then + local highest + highest=$(printf '%s\n%s' "${current_latest_tag}" "${release_name}" | sort -V | tail -n 1) + if [[ "${release_name}" != "${highest}" ]]; then + echo "+++ Version ${release_name} is older than ${current_latest_tag}. Will NOT mark as latest." + latest_flag="false" + fi + fi + else + echo "+++ Warning: Could not determine current latest release tag. Will NOT mark as latest to be safe." + latest_flag="false" + fi fi + + # Use a subshell so that the EXIT trap for temp file cleanup does not + # affect the outer script's traps. + ( + notes_file="$(mktemp)" + trap 'rm -f "$notes_file"' EXIT + get_release_page > "$notes_file" + + echo "+++ Deploying to GitHub (Tag: ${tag_to_deploy}, Latest: ${latest_flag})" + + GH_TOKEN="${github_token}" gh release create "${tag_to_deploy}" \ + "${artifact_dir}"/* \ + --repo "bazelbuild/bazel" \ + --target "${release_branch}" \ + --title "${tag_to_deploy}" \ + --notes-file "$notes_file" \ + ${prerelease_flag} \ + --latest="${latest_flag}" + ) fi } @@ -180,8 +220,7 @@ } # Deploy a release candidate to Google Cloud Storage. -# It requires to have gsutil installed. You can force the path to gsutil -# by setting the GSUTIL environment variable. +# It requires to have gcloud storage installed. # This methods expects the following arguments: # $1..$n files generated by package_build function release_to_gcs() { @@ -199,9 +238,9 @@ release_path="${release_name}/rc${rc}" fi create_index_html "${artifact_dir}" > "${artifact_dir}/index.html" - gsutil -m cp "${artifact_dir}/**" "gs://bazel/${release_path}" + gcloud storage cp "${artifact_dir}/**" "gs://bazel/${release_path}" # Set the content type on index.html so it isn't autodetected incorrectly by the browser. - gsutil setmeta -h "Content-Type: text/html; charset=utf-8" "gs://bazel/${release_path}/index.html" + gcloud storage objects update "gs://bazel/${release_path}/index.html" --content-type="text/html; charset=utf-8" fi } @@ -209,8 +248,7 @@ if ! gpg --list-secret-keys | grep "${APT_GPG_KEY_ID}" > /dev/null; then keyfile=$(mktemp --tmpdir) chmod 0600 "${keyfile}" - gsutil cat "gs://bazel-trusted-encrypted-secrets/release-key.gpg.enc" | \ - gcloud kms decrypt --location "global" --keyring "buildkite" --key "bazel-release-key" --ciphertext-file "-" --plaintext-file "${keyfile}" + gcloud secrets versions access latest --secret="bazel-release-key" --project="bazel-public" --out-file="${keyfile}" gpg --allow-secret-key-import --import "${keyfile}" rm -f "${keyfile}" fi @@ -261,7 +299,7 @@ local distribution="$1" # Download the metadata info from previous distribution mkdir -p previous - gsutil -m cp -r "gs://bazel-apt/dists" "./previous" + gcloud storage cp -r "gs://bazel-apt/dists" "./previous" # Merge Packages and Packages.gz file cat "previous/dists/${distribution}/jdk1.8/binary-amd64/Packages" >> "dists/${distribution}/jdk1.8/binary-amd64/Packages" @@ -366,7 +404,7 @@ merge_previous_dists "${distribution}" - gsutil -m cp -r dists pool "gs://bazel-apt" + gcloud storage cp -r dists pool "gs://bazel-apt" } function release_to_apt() { @@ -437,4 +475,3 @@ rm -f "${github_working_dir}/bazel_${release_label}"*.{dsc,tar.gz}{,.sha256,.sig} release_to_github "${github_working_dir}" } -
diff --git a/scripts/release/release.sh b/scripts/release/release.sh index f5bb2db..0f57508 100755 --- a/scripts/release/release.sh +++ b/scripts/release/release.sh
@@ -191,16 +191,30 @@ # destroying the release branch, updating the master's CHANGELOG.md # and pushing everything to GitHub. function __do_release() { - local branch=$(get_release_branch) + local current_branch=$(get_release_branch) local tag_name=$(get_release_name) local candidate=$(get_release_candidate) + local release_branch="release-${tag_name}" - echo -n "You are about to release branch ${branch} in tag ${tag_name}, confirm? [y/N] " + local current_commit="$(__git_commit_hash "${current_branch}")" + local release_commit="$(__git_commit_hash "${release_branch}" 2>/dev/null || true)" + if [ -z "${candidate}" ] || [ -z "${release_commit}" ] || [ "${current_commit}" != "${release_commit}" ]; then + echo "The current branch ${current_branch} must be the latest release candidate (release-X.Y.ZrcN) and at the same commit as branch ${release_branch}." >&2 + exit 1 + fi + + echo -n "You are about to release branch ${release_branch} in tag ${tag_name}, confirm? [y/N] " read answer if [ "$answer" = "y" ] || [ "$answer" = "Y" ]; then + echo "Switching to the release branch" + git checkout "${release_branch}" + echo "Creating the release commit" __create_release_commit "${tag_name}" + echo "Pushing the release branch" + __push_ref "${release_branch}" + echo "Creating the tag" git tag ${tag_name} @@ -213,10 +227,10 @@ # We do not cherry-pick because we might have conflict if the baseline # does not contains the latest CHANGELOG.md file, so trick it. local changelog_path="$PWD/CHANGELOG.md" - git show "${branch}:CHANGELOG.md" > "${changelog_path}" + git show "${release_branch}:CHANGELOG.md" > "${changelog_path}" local tmpfile=$(mktemp --tmpdir relnotes-XXXXXXXX) trap 'rm -f ${tmpfile}' EXIT - git_commit_msg "${branch}" > "${tmpfile}" + git_commit_msg "${release_branch}" > "${tmpfile}" git add "${changelog_path}" git commit --no-verify -F "${tmpfile}" --no-edit --author "${RELEASE_AUTHOR}" rm -f "${tmpfile}"
diff --git a/scripts/release/relnotes.py b/scripts/release/relnotes.py index 8d42ed3..9e4d034 100644 --- a/scripts/release/relnotes.py +++ b/scripts/release/relnotes.py
@@ -96,13 +96,20 @@ def get_label(issue_id): """Get team-X label added to issue.""" - auth = subprocess.check_output( - "gsutil cat" - " gs://bazel-trusted-encrypted-secrets/github-trusted-token.enc |" - " gcloud kms decrypt --project bazel-public --location global" - " --keyring buildkite --key github-trusted-token --ciphertext-file" - " - --plaintext-file -", shell=True - ).decode("utf-8").strip().split("\n")[0] + auth = ( + subprocess.check_output([ + "gcloud", + "secrets", + "versions", + "access", + "latest", + "--secret=github-trusted-token", + "--project=bazel-public", + ]) + .decode("utf-8") + .strip() + .split("\n")[0] + ) headers = { "Authorization": "Bearer " + auth, "Accept": "application/vnd.github+json",
diff --git a/src/create_java_tools_release.sh b/src/create_java_tools_release.sh index fd3f770..51ba074 100755 --- a/src/create_java_tools_release.sh +++ b/src/create_java_tools_release.sh
@@ -19,7 +19,7 @@ # binaries pipeline was previously run and generated java_tools artifacts at # a commit hash. # -# The script is using gsutil to copy artifacts. +# The script is using gcloud storage to copy artifacts. # # Mandatory flags: # --java_tools_version The version number of the java_tools to be released. @@ -84,7 +84,7 @@ mirror_url=${mirror_prefix}/${release_artifact} urls='"mirror_url" : "'${mirror_url}'", "github_url" : "'${github_url}'"' else - tmp_url=$(gsutil ls -lh ${gcs_bucket}/tmp/build/${commit_hash}/java/java_tools_${platform}* | sort -k 2 | grep gs -m 1 | awk '{print $4}') + tmp_url=$(gcloud storage ls -l ${gcs_bucket}/tmp/build/${commit_hash}/java/java_tools_${platform}* | sort -k 2 | grep gs -m 1 | awk '{print $3}') # Make the generated artifact a release candidate for the current platform. # Don't overwrite existing file. @@ -98,7 +98,7 @@ # java_tools in Bazel). # Don't overwrite existing file. local_zip="$tmp_dir/java_tools$platform.zip" - gsutil -q cp -n ${gcs_bucket}/${rc_url} ${local_zip} + gcloud --quiet storage cp -n ${gcs_bucket}/${rc_url} ${local_zip} file_hash=$(sha256sum ${local_zip} | cut -d' ' -f1) platform_output+='"java_tools_'${platform}'" : {'${urls}', "sha": "'${file_hash}'"},' @@ -122,13 +122,13 @@ mirror_url=${mirror_prefix}/${release_artifact} urls='"mirror_url" : "'${mirror_url}'", "github_url" : "'${github_url}'"' else - tmp_url=$(gsutil ls -lh ${gcs_bucket}/tmp/build/${commit_hash}/java/java_tools-* | sort -k 2 | grep gs -m 1 | awk '{print $4}') + tmp_url=$(gcloud storage ls -l ${gcs_bucket}/tmp/build/${commit_hash}/java/java_tools-* | sort -k 2 | grep gs -m 1 | awk '{print $3}') gsutil -q cp -n ${tmp_url} "${gcs_bucket}/${rc_url}" # Copy the associated zip file that contains the sources of the release zip. # Don't overwrite existing file. - tmp_sources_url=$(gsutil ls -lh ${gcs_bucket}/tmp/sources/${commit_hash}/java/java_tools-* | sort -k 2 | grep gs -m 1 | awk '{print $4}') + tmp_sources_url=$(gcloud storage ls -l ${gcs_bucket}/tmp/sources/${commit_hash}/java/java_tools-* | sort -k 2 | grep gs -m 1 | awk '{print $3}') gsutil -q cp -n ${tmp_sources_url} ${gcs_bucket}/${rc_sources_url} mirror_url=${mirror_prefix}/${rc_url} @@ -139,7 +139,7 @@ # java_tools in Bazel). # Don't overwrite existing file. local_zip="$tmp_dir/java_tools.zip" -gsutil -q cp -n ${gcs_bucket}/${rc_url} ${local_zip} +gcloud --quiet storage cp -n ${gcs_bucket}/${rc_url} ${local_zip} file_hash=$(sha256sum ${local_zip} | cut -d' ' -f1) java_tools_output='"java_tools" : {'${urls}', "sha" : "'${file_hash}'"}'
diff --git a/src/upload_java_tools.sh b/src/upload_java_tools.sh index e9a6fa8..f8a9e69 100755 --- a/src/upload_java_tools.sh +++ b/src/upload_java_tools.sh
@@ -104,9 +104,9 @@ # Add the README.md file to the temp zip. zip -rv "${tmp_zip}" "${readme_file}" -gsutil_cmd="gsutil" +gcloud_cmd="gcloud" if "$is_windows"; then - gsutil_cmd="gsutil.cmd" + gcloud_cmd="gcloud.cmd" fi @@ -118,5 +118,5 @@ fi # Upload the zip that contains the README.md to GCS. -"$gsutil_cmd" cp "$zip_url" \ +"$gcloud_cmd" storage cp "$zip_url" \ "gs://bazel-mirror/bazel_java_tools/${gcs_java_tools_dir}/${commit_hash}/java/java_tools${platform:+"_${platform}"}-${timestamp}.zip"
diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/build_and_upload.sh b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/build_and_upload.sh index bc4dd57..be9c0e7 100755 --- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/build_and_upload.sh +++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/build_and_upload.sh
@@ -16,6 +16,6 @@ bazel build //tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:coverage_output_generator.zip DEST=bazel_coverage_output_generator/coverage_output_generator-$(git rev-parse HEAD)-$(date +%s).zip -gsutil cp "tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/coverage_output_generator.zip" \ +gcloud storage cp "tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/coverage_output_generator.zip" \ "gs://bazel-mirror/${DEST}" echo "Uploaded to $DEST"
diff --git a/tools/upload_bazel_mirror.sh b/tools/upload_bazel_mirror.sh index 9462958..a797ed90 100755 --- a/tools/upload_bazel_mirror.sh +++ b/tools/upload_bazel_mirror.sh
@@ -63,5 +63,5 @@ # Upload the tarball to GCS. # -n for no-clobber, so we don't overwrite existing files -gsutil cp -n "${versioned_archive}" \ +gcloud storage cp -n "${versioned_archive}" \ "gs://bazel-mirror/github.com/bazelbuild/${REPO}/archive/${VERSION}.tar.gz"