Fetch Buildkite agent tokens from Secret Manager (#2772)

...instead of decrypting files from GCS buckets.
This makes it easier to rotate any tokens.

Note: There are still other tokens left (GitHub etc) that use the old
mechanism. We should migrate them, too.

agent_metrics/README.md

diff --git a/agent_metrics/README.md b/agent_metrics/README.md
index 1a2544f..fb87f5d 100644
--- a/agent_metrics/README.md
+++ b/agent_metrics/README.md
@@ -24,6 +24,6 @@
 
 The service account used for the VM must have at least the following permissions:
 
-- `Cloud KMS Decryption` for the Buildkite agent tokens.
+- `Secret Manager Secret Accessor` for the secrets that store the Buildkite agent tokens.
 - `Logging > Logs Writer` to write logs to Stackdriver Logging.
 - `Monitoring > Monitoring Metric Writer` to write to the Stackdriver Metrics.

agent_metrics/start.sh

diff --git a/agent_metrics/start.sh b/agent_metrics/start.sh
index fbf4fca..8eb3049 100755
--- a/agent_metrics/start.sh
+++ b/agent_metrics/start.sh
@@ -5,18 +5,15 @@
 # Get all the Buildkite agent tokens.
 mkdir -p /etc/buildkite-agent-metrics
 cat > /etc/buildkite-agent-metrics/trusted <<EOF
-BUILDKITE_AGENT_TOKEN=$(gsutil cat "gs://bazel-trusted-encrypted-secrets/buildkite-trusted-agent-token.enc" | \
-    gcloud kms decrypt --project bazel-public --location global --keyring buildkite --key buildkite-trusted-agent-token --ciphertext-file - --plaintext-file -)
+BUILDKITE_AGENT_TOKEN=$(gcloud secrets versions access latest --secret="bazel-trusted-buildkite-agent-token" --project bazel-public)
 GCP_PROJECT=bazel-public
 EOF
 cat > /etc/buildkite-agent-metrics/testing <<EOF
-BUILDKITE_AGENT_TOKEN=$(gsutil cat "gs://bazel-testing-encrypted-secrets/buildkite-testing-agent-token.enc" | \
-    gcloud kms decrypt --project bazel-untrusted --location global --keyring buildkite --key buildkite-testing-agent-token --ciphertext-file - --plaintext-file -)
+BUILDKITE_AGENT_TOKEN=$(gcloud secrets versions access latest --secret="bazel-testing-buildkite-agent-token" --project bazel-untrusted)
 GCP_PROJECT=bazel-untrusted
 EOF
 cat > /etc/buildkite-agent-metrics/untrusted <<EOF
-BUILDKITE_AGENT_TOKEN=$(gsutil cat "gs://bazel-untrusted-encrypted-secrets/buildkite-untrusted-agent-token.enc" | \
-    gcloud kms decrypt --project bazel-untrusted --location global --keyring buildkite --key buildkite-untrusted-agent-token --ciphertext-file - --plaintext-file -)
+BUILDKITE_AGENT_TOKEN=$(gcloud secrets versions access latest --secret="bazel-buildkite-agent-token" --project bazel-untrusted)
 GCP_PROJECT=bazel-untrusted
 EOF
 

buildkite/startup-docker-pdssd.sh

diff --git a/buildkite/startup-docker-pdssd.sh b/buildkite/startup-docker-pdssd.sh
index 4c411ec..5d1e4fa 100644
--- a/buildkite/startup-docker-pdssd.sh
+++ b/buildkite/startup-docker-pdssd.sh
@@ -33,20 +33,17 @@
 case $(hostname -f) in
   *.bazel-public.*)
     ARTIFACT_BUCKET="bazel-trusted-buildkite-artifacts"
-    BUILDKITE_TOKEN=$(gsutil cat "gs://bazel-trusted-encrypted-secrets/buildkite-trusted-agent-token.enc" | \
-        gcloud kms decrypt --project bazel-public --location global --keyring buildkite --key buildkite-trusted-agent-token --ciphertext-file - --plaintext-file -)
+    BUILDKITE_TOKEN=$(gcloud secrets versions access latest --secret="bazel-trusted-buildkite-agent-token" --project bazel-public)
     ;;
   *.bazel-untrusted.*)
     case $(hostname -f) in
       *-testing-*)
         ARTIFACT_BUCKET="bazel-testing-buildkite-artifacts"
-        BUILDKITE_TOKEN=$(gsutil cat "gs://bazel-testing-encrypted-secrets/buildkite-testing-agent-token.enc" | \
-            gcloud kms decrypt --project bazel-untrusted --location global --keyring buildkite --key buildkite-testing-agent-token --ciphertext-file - --plaintext-file -)
+        BUILDKITE_TOKEN=$(gcloud secrets versions access latest --secret="bazel-testing-buildkite-agent-token" --project bazel-untrusted)
         ;;
       *)
         ARTIFACT_BUCKET="bazel-untrusted-buildkite-artifacts"
-        BUILDKITE_TOKEN=$(gsutil cat "gs://bazel-untrusted-encrypted-secrets/buildkite-untrusted-agent-token.enc" | \
-            gcloud kms decrypt --project bazel-untrusted --location global --keyring buildkite --key buildkite-untrusted-agent-token --ciphertext-file - --plaintext-file -)
+        BUILDKITE_TOKEN=$(gcloud secrets versions access latest --secret="bazel-buildkite-agent-token" --project bazel-untrusted)
         ;;
     esac
 esac

buildkite/startup-windows-pdssd.ps1

diff --git a/buildkite/startup-windows-pdssd.ps1 b/buildkite/startup-windows-pdssd.ps1
index 7f07306..10dd724 100644
--- a/buildkite/startup-windows-pdssd.ps1
+++ b/buildkite/startup-windows-pdssd.ps1
@@ -121,46 +121,14 @@
 ## Write encrypted buildkite agent token into a file.

 $myhostname = [System.Net.Dns]::GetHostName()

 if ($myhostname -like "*trusted*") {

-  $buildkite_agent_token_url = "https://storage.googleapis.com/bazel-trusted-encrypted-secrets/buildkite-trusted-agent-token.enc"

   $project = "bazel-public"

-  $key = "buildkite-trusted-agent-token"

+  $secret = "bazel-trusted-buildkite-agent-token"

 } elseif ($myhostname -like "*testing*") {

-  $buildkite_agent_token_url = "https://storage.googleapis.com/bazel-testing-encrypted-secrets/buildkite-testing-agent-token.enc"

   $project = "bazel-untrusted"

-  $key = "buildkite-testing-agent-token"

+  $secret = "bazel-testing-buildkite-agent-token"

 } else {

-  $buildkite_agent_token_url = "https://storage.googleapis.com/bazel-untrusted-encrypted-secrets/buildkite-untrusted-agent-token.enc"

   $project = "bazel-untrusted"

-  $key = "buildkite-untrusted-agent-token"

-}

-$buildkite_agent_token_file = "c:\buildkite\buildkite_agent_token.enc"

-

-Write-Host "Getting Buildkite Agent token from GCS..."

-

-# Try at most 30 times to download file, if not succeeding, shutdown the VM.

-$maxAttempts = 30

-$attemptCount = 0

-

-while ($attemptCount -lt $maxAttempts) {

-  try {

-    (New-Object Net.WebClient).DownloadFile($buildkite_agent_token_url, $buildkite_agent_token_file)

-    Write-Host "Token downloaded successfully."

-    break

-  } catch {

-    $msg = $_.Exception.Message

-    Write-Host "Failed to download token: $msg"

-

-    $attemptCount++

-    Write-Host "Attempt $attemptCount of $maxAttempts"

-

-    Start-Sleep -Seconds 10

-  }

-}

-

-# Check if maximum attempts were reached and shut down if necessary

-if ($attemptCount -eq $maxAttempts) {

-  Write-Host "Maximum attempts reached. Shutting down the machine..."

-  Stop-Computer

+  $secret = "bazel-buildkite-agent-token"

 }

 

 # Fix ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1129)

@@ -172,10 +140,9 @@
 & openssl.exe pkcs12 -export -nokeys -out certs.pfx -in cacert.pem -passout pass:$plaintext_pw;

 Import-PfxCertificate -Password $secure_pw  -CertStoreLocation Cert:\LocalMachine\Root -FilePath certs.pfx;

 

-## Decrypt the Buildkite agent token.

-Write-Host "Decrypting Buildkite Agent token using KMS..."

-$buildkite_agent_token = & gcloud kms decrypt --project $project --location global --keyring buildkite --key $key --ciphertext-file $buildkite_agent_token_file --plaintext-file -

-Remove-Item $buildkite_agent_token_file

+## Get the Buildkite agent token.

+Write-Host "Retrieving Buildkite Agent token from Secret Manager..."

+$buildkite_agent_token = & gcloud secrets versions access latest --secret=$secret --project $project

 

 ## Configure the Buildkite agent.

 Write-Host "Configuring Buildkite Agent..."