Fix Linux VM image creation and handle gcloud SQLite concurrency collisions (#2769)
### Summary of Changes
1. **`buildkite/create_images.py`**:
- Use machine type `c4a-standard-8` for the `bk-testing-docker-arm64`
image builder VM. Local SSDs (`-lssd`) are ephemeral, not captured in
custom GCE images, and caused capacity stockouts
(`ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS`) in `us-central1`.
2. **`buildkite/gcloud.py`**:
- Add automatic retry for transient `gcloud crashed (OperationalError)`
/ SQLite schema migration collisions when multiple threads invoke
`gcloud` concurrently.
- Example failure:
https://buildkite.com/bazel-trusted/create-linux-vm-image/builds/43#019ff552-6791-4b7a-95c4-2c7c882bcfd5
diff --git a/buildkite/create_images.py b/buildkite/create_images.py
index 81a923e..b0dfd93 100755
--- a/buildkite/create_images.py
+++ b/buildkite/create_images.py
@@ -48,7 +48,7 @@
"licenses": [
"https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx"
],
- "machine_type": "c4a-standard-8-lssd",
+ "machine_type": "c4a-standard-8",
"boot_disk_type": "hyperdisk-balanced",
},
"bk-testing-windows": {
diff --git a/buildkite/gcloud.py b/buildkite/gcloud.py
index 6d8b585..f55f53e 100755
--- a/buildkite/gcloud.py
+++ b/buildkite/gcloud.py
@@ -19,6 +19,7 @@
import re
import subprocess
import threading
+import time
DEBUG = True
PRINT_LOCK = threading.Lock()
@@ -57,9 +58,24 @@
if not "get-serial-port-output" in cmd:
debug("Running: " + " ".join(cmd))
- return subprocess.run(
- cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, check=True
- )
+ max_attempts = 3
+ for attempt in range(max_attempts):
+ try:
+ return subprocess.run(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, check=True
+ )
+ except subprocess.CalledProcessError as e:
+ # When multiple threads invoke gcloud concurrently in a fresh environment,
+ # multiple processes race to initialize and migrate gcloud's local SQLite
+ # database (~/.config/gcloud/), causing transient collisions such as:
+ # "ERROR: gcloud crashed (OperationalError): duplicate column name: regional_access_boundary".
+ # Example failure: https://buildkite.com/bazel-trusted/create-linux-vm-image/builds/43#019ff552-6791-4b7a-95c4-2c7c882bcfd5
+ # Retrying allows subsequent attempts to succeed once the initial migration completes.
+ if attempt < max_attempts - 1 and "OperationalError" in e.stderr:
+ debug(f"gcloud crashed with OperationalError on attempt {attempt + 1}, retrying...")
+ time.sleep(1)
+ continue
+ raise
def create_instance(name, **kwargs):