Add a gcloud wrapper that can be used by our infra scripts.
Refactor create_images.py and create_instances.py to use it.
diff --git a/buildkite/create_images.py b/buildkite/create_images.py
index 40ca2c8..96f88e3 100755
--- a/buildkite/create_images.py
+++ b/buildkite/create_images.py
@@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from datetime import datetime
import json
import os
import queue
@@ -23,7 +24,9 @@
import tempfile
import threading
import time
-from datetime import datetime
+
+import gcloud
+import gcloud_utils
DEBUG = False
@@ -81,6 +84,26 @@
'https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx'
]
},
+ 'philwo-ubuntu1604': {
+ 'source_image_project': 'ubuntu-os-cloud',
+ 'source_image_family': 'ubuntu-1604-lts',
+ 'target_image_family': 'philwo-ubuntu1604',
+ 'scripts': [
+ 'shell-utils.sh',
+ 'setup-ubuntu.sh',
+ 'install-azul-zulu.sh',
+ 'install-bazel.sh',
+ 'install-buildkite-agent.sh',
+ 'install-docker.sh',
+ 'install-nodejs.sh',
+ 'install-python36.sh',
+ 'install-android-sdk.sh',
+ 'shutdown.sh'
+ ],
+ 'licenses': [
+ 'https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx'
+ ]
+ },
'buildkite-pipeline-ubuntu1604': {
'source_image_project': 'ubuntu-os-cloud',
'source_image_family': 'ubuntu-1604-lts',
@@ -104,71 +127,15 @@
}
}
-PRINT_LOCK = threading.Lock()
WORK_QUEUE = queue.Queue()
-def debug(*args, **kwargs):
- if DEBUG:
- with PRINT_LOCK:
- print(*args, **kwargs)
-
-
def run(args, **kwargs):
- debug('Running: %s' % ' '.join(args))
return subprocess.run(args, **kwargs)
-def wait_for_vm(vm, status):
- while True:
- cmd = ['gcloud', 'compute', 'instances', 'describe', vm]
- cmd += ['--zone', LOCATION, '--format', 'json']
- result = run(cmd, check=True, stdout=subprocess.PIPE)
- current_status = json.loads(result.stdout)['status']
- if current_status == status:
- debug("wait_for_vm: VM %s reached status %s" % (vm, status))
- break
- else:
- debug("wait_for_vm: Waiting for VM %s to transition from status %s -> %s" %
- (vm, current_status, status))
- time.sleep(1)
-
-
-def print_pretty_logs(vm, log):
- with PRINT_LOCK:
- for line in log.splitlines():
- # Skip empty lines.
- if not line:
- continue
- if 'ubuntu' in vm:
- match = re.match(r'.*INFO startup-script: (.*)', line)
- if match:
- print("%s: %s" % (vm, match.group(1)))
- # elif 'windows' in vm:
- # match = re.match(r'.*windows-startup-script-ps1: (.*)', line)
- # if match:
- # print(match.group(1))
- else:
- print("%s: %s" % (vm, line))
-
-
-def tail_serial_console(vm, start=None, until=None):
- next_start = start if start else '0'
- while True:
- cmd = ['gcloud', 'compute', 'instances', 'get-serial-port-output', vm]
- cmd += ['--zone', LOCATION, '--start', next_start]
- result = run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
- if result.returncode != 0:
- break
- print_pretty_logs(vm, result.stdout)
- next_start = re.search(r'--start=(\d*)', result.stderr).group(1)
- if until and until in result.stdout:
- break
- return next_start
-
-
-def merge_setup_scripts(vm, scripts):
- newline = '\r\n' if 'windows' in vm else None
+def merge_setup_scripts(instance_name, scripts):
+ newline = '\r\n' if 'windows' in instance_name else None
# Merge all setup scripts into one.
merged_script_path = tempfile.mkstemp()[1]
with open(merged_script_path, 'w', newline=newline) as merged_script_file:
@@ -178,44 +145,37 @@
return merged_script_path
-def create_vm(vm, params):
- merged_script_path = merge_setup_scripts(vm, params['scripts'])
+def create_instance(instance_name, params):
+ merged_script_path = merge_setup_scripts(instance_name, params['scripts'])
try:
- cmd = ['gcloud', 'compute', 'instances', 'create', vm]
- cmd += ['--zone', LOCATION]
- cmd += ['--machine-type', 'n1-standard-8']
- cmd += ['--network', 'buildkite']
- if 'windows' in vm:
- cmd += ['--metadata-from-file', 'windows-startup-script-ps1=' + merged_script_path]
+ if 'windows' in instance_name:
+ startup_script = 'windows-startup-script-ps1=' + merged_script_path
else:
- cmd += ['--metadata-from-file', 'startup-script=' + merged_script_path]
- cmd += ['--min-cpu-platform', 'Intel Skylake']
- cmd += ['--boot-disk-type', 'pd-ssd']
- cmd += ['--boot-disk-size', '50GB']
+ startup_script = 'startup-script=' + merged_script_path
+
if 'source_image' in params:
- cmd += ['--image', params['source_image']]
+ image = {
+ 'image': params['source_image']
+ }
else:
- cmd += ['--image-project', params['source_image_project']]
- cmd += ['--image-family', params['source_image_family']]
- run(cmd)
+ image = {
+ 'image-project': params['source_image_project'],
+ 'image-family': params['source_image_family']
+ }
+
+ gcloud.create_instance(instance_name,
+ zone=LOCATION,
+ machine_type='n1-standard-8',
+ network='buildkite',
+ metadata_from_file=startup_script,
+ min_cpu_platform='Intel Skylake',
+ boot_disk_type='pd-ssd',
+ boot_disk_size='50GB',
+ **image)
finally:
os.remove(merged_script_path)
-def delete_vm(vm):
- run(['gcloud', 'compute', 'instances', 'delete', vm, '--quiet', '--zone', LOCATION])
-
-
-def create_image(vm, params):
- cmd = ['gcloud', 'compute', 'images', 'create', vm]
- cmd += ['--family', params['target_image_family']]
- cmd += ['--source-disk', vm]
- cmd += ['--source-disk-zone', LOCATION]
- for license in params.get('licenses', []):
- cmd += ['--licenses', license]
- run(cmd)
-
-
# https://stackoverflow.com/a/25802742
def write_to_clipboard(output):
process = subprocess.Popen(
@@ -223,25 +183,23 @@
process.communicate(output.encode('utf-8'))
-def print_windows_instructions(vm):
- tail_start = tail_serial_console(vm, until='Finished running startup scripts')
+def print_windows_instructions(instance_name):
+ tail_start = gcloud_utils.tail_serial_console(instance_name, zone=LOCATION, until='Finished running startup scripts')
- cmd = ['gcloud', 'compute', 'reset-windows-password', vm]
- cmd += ['--format', 'json', '--quiet', '--zone', LOCATION]
-
- pw = json.loads(run(cmd, check=True, stdout=subprocess.PIPE).stdout)
+ pw = json.loads(gcloud.reset_windows_password(instance_name, format='json', zone=LOCATION).stdout)
rdp_file = tempfile.mkstemp(suffix='.rdp')[1]
with open(rdp_file, 'w') as f:
f.write('full address:s:' + pw['ip_address'] + '\n')
f.write('username:s:' + pw['username'] + '\n')
- run(['open', rdp_file])
+ subprocess.run(['open', rdp_file])
write_to_clipboard(pw['password'])
- with PRINT_LOCK:
+ with gcloud.PRINT_LOCK:
print('Use this password to connect to the Windows VM: ' + pw['password'])
print('Please run the setup script C:\\setup.ps1 once you\'re logged in.')
# Wait until the VM reboots once, then open RDP again.
- tail_start = tail_serial_console(vm, start=tail_start, until='Finished running startup scripts')
+ tail_start = gcloud_utils.tail_serial_console(
+ instance_name, zone=LOCATION, start=tail_start, until='Finished running startup scripts')
print('Connecting via RDP a second time to finish the setup...')
write_to_clipboard(pw['password'])
run(['open', rdp_file])
@@ -249,30 +207,34 @@
def workflow(name, params):
- vm = "%s-image-%s" % (name, int(datetime.now().timestamp()))
+ instance_name = "%s-image-%s" % (name, int(datetime.now().timestamp()))
try:
# Create the VM.
- create_vm(vm, params)
+ create_instance(instance_name, params)
# Wait for the VM to become ready.
- wait_for_vm(vm, 'RUNNING')
+ gcloud_utils.wait_for_instance(instance_name, zone=LOCATION, status='RUNNING')
- if 'windows' in vm:
+ if 'windows' in instance_name:
# Wait for VM to be ready, then print setup instructions.
- tail_start = print_windows_instructions(vm)
+ tail_start = print_windows_instructions(instance_name)
# Continue printing the serial console until the VM shuts down.
- tail_serial_console(vm, start=tail_start)
+ gcloud_utils.tail_serial_console(instance_name, zone=LOCATION, start=tail_start)
else:
# Continuously print the serial console.
- tail_serial_console(vm)
+ gcloud_utils.tail_serial_console(instance_name, zone=LOCATION)
# Wait for the VM to completely shutdown.
- wait_for_vm(vm, 'TERMINATED')
+ gcloud_utils.wait_for_instance(instance_name, zone=LOCATION, status='TERMINATED')
# Create a new image from our VM.
- create_image(vm, params)
+ gcloud.create_image(instance_name,
+ family=params['target_image_family'],
+ source_disk=instance_name,
+ source_disk_zone=LOCATION,
+ licenses=params.get('licenses', []))
finally:
- delete_vm(vm)
+ gcloud.delete_instance(instance_name)
def worker():
diff --git a/buildkite/create_instances.py b/buildkite/create_instances.py
index cfaf184..3caf5a3 100755
--- a/buildkite/create_instances.py
+++ b/buildkite/create_instances.py
@@ -21,6 +21,9 @@
import sys
import threading
+import gcloud
+
+
DEBUG = True
LOCATION = 'europe-west1-d'
@@ -35,68 +38,63 @@
# - "$USER": This is a VM used by one specific engineer for tests. It does not run the Buildkite
# agent.
#
+DEFAULT_VM = {
+ 'boot_disk_size': '50GB',
+ 'boot_disk_type': 'pd-ssd',
+ 'image_project': 'bazel-public',
+ 'machine_type': 'n1-standard-32',
+ 'min_cpu_platform': 'Intel Skylake',
+ 'network': 'buildkite',
+ 'scopes': 'cloud-platform',
+ 'service_account': 'remote-account@bazel-public.iam.gserviceaccount.com',
+}
+
INSTANCE_GROUPS = {
'buildkite-ubuntu1404': {
'count': 8,
- 'startup_script': 'startup-ubuntu.sh',
- 'machine_type': 'n1-standard-32',
+ 'image_family': 'buildkite-ubuntu1404',
'local_ssd': 'interface=nvme',
+ 'metadata_from_file': 'startup-script=startup-ubuntu.sh',
},
'buildkite-ubuntu1604': {
'count': 8,
- 'startup_script': 'startup-ubuntu.sh',
- 'machine_type': 'n1-standard-32',
+ 'image_family': 'buildkite-ubuntu1604',
'local_ssd': 'interface=nvme',
+ 'metadata_from_file': 'startup-script=startup-ubuntu.sh',
},
'buildkite-windows': {
'count': 4,
- 'startup_script': 'startup-windows.ps1',
- 'machine_type': 'n1-standard-32',
+ 'image_family': 'buildkite-windows',
'local_ssd': 'interface=scsi',
+ 'metadata_from_file': 'windows-startup-script-ps1=startup-windows.ps1',
},
}
SINGLE_INSTANCES = {
'buildkite-pipeline-ubuntu1604': {
- 'startup_script': 'startup-ubuntu.sh',
+ 'image_family': 'buildkite-pipeline-ubuntu1604',
'machine_type': 'n1-standard-8',
- 'persistent_disk': 'buildkite-pipeline-persistent'
+ 'metadata_from_file': 'startup-script=startup-ubuntu.sh',
+ 'persistent_disk': 'name={0},device-name={0},mode=rw,boot=no'.format('buildkite-pipeline-persistent'),
},
'testing-ubuntu1404': {
'image_family': 'buildkite-ubuntu1404',
- 'startup_script': 'startup-ubuntu.sh',
- 'machine_type': 'n1-standard-32',
- 'persistent_disk': 'testing-ubuntu1404-persistent'
+ 'metadata_from_file': 'startup-script=startup-ubuntu.sh',
+ 'persistent_disk': 'name={0},device-name={0},mode=rw,boot=no'.format('testing-ubuntu1404-persistent'),
},
'testing-ubuntu1604': {
'image_family': 'buildkite-ubuntu1604',
- 'startup_script': 'startup-ubuntu.sh',
- 'machine_type': 'n1-standard-32',
- 'persistent_disk': 'testing-ubuntu1604-persistent'
+ 'metadata_from_file': 'startup-script=startup-ubuntu.sh',
+ 'persistent_disk': 'name={0},device-name={0},mode=rw,boot=no'.format('testing-ubuntu1604-persistent'),
+ },
+ 'philwo-ubuntu1604': {
+ 'image_family': 'buildkite-ubuntu1604',
+ 'metadata_from_file': 'startup-script=startup-ubuntu.sh',
},
'testing-windows': {
+ 'boot_disk_size': '500GB',
'image_family': 'buildkite-windows',
- 'machine_type': 'n1-standard-32',
- 'boot_disk_size': '500GB'
},
- '{}-ubuntu1404'.format(getpass.getuser()): {
- 'image_family': 'buildkite-ubuntu1404',
- 'startup_script': 'startup-ubuntu.sh',
- 'machine_type': 'n1-standard-32',
- 'local_ssd': 'interface=nvme',
- },
- '{}-ubuntu1604'.format(getpass.getuser()): {
- 'image_family': 'buildkite-ubuntu1604',
- 'startup_script': 'startup-ubuntu.sh',
- 'machine_type': 'n1-standard-32',
- 'local_ssd': 'interface=nvme',
- },
- '{}-windows'.format(getpass.getuser()): {
- 'image_family': 'buildkite-windows',
- 'startup_script': 'startup-windows.ps1',
- 'machine_type': 'n1-standard-32',
- 'local_ssd': 'interface=scsi',
- }
}
PRINT_LOCK = threading.Lock()
@@ -113,105 +111,27 @@
return subprocess.run(args, **kwargs)
-def flags_for_instance(image_family, params):
- cmd = ['--machine-type', params['machine_type']]
- cmd.extend(['--network', 'buildkite'])
- if 'startup_script' in params:
- if 'windows' in image_family:
- cmd.extend(['--metadata-from-file',
- 'windows-startup-script-ps1=' + params['startup_script']])
- else:
- cmd.extend(['--metadata-from-file', 'startup-script=' + params['startup_script']])
- cmd.extend(['--min-cpu-platform', 'Intel Skylake'])
- cmd.extend(['--boot-disk-type', 'pd-ssd'])
- cmd.extend(['--boot-disk-size', params.get('boot_disk_size', '50GB')])
- if 'local_ssd' in params:
- cmd.extend(['--local-ssd', params['local_ssd']])
- if 'persistent_disk' in params:
- cmd.extend(['--disk',
- 'name={0},device-name={0},mode=rw,boot=no'.format(params['persistent_disk'])])
- cmd.extend(['--image-project', 'bazel-public'])
- cmd.extend(['--image-family', image_family])
- cmd.extend(['--service-account', 'remote-account@bazel-public.iam.gserviceaccount.com'])
- cmd.extend(['--scopes', 'cloud-platform'])
- return cmd
-
-
-def delete_instance_template(template_name):
- cmd = ['gcloud', 'compute', 'instance-templates', 'delete', template_name, '--quiet']
- result = run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
- if result.returncode != 0:
- # It's not an error if 'delete' failed, because the template didn't exist in the first place.
- # But we do want to error out on other unexpected errors.
- if not re.search(r'The resource .* was not found', result.stdout):
- raise Exception('"gcloud compute instance-templates delete" returned unexpected error:\n{}'.format(result.stdout))
- return result
-
-
-def create_instance_template(template_name, image_family, params):
- cmd = ['gcloud', 'compute', 'instance-templates', 'create', template_name]
- cmd.extend(flags_for_instance(image_family, params))
- run(cmd)
-
-
-def delete_instance(instance_name):
- cmd = ['gcloud', 'compute', 'instances', 'delete', '--quiet', instance_name]
- result = run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
- if result.returncode != 0:
- # It's not an error if 'delete' failed, because the template didn't exist in the first place.
- # But we do want to error out on other unexpected errors.
- if not re.search(r'The resource .* was not found', result.stdout):
- raise Exception('"gcloud compute instance delete" returned unexpected error:\n{}'.format(result.stdout))
- return result
-
-
-def create_instance(instance_name, image_family, params):
- cmd = ['gcloud', 'compute', 'instances', 'create', instance_name]
- cmd.extend(['--zone', LOCATION])
- cmd.extend(flags_for_instance(image_family, params))
- run(cmd)
-
-
-def delete_instance_group(instance_group_name):
- cmd = ['gcloud', 'compute', 'instance-groups', 'managed', 'delete', instance_group_name]
- cmd.extend(['--zone', LOCATION])
- cmd.extend(['--quiet'])
- result = run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
- if result.returncode != 0:
- # It's not an error if 'delete' failed, because the template didn't exist in the first place.
- # But we do want to error out on other unexpected errors.
- if not re.search(r'The resource .* was not found', result.stdout):
- raise Exception('"gcloud compute instance-groups managed delete" returned unexpected error:\n{}'.format(result.stdout))
- return result
-
-
-def create_instance_group(instance_group_name, template_name, count):
- cmd = ['gcloud', 'compute', 'instance-groups', 'managed', 'create', instance_group_name]
- cmd.extend(['--zone', LOCATION])
- cmd.extend(['--base-instance-name', instance_group_name])
- cmd.extend(['--template', template_name])
- cmd.extend(['--size', str(count)])
- return run(cmd)
-
-
-def instance_group_task(instance_group_name, params):
- image_family = params.get('image_family', instance_group_name)
+def instance_group_task(instance_group_name, count, **kwargs):
template_name = instance_group_name + '-template'
- if delete_instance_group(instance_group_name).returncode == 0:
+ if gcloud.delete_instance_group(instance_group_name, zone=LOCATION).returncode == 0:
print('Deleted existing instance group: {}'.format(instance_group_name))
- if delete_instance_template(template_name).returncode == 0:
+
+ if gcloud.delete_instance_template(template_name, zone=LOCATION).returncode == 0:
print('Deleted existing VM template: {}'.format(template_name))
- create_instance_template(template_name, image_family, params)
- create_instance_group(instance_group_name, template_name, params['count'])
+
+ gcloud.create_instance_template(template_name, **kwargs)
+
+ gcloud.create_instance_group(instance_group_name,
+ zone=LOCATION, base_instance_name=instance_group_name, template=template_name,
+ size=count)
-def single_instance_task(instance_name, params):
- image_family = params.get('image_family', instance_name)
-
- if delete_instance(instance_name).returncode == 0:
+def single_instance_task(instance_name, **kwargs):
+ if gcloud.delete_instance(instance_name, zone=LOCATION).returncode == 0:
print('Deleted existing instance: {}'.format(instance_name))
- create_instance(instance_name, image_family, params)
+
+ gcloud.create_instance(instance_name, zone=LOCATION, **kwargs)
def worker():
@@ -240,13 +160,10 @@
# instances, otherwise we process all.
if argv and instance_group_name not in argv:
continue
- # Do not automatically create user-specific instances. These must be specified explicitly
- # on the command-line.
- if instance_group_name.startswith(getpass.getuser()) and instance_group_name not in argv:
- continue
WORK_QUEUE.put({
+ **DEFAULT_VM,
'instance_group_name': instance_group_name,
- 'params': params
+ **params
})
for instance_name, params in SINGLE_INSTANCES.items():
@@ -254,13 +171,10 @@
# instances, otherwise we process all.
if argv and instance_name not in argv:
continue
- # Do not automatically create user-specific instances. These must be specified explicitly
- # on the command-line.
- if instance_name.startswith(getpass.getuser()) and instance_name not in argv:
- continue
WORK_QUEUE.put({
+ **DEFAULT_VM,
'instance_name': instance_name,
- 'params': params
+ **params
})
# Spawn worker threads that will create the VMs.
diff --git a/buildkite/gcloud.py b/buildkite/gcloud.py
new file mode 100644
index 0000000..40cf4d1
--- /dev/null
+++ b/buildkite/gcloud.py
@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+#
+# Copyright 2018 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+import re
+import subprocess
+import threading
+
+DEBUG = True
+PRINT_LOCK = threading.Lock()
+
+
+def debug(*args, **kwargs):
+ if DEBUG:
+ with PRINT_LOCK:
+ print(*args, **kwargs)
+
+
+def is_sequence(seq):
+ return isinstance(seq, collections.Sequence) and not isinstance(seq, str)
+
+
+def gcloud(*args, **kwargs):
+ cmd = ['gcloud']
+ cmd += args
+ for flag, value in kwargs.items():
+ # Python uses underscores as word delimiters in kwargs, but gcloud wants dashes.
+ flag = flag.replace('_', '-')
+ # We convert key=[a, b] into two flags: --key=a --key=b.
+ if is_sequence(value):
+ for item in value:
+ cmd += ['--' + flag, str(item)]
+ else:
+ cmd += ['--' + flag, str(value)]
+ # Throws `subprocess.CalledProcessError` if the process exits with return code > 0.
+ return subprocess.run(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, check=True)
+
+
+def create_instance(name, **kwargs):
+ try:
+ return gcloud('compute', 'instances', 'create', name, **kwargs)
+ except subprocess.CalledProcessError as e:
+ raise Exception('"gcloud compute instance create" returned unexpected error:\n{}'.format(e.stderr))
+
+
+def delete_instance(name, **kwargs):
+ try:
+ return gcloud('compute', 'instances', 'delete', name, '--quiet', **kwargs)
+ except subprocess.CalledProcessError as e:
+ # It's not an error if 'delete' failed, because the object didn't exist in the first place.
+ # But we do want to error out on other unexpected errors.
+ if not re.search(r'The resource .* was not found', e.stderr):
+ raise Exception('"gcloud compute instance delete" returned unexpected error:\n{}'.format(e.stderr))
+ return e
+
+
+def describe_instance(name, **kwargs):
+ try:
+ return gcloud('compute', 'instances', 'describe', name, **kwargs)
+ except subprocess.CalledProcessError as e:
+ raise Exception('"gcloud compute instance describe" returned unexpected error:\n{}'.format(e.stderr))
+
+
+def create_instance_group(name, **kwargs):
+ try:
+ return gcloud('compute', 'instance-groups', 'managed', 'create', name, **kwargs)
+ except subprocess.CalledProcessError as e:
+ raise Exception('"gcloud compute instance-groups managed create" returned unexpected error:\n{}'.format(e.stderr))
+
+
+def delete_instance_group(name, **kwargs):
+ try:
+ return gcloud('compute', 'instance-groups', 'managed', 'delete', name, '--quiet', **kwargs)
+ except subprocess.CalledProcessError as e:
+ # It's not an error if 'delete' failed, because the object didn't exist in the first place.
+ # But we do want to error out on other unexpected errors.
+ if not re.search(r'The resource .* was not found', e.stderr):
+ raise Exception('"gcloud compute instance-groups managed delete" returned unexpected error:\n{}'.format(e.stderr))
+ return e
+
+
+def create_instance_template(name, **kwargs):
+ try:
+ return gcloud('compute', 'instance-templates', 'create', name, **kwargs)
+ except subprocess.CalledProcessError as e:
+ raise Exception('"gcloud compute instance-templates create" returned unexpected error:\n{}'.format(e.stderr))
+
+
+def delete_instance_template(name, **kwargs):
+ try:
+ return gcloud('compute', 'instance-templates', 'delete', name, '--quiet', **kwargs)
+ except subprocess.CalledProcessError as e:
+ # It's not an error if 'delete' failed, because the object didn't exist in the first place.
+ # But we do want to error out on other unexpected errors.
+ if not re.search(r'The resource .* was not found', e.stderr):
+ raise Exception('"gcloud compute instance-templates delete" returned unexpected error:\n{}'.format(e.stderr))
+ return e
+
+
+def create_image(name, **kwargs):
+ try:
+ return gcloud('compute', 'images', 'create', name, **kwargs)
+ except subprocess.CalledProcessError as e:
+ raise Exception('"gcloud compute images create" returned unexpected error:\n{}'.format(e.stderr))
+
+
+def reset_windows_password(name, **kwargs):
+ try:
+ return gcloud('compute', 'reset-windows-password', name, '--quiet', **kwargs)
+ except subprocess.CalledProcessError as e:
+ raise Exception('"gcloud compute reset-windows-password" returned unexpected error:\n{}'.format(e.stderr))
+
+
+def get_serial_port_output(name, **kwargs):
+ return gcloud('compute', 'instances', 'get-serial-port-output', name, **kwargs)
diff --git a/buildkite/gcloud_utils.py b/buildkite/gcloud_utils.py
new file mode 100644
index 0000000..3d7f41d
--- /dev/null
+++ b/buildkite/gcloud_utils.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+#
+# Copyright 2018 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import gcloud
+import json
+import subprocess
+import time
+import threading
+import re
+
+
+def wait_for_instance(instance_name, zone, status):
+ while True:
+ result = gcloud.describe_instance(instance_name, zone=zone, format='json')
+ current_status = json.loads(result.stdout)['status']
+ if current_status == status:
+ gcloud.debug('wait_for_instance: {}/{} arrived at status {}'.format(zone, instance_name, status))
+ break
+ else:
+ gcloud.debug('wait_for_instance: Waiting for {}/{} to go from status {} to status {}'.format(
+ zone, instance_name, current_status, status))
+ time.sleep(5)
+
+
+def prettify_logs(instance_name, log, with_prefix=True):
+ for line in log.splitlines():
+ # Skip empty lines.
+ if not line:
+ continue
+
+ # Filter for log lines printed by our startup script, ignore the rest.
+ # Then drop the common prefix to make the output easier to read.
+ # For unknown platforms, we just take every line unmodified.
+ if 'ubuntu' in instance_name:
+ match = re.match(r'.*INFO startup-script: (.*)', line)
+ if not match:
+ continue
+ line = match.group(1)
+ elif 'windows' in instance_name:
+ match = re.match(r'.*windows-startup-script-ps1: (.*)', line)
+ if not match:
+ continue
+ line = match.group(1)
+
+ if with_prefix:
+ yield "{}: {}".format(instance_name, line)
+ else:
+ yield line
+
+
+def print_pretty_logs(instance_name, log):
+ lines = ('\n'.join(prettify_logs(instance_name, log))).strip()
+ if lines:
+ with gcloud.PRINT_LOCK:
+ print(lines)
+
+
+def tail_serial_console(instance_name, zone, start=None, until=None):
+ next_start = start if start else '0'
+ while True:
+ try:
+ result = gcloud.get_serial_port_output(instance_name, zone=zone, start=next_start)
+ except subprocess.CalledProcessError as e:
+ gcloud.debug('tail_serial_console: Done, because got exception: {}'.format(e))
+ if e.stdout:
+ gcloud.debug('stdout: ' + e.stdout)
+ if e.stderr:
+ gcloud.debug('stderr: ' + e.stderr)
+ break
+ print_pretty_logs(instance_name, result.stdout)
+ next_start = re.search(r'--start=(\d*)', result.stderr).group(1)
+ if until and until in result.stdout:
+ gcloud.debug('tail_serial_console: Done, because found string "{}"'.format(until))
+ break
+ time.sleep(5)
+ return next_start
