buildkite/create_images.py

#!/usr/bin/env python3

import json
import os
import re
import subprocess
import sys
import tempfile
import time
from datetime import datetime

DEBUG = False

LOCATION = 'europe-west1-d'

IMAGE_CREATION_VMS = {
    # 'buildkite-freebsd11-image': {
    #     'source_image': 'https://www.googleapis.com/compute/v1/projects/freebsd-org-cloud-dev/global/images/freebsd-11-1-stable-amd64-2017-12-28',
    #     'target_image_family': 'bazel-freebsd11',
    #     'scripts': [
    #         'setup-freebsd.sh',
    #         'install-buildkite-agent.sh'
    #     ]
    # },
    'buildkite-ubuntu1404-image': {
        'source_image_project': 'ubuntu-os-cloud',
        'source_image_family': 'ubuntu-1404-lts',
        'target_image_family': 'buildkite-ubuntu1404',
        'scripts': [
            'shell-utils.sh',
            'setup-ubuntu.sh',
            'install-azul-zulu.sh',
            'install-bazel.sh',
            'install-buildkite-agent.sh',
            'install-docker.sh',
            'install-nodejs.sh',
            'install-android-sdk.sh',
            'shutdown.sh'
        ]
    },
    'buildkite-ubuntu1604-image': {
        'source_image_project': 'ubuntu-os-cloud',
        'source_image_family': 'ubuntu-1604-lts',
        'target_image_family': 'buildkite-ubuntu1604',
        'scripts': [
            'shell-utils.sh',
            'setup-ubuntu.sh',
            'install-azul-zulu.sh',
            'install-bazel.sh',
            'install-buildkite-agent.sh',
            'install-docker.sh',
            'install-nodejs.sh',
            'install-android-sdk.sh',
            'shutdown.sh'
        ]
    },
    # 'buildkite-windows2016-image': {
    #     'source_image_project': 'windows-cloud',
    #     'source_image_family': 'windows-2016',
    #     'target_image_family': 'bazel-windows2016',
    #     'scripts': [
    #         'setup-windows2016.ps1'
    #     ]
    # }
}


def debug(*args, **kwargs):
  if DEBUG:
    print(*args, **kwargs)


def run(args, **kwargs):
  if DEBUG:
    print('Running: %s' % ' '.join(args))
  return subprocess.run(args, **kwargs)


def wait_for_vm(vm, status):
  while True:
    result = run(['gcloud', 'compute', 'instances', 'describe', '--zone', LOCATION, '--format', 'json', vm], check=True, stdout=subprocess.PIPE)
    current_status = json.loads(result.stdout)['status']
    if current_status == status:
      debug("wait_for_vm: VM %s reached status %s" % (vm, status))
      break
    else:
      debug("wait_for_vm: Waiting for VM %s to transition from status %s -> %s" % (vm, current_status, status))
    time.sleep(1)


def print_pretty_logs(vm, log):
  for line in log.splitlines():
    # Skip empty lines.
    if not line:
      continue
    if 'ubuntu' in vm:
      match = re.match(r'.*INFO startup-script: (.*)', line)
      if match:
        print(match.group(1))
    elif 'windows' in vm:
      match = re.match(r'.*windows-startup-script-ps1: (.*)', line)
      if match:
        print(match.group(1))
    else:
      print(line)


def tail_serial_console(vm):
  next_start = '0'
  while True:
    result = run(['gcloud', 'compute', 'instances', 'get-serial-port-output', '--zone', LOCATION, '--start', next_start, vm], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
    if result.returncode != 0:
      break
    print_pretty_logs(vm, result.stdout)
    next_start = re.search(r'--start=(\d*)', result.stderr).group(1)


def merge_setup_scripts(scripts):
  # Merge all setup scripts into one.
  merged_script_path = tempfile.mkstemp()[1]
  with open(merged_script_path, 'w') as merged_script_file:
    for script in scripts:
      with open(script, 'r') as script_file:
        script_contents = script_file.read()
        script_contents.replace('BUILDKITE_TOKEN="xxx"', 'BUILDKITE_TOKEN="%s"' % os.environ['BUILDKITE_TOKEN'])
        merged_script_file.write(script_contents + '\n')
  return merged_script_path


def create_vm(vm, params):
  merged_script_path = merge_setup_scripts(params['scripts'])
  try:
    cmd = ['gcloud', 'compute', 'instances', 'create', vm]
    cmd.extend(['--zone', LOCATION])
    cmd.extend(['--machine-type', 'n1-standard-32'])
    cmd.extend(['--network', 'buildkite'])
    if 'windows' in vm:
      cmd.extend(['--metadata-from-file', 'windows-startup-script-ps1=' + merged_script_path])
    else:
      cmd.extend(['--metadata-from-file', 'startup-script=' + merged_script_path])
    cmd.extend(['--min-cpu-platform', 'Intel Skylake'])
    cmd.extend(['--boot-disk-type', 'pd-ssd'])
    cmd.extend(['--boot-disk-size', '25GB'])
    if 'source_image' in params:
      cmd.extend(['--image', params['source_image']])
    else:
      cmd.extend(['--image-project', params['source_image_project']])
      cmd.extend(['--image-family', params['source_image_family']])
    run(cmd)
  finally:
    os.remove(merged_script_path)


def delete_vm(vm):
  run(['gcloud', 'compute', 'instances', 'delete', '--quiet', vm])


def create_image(vm, target_image_family):
  run(['gcloud', 'compute', 'images', 'create', vm, '--family', target_image_family, '--source-disk', vm, '--source-disk-zone', LOCATION])


def main(argv=None):
  if argv is None:
    argv = sys.argv[1:]

  if not 'BUILDKITE_TOKEN' in os.environ:
    print("Please set the BUILDKITE_TOKEN environment variable.")
    print("You can get the token from: https://buildkite.com/organizations/bazel/agents")
    return 1

  for vm, params in IMAGE_CREATION_VMS.items():
    if argv and not vm in argv:
        continue
    vm = "%s-%s" % (vm, int(datetime.now().timestamp()))
    try:
      # Create the VM.
      create_vm(vm, params)

      # Wait for the VM to become ready.
      wait_for_vm(vm, 'RUNNING')

      # Continuously print the serial console.
      tail_serial_console(vm)

      # Wait for the VM to shutdown.
      wait_for_vm(vm, 'TERMINATED')

      # Create a new image from our VM.
      create_image(vm, params['target_image_family'])
    finally:
      delete_vm(vm)
  return 0

if __name__ == '__main__':
  sys.exit(main())