Add Buildkite infrastructure scripts.
diff --git a/buildkite/acceptance-test.sh b/buildkite/acceptance-test.sh
new file mode 100755
index 0000000..2b2bbe9
--- /dev/null
+++ b/buildkite/acceptance-test.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# $BAZEL_VERSION is set in install-bazel.sh.
+mkdir -p /root/bazel
+cd /root/bazel
+curl -sSLo /root/bazel-dist.zip "https://releases.bazel.build/${BAZEL_VERSION}/release/bazel-${BAZEL_VERSION}-dist.zip"
+unzip /root/bazel-dist.zip > /dev/null
+rm /root/bazel-dist.zip
+
+source /etc/buildkite-agent/hooks/environment
+bazel build --spawn_strategy=linux-sandbox -- //src:bazel //src/test/... -//src/test/docker/...
+bazel clean --expunge
+
+cd /root
+rm -rf /root/bazel
diff --git a/buildkite/create_images.py b/buildkite/create_images.py
new file mode 100755
index 0000000..0a91725
--- /dev/null
+++ b/buildkite/create_images.py
@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import time
+from datetime import datetime
+
+DEBUG = False
+
+LOCATION = 'europe-west1-d'
+
+IMAGE_CREATION_VMS = {
+ # 'buildkite-freebsd11-image': {
+ # 'source_image': 'https://www.googleapis.com/compute/v1/projects/freebsd-org-cloud-dev/global/images/freebsd-11-1-stable-amd64-2017-12-28',
+ # 'target_image_family': 'bazel-freebsd11',
+ # 'scripts': [
+ # 'setup-freebsd.sh',
+ # 'install-buildkite-agent.sh'
+ # ]
+ # },
+ 'buildkite-ubuntu1404-image': {
+ 'source_image_project': 'ubuntu-os-cloud',
+ 'source_image_family': 'ubuntu-1404-lts',
+ 'target_image_family': 'buildkite-ubuntu1404',
+ 'scripts': [
+ 'shell-utils.sh',
+ 'setup-ubuntu.sh',
+ 'install-azul-zulu.sh',
+ 'install-bazel.sh',
+ 'install-buildkite-agent.sh',
+ 'install-docker.sh',
+ 'install-nodejs.sh',
+ 'install-android-sdk.sh',
+ 'shutdown.sh'
+ ]
+ },
+ 'buildkite-ubuntu1604-image': {
+ 'source_image_project': 'ubuntu-os-cloud',
+ 'source_image_family': 'ubuntu-1604-lts',
+ 'target_image_family': 'buildkite-ubuntu1604',
+ 'scripts': [
+ 'shell-utils.sh',
+ 'setup-ubuntu.sh',
+ 'install-azul-zulu.sh',
+ 'install-bazel.sh',
+ 'install-buildkite-agent.sh',
+ 'install-docker.sh',
+ 'install-nodejs.sh',
+ 'install-android-sdk.sh',
+ 'shutdown.sh'
+ ]
+ },
+ # 'buildkite-windows2016-image': {
+ # 'source_image_project': 'windows-cloud',
+ # 'source_image_family': 'windows-2016',
+ # 'target_image_family': 'bazel-windows2016',
+ # 'scripts': [
+ # 'setup-windows2016.ps1'
+ # ]
+ # }
+}
+
+
+def debug(*args, **kwargs):
+ if DEBUG:
+ print(*args, **kwargs)
+
+
+def run(args, **kwargs):
+ if DEBUG:
+ print('Running: %s' % ' '.join(args))
+ return subprocess.run(args, **kwargs)
+
+
+def wait_for_vm(vm, status):
+ while True:
+ result = run(['gcloud', 'compute', 'instances', 'describe', '--zone', LOCATION, '--format', 'json', vm], check=True, stdout=subprocess.PIPE)
+ current_status = json.loads(result.stdout)['status']
+ if current_status == status:
+ debug("wait_for_vm: VM %s reached status %s" % (vm, status))
+ break
+ else:
+ debug("wait_for_vm: Waiting for VM %s to transition from status %s -> %s" % (vm, current_status, status))
+ time.sleep(1)
+
+
+def print_pretty_logs(vm, log):
+ for line in log.splitlines():
+ # Skip empty lines.
+ if not line:
+ continue
+ if 'ubuntu' in vm:
+ match = re.match(r'.*INFO startup-script: (.*)', line)
+ if match:
+ print(match.group(1))
+ elif 'windows' in vm:
+ match = re.match(r'.*windows-startup-script-ps1: (.*)', line)
+ if match:
+ print(match.group(1))
+ else:
+ print(line)
+
+
+def tail_serial_console(vm):
+ next_start = '0'
+ while True:
+ result = run(['gcloud', 'compute', 'instances', 'get-serial-port-output', '--zone', LOCATION, '--start', next_start, vm], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+ if result.returncode != 0:
+ break
+ print_pretty_logs(vm, result.stdout)
+ next_start = re.search(r'--start=(\d*)', result.stderr).group(1)
+
+
+def merge_setup_scripts(scripts):
+ # Merge all setup scripts into one.
+ merged_script_path = tempfile.mkstemp()[1]
+ with open(merged_script_path, 'w') as merged_script_file:
+ for script in scripts:
+ with open(script, 'r') as script_file:
+ script_contents = script_file.read()
+ script_contents.replace('BUILDKITE_TOKEN="xxx"', 'BUILDKITE_TOKEN="%s"' % os.environ['BUILDKITE_TOKEN'])
+ merged_script_file.write(script_contents + '\n')
+ return merged_script_path
+
+
+def create_vm(vm, params):
+ merged_script_path = merge_setup_scripts(params['scripts'])
+ try:
+ cmd = ['gcloud', 'compute', 'instances', 'create', vm]
+ cmd.extend(['--zone', LOCATION])
+ cmd.extend(['--machine-type', 'n1-standard-32'])
+ cmd.extend(['--network', 'buildkite'])
+ if 'windows' in vm:
+ cmd.extend(['--metadata-from-file', 'windows-startup-script-ps1=' + merged_script_path])
+ else:
+ cmd.extend(['--metadata-from-file', 'startup-script=' + merged_script_path])
+ cmd.extend(['--min-cpu-platform', 'Intel Skylake'])
+ cmd.extend(['--boot-disk-type', 'pd-ssd'])
+ cmd.extend(['--boot-disk-size', '25GB'])
+ if 'source_image' in params:
+ cmd.extend(['--image', params['source_image']])
+ else:
+ cmd.extend(['--image-project', params['source_image_project']])
+ cmd.extend(['--image-family', params['source_image_family']])
+ run(cmd)
+ finally:
+ os.remove(merged_script_path)
+
+
+def delete_vm(vm):
+ run(['gcloud', 'compute', 'instances', 'delete', '--quiet', vm])
+
+
+def create_image(vm, target_image_family):
+ run(['gcloud', 'compute', 'images', 'create', vm, '--family', target_image_family, '--source-disk', vm, '--source-disk-zone', LOCATION])
+
+
+def main(argv=None):
+ if argv is None:
+ argv = sys.argv[1:]
+
+ if not 'BUILDKITE_TOKEN' in os.environ:
+ print("Please set the BUILDKITE_TOKEN environment variable.")
+ print("You can get the token from: https://buildkite.com/organizations/bazel/agents")
+ return 1
+
+ for vm, params in IMAGE_CREATION_VMS.items():
+ if argv and not vm in argv:
+ continue
+ vm = "%s-%s" % (vm, int(datetime.now().timestamp()))
+ try:
+ # Create the VM.
+ create_vm(vm, params)
+
+ # Wait for the VM to become ready.
+ wait_for_vm(vm, 'RUNNING')
+
+ # Continuously print the serial console.
+ tail_serial_console(vm)
+
+ # Wait for the VM to shutdown.
+ wait_for_vm(vm, 'TERMINATED')
+
+ # Create a new image from our VM.
+ create_image(vm, params['target_image_family'])
+ finally:
+ delete_vm(vm)
+ return 0
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/buildkite/install-android-sdk.sh b/buildkite/install-android-sdk.sh
new file mode 100755
index 0000000..399e7ef
--- /dev/null
+++ b/buildkite/install-android-sdk.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+# Copyright 2015 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Android NDK
+cd /opt
+curl -sSLo android-ndk.zip https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip
+unzip android-ndk.zip > /dev/null
+rm android-ndk.zip
+
+# Android SDK
+mkdir -p /opt/android-sdk-linux
+cd /opt/android-sdk-linux
+curl -sSLo android-sdk.zip https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip
+unzip android-sdk.zip > /dev/null
+rm android-sdk.zip
+expect -c '
+set timeout -1
+log_user 0
+spawn tools/bin/sdkmanager --update
+expect {
+ "Accept? (y/N)" { exp_send "y\r" ; exp_continue }
+ eof
+}
+'
+
+# This should be kept in sync with mac/mac-android.sh.
+tools/bin/sdkmanager \
+ "platform-tools" \
+ "build-tools;27.0.3" \
+ "platforms;android-24" \
+ "platforms;android-25" \
+ "platforms;android-26" \
+ "platforms;android-27" \
+ "extras;android;m2repository" > /dev/null
+chown -R root:root /opt/android*
diff --git a/buildkite/install-azul-zulu.sh b/buildkite/install-azul-zulu.sh
new file mode 100755
index 0000000..173bfa9
--- /dev/null
+++ b/buildkite/install-azul-zulu.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0x219BD9C9
+apt-add-repository 'deb http://repos.azulsystems.com/ubuntu stable main'
+apt-get -qqy update
+apt-get -qqy install zulu-8 > /dev/null
diff --git a/buildkite/install-bazel.sh b/buildkite/install-bazel.sh
new file mode 100755
index 0000000..7626690
--- /dev/null
+++ b/buildkite/install-bazel.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Install a bootstrap bazel; we use the latest released version
+PLATFORM=$(uname -s | tr '[:upper:]' '[:lower:]')-$(uname -m)
+BAZEL_VERSION=$(curl -sSI https://github.com/bazelbuild/bazel/releases/latest | grep '^Location: ' | sed 's|.*/||' | sed $'s/\r//')
+
+curl -sSLo install.sh "https://releases.bazel.build/${BAZEL_VERSION}/release/bazel-${BAZEL_VERSION}-without-jdk-installer-${PLATFORM}.sh"
+bash install.sh > /dev/null
+bazel version
diff --git a/buildkite/install-buildkite-agent.sh b/buildkite/install-buildkite-agent.sh
new file mode 100755
index 0000000..14ff9fc
--- /dev/null
+++ b/buildkite/install-buildkite-agent.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 32A37959C2FA5C3C99EFBC32A79206696452D198 &> /dev/null
+add-apt-repository -y "deb https://apt.buildkite.com/buildkite-agent stable main"
+
+apt-get -qqy update > /dev/null
+apt-get -qqy install buildkite-agent > /dev/null
+
+# Our very secret Buildkite token used to authenticate the agent.
+BUILDKITE_TOKEN="xxx"
+
+# Deduce the operating system from the hostname and put it into the metadata.
+case $(hostname) in
+ *ubuntu1404*)
+ osname="ubuntu1404"
+ ;;
+ *ubuntu1604*)
+ osname="ubuntu1604"
+ ;;
+ *freebsd11*)
+ osname="freebsd11"
+ ;;
+ default)
+ echo "Could not deduce operating system from hostname: $(hostname)!"
+ exit 1
+esac
+
+# Create configuration file for buildkite-agent.
+sed -i \
+ -e "s/^\(# \)*token=.*/token=\"${BUILDKITE_TOKEN}\"/g" \
+ -e "s/^\(# \)*name=.*/name=\"%hostname\"/g" \
+ -e "s/^\(# \)*meta-data=.*/meta-data=\"os=$osname\"/g" \
+ /etc/buildkite-agent/buildkite-agent.cfg
+
+cat > /etc/buildkite-agent/hooks/environment <<'EOF'
+#!/bin/bash
+
+# The `environment` hook will run before all other commands, and can be used
+# to set up secrets, data, etc. Anything exported in hooks will be available
+# to the build script.
+#
+# For example:
+#
+# export SECRET_VAR=token
+
+set -e
+
+export ANDROID_HOME="/opt/android-sdk-linux"
+export ANDROID_NDK_HOME="/opt/android-ndk-r14b"
+export BUILDKITE_ARTIFACT_UPLOAD_DESTINATION="gs://bazel-buildkite-artifacts/$BUILDKITE_JOB_ID"
+EOF
+
+chmod 0400 /etc/buildkite-agent/buildkite-agent.cfg
+chmod 0500 /etc/buildkite-agent/hooks/*
+chown -R buildkite-agent:buildkite-agent /etc/buildkite-agent
+
+# Do not start buildkite-agent automatically. The startup script will prepare
+# the local SSD first and then start it.
+if [ "$osname" = "ubuntu1404" ]; then
+ echo manual > /etc/init/buildkite-agent.override
+fi
diff --git a/buildkite/install-docker.sh b/buildkite/install-docker.sh
new file mode 100755
index 0000000..3dd8bb2
--- /dev/null
+++ b/buildkite/install-docker.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apt-get -qqy install apt-transport-https ca-certificates > /dev/null
+
+# From https://download.docker.com/linux/ubuntu/gpg
+curl -sSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+
+apt-get -qqy update > /dev/null
+apt-get -qqy install docker-ce > /dev/null
+
+# Docker group
+usermod -aG docker buildkite-agent
diff --git a/buildkite/install-nodejs.sh b/buildkite/install-nodejs.sh
new file mode 100755
index 0000000..28d569c
--- /dev/null
+++ b/buildkite/install-nodejs.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+curl -sSL https://deb.nodesource.com/setup_8.x | bash - > /dev/null
+apt-get -qqy install nodejs > /dev/null
+npm install -g typescript fried-twinkie
diff --git a/buildkite/install-oracle-java.sh b/buildkite/install-oracle-java.sh
new file mode 100755
index 0000000..88768e5
--- /dev/null
+++ b/buildkite/install-oracle-java.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add-apt-repository ppa:webupd8team/java
+echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
+apt-get -qqy install oracle-java8-installer oracle-java8-set-default > /dev/null
diff --git a/buildkite/setup-freebsd.sh b/buildkite/setup-freebsd.sh
new file mode 100755
index 0000000..14754c5
--- /dev/null
+++ b/buildkite/setup-freebsd.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Setup script for FreeBSD.
+set -eux
+
+## Install Bazel and its dependencies.
+pkg install -y \
+ bazel \
+ git \
+ wget \
+ zip
+
+## Mount procfs and fddescfs.
+mount -t fdescfs fdesc /dev/fd
+mount -t procfs proc /proc
+cat >> /etc/fstab <<EOF
+fdesc /dev/fd fdescfs rw 0 0
+proc /proc procfs rw 0 0
+EOF
diff --git a/buildkite/setup-ubuntu.sh b/buildkite/setup-ubuntu.sh
new file mode 100755
index 0000000..e30c046
--- /dev/null
+++ b/buildkite/setup-ubuntu.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Setup script for Ubuntu 14.04 LTS and 16.04 LTS.
+
+# Prevent dpkg / apt-get / debconf from trying to access stdin.
+export DEBIAN_FRONTEND=noninteractive
+
+# Android SDK requires 32-bits libraries.
+dpkg --add-architecture i386
+
+apt-get -qqy update > /dev/null
+apt-get -qqy dist-upgrade > /dev/null
+
+packages=(
+ # Bazel dependencies.
+ build-essential
+ curl
+ git
+ python
+ python3
+ realpath
+ unzip
+ wget
+ xvfb
+ zip
+ zlib1g-dev
+
+ # Dependencies for Android SDK.
+ # https://developer.android.com/studio/troubleshoot.html#linux-libraries
+ # https://code.google.com/p/android/issues/detail?id=207212
+ expect
+ libbz2-1.0:i386
+ libncurses5:i386
+ libstdc++6:i386
+ libz1:i386
+
+ # Dependencies for TensorFlow.
+ libcurl3-dev
+ python-dev
+ python-numpy
+ python-pip
+ #python-wheel
+ swig
+)
+apt-get -qqy install "${packages[@]}" > /dev/null
+
+pip install mock
diff --git a/buildkite/setup-windows2016.ps1 b/buildkite/setup-windows2016.ps1
new file mode 100755
index 0000000..8290330
--- /dev/null
+++ b/buildkite/setup-windows2016.ps1
@@ -0,0 +1,106 @@
+# Stop on action error.
+$ErrorActionPreference = "Stop"
+
+# Install Chocolatey
+Invoke-Expression ((New-Object Net.WebClient).DownloadString("https://chocolatey.org/install.ps1"))
+
+if (-Not (Test-Path c:\bazel_ci)) {
+ New-Item c:\bazel_ci -type directory
+}
+Set-Location c:\bazel_ci
+
+# TODO(philwo) remove if it turns out we don't need Chocolatey.
+# Upgrade Chocolatey.
+# Write-Host "Updating Chocolatey..."
+# & "C:\ProgramData\chocolatey\choco.exe" upgrade chocolatey
+# & "C:\ProgramData\chocolatey\choco.exe" upgrade all
+
+# Update MSYS2 once.
+Write-Host "Updating MSYS2 packages (round 1)..."
+Start-Process -Wait "c:\msys64\msys2_shell.cmd" -ArgumentList "-c", "pacman --noconfirm -Syuu"
+
+# Update again, in case the first round only upgraded core packages.
+Write-Host "Updating MSYS2 packages (round 2)..."
+Start-Process -Wait "c:\msys64\msys2_shell.cmd" -ArgumentList "-c", "pacman --noconfirm -Syuu"
+
+# Install a couple of Python modules required by TensorFlow.
+Write-Host "Updating Python packages..."
+& "C:\Python3\Scripts\pip.exe" install --upgrade `
+ autograd `
+ numpy `
+ portpicker `
+ protobuf `
+ pyreadline `
+ six `
+ wheel
+
+# Fetch the instance ID from GCE.
+Write-Host "Fetching instance ID from GCE..."
+$webclient = (New-Object Net.WebClient)
+$webclient.Headers.Add("Metadata-Flavor", "Google")
+$jenkins_node = $webclient.DownloadString("http://metadata/computeMetadata/v1/instance/attributes/jenkins_node")
+
+# Get the latest release version number of Bazel.
+Write-Host "Grabbing latest Bazel version number from GitHub..."
+$url = "https://github.com/bazelbuild/bazel/releases/latest"
+$req = [system.Net.HttpWebRequest]::Create($url);
+$res = $req.getresponse();
+$res.Close();
+$bazel_version = $res.ResponseUri.AbsolutePath.TrimStart("/bazelbuild/bazel/releases/tag/")
+
+# Download the latest bazel.
+$folder = "c:\bazel_ci\installs\${bazel_version}"
+if (-Not (Test-Path "${folder}\bazel.exe")) {
+ Write-Host "Downloading Bazel ${bazel_version}..."
+ $url = "https://releases.bazel.build/${bazel_version}/release/bazel-${bazel_version}-without-jdk-windows-x86_64.exe"
+ New-Item $folder -type directory -force
+ (New-Object Net.WebClient).DownloadFile("${url}", "${folder}\bazel.exe")
+} else {
+ Write-Host "Bazel ${bazel_version} was already downloaded, skipping..."
+}
+
+# Create a junction to the latest release.
+Write-Host "Creating helper junctions..."
+$latest_folder = "C:\bazel_ci\installs\latest"
+if (Test-Path $latest_folder) {
+ Remove-Item -Force -Recurse $latest_folder
+}
+New-Item -ItemType Junction $latest_folder -Value $folder
+
+$bootstrap_folder = "C:\bazel_ci\installs\bootstrap"
+if (Test-Path $bootstrap_folder) {
+ Remove-Item -Force -Recurse $bootstrap_folder
+}
+New-Item -ItemType Junction $bootstrap_folder -Value $folder
+
+# Find the JDK. The path changes frequently, so hardcoding it is not enough.
+$java = Get-ChildItem "c:\Program Files\Java\jdk*" | Select-Object -Index 0 | foreach { $_.FullName }
+Write-Host "Found latest JDK at ${java}..."
+
+# Save the Jenkins slave.jar to a suitable location.
+Write-Host "Downloading https://ci.bazel.build/jnlpJars/slave.jar..."
+Invoke-WebRequest https://ci.bazel.build/jnlpJars/slave.jar -OutFile slave.jar
+
+# Create the service that runs the Jenkins slave
+# We can't execute Java directly because then it mysteriously fails with
+# "Sockets error: 10106: create", so we redirect through Powershell
+# The path change is needed because Jenkins cannot execute a different git
+# binary on different nodes, so we need to simply use "git"
+Write-Host "Creating Jenkins slave startup script..."
+$jnlpUrl = "https://ci.bazel.build/computer/${jenkins_node}/slave-agent.jnlp"
+$agent_script = @"
+`$env:path="c:\tools\msys64\usr\bin;`$env:path"
+cd c:\bazel_ci
+# A path name with c:\ in the JNLP URL makes Java hang. I don't know why.
+# Jenkins tries to reconnect to the wrong port if the server is restarted.
+# -noReconnect makes the agent die, and it is then subsequently restarted by
+# Windows because it is a service, and then all is well.
+& "${java}\bin\java" -jar c:\bazel_ci\slave.jar -jnlpUrl ${jnlpUrl} -noReconnect
+"@
+Write-Output $agent_script | Out-File -Encoding ascii agent_script.ps1
+
+Write-Host "Creating Jenkins slave service..."
+& nssm install bazel_ci powershell c:\bazel_ci\agent_script.ps1
+& nssm set bazel_ci AppStdout c:\bazel_ci\stdout.log
+& nssm set bazel_ci AppStderr c:\bazel_ci\stderr.log
+& nssm start bazel_ci
diff --git a/buildkite/shell-utils.sh b/buildkite/shell-utils.sh
new file mode 100755
index 0000000..4cf353b
--- /dev/null
+++ b/buildkite/shell-utils.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Fail on errors.
+# Fail when using undefined variables.
+# Print all executed commands.
+# Fail when any command in a pipe fails.
+set -euxo pipefail
diff --git a/buildkite/shutdown.sh b/buildkite/shutdown.sh
new file mode 100755
index 0000000..f96e1a3
--- /dev/null
+++ b/buildkite/shutdown.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+poweroff
diff --git a/buildkite/start_worker.py b/buildkite/start_worker.py
new file mode 100755
index 0000000..a78e209
--- /dev/null
+++ b/buildkite/start_worker.py
@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+
+import sys
+import subprocess
+import threading
+import queue
+
+DEBUG = True
+
+LOCATION = 'europe-west1-d'
+
+AGENTS = {
+ "buildkite-ubuntu1404": {
+ 'count': 4,
+ 'startup_script': 'startup-ubuntu1404.sh',
+ 'machine_type': 'n1-standard-32',
+ 'local_ssd': 'interface=nvme',
+ },
+ "buildkite-ubuntu1604": {
+ 'count': 4,
+ 'startup_script': 'startup-ubuntu1604.sh',
+ 'machine_type': 'n1-standard-32',
+ 'local_ssd': 'interface=nvme',
+ },
+}
+
+PRINT_LOCK = threading.Lock()
+WORK_QUEUE = queue.Queue()
+
+
+def debug(*args, **kwargs):
+ if DEBUG:
+ with PRINT_LOCK:
+ print(*args, **kwargs)
+
+
+def run(args, **kwargs):
+ debug('Running: %s' % ' '.join(args))
+ return subprocess.run(args, **kwargs)
+
+
+def delete_vm(vm):
+ return run(['gcloud', 'compute', 'instances', 'delete', '--quiet', vm])
+
+
+def create_vm(image_family, idx, params):
+ vm = '%s-%s' % (image_family, idx)
+ if delete_vm(vm).returncode == 0:
+ with PRINT_LOCK:
+ print("Deleted existing VM: %s" % vm)
+ cmd = ['gcloud', 'compute', 'instances', 'create', vm]
+ cmd.extend(['--zone', LOCATION])
+ cmd.extend(['--machine-type', params['machine_type']])
+ cmd.extend(['--network', 'buildkite'])
+ if 'windows' in vm:
+ cmd.extend(['--metadata-from-file', 'windows-startup-script-ps1=' + params['startup_script']])
+ else:
+ cmd.extend(['--metadata-from-file', 'startup-script=' + params['startup_script']])
+ cmd.extend(['--min-cpu-platform', 'Intel Skylake'])
+ cmd.extend(['--boot-disk-type', 'pd-ssd'])
+ cmd.extend(['--boot-disk-size', '25GB'])
+ if 'local_ssd' in params:
+ cmd.extend(['--local-ssd', params['local_ssd']])
+ cmd.extend(['--image-project', 'bazel-public'])
+ cmd.extend(['--image-family', image_family])
+ cmd.extend(['--service-account', 'remote-account@bazel-public.iam.gserviceaccount.com'])
+ cmd.extend(['--scopes', 'cloud-platform'])
+ run(cmd)
+
+
+def worker():
+ while True:
+ item = WORK_QUEUE.get()
+ if not item:
+ break
+ try:
+ create_vm(**item)
+ finally:
+ WORK_QUEUE.task_done()
+
+
+def main(argv=None):
+ if argv is None:
+ argv = sys.argv[1:]
+
+ # Put VM creation instructions into the work queue.
+ worker_count = 0
+ for image_family, params in AGENTS.items():
+ if argv and not image_family in argv:
+ continue
+ worker_count += params['count']
+ for idx in range(1, params['count'] + 1):
+ WORK_QUEUE.put({
+ 'image_family': image_family,
+ 'idx': idx,
+ 'params': params
+ })
+
+ # Spawn worker threads that will create the VMs.
+ threads = []
+ for _ in range(worker_count):
+ t = threading.Thread(target=worker)
+ t.start()
+ threads.append(t)
+
+ # Wait for all VMs to be created.
+ WORK_QUEUE.join()
+
+ # Signal worker threads to exit.
+ for _ in range(worker_count):
+ WORK_QUEUE.put(None)
+
+ # Wait for worker threads to exit.
+ for t in threads:
+ t.join()
+
+ return 0
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/buildkite/startup-freebsd11.sh b/buildkite/startup-freebsd11.sh
new file mode 100755
index 0000000..05db1e5
--- /dev/null
+++ b/buildkite/startup-freebsd11.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+exit 0
diff --git a/buildkite/startup-ubuntu1404.sh b/buildkite/startup-ubuntu1404.sh
new file mode 100755
index 0000000..d00c520
--- /dev/null
+++ b/buildkite/startup-ubuntu1404.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+# Use a local SSD if available, otherwise use a RAM disk for our builds.
+if [ -e /dev/nvme0n1 ]; then
+ mkfs.ext4 -F /dev/nvme0n1
+ mount /dev/nvme0n1 /var/lib/buildkite-agent
+ chown -R buildkite-agent:buildkite-agent /var/lib/buildkite-agent
+ chmod 0755 /var/lib/buildkite-agent
+else
+ mount -t tmpfs -o mode=0755,uid=buildkite-agent,gid=buildkite-agent tmpfs /var/lib/buildkite-agent
+fi
+
+service buildkite-agent start
+
+exit 0
diff --git a/buildkite/startup-ubuntu1604.sh b/buildkite/startup-ubuntu1604.sh
new file mode 100755
index 0000000..05800f4
--- /dev/null
+++ b/buildkite/startup-ubuntu1604.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+# Use a local SSD if available, otherwise use a RAM disk for our builds.
+if [ -e /dev/nvme0n1 ]; then
+ mkfs.ext4 -F /dev/nvme0n1
+ mount /dev/nvme0n1 /var/lib/buildkite-agent
+ chown -R buildkite-agent:buildkite-agent /var/lib/buildkite-agent
+ chmod 0755 /var/lib/buildkite-agent
+else
+ mount -t tmpfs -o mode=0755,uid=buildkite-agent,gid=buildkite-agent tmpfs /var/lib/buildkite-agent
+fi
+
+systemctl start buildkite-agent
+
+exit 0
diff --git a/buildkite/startup-windows2016.ps1 b/buildkite/startup-windows2016.ps1
new file mode 100755
index 0000000..e69de29
--- /dev/null
+++ b/buildkite/startup-windows2016.ps1
