Use Docker images for Linux jobs (#441)
diff --git a/buildkite/bazelci.py b/buildkite/bazelci.py index 0389f0d..adf72f7 100644 --- a/buildkite/bazelci.py +++ b/buildkite/bazelci.py
@@ -242,6 +242,7 @@ "agent-directory": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}", "publish_binary": True, "java": "8", + "docker-image": "gcr.io/bazel-untrusted/ubuntu1404:java8", }, "ubuntu1604": { "name": "Ubuntu 16.04, JDK 8", @@ -249,6 +250,7 @@ "agent-directory": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}", "publish_binary": False, "java": "8", + "docker-image": "gcr.io/bazel-untrusted/ubuntu1604:java8", }, "ubuntu1804": { "name": "Ubuntu 18.04, JDK 8", @@ -256,6 +258,7 @@ "agent-directory": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}", "publish_binary": False, "java": "8", + "docker-image": "gcr.io/bazel-untrusted/ubuntu1804:java8", }, "ubuntu1804_nojava": { "name": "Ubuntu 18.04, no JDK", @@ -263,6 +266,7 @@ "agent-directory": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}", "publish_binary": False, "java": "no", + "docker-image": "gcr.io/bazel-untrusted/ubuntu1804:nojava", }, "ubuntu1804_java9": { "name": "Ubuntu 18.04, JDK 9", @@ -270,6 +274,7 @@ "agent-directory": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}", "publish_binary": False, "java": "9", + "docker-image": "gcr.io/bazel-untrusted/ubuntu1804:java9", }, "ubuntu1804_java10": { "name": "Ubuntu 18.04, JDK 10", @@ -277,6 +282,7 @@ "agent-directory": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}", "publish_binary": False, "java": "10", + "docker-image": "gcr.io/bazel-untrusted/ubuntu1804:java10", }, "macos": { "name": "macOS, JDK 8", @@ -299,9 +305,19 @@ "publish_binary": False, "host-platform": "ubuntu1604", "java": "8", + "docker-image": "gcr.io/bazel-untrusted/ubuntu1604:java8", }, } +# The platform used for various steps (e.g. stuff that formerly ran on the "pipeline" workers). +DEFAULT_PLATFORM = "ubuntu1804" + +ENCRYPTED_SAUCELABS_TOKEN = """ +CiQAGuqy23f9LNPzp0AetddpO5CXjducZuBB/dfp6ccpX4LxM+8STQBj1BIUMJMXFAWd9BxYJmcM +W7hzbbFFEfpDuqwVwzD2xF3KugY3Otwv+lPLf6K+8ZI55SbpryFFbt7eSlvVTJIBlElfwIU6OpuK +OuI/ +""".strip() + class BuildkiteException(Exception): """ @@ -472,7 +488,7 @@ if config.get("sauce", None): print_collapsed_group(":saucelabs: Starting Sauce Connect Proxy") os.environ["SAUCE_USERNAME"] = "bazel_rules_webtesting" - os.environ["SAUCE_ACCESS_KEY"] = fetch_saucelabs_token() + os.environ["SAUCE_ACCESS_KEY"] = saucelabs_token() os.environ["TUNNEL_IDENTIFIER"] = str(uuid.uuid4()) os.environ["BUILD_TAG"] = str(uuid.uuid4()) readyfile = os.path.join(tmpdir, "sc_is_ready") @@ -548,89 +564,30 @@ return set(label for label, _ in test_logs_for_status(bep_file, status=status)) -__saucelabs_token__ = None -__buildkite_token__ = None - - -def fetch_saucelabs_token(): - global __saucelabs_token__ - if __saucelabs_token__: - return __saucelabs_token__ - try: - execute_command( +def saucelabs_token(): + return ( + subprocess.check_output( [ - gsutil_command(), - "cp", - "gs://bazel-encrypted-secrets/saucelabs-access-key.enc", - "saucelabs-access-key.enc", - ] + gcloud_command(), + "kms", + "decrypt", + "--location", + "global", + "--keyring", + "buildkite", + "--key", + "saucelabs-access-key", + "--ciphertext-file", + "-", + "--plaintext-file", + "-", + ], + input=base64.b64decode(ENCRYPTED_SAUCELABS_TOKEN), + env=os.environ, ) - __saucelabs_token__ = ( - subprocess.check_output( - [ - gcloud_command(), - "kms", - "decrypt", - "--location", - "global", - "--keyring", - "buildkite", - "--key", - "saucelabs-access-key", - "--ciphertext-file", - "saucelabs-access-key.enc", - "--plaintext-file", - "-", - ], - env=os.environ, - ) - .decode("utf-8") - .strip() - ) - return __saucelabs_token__ - finally: - os.remove("saucelabs-access-key.enc") - - -def fetch_buildkite_token(): - """This function is used in buildkite/incompatible_flag_verbose_failures.py""" - global __buildkite_token__ - if __buildkite_token__: - return __buildkite_token__ - try: - execute_command( - [ - gsutil_command(), - "cp", - "gs://bazel-encrypted-secrets/buildkite-api-token.enc", - "buildkite-api-token.enc", - ] - ) - __buildkite_token__ = ( - subprocess.check_output( - [ - gcloud_command(), - "kms", - "decrypt", - "--location", - "global", - "--keyring", - "buildkite", - "--key", - "buildkite-api-token", - "--ciphertext-file", - "buildkite-api-token.enc", - "--plaintext-file", - "-", - ], - env=os.environ, - ) - .decode("utf-8") - .strip() - ) - return __buildkite_token__ - finally: - os.remove("buildkite-api-token.enc") + .decode("utf-8") + .strip() + ) def is_pull_request(): @@ -822,7 +779,7 @@ "--remote_max_connections=200", '--experimental_remote_platform_override=properties:{name:"platform" value:"%s"}' % platform, - "--remote_http_cache=https://storage.googleapis.com/bazel-buildkite-cache", + "--remote_http_cache=https://storage.googleapis.com/bazel-untrusted-buildkite-cache", ] @@ -882,7 +839,7 @@ # Enable remote execution via RBE. flags = [ "--remote_executor=remotebuildexecution.googleapis.com", - "--remote_instance_name=projects/bazel-public/instances/default_instance", + "--remote_instance_name=projects/bazel-untrusted/instances/default_instance", "--remote_timeout=3600", "--spawn_strategy=remote", "--strategy=Javac=remote", @@ -897,7 +854,7 @@ flags += [ "--bes_backend=buildeventservice.googleapis.com", "--bes_timeout=360s", - "--project_id=bazel-public", + "--project_id=bazel-untrusted", ] if not accept_cached: @@ -1139,19 +1096,49 @@ # If all builds succeed, update the last green commit of this project pipeline_steps.append( - { - "label": "Try Update Last Green Commit", - "command": [ + step( + label="Try Update Last Green Commit", + commands=[ fetch_bazelcipy_command(), python_binary() + " bazelci.py try_update_last_green_commit", ], - "agents": {"kind": "pipeline"}, - } + ) ) print(yaml.dump({"steps": pipeline_steps})) +def step(label, commands, platform=DEFAULT_PLATFORM): + host_platform = PLATFORMS[platform].get("host-platform", platform) + if "docker-image" in PLATFORMS[platform]: + return { + "label": label, + "command": commands, + "agents": {"kind": "docker", "os": "linux"}, + "plugins": { + "philwo/docker": { + "always-pull": True, + "debug": True, + "environment": ["BUILDKITE_ARTIFACT_UPLOAD_DESTINATION", "BUILDKITE_GS_ACL"], + "image": PLATFORMS[platform]["docker-image"], + "privileged": True, + "propagate-environment": True, + "tmpfs": ["/home/bazel/.cache:exec,uid=999,gid=999"], + } + }, + } + else: + return { + "label": label, + "command": commands, + "agents": { + "kind": "worker", + "java": PLATFORMS[platform]["java"], + "os": rchop(host_platform, "_nojava", "_java8", "_java9", "_java10"), + }, + } + + def runner_step( platform, project_name=None, @@ -1180,15 +1167,7 @@ for flag in incompatible_flags or []: command += " --incompatible_flag=" + flag label = create_label(platform, project_name) - return { - "label": label, - "command": [fetch_bazelcipy_command(), command], - "agents": { - "kind": "worker", - "java": PLATFORMS[platform]["java"], - "os": rchop(host_platform, "_nojava", "_java8", "_java9", "_java10"), - }, - } + return step(label=label, commands=[fetch_bazelcipy_command(), command], platform=platform) def fetch_bazelcipy_command(): @@ -1218,11 +1197,10 @@ pipeline_command += " --file_config=" + file_config pipeline_command += " | buildkite-agent pipeline upload" - return { - "label": "Setup {0}".format(project_name), - "command": [fetch_bazelcipy_command(), pipeline_command], - "agents": {"kind": "pipeline"}, - } + return step( + label="Setup {0}".format(project_name), + commands=[fetch_bazelcipy_command(), pipeline_command], + ) def create_label(platform, project_name, build_only=False, test_only=False): @@ -1262,15 +1240,11 @@ pipeline_command += " --file_config=" + file_config pipeline_command += " --platform=" + platform - return { - "label": create_label(platform, project_name, build_only, test_only), - "command": [fetch_bazelcipy_command(), pipeline_command], - "agents": { - "kind": "worker", - "java": PLATFORMS[platform]["java"], - "os": rchop(host_platform, "_nojava", "_java8", "_java9", "_java10"), - }, - } + return step( + label=create_label(platform, project_name, build_only, test_only), + commands=[fetch_bazelcipy_command(), pipeline_command], + platform=platform, + ) def print_bazel_publish_binaries_pipeline(configs, http_config, file_config): @@ -1302,14 +1276,10 @@ # If all builds succeed, publish the Bazel binaries to GCS. pipeline_steps.append( - { - "label": "Publish Bazel Binaries", - "command": [ - fetch_bazelcipy_command(), - python_binary() + " bazelci.py publish_binaries", - ], - "agents": {"kind": "pipeline"}, - } + step( + label="Publish Bazel Binaries", + commands=[fetch_bazelcipy_command(), python_binary() + " bazelci.py publish_binaries"], + ) ) print(yaml.dump({"steps": pipeline_steps})) @@ -1324,13 +1294,12 @@ if len(info_text) == 1: return None - return { - "label": ":sadpanda:", - "command": [ + return step( + label=":sadpanda:", + commands=[ 'buildkite-agent annotate --append --style=info "\n' + "\n".join(info_text) + '\n"' ], - "agents": {"kind": "pipeline"}, - } + ) def print_incompatible_flags_info_box_step(incompatible_flags_map): @@ -1341,13 +1310,12 @@ if len(info_text) == 1: return None - return { - "label": "Incompatible flags info", - "command": [ + return step( + label="Incompatible flags info", + commands=[ 'buildkite-agent annotate --append --style=info "\n' + "\n".join(info_text) + '\n"' ], - "agents": {"kind": "pipeline"}, - } + ) def fetch_incompatible_flags_from_github(): @@ -1441,17 +1409,16 @@ if not current_build_number: raise BuildkiteException("Not running inside Buildkite") pipeline_steps.append( - { - "label": "Test failing jobs with incompatible flag separately", - "command": [ + step( + label="Test failing jobs with incompatible flag separately", + commands=[ fetch_bazelcipy_command(), fetch_incompatible_flag_verbose_failures_command(), python_binary() + " incompatible_flag_verbose_failures.py --build_number=%s | buildkite-agent pipeline upload" % current_build_number, ], - "agents": {"kind": "pipeline"}, - } + ) ) print(yaml.dump({"steps": pipeline_steps}))
diff --git a/buildkite/create-docker-cache.sh b/buildkite/create-docker-cache.sh index 61153f9..b5ba68e 100644 --- a/buildkite/create-docker-cache.sh +++ b/buildkite/create-docker-cache.sh
@@ -2,7 +2,7 @@ set -euxo pipefail -gcloud compute instances delete docker-cache --zone=europe-west1-c --quiet +gcloud compute instances delete docker-cache --project=bazel-public --zone=europe-west1-c --quiet gcloud compute instances create-with-container \ --project bazel-public \ @@ -19,3 +19,21 @@ --network buildkite \ --network-tier PREMIUM \ docker-cache + +gcloud compute instances delete docker-cache --project=bazel-untrusted --zone=europe-north1-a --quiet + +gcloud compute instances create-with-container \ + --project bazel-untrusted \ + --zone "europe-north1-a" \ + --boot-disk-device-name "docker-cache" \ + --boot-disk-size 250GB \ + --boot-disk-type pd-ssd \ + --container-env "REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io" \ + --container-image "registry:2" \ + --container-restart-policy always \ + --image-family cos-stable \ + --image-project cos-cloud \ + --machine-type n1-standard-4 \ + --network buildkite \ + --network-tier PREMIUM \ + docker-cache
diff --git a/buildkite/create_images.py b/buildkite/create_images.py index 0452650..d2312f3 100755 --- a/buildkite/create_images.py +++ b/buildkite/create_images.py
@@ -29,7 +29,8 @@ DEBUG = False -LOCATION = "europe-west1-c" +PROJECT = "bazel-untrusted" +LOCATION = "europe-north1-a" IMAGE_CREATION_VMS = { # Find the newest FreeBSD 11 image via: @@ -50,37 +51,6 @@ "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" ], }, - ("bk-worker-ubuntu1404-java8",): { - "source_image_project": "ubuntu-os-cloud", - "source_image_family": "ubuntu-1404-lts", - "setup_script": "setup-ubuntu.sh", - "licenses": [ - "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" - ], - }, - ("bk-worker-ubuntu1604-java8",): { - "source_image_project": "ubuntu-os-cloud", - "source_image_family": "ubuntu-1604-lts", - "setup_script": "setup-ubuntu.sh", - "licenses": [ - "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" - ], - }, - ( - "bk-pipeline-ubuntu1804-java8", - "bk-trusted-ubuntu1804-java8", - "bk-worker-ubuntu1804-nojava", - "bk-worker-ubuntu1804-java8", - "bk-worker-ubuntu1804-java9", - "bk-worker-ubuntu1804-java10", - ): { - "source_image_project": "ubuntu-os-cloud", - "source_image_family": "ubuntu-1804-lts", - "setup_script": "setup-ubuntu.sh", - "licenses": [ - "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" - ], - }, ( "bk-worker-windows-java8", # 'bk-worker-windows-java9', @@ -113,7 +83,7 @@ return output_file -def create_instance(instance_name, params, git_commit): +def create_instance(instance_name, params): is_windows = "windows" in instance_name setup_script = preprocess_setup_script(params["setup_script"], is_windows) try: @@ -132,10 +102,10 @@ gcloud.create_instance( instance_name, + project=PROJECT, zone=LOCATION, machine_type="n1-standard-8", network="buildkite", - metadata="image-version={}".format(git_commit), metadata_from_file=startup_script, min_cpu_platform="Intel Skylake", boot_disk_type="pd-ssd", @@ -154,11 +124,13 @@ def print_windows_instructions(instance_name): tail_start = gcloud_utils.tail_serial_console( - instance_name, zone=LOCATION, until="Finished running startup scripts" + instance_name, project=PROJECT, zone=LOCATION, until="Finished running startup scripts" ) pw = json.loads( - gcloud.reset_windows_password(instance_name, format="json", zone=LOCATION).stdout + gcloud.reset_windows_password( + instance_name, format="json", project=PROJECT, zone=LOCATION + ).stdout ) rdp_file = tempfile.mkstemp(suffix=".rdp")[1] with open(rdp_file, "w") as f: @@ -172,7 +144,11 @@ # Wait until the VM reboots once, then open RDP again. tail_start = gcloud_utils.tail_serial_console( - instance_name, zone=LOCATION, start=tail_start, until="Finished running startup scripts" + instance_name, + project=PROJECT, + zone=LOCATION, + start=tail_start, + until="Finished running startup scripts", ) print("Connecting via RDP a second time to finish the setup...") write_to_clipboard(pw["password"]) @@ -180,37 +156,44 @@ return tail_start -def workflow(name, params, git_commit): +def workflow(name, params): instance_name = "%s-image-%s" % (name, int(datetime.now().timestamp())) try: # Create the VM. - create_instance(instance_name, params, git_commit) + create_instance(instance_name, params) # Wait for the VM to become ready. - gcloud_utils.wait_for_instance(instance_name, zone=LOCATION, status="RUNNING") + gcloud_utils.wait_for_instance( + instance_name, project=PROJECT, zone=LOCATION, status="RUNNING" + ) if "windows" in instance_name: # Wait for VM to be ready, then print setup instructions. tail_start = print_windows_instructions(instance_name) # Continue printing the serial console until the VM shuts down. - gcloud_utils.tail_serial_console(instance_name, zone=LOCATION, start=tail_start) + gcloud_utils.tail_serial_console( + instance_name, project=PROJECT, zone=LOCATION, start=tail_start + ) else: # Continuously print the serial console. - gcloud_utils.tail_serial_console(instance_name, zone=LOCATION) + gcloud_utils.tail_serial_console(instance_name, project=PROJECT, zone=LOCATION) # Wait for the VM to completely shutdown. - gcloud_utils.wait_for_instance(instance_name, zone=LOCATION, status="TERMINATED") + gcloud_utils.wait_for_instance( + instance_name, project=PROJECT, zone=LOCATION, status="TERMINATED" + ) # Create a new image from our VM. gcloud.create_image( instance_name, + project=PROJECT, family=name, source_disk=instance_name, source_disk_zone=LOCATION, licenses=params.get("licenses", []), ) finally: - gcloud.delete_instance(instance_name, zone=LOCATION) + gcloud.delete_instance(instance_name, project=PROJECT, zone=LOCATION) def worker(): @@ -236,18 +219,6 @@ ) return 1 - try: - git_commit = subprocess.check_output( - ["git", "rev-parse", "--verify", "HEAD"], universal_newlines=True - ).strip() - except subprocess.CalledProcessError: - print( - "Could not get current Git commit hash. You have to run create_images.py " - "from a Git repository.", - file=sys.stderr, - ) - return 1 - if subprocess.check_output(["git", "status", "--porcelain"], universal_newlines=True).strip(): print( "There are pending changes in your Git repository. You have to commit " @@ -261,7 +232,7 @@ for name in names: if argv and name not in argv: continue - WORK_QUEUE.put({"name": name, "params": params, "git_commit": git_commit}) + WORK_QUEUE.put({"name": name, "params": params}) # Spawn worker threads that will create the VMs. threads = []
diff --git a/buildkite/create_instances.py b/buildkite/create_instances.py index d42fd12..08072a4 100755 --- a/buildkite/create_instances.py +++ b/buildkite/create_instances.py
@@ -25,7 +25,6 @@ import gcloud - CONFIG_URL = "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/instances.yml" LOCAL_CONFIG_FILE_NAME = "instances.yml" @@ -38,30 +37,56 @@ if not item: break try: - # We take three keys out of the config item. The rest is passed + # We take a few keys out of the config item. The rest is passed # as-is to create_instance_template() and thus to the gcloud # command line tool. count = item.pop("count") instance_group_name = item.pop("name") - zone = item.pop("zone") + project = item.pop("project") + zone = item.pop("zone", None) + region = item.pop("region", None) + + if not project: + raise Exception("Invalid instance config, no project name set") + + if not zone and not region: + raise Exception( + "Invalid instance config, either zone or region must be specified" + ) template_name = instance_group_name + "-template" - if gcloud.delete_instance_group(instance_group_name, zone=zone).returncode == 0: - print("Deleted existing instance group: {}".format(instance_group_name)) + if zone is not None: + if (gcloud.delete_instance_group( + instance_group_name, project=project, + zone=zone).returncode == 0): + print("Deleted existing instance group: {}".format( + instance_group_name)) + elif region is not None: + if (gcloud.delete_instance_group( + instance_group_name, project=project, + region=region).returncode == 0): + print("Deleted existing instance group: {}".format( + instance_group_name)) - if gcloud.delete_instance_template(template_name).returncode == 0: + if gcloud.delete_instance_template( + template_name, project=project).returncode == 0: print("Deleted existing VM template: {}".format(template_name)) - gcloud.create_instance_template(template_name, **item) + gcloud.create_instance_template( + template_name, project=project, **item) - gcloud.create_instance_group( - instance_group_name, - zone=zone, - base_instance_name=instance_group_name, - template=template_name, - size=count, - ) + kwargs = { + "project": project, + "base_instance_name": instance_group_name, + "size": count, + "template": template_name, + } + if zone is not None: + kwargs["zone"] = zone + elif region is not None: + kwargs["region"] = region + gcloud.create_instance_group(instance_group_name, **kwargs) finally: WORK_QUEUE.task_done() @@ -94,7 +119,8 @@ parser.add_argument( "--local_config", action="store_true", - help="Whether to read the configuration from CWD/%s" % LOCAL_CONFIG_FILE_NAME, + help="Whether to read the configuration from CWD/%s" % + LOCAL_CONFIG_FILE_NAME, ) args = parser.parse_args(argv) @@ -105,7 +131,8 @@ for name in args.names: if name not in valid_names: print("Unknown instance name: {}!".format(name)) - print("\nValid instance names are: {}".format(" ".join(valid_names))) + print("\nValid instance names are: {}".format( + " ".join(valid_names))) return 1 if not args.names: parser.print_help()
diff --git a/buildkite/docker/Dockerfile b/buildkite/docker/Dockerfile new file mode 100644 index 0000000..68676d5 --- /dev/null +++ b/buildkite/docker/Dockerfile
@@ -0,0 +1,356 @@ +### Install Android NDK. +FROM ubuntu AS builder +ENV DEBIAN_FRONTEND="noninteractive" +RUN apt-get -qqy update && \ + apt-get -qqy install curl openjdk-8-jre-headless unzip && \ + rm -rf /var/lib/apt/lists/* + +FROM builder AS android_ndk +RUN cd /opt && \ + curl -Lo android-ndk.zip https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip && \ + unzip android-ndk.zip > /dev/null && \ + rm android-ndk.zip && \ + chown -R root:root /opt/android-ndk-r15c + +FROM builder AS android_sdk +RUN mkdir -p /opt/android-sdk-linux && \ + cd /opt/android-sdk-linux && \ + curl -Lo android-sdk.zip https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip && \ + unzip android-sdk.zip > /dev/null && \ + rm android-sdk.zip && \ + yes | tools/bin/sdkmanager --licenses > /dev/null && \ + tools/bin/sdkmanager --update && \ + tools/bin/sdkmanager \ + "build-tools;27.0.3" \ + "build-tools;28.0.2" \ + "emulator" \ + "extras;android;m2repository" \ + "platform-tools" \ + "platforms;android-24" \ + "platforms;android-28" \ + "system-images;android-19;default;x86" \ + "system-images;android-21;default;x86" \ + "system-images;android-22;default;x86" \ + "system-images;android-23;default;x86" && \ + chown -R root:root /opt/android-sdk-linux + +FROM builder AS bazelisk +RUN curl -Lo /usr/local/bin/bazel https://raw.githubusercontent.com/philwo/bazelisk/master/bazelisk.py && \ + chown root:root /usr/local/bin/bazel && \ + chmod 0755 /usr/local/bin/bazel + +### Install tools required by the release process. +FROM builder AS github-release +RUN curl -L https://github.com/c4milo/github-release/releases/download/v1.1.0/github-release_v1.1.0_linux_amd64.tar.gz | \ + tar xz -C /usr/local/bin && \ + chown root:root /usr/local/bin/github-release && \ + chmod 0755 /usr/local/bin/github-release + +### Install Sauce Connect (for rules_webtesting). +FROM builder AS saucelabs +RUN curl -L https://saucelabs.com/downloads/sc-4.5.1-linux.tar.gz | \ + tar xz -C /usr/local --strip=1 sc-4.5.1-linux/bin/sc && \ + chown root:root /usr/local/bin/sc && \ + chmod 0755 /usr/local/bin/sc + +############################################################################### +### UBUNTU 14.04 ############################################################## +############################################################################### + +FROM ubuntu:14.04 as ubuntu1404-nojava +ENV DEBIAN_FRONTEND="noninteractive" \ + ANDROID_HOME="/opt/android-sdk-linux" \ + ANDROID_NDK_HOME="/opt/android-ndk-r15c" +COPY --from=android_ndk /opt/android-ndk-r15c /opt/android-ndk-r15c/ +COPY --from=android_sdk /opt/android-sdk-linux /opt/android-sdk-linux/ +COPY --from=bazelisk /usr/local/bin/bazel /usr/local/bin/bazel +COPY --from=github-release /usr/local/bin/github-release /usr/local/bin/github-release +COPY --from=saucelabs /usr/local/bin/sc /usr/local/bin/sc + +### Install required packages. +RUN dpkg --add-architecture i386 && \ + apt-get -qqy update && \ + echo "Installing base packages" && \ + apt-get -qqy install apt-utils curl lsb-release software-properties-common && \ + echo "Installing packages required by Bazel" && \ + apt-get -qqy install build-essential clang curl ed git iproute2 iputils-ping netcat-openbsd python python-dev python3 python3-dev unzip wget xvfb zip zlib1g-dev && \ + echo "Installing packages required by Android SDK" && \ + apt-get -qqy install expect libbz2-1.0:i386 libncurses5:i386 libstdc++6:i386 libz1:i386 && \ + echo "Installing packages required by Tensorflow" && \ + apt-get -qqy install libcurl3-dev swig python-enum34 python-mock python-numpy python-pip python-wheel python3-mock python3-numpy python3-pip python3-wheel && \ + echo "Installing packages required by Envoy" && \ + apt-get -qqy install automake autotools-dev cmake libtool m4 && \ + echo "Installing packages required by Android emulator" && \ + apt-get -qqy install cpu-checker qemu-system-x86 unzip xvfb && \ + echo "Installing packages required by Bazel release process" && \ + apt-get -qqy install devscripts gnupg pandoc reprepro ssmtp && \ + echo "Installing packages required by C++ coverage tests" && \ + apt-get -qqy install lcov llvm && \ + echo "Installing packages required by Swift toolchain" && \ + apt-get -qqy install clang libicu-dev && \ + echo "Installing packages required by Bazel (Ubuntu 14.04 and 16.04 only)" && \ + apt-get -qqy install realpath libssl-dev && \ + apt-get -qqy purge apport && \ + rm -rf /var/lib/apt/lists/* + +### Install Google Cloud SDK. +### https://cloud.google.com/sdk/docs/quickstart-debian-ubuntu +RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ + echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ + curl -L https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ + apt-get -qqy update && apt-get -qqy install google-cloud-sdk && \ + rm -rf /var/lib/apt/lists/* + +### Install Mono (required by rules_dotnet). +# RUN apt-get -qqy install apt-transport-https ca-certificates && \ +# apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF && \ +# add-apt-repository "deb https://download.mono-project.com/repo/ubuntu stable-$(lsb_release -cs) main" && \ +# apt-get -qqy update && \ +# apt-get -qqy install mono-devel mono-complete + +### Install Node.js and packages required by Gerrit. +### (see https://gerrit.googlesource.com/gerrit/+show/master/polygerrit-ui/README.md) +RUN curl -L https://deb.nodesource.com/setup_8.x | bash - && \ + apt-get -qqy install nodejs && \ + npm install -g typescript fried-twinkie@0.0.15 + +### Install Python (required our own bazelci.py script). +RUN export PYTHON_VERSION="3.6.8" && \ + mkdir -p /usr/local/src && \ + cd /usr/local/src && \ + curl -LO "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz" && \ + tar xfJ "Python-${PYTHON_VERSION}.tar.xz" && \ + rm "Python-${PYTHON_VERSION}.tar.xz" && \ + cd "Python-${PYTHON_VERSION}" && \ + echo "_ssl _ssl.c -DUSE_SSL -I/usr/include -I/usr/include/openssl -L/usr/lib -lssl -lcrypto" >> Modules/Setup.dist && \ + echo "Compiling Python ${PYTHON_VERSION} ..." && \ + ./configure --quiet --enable-ipv6 && \ + make -s -j all && \ + echo "Installing Python ${PYTHON_VERSION} ..." && \ + make -s altinstall && \ + pip3.6 install requests uritemplate pyyaml github3.py && \ + rm -rf "/usr/local/src/Python-${PYTHON_VERSION}" + +### Install Swift toolchain (required by rules_swift). +ENV SWIFT_HOME "/opt/swift" +ENV PATH "${PATH}:${SWIFT_HOME}/usr/bin" +RUN mkdir -p /opt/swift && \ + curl -L https://swift.org/builds/swift-4.2.1-release/ubuntu1404/swift-4.2.1-RELEASE/swift-4.2.1-RELEASE-ubuntu14.04.tar.gz | \ + tar xz -C /opt/swift --strip 1 + +RUN groupadd --gid 999 bazel && \ + useradd --gid 999 --uid 999 --create-home --shell /bin/bash bazel +USER bazel +RUN bazel version + +FROM ubuntu1404-nojava AS ubuntu1404-java8 +USER root +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0x219BD9C9 && \ + apt-add-repository 'deb http://repos.azulsystems.com/ubuntu stable main' && \ + apt-get -qqy update && \ + apt-get -qqy install zulu-8 +USER bazel + +############################################################################### +### UBUNTU 16.04 ############################################################## +############################################################################### + +FROM ubuntu:16.04 as ubuntu1604-nojava +ENV DEBIAN_FRONTEND="noninteractive" \ + ANDROID_HOME="/opt/android-sdk-linux" \ + ANDROID_NDK_HOME="/opt/android-ndk-r15c" +COPY --from=android_ndk /opt/android-ndk-r15c /opt/android-ndk-r15c/ +COPY --from=android_sdk /opt/android-sdk-linux /opt/android-sdk-linux/ +COPY --from=bazelisk /usr/local/bin/bazel /usr/local/bin/bazel +COPY --from=github-release /usr/local/bin/github-release /usr/local/bin/github-release +COPY --from=saucelabs /usr/local/bin/sc /usr/local/bin/sc + +### Install required packages. +RUN dpkg --add-architecture i386 && \ + apt-get -qqy update && \ + echo "Installing base packages" && \ + apt-get -qqy install apt-utils curl lsb-release software-properties-common && \ + echo "Installing packages required by Bazel" && \ + apt-get -qqy install build-essential clang curl ed git iproute2 iputils-ping netcat-openbsd python python-dev python3 python3-dev unzip wget xvfb zip zlib1g-dev && \ + echo "Installing packages required by Android SDK" && \ + apt-get -qqy install expect libbz2-1.0:i386 libncurses5:i386 libstdc++6:i386 libz1:i386 && \ + echo "Installing packages required by Tensorflow" && \ + apt-get -qqy install libcurl3-dev swig python-enum34 python-mock python-numpy python-pip python-wheel python3-mock python3-numpy python3-pip python3-wheel && \ + echo "Installing packages required by Envoy" && \ + apt-get -qqy install automake autotools-dev cmake libtool m4 && \ + echo "Installing packages required by Android emulator" && \ + apt-get -qqy install cpu-checker qemu-system-x86 unzip xvfb && \ + echo "Installing packages required by Bazel release process" && \ + apt-get -qqy install devscripts gnupg pandoc reprepro ssmtp && \ + echo "Installing packages required by C++ coverage tests" && \ + apt-get -qqy install lcov llvm && \ + echo "Installing packages required by Swift toolchain" && \ + apt-get -qqy install clang libicu-dev && \ + echo "Installing packages required by Bazel (Ubuntu 14.04 and 16.04 only)" && \ + apt-get -qqy install realpath libssl-dev && \ + apt-get -qqy purge apport && \ + rm -rf /var/lib/apt/lists/* + +### Install Python packages required by Tensorflow. +RUN pip install keras_applications keras_preprocessing && \ + pip3 install keras_applications keras_preprocessing + +### Install Google Cloud SDK. +### https://cloud.google.com/sdk/docs/quickstart-debian-ubuntu +RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ + echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ + curl -L https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ + apt-get -qqy update && apt-get -qqy install google-cloud-sdk && \ + rm -rf /var/lib/apt/lists/* + +### Install Mono (required by rules_dotnet). +# RUN apt-get -qqy install apt-transport-https ca-certificates && \ +# apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF && \ +# add-apt-repository "deb https://download.mono-project.com/repo/ubuntu stable-$(lsb_release -cs) main" && \ +# apt-get -qqy update && \ +# apt-get -qqy install mono-devel mono-complete + +### Install Node.js and packages required by Gerrit. +### (see https://gerrit.googlesource.com/gerrit/+show/master/polygerrit-ui/README.md) +RUN curl -L https://deb.nodesource.com/setup_8.x | bash - && \ + apt-get -qqy install nodejs && \ + npm install -g typescript fried-twinkie@0.0.15 + +### Install Python (required by our own bazelci.py script). +RUN export PYTHON_VERSION="3.6.8" && \ + mkdir -p /usr/local/src && \ + cd /usr/local/src && \ + curl -LO "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz" && \ + tar xfJ "Python-${PYTHON_VERSION}.tar.xz" && \ + rm "Python-${PYTHON_VERSION}.tar.xz" && \ + cd "Python-${PYTHON_VERSION}" && \ + echo "_ssl _ssl.c -DUSE_SSL -I/usr/include -I/usr/include/openssl -L/usr/lib -lssl -lcrypto" >> Modules/Setup.dist && \ + echo "Compiling Python ${PYTHON_VERSION} ..." && \ + ./configure --quiet --enable-ipv6 && \ + make -s -j all && \ + echo "Installing Python ${PYTHON_VERSION} ..." && \ + make -s altinstall && \ + pip3.6 install requests uritemplate pyyaml github3.py && \ + rm -rf "/usr/local/src/Python-${PYTHON_VERSION}" + +### Install Swift toolchain (required by rules_swift). +ENV SWIFT_HOME "/opt/swift" +ENV PATH "${PATH}:${SWIFT_HOME}/usr/bin" +RUN mkdir -p /opt/swift && \ + curl -L https://swift.org/builds/swift-4.2.1-release/ubuntu1604/swift-4.2.1-RELEASE/swift-4.2.1-RELEASE-ubuntu16.04.tar.gz | \ + tar xz -C /opt/swift --strip 1 + +RUN groupadd --gid 999 bazel && \ + useradd --gid 999 --uid 999 --create-home --shell /bin/bash bazel +USER bazel +RUN bazel version + +FROM ubuntu1604-nojava AS ubuntu1604-java8 +USER root +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0x219BD9C9 && \ + apt-add-repository 'deb http://repos.azulsystems.com/ubuntu stable main' && \ + apt-get -qqy update && \ + apt-get -qqy install zulu-8 +USER bazel + +############################################################################### +### UBUNTU 18.04 ############################################################## +############################################################################### + +FROM ubuntu:18.04 as ubuntu1804-nojava +ENV DEBIAN_FRONTEND="noninteractive" \ + ANDROID_HOME="/opt/android-sdk-linux" \ + ANDROID_NDK_HOME="/opt/android-ndk-r15c" +COPY --from=android_ndk /opt/android-ndk-r15c /opt/android-ndk-r15c/ +COPY --from=android_sdk /opt/android-sdk-linux /opt/android-sdk-linux/ +COPY --from=bazelisk /usr/local/bin/bazel /usr/local/bin/bazel +COPY --from=github-release /usr/local/bin/github-release /usr/local/bin/github-release +COPY --from=saucelabs /usr/local/bin/sc /usr/local/bin/sc + +### Install required packages. +RUN dpkg --add-architecture i386 && \ + apt-get -qqy update && \ + echo "Installing base packages" && \ + apt-get -qqy install apt-utils curl lsb-release software-properties-common && \ + echo "Installing packages required by Bazel" && \ + apt-get -qqy install build-essential clang curl ed git iproute2 iputils-ping netcat-openbsd python python-dev python3 python3-dev unzip wget xvfb zip zlib1g-dev && \ + echo "Installing packages required by Android SDK" && \ + apt-get -qqy install expect libbz2-1.0:i386 libncurses5:i386 libstdc++6:i386 libz1:i386 && \ + echo "Installing packages required by Tensorflow" && \ + apt-get -qqy install libcurl3-dev swig python-enum34 python-mock python-numpy python-pip python-wheel python3-mock python3-numpy python3-pip python3-wheel && \ + echo "Installing packages required by Envoy" && \ + apt-get -qqy install automake autotools-dev cmake libtool m4 && \ + echo "Installing packages required by Android emulator" && \ + apt-get -qqy install cpu-checker qemu-system-x86 unzip xvfb && \ + echo "Installing packages required by Bazel release process" && \ + apt-get -qqy install devscripts gnupg pandoc reprepro ssmtp && \ + echo "Installing packages required by C++ coverage tests" && \ + apt-get -qqy install lcov llvm && \ + echo "Installing packages required by Swift toolchain" && \ + apt-get -qqy install clang libicu-dev && \ + apt-get -qqy purge apport && \ + rm -rf /var/lib/apt/lists/* + +### Install Python packages required by Tensorflow. +RUN pip install keras_applications keras_preprocessing && \ + pip3 install keras_applications keras_preprocessing + +### Install Google Cloud SDK. +### https://cloud.google.com/sdk/docs/quickstart-debian-ubuntu +RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ + echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ + curl -L https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ + apt-get -qqy update && apt-get -qqy install google-cloud-sdk && \ + rm -rf /var/lib/apt/lists/* + +### Install Mono (required by rules_dotnet). +# RUN apt-get -qqy install apt-transport-https ca-certificates && \ +# apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF && \ +# add-apt-repository "deb https://download.mono-project.com/repo/ubuntu stable-$(lsb_release -cs) main" && \ +# apt-get -qqy update && \ +# apt-get -qqy install mono-devel mono-complete + +### Install Node.js and packages required by Gerrit. +### (see https://gerrit.googlesource.com/gerrit/+show/master/polygerrit-ui/README.md) +RUN curl -L https://deb.nodesource.com/setup_8.x | bash - && \ + apt-get -qqy install nodejs && \ + npm install -g typescript fried-twinkie@0.0.15 + +### Install Python dependencies required by our own bazelci.py script. +RUN pip3 install requests uritemplate pyyaml github3.py + +### Install Swift toolchain (required by rules_swift). +ENV SWIFT_HOME "/opt/swift" +ENV PATH "${PATH}:${SWIFT_HOME}/usr/bin" +RUN mkdir -p /opt/swift && \ + curl -L https://swift.org/builds/swift-4.2.1-release/ubuntu1804/swift-4.2.1-RELEASE/swift-4.2.1-RELEASE-ubuntu18.04.tar.gz | \ + tar xz -C /opt/swift --strip 1 + +RUN groupadd --gid 999 bazel && \ + useradd --gid 999 --uid 999 --create-home --shell /bin/bash bazel +USER bazel +RUN bazel version + +FROM ubuntu1804-nojava AS ubuntu1804-java8 +USER root +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0x219BD9C9 && \ + apt-add-repository 'deb http://repos.azulsystems.com/ubuntu stable main' && \ + apt-get -qqy update && \ + apt-get -qqy install zulu-8 +USER bazel + +FROM ubuntu1804-nojava AS ubuntu1804-java9 +USER root +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0x219BD9C9 && \ + apt-add-repository 'deb http://repos.azulsystems.com/ubuntu stable main' && \ + apt-get -qqy update && \ + apt-get -qqy install zulu-9 +USER bazel + +FROM ubuntu1804-nojava AS ubuntu1804-java10 +USER root +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0x219BD9C9 && \ + apt-add-repository 'deb http://repos.azulsystems.com/ubuntu stable main' && \ + apt-get -qqy update && \ + apt-get -qqy install zulu-10 +USER bazel
diff --git a/buildkite/docker/build.sh b/buildkite/docker/build.sh index 90f349a..c0b153f 100755 --- a/buildkite/docker/build.sh +++ b/buildkite/docker/build.sh
@@ -2,16 +2,13 @@ set -euxo pipefail -cd ubuntu1404 -docker build --target java8 -t gcr.io/bazel-public/ubuntu1404:java8 . +docker build --target ubuntu1404-java8 -t gcr.io/bazel-public/ubuntu1404:java8 . docker push gcr.io/bazel-public/ubuntu1404:java8 -cd ../ubuntu1604 -docker build --target java8 -t gcr.io/bazel-public/ubuntu1604:java8 . +docker build --target ubuntu1604-java8 -t gcr.io/bazel-public/ubuntu1604:java8 . docker push gcr.io/bazel-public/ubuntu1604:java8 -cd ../ubuntu1804 for java in java8 java9 java10 nojava; do - docker build --target $java -t gcr.io/bazel-public/ubuntu1804:$java . + docker build --target ubuntu1804-$java -t gcr.io/bazel-public/ubuntu1804:$java . docker push gcr.io/bazel-public/ubuntu1804:$java done
diff --git a/buildkite/docker/ubuntu1404/Dockerfile b/buildkite/docker/ubuntu1404/Dockerfile deleted file mode 100644 index 9a23749..0000000 --- a/buildkite/docker/ubuntu1404/Dockerfile +++ /dev/null
@@ -1,129 +0,0 @@ -FROM ubuntu:14.04 as java8 - -ENV DEBIAN_FRONTEND "noninteractive" - -RUN apt-get -qqy update && \ - apt-get -qqy install lsb-release curl apt-utils software-properties-common && \ - rm -rf /var/lib/apt/lists/* - -### Install Google Cloud SDK. -### https://cloud.google.com/sdk/docs/quickstart-debian-ubuntu -RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ - echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ - curl -L https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ - apt-get -qqy update && apt-get -qqy install google-cloud-sdk && \ - rm -rf /var/lib/apt/lists/* - -### Install required packages. -RUN dpkg --add-architecture i386 && \ - apt-get -qqy update && \ - echo "Installing packages required by Bazel" && \ - apt-get -qqy install build-essential clang curl git python python-dev python3 python3-dev unzip wget xvfb zip zlib1g-dev && \ - echo "Installing packages required by Android SDK" && \ - apt-get -qqy install expect libbz2-1.0:i386 libncurses5:i386 libstdc++6:i386 libz1:i386 && \ - echo "Installing packages required by Tensorflow" && \ - apt-get -qqy install libcurl3-dev swig python-enum34 python-mock python-numpy python-pip python-wheel python3-mock python3-numpy python3-pip python3-wheel && \ - echo "Installing packages required by Envoy" && \ - apt-get -qqy install automake autotools-dev cmake libtool m4 && \ - echo "Installing packages required by Android emulator" && \ - apt-get -qqy install cpu-checker qemu-system-x86 unzip xvfb && \ - echo "Installing packages required by Bazel release process" && \ - apt-get -qqy install devscripts gnupg pandoc reprepro ssmtp && \ - echo "Installing packages required by C++ coverage tests" && \ - apt-get -qqy install lcov llvm && \ - echo "Installing packages required by Swift toolchain" && \ - apt-get -qqy install clang libicu-dev && \ - echo "Installing packages required by Bazel (Ubuntu 14.04 and 16.04 only)" && \ - apt-get -qqy install realpath libssl-dev && \ - apt-get -qqy purge apport && \ - rm -rf /var/lib/apt/lists/* - -### Install Bazelisk (as Bazel). -RUN curl -Lo /usr/local/bin/bazel https://raw.githubusercontent.com/philwo/bazelisk/master/bazelisk.py && \ - chmod 0755 /usr/local/bin/bazel - -### Install Azul Zulu. -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0x219BD9C9 && \ - apt-add-repository 'deb http://repos.azulsystems.com/ubuntu stable main' && \ - apt-get -qqy update && \ - apt-get -qqy install zulu-8 - -### Install Mono. -RUN apt-get -qqy install apt-transport-https ca-certificates && \ - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF && \ - add-apt-repository "deb https://download.mono-project.com/repo/ubuntu stable-$(lsb_release -cs) main" && \ - apt-get -qqy update && \ - apt-get -qqy install mono-devel mono-complete - -### Install Node.js. -RUN curl -L https://deb.nodesource.com/setup_8.x | bash - && \ - apt-get -qqy install nodejs - -### Required by Gerrit. -### https://gerrit.googlesource.com/gerrit/+show/master/polygerrit-ui/README.md -RUN npm install -g typescript fried-twinkie@0.0.15 - -### Install Python. -RUN export PYTHON_VERSION="3.6.8" && \ - mkdir -p /usr/local/src && \ - cd /usr/local/src && \ - curl -LO "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz" && \ - tar xfJ "Python-${PYTHON_VERSION}.tar.xz" && \ - rm "Python-${PYTHON_VERSION}.tar.xz" && \ - cd "Python-${PYTHON_VERSION}" && \ - echo "_ssl _ssl.c -DUSE_SSL -I/usr/include -I/usr/include/openssl -L/usr/lib -lssl -lcrypto" >> Modules/Setup.dist && \ - echo "Compiling Python ${PYTHON_VERSION} ..." && \ - ./configure --quiet --enable-ipv6 && \ - make -s -j all && \ - echo "Installing Python ${PYTHON_VERSION} ..." && \ - make -s altinstall && \ - pip3.6 install requests uritemplate pyyaml github3.py && \ - rm -rf "/usr/local/src/Python-${PYTHON_VERSION}" - -### Install Android NDK. -ENV ANDROID_NDK_HOME "/opt/android-ndk-r15c" -RUN cd /opt && \ - curl -Lo android-ndk.zip https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip && \ - unzip android-ndk.zip > /dev/null && \ - rm android-ndk.zip - -### Install Android SDK. -ENV ANDROID_HOME "/opt/android-sdk-linux" -RUN mkdir -p /opt/android-sdk-linux && \ - cd /opt/android-sdk-linux && \ - curl -Lo android-sdk.zip https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip && \ - unzip android-sdk.zip > /dev/null && \ - rm android-sdk.zip && \ - yes | tools/bin/sdkmanager --licenses > /dev/null && \ - tools/bin/sdkmanager --update && \ - tools/bin/sdkmanager \ - "build-tools;27.0.3" \ - "build-tools;28.0.2" \ - "emulator" \ - "extras;android;m2repository" \ - "platform-tools" \ - "platforms;android-24" \ - "platforms;android-28" \ - "system-images;android-19;default;x86" \ - "system-images;android-21;default;x86" \ - "system-images;android-22;default;x86" \ - "system-images;android-23;default;x86" - -### Install Swift toolchain (for rules_swift). -ENV SWIFT_HOME "/opt/swift" -ENV PATH "${PATH}:${SWIFT_HOME}/usr/bin" -RUN mkdir -p /opt/swift && \ - curl -L https://swift.org/builds/swift-4.2.1-release/ubuntu1404/swift-4.2.1-RELEASE/swift-4.2.1-RELEASE-ubuntu14.04.tar.gz | \ - tar xz -C /opt/swift --strip 1 - -### Install tools required by the release process. -RUN curl -L https://github.com/c4milo/github-release/releases/download/v1.1.0/github-release_v1.1.0_linux_amd64.tar.gz | \ - tar xz -C /usr/local/bin && \ - chown root:root /usr/local/bin/github-release && \ - chmod 0755 /usr/local/bin/github-release - -### Install Sauce Connect (for rules_webtesting). -RUN curl -L https://saucelabs.com/downloads/sc-4.5.1-linux.tar.gz | \ - tar xz -C /opt && \ - chown -R root:root /opt/sc-4.5.1-linux && \ - ln -s /opt/sc-4.5.1-linux/bin/sc /usr/local/bin/sc
diff --git a/buildkite/docker/ubuntu1604/Dockerfile b/buildkite/docker/ubuntu1604/Dockerfile deleted file mode 100755 index d9fc380..0000000 --- a/buildkite/docker/ubuntu1604/Dockerfile +++ /dev/null
@@ -1,131 +0,0 @@ -FROM ubuntu:16.04 as java8 - -ENV DEBIAN_FRONTEND "noninteractive" - -RUN apt-get -qqy update && \ - apt-get -qqy install lsb-release curl apt-utils software-properties-common && \ - rm -rf /var/lib/apt/lists/* - -### Install Google Cloud SDK. -### https://cloud.google.com/sdk/docs/quickstart-debian-ubuntu -RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ - echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ - curl -L https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ - apt-get -qqy update && apt-get -qqy install google-cloud-sdk && \ - rm -rf /var/lib/apt/lists/* - -### Install required packages. -RUN dpkg --add-architecture i386 && \ - apt-get -qqy update && \ - echo "Installing packages required by Bazel" && \ - apt-get -qqy install build-essential clang curl git python python-dev python3 python3-dev unzip wget xvfb zip zlib1g-dev && \ - echo "Installing packages required by Android SDK" && \ - apt-get -qqy install expect libbz2-1.0:i386 libncurses5:i386 libstdc++6:i386 libz1:i386 && \ - echo "Installing packages required by Tensorflow" && \ - apt-get -qqy install libcurl3-dev swig python-enum34 python-mock python-numpy python-pip python-wheel python3-mock python3-numpy python3-pip python3-wheel && \ - pip install keras_applications keras_preprocessing && \ - pip3 install keras_applications keras_preprocessing && \ - echo "Installing packages required by Envoy" && \ - apt-get -qqy install automake autotools-dev cmake libtool m4 && \ - echo "Installing packages required by Android emulator" && \ - apt-get -qqy install cpu-checker qemu-system-x86 unzip xvfb && \ - echo "Installing packages required by Bazel release process" && \ - apt-get -qqy install devscripts gnupg pandoc reprepro ssmtp && \ - echo "Installing packages required by C++ coverage tests" && \ - apt-get -qqy install lcov llvm && \ - echo "Installing packages required by Swift toolchain" && \ - apt-get -qqy install clang libicu-dev && \ - echo "Installing packages required by Bazel (Ubuntu 14.04 and 16.04 only)" && \ - apt-get -qqy install realpath libssl-dev && \ - apt-get -qqy purge apport && \ - rm -rf /var/lib/apt/lists/* - -### Install Bazelisk (as Bazel). -RUN curl -Lo /usr/local/bin/bazel https://raw.githubusercontent.com/philwo/bazelisk/master/bazelisk.py && \ - chmod 0755 /usr/local/bin/bazel - -### Install Azul Zulu. -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0x219BD9C9 && \ - apt-add-repository 'deb http://repos.azulsystems.com/ubuntu stable main' && \ - apt-get -qqy update && \ - apt-get -qqy install zulu-8 - -### Install Mono. -RUN apt-get -qqy install apt-transport-https ca-certificates && \ - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF && \ - add-apt-repository "deb https://download.mono-project.com/repo/ubuntu stable-$(lsb_release -cs) main" && \ - apt-get -qqy update && \ - apt-get -qqy install mono-devel mono-complete - -### Install Node.js. -RUN curl -L https://deb.nodesource.com/setup_8.x | bash - && \ - apt-get -qqy install nodejs - -### Required by Gerrit. -### https://gerrit.googlesource.com/gerrit/+show/master/polygerrit-ui/README.md -RUN npm install -g typescript fried-twinkie@0.0.15 - -### Install Python. -RUN export PYTHON_VERSION="3.6.8" && \ - mkdir -p /usr/local/src && \ - cd /usr/local/src && \ - curl -LO "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz" && \ - tar xfJ "Python-${PYTHON_VERSION}.tar.xz" && \ - rm "Python-${PYTHON_VERSION}.tar.xz" && \ - cd "Python-${PYTHON_VERSION}" && \ - echo "_ssl _ssl.c -DUSE_SSL -I/usr/include -I/usr/include/openssl -L/usr/lib -lssl -lcrypto" >> Modules/Setup.dist && \ - echo "Compiling Python ${PYTHON_VERSION} ..." && \ - ./configure --quiet --enable-ipv6 && \ - make -s -j all && \ - echo "Installing Python ${PYTHON_VERSION} ..." && \ - make -s altinstall && \ - pip3.6 install requests uritemplate pyyaml github3.py && \ - rm -rf "/usr/local/src/Python-${PYTHON_VERSION}" - -### Install Android NDK. -ENV ANDROID_NDK_HOME "/opt/android-ndk-r15c" -RUN cd /opt && \ - curl -Lo android-ndk.zip https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip && \ - unzip android-ndk.zip > /dev/null && \ - rm android-ndk.zip - -### Install Android SDK. -ENV ANDROID_HOME "/opt/android-sdk-linux" -RUN mkdir -p /opt/android-sdk-linux && \ - cd /opt/android-sdk-linux && \ - curl -Lo android-sdk.zip https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip && \ - unzip android-sdk.zip > /dev/null && \ - rm android-sdk.zip && \ - yes | tools/bin/sdkmanager --licenses > /dev/null && \ - tools/bin/sdkmanager --update && \ - tools/bin/sdkmanager \ - "build-tools;27.0.3" \ - "build-tools;28.0.2" \ - "emulator" \ - "extras;android;m2repository" \ - "platform-tools" \ - "platforms;android-24" \ - "platforms;android-28" \ - "system-images;android-19;default;x86" \ - "system-images;android-21;default;x86" \ - "system-images;android-22;default;x86" \ - "system-images;android-23;default;x86" - -### Install Swift toolchain (for rules_swift). -ENV SWIFT_HOME "/opt/swift" -ENV PATH "${PATH}:${SWIFT_HOME}/usr/bin" -RUN mkdir -p /opt/swift && \ - curl -L https://swift.org/builds/swift-4.2.1-release/ubuntu1604/swift-4.2.1-RELEASE/swift-4.2.1-RELEASE-ubuntu16.04.tar.gz | \ - tar xz -C /opt/swift --strip 1 - -### Install tools required by the release process. -RUN curl -L https://github.com/c4milo/github-release/releases/download/v1.1.0/github-release_v1.1.0_linux_amd64.tar.gz | \ - tar xz -C /usr/local/bin && \ - chown root:root /usr/local/bin/github-release && \ - chmod 0755 /usr/local/bin/github-release - -### Install Sauce Connect (for rules_webtesting). -RUN curl -L https://saucelabs.com/downloads/sc-4.5.1-linux.tar.gz | \ - tar xz -C /opt && \ - chown -R root:root /opt/sc-4.5.1-linux && \ - ln -s /opt/sc-4.5.1-linux/bin/sc /usr/local/bin/sc
diff --git a/buildkite/docker/ubuntu1804/Dockerfile b/buildkite/docker/ubuntu1804/Dockerfile deleted file mode 100755 index f431699..0000000 --- a/buildkite/docker/ubuntu1804/Dockerfile +++ /dev/null
@@ -1,131 +0,0 @@ -FROM ubuntu:18.04 as java8 - -ENV DEBIAN_FRONTEND "noninteractive" - -RUN apt-get -qqy update && \ - apt-get -qqy install lsb-release curl apt-utils software-properties-common && \ - rm -rf /var/lib/apt/lists/* - -### Install Google Cloud SDK. -### https://cloud.google.com/sdk/docs/quickstart-debian-ubuntu -RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ - echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ - curl -L https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ - apt-get -qqy update && apt-get -qqy install google-cloud-sdk && \ - rm -rf /var/lib/apt/lists/* - -### Install required packages. -RUN dpkg --add-architecture i386 && \ - apt-get -qqy update && \ - echo "Installing packages required by Bazel" && \ - apt-get -qqy install build-essential clang curl git python python-dev python3 python3-dev unzip wget xvfb zip zlib1g-dev && \ - echo "Installing packages required by Android SDK" && \ - apt-get -qqy install expect libbz2-1.0:i386 libncurses5:i386 libstdc++6:i386 libz1:i386 && \ - echo "Installing packages required by Tensorflow" && \ - apt-get -qqy install libcurl3-dev swig python-enum34 python-mock python-numpy python-pip python-wheel python3-mock python3-numpy python3-pip python3-wheel && \ - pip install keras_applications keras_preprocessing && \ - pip3 install keras_applications keras_preprocessing && \ - echo "Installing packages required by Envoy" && \ - apt-get -qqy install automake autotools-dev cmake libtool m4 && \ - echo "Installing packages required by Android emulator" && \ - apt-get -qqy install cpu-checker qemu-system-x86 unzip xvfb && \ - echo "Installing packages required by Bazel release process" && \ - apt-get -qqy install devscripts gnupg pandoc reprepro ssmtp && \ - echo "Installing packages required by C++ coverage tests" && \ - apt-get -qqy install lcov llvm && \ - echo "Installing packages required by Swift toolchain" && \ - apt-get -qqy install clang libicu-dev && \ - echo "Installing packages required by Bazel (Ubuntu 18.04 only)" && \ - apt-get -qqy install coreutils && \ - apt-get -qqy purge apport && \ - rm -rf /var/lib/apt/lists/* - -### Install Bazelisk (as Bazel). -RUN curl -Lo /usr/local/bin/bazel https://raw.githubusercontent.com/philwo/bazelisk/master/bazelisk.py && \ - chmod 0755 /usr/local/bin/bazel - -### Install Azul Zulu. -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0x219BD9C9 && \ - apt-add-repository 'deb http://repos.azulsystems.com/ubuntu stable main' && \ - apt-get -qqy update && \ - apt-get -qqy install zulu-8 - -### Install Mono. -RUN apt-get -qqy install apt-transport-https ca-certificates && \ - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF && \ - add-apt-repository "deb https://download.mono-project.com/repo/ubuntu stable-$(lsb_release -cs) main" && \ - apt-get -qqy update && \ - apt-get -qqy install mono-devel mono-complete - -### Install Node.js. -RUN curl -L https://deb.nodesource.com/setup_8.x | bash - && \ - apt-get -qqy install nodejs - -### Required by Gerrit. -### https://gerrit.googlesource.com/gerrit/+show/master/polygerrit-ui/README.md -RUN npm install -g typescript fried-twinkie@0.0.15 - -### Install required Python packages. -RUN pip3 install requests uritemplate pyyaml github3.py - -### Install Android NDK. -ENV ANDROID_NDK_HOME "/opt/android-ndk-r15c" -RUN cd /opt && \ - curl -Lo android-ndk.zip https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip && \ - unzip android-ndk.zip > /dev/null && \ - rm android-ndk.zip - -### Install Android SDK. -ENV ANDROID_HOME "/opt/android-sdk-linux" -RUN mkdir -p /opt/android-sdk-linux && \ - cd /opt/android-sdk-linux && \ - curl -Lo android-sdk.zip https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip && \ - unzip android-sdk.zip > /dev/null && \ - rm android-sdk.zip && \ - yes | tools/bin/sdkmanager --licenses > /dev/null && \ - tools/bin/sdkmanager --update && \ - tools/bin/sdkmanager \ - "build-tools;27.0.3" \ - "build-tools;28.0.2" \ - "emulator" \ - "extras;android;m2repository" \ - "platform-tools" \ - "platforms;android-24" \ - "platforms;android-28" \ - "system-images;android-19;default;x86" \ - "system-images;android-21;default;x86" \ - "system-images;android-22;default;x86" \ - "system-images;android-23;default;x86" - -### Install Swift toolchain (for rules_swift). -ENV SWIFT_HOME "/opt/swift" -ENV PATH "${PATH}:${SWIFT_HOME}/usr/bin" -RUN mkdir -p /opt/swift && \ - curl -L https://swift.org/builds/swift-4.2.1-release/ubuntu1804/swift-4.2.1-RELEASE/swift-4.2.1-RELEASE-ubuntu18.04.tar.gz | \ - tar xz -C /opt/swift --strip 1 - -### Install tools required by the release process. -RUN curl -L https://github.com/c4milo/github-release/releases/download/v1.1.0/github-release_v1.1.0_linux_amd64.tar.gz | \ - tar xz -C /usr/local/bin && \ - chown root:root /usr/local/bin/github-release && \ - chmod 0755 /usr/local/bin/github-release - -### Install Sauce Connect (for rules_webtesting). -RUN curl -L https://saucelabs.com/downloads/sc-4.5.1-linux.tar.gz | \ - tar xz -C /opt && \ - chown -R root:root /opt/sc-4.5.1-linux && \ - ln -s /opt/sc-4.5.1-linux/bin/sc /usr/local/bin/sc - -FROM java8 as nojava -RUN apt-get -qqy purge *openjdk* *zulu* && \ - apt-get -qqy autoremove --purge - -FROM nojava as java9 -RUN apt-get -qqy update && \ - apt-get -qqy install zulu-9 && \ - rm -rf /var/lib/apt/lists/* - -FROM nojava as java10 -RUN apt-get -qqy update && \ - apt-get -qqy install zulu-10 && \ - rm -rf /var/lib/apt/lists/*
diff --git a/buildkite/gcloud.py b/buildkite/gcloud.py index e50c59e..4bca4fc 100644 --- a/buildkite/gcloud.py +++ b/buildkite/gcloud.py
@@ -37,6 +37,10 @@ cmd = ["gcloud"] cmd += args for flag, value in kwargs.items(): + # Optionally strip counter suffixes to make it possible to specify the same flag multiple + # times, even though it's passed here via a dict. + if re.search(r"_\d$", flag): + flag = flag[:-2] # Python uses underscores as word delimiters in kwargs, but gcloud wants dashes. flag = flag.replace("_", "-") if isinstance(value, bool):
diff --git a/buildkite/gcloud_utils.py b/buildkite/gcloud_utils.py index cddc3cd..53e1d14 100644 --- a/buildkite/gcloud_utils.py +++ b/buildkite/gcloud_utils.py
@@ -18,13 +18,12 @@ import json import subprocess import time -import threading import re -def wait_for_instance(instance_name, zone, status): +def wait_for_instance(instance_name, project, zone, status): while True: - result = gcloud.describe_instance(instance_name, zone=zone, format="json") + result = gcloud.describe_instance(instance_name, project=project, zone=zone, format="json") current_status = json.loads(result.stdout)["status"] if current_status == status: gcloud.debug( @@ -73,11 +72,13 @@ print(lines) -def tail_serial_console(instance_name, zone, start=None, until=None): +def tail_serial_console(instance_name, project, zone, start=None, until=None): next_start = start if start else "0" while True: try: - result = gcloud.get_serial_port_output(instance_name, zone=zone, start=next_start) + result = gcloud.get_serial_port_output( + instance_name, project=project, zone=zone, start=next_start + ) except subprocess.CalledProcessError as e: if "Could not fetch serial port output: TIMEOUT" in e.stderr: gcloud.debug("tail_serial_console: Retrying after TIMEOUT")
diff --git a/buildkite/incompatible_flag_verbose_failures.py b/buildkite/incompatible_flag_verbose_failures.py index 697b22a..5762d8e 100644 --- a/buildkite/incompatible_flag_verbose_failures.py +++ b/buildkite/incompatible_flag_verbose_failures.py
@@ -15,23 +15,54 @@ # limitations under the License. import argparse +import base64 import json import os -import sys -import subprocess -import yaml import re +import subprocess +import sys +import yaml + import bazelci from bazelci import BuildkiteException BUILD_STATUS_API_URL = "https://api.buildkite.com/v2/organizations/bazel/pipelines/bazel-at-release-plus-incompatible-flags/builds/" +ENCRYPTED_BUILDKITE_API_TOKEN = """ +CiQAFKMEShE9EMWmhqEpX4gPRLdVe8SgL84SVyMAWLTLXd6VssASUADXsQCefXM1gXeKWD5qLKVT +VouWIY1h9jpCEd9bgy/UgMuf19M0dcklP4wpfPGL/+ZyuixQh0Ih+TfD8UAghCpN7VA6MLDNrV0D +k/lpv7B5 +""".strip() + + +def buildkite_token(): + return ( + subprocess.check_output( + [ + bazelci.gcloud_command(), + "kms", + "decrypt", + "--location", + "global", + "--keyring", + "buildkite", + "--key", + "buildkite-api-token", + "--ciphertext-file", + "-", + "--plaintext-file", + "-", + ], + input=base64.b64decode(ENCRYPTED_BUILDKITE_API_TOKEN), + env=os.environ, + ) + .decode("utf-8") + .strip() + ) + def get_build_status_api_url(build_number): - return BUILD_STATUS_API_URL + "%s?access_token=%s" % ( - build_number, - bazelci.fetch_buildkite_token(), - ) + return BUILD_STATUS_API_URL + "%s?access_token=%s" % (build_number, buildkite_token()) def get_build_info(build_number):
diff --git a/buildkite/instances.yml b/buildkite/instances.yml index 6483b01..d825dbb 100644 --- a/buildkite/instances.yml +++ b/buildkite/instances.yml
@@ -14,63 +14,46 @@ default_vm: boot_disk_size: 50GB boot_disk_type: pd-ssd - image_project: bazel-public + image_project: bazel-untrusted machine_type: n1-standard-32 min_cpu_platform: Intel Skylake network: buildkite scopes: cloud-platform - service_account: remote-account@bazel-public.iam.gserviceaccount.com - zone: europe-west1-c instance_groups: - - name: bk-pipeline-ubuntu1804-java8 - count: 1 - image_family: bk-pipeline-ubuntu1804-java8 + - name: bk-docker + count: 60 + project: bazel-untrusted + region: europe-north1 + service_account: buildkite@bazel-untrusted.iam.gserviceaccount.com + image_family: bk-docker local_ssd: interface=nvme - machine_type: n1-standard-8 - metadata_from_file: startup-script=startup-ubuntu.sh - - name: bk-trusted-ubuntu1804-java8 - count: 1 - image_family: bk-trusted-ubuntu1804-java8 - local_ssd: interface=nvme - machine_type: n1-standard-8 - metadata_from_file: startup-script=startup-ubuntu.sh - service_account: bazel-release-process@bazel-public.iam.gserviceaccount.com - - name: bk-worker-ubuntu1404-java8 - count: 12 - image_family: bk-worker-ubuntu1404-java8 - local_ssd: interface=nvme - metadata_from_file: startup-script=startup-ubuntu.sh - - name: bk-worker-ubuntu1604-java8 - count: 16 - image_family: bk-worker-ubuntu1604-java8 - local_ssd: interface=nvme - metadata_from_file: startup-script=startup-ubuntu.sh - - name: bk-worker-ubuntu1804-java8 + metadata_from_file: startup-script=startup-docker.sh + restart-on-failure: False + - name: bk-windows-java8 count: 8 - image_family: bk-worker-ubuntu1804-java8 - local_ssd: interface=nvme - metadata_from_file: startup-script=startup-ubuntu.sh - - name: bk-worker-ubuntu1804-java9 - count: 8 - image_family: bk-worker-ubuntu1804-java9 - local_ssd: interface=nvme - metadata_from_file: startup-script=startup-ubuntu.sh - - name: bk-worker-ubuntu1804-java10 - count: 8 - image_family: bk-worker-ubuntu1804-java10 - local_ssd: interface=nvme - metadata_from_file: startup-script=startup-ubuntu.sh - - name: bk-worker-ubuntu1804-nojava - count: 8 - image_family: bk-worker-ubuntu1804-nojava - local_ssd: interface=nvme - metadata_from_file: startup-script=startup-ubuntu.sh - - name: bk-worker-windows-java8 - count: 8 + project: bazel-untrusted + region: europe-north1 + service_account: buildkite@bazel-untrusted.iam.gserviceaccount.com image_family: bk-worker-windows-java8 local_ssd: interface=scsi metadata_from_file: windows-startup-script-ps1=startup-windows.ps1 restart-on-failure: False + # - name: bk-trusted-docker + # count: 1 + # image_family: bk-trusted-ubuntu1804-java8 + # local_ssd: interface=nvme + # machine_type: n1-standard-8 + # metadata_from_file: startup-script=startup-ubuntu.sh + # service_account: bazel-release-process@bazel-public.iam.gserviceaccount.com + # region: europe-west1 + # zone: europe-west1-c + # service_account: remote-account@bazel-public.iam.gserviceaccount.com + # - name: bk-trusted-windows-java8 + # count: 8 + # image_family: bk-worker-windows-java8 + # local_ssd: interface=scsi + # metadata_from_file: windows-startup-script-ps1=startup-windows.ps1 + # restart-on-failure: False physical_clusters: - name: buildkite-imacpro count: 10
diff --git a/buildkite/setup-docker.sh b/buildkite/setup-docker.sh index a534242..21372f3 100644 --- a/buildkite/setup-docker.sh +++ b/buildkite/setup-docker.sh
@@ -28,7 +28,8 @@ ### Install base packages. { apt-get -qqy update - apt-get -qqy dist-upgrade > /dev/null + apt-get -qqy dist-upgrade + apt-get -qqy install zfsutils-linux } ### Increase file descriptor limits @@ -42,46 +43,32 @@ ### Install the Buildkite Agent on production images. { apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 \ - --recv-keys 32A37959C2FA5C3C99EFBC32A79206696452D198 &> /dev/null + --recv-keys 32A37959C2FA5C3C99EFBC32A79206696452D198 add-apt-repository -y "deb https://apt.buildkite.com/buildkite-agent stable main" apt-get -qqy update - apt-get -qqy install buildkite-agent > /dev/null + apt-get -qqy install buildkite-agent - # Write the Buildkite agent configuration. - cat > /etc/buildkite-agent/buildkite-agent.cfg <<EOF -token="xxx" -name="%hostname-%n" -tags="kind=docker,os=linux" -build-path="/var/lib/buildkite-agent/builds" -hooks-path="/etc/buildkite-agent/hooks" -plugins-path="/etc/buildkite-agent/plugins" -git-clone-flags="-v --reference /var/lib/bazelbuild" -disconnect-after-job=true -disconnect-after-job-timeout=86400 -EOF - - # Add the Buildkite agent hooks. - cat > /etc/buildkite-agent/hooks/environment <<'EOF' -#!/bin/bash - -set -euo pipefail - -export PATH=$PATH:/usr/lib/google-cloud-sdk/bin:/snap/bin:/snap/google-cloud-sdk/current/bin -export BUILDKITE_ARTIFACT_UPLOAD_DESTINATION="gs://bazel-buildkite-artifacts/$BUILDKITE_JOB_ID" -export BUILDKITE_GS_ACL="publicRead" - -gcloud auth configure-docker --quiet -EOF - - # This is a normal worker machine with systemd (e.g. Ubuntu 16.04 LTS). + # Disable the Buildkite agent service, as the startup script has to mount /var/lib/buildkite-agent + # first. systemctl disable buildkite-agent + mkdir /etc/systemd/system/buildkite-agent.service.d cat > /etc/systemd/system/buildkite-agent.service.d/override.conf <<'EOF' [Service] Restart=always PermissionsStartOnly=true -# Immediately force a shutdown of the system when the Buildkite agent terminates. -ExecStopPost=/sbin/poweroff --force --force +# Disable tasks accounting, because Bazel is prone to run into resource limits there. +# This fixes the "cgroup: fork rejected by pids controller" error that some CI jobs triggered. +TasksAccounting=no +EOF + + mkdir /etc/systemd/system/buildkite-agent@.service.d + cat > /etc/systemd/system/buildkite-agent@.service.d/override.conf <<'EOF' +[Service] +Restart=always +PermissionsStartOnly=true +Environment=BUILDKITE_AGENT_NAME=%%hostname-%i +Environment=BUILDKITE_AGENT_PRIORITY=%i # Disable tasks accounting, because Bazel is prone to run into resource limits there. # This fixes the "cgroup: fork rejected by pids controller" error that some CI jobs triggered. TasksAccounting=no @@ -90,37 +77,23 @@ ### Install Docker. { - apt-get -qqy install apt-transport-https ca-certificates > /dev/null + apt-get -qqy install apt-transport-https ca-certificates curl -sSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" apt-get -qqy update - apt-get -qqy install docker-ce > /dev/null + apt-get -qqy install docker-ce # Allow the buildkite-agent user access to Docker. usermod -aG docker buildkite-agent - # Use our caching Docker registry. - cat > /etc/docker/daemon.json <<'EOF' -{ - "insecure-registries" : ["docker-cache.europe-west1-c.c.bazel-public.internal:5000"] -} -EOF - - # Disable the Docker service, as the startup script has to mount + # Disable the Docker service and related stuff, as the startup script has to mount # /var/lib/docker first. + systemctl disable containerd systemctl disable docker } -### Download a static bundle of all our Git repositories and unpack it. -{ - rm -rf /var/lib/bazelbuild - curl -sS https://storage.googleapis.com/bazel-git-mirror/bazelbuild.tar | tar x -C /var/lib - chown -R root:root /var/lib/bazelbuild - chmod -R 0755 /var/lib/bazelbuild -} - ### Clean up and trim the filesystem (potentially reduces the final image size). { rm -rf /var/lib/apt/lists/*
diff --git a/buildkite/setup-ubuntu.sh b/buildkite/setup-ubuntu.sh index eacb7cc..d0dc609 100755 --- a/buildkite/setup-ubuntu.sh +++ b/buildkite/setup-ubuntu.sh
@@ -181,13 +181,6 @@ pip3 install keras_applications keras_preprocessing } -### Fetch and save image version to file. -{ - IMAGE_VERSION=$(curl -sS "http://metadata.google.internal/computeMetadata/v1/instance/attributes/image-version" -H "Metadata-Flavor: Google") - echo -n "${IMAGE_VERSION}" > /etc/image-version -} - - ### Install Azul Zulu (OpenJDK). if [[ "${config_java}" != "no" ]]; then apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0x219BD9C9 @@ -219,7 +212,7 @@ cat > /etc/buildkite-agent/buildkite-agent.cfg <<EOF token="xxx" name="%hostname-%n" -tags="kind=${config_kind},os=${config_os},java=${config_java},image-version=${IMAGE_VERSION}" +tags="kind=${config_kind},os=${config_os},java=${config_java}" build-path="/var/lib/buildkite-agent/builds" hooks-path="/etc/buildkite-agent/hooks" plugins-path="/etc/buildkite-agent/plugins"
diff --git a/buildkite/startup-docker.sh b/buildkite/startup-docker.sh index a879425..690a8ae 100644 --- a/buildkite/startup-docker.sh +++ b/buildkite/startup-docker.sh
@@ -14,35 +14,101 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Fail on errors. +# Fail when using undefined variables. +# Print all executed commands. +# Fail when any command in a pipe fails. set -euxo pipefail # Wait for all snaps to become available. snap wait system seed.loaded +# Prevent dpkg / apt-get / debconf from trying to access stdin. +export DEBIAN_FRONTEND="noninteractive" + # Ubuntu 18.04 installs gcloud, gsutil, etc. commands in /snap/bin -export PATH=$PATH:/snap/bin +export PATH=$PATH:/snap/bin:/snap/google-cloud-sdk/current/bin -# If available: Use the local SSD as swap space. -if [[ -e /dev/nvme0n1 ]]; then - mkswap -f /dev/nvme0n1 - swapon /dev/nvme0n1 +# Optimize the CPU scheduler for throughput. +# (see https://unix.stackexchange.com/questions/466722/how-to-change-the-length-of-time-slices-used-by-the-linux-cpu-scheduler/466723) +sysctl -w kernel.sched_min_granularity_ns=10000000 +sysctl -w kernel.sched_wakeup_granularity_ns=15000000 +sysctl -w vm.dirty_ratio=40 +#echo always > /sys/kernel/mm/transparent_hugepage/enabled - # Move fast and lose data. - mount -t tmpfs -o mode=1777,uid=root,gid=root,size=$((100 * 1024 * 1024 * 1024)) tmpfs /tmp - mount -t tmpfs -o mode=0711,uid=root,gid=root,size=$((100 * 1024 * 1024 * 1024)) tmpfs /var/lib/docker - mount -t tmpfs -o mode=0755,uid=buildkite-agent,gid=buildkite-agent,size=$((100 * 1024 * 1024 * 1024)) tmpfs /var/lib/buildkite-agent -fi +# Use the local SSDs as fast storage for Docker and the Buildkite agent. +zpool destroy -f bazel || true +zpool create -f \ + -o ashift=12 \ + -O canmount=off \ + -O compression=lz4 \ + -O normalization=formD \ + -O relatime=on \ + -O sync=disabled \ + -O xattr=sa \ + bazel /dev/nvme0n? -# Start Docker. +rm -rf /var/lib/bazelbuild +zfs create -o mountpoint=/var/lib/bazelbuild bazel/bazelbuild +curl https://storage.googleapis.com/bazel-git-mirror/bazelbuild.tar | tar x -C /var/lib +chown -R root:root /var/lib/bazelbuild +chmod -R 0755 /var/lib/bazelbuild + +rm -rf /var/lib/buildkite-agent +zfs create -o mountpoint=/var/lib/buildkite-agent bazel/buildkite-agent +chown buildkite-agent:buildkite-agent /var/lib/buildkite-agent +chmod 0755 /var/lib/buildkite-agent + +rm -rf /var/lib/docker +zfs create -o mountpoint=/var/lib/docker bazel/docker +chown root:root /var/lib/docker +chmod 0711 /var/lib/docker + +# Configure and start Docker. +cat > /etc/docker/daemon.json <<'EOF' +{ + "insecure-registries" : ["docker-cache.europe-north1-a.c.bazel-untrusted.internal:5000"], + "storage-driver": "zfs" +} +EOF systemctl start docker -# Get the Buildkite Token from GCS and decrypt it using KMS. -BUILDKITE_TOKEN=$(gsutil cat "gs://bazel-encrypted-secrets/buildkite-agent-token.enc" | \ - gcloud kms decrypt --location global --keyring buildkite --key buildkite-agent-token --ciphertext-file - --plaintext-file -) +# Pull some known images so that we don't have to download / extract them on each CI job. +gcloud auth configure-docker --quiet +docker pull gcr.io/bazel-untrusted/ubuntu1404:java8 & +docker pull gcr.io/bazel-untrusted/ubuntu1604:java8 & +for java in java8 java9 java10 nojava; do + docker pull gcr.io/bazel-untrusted/ubuntu1804:$java & +done +wait -# Insert the Buildkite Token into the agent configuration. -sed -i "s/token=\"xxx\"/token=\"${BUILDKITE_TOKEN}\"/" /etc/buildkite-agent/buildkite-agent.cfg -sed -i "s/name=.*/name=%hostname/" /etc/buildkite-agent/buildkite-agent.cfg +# Get the Buildkite Token from GCS and decrypt it using KMS. +BUILDKITE_TOKEN=$(gsutil cat "gs://bazel-untrusted-encrypted-secrets/buildkite-untrusted-agent-token.enc" | \ + gcloud kms decrypt --project bazel-untrusted --location global --keyring buildkite --key buildkite-untrusted-agent-token --ciphertext-file - --plaintext-file -) + +# Write the Buildkite agent configuration. +cat > /etc/buildkite-agent/buildkite-agent.cfg <<EOF +token="${BUILDKITE_TOKEN}" +name="%hostname" +tags="kind=docker,os=linux" +build-path="/var/lib/buildkite-agent/builds" +hooks-path="/etc/buildkite-agent/hooks" +plugins-path="/etc/buildkite-agent/plugins" +git-clone-flags="-v --reference /var/lib/bazelbuild" +EOF + +# Add the Buildkite agent hooks. +cat > /etc/buildkite-agent/hooks/environment <<'EOF' +#!/bin/bash + +set -euo pipefail + +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin:/snap/google-cloud-sdk/current/bin" +export BUILDKITE_ARTIFACT_UPLOAD_DESTINATION="gs://bazel-untrusted-buildkite-artifacts/$BUILDKITE_JOB_ID" +export BUILDKITE_GS_ACL="publicRead" + +gcloud auth configure-docker --quiet +EOF # Fix permissions of the Buildkite agent configuration files and hooks. chmod 0400 /etc/buildkite-agent/buildkite-agent.cfg