blob: 26b241f46d0bd4a889d35746aa6e732192c6c681 [file] [log] [blame]
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01001#!/usr/bin/env python3
2#
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01003# Copyright 2018 The Bazel Authors. All rights reserved.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010017import argparse
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +010018import base64
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010019import codecs
Jakob Buchgraber12807052018-02-25 17:04:56 +010020import datetime
Philipp Wollermann2b4ee9f2021-02-11 16:32:35 +010021from glob import glob
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +010022import hashlib
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010023import json
Jakob Buchgraber6db0f262018-02-17 15:45:54 +010024import multiprocessing
Philipp Wollermann0a04cf32018-02-21 17:07:22 +010025import os
Philipp Wollermanndcaddd92018-02-21 14:13:43 +010026import os.path
Jakob Buchgraber257693b2018-02-20 00:03:56 +010027import random
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010028import re
Yun Peng9337bb32020-02-28 13:31:29 +010029import requests
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010030import shutil
Philipp Wollermanndcaddd92018-02-21 14:13:43 +010031import stat
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010032import subprocess
33import sys
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010034import tempfile
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +020035import threading
Philipp Wollermanne1318eb2018-08-13 15:08:01 +020036import time
Philipp Wollermannce986af2019-07-18 14:46:05 +020037import urllib.error
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010038import urllib.request
Philipp Wollermanne1318eb2018-08-13 15:08:01 +020039import uuid
Jakob Buchgraber25bb50f2018-02-22 18:06:21 +010040import yaml
Philipp Wollermanndcaddd92018-02-21 14:13:43 +010041
42# Initialize the random number generator.
43random.seed()
44
Philipp Wollermanne67eec42019-05-24 15:18:20 +020045BUILDKITE_ORG = os.environ["BUILDKITE_ORGANIZATION_SLUG"]
Florian Weikertc2745512021-02-17 16:13:55 +010046THIS_IS_PRODUCTION = BUILDKITE_ORG == "bazel"
Philipp Wollermanne67eec42019-05-24 15:18:20 +020047THIS_IS_TESTING = BUILDKITE_ORG == "bazel-testing"
48THIS_IS_TRUSTED = BUILDKITE_ORG == "bazel-trusted"
49THIS_IS_SPARTA = True
50
51CLOUD_PROJECT = "bazel-public" if THIS_IS_TRUSTED else "bazel-untrusted"
52
53GITHUB_BRANCH = {"bazel": "master", "bazel-trusted": "master", "bazel-testing": "testing"}[
54 BUILDKITE_ORG
55]
56
57SCRIPT_URL = "https://raw.githubusercontent.com/bazelbuild/continuous-integration/{}/buildkite/bazelci.py?{}".format(
58 GITHUB_BRANCH, int(time.time())
Philipp Wollermannc05ac682019-01-19 12:37:28 +010059)
Jakob Buchgraber95e3d572018-02-21 18:48:49 +010060
Philipp Wollermanne67eec42019-05-24 15:18:20 +020061INCOMPATIBLE_FLAG_VERBOSE_FAILURES_URL = "https://raw.githubusercontent.com/bazelbuild/continuous-integration/{}/buildkite/incompatible_flag_verbose_failures.py?{}".format(
62 GITHUB_BRANCH, int(time.time())
63)
64
65AGGREGATE_INCOMPATIBLE_TEST_RESULT_URL = "https://raw.githubusercontent.com/bazelbuild/continuous-integration/{}/buildkite/aggregate_incompatible_flags_test_result.py?{}".format(
66 GITHUB_BRANCH, int(time.time())
67)
68
69EMERGENCY_FILE_URL = "https://raw.githubusercontent.com/bazelbuild/continuous-integration/{}/buildkite/emergency.yml?{}".format(
70 GITHUB_BRANCH, int(time.time())
71)
72
73FLAKY_TESTS_BUCKET = {
74 "bazel-testing": "gs://bazel-testing-buildkite-stats/flaky-tests-bep/",
75 "bazel-trusted": "gs://bazel-buildkite-stats/flaky-tests-bep/",
76 "bazel": "gs://bazel-buildkite-stats/flaky-tests-bep/",
77}[BUILDKITE_ORG]
78
Chi Wangb2b65682020-08-27 10:36:15 +080079KZIPS_BUCKET = {
80 "bazel-testing": "gs://bazel-kzips-testing/",
81 "bazel-trusted": "gs://bazel-kzips/",
82 "bazel": "gs://bazel-kzips/",
83}[BUILDKITE_ORG]
84
Florian Weikert797787b2019-12-19 15:33:07 +010085# Projects can opt out of receiving GitHub issues from --notify by adding `"do_not_notify": True` to their respective downstream entry.
Philipp Wollermanne67eec42019-05-24 15:18:20 +020086DOWNSTREAM_PROJECTS_PRODUCTION = {
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +010087 "Android Studio Plugin": {
88 "git_repository": "https://github.com/bazelbuild/intellij.git",
89 "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/master/.bazelci/android-studio.yml",
90 "pipeline_slug": "android-studio-plugin",
91 },
Yun Peng996efad2018-11-27 17:19:44 +010092 "Android Testing": {
93 "git_repository": "https://github.com/googlesamples/android-testing.git",
Jingwenbde72602018-12-13 10:57:43 -050094 "http_config": "https://raw.githubusercontent.com/googlesamples/android-testing/master/bazelci/buildkite-pipeline.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +010095 "pipeline_slug": "android-testing",
Yun Peng996efad2018-11-27 17:19:44 +010096 },
Yun Peng8910fa32019-01-03 08:58:16 +010097 "Bazel": {
98 "git_repository": "https://github.com/bazelbuild/bazel.git",
99 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel/master/.bazelci/postsubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100100 "pipeline_slug": "bazel-bazel",
Yun Peng8910fa32019-01-03 08:58:16 +0100101 },
Tobias Werthd848eca2019-05-14 15:08:35 +0200102 "Bazel Bench": {
103 "git_repository": "https://github.com/bazelbuild/bazel-bench.git",
joeleba92ffec82019-05-22 14:50:15 +0200104 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-bench/master/.bazelci/postsubmit.yml",
Tobias Werthd848eca2019-05-14 15:08:35 +0200105 "pipeline_slug": "bazel-bench",
106 },
Philipp Wollermannfefcbf42019-05-28 14:28:40 +0200107 "Bazel Codelabs": {
108 "git_repository": "https://github.com/bazelbuild/codelabs.git",
109 "http_config": "https://raw.githubusercontent.com/bazelbuild/codelabs/master/.bazelci/presubmit.yml",
110 "pipeline_slug": "bazel-codelabs",
Yun Peng88d80ae2020-11-19 16:23:49 +0100111 "disabled_reason": "https://github.com/bazelbuild/codelabs/issues/38",
Philipp Wollermannfefcbf42019-05-28 14:28:40 +0200112 },
Jinfce9b302019-08-08 15:18:26 -0400113 "Bazel Examples": {
114 "git_repository": "https://github.com/bazelbuild/examples.git",
115 "http_config": "https://raw.githubusercontent.com/bazelbuild/examples/master/.bazelci/presubmit.yml",
116 "pipeline_slug": "bazel-bazel-examples",
117 },
Florian Weikert4b3ec672019-08-14 19:05:12 +0200118 "Bazel Federation": {
119 "git_repository": "https://github.com/bazelbuild/bazel-federation.git",
120 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-federation/master/.bazelci/presubmit.yml",
121 "pipeline_slug": "bazel-federation",
Yun Pengc263eff2020-11-19 15:20:15 +0100122 "disabled_reason": "https://github.com/bazelbuild/bazel-federation/issues/126",
Florian Weikert4b3ec672019-08-14 19:05:12 +0200123 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100124 "Bazel Remote Cache": {
125 "git_repository": "https://github.com/buchgr/bazel-remote.git",
126 "http_config": "https://raw.githubusercontent.com/buchgr/bazel-remote/master/.bazelci/presubmit.yml",
127 "pipeline_slug": "bazel-remote-cache",
Yun Peng996efad2018-11-27 17:19:44 +0100128 },
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200129 "Bazel integration testing": {
130 "git_repository": "https://github.com/bazelbuild/bazel-integration-testing.git",
131 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-integration-testing/master/.bazelci/presubmit.yml",
132 "pipeline_slug": "bazel-integration-testing",
133 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100134 "Bazel skylib": {
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200135 "git_repository": "https://github.com/bazelbuild/bazel-skylib.git",
Alexandre Rostovtsevd414d0d2021-04-16 13:44:35 -0400136 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-skylib/main/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100137 "pipeline_slug": "bazel-skylib",
Yun Peng667750b2020-02-20 14:06:43 +0100138 "owned_by_bazel": True,
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200139 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100140 "Bazel toolchains": {
141 "git_repository": "https://github.com/bazelbuild/bazel-toolchains.git",
142 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-toolchains/master/.bazelci/presubmit.yml",
143 "pipeline_slug": "bazel-toolchains",
144 },
145 "Bazel watcher": {
146 "git_repository": "https://github.com/bazelbuild/bazel-watcher.git",
147 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-watcher/master/.bazelci/presubmit.yml",
148 "pipeline_slug": "bazel-watcher",
149 },
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200150 "Bazelisk": {
151 "git_repository": "https://github.com/bazelbuild/bazelisk.git",
152 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazelisk/master/.bazelci/config.yml",
153 "pipeline_slug": "bazelisk",
154 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100155 "Buildfarm": {
156 "git_repository": "https://github.com/bazelbuild/bazel-buildfarm.git",
Philipp Wollermanndf1e19b2021-09-09 12:33:37 +0200157 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-buildfarm/main/.bazelci/presubmit.yml",
Philipp Wollermann89e5d882021-09-09 12:44:29 +0200158 "pipeline_slug": "buildfarm-farmer",
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100159 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100160 "Buildtools": {
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200161 "git_repository": "https://github.com/bazelbuild/buildtools.git",
Yun Peng996efad2018-11-27 17:19:44 +0100162 "http_config": "https://raw.githubusercontent.com/bazelbuild/buildtools/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100163 "pipeline_slug": "buildtools",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200164 },
Yun Peng175bf3e2021-02-23 16:37:35 +0100165 "Cargo-Raze": {
166 "git_repository": "https://github.com/google/cargo-raze.git",
167 "http_config": "https://raw.githubusercontent.com/google/cargo-raze/master/.bazelci/presubmit.yml",
168 "pipeline_slug": "cargo-raze",
169 },
Yun Peng39a42582018-11-09 10:59:47 +0100170 "CLion Plugin": {
171 "git_repository": "https://github.com/bazelbuild/intellij.git",
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100172 "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/master/.bazelci/clion.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100173 "pipeline_slug": "clion-plugin",
Yun Peng39a42582018-11-09 10:59:47 +0100174 },
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200175 "Cartographer": {
176 "git_repository": "https://github.com/googlecartographer/cartographer.git",
177 "http_config": "https://raw.githubusercontent.com/googlecartographer/cartographer/master/.bazelci/presubmit.yml",
178 "pipeline_slug": "cartographer",
179 },
Philipp Wollermannee850782019-02-05 22:56:04 +0100180 "Cloud Robotics Core": {
Stefan Sauerb4dd3f92019-02-05 22:44:28 +0100181 "git_repository": "https://github.com/googlecloudrobotics/core.git",
Philipp Wollermanndd8cf922021-10-01 15:43:33 +0200182 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/pipelines/cloud-robotics.yml",
Stefan Sauerb4dd3f92019-02-05 22:44:28 +0100183 "pipeline_slug": "cloud-robotics-core",
184 },
Keith Smiley3b0ba602019-05-15 04:42:19 -0700185 "Envoy": {
186 "git_repository": "https://github.com/envoyproxy/envoy.git",
Philipp Wollermanndd8cf922021-10-01 15:43:33 +0200187 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/pipelines/envoy.yml",
Keith Smiley3b0ba602019-05-15 04:42:19 -0700188 "pipeline_slug": "envoy",
189 },
Florian Weikert1fe28b72019-07-02 12:47:55 +0200190 "FlatBuffers": {
191 "git_repository": "https://github.com/google/flatbuffers.git",
192 "http_config": "https://raw.githubusercontent.com/google/flatbuffers/master/.bazelci/presubmit.yml",
193 "pipeline_slug": "flatbuffers",
Florian Weikert1d08af62021-08-07 08:29:20 +0200194 "disabled_reason": "https://github.com/bazelbuild/bazel/issues/13811",
Florian Weikert1fe28b72019-07-02 12:47:55 +0200195 },
Philipp Wollermannf3750fa2019-05-21 17:11:59 +0200196 "Flogger": {
197 "git_repository": "https://github.com/google/flogger.git",
Philipp Wollermanndd8cf922021-10-01 15:43:33 +0200198 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/pipelines/flogger.yml",
Philipp Wollermannf3750fa2019-05-21 17:11:59 +0200199 "pipeline_slug": "flogger",
200 },
Marcel Hlopkoc8840772018-10-23 12:51:46 +0200201 "Gerrit": {
202 "git_repository": "https://gerrit.googlesource.com/gerrit.git",
Philipp Wollermanndd8cf922021-10-01 15:43:33 +0200203 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/pipelines/gerrit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100204 "pipeline_slug": "gerrit",
Florian Weikert8bc0c772021-06-17 10:24:31 +0200205 "disabled_reason": "https://github.com/bazelbuild/continuous-integration/issues/1182",
Marcel Hlopkoc8840772018-10-23 12:51:46 +0200206 },
Yun Pengd6622022018-11-05 13:10:26 +0100207 "Google Logging": {
208 "git_repository": "https://github.com/google/glog.git",
Philipp Wollermann17e5fc62021-02-15 14:48:36 +0100209 "http_config": "https://raw.githubusercontent.com/google/glog/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100210 "pipeline_slug": "google-logging",
Yun Pengd6622022018-11-05 13:10:26 +0100211 },
Yun Peng9586db52018-11-02 10:48:40 +0100212 "IntelliJ Plugin": {
213 "git_repository": "https://github.com/bazelbuild/intellij.git",
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100214 "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/master/.bazelci/intellij.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100215 "pipeline_slug": "intellij-plugin",
Yun Peng9586db52018-11-02 10:48:40 +0100216 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100217 "IntelliJ Plugin Aspect": {
218 "git_repository": "https://github.com/bazelbuild/intellij.git",
219 "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/master/.bazelci/aspect.yml",
220 "pipeline_slug": "intellij-plugin-aspect",
221 },
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200222 "Kythe": {
223 "git_repository": "https://github.com/kythe/kythe.git",
224 "http_config": "https://raw.githubusercontent.com/kythe/kythe/master/.bazelci/presubmit.yml",
225 "pipeline_slug": "kythe",
226 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100227 "Protobuf": {
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200228 "git_repository": "https://github.com/google/protobuf.git",
Philipp Wollermanndd8cf922021-10-01 15:43:33 +0200229 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/pipelines/protobuf.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100230 "pipeline_slug": "protobuf",
Yun Peng667750b2020-02-20 14:06:43 +0100231 "owned_by_bazel": True,
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200232 },
Laurent Le Brunf6326d62020-07-28 18:24:10 +0200233 "Stardoc": {
234 "git_repository": "https://github.com/bazelbuild/stardoc.git",
235 "http_config": "https://raw.githubusercontent.com/bazelbuild/stardoc/master/.bazelci/presubmit.yml",
236 "pipeline_slug": "stardoc",
Yun Peng667750b2020-02-20 14:06:43 +0100237 "owned_by_bazel": True,
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200238 },
239 "Subpar": {
240 "git_repository": "https://github.com/google/subpar.git",
Philipp Wollermanndd8cf922021-10-01 15:43:33 +0200241 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/pipelines/subpar.yml",
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200242 "pipeline_slug": "subpar",
Yun Peng667750b2020-02-20 14:06:43 +0100243 "owned_by_bazel": True,
Xùdōng Yáng26e280f2021-07-20 23:33:07 +1000244 "disabled_reason": "https://github.com/google/subpar/issues/133",
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200245 },
246 "TensorFlow": {
247 "git_repository": "https://github.com/tensorflow/tensorflow.git",
Philipp Wollermanndd8cf922021-10-01 15:43:33 +0200248 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/pipelines/tensorflow.yml",
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200249 "pipeline_slug": "tensorflow",
Florian Weikert1d08af62021-08-07 08:29:20 +0200250 "disabled_reason": "https://github.com/bazelbuild/bazel/issues/13811",
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200251 },
252 "Tulsi": {
253 "git_repository": "https://github.com/bazelbuild/tulsi.git",
254 "http_config": "https://raw.githubusercontent.com/bazelbuild/tulsi/master/.bazelci/presubmit.yml",
255 "pipeline_slug": "tulsi-bazel-darwin",
256 },
257 "re2": {
258 "git_repository": "https://github.com/google/re2.git",
Philipp Wollermanndd8cf922021-10-01 15:43:33 +0200259 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/pipelines/re2.yml",
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200260 "pipeline_slug": "re2",
261 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100262 "rules_android": {
263 "git_repository": "https://github.com/bazelbuild/rules_android.git",
264 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_android/master/.bazelci/postsubmit.yml",
265 "pipeline_slug": "rules-android",
Yun Pengc0575c82020-06-03 11:27:45 +0200266 "disabled_reason": "https://github.com/bazelbuild/rules_android/issues/15",
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100267 },
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200268 "rules_appengine": {
269 "git_repository": "https://github.com/bazelbuild/rules_appengine.git",
Yun Peng996efad2018-11-27 17:19:44 +0100270 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_appengine/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100271 "pipeline_slug": "rules-appengine-appengine",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200272 },
Yun Peng809f27b2018-11-13 10:15:39 +0100273 "rules_apple": {
274 "git_repository": "https://github.com/bazelbuild/rules_apple.git",
Yun Peng996efad2018-11-27 17:19:44 +0100275 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_apple/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100276 "pipeline_slug": "rules-apple-darwin",
Yun Peng809f27b2018-11-13 10:15:39 +0100277 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100278 "rules_cc": {
279 "git_repository": "https://github.com/bazelbuild/rules_cc.git",
aiuto560809f2021-08-17 14:51:32 -0400280 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_cc/main/.bazelci/presubmit.yml",
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100281 "pipeline_slug": "rules-cc",
Yun Peng667750b2020-02-20 14:06:43 +0100282 "owned_by_bazel": True,
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100283 },
Marcel Hlopko340dfd22018-10-19 11:33:01 +0200284 "rules_closure": {
285 "git_repository": "https://github.com/bazelbuild/rules_closure.git",
Yun Peng996efad2018-11-27 17:19:44 +0100286 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_closure/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100287 "pipeline_slug": "rules-closure-closure-compiler",
Yun Peng667750b2020-02-20 14:06:43 +0100288 "owned_by_bazel": True,
Marcel Hlopko340dfd22018-10-19 11:33:01 +0200289 },
Yun Peng51ce6692019-01-09 14:31:46 +0100290 "rules_d": {
291 "git_repository": "https://github.com/bazelbuild/rules_d.git",
292 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_d/master/.bazelci/presubmit.yml",
293 "pipeline_slug": "rules-d",
294 },
Yun Peng996efad2018-11-27 17:19:44 +0100295 "rules_docker": {
296 "git_repository": "https://github.com/bazelbuild/rules_docker.git",
297 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_docker/master/.bazelci/presubmit.yml",
Jakob Buchgrabera6a8ea82018-12-07 13:51:02 +0100298 "pipeline_slug": "rules-docker-docker",
Yun Peng996efad2018-11-27 17:19:44 +0100299 },
Florian Weikert5569c112021-04-01 14:04:33 +0200300 "rules_dotnet": {
301 "git_repository": "https://github.com/bazelbuild/rules_dotnet.git",
302 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_dotnet/master/.bazelci/presubmit.yml",
303 "pipeline_slug": "rules-dotnet-edge",
304 },
Yun Peng996efad2018-11-27 17:19:44 +0100305 "rules_foreign_cc": {
306 "git_repository": "https://github.com/bazelbuild/rules_foreign_cc.git",
mai932d494522021-03-30 18:34:05 +0200307 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_foreign_cc/main/.bazelci/config.yaml",
Yun Peng996efad2018-11-27 17:19:44 +0100308 "pipeline_slug": "rules-foreign-cc",
Yun Peng667750b2020-02-20 14:06:43 +0100309 "owned_by_bazel": True,
Yun Peng996efad2018-11-27 17:19:44 +0100310 },
Xindb02c012018-11-07 14:10:54 -0500311 "rules_go": {
312 "git_repository": "https://github.com/bazelbuild/rules_go.git",
Yun Peng996efad2018-11-27 17:19:44 +0100313 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_go/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100314 "pipeline_slug": "rules-go-golang",
Yun Pengb7247ff2018-11-15 13:52:39 +0100315 },
Yun Peng7deea572018-11-05 10:47:45 +0100316 "rules_groovy": {
Yun Peng996efad2018-11-27 17:19:44 +0100317 "git_repository": "https://github.com/bazelbuild/rules_groovy.git",
318 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_groovy/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100319 "pipeline_slug": "rules-groovy",
Yun Peng996efad2018-11-27 17:19:44 +0100320 },
321 "rules_gwt": {
322 "git_repository": "https://github.com/bazelbuild/rules_gwt.git",
323 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_gwt/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100324 "pipeline_slug": "rules-gwt",
Xùdōng Yáng99b86b32021-08-18 23:42:05 +1000325 "disabled_reason": "https://github.com/bazelbuild/continuous-integration/issues/1202",
Philipp Wollermann2a160432019-09-19 15:57:28 +0200326 },
Florian Weikertff6444e2019-09-16 16:08:57 +0200327 "rules_haskell": {
328 "git_repository": "https://github.com/tweag/rules_haskell.git",
329 "http_config": "https://raw.githubusercontent.com/tweag/rules_haskell/master/.bazelci/presubmit.yml",
330 "pipeline_slug": "rules-haskell-haskell",
Philipp Wollermann2a160432019-09-19 15:57:28 +0200331 },
Yun Peng996efad2018-11-27 17:19:44 +0100332 "rules_jsonnet": {
333 "git_repository": "https://github.com/bazelbuild/rules_jsonnet.git",
334 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_jsonnet/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100335 "pipeline_slug": "rules-jsonnet",
Yun Peng996efad2018-11-27 17:19:44 +0100336 },
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200337 "rules_jvm_external": {
338 "git_repository": "https://github.com/bazelbuild/rules_jvm_external.git",
339 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_jvm_external/master/.bazelci/presubmit.yml",
340 "pipeline_slug": "rules-jvm-external",
Yun Peng667750b2020-02-20 14:06:43 +0100341 "owned_by_bazel": True,
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200342 },
343 "rules_jvm_external - examples": {
344 "git_repository": "https://github.com/bazelbuild/rules_jvm_external.git",
345 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_jvm_external/master/.bazelci/examples.yml",
346 "pipeline_slug": "rules-jvm-external-examples",
Yun Peng667750b2020-02-20 14:06:43 +0100347 "owned_by_bazel": True,
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200348 },
Yun Peng996efad2018-11-27 17:19:44 +0100349 "rules_k8s": {
350 "git_repository": "https://github.com/bazelbuild/rules_k8s.git",
351 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_k8s/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100352 "pipeline_slug": "rules-k8s-k8s",
Yun Peng996efad2018-11-27 17:19:44 +0100353 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100354 "rules_kotlin": {
355 "git_repository": "https://github.com/bazelbuild/rules_kotlin.git",
356 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_kotlin/master/.bazelci/presubmit.yml",
357 "pipeline_slug": "rules-kotlin-kotlin",
358 },
Yun Penga5650e12018-11-14 10:16:06 +0100359 "rules_nodejs": {
360 "git_repository": "https://github.com/bazelbuild/rules_nodejs.git",
Ivo List23ce48d2020-11-18 13:15:34 +0100361 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_nodejs/stable/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100362 "pipeline_slug": "rules-nodejs-nodejs",
Yun Penga5650e12018-11-14 10:16:06 +0100363 },
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200364 "rules_perl": {
365 "git_repository": "https://github.com/bazelbuild/rules_perl.git",
Yun Peng996efad2018-11-27 17:19:44 +0100366 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_perl/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100367 "pipeline_slug": "rules-perl",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200368 },
Yannic6110b3c2019-08-12 15:09:37 +0000369 "rules_proto": {
370 "git_repository": "https://github.com/bazelbuild/rules_proto.git",
371 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_proto/master/.bazelci/presubmit.yml",
372 "pipeline_slug": "rules-proto",
Yun Peng667750b2020-02-20 14:06:43 +0100373 "owned_by_bazel": True,
Yannic6110b3c2019-08-12 15:09:37 +0000374 },
Yun Peng3d5a8a62018-11-19 11:42:01 +0100375 "rules_python": {
376 "git_repository": "https://github.com/bazelbuild/rules_python.git",
Florian Weikertf126dfc2021-07-05 15:39:24 +0200377 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_python/main/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100378 "pipeline_slug": "rules-python-python",
Yun Peng667750b2020-02-20 14:06:43 +0100379 "owned_by_bazel": True,
Yun Peng3d5a8a62018-11-19 11:42:01 +0100380 },
Xindb02c012018-11-07 14:10:54 -0500381 "rules_rust": {
382 "git_repository": "https://github.com/bazelbuild/rules_rust.git",
Yun Peng996efad2018-11-27 17:19:44 +0100383 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_rust/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100384 "pipeline_slug": "rules-rust-rustlang",
Xindb02c012018-11-07 14:10:54 -0500385 },
Yun Pengca62fff2018-10-31 11:22:03 +0100386 "rules_sass": {
387 "git_repository": "https://github.com/bazelbuild/rules_sass.git",
Yun Peng996efad2018-11-27 17:19:44 +0100388 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_sass/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100389 "pipeline_slug": "rules-sass",
Yun Pengca62fff2018-10-31 11:22:03 +0100390 },
Xindb02c012018-11-07 14:10:54 -0500391 "rules_scala": {
392 "git_repository": "https://github.com/bazelbuild/rules_scala.git",
Yun Peng996efad2018-11-27 17:19:44 +0100393 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_scala/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100394 "pipeline_slug": "rules-scala-scala",
Xùdōng Yánga1b883a2021-02-27 03:00:43 +1100395 "disabled_reason": "https://github.com/bazelbuild/rules_scala/issues/1224",
Xindb02c012018-11-07 14:10:54 -0500396 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100397 "rules_swift": {
398 "git_repository": "https://github.com/bazelbuild/rules_swift.git",
399 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_swift/master/.bazelci/presubmit.yml",
400 "pipeline_slug": "rules-swift-swift",
Florian Weikertbaa683f2019-12-27 18:09:58 +0100401 "do_not_notify": "https://github.com/bazelbuild/continuous-integration/issues/915",
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100402 },
Yun Peng996efad2018-11-27 17:19:44 +0100403 "rules_webtesting": {
404 "git_repository": "https://github.com/bazelbuild/rules_webtesting.git",
Yun Pengc2fab332019-01-04 10:53:49 +0100405 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_webtesting/master/.bazelci/presubmit.yml",
Yun Peng996efad2018-11-27 17:19:44 +0100406 "pipeline_slug": "rules-webtesting-saucelabs",
Yun Peng996efad2018-11-27 17:19:44 +0100407 },
Philipp Wollermann389acd82019-05-21 17:41:48 +0200408 "upb": {
409 "git_repository": "https://github.com/protocolbuffers/upb.git",
410 "http_config": "https://raw.githubusercontent.com/protocolbuffers/upb/master/.bazelci/presubmit.yml",
411 "pipeline_slug": "upb",
412 },
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200413}
414
Philipp Wollermanne67eec42019-05-24 15:18:20 +0200415DOWNSTREAM_PROJECTS_TESTING = {
Philipp Wollermannbed211d2019-06-07 11:38:59 +0200416 "Bazel": DOWNSTREAM_PROJECTS_PRODUCTION["Bazel"],
417 "Bazelisk": DOWNSTREAM_PROJECTS_PRODUCTION["Bazelisk"],
Florian Weikert8fda2a72019-05-31 15:21:54 +0200418 "Federation": {
419 "git_repository": "https://github.com/fweikert/bazel-federation.git",
420 "http_config": "https://raw.githubusercontent.com/fweikert/bazel-federation/master/.bazelci/presubmit.yml",
421 "pipeline_slug": "bazel-federation",
422 },
Philipp Wollermannbed211d2019-06-07 11:38:59 +0200423 "rules_docker": DOWNSTREAM_PROJECTS_PRODUCTION["rules_docker"],
424 "rules_go": DOWNSTREAM_PROJECTS_PRODUCTION["rules_go"],
425 "rules_groovy": DOWNSTREAM_PROJECTS_PRODUCTION["rules_groovy"],
426 "rules_kotlin": DOWNSTREAM_PROJECTS_PRODUCTION["rules_kotlin"],
427 "rules_nodejs": DOWNSTREAM_PROJECTS_PRODUCTION["rules_nodejs"],
428 "rules_rust": DOWNSTREAM_PROJECTS_PRODUCTION["rules_rust"],
429 "rules_scala": DOWNSTREAM_PROJECTS_PRODUCTION["rules_scala"],
Philipp Wollermanne67eec42019-05-24 15:18:20 +0200430}
431
432DOWNSTREAM_PROJECTS = {
433 "bazel-testing": DOWNSTREAM_PROJECTS_TESTING,
434 "bazel-trusted": {},
435 "bazel": DOWNSTREAM_PROJECTS_PRODUCTION,
436}[BUILDKITE_ORG]
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100437
Philipp Wollermann81a88412019-07-12 10:34:33 +0200438DOCKER_REGISTRY_PREFIX = {
439 "bazel-testing": "bazel-public/testing",
440 "bazel-trusted": "bazel-public",
441 "bazel": "bazel-public",
442}[BUILDKITE_ORG]
443
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200444# A map containing all supported platform names as keys, with the values being
445# the platform name in a human readable format, and a the buildkite-agent's
446# working directory.
447PLATFORMS = {
Philipp Wollermanneffcd6e2019-06-21 18:30:34 +0200448 "centos7": {
449 "name": "CentOS 7, Java 8",
450 "emoji-name": ":centos: 7 (Java 8)",
451 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermannf4aabb72019-06-25 15:59:00 +0200452 "publish_binary": ["ubuntu1404", "centos7", "linux"],
Philipp Wollermann5d6765d2020-02-17 17:12:02 +0100453 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/centos7-java8",
Philipp Wollermanneffcd6e2019-06-21 18:30:34 +0200454 "python": "python3.6",
455 },
Philipp Wollermann67fc3712019-06-12 15:39:21 +0200456 "debian10": {
457 "name": "Debian Buster, OpenJDK 11",
458 "emoji-name": ":debian: Buster (OpenJDK 11)",
459 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
460 "publish_binary": [],
Philipp Wollermann5d6765d2020-02-17 17:12:02 +0100461 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/debian10-java11",
Philipp Wollermann67fc3712019-06-12 15:39:21 +0200462 "python": "python3.7",
463 },
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200464 "ubuntu1604": {
Philipp Wollermanndb877332019-04-23 17:58:01 +0200465 "name": "Ubuntu 16.04, OpenJDK 8",
466 "emoji-name": ":ubuntu: 16.04 (OpenJDK 8)",
Philipp Wollermannd551bf62019-05-18 22:04:35 +0200467 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermannf4aabb72019-06-25 15:59:00 +0200468 "publish_binary": ["ubuntu1604"],
Philipp Wollermann5d6765d2020-02-17 17:12:02 +0100469 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu1604-java8",
Philipp Wollermann57b32682019-05-18 22:09:27 +0200470 "python": "python3.6",
Philipp Wollermann438ec242018-09-05 14:39:24 +0200471 },
472 "ubuntu1804": {
Philipp Wollermannf5a2feb2019-04-25 11:13:46 +0200473 "name": "Ubuntu 18.04, OpenJDK 11",
474 "emoji-name": ":ubuntu: 18.04 (OpenJDK 11)",
Philipp Wollermannd551bf62019-05-18 22:04:35 +0200475 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermann783d1672019-06-06 13:35:30 +0200476 "publish_binary": ["ubuntu1804"],
Philipp Wollermann5d6765d2020-02-17 17:12:02 +0100477 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu1804-java11",
Philipp Wollermann57b32682019-05-18 22:09:27 +0200478 "python": "python3.6",
Philipp Wollermann438ec242018-09-05 14:39:24 +0200479 },
Mostyn Bramley-Mooreab4599e2020-06-23 20:31:01 +0200480 "ubuntu2004": {
481 "name": "Ubuntu 20.04, OpenJDK 11",
482 "emoji-name": ":ubuntu: 20.04 (OpenJDK 11)",
483 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermann55f72ac2020-09-21 22:22:05 +0200484 "publish_binary": [],
Mostyn Bramley-Mooreab4599e2020-06-23 20:31:01 +0200485 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu2004-java11",
486 "python": "python3.8",
487 },
Chi Wang6357efe2020-08-25 16:23:38 +0800488 "kythe_ubuntu2004": {
489 "name": "Kythe (Ubuntu 20.04, OpenJDK 11)",
490 "emoji-name": "Kythe (:ubuntu: 20.04, OpenJDK 11)",
491 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
492 "publish_binary": [],
493 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu2004-java11-kythe",
494 "python": "python3.8",
495 },
Philipp Wollermann438ec242018-09-05 14:39:24 +0200496 "macos": {
Philipp Wollermanndb877332019-04-23 17:58:01 +0200497 "name": "macOS, OpenJDK 8",
498 "emoji-name": ":darwin: (OpenJDK 8)",
Philipp Wollermann51147bf2019-05-08 15:50:10 +0200499 "downstream-root": "/Users/buildkite/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermann783d1672019-06-06 13:35:30 +0200500 "publish_binary": ["macos"],
Philipp Wollermann7a185322019-05-18 22:15:48 +0200501 "queue": "macos",
Philipp Wollermann89d36492021-02-16 11:59:09 +0100502 "python": "python3",
Philipp Wollermann438ec242018-09-05 14:39:24 +0200503 },
Yun Peng46d43912021-04-21 09:49:53 +0200504 "macos_arm64": {
505 "name": "macOS (arm64), OpenJDK 8",
506 "emoji-name": ":darwin: (arm64) (OpenJDK 8)",
507 "downstream-root": "/Users/buildkite/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Yun Peng83f32772021-04-21 11:22:35 +0200508 "publish_binary": ["macos_arm64"],
Yun Peng46d43912021-04-21 09:49:53 +0200509 # TODO(pcloudy): Switch to macos_arm64 queue when Apple Silicon machines are available,
510 # current we just use x86_64 machines to do cross compile.
511 "queue": "macos",
512 "python": "python3",
513 },
Philipp Wollermann438ec242018-09-05 14:39:24 +0200514 "windows": {
Philipp Wollermanndb877332019-04-23 17:58:01 +0200515 "name": "Windows, OpenJDK 8",
516 "emoji-name": ":windows: (OpenJDK 8)",
Philipp Wollermannd5ab3d92020-02-05 16:55:13 +0100517 "downstream-root": "c:/b/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermann783d1672019-06-06 13:35:30 +0200518 "publish_binary": ["windows"],
Philipp Wollermann7a185322019-05-18 22:15:48 +0200519 "queue": "windows",
Philipp Wollermann57b32682019-05-18 22:09:27 +0200520 "python": "python.exe",
Philipp Wollermann438ec242018-09-05 14:39:24 +0200521 },
522 "rbe_ubuntu1604": {
Philipp Wollermanndb877332019-04-23 17:58:01 +0200523 "name": "RBE (Ubuntu 16.04, OpenJDK 8)",
Jakob Buchgraber1f37fbd2019-07-17 17:08:28 +0200524 "emoji-name": "RBE (:ubuntu: 16.04, OpenJDK 8)",
Philipp Wollermannd551bf62019-05-18 22:04:35 +0200525 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermann783d1672019-06-06 13:35:30 +0200526 "publish_binary": [],
Philipp Wollermann5d6765d2020-02-17 17:12:02 +0100527 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu1604-java8",
Philipp Wollermann57b32682019-05-18 22:09:27 +0200528 "python": "python3.6",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100529 },
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200530}
531
Philipp Wollermannfce92bf2019-05-22 15:14:32 +0200532BUILDIFIER_DOCKER_IMAGE = "gcr.io/bazel-public/buildifier"
Florian Weikertf20ae6f2019-01-16 14:32:09 +0100533
Philipp Wollermann1403d2c2019-01-10 13:15:51 +0100534# The platform used for various steps (e.g. stuff that formerly ran on the "pipeline" workers).
535DEFAULT_PLATFORM = "ubuntu1804"
536
Philipp Wollermannf4aabb72019-06-25 15:59:00 +0200537# In order to test that "the one Linux binary" that we build for our official releases actually
538# works on all Linux distributions that we test on, we use the Linux binary built on our official
539# release platform for all Linux downstream tests.
540LINUX_BINARY_PLATFORM = "centos7"
541
Philipp Wollermann14c12b22021-02-15 16:06:54 +0100542DEFAULT_XCODE_VERSION = "12.4"
Philipp Wollermann380f1e62019-04-12 16:45:27 +0200543XCODE_VERSION_REGEX = re.compile(r"^\d+\.\d+(\.\d+)?$")
Florian Weikertdb832a02020-11-19 19:14:48 +0100544XCODE_VERSION_OVERRIDES = {"10.2.1": "10.3", "11.2": "11.2.1", "11.3": "11.3.1"}
Philipp Wollermann380f1e62019-04-12 16:45:27 +0200545
Florian Weikertc8642af2019-02-03 23:58:51 +0100546BUILD_LABEL_PATTERN = re.compile(r"^Build label: (\S+)$", re.MULTILINE)
547
Florian Weikert29cb7ec2019-03-07 14:52:18 +0100548BUILDIFIER_VERSION_ENV_VAR = "BUILDIFIER_VERSION"
549
Florian Weikert85208912019-03-07 17:08:39 +0100550BUILDIFIER_WARNINGS_ENV_VAR = "BUILDIFIER_WARNINGS"
551
Florian Weikertde96a6f2019-03-07 14:57:50 +0100552BUILDIFIER_STEP_NAME = "Buildifier"
553
Florian Weikert5f5d3cb2019-04-15 15:36:27 +0200554SKIP_TASKS_ENV_VAR = "CI_SKIP_TASKS"
555
Philipp Wollermannce986af2019-07-18 14:46:05 +0200556CONFIG_FILE_EXTENSIONS = {".yml", ".yaml"}
Florian Weikert778251c2019-04-25 15:14:44 +0200557
Chi Wang6357efe2020-08-25 16:23:38 +0800558KYTHE_DIR = "/usr/local/kythe"
559
560INDEX_UPLOAD_POLICY_ALWAYS = "Always"
561
562INDEX_UPLOAD_POLICY_IF_BUILD_SUCCESS = "IfBuildSuccess"
563
564INDEX_UPLOAD_POLICY_NEVER = "Never"
Florian Weikert13215a82019-05-10 12:42:21 +0200565
Florian Weikertdb832a02020-11-19 19:14:48 +0100566
Philipp Wollermanndcaddd92018-02-21 14:13:43 +0100567class BuildkiteException(Exception):
568 """
569 Raised whenever something goes wrong and we should exit with an error.
570 """
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100571
Philipp Wollermanndcaddd92018-02-21 14:13:43 +0100572 pass
573
574
575class BinaryUploadRaceException(Exception):
576 """
577 Raised when try_publish_binaries wasn't able to publish a set of binaries,
578 because the generation of the current file didn't match the expected value.
579 """
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100580
Philipp Wollermanndcaddd92018-02-21 14:13:43 +0100581 pass
582
583
Florian Weikerta0e74592019-03-07 11:56:12 +0100584class BuildkiteClient(object):
585
586 _ENCRYPTED_BUILDKITE_API_TOKEN = """
587CiQA4DEB9ldzC+E39KomywtqXfaQ86hhulgeDsicds2BuvbCYzsSUAAqwcvXZPh9IMWlwWh94J2F
588exosKKaWB0tSRJiPKnv2NPDfEqGul0ZwVjtWeASpugwxxKeLhFhPMcgHMPfndH6j2GEIY6nkKRbP
589uwoRMCwe
590""".strip()
591
Philipp Wollermanne67eec42019-05-24 15:18:20 +0200592 _ENCRYPTED_BUILDKITE_API_TESTING_TOKEN = """
593CiQAMTBkWjL1C+F5oon3+cC1vmum5+c1y5+96WQY44p0Lxd0PeASUQAy7iU0c6E3W5EOSFYfD5fA
594MWy/SHaMno1NQSUa4xDOl5yc2kizrtxPPVkX4x9pLNuGUY/xwAn2n1DdiUdWZNWlY1bX2C4ex65e
595P9w8kNhEbw==
596""".strip()
597
Florian Weikertde96a6f2019-03-07 14:57:50 +0100598 _BUILD_STATUS_URL_TEMPLATE = (
599 "https://api.buildkite.com/v2/organizations/{}/pipelines/{}/builds/{}"
600 )
Florian Weikerta0e74592019-03-07 11:56:12 +0100601
Florian Weikertdb832a02020-11-19 19:14:48 +0100602 _NEW_BUILD_URL_TEMPLATE = "https://api.buildkite.com/v2/organizations/{}/pipelines/{}/builds"
Yun Peng9337bb32020-02-28 13:31:29 +0100603
604 _RETRY_JOB_URL_TEMPLATE = (
605 "https://api.buildkite.com/v2/organizations/{}/pipelines/{}/builds/{}/jobs/{}/retry"
606 )
607
Florian Weikerta0e74592019-03-07 11:56:12 +0100608 def __init__(self, org, pipeline):
609 self._org = org
610 self._pipeline = pipeline
611 self._token = self._get_buildkite_token()
612
613 def _get_buildkite_token(self):
Florian Weikert849afb22019-12-14 12:22:29 -0800614 return decrypt_token(
615 encrypted_token=self._ENCRYPTED_BUILDKITE_API_TESTING_TOKEN
616 if THIS_IS_TESTING
617 else self._ENCRYPTED_BUILDKITE_API_TOKEN,
618 kms_key="buildkite-testing-api-token"
619 if THIS_IS_TESTING
620 else "buildkite-untrusted-api-token",
Florian Weikerta0e74592019-03-07 11:56:12 +0100621 )
622
Florian Weikertdb832a02020-11-19 19:14:48 +0100623 def _open_url(self, url, params=[]):
Florian Weikert60661912019-12-18 15:17:10 +0100624 try:
Yun Peng9337bb32020-02-28 13:31:29 +0100625 params_str = "".join("&{}={}".format(k, v) for k, v in params)
Florian Weikert60661912019-12-18 15:17:10 +0100626 return (
Yun Peng9337bb32020-02-28 13:31:29 +0100627 urllib.request.urlopen("{}?access_token={}{}".format(url, self._token, params_str))
Florian Weikert60661912019-12-18 15:17:10 +0100628 .read()
Yun Pengdbedc122020-02-28 13:32:04 +0100629 .decode("utf-8", "ignore")
Florian Weikert60661912019-12-18 15:17:10 +0100630 )
631 except urllib.error.HTTPError as ex:
632 raise BuildkiteException("Failed to open {}: {} - {}".format(url, ex.code, ex.reason))
Florian Weikerta0e74592019-03-07 11:56:12 +0100633
634 def get_build_info(self, build_number):
Yun Peng9337bb32020-02-28 13:31:29 +0100635 """Get build info for a pipeline with a given build number
636 See https://buildkite.com/docs/apis/rest-api/builds#get-a-build
637
638 Parameters
639 ----------
640 build_number : the build number
641
642 Returns
643 -------
644 dict
645 the metadata for the build
646 """
Florian Weikerta0e74592019-03-07 11:56:12 +0100647 url = self._BUILD_STATUS_URL_TEMPLATE.format(self._org, self._pipeline, build_number)
648 output = self._open_url(url)
649 return json.loads(output)
650
Yun Peng9337bb32020-02-28 13:31:29 +0100651 def get_build_info_list(self, params):
652 """Get a list of build infos for this pipeline
653 See https://buildkite.com/docs/apis/rest-api/builds#list-builds-for-a-pipeline
654
655 Parameters
656 ----------
657 params : the parameters to filter the result
658
659 Returns
660 -------
661 list of dict
662 the metadata for a list of builds
663 """
664 url = self._BUILD_STATUS_URL_TEMPLATE.format(self._org, self._pipeline, "")
665 output = self._open_url(url, params)
666 return json.loads(output)
667
Florian Weikerta0e74592019-03-07 11:56:12 +0100668 def get_build_log(self, job):
669 return self._open_url(job["raw_log_url"])
670
Yun Peng9337bb32020-02-28 13:31:29 +0100671 @staticmethod
672 def _check_response(response, expected_status_code):
673 if response.status_code != expected_status_code:
674 eprint("Exit code:", response.status_code)
675 eprint("Response:\n", response.text)
676 response.raise_for_status()
677
Florian Weikertdb832a02020-11-19 19:14:48 +0100678 def trigger_new_build(self, commit, message=None, env={}):
Yun Peng9337bb32020-02-28 13:31:29 +0100679 """Trigger a new build at a given commit and return the build metadata.
680 See https://buildkite.com/docs/apis/rest-api/builds#create-a-build
681
682 Parameters
683 ----------
684 commit : the commit we want to build at
685 message : the message we should as the build titile
686 env : (optional) the environment variables to set
687
688 Returns
689 -------
690 dict
691 the metadata for the build
692 """
693 url = self._NEW_BUILD_URL_TEMPLATE.format(self._org, self._pipeline)
694 data = {
695 "commit": commit,
696 "branch": "master",
697 "message": message if message else f"Trigger build at {commit}",
698 "env": env,
699 }
Florian Weikertdb832a02020-11-19 19:14:48 +0100700 response = requests.post(url + "?access_token=" + self._token, json=data)
Yun Peng9337bb32020-02-28 13:31:29 +0100701 BuildkiteClient._check_response(response, requests.codes.created)
702 return json.loads(response.text)
703
Yun Peng9337bb32020-02-28 13:31:29 +0100704 def trigger_job_retry(self, build_number, job_id):
705 """Trigger a job retry and return the job metadata.
706 See https://buildkite.com/docs/apis/rest-api/jobs#retry-a-job
707
708 Parameters
709 ----------
710 build_number : the number of the build we want to retry
711 job_id : the id of the job we want to retry
712
713 Returns
714 -------
715 dict
716 the metadata for the job
717 """
718 url = self._RETRY_JOB_URL_TEMPLATE.format(self._org, self._pipeline, build_number, job_id)
719 response = requests.put(url + "?access_token=" + self._token)
720 BuildkiteClient._check_response(response, requests.codes.ok)
721 return json.loads(response.text)
722
Yun Peng9337bb32020-02-28 13:31:29 +0100723 def wait_job_to_finish(self, build_number, job_id, interval_time=30, logger=None):
724 """Wait a job to finish and return the job metadata
725
726 Parameters
727 ----------
728 build_number : the number of the build we want to wait
729 job_id : the id of the job we want to wait
730 interval_time : (optional) the interval time to check the build status, default to 30s
731 logger : (optional) a logger to report progress
732
733 Returns
734 -------
735 dict
736 the latest metadata for the job
737 """
738 t = 0
739 build_info = self.get_build_info(build_number)
740 while True:
741 for job in build_info["jobs"]:
742 if job["id"] == job_id:
743 state = job["state"]
744 if state != "scheduled" and state != "running" and state != "assigned":
745 return job
746 break
747 else:
Florian Weikertdb832a02020-11-19 19:14:48 +0100748 raise BuildkiteException(
749 f"job id {job_id} doesn't exist in build " + build_info["web_url"]
750 )
Yun Peng9337bb32020-02-28 13:31:29 +0100751 url = build_info["web_url"]
752 if logger:
753 logger.log(f"Waiting for {url}, waited {t} seconds...")
754 time.sleep(interval_time)
755 t += interval_time
756 build_info = self.get_build_info(build_number)
757
Yun Peng9337bb32020-02-28 13:31:29 +0100758 def wait_build_to_finish(self, build_number, interval_time=30, logger=None):
759 """Wait a build to finish and return the build metadata
760
761 Parameters
762 ----------
763 build_number : the number of the build we want to wait
764 interval_time : (optional) the interval time to check the build status, default to 30s
765 logger : (optional) a logger to report progress
766
767 Returns
768 -------
769 dict
770 the latest metadata for the build
771 """
772 t = 0
773 build_info = self.get_build_info(build_number)
774 while build_info["state"] == "scheduled" or build_info["state"] == "running":
775 url = build_info["web_url"]
776 if logger:
777 logger.log(f"Waiting for {url}, waited {t} seconds...")
778 time.sleep(interval_time)
779 t += interval_time
780 build_info = self.get_build_info(build_number)
781 return build_info
782
783
Florian Weikert849afb22019-12-14 12:22:29 -0800784def decrypt_token(encrypted_token, kms_key):
785 return (
786 subprocess.check_output(
787 [
788 gcloud_command(),
789 "kms",
790 "decrypt",
791 "--project",
792 "bazel-untrusted",
793 "--location",
794 "global",
795 "--keyring",
796 "buildkite",
797 "--key",
798 kms_key,
799 "--ciphertext-file",
800 "-",
801 "--plaintext-file",
802 "-",
803 ],
804 input=base64.b64decode(encrypted_token),
805 env=os.environ,
806 )
807 .decode("utf-8")
808 .strip()
809 )
810
811
Philipp Wollermanndcaddd92018-02-21 14:13:43 +0100812def eprint(*args, **kwargs):
813 """
814 Print to stderr and flush (just in case).
815 """
816 print(*args, flush=True, file=sys.stderr, **kwargs)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +0100817
818
Jakob Buchgraber9f153542018-02-27 10:56:04 +0100819def is_windows():
Jakob Buchgraber09048fa2018-02-27 11:39:39 +0100820 return os.name == "nt"
Jakob Buchgraber9f153542018-02-27 10:56:04 +0100821
Jakob Buchgrabere6de16b2018-02-28 12:42:12 +0100822
Jakob Buchgraber9f153542018-02-27 10:56:04 +0100823def gsutil_command():
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200824 return "gsutil.cmd" if is_windows() else "gsutil"
Jakob Buchgraber9f153542018-02-27 10:56:04 +0100825
Jakob Buchgrabere6de16b2018-02-28 12:42:12 +0100826
Jakob Buchgraber9f153542018-02-27 10:56:04 +0100827def gcloud_command():
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200828 return "gcloud.cmd" if is_windows() else "gcloud"
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +0100829
Jakob Buchgrabere6de16b2018-02-28 12:42:12 +0100830
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +0100831def downstream_projects_root(platform):
Philipp Wollermann51147bf2019-05-08 15:50:10 +0200832 downstream_root = os.path.expandvars(PLATFORMS[platform]["downstream-root"])
Philipp Wollermannd5ab3d92020-02-05 16:55:13 +0100833 if platform == "windows" and os.path.exists("d:/b"):
834 # If this is a Windows machine with a local SSD, the build directory is
835 # on drive D.
836 downstream_root = downstream_root.replace("c:/b/", "d:/b/")
Philipp Wollermann51147bf2019-05-08 15:50:10 +0200837 if not os.path.exists(downstream_root):
838 os.makedirs(downstream_root)
839 return downstream_root
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +0100840
841
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +0100842def fetch_configs(http_url, file_config):
Philipp Wollermanndb024862018-02-19 17:16:56 +0100843 """
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +0100844 If specified fetches the build configuration from file_config or http_url, else tries to
Jakob Buchgraber25bb50f2018-02-22 18:06:21 +0100845 read it from .bazelci/presubmit.yml.
Philipp Wollermann598b4a42018-02-19 17:03:36 +0100846 Returns the json configuration as a python data structure.
Philipp Wollermanndb024862018-02-19 17:16:56 +0100847 """
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +0100848 if file_config is not None and http_url is not None:
849 raise BuildkiteException("file_config and http_url cannot be set at the same time")
850
Florian Weikertc8b3ed22019-05-31 16:14:12 +0200851 return load_config(http_url, file_config)
852
853
854def load_config(http_url, file_config, allow_imports=True):
Florian Weikertc8b3ed22019-05-31 16:14:12 +0200855 if http_url:
856 config = load_remote_yaml_file(http_url)
857 else:
858 file_config = file_config or ".bazelci/presubmit.yml"
859 with open(file_config, "r") as fd:
860 config = yaml.safe_load(fd)
861
Florian Weikert843d7a02019-02-03 17:24:50 +0100862 # Legacy mode means that there is exactly one task per platform (e.g. ubuntu1604_nojdk),
863 # which means that we can get away with using the platform name as task ID.
864 # No other updates are needed since get_platform_for_task() falls back to using the
865 # task ID as platform if there is no explicit "platforms" field.
866 if "platforms" in config:
867 config["tasks"] = config.pop("platforms")
868
Florian Weikertc8b3ed22019-05-31 16:14:12 +0200869 if "tasks" not in config:
870 config["tasks"] = {}
871
872 imports = config.pop("imports", None)
873 if imports:
874 if not allow_imports:
875 raise BuildkiteException("Nested imports are not allowed")
876
877 for i in imports:
878 imported_tasks = load_imported_tasks(i, http_url, file_config)
879 config["tasks"].update(imported_tasks)
880
Florian Weikert843d7a02019-02-03 17:24:50 +0100881 return config
882
883
Florian Weikert13215a82019-05-10 12:42:21 +0200884def load_remote_yaml_file(http_url):
885 with urllib.request.urlopen(http_url) as resp:
886 reader = codecs.getreader("utf-8")
Philipp Wollermannd00107e2019-05-18 23:50:59 +0200887 return yaml.safe_load(reader(resp))
Florian Weikert13215a82019-05-10 12:42:21 +0200888
889
Florian Weikertc8b3ed22019-05-31 16:14:12 +0200890def load_imported_tasks(import_name, http_url, file_config):
891 if "/" in import_name:
892 raise BuildkiteException("Invalid import '%s'" % import_name)
893
894 old_path = http_url or file_config
895 new_path = "%s%s" % (old_path[: old_path.rfind("/") + 1], import_name)
896 if http_url:
897 http_url = new_path
898 else:
899 file_config = new_path
900
901 imported_config = load_config(http_url=http_url, file_config=file_config, allow_imports=False)
902
903 namespace = import_name.partition(".")[0]
904 tasks = {}
905 for task_name, task_config in imported_config["tasks"].items():
Florian Weikert61f29b82019-08-12 16:56:56 +0200906 fix_imported_task_platform(task_name, task_config)
907 fix_imported_task_name(namespace, task_config)
908 fix_imported_task_working_directory(namespace, task_config)
Florian Weikertc8b3ed22019-05-31 16:14:12 +0200909 tasks["%s_%s" % (namespace, task_name)] = task_config
910
911 return tasks
912
913
Florian Weikert61f29b82019-08-12 16:56:56 +0200914def fix_imported_task_platform(task_name, task_config):
915 if "platform" not in task_config:
916 task_config["platform"] = task_name
917
918
919def fix_imported_task_name(namespace, task_config):
920 old_name = task_config.get("name")
921 task_config["name"] = "%s (%s)" % (namespace, old_name) if old_name else namespace
922
923
924def fix_imported_task_working_directory(namespace, task_config):
925 old_dir = task_config.get("working_directory")
926 task_config["working_directory"] = os.path.join(namespace, old_dir) if old_dir else namespace
927
928
Jakob Buchgraber3120f7a2018-02-18 13:28:02 +0100929def print_collapsed_group(name):
Jakob Buchgraber5c9b13d2018-02-21 22:28:14 +0100930 eprint("\n\n--- {0}\n\n".format(name))
Jakob Buchgraber3120f7a2018-02-18 13:28:02 +0100931
Jakob Buchgraber9c83de72018-02-18 15:32:44 +0100932
Jakob Buchgraber3120f7a2018-02-18 13:28:02 +0100933def print_expanded_group(name):
Jakob Buchgraber5c9b13d2018-02-21 22:28:14 +0100934 eprint("\n\n+++ {0}\n\n".format(name))
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +0100935
Jakob Buchgraber9c83de72018-02-18 15:32:44 +0100936
Yun Peng8975c6b2019-02-28 11:55:55 +0100937def use_bazelisk_migrate():
938 """
939 If USE_BAZELISK_MIGRATE is set, we use `bazelisk --migrate` to test incompatible flags.
940 """
Florian Weikertde96a6f2019-03-07 14:57:50 +0100941 return bool(os.environ.get("USE_BAZELISK_MIGRATE"))
Yun Peng8975c6b2019-02-28 11:55:55 +0100942
943
944def bazelisk_flags():
945 return ["--migrate"] if use_bazelisk_migrate() else []
946
947
Chi Wang54595a22021-06-11 17:49:58 +0800948def calculate_flags(task_config, task_config_key, action_key, tmpdir, test_env_vars):
Chi Wang6357efe2020-08-25 16:23:38 +0800949 include_json_profile = task_config.get("include_json_profile", [])
Chi Wang54595a22021-06-11 17:49:58 +0800950 capture_corrupted_outputs = task_config.get("capture_corrupted_outputs", [])
Chi Wang6357efe2020-08-25 16:23:38 +0800951
952 json_profile_flags = []
953 json_profile_out = None
Chi Wang54595a22021-06-11 17:49:58 +0800954 if action_key in include_json_profile:
955 json_profile_out = os.path.join(tmpdir, "{}.profile.gz".format(action_key))
Tobias Werthc22e4c42021-10-08 12:03:14 +0200956 json_profile_flags = ["--profile={}".format(json_profile_out)]
957
Yun Peng92c0ef82021-07-26 10:41:21 +0200958
Chi Wang54595a22021-06-11 17:49:58 +0800959 capture_corrupted_outputs_flags = []
960 capture_corrupted_outputs_dir = None
961 if action_key in capture_corrupted_outputs:
Philipp Wollermannf436e742021-08-11 11:06:55 +0200962 capture_corrupted_outputs_dir = os.path.join(
963 tmpdir, "{}_corrupted_outputs".format(action_key)
964 )
965 capture_corrupted_outputs_flags = [
966 "--experimental_remote_capture_corrupted_outputs={}".format(
967 capture_corrupted_outputs_dir
968 )
969 ]
Chi Wang6357efe2020-08-25 16:23:38 +0800970
971 flags = task_config.get(task_config_key) or []
972 flags += json_profile_flags
Chi Wang54595a22021-06-11 17:49:58 +0800973 flags += capture_corrupted_outputs_flags
Chi Wang6357efe2020-08-25 16:23:38 +0800974 # We have to add --test_env flags to `build`, too, otherwise Bazel
975 # discards its analysis cache between `build` and `test`.
976 if test_env_vars:
977 flags += ["--test_env={}".format(v) for v in test_env_vars]
978
Chi Wang54595a22021-06-11 17:49:58 +0800979 return flags, json_profile_out, capture_corrupted_outputs_dir
Chi Wang6357efe2020-08-25 16:23:38 +0800980
981
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100982def execute_commands(
Florian Weikertc8642af2019-02-03 23:58:51 +0100983 task_config,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100984 platform,
985 git_repository,
986 git_commit,
Yun Peng5012a862021-09-16 16:35:43 +0200987 repo_location,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100988 use_bazel_at_commit,
989 use_but,
990 save_but,
Yun Peng4d1d6542019-01-17 18:30:33 +0100991 needs_clean,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100992 build_only,
993 test_only,
994 monitor_flaky_tests,
995 incompatible_flags,
Florian Weikertc8642af2019-02-03 23:58:51 +0100996 bazel_version=None,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100997):
Yun Pengf50f7b72019-02-28 19:09:52 +0100998 # If we want to test incompatible flags, we ignore bazel_version and always use
999 # the latest Bazel version through Bazelisk.
1000 if incompatible_flags:
1001 bazel_version = None
Florian Weikert13215a82019-05-10 12:42:21 +02001002 if not bazel_version:
1003 # The last good version of Bazel can be specified in an emergency file.
1004 # However, we only use last_good_bazel for pipelines that do not
1005 # explicitly specify a version of Bazel.
1006 try:
1007 emergency_settings = load_remote_yaml_file(EMERGENCY_FILE_URL)
1008 bazel_version = emergency_settings.get("last_good_bazel")
1009 except urllib.error.HTTPError:
1010 # Ignore this error. The Setup step will have already complained about
1011 # it by showing an error message.
1012 pass
Yun Pengf50f7b72019-02-28 19:09:52 +01001013
Jakob Buchgraberfb95a2f2018-02-22 11:46:25 +01001014 if build_only and test_only:
1015 raise BuildkiteException("build_only and test_only cannot be true at the same time")
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02001016
Yun Peng20d45602018-10-18 13:27:05 +02001017 if use_bazel_at_commit and use_but:
1018 raise BuildkiteException("use_bazel_at_commit cannot be set when use_but is true")
1019
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02001020 tmpdir = tempfile.mkdtemp()
1021 sc_process = None
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001022 try:
Yun Pengda319b52021-05-20 17:00:36 +02001023 if platform == "macos" or platform == "macos_arm64":
Florian Weikertee84c5c2019-05-28 11:21:51 +02001024 activate_xcode(task_config)
Philipp Wollermann380f1e62019-04-12 16:45:27 +02001025
Florian Weikert4ee0bed2019-02-21 18:03:00 +01001026 # If the CI worker runs Bazelisk, we need to forward all required env variables to the test.
1027 # Otherwise any integration test that invokes Bazel (=Bazelisk in this case) will fail.
Marcel Hlopko198328b2019-02-25 09:19:55 +01001028 test_env_vars = ["LocalAppData"] if platform == "windows" else ["HOME"]
Florian Weikertc12580c2021-07-13 17:09:25 +02001029
1030 # CI should have its own user agent so that we can remove it from Bazel download statistics.
1031 os.environ["BAZELISK_USER_AGENT"] = "Bazelisk/BazelCI"
1032 test_env_vars.append("BAZELISK_USER_AGENT")
1033
Yun Peng5012a862021-09-16 16:35:43 +02001034 if repo_location:
1035 os.chdir(repo_location)
Yun Peng376d2b32018-11-29 10:24:54 +01001036 elif git_repository:
1037 clone_git_repository(git_repository, platform, git_commit)
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001038
Philipp Wollermannf4aabb72019-06-25 15:59:00 +02001039 # We use one binary for all Linux platforms (because we also just release one binary for all
1040 # Linux versions and we have to ensure that it works on all of them).
1041 binary_platform = platform if platform in ["macos", "windows"] else LINUX_BINARY_PLATFORM
1042
Yun Peng20d45602018-10-18 13:27:05 +02001043 if use_bazel_at_commit:
1044 print_collapsed_group(":gcloud: Downloading Bazel built at " + use_bazel_at_commit)
Philipp Wollermannf4aabb72019-06-25 15:59:00 +02001045 bazel_binary = download_bazel_binary_at_commit(
1046 tmpdir, binary_platform, use_bazel_at_commit
1047 )
Yun Pengf0a66e22019-10-14 12:45:42 +02001048 os.environ["USE_BAZEL_VERSION"] = bazel_binary
Philipp Wollermann639c0452019-01-03 11:23:54 +01001049 elif use_but:
Jakob Buchgraber92755d72018-02-22 15:33:37 +01001050 print_collapsed_group(":gcloud: Downloading Bazel Under Test")
Philipp Wollermannf4aabb72019-06-25 15:59:00 +02001051 bazel_binary = download_bazel_binary(tmpdir, binary_platform)
Yun Pengf0a66e22019-10-14 12:45:42 +02001052 os.environ["USE_BAZEL_VERSION"] = bazel_binary
Philipp Wollermann639c0452019-01-03 11:23:54 +01001053 else:
1054 bazel_binary = "bazel"
Florian Weikertc8642af2019-02-03 23:58:51 +01001055 if bazel_version:
Florian Weikertc8642af2019-02-03 23:58:51 +01001056 os.environ["USE_BAZEL_VERSION"] = bazel_version
Philipp Wollermann87b45252020-01-22 12:43:42 +01001057 if "USE_BAZEL_VERSION" in os.environ and not task_config.get(
1058 "skip_use_bazel_version_for_test", False
1059 ):
Yun Pengf0a66e22019-10-14 12:45:42 +02001060 # This will only work if the bazel binary in $PATH is actually a bazelisk binary
1061 # (https://github.com/bazelbuild/bazelisk).
1062 test_env_vars.append("USE_BAZEL_VERSION")
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001063
Philipp Wollermann5c7ea412019-05-24 15:26:57 +02001064 for key, value in task_config.get("environment", {}).items():
Philipp Wollermann4ad4aac2019-05-24 15:23:09 +02001065 # We have to explicitly convert the value to a string, because sometimes YAML tries to
1066 # be smart and converts strings like "true" and "false" to booleans.
Philipp Wollermann7df2d0d2021-08-11 13:26:04 +02001067 os.environ[key] = os.path.expandvars(str(value))
Philipp Wollermann213ac9d2019-02-06 11:50:05 +01001068
Yun Peng366f04c2020-08-10 16:55:58 +02001069 # Set BAZELISK_SHUTDOWN to 1 when we use bazelisk --migrate on Windows.
1070 # This is a workaround for https://github.com/bazelbuild/continuous-integration/issues/1012
1071 if use_bazelisk_migrate() and platform == "windows":
1072 os.environ["BAZELISK_SHUTDOWN"] = "1"
1073
Florian Weikerta8c020b2019-08-12 16:56:38 +02001074 cmd_exec_func = execute_batch_commands if platform == "windows" else execute_shell_commands
1075 cmd_exec_func(task_config.get("setup", None))
1076
Philipp Wollermanna5aee2c2019-02-11 16:55:19 +01001077 # Allow the config to override the current working directory.
1078 required_prefix = os.getcwd()
1079 requested_working_dir = os.path.abspath(task_config.get("working_directory", ""))
1080 if os.path.commonpath([required_prefix, requested_working_dir]) != required_prefix:
1081 raise BuildkiteException("working_directory refers to a path outside the workspace")
1082 os.chdir(requested_working_dir)
1083
Florian Weikerte72d68a2019-03-08 18:56:33 +01001084 if platform == "windows":
1085 execute_batch_commands(task_config.get("batch_commands", None))
1086 else:
1087 execute_shell_commands(task_config.get("shell_commands", None))
1088
Florian Weikertc8642af2019-02-03 23:58:51 +01001089 bazel_version = print_bazel_version_info(bazel_binary, platform)
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001090
Yun Penga5a1ee02018-12-05 15:00:58 +01001091 print_environment_variables_info()
1092
Yun Pengd0217ed2018-11-30 14:51:11 +01001093 if incompatible_flags:
1094 print_expanded_group("Build and test with the following incompatible flags:")
1095 for flag in incompatible_flags:
1096 eprint(flag + "\n")
1097
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001098 execute_bazel_run(
Florian Weikertc8642af2019-02-03 23:58:51 +01001099 bazel_binary, platform, task_config.get("run_targets", None), incompatible_flags
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001100 )
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001101
Yun Peng4d1d6542019-01-17 18:30:33 +01001102 if needs_clean:
Yun Pengea0359e2019-01-17 15:37:47 +01001103 execute_bazel_clean(bazel_binary, platform)
1104
Chi Wang6357efe2020-08-25 16:23:38 +08001105 build_targets, test_targets, index_targets = calculate_targets(
Florian Weikert736d06e2019-05-08 13:16:42 +02001106 task_config, platform, bazel_binary, build_only, test_only
1107 )
Florian Weikert736d06e2019-05-08 13:16:42 +02001108
1109 if build_targets:
Philipp Wollermannf436e742021-08-11 11:06:55 +02001110 (
1111 build_flags,
1112 json_profile_out_build,
1113 capture_corrupted_outputs_dir_build,
1114 ) = calculate_flags(task_config, "build_flags", "build", tmpdir, test_env_vars)
joeleba76887952019-05-16 15:22:17 +02001115 try:
Yun Pengc8bb9e92021-07-28 11:56:53 +02001116 release_name = get_release_name_from_branch_name()
joeleba76887952019-05-16 15:22:17 +02001117 execute_bazel_build(
1118 bazel_version,
1119 bazel_binary,
1120 platform,
Philipp Wollermannf436e742021-08-11 11:06:55 +02001121 build_flags
1122 + (
1123 ["--stamp", "--embed_label=%s" % release_name]
1124 if save_but and release_name
1125 else []
1126 ),
joeleba76887952019-05-16 15:22:17 +02001127 build_targets,
1128 None,
1129 incompatible_flags,
1130 )
1131 if save_but:
1132 upload_bazel_binary(platform)
1133 finally:
Chi Wang6357efe2020-08-25 16:23:38 +08001134 if json_profile_out_build:
Philipp Wollermann92cf51e2019-05-16 15:31:11 +02001135 upload_json_profile(json_profile_out_build, tmpdir)
Chi Wang54595a22021-06-11 17:49:58 +08001136 if capture_corrupted_outputs_dir_build:
1137 upload_corrupted_outputs(capture_corrupted_outputs_dir_build, tmpdir)
Philipp Wollermann639c0452019-01-03 11:23:54 +01001138
Florian Weikert736d06e2019-05-08 13:16:42 +02001139 if test_targets:
Chi Wang54595a22021-06-11 17:49:58 +08001140 test_flags, json_profile_out_test, capture_corrupted_outputs_dir_test = calculate_flags(
Florian Weikertdb832a02020-11-19 19:14:48 +01001141 task_config, "test_flags", "test", tmpdir, test_env_vars
1142 )
Florian Weikert4901c662019-02-26 13:20:11 +01001143 if not is_windows():
1144 # On platforms that support sandboxing (Linux, MacOS) we have
1145 # to allow access to Bazelisk's cache directory.
1146 # However, the flag requires the directory to exist,
1147 # so we create it here in order to not crash when a test
1148 # does not invoke Bazelisk.
1149 bazelisk_cache_dir = get_bazelisk_cache_directory(platform)
1150 os.makedirs(bazelisk_cache_dir, mode=0o755, exist_ok=True)
1151 test_flags.append("--sandbox_writable_path={}".format(bazelisk_cache_dir))
Florian Weikert5b890332019-02-25 14:57:43 +01001152
Philipp Wollermannce986af2019-07-18 14:46:05 +02001153 test_bep_file = os.path.join(tmpdir, "test_bep.json")
Philipp Wollermannce986af2019-07-18 14:46:05 +02001154 upload_thread = threading.Thread(
Chi Wangd279d582021-09-29 10:59:06 +08001155 target=upload_test_logs_from_bep, args=(test_bep_file, tmpdir, binary_platform, monitor_flaky_tests)
Philipp Wollermannce986af2019-07-18 14:46:05 +02001156 )
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01001157 try:
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001158 upload_thread.start()
1159 try:
1160 execute_bazel_test(
1161 bazel_version,
1162 bazel_binary,
1163 platform,
1164 test_flags,
1165 test_targets,
1166 test_bep_file,
1167 monitor_flaky_tests,
1168 incompatible_flags,
1169 )
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001170 finally:
Chi Wang6357efe2020-08-25 16:23:38 +08001171 if json_profile_out_test:
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001172 upload_json_profile(json_profile_out_test, tmpdir)
Chi Wang54595a22021-06-11 17:49:58 +08001173 if capture_corrupted_outputs_dir_test:
1174 upload_corrupted_outputs(capture_corrupted_outputs_dir_test, tmpdir)
Philipp Wollermann639c0452019-01-03 11:23:54 +01001175 finally:
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001176 upload_thread.join()
Chi Wang6357efe2020-08-25 16:23:38 +08001177
1178 if index_targets:
Philipp Wollermannf436e742021-08-11 11:06:55 +02001179 (
1180 index_flags,
1181 json_profile_out_index,
1182 capture_corrupted_outputs_dir_index,
1183 ) = calculate_flags(task_config, "index_flags", "index", tmpdir, test_env_vars)
Chi Wang6357efe2020-08-25 16:23:38 +08001184 index_upload_policy = task_config.get("index_upload_policy", "IfBuildSuccess")
Chi Wangb2b65682020-08-27 10:36:15 +08001185 index_upload_gcs = task_config.get("index_upload_gcs", False)
Chi Wang6357efe2020-08-25 16:23:38 +08001186
1187 try:
Florian Weikertdb832a02020-11-19 19:14:48 +01001188 should_upload_kzip = (
1189 True if index_upload_policy == INDEX_UPLOAD_POLICY_ALWAYS else False
1190 )
Chi Wang6357efe2020-08-25 16:23:38 +08001191 try:
1192 execute_bazel_build_with_kythe(
1193 bazel_version,
1194 bazel_binary,
1195 platform,
1196 index_flags,
1197 index_targets,
1198 None,
Florian Weikertdb832a02020-11-19 19:14:48 +01001199 incompatible_flags,
Chi Wang6357efe2020-08-25 16:23:38 +08001200 )
1201
1202 if index_upload_policy == INDEX_UPLOAD_POLICY_IF_BUILD_SUCCESS:
1203 should_upload_kzip = True
1204 except subprocess.CalledProcessError as e:
1205 # If not running with Always policy, raise the build error.
1206 if index_upload_policy != INDEX_UPLOAD_POLICY_ALWAYS:
1207 handle_bazel_failure(e, "build")
1208
Philipp Wollermanna038b002021-05-05 22:04:38 +02001209 if should_upload_kzip and not is_pull_request():
Chi Wang6357efe2020-08-25 16:23:38 +08001210 try:
Chi Wangb2b65682020-08-27 10:36:15 +08001211 merge_and_upload_kythe_kzip(platform, index_upload_gcs)
Chi Wang6357efe2020-08-25 16:23:38 +08001212 except subprocess.CalledProcessError:
1213 raise BuildkiteException("Failed to upload kythe kzip")
1214 finally:
1215 if json_profile_out_index:
1216 upload_json_profile(json_profile_out_index, tmpdir)
Chi Wang54595a22021-06-11 17:49:58 +08001217 if capture_corrupted_outputs_dir_index:
1218 upload_corrupted_outputs(capture_corrupted_outputs_dir_index, tmpdir)
Chi Wang6357efe2020-08-25 16:23:38 +08001219
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001220 finally:
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001221 terminate_background_process(sc_process)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001222 if tmpdir:
1223 shutil.rmtree(tmpdir)
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01001224
Philipp Wollermann3c8b8512019-07-16 15:28:03 +02001225
Florian Weikertee84c5c2019-05-28 11:21:51 +02001226def activate_xcode(task_config):
1227 # Get the Xcode version from the config.
Philipp Wollermann2b4ee9f2021-02-11 16:32:35 +01001228 wanted_xcode_version = task_config.get("xcode_version", DEFAULT_XCODE_VERSION)
1229 print_collapsed_group(":xcode: Activating Xcode {}...".format(wanted_xcode_version))
Florian Weikertee84c5c2019-05-28 11:21:51 +02001230
1231 # Ensure it's a valid version number.
Philipp Wollermann2b4ee9f2021-02-11 16:32:35 +01001232 if not isinstance(wanted_xcode_version, str):
Florian Weikertee84c5c2019-05-28 11:21:51 +02001233 raise BuildkiteException(
1234 "Version number '{}' is not a string. Did you forget to put it in quotes?".format(
Philipp Wollermann2b4ee9f2021-02-11 16:32:35 +01001235 wanted_xcode_version
Florian Weikertee84c5c2019-05-28 11:21:51 +02001236 )
1237 )
Philipp Wollermann2b4ee9f2021-02-11 16:32:35 +01001238 if not XCODE_VERSION_REGEX.match(wanted_xcode_version):
Florian Weikertee84c5c2019-05-28 11:21:51 +02001239 raise BuildkiteException(
1240 "Invalid Xcode version format '{}', must match the format X.Y[.Z].".format(
Philipp Wollermann2b4ee9f2021-02-11 16:32:35 +01001241 wanted_xcode_version
Florian Weikertee84c5c2019-05-28 11:21:51 +02001242 )
1243 )
1244
Philipp Wollermann06e56972020-01-29 14:46:41 +01001245 # This is used to replace e.g. 11.2 with 11.2.1 without having to update all configs.
Philipp Wollermann2b4ee9f2021-02-11 16:32:35 +01001246 xcode_version = XCODE_VERSION_OVERRIDES.get(wanted_xcode_version, wanted_xcode_version)
1247
1248 # This falls back to a default version if the selected version is not available.
1249 supported_versions = sorted(
1250 # Stripping "Xcode" prefix and ".app" suffix from e.g. "Xcode12.0.1.app" leaves just the version number.
Florian Weikert9d5e4c02021-05-27 15:10:20 +02001251 [os.path.basename(x)[5:-4] for x in glob("/Applications/Xcode*.app")],
1252 reverse=True,
Philipp Wollermann2b4ee9f2021-02-11 16:32:35 +01001253 )
1254 if xcode_version not in supported_versions:
1255 xcode_version = DEFAULT_XCODE_VERSION
1256 if xcode_version != wanted_xcode_version:
1257 print_collapsed_group(
1258 ":xcode: Fixed Xcode version: {} -> {}...".format(wanted_xcode_version, xcode_version)
1259 )
1260 lines = [
Florian Weikert9d5e4c02021-05-27 15:10:20 +02001261 "Your selected Xcode version {} was not available on the machine.".format(
1262 wanted_xcode_version
1263 ),
Philipp Wollermann2b4ee9f2021-02-11 16:32:35 +01001264 "Bazel CI automatically picked a fallback version: {}.".format(xcode_version),
1265 "Available versions are: {}.".format(supported_versions),
1266 ]
Florian Weikert9d5e4c02021-05-27 15:10:20 +02001267 execute_command(
1268 [
1269 "buildkite-agent",
1270 "annotate",
1271 "--style=warning",
1272 "\n".join(lines),
1273 "--context",
1274 "ctx-xcode_version_fixed",
1275 ]
1276 )
Philipp Wollermann06e56972020-01-29 14:46:41 +01001277
Florian Weikertee84c5c2019-05-28 11:21:51 +02001278 # Check that the selected Xcode version is actually installed on the host.
1279 xcode_path = "/Applications/Xcode{}.app".format(xcode_version)
1280 if not os.path.exists(xcode_path):
1281 raise BuildkiteException("Xcode not found at '{}'.".format(xcode_path))
1282
1283 # Now activate the specified Xcode version and let it install its required components.
1284 # The CI machines have a sudoers config that allows the 'buildkite' user to run exactly
1285 # these two commands, so don't change them without also modifying the file there.
1286 execute_command(["/usr/bin/sudo", "/usr/bin/xcode-select", "--switch", xcode_path])
1287 execute_command(["/usr/bin/sudo", "/usr/bin/xcodebuild", "-runFirstLaunch"])
1288
1289
Florian Weikert4901c662019-02-26 13:20:11 +01001290def get_bazelisk_cache_directory(platform):
1291 # The path relies on the behavior of Go's os.UserCacheDir()
1292 # and of the Go version of Bazelisk.
Philipp Wollermannce986af2019-07-18 14:46:05 +02001293 cache_dir = "Library/Caches" if platform == "macos" else ".cache"
1294 return os.path.join(os.environ.get("HOME"), cache_dir, "bazelisk")
Florian Weikert4901c662019-02-26 13:20:11 +01001295
Florian Weikert5b890332019-02-25 14:57:43 +01001296
Philipp Wollermann1b5ecdc2021-06-10 21:52:55 +02001297def current_branch_is_main_branch():
1298 return os.getenv("BUILDKITE_BRANCH") in ("master", "stable", "main")
1299
1300
Yun Peng92c0ef82021-07-26 10:41:21 +02001301def get_release_name_from_branch_name():
Yun Peng26e01ff2021-07-26 12:05:53 +02001302 res = re.match(r"release-(\d+\.\d+\.\d+(rc\d+)?).*", os.getenv("BUILDKITE_BRANCH"))
Yun Peng92c0ef82021-07-26 10:41:21 +02001303 return res.group(1) if res else ""
1304
1305
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01001306def is_pull_request():
Jakob Buchgraber67761d32018-02-21 19:00:21 +01001307 third_party_repo = os.getenv("BUILDKITE_PULL_REQUEST_REPO", "")
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01001308 return len(third_party_repo) > 0
1309
1310
Yun Penge3cf12d2018-12-05 15:01:09 +01001311def print_bazel_version_info(bazel_binary, platform):
Jakob Buchgraber99c4bbb2018-02-22 11:59:31 +01001312 print_collapsed_group(":information_source: Bazel Info")
Philipp Wollermannf13804b2019-02-05 21:08:30 +01001313 version_output = execute_command_and_get_output(
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001314 [bazel_binary]
1315 + common_startup_flags(platform)
1316 + ["--nomaster_bazelrc", "--bazelrc=/dev/null", "version"]
1317 )
1318 execute_command(
1319 [bazel_binary]
1320 + common_startup_flags(platform)
1321 + ["--nomaster_bazelrc", "--bazelrc=/dev/null", "info"]
1322 )
Jakob Buchgraber7e690a72018-02-18 13:22:15 +01001323
Florian Weikertc8642af2019-02-03 23:58:51 +01001324 match = BUILD_LABEL_PATTERN.search(version_output)
1325 return match.group(1) if match else "unreleased binary"
1326
Jakob Buchgraber7e690a72018-02-18 13:22:15 +01001327
Yun Penga5a1ee02018-12-05 15:00:58 +01001328def print_environment_variables_info():
1329 print_collapsed_group(":information_source: Environment Variables")
1330 for key, value in os.environ.items():
1331 eprint("%s=(%s)" % (key, value))
1332
1333
Jakob Buchgraber426399e2018-03-20 19:45:46 +01001334def upload_bazel_binary(platform):
Jakob Buchgraber7d1d3bb2018-02-21 22:38:22 +01001335 print_collapsed_group(":gcloud: Uploading Bazel Under Test")
Jakob Buchgraber426399e2018-03-20 19:45:46 +01001336 if platform == "windows":
Philipp Wollermann10183212020-02-04 21:54:14 +01001337 binary_dir = r"bazel-bin\src"
1338 binary_name = r"bazel.exe"
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04001339 binary_nojdk_name = r"bazel_nojdk.exe"
Philipp Wollermann10183212020-02-04 21:54:14 +01001340 else:
1341 binary_dir = "bazel-bin/src"
1342 binary_name = "bazel"
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04001343 binary_nojdk_name = "bazel_nojdk"
Philipp Wollermann10183212020-02-04 21:54:14 +01001344 execute_command(["buildkite-agent", "artifact", "upload", binary_name], cwd=binary_dir)
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04001345 execute_command(["buildkite-agent", "artifact", "upload", binary_nojdk_name], cwd=binary_dir)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001346
1347
Chi Wangb2b65682020-08-27 10:36:15 +08001348def merge_and_upload_kythe_kzip(platform, index_upload_gcs):
Chi Wang6357efe2020-08-25 16:23:38 +08001349 print_collapsed_group(":gcloud: Uploading kythe kzip")
1350
Philipp Wollermann2b4ee9f2021-02-11 16:32:35 +01001351 kzips = glob("bazel-out/*/extra_actions/**/*.kzip", recursive=True)
Chi Wang6357efe2020-08-25 16:23:38 +08001352
Chi Wang6357efe2020-08-25 16:23:38 +08001353 build_number = os.getenv("BUILDKITE_BUILD_NUMBER")
Chi Wangb2b65682020-08-27 10:36:15 +08001354 git_commit = os.getenv("BUILDKITE_COMMIT")
1355 final_kzip_name = "{}-{}-{}.kzip".format(build_number, platform, git_commit)
Chi Wang6357efe2020-08-25 16:23:38 +08001356
Chi Wangb2b65682020-08-27 10:36:15 +08001357 execute_command([f"{KYTHE_DIR}/tools/kzip", "merge", "--output", final_kzip_name] + kzips)
Chi Wang6357efe2020-08-25 16:23:38 +08001358 execute_command(["buildkite-agent", "artifact", "upload", final_kzip_name])
1359
Chi Wangb2b65682020-08-27 10:36:15 +08001360 if index_upload_gcs:
1361 pipeline = os.getenv("BUILDKITE_PIPELINE_SLUG")
1362 destination = KZIPS_BUCKET + pipeline + "/" + final_kzip_name
1363 print("Uploading to GCS {}".format(destination))
Florian Weikertdb832a02020-11-19 19:14:48 +01001364 execute_command([gsutil_command(), "cp", final_kzip_name, destination])
Chi Wangb2b65682020-08-27 10:36:15 +08001365
Chi Wang6357efe2020-08-25 16:23:38 +08001366
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04001367def download_binary(dest_dir, platform, binary_name):
Philipp Wollermannc52e26a2019-05-18 22:10:47 +02001368 source_step = create_label(platform, "Bazel", build_only=True)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001369 execute_command(
Philipp Wollermann10183212020-02-04 21:54:14 +01001370 ["buildkite-agent", "artifact", "download", binary_name, dest_dir, "--step", source_step]
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001371 )
Philipp Wollermann10183212020-02-04 21:54:14 +01001372 bazel_binary_path = os.path.join(dest_dir, binary_name)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001373 st = os.stat(bazel_binary_path)
1374 os.chmod(bazel_binary_path, st.st_mode | stat.S_IEXEC)
1375 return bazel_binary_path
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001376
1377
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04001378def download_bazel_binary(dest_dir, platform):
1379 binary_name = "bazel.exe" if platform == "windows" else "bazel"
1380 return download_binary(dest_dir, platform, binary_name)
1381
1382
1383def download_bazel_nojdk_binary(dest_dir, platform):
1384 binary_name = "bazel_nojdk.exe" if platform == "windows" else "bazel_nojdk"
1385 return download_binary(dest_dir, platform, binary_name)
1386
1387
Florian Weikertdb832a02020-11-19 19:14:48 +01001388def download_binary_at_commit(
1389 dest_dir, platform, bazel_git_commit, bazel_binary_url, bazel_binary_path
1390):
Yun Peng02312732019-01-17 18:17:05 +01001391 try:
Florian Weikertdb832a02020-11-19 19:14:48 +01001392 execute_command([gsutil_command(), "cp", bazel_binary_url, bazel_binary_path])
Yun Peng02312732019-01-17 18:17:05 +01001393 except subprocess.CalledProcessError as e:
Philipp Wollermannc05ac682019-01-19 12:37:28 +01001394 raise BuildkiteException(
1395 "Failed to download Bazel binary at %s, error message:\n%s" % (bazel_git_commit, str(e))
1396 )
Yun Peng20d45602018-10-18 13:27:05 +02001397 st = os.stat(bazel_binary_path)
1398 os.chmod(bazel_binary_path, st.st_mode | stat.S_IEXEC)
1399 return bazel_binary_path
1400
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001401
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04001402def download_bazel_binary_at_commit(dest_dir, platform, bazel_git_commit):
1403 url = bazelci_builds_gs_url(platform, bazel_git_commit)
1404 path = os.path.join(dest_dir, "bazel.exe" if platform == "windows" else "bazel")
1405 return download_binary_at_commit(dest_dir, platform, bazel_git_commit, url, path)
1406
1407
1408def download_bazel_nojdk_binary_at_commit(dest_dir, platform, bazel_git_commit):
1409 url = bazelci_builds_nojdk_gs_url(platform, bazel_git_commit)
1410 path = os.path.join(dest_dir, "bazel_nojdk.exe" if platform == "windows" else "bazel_nojdk")
1411 return download_binary_at_commit(dest_dir, platform, bazel_git_commit, url, path)
1412
Chi Wangd279d582021-09-29 10:59:06 +08001413def download_bazelci_agent(dest_dir, platform, version):
1414 postfix = ""
1415 if platform == "windows":
1416 postfix = "x86_64-pc-windows-msvc.exe"
1417 elif platform == "macos":
1418 postfix = "x86_64-apple-darwin"
1419 else:
1420 postfix = "x86_64-unknown-linux-musl"
1421
1422 name = "bazelci-agent-{}-{}".format(version, postfix)
1423 url = "https://github.com/bazelbuild/continuous-integration/releases/download/agent-{}/{}".format(version, name)
1424 path = os.path.join(dest_dir, "bazelci-agent.exe" if platform == "windows" else "bazelci-agent")
1425 execute_command(["curl", "-sSL", url, "-o", path])
1426 st = os.stat(path)
1427 os.chmod(path, st.st_mode | stat.S_IEXEC)
1428 return path
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04001429
joeleba7050d842019-05-23 17:03:31 +02001430def get_mirror_path(git_repository, platform):
1431 mirror_root = {
1432 "macos": "/usr/local/var/bazelbuild/",
1433 "windows": "c:\\buildkite\\bazelbuild\\",
1434 }.get(platform, "/var/lib/bazelbuild/")
1435
1436 return mirror_root + re.sub(r"[^0-9A-Za-z]", "-", git_repository)
1437
1438
Yun Peng376d2b32018-11-29 10:24:54 +01001439def clone_git_repository(git_repository, platform, git_commit=None):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001440 root = downstream_projects_root(platform)
Philipp Wollermannff39ef52018-02-21 14:18:52 +01001441 project_name = re.search(r"/([^/]+)\.git$", git_repository).group(1)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001442 clone_path = os.path.join(root, project_name)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001443 print_collapsed_group(
1444 "Fetching %s sources at %s" % (project_name, git_commit if git_commit else "HEAD")
1445 )
Philipp Wollermann438ec242018-09-05 14:39:24 +02001446
joeleba7050d842019-05-23 17:03:31 +02001447 mirror_path = get_mirror_path(git_repository, platform)
Philipp Wollermannea128282019-05-08 11:56:14 +02001448
Philipp Wollermann414703d2018-08-28 16:40:38 +02001449 if not os.path.exists(clone_path):
Philipp Wollermann62f4a032019-05-08 17:44:14 +02001450 if os.path.exists(mirror_path):
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001451 execute_command(
Philipp Wollermann62f4a032019-05-08 17:44:14 +02001452 ["git", "clone", "-v", "--reference", mirror_path, git_repository, clone_path]
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001453 )
Philipp Wollermannd4cd0d82018-05-01 09:56:24 +02001454 else:
Philipp Wollermannea128282019-05-08 11:56:14 +02001455 execute_command(["git", "clone", "-v", git_repository, clone_path])
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001456
Philipp Wollermann414703d2018-08-28 16:40:38 +02001457 os.chdir(clone_path)
1458 execute_command(["git", "remote", "set-url", "origin", git_repository])
1459 execute_command(["git", "clean", "-fdqx"])
Florian Weikertd8f497c2019-06-19 15:44:20 +02001460 execute_command(["git", "submodule", "foreach", "--recursive", "git clean -fdqx"])
Philipp Wollermann414703d2018-08-28 16:40:38 +02001461 execute_command(["git", "fetch", "origin"])
Yun Peng376d2b32018-11-29 10:24:54 +01001462 if git_commit:
1463 # sync to a specific commit of this repository
1464 execute_command(["git", "reset", git_commit, "--hard"])
1465 else:
1466 # sync to the latest commit of HEAD. Unlikely git pull this also works after a force push.
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001467 remote_head = (
1468 subprocess.check_output(["git", "symbolic-ref", "refs/remotes/origin/HEAD"])
1469 .decode("utf-8")
1470 .rstrip()
1471 )
Yun Peng376d2b32018-11-29 10:24:54 +01001472 execute_command(["git", "reset", remote_head, "--hard"])
Philipp Wollermann414703d2018-08-28 16:40:38 +02001473 execute_command(["git", "submodule", "sync", "--recursive"])
1474 execute_command(["git", "submodule", "update", "--init", "--recursive", "--force"])
Florian Weikertd8f497c2019-06-19 15:44:20 +02001475 execute_command(["git", "submodule", "foreach", "--recursive", "git reset --hard"])
Philipp Wollermann414703d2018-08-28 16:40:38 +02001476 execute_command(["git", "clean", "-fdqx"])
Florian Weikertd8f497c2019-06-19 15:44:20 +02001477 execute_command(["git", "submodule", "foreach", "--recursive", "git clean -fdqx"])
Yun Peng20d45602018-10-18 13:27:05 +02001478 return clone_path
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001479
Philipp Wollermann438ec242018-09-05 14:39:24 +02001480
Yun Penga935a542018-05-18 15:08:53 +02001481def execute_batch_commands(commands):
1482 if not commands:
1483 return
1484 print_collapsed_group(":batch: Setup (Batch Commands)")
1485 batch_commands = "&".join(commands)
Jakob Buchgraber4a824412018-06-22 12:56:10 +02001486 return subprocess.run(batch_commands, shell=True, check=True, env=os.environ).returncode
Yun Penga935a542018-05-18 15:08:53 +02001487
Philipp Wollermann414703d2018-08-28 16:40:38 +02001488
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001489def execute_shell_commands(commands):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001490 if not commands:
1491 return
Jakob Buchgraber94d5c212018-02-22 09:57:08 +01001492 print_collapsed_group(":bash: Setup (Shell Commands)")
mostynb14440912020-03-17 17:11:47 +01001493 shell_command = "\n".join(["set -e"] + commands)
Philipp Wollermann3e1a7712018-02-19 17:34:24 +01001494 execute_command([shell_command], shell=True)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001495
1496
Yun Peng0a6a98a2019-03-06 13:07:54 +01001497def handle_bazel_failure(exception, action):
1498 msg = "bazel {0} failed with exit code {1}".format(action, exception.returncode)
1499 if use_bazelisk_migrate():
1500 print_collapsed_group(msg)
1501 else:
1502 raise BuildkiteException(msg)
1503
1504
Yun Peng4be92b32018-11-30 09:48:29 +01001505def execute_bazel_run(bazel_binary, platform, targets, incompatible_flags):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001506 if not targets:
1507 return
1508 print_collapsed_group("Setup (Run Targets)")
Florian Weikert474d7972019-03-01 02:12:01 +01001509 # When using bazelisk --migrate to test incompatible flags,
1510 # incompatible flags set by "INCOMPATIBLE_FLAGS" env var will be ignored.
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001511 incompatible_flags_to_use = (
1512 [] if (use_bazelisk_migrate() or not incompatible_flags) else incompatible_flags
1513 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001514 for target in targets:
Yun Peng0a6a98a2019-03-06 13:07:54 +01001515 try:
1516 execute_command(
1517 [bazel_binary]
1518 + bazelisk_flags()
1519 + common_startup_flags(platform)
1520 + ["run"]
1521 + common_build_flags(None, platform)
1522 + incompatible_flags_to_use
1523 + [target]
1524 )
1525 except subprocess.CalledProcessError as e:
1526 handle_bazel_failure(e, "run")
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001527
1528
Jakob Buchgraber4f1d2712018-02-20 10:22:47 +01001529def remote_caching_flags(platform):
Philipp Wollermanne67eec42019-05-24 15:18:20 +02001530 # Only enable caching for untrusted and testing builds.
Philipp Wollermannfce92bf2019-05-22 15:14:32 +02001531 if CLOUD_PROJECT not in ["bazel-untrusted"]:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001532 return []
Philipp Wollermann02955272019-04-18 18:00:48 +02001533
Philipp Wollermanne67eec42019-05-24 15:18:20 +02001534 platform_cache_key = [BUILDKITE_ORG.encode("utf-8")]
Jakob Buchgraber89df3982019-08-06 13:07:02 +02001535 # Whenever the remote cache was known to have been poisoned increase the number below
Yun Penge04dd1d2021-08-11 13:14:42 +02001536 platform_cache_key += ["cache-poisoning-20210811".encode("utf-8")]
Philipp Wollermannfce92bf2019-05-22 15:14:32 +02001537
Philipp Wollermann380f1e62019-04-12 16:45:27 +02001538 if platform == "macos":
Philipp Wollermannfce92bf2019-05-22 15:14:32 +02001539 platform_cache_key += [
Philipp Wollermannef89d2f2019-04-18 15:52:24 +02001540 # macOS version:
1541 subprocess.check_output(["/usr/bin/sw_vers", "-productVersion"]),
1542 # Path to Xcode:
1543 subprocess.check_output(["/usr/bin/xcode-select", "-p"]),
1544 # Xcode version:
1545 subprocess.check_output(["/usr/bin/xcodebuild", "-version"]),
1546 ]
1547 # Use a local cache server for our macOS machines.
Philipp Wollermannb9e96282020-02-18 13:59:27 +01001548 flags = ["--remote_cache=http://100.107.73.148"]
Philipp Wollermannef89d2f2019-04-18 15:52:24 +02001549 else:
Philipp Wollermannfce92bf2019-05-22 15:14:32 +02001550 platform_cache_key += [
Philipp Wollermannef89d2f2019-04-18 15:52:24 +02001551 # Platform name:
1552 platform.encode("utf-8")
1553 ]
Philipp Wollermanne74da4e2019-06-07 11:31:02 +02001554 # Use RBE for caching builds running on GCE.
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001555 flags = [
1556 "--google_default_credentials",
Philipp Wollermanne74da4e2019-06-07 11:31:02 +02001557 "--remote_cache=remotebuildexecution.googleapis.com",
1558 "--remote_instance_name=projects/{}/instances/default_instance".format(CLOUD_PROJECT),
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001559 ]
Philipp Wollermann380f1e62019-04-12 16:45:27 +02001560
1561 platform_cache_digest = hashlib.sha256()
1562 for key in platform_cache_key:
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001563 eprint("Adding to platform cache key: {}".format(key))
Philipp Wollermann380f1e62019-04-12 16:45:27 +02001564 platform_cache_digest.update(key)
1565 platform_cache_digest.update(b":")
1566
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001567 flags += [
Philipp Wollermann94937722019-01-11 14:33:18 +01001568 "--remote_timeout=60",
Philipp Wollermann639c0452019-01-03 11:23:54 +01001569 "--remote_max_connections=200",
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001570 '--remote_default_platform_properties=properties:{name:"cache-silo-key" value:"%s"}'
1571 % platform_cache_digest.hexdigest(),
Philipp Wollermann639c0452019-01-03 11:23:54 +01001572 ]
Jakob Buchgraber4f1d2712018-02-20 10:22:47 +01001573
Philipp Wollermannd96d8fa2019-01-11 14:37:47 +01001574 return flags
1575
Jakob Buchgraber4f1d2712018-02-20 10:22:47 +01001576
Jakob Buchgraberb4342cd2018-02-20 16:35:07 +01001577def remote_enabled(flags):
1578 # Detect if the project configuration enabled its own remote caching / execution.
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001579 remote_flags = ["--remote_executor", "--remote_cache", "--remote_http_cache"]
Jakob Buchgraberb4342cd2018-02-20 16:35:07 +01001580 for flag in flags:
1581 for remote_flag in remote_flags:
1582 if flag.startswith(remote_flag):
1583 return True
1584 return False
1585
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01001586
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001587def concurrent_jobs(platform):
1588 return "75" if platform.startswith("rbe_") else str(multiprocessing.cpu_count())
Jakob Buchgraber51a83662018-02-22 19:49:24 +01001589
1590
Philipp Wollermann3e28d3b2018-02-23 23:19:37 +01001591def concurrent_test_jobs(platform):
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001592 if platform.startswith("rbe_"):
1593 return "75"
1594 elif platform == "windows":
Jakob Buchgrabere3ccda32018-06-22 23:29:48 +02001595 return "8"
Philipp Wollermann85877cb2021-07-02 19:20:33 +02001596 elif platform.startswith("macos") and THIS_IS_TESTING:
1597 return "4"
1598 elif platform.startswith("macos"):
Philipp Wollermann111adfb2018-11-22 10:26:03 +01001599 return "8"
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001600 return "12"
Philipp Wollermann3e28d3b2018-02-23 23:19:37 +01001601
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001602
Yun Peng58977d62018-11-16 12:19:20 +01001603def common_startup_flags(platform):
Philipp Wollermannd5ab3d92020-02-05 16:55:13 +01001604 if platform == "windows":
1605 if os.path.exists("D:/b"):
1606 # This machine has a local SSD mounted as drive D.
1607 return ["--output_user_root=D:/b"]
1608 else:
1609 # This machine uses its PD-SSD as the build directory.
1610 return ["--output_user_root=C:/b"]
1611 return []
Yun Peng58977d62018-11-16 12:19:20 +01001612
1613
1614def common_build_flags(bep_file, platform):
Yun Peng088cc932018-11-16 12:11:46 +01001615 flags = [
Yun Pengf51e7842018-11-16 11:35:43 +01001616 "--show_progress_rate_limit=5",
1617 "--curses=yes",
1618 "--color=yes",
Philipp Wollermannd99414c2019-05-28 17:26:09 +02001619 "--terminal_columns=143",
Philipp Wollermann4c8391e2019-05-28 18:03:35 +02001620 "--show_timestamps",
Yun Pengf51e7842018-11-16 11:35:43 +01001621 "--verbose_failures",
Yun Pengf51e7842018-11-16 11:35:43 +01001622 "--jobs=" + concurrent_jobs(platform),
Yun Pengf51e7842018-11-16 11:35:43 +01001623 "--announce_rc",
Philipp Wollermannb97f9102019-04-16 18:05:56 +02001624 "--experimental_repository_cache_hardlinks",
Philipp Wollermannef89d2f2019-04-18 15:52:24 +02001625 # Some projects set --disk_cache in their project-specific bazelrc, which we never want on
1626 # CI, so let's just disable it explicitly.
1627 "--disk_cache=",
Yun Peng088cc932018-11-16 12:11:46 +01001628 ]
Philipp Wollermann639c0452019-01-03 11:23:54 +01001629
Philipp Wollermannb97f9102019-04-16 18:05:56 +02001630 if platform == "windows":
1631 pass
1632 elif platform == "macos":
1633 flags += [
1634 "--sandbox_writable_path=/var/tmp/_bazel_buildkite/cache/repos/v1",
1635 "--test_env=REPOSITORY_CACHE=/var/tmp/_bazel_buildkite/cache/repos/v1",
1636 ]
1637 else:
Philipp Wollermann639c0452019-01-03 11:23:54 +01001638 flags += ["--sandbox_tmpfs_path=/tmp"]
1639
Yun Peng088cc932018-11-16 12:11:46 +01001640 if bep_file:
Philipp Wollermann639c0452019-01-03 11:23:54 +01001641 flags += [
1642 "--experimental_build_event_json_file_path_conversion=false",
1643 "--build_event_json_file=" + bep_file,
1644 ]
1645
Yun Peng088cc932018-11-16 12:11:46 +01001646 return flags
Philipp Wollermann94bd9e32018-04-30 15:32:28 +02001647
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001648
Yun Pengb7247ff2018-11-15 13:52:39 +01001649def rbe_flags(original_flags, accept_cached):
Philipp Wollermannbcfd9da2018-08-09 15:31:18 +02001650 # Enable remote execution via RBE.
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001651 flags = [
1652 "--remote_executor=remotebuildexecution.googleapis.com",
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01001653 "--remote_instance_name=projects/bazel-untrusted/instances/default_instance",
Philipp Wollermann2df93482021-06-16 04:39:49 +02001654 "--remote_timeout=3600",
Philipp Wollermann57dadb82020-02-17 14:32:24 +01001655 "--incompatible_strict_action_env",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001656 "--google_default_credentials",
Philipp Wollermann57dadb82020-02-17 14:32:24 +01001657 "--toolchain_resolution_debug",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001658 ]
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02001659
Philipp Wollermannbcfd9da2018-08-09 15:31:18 +02001660 # Enable BES / Build Results reporting.
1661 flags += [
1662 "--bes_backend=buildeventservice.googleapis.com",
Philipp Wollermannbcfd9da2018-08-09 15:31:18 +02001663 "--bes_timeout=360s",
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01001664 "--project_id=bazel-untrusted",
Philipp Wollermannbcfd9da2018-08-09 15:31:18 +02001665 ]
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02001666
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001667 if not accept_cached:
1668 flags += ["--noremote_accept_cached"]
Philipp Wollermannbcfd9da2018-08-09 15:31:18 +02001669
Nicolas Lopez36996222019-05-28 12:21:28 -04001670 # Adapted from https://github.com/bazelbuild/bazel-toolchains/blob/master/bazelrc/.bazelrc
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001671 flags += [
Nicolas Lopez36996222019-05-28 12:21:28 -04001672 # These should NOT longer need to be modified.
1673 # All that is needed is updating the @bazel_toolchains repo pin
1674 # in projects' WORKSPACE files.
Xindb02c012018-11-07 14:10:54 -05001675 #
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001676 # Toolchain related flags to append at the end of your .bazelrc file.
Nicolas Lopez36996222019-05-28 12:21:28 -04001677 "--host_javabase=@buildkite_config//java:jdk",
1678 "--javabase=@buildkite_config//java:jdk",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001679 "--host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8",
1680 "--java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8",
Nicolas Lopez36996222019-05-28 12:21:28 -04001681 "--crosstool_top=@buildkite_config//cc:toolchain",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001682 "--action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001683 ]
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02001684
Yun Pengb7247ff2018-11-15 13:52:39 +01001685 # Platform flags:
1686 # The toolchain container used for execution is defined in the target indicated
1687 # by "extra_execution_platforms", "host_platform" and "platforms".
Xin03a88ab2019-03-11 19:18:50 -04001688 # If you are using your own toolchain container, you need to create a platform
1689 # target with "constraint_values" that allow for the toolchain specified with
1690 # "extra_toolchains" to be selected (given constraints defined in
1691 # "exec_compatible_with").
Yun Pengb7247ff2018-11-15 13:52:39 +01001692 # More about platforms: https://docs.bazel.build/versions/master/platforms.html
1693 # Don't add platform flags if they are specified already.
1694 platform_flags = {
Nicolas Lopez36996222019-05-28 12:21:28 -04001695 "--extra_toolchains": "@buildkite_config//config:cc-toolchain",
1696 "--extra_execution_platforms": "@buildkite_config//config:platform",
1697 "--host_platform": "@buildkite_config//config:platform",
1698 "--platforms": "@buildkite_config//config:platform",
Yun Pengb7247ff2018-11-15 13:52:39 +01001699 }
Yun Peng67ab5062018-11-15 13:58:15 +01001700 for platform_flag, value in list(platform_flags.items()):
Yun Pengb7247ff2018-11-15 13:52:39 +01001701 found = False
1702 for original_flag in original_flags:
1703 if original_flag.startswith(platform_flag):
1704 found = True
1705 break
1706 if not found:
1707 flags += [platform_flag + "=" + value]
1708
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001709 return flags
1710
1711
Philipp Wollermann87b45252020-01-22 12:43:42 +01001712def compute_flags(
1713 platform, flags, incompatible_flags, bep_file, bazel_binary, enable_remote_cache=False
1714):
Yun Peng58977d62018-11-16 12:19:20 +01001715 aggregated_flags = common_build_flags(bep_file, platform)
Yun Peng32dbec32018-11-02 12:47:41 +01001716 if not remote_enabled(flags):
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001717 if platform.startswith("rbe_"):
Florian Weikert24a4b482018-11-30 19:05:38 +01001718 aggregated_flags += rbe_flags(flags, accept_cached=enable_remote_cache)
1719 elif enable_remote_cache:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001720 aggregated_flags += remote_caching_flags(platform)
Yun Peng75786552018-11-13 15:23:30 +01001721 aggregated_flags += flags
Yun Peng4be92b32018-11-30 09:48:29 +01001722 if incompatible_flags:
1723 aggregated_flags += incompatible_flags
Yun Peng53598002018-12-03 10:42:02 +01001724
Philipp Wollermann87b45252020-01-22 12:43:42 +01001725 for i, flag in enumerate(aggregated_flags):
1726 if "$HOME" in flag:
Philipp Wollermannd5ab3d92020-02-05 16:55:13 +01001727 if platform == "windows":
1728 if os.path.exists("D:/"):
1729 home = "D:"
1730 else:
1731 home = "C:/b"
1732 elif platform == "macos":
1733 home = "/Users/buildkite"
1734 else:
1735 home = "/var/lib/buildkite-agent"
Philipp Wollermann87b45252020-01-22 12:43:42 +01001736 aggregated_flags[i] = flag.replace("$HOME", home)
1737 if "$OUTPUT_BASE" in flag:
1738 output_base = execute_command_and_get_output(
Philipp Wollermann0e394412020-01-26 15:52:32 +01001739 [bazel_binary] + common_startup_flags(platform) + ["info", "output_base"],
1740 print_output=False,
Philipp Wollermann87b45252020-01-22 12:43:42 +01001741 ).strip()
1742 aggregated_flags[i] = flag.replace("$OUTPUT_BASE", output_base)
1743
Florian Weikert24a4b482018-11-30 19:05:38 +01001744 return aggregated_flags
Yun Peng53598002018-12-03 10:42:02 +01001745
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001746
Yun Pengea0359e2019-01-17 15:37:47 +01001747def execute_bazel_clean(bazel_binary, platform):
1748 print_expanded_group(":bazel: Clean")
1749
1750 try:
Philipp Wollermannc05ac682019-01-19 12:37:28 +01001751 execute_command([bazel_binary] + common_startup_flags(platform) + ["clean", "--expunge"])
Yun Pengea0359e2019-01-17 15:37:47 +01001752 except subprocess.CalledProcessError as e:
1753 raise BuildkiteException("bazel clean failed with exit code {}".format(e.returncode))
1754
1755
Chi Wang6357efe2020-08-25 16:23:38 +08001756def kythe_startup_flags():
1757 return [f"--bazelrc={KYTHE_DIR}/extractors.bazelrc"]
1758
1759
1760def kythe_build_flags():
Philipp Wollermannf61a20b2021-05-05 22:04:20 +02001761 return [
1762 "--experimental_convenience_symlinks=normal",
Florian Weikert9d5e4c02021-05-27 15:10:20 +02001763 f"--override_repository=kythe_release={KYTHE_DIR}",
Philipp Wollermannf61a20b2021-05-05 22:04:20 +02001764 ]
Chi Wang6357efe2020-08-25 16:23:38 +08001765
1766
Florian Weikertc8642af2019-02-03 23:58:51 +01001767def execute_bazel_build(
1768 bazel_version, bazel_binary, platform, flags, targets, bep_file, incompatible_flags
1769):
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001770 print_collapsed_group(":bazel: Computing flags for build step")
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001771 aggregated_flags = compute_flags(
Yun Peng8975c6b2019-02-28 11:55:55 +01001772 platform,
1773 flags,
1774 # When using bazelisk --migrate to test incompatible flags,
1775 # incompatible flags set by "INCOMPATIBLE_FLAGS" env var will be ignored.
1776 [] if (use_bazelisk_migrate() or not incompatible_flags) else incompatible_flags,
1777 bep_file,
Philipp Wollermann87b45252020-01-22 12:43:42 +01001778 bazel_binary,
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001779 enable_remote_cache=True,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001780 )
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001781
1782 print_expanded_group(":bazel: Build ({})".format(bazel_version))
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01001783 try:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001784 execute_command(
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001785 [bazel_binary]
1786 + bazelisk_flags()
1787 + common_startup_flags(platform)
1788 + ["build"]
1789 + aggregated_flags
Philipp Wollermann2a160432019-09-19 15:57:28 +02001790 + ["--"]
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001791 + targets
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001792 )
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01001793 except subprocess.CalledProcessError as e:
Yun Peng0a6a98a2019-03-06 13:07:54 +01001794 handle_bazel_failure(e, "build")
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001795
1796
Chi Wang6357efe2020-08-25 16:23:38 +08001797def execute_bazel_build_with_kythe(
1798 bazel_version, bazel_binary, platform, flags, targets, bep_file, incompatible_flags
1799):
1800 print_collapsed_group(":bazel: Computing flags for build step")
1801 aggregated_flags = compute_flags(
1802 platform,
1803 flags,
1804 # When using bazelisk --migrate to test incompatible flags,
1805 # incompatible flags set by "INCOMPATIBLE_FLAGS" env var will be ignored.
1806 [] if (use_bazelisk_migrate() or not incompatible_flags) else incompatible_flags,
1807 bep_file,
1808 bazel_binary,
1809 enable_remote_cache=False,
1810 )
1811
1812 print_expanded_group(":bazel: Build ({})".format(bazel_version))
1813
1814 execute_command(
1815 [bazel_binary]
1816 + bazelisk_flags()
1817 + common_startup_flags(platform)
1818 + kythe_startup_flags()
1819 + ["build"]
1820 + kythe_build_flags()
1821 + aggregated_flags
1822 + ["--"]
1823 + targets
1824 )
1825
1826
Florian Weikert736d06e2019-05-08 13:16:42 +02001827def calculate_targets(task_config, platform, bazel_binary, build_only, test_only):
1828 build_targets = [] if test_only else task_config.get("build_targets", [])
1829 test_targets = [] if build_only else task_config.get("test_targets", [])
Chi Wang6357efe2020-08-25 16:23:38 +08001830 index_targets = [] if (build_only or test_only) else task_config.get("index_targets", [])
1831
Florian Weikertdb832a02020-11-19 19:14:48 +01001832 index_targets_query = (
1833 None if (build_only or test_only) else task_config.get("index_targets_query", None)
1834 )
Chi Wang6357efe2020-08-25 16:23:38 +08001835 if index_targets_query:
1836 output = execute_command_and_get_output(
1837 [bazel_binary]
1838 + common_startup_flags(platform)
Florian Weikertdb832a02020-11-19 19:14:48 +01001839 + ["--nomaster_bazelrc", "--bazelrc=/dev/null", "query", index_targets_query],
Chi Wang6357efe2020-08-25 16:23:38 +08001840 print_output=False,
1841 )
1842 index_targets += output.strip().split("\n")
Florian Weikert736d06e2019-05-08 13:16:42 +02001843
Philipp Wollermann2a160432019-09-19 15:57:28 +02001844 # Remove the "--" argument splitter from the list that some configs explicitly
1845 # include. We'll add it back again later where needed.
1846 build_targets = [x.strip() for x in build_targets if x.strip() != "--"]
1847 test_targets = [x.strip() for x in test_targets if x.strip() != "--"]
Chi Wang6357efe2020-08-25 16:23:38 +08001848 index_targets = [x.strip() for x in index_targets if x.strip() != "--"]
Philipp Wollermann2a160432019-09-19 15:57:28 +02001849
Florian Weikert736d06e2019-05-08 13:16:42 +02001850 shard_id = int(os.getenv("BUILDKITE_PARALLEL_JOB", "-1"))
1851 shard_count = int(os.getenv("BUILDKITE_PARALLEL_JOB_COUNT", "-1"))
1852 if shard_id > -1 and shard_count > -1:
1853 print_collapsed_group(
1854 ":female-detective: Calculating targets for shard {}/{}".format(
1855 shard_id + 1, shard_count
1856 )
1857 )
1858 expanded_test_targets = expand_test_target_patterns(bazel_binary, platform, test_targets)
Philipp Wollermann87b45252020-01-22 12:43:42 +01001859 test_targets = get_targets_for_shard(expanded_test_targets, shard_id, shard_count)
Florian Weikert736d06e2019-05-08 13:16:42 +02001860
Chi Wang6357efe2020-08-25 16:23:38 +08001861 return build_targets, test_targets, index_targets
Florian Weikert736d06e2019-05-08 13:16:42 +02001862
1863
1864def expand_test_target_patterns(bazel_binary, platform, test_targets):
Philipp Wollermann2a160432019-09-19 15:57:28 +02001865 included_targets, excluded_targets = partition_targets(test_targets)
Florian Weikert736d06e2019-05-08 13:16:42 +02001866 excluded_string = (
1867 " except tests(set({}))".format(" ".join("'{}'".format(t) for t in excluded_targets))
1868 if excluded_targets
1869 else ""
1870 )
1871
Philipp Wollermann2a160432019-09-19 15:57:28 +02001872 exclude_manual = ' except tests(attr("tags", "manual", set({})))'.format(
1873 " ".join("'{}'".format(t) for t in included_targets)
1874 )
1875
Florian Weikert736d06e2019-05-08 13:16:42 +02001876 eprint("Resolving test targets via bazel query")
1877 output = execute_command_and_get_output(
1878 [bazel_binary]
1879 + common_startup_flags(platform)
1880 + [
1881 "--nomaster_bazelrc",
1882 "--bazelrc=/dev/null",
1883 "query",
Philipp Wollermann2a160432019-09-19 15:57:28 +02001884 "tests(set({})){}{}".format(
1885 " ".join("'{}'".format(t) for t in included_targets),
1886 excluded_string,
1887 exclude_manual,
Florian Weikert736d06e2019-05-08 13:16:42 +02001888 ),
1889 ],
1890 print_output=False,
Philipp Wollermann54719fe2021-07-02 21:54:08 +02001891 ).strip()
1892 return output.split("\n") if output else []
Florian Weikert736d06e2019-05-08 13:16:42 +02001893
1894
Philipp Wollermann2a160432019-09-19 15:57:28 +02001895def partition_targets(targets):
Florian Weikert736d06e2019-05-08 13:16:42 +02001896 included_targets, excluded_targets = [], []
Philipp Wollermann2a160432019-09-19 15:57:28 +02001897 for target in targets:
1898 if target.startswith("-"):
Florian Weikert736d06e2019-05-08 13:16:42 +02001899 excluded_targets.append(target[1:])
1900 else:
1901 included_targets.append(target)
1902
1903 return included_targets, excluded_targets
1904
1905
Philipp Wollermann87b45252020-01-22 12:43:42 +01001906def get_targets_for_shard(test_targets, shard_id, shard_count):
Florian Weikert736d06e2019-05-08 13:16:42 +02001907 # TODO(fweikert): implement a more sophisticated algorithm
Philipp Wollermann87b45252020-01-22 12:43:42 +01001908 return sorted(test_targets)[shard_id::shard_count]
Florian Weikert736d06e2019-05-08 13:16:42 +02001909
1910
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001911def execute_bazel_test(
Florian Weikertc8642af2019-02-03 23:58:51 +01001912 bazel_version,
1913 bazel_binary,
1914 platform,
1915 flags,
1916 targets,
1917 bep_file,
1918 monitor_flaky_tests,
1919 incompatible_flags,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001920):
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001921 aggregated_flags = [
1922 "--flaky_test_attempts=3",
1923 "--build_tests_only",
1924 "--local_test_jobs=" + concurrent_test_jobs(platform),
1925 ]
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001926 # Don't enable remote caching if the user enabled remote execution / caching themselves
Jakob Buchgraberc340f582018-06-22 13:48:33 +02001927 # or flaky test monitoring is enabled, as remote caching makes tests look less flaky than
1928 # they are.
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001929 print_collapsed_group(":bazel: Computing flags for test step")
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001930 aggregated_flags += compute_flags(
Yun Peng8975c6b2019-02-28 11:55:55 +01001931 platform,
1932 flags,
1933 # When using bazelisk --migrate to test incompatible flags,
1934 # incompatible flags set by "INCOMPATIBLE_FLAGS" env var will be ignored.
1935 [] if (use_bazelisk_migrate() or not incompatible_flags) else incompatible_flags,
1936 bep_file,
Philipp Wollermann87b45252020-01-22 12:43:42 +01001937 bazel_binary,
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001938 enable_remote_cache=not monitor_flaky_tests,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001939 )
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001940
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001941 print_expanded_group(":bazel: Test ({})".format(bazel_version))
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01001942 try:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001943 execute_command(
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001944 [bazel_binary]
1945 + bazelisk_flags()
1946 + common_startup_flags(platform)
1947 + ["test"]
1948 + aggregated_flags
Philipp Wollermann2a160432019-09-19 15:57:28 +02001949 + ["--"]
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001950 + targets
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001951 )
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01001952 except subprocess.CalledProcessError as e:
Yun Peng0a6a98a2019-03-06 13:07:54 +01001953 handle_bazel_failure(e, "test")
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001954
1955
Chi Wangd279d582021-09-29 10:59:06 +08001956def upload_test_logs_from_bep(bep_file, tmpdir, binary_platform, monitor_flaky_tests):
1957 bazelci_agent_binary = download_bazelci_agent(tmpdir, binary_platform, "0.1.1")
1958 execute_command(
1959 [bazelci_agent_binary, "artifact", "upload", "--delay=5", "--mode=buildkite", "--build_event_json_file={}".format(bep_file)]
1960 + (["--monitor_flaky_tests"] if monitor_flaky_tests else [])
1961 )
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001962
Philipp Wollermann3c8b8512019-07-16 15:28:03 +02001963
joeleba76887952019-05-16 15:22:17 +02001964def upload_json_profile(json_profile_path, tmpdir):
1965 if not os.path.exists(json_profile_path):
1966 return
1967 print_collapsed_group(":gcloud: Uploading JSON Profile")
Philipp Wollermann92cf51e2019-05-16 15:31:11 +02001968 execute_command(["buildkite-agent", "artifact", "upload", json_profile_path], cwd=tmpdir)
joeleba76887952019-05-16 15:22:17 +02001969
Philipp Wollermannf436e742021-08-11 11:06:55 +02001970
Chi Wang54595a22021-06-11 17:49:58 +08001971def upload_corrupted_outputs(capture_corrupted_outputs_dir, tmpdir):
1972 if not os.path.exists(capture_corrupted_outputs_dir):
1973 return
1974 print_collapsed_group(":gcloud: Uploading corrupted outputs")
Philipp Wollermannf436e742021-08-11 11:06:55 +02001975 execute_command(
1976 ["buildkite-agent", "artifact", "upload", "{}/**/*".format(capture_corrupted_outputs_dir)],
1977 cwd=tmpdir,
1978 )
1979
Philipp Wollermann5b00a702019-07-18 11:21:38 +02001980
Philipp Wollermannaf35abf2019-05-22 17:52:01 +02001981def execute_command_and_get_output(args, shell=False, fail_if_nonzero=True, print_output=True):
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01001982 eprint(" ".join(args))
Florian Weikertc8642af2019-02-03 23:58:51 +01001983 process = subprocess.run(
1984 args,
1985 shell=shell,
1986 check=fail_if_nonzero,
1987 env=os.environ,
1988 stdout=subprocess.PIPE,
Philipp Wollermannf13804b2019-02-05 21:08:30 +01001989 errors="replace",
Florian Weikertc8642af2019-02-03 23:58:51 +01001990 universal_newlines=True,
1991 )
Florian Weikert736d06e2019-05-08 13:16:42 +02001992 if print_output:
1993 eprint(process.stdout)
1994
Florian Weikertc8642af2019-02-03 23:58:51 +01001995 return process.stdout
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001996
1997
Yun Peng9337bb32020-02-28 13:31:29 +01001998def execute_command(args, shell=False, fail_if_nonzero=True, cwd=None, print_output=True):
1999 if print_output:
2000 eprint(" ".join(args))
Philipp Wollermann92cf51e2019-05-16 15:31:11 +02002001 return subprocess.run(
2002 args, shell=shell, check=fail_if_nonzero, env=os.environ, cwd=cwd
2003 ).returncode
Philipp Wollermannf13804b2019-02-05 21:08:30 +01002004
2005
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02002006def execute_command_background(args):
2007 eprint(" ".join(args))
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02002008 return subprocess.Popen(args, env=os.environ)
2009
Philipp Wollermann3c8b8512019-07-16 15:28:03 +02002010
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02002011def terminate_background_process(process):
2012 if process:
2013 process.terminate()
2014 try:
2015 process.wait(timeout=10)
2016 except subprocess.TimeoutExpired:
2017 process.kill()
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02002018
Philipp Wollermann3c8b8512019-07-16 15:28:03 +02002019
mai93b49bad72021-05-06 00:50:34 +02002020def create_step(label, commands, platform, shards=1, soft_fail=None):
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002021 if "docker-image" in PLATFORMS[platform]:
Florian Weikert736d06e2019-05-08 13:16:42 +02002022 step = create_docker_step(
Philipp Wollermannc05ac682019-01-19 12:37:28 +01002023 label, image=PLATFORMS[platform]["docker-image"], commands=commands
2024 )
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002025 else:
Philipp Wollermannf3750fa2019-05-21 17:11:59 +02002026 step = {
2027 "label": label,
2028 "command": commands,
2029 "agents": {"queue": PLATFORMS[platform]["queue"]},
2030 }
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002031
Florian Weikert736d06e2019-05-08 13:16:42 +02002032 if shards > 1:
2033 step["label"] += " (shard %n)"
2034 step["parallelism"] = shards
2035
mai93b49bad72021-05-06 00:50:34 +02002036 if soft_fail is not None:
2037 step["soft_fail"] = soft_fail
2038
Philipp Wollermann5b2f3fc2019-05-18 22:36:17 +02002039 # Enforce a global 8 hour job timeout.
2040 step["timeout_in_minutes"] = 8 * 60
2041
2042 # Automatically retry when an agent got lost (usually due to an infra flake).
Philipp Wollermannf22bba32019-07-18 11:22:50 +02002043 step["retry"] = {
2044 "automatic": [
2045 {"exit_status": -1, "limit": 3}, # Buildkite internal "agent lost" exit code
2046 {"exit_status": 137, "limit": 3}, # SIGKILL
2047 {"exit_status": 143, "limit": 3}, # SIGTERM
2048 ]
2049 }
Philipp Wollermann5b2f3fc2019-05-18 22:36:17 +02002050
Florian Weikert736d06e2019-05-08 13:16:42 +02002051 return step
2052
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002053
Florian Weikert29cb7ec2019-03-07 14:52:18 +01002054def create_docker_step(label, image, commands=None, additional_env_vars=None):
Philipp Wollermann0e051dd2019-05-16 11:37:52 +02002055 env = ["ANDROID_HOME", "ANDROID_NDK_HOME", "BUILDKITE_ARTIFACT_UPLOAD_DESTINATION"]
Florian Weikert29cb7ec2019-03-07 14:52:18 +01002056 if additional_env_vars:
2057 env += ["{}={}".format(k, v) for k, v in additional_env_vars.items()]
2058
Philipp Wollermannc05ac682019-01-19 12:37:28 +01002059 step = {
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002060 "label": label,
2061 "command": commands,
Philipp Wollermannb2fa2f62019-05-18 23:33:23 +02002062 "agents": {"queue": "default"},
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002063 "plugins": {
Philipp Wollermann9fa03542021-07-01 23:27:53 +02002064 "docker#v3.8.0": {
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002065 "always-pull": True,
Florian Weikert29cb7ec2019-03-07 14:52:18 +01002066 "environment": env,
Philipp Wollermannc05ac682019-01-19 12:37:28 +01002067 "image": image,
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002068 "network": "host",
2069 "privileged": True,
2070 "propagate-environment": True,
Philipp Wollermann7aa95492019-05-18 22:03:24 +02002071 "propagate-uid-gid": True,
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002072 "volumes": [
Philipp Wollermann7aa95492019-05-18 22:03:24 +02002073 "/etc/group:/etc/group:ro",
2074 "/etc/passwd:/etc/passwd:ro",
Philipp Wollermann338db4a2019-05-18 11:21:04 +02002075 "/opt:/opt:ro",
Philipp Wollermann7aa95492019-05-18 22:03:24 +02002076 "/var/lib/buildkite-agent:/var/lib/buildkite-agent",
Philipp Wollermann338db4a2019-05-18 11:21:04 +02002077 "/var/lib/gitmirrors:/var/lib/gitmirrors:ro",
Philipp Wollermanna65944a2020-02-03 12:45:22 +01002078 "/var/run/docker.sock:/var/run/docker.sock",
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002079 ],
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002080 }
2081 },
2082 }
Philipp Wollermannc05ac682019-01-19 12:37:28 +01002083 if not step["command"]:
2084 del step["command"]
2085 return step
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002086
2087
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002088def print_project_pipeline(
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002089 configs,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002090 project_name,
2091 http_config,
2092 file_config,
2093 git_repository,
2094 monitor_flaky_tests,
2095 use_but,
2096 incompatible_flags,
Florian Weikert60661912019-12-18 15:17:10 +01002097 notify,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002098):
Florian Weikert843d7a02019-02-03 17:24:50 +01002099 task_configs = configs.get("tasks", None)
2100 if not task_configs:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002101 raise BuildkiteException("{0} pipeline configuration is empty.".format(project_name))
2102
Jakob Buchgraberaa2af382018-02-21 19:56:54 +01002103 pipeline_steps = []
mai93f2e116c2020-10-19 09:33:14 +02002104 # If the repository is hosted on Git-on-borg, we show the link to the commit Gerrit review
2105 buildkite_repo = os.getenv("BUILDKITE_REPO")
2106 if is_git_on_borg_repo(buildkite_repo):
2107 show_gerrit_review_link(buildkite_repo, pipeline_steps)
2108
Florian Weikert5f5d3cb2019-04-15 15:36:27 +02002109 task_configs = filter_tasks_that_should_be_skipped(task_configs, pipeline_steps)
Jakob Buchgraberff2bdad2018-02-25 13:06:30 +01002110
Philipp Wollermanndac65512019-02-05 22:14:10 +01002111 # In Bazel Downstream Project pipelines, git_repository and project_name must be specified.
2112 is_downstream_project = (use_but or incompatible_flags) and git_repository and project_name
2113
Florian Weikert85208912019-03-07 17:08:39 +01002114 buildifier_config = configs.get("buildifier")
Philipp Wollermanndac65512019-02-05 22:14:10 +01002115 # Skip Buildifier when we test downstream projects.
Florian Weikert85208912019-03-07 17:08:39 +01002116 if buildifier_config and not is_downstream_project:
2117 buildifier_env_vars = {}
2118 if isinstance(buildifier_config, str):
2119 # Simple format:
2120 # ---
2121 # buildifier: latest
2122 buildifier_env_vars[BUILDIFIER_VERSION_ENV_VAR] = buildifier_config
2123 else:
2124 # Advanced format:
2125 # ---
2126 # buildifier:
2127 # version: latest
2128 # warnings: all
2129
Philipp Wollermannce986af2019-07-18 14:46:05 +02002130 def set_env_var(config_key, env_var_name):
Florian Weikert85208912019-03-07 17:08:39 +01002131 if config_key in buildifier_config:
2132 buildifier_env_vars[env_var_name] = buildifier_config[config_key]
2133
Philipp Wollermannce986af2019-07-18 14:46:05 +02002134 set_env_var("version", BUILDIFIER_VERSION_ENV_VAR)
2135 set_env_var("warnings", BUILDIFIER_WARNINGS_ENV_VAR)
Florian Weikert85208912019-03-07 17:08:39 +01002136
2137 if not buildifier_env_vars:
2138 raise BuildkiteException(
2139 'Invalid buildifier configuration entry "{}"'.format(buildifier_config)
2140 )
2141
Florian Weikert29cb7ec2019-03-07 14:52:18 +01002142 pipeline_steps.append(
2143 create_docker_step(
Florian Weikertde96a6f2019-03-07 14:57:50 +01002144 BUILDIFIER_STEP_NAME,
Florian Weikert29cb7ec2019-03-07 14:52:18 +01002145 image=BUILDIFIER_DOCKER_IMAGE,
Florian Weikert85208912019-03-07 17:08:39 +01002146 additional_env_vars=buildifier_env_vars,
Florian Weikert29cb7ec2019-03-07 14:52:18 +01002147 )
2148 )
Philipp Wollermannc05ac682019-01-19 12:37:28 +01002149
Philipp Wollermanndac65512019-02-05 22:14:10 +01002150 # In Bazel Downstream Project pipelines, we should test the project at the last green commit.
Yun Peng376d2b32018-11-29 10:24:54 +01002151 git_commit = None
Philipp Wollermanndac65512019-02-05 22:14:10 +01002152 if is_downstream_project:
Florian Weikert35906542019-04-01 11:53:53 +02002153 last_green_commit_url = bazelci_last_green_commit_url(
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002154 git_repository, DOWNSTREAM_PROJECTS[project_name]["pipeline_slug"]
2155 )
Florian Weikert35906542019-04-01 11:53:53 +02002156 git_commit = get_last_green_commit(last_green_commit_url)
Philipp Wollermanndac65512019-02-05 22:14:10 +01002157
Florian Weikert854fd852019-06-04 16:44:19 +02002158 config_hashes = set()
Florian Weikertdb832a02020-11-19 19:14:48 +01002159 skipped_due_to_bazel_version = []
Florian Weikert843d7a02019-02-03 17:24:50 +01002160 for task, task_config in task_configs.items():
Florian Weikertdb832a02020-11-19 19:14:48 +01002161 platform = get_platform_for_task(task, task_config)
2162 task_name = task_config.get("name")
mai93b49bad72021-05-06 00:50:34 +02002163 soft_fail = task_config.get("soft_fail")
Florian Weikertdb832a02020-11-19 19:14:48 +01002164
Florian Weikert854fd852019-06-04 16:44:19 +02002165 # We override the Bazel version in downstream pipelines. This means that two tasks that
2166 # only differ in the value of their explicit "bazel" field will be identical in the
2167 # downstream pipeline, thus leading to duplicate work.
2168 # Consequently, we filter those duplicate tasks here.
2169 if is_downstream_project:
Florian Weikertdb832a02020-11-19 19:14:48 +01002170 # Skip tasks that require a specific Bazel version
2171 bazel = task_config.get("bazel")
2172 if bazel and bazel != "latest":
2173 skipped_due_to_bazel_version.append(
2174 "{}: '{}'".format(
2175 create_label(platform, project_name, task_name=task_name), bazel
2176 )
2177 )
2178 continue
2179
Florian Weikert854fd852019-06-04 16:44:19 +02002180 h = hash_task_config(task, task_config)
2181 if h in config_hashes:
2182 continue
Florian Weikert854fd852019-06-04 16:44:19 +02002183 config_hashes.add(h)
2184
Florian Weikert736d06e2019-05-08 13:16:42 +02002185 shards = task_config.get("shards", "1")
2186 try:
2187 shards = int(shards)
2188 except ValueError:
2189 raise BuildkiteException("Task {} has invalid shard value '{}'".format(task, shards))
2190
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002191 step = runner_step(
Florian Weikertdb832a02020-11-19 19:14:48 +01002192 platform=platform,
Florian Weikert843d7a02019-02-03 17:24:50 +01002193 task=task,
Florian Weikertdb832a02020-11-19 19:14:48 +01002194 task_name=task_name,
Florian Weikert843d7a02019-02-03 17:24:50 +01002195 project_name=project_name,
2196 http_config=http_config,
2197 file_config=file_config,
2198 git_repository=git_repository,
2199 git_commit=git_commit,
2200 monitor_flaky_tests=monitor_flaky_tests,
2201 use_but=use_but,
2202 incompatible_flags=incompatible_flags,
Florian Weikert736d06e2019-05-08 13:16:42 +02002203 shards=shards,
mai93b49bad72021-05-06 00:50:34 +02002204 soft_fail=soft_fail,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002205 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002206 pipeline_steps.append(step)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002207
Florian Weikertdb832a02020-11-19 19:14:48 +01002208 if skipped_due_to_bazel_version:
Xùdōng Yáng045c9812021-08-18 01:42:35 +10002209 lines = ["\n- {}".format(s) for s in skipped_due_to_bazel_version]
Florian Weikertdb832a02020-11-19 19:14:48 +01002210 commands = [
Xùdōng Yáng2f2245a2021-08-17 00:21:58 +10002211 "buildkite-agent annotate --style=info '{}' --append --context 'ctx-skipped_due_to_bazel_version'".format(
Xùdōng Yáng045c9812021-08-18 01:42:35 +10002212 "".join(lines)
2213 ),
2214 "buildkite-agent meta-data set 'has-skipped-steps' 'true'",
Florian Weikertdb832a02020-11-19 19:14:48 +01002215 ]
2216 pipeline_steps.append(
2217 create_step(
2218 label=":pipeline: Print information about skipped tasks due to different Bazel versions",
2219 commands=commands,
2220 platform=DEFAULT_PLATFORM,
2221 )
2222 )
2223
Yun Peng996efad2018-11-27 17:19:44 +01002224 pipeline_slug = os.getenv("BUILDKITE_PIPELINE_SLUG")
2225 all_downstream_pipeline_slugs = []
2226 for _, config in DOWNSTREAM_PROJECTS.items():
2227 all_downstream_pipeline_slugs.append(config["pipeline_slug"])
Ivo List23ce48d2020-11-18 13:15:34 +01002228 # We update last green commit in the following cases:
2229 # 1. This job runs on master, stable or main branch (could be a custom build launched manually)
2230 # 2. We intend to run the same job in downstream with Bazel@HEAD (eg. google-bazel-presubmit)
2231 # 3. This job is not:
2232 # - a GitHub pull request
2233 # - uses a custom built Bazel binary (in Bazel Downstream Projects pipeline)
2234 # - testing incompatible flags
2235 # - running `bazelisk --migrate` in a non-downstream pipeline
Florian Weikertdb832a02020-11-19 19:14:48 +01002236 if (
Philipp Wollermann1b5ecdc2021-06-10 21:52:55 +02002237 current_branch_is_main_branch()
Ivo List23ce48d2020-11-18 13:15:34 +01002238 and pipeline_slug in all_downstream_pipeline_slugs
2239 and not (is_pull_request() or use_but or incompatible_flags or use_bazelisk_migrate())
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002240 ):
Florian Weikertde96a6f2019-03-07 14:57:50 +01002241 # We need to call "Try Update Last Green Commit" even if there are failures,
2242 # since we don't want a failing Buildifier step to block the update of
2243 # the last green commit for this project.
2244 # try_update_last_green_commit() ensures that we don't update the commit
2245 # if any build or test steps fail.
2246 pipeline_steps.append({"wait": None, "continue_on_failure": True})
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002247 pipeline_steps.append(
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002248 create_step(
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002249 label="Try Update Last Green Commit",
2250 commands=[
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002251 fetch_bazelcipy_command(),
Philipp Wollermann57b32682019-05-18 22:09:27 +02002252 PLATFORMS[DEFAULT_PLATFORM]["python"]
2253 + " bazelci.py try_update_last_green_commit",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002254 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002255 platform=DEFAULT_PLATFORM,
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002256 )
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002257 )
Yun Peng43239b02018-11-23 13:57:34 +01002258
Florian Weikert778251c2019-04-25 15:14:44 +02002259 if "validate_config" in configs:
Florian Weikertf52f91a2019-05-08 15:19:30 +02002260 pipeline_steps += create_config_validation_steps()
Florian Weikert778251c2019-04-25 15:14:44 +02002261
Florian Weikert09813a02019-10-26 19:34:33 +02002262 if use_bazelisk_migrate() and not is_downstream_project:
2263 # Print results of bazelisk --migrate in project pipelines that explicitly set
2264 # the USE_BAZELISK_MIGRATE env var, but that are not being run as part of a
2265 # downstream pipeline.
2266 number = os.getenv("BUILDKITE_BUILD_NUMBER")
Florian Weikert60661912019-12-18 15:17:10 +01002267 pipeline_steps += get_steps_for_aggregating_migration_results(number, notify)
Florian Weikert09813a02019-10-26 19:34:33 +02002268
Florian Weikertd79dc502019-05-13 09:51:05 +02002269 print_pipeline_steps(pipeline_steps, handle_emergencies=not is_downstream_project)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002270
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002271
mai93f2e116c2020-10-19 09:33:14 +02002272def show_gerrit_review_link(git_repository, pipeline_steps):
2273 host = re.search(r"https://(.+?)\.googlesource", git_repository).group(1)
2274 if not host:
2275 raise BuildkiteException("Couldn't get host name from %s" % git_repository)
Florian Weikertdb832a02020-11-19 19:14:48 +01002276 text = "The transformed code used in this pipeline can be found under https://{}-review.googlesource.com/q/{}".format(
2277 host, os.getenv("BUILDKITE_COMMIT")
2278 )
mai93f2e116c2020-10-19 09:33:14 +02002279 commands = ["buildkite-agent annotate --style=info '{}'".format(text)]
2280 pipeline_steps.append(
2281 create_step(
2282 label=":pipeline: Print information about Gerrit Review Link",
2283 commands=commands,
2284 platform=DEFAULT_PLATFORM,
2285 )
2286 )
2287
2288
2289def is_git_on_borg_repo(git_repository):
2290 return git_repository and "googlesource.com" in git_repository
2291
2292
Florian Weikert854fd852019-06-04 16:44:19 +02002293def hash_task_config(task_name, task_config):
2294 # Two task configs c1 and c2 have the same hash iff they lead to two functionally identical jobs
2295 # in the downstream pipeline. This function discards the "bazel" field (since it's being
Philipp Wollermannce986af2019-07-18 14:46:05 +02002296 # overridden) and the "name" field (since it has no effect on the actual work).
Florian Weikert854fd852019-06-04 16:44:19 +02002297 # Moreover, it adds an explicit "platform" field if that's missing.
2298 cpy = task_config.copy()
2299 cpy.pop("bazel", None)
2300 cpy.pop("name", None)
2301 if "platform" not in cpy:
2302 cpy["platform"] = task_name
2303
2304 m = hashlib.md5()
2305 for key in sorted(cpy):
Florian Weikert8186c392019-06-05 12:53:39 +02002306 value = "%s:%s;" % (key, cpy[key])
2307 m.update(value.encode("utf-8"))
Florian Weikert854fd852019-06-04 16:44:19 +02002308
2309 return m.digest()
2310
2311
Florian Weikert843d7a02019-02-03 17:24:50 +01002312def get_platform_for_task(task, task_config):
2313 # Most pipeline configurations have exactly one task per platform, which makes it
2314 # convenient to use the platform name as task ID. Consequently, we use the
2315 # task ID as platform if there is no explicit "platform" field.
2316 return task_config.get("platform", task)
2317
2318
Florian Weikertf52f91a2019-05-08 15:19:30 +02002319def create_config_validation_steps():
2320 output = execute_command_and_get_output(
2321 ["git", "diff-tree", "--no-commit-id", "--name-only", "-r", os.getenv("BUILDKITE_COMMIT")]
2322 )
2323 config_files = [
Philipp Wollermann67225ec2021-08-11 11:12:51 +02002324 path
2325 for path in output.split("\n")
2326 if path.startswith(".bazelci/") and os.path.splitext(path)[1] in CONFIG_FILE_EXTENSIONS
Florian Weikertf52f91a2019-05-08 15:19:30 +02002327 ]
Florian Weikertf52f91a2019-05-08 15:19:30 +02002328 return [
2329 create_step(
2330 label=":cop: Validate {}".format(f),
2331 commands=[
2332 fetch_bazelcipy_command(),
2333 "{} bazelci.py project_pipeline --file_config={}".format(
Philipp Wollermann57b32682019-05-18 22:09:27 +02002334 PLATFORMS[DEFAULT_PLATFORM]["python"], f
Florian Weikertf52f91a2019-05-08 15:19:30 +02002335 ),
2336 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002337 platform=DEFAULT_PLATFORM,
Florian Weikertf52f91a2019-05-08 15:19:30 +02002338 )
2339 for f in config_files
2340 ]
2341
2342
Florian Weikertd79dc502019-05-13 09:51:05 +02002343def print_pipeline_steps(pipeline_steps, handle_emergencies=True):
2344 if handle_emergencies:
2345 emergency_step = create_emergency_announcement_step_if_necessary()
2346 if emergency_step:
2347 pipeline_steps.insert(0, emergency_step)
Florian Weikert13215a82019-05-10 12:42:21 +02002348
2349 print(yaml.dump({"steps": pipeline_steps}))
2350
2351
2352def create_emergency_announcement_step_if_necessary():
2353 style = "error"
2354 message, issue_url, last_good_bazel = None, None, None
2355 try:
2356 emergency_settings = load_remote_yaml_file(EMERGENCY_FILE_URL)
2357 message = emergency_settings.get("message")
2358 issue_url = emergency_settings.get("issue_url")
2359 last_good_bazel = emergency_settings.get("last_good_bazel")
2360 except urllib.error.HTTPError as ex:
2361 message = str(ex)
2362 style = "warning"
2363
2364 if not any([message, issue_url, last_good_bazel]):
2365 return
2366
2367 text = '<span class="h1">:rotating_light: Emergency :rotating_light:</span>\n'
2368 if message:
2369 text += "- {}\n".format(message)
2370 if issue_url:
2371 text += '- Please check this <a href="{}">issue</a> for more details.\n'.format(issue_url)
2372 if last_good_bazel:
2373 text += (
2374 "- Default Bazel version is *{}*, "
2375 "unless the pipeline configuration specifies an explicit version."
2376 ).format(last_good_bazel)
2377
2378 return create_step(
2379 label=":rotating_light: Emergency :rotating_light:",
Philipp Wollermann7590b962019-05-16 11:35:03 +02002380 commands=[
2381 'buildkite-agent annotate --append --style={} --context "omg" "{}"'.format(style, text)
2382 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002383 platform=DEFAULT_PLATFORM,
Florian Weikert13215a82019-05-10 12:42:21 +02002384 )
2385
2386
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002387def runner_step(
2388 platform,
Florian Weikert843d7a02019-02-03 17:24:50 +01002389 task,
2390 task_name=None,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002391 project_name=None,
2392 http_config=None,
2393 file_config=None,
2394 git_repository=None,
2395 git_commit=None,
2396 monitor_flaky_tests=False,
2397 use_but=False,
2398 incompatible_flags=None,
Florian Weikert736d06e2019-05-08 13:16:42 +02002399 shards=1,
mai93b49bad72021-05-06 00:50:34 +02002400 soft_fail=None,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002401):
Philipp Wollermann57b32682019-05-18 22:09:27 +02002402 command = PLATFORMS[platform]["python"] + " bazelci.py runner --task=" + task
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002403 if http_config:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002404 command += " --http_config=" + http_config
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01002405 if file_config:
2406 command += " --file_config=" + file_config
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002407 if git_repository:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002408 command += " --git_repository=" + git_repository
Yun Peng376d2b32018-11-29 10:24:54 +01002409 if git_commit:
2410 command += " --git_commit=" + git_commit
Jakob Buchgraberc340f582018-06-22 13:48:33 +02002411 if monitor_flaky_tests:
2412 command += " --monitor_flaky_tests"
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002413 if use_but:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002414 command += " --use_but"
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002415 for flag in incompatible_flags or []:
Yun Peng4be92b32018-11-30 09:48:29 +01002416 command += " --incompatible_flag=" + flag
Florian Weikert843d7a02019-02-03 17:24:50 +01002417 label = create_label(platform, project_name, task_name=task_name)
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002418 return create_step(
Florian Weikert9d5e4c02021-05-27 15:10:20 +02002419 label=label,
2420 commands=[fetch_bazelcipy_command(), command],
2421 platform=platform,
2422 shards=shards,
2423 soft_fail=soft_fail,
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002424 )
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002425
2426
2427def fetch_bazelcipy_command():
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002428 return "curl -sS {0} -o bazelci.py".format(SCRIPT_URL)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002429
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002430
Yun Peng002eab92018-12-17 18:28:14 +01002431def fetch_incompatible_flag_verbose_failures_command():
Philipp Wollermannfe145a52019-01-11 13:16:48 +01002432 return "curl -sS {0} -o incompatible_flag_verbose_failures.py".format(
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002433 INCOMPATIBLE_FLAG_VERBOSE_FAILURES_URL
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002434 )
Yun Peng002eab92018-12-17 18:28:14 +01002435
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002436
Yun Peng8975c6b2019-02-28 11:55:55 +01002437def fetch_aggregate_incompatible_flags_test_result_command():
2438 return "curl -sS {0} -o aggregate_incompatible_flags_test_result.py".format(
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002439 AGGREGATE_INCOMPATIBLE_TEST_RESULT_URL
Yun Peng8975c6b2019-02-28 11:55:55 +01002440 )
2441
2442
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002443def upload_project_pipeline_step(
2444 project_name, git_repository, http_config, file_config, incompatible_flags
2445):
2446 pipeline_command = (
2447 '{0} bazelci.py project_pipeline --project_name="{1}" ' + "--git_repository={2}"
Philipp Wollermann57b32682019-05-18 22:09:27 +02002448 ).format(PLATFORMS[DEFAULT_PLATFORM]["python"], project_name, git_repository)
Philipp Wollermann639c0452019-01-03 11:23:54 +01002449 if incompatible_flags is None:
Yun Peng4be92b32018-11-30 09:48:29 +01002450 pipeline_command += " --use_but"
Yun Peng95908792018-11-30 15:03:55 +01002451 else:
2452 for flag in incompatible_flags:
2453 pipeline_command += " --incompatible_flag=" + flag
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002454 if http_config:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002455 pipeline_command += " --http_config=" + http_config
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01002456 if file_config:
2457 pipeline_command += " --file_config=" + file_config
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002458 pipeline_command += " | buildkite-agent pipeline upload"
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002459
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002460 return create_step(
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002461 label="Setup {0}".format(project_name),
2462 commands=[fetch_bazelcipy_command(), pipeline_command],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002463 platform=DEFAULT_PLATFORM,
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002464 )
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002465
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002466
Florian Weikert843d7a02019-02-03 17:24:50 +01002467def create_label(platform, project_name, build_only=False, test_only=False, task_name=None):
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002468 if build_only and test_only:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002469 raise BuildkiteException("build_only and test_only cannot be true at the same time")
Florian Weikert843d7a02019-02-03 17:24:50 +01002470 platform_display_name = PLATFORMS[platform]["emoji-name"]
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002471
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002472 if build_only:
2473 label = "Build "
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002474 elif test_only:
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002475 label = "Test "
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002476 else:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002477 label = ""
2478
Florian Weikert843d7a02019-02-03 17:24:50 +01002479 platform_label = (
2480 "{0} on {1}".format(task_name, platform_display_name)
2481 if task_name
2482 else platform_display_name
2483 )
2484
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002485 if project_name:
Florian Weikert843d7a02019-02-03 17:24:50 +01002486 label += "{0} ({1})".format(project_name, platform_label)
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002487 else:
Florian Weikert843d7a02019-02-03 17:24:50 +01002488 label += platform_label
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002489
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002490 return label
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002491
2492
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002493def bazel_build_step(
Florian Weikert843d7a02019-02-03 17:24:50 +01002494 task,
2495 platform,
2496 project_name,
2497 http_config=None,
2498 file_config=None,
2499 build_only=False,
2500 test_only=False,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002501):
Philipp Wollermann57b32682019-05-18 22:09:27 +02002502 pipeline_command = PLATFORMS[platform]["python"] + " bazelci.py runner"
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002503 if build_only:
Philipp Wollermannc52e26a2019-05-18 22:10:47 +02002504 pipeline_command += " --build_only --save_but"
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002505 if test_only:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002506 pipeline_command += " --test_only"
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002507 if http_config:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002508 pipeline_command += " --http_config=" + http_config
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01002509 if file_config:
2510 pipeline_command += " --file_config=" + file_config
Florian Weikert843d7a02019-02-03 17:24:50 +01002511 pipeline_command += " --task=" + task
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002512
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002513 return create_step(
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002514 label=create_label(platform, project_name, build_only, test_only),
2515 commands=[fetch_bazelcipy_command(), pipeline_command],
2516 platform=platform,
2517 )
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002518
2519
Florian Weikert5f5d3cb2019-04-15 15:36:27 +02002520def filter_tasks_that_should_be_skipped(task_configs, pipeline_steps):
2521 skip_tasks = get_skip_tasks()
2522 if not skip_tasks:
Florian Weikert5f5d3cb2019-04-15 15:36:27 +02002523 return task_configs
2524
2525 actually_skipped = []
2526 skip_tasks = set(skip_tasks)
2527 for task in list(task_configs.keys()):
2528 if task in skip_tasks:
2529 actually_skipped.append(task)
2530 del task_configs[task]
2531 skip_tasks.remove(task)
2532
2533 if not task_configs:
2534 raise BuildkiteException(
2535 "Nothing to do since all tasks in the configuration should be skipped."
2536 )
2537
2538 annotations = []
2539 if actually_skipped:
2540 annotations.append(
2541 ("info", "Skipping the following task(s): {}".format(", ".join(actually_skipped)))
2542 )
2543
2544 if skip_tasks:
2545 annotations.append(
2546 (
2547 "warning",
2548 (
2549 "The following tasks should have been skipped, "
2550 "but were not part of the configuration: {}"
2551 ).format(", ".join(skip_tasks)),
2552 )
2553 )
2554
2555 if annotations:
2556 print_skip_task_annotations(annotations, pipeline_steps)
2557
2558 return task_configs
2559
2560
2561def get_skip_tasks():
2562 value = os.getenv(SKIP_TASKS_ENV_VAR, "")
2563 return [v for v in value.split(",") if v]
2564
2565
2566def print_skip_task_annotations(annotations, pipeline_steps):
2567 commands = [
2568 "buildkite-agent annotate --style={} '{}' --context 'ctx-{}'".format(s, t, hash(t))
2569 for s, t in annotations
2570 ]
2571 pipeline_steps.append(
Philipp Wollermann7a185322019-05-18 22:15:48 +02002572 create_step(
2573 label=":pipeline: Print information about skipped tasks",
2574 commands=commands,
2575 platform=DEFAULT_PLATFORM,
2576 )
Florian Weikert5f5d3cb2019-04-15 15:36:27 +02002577 )
2578
2579
Florian Weikert843d7a02019-02-03 17:24:50 +01002580def print_bazel_publish_binaries_pipeline(task_configs, http_config, file_config):
2581 if not task_configs:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002582 raise BuildkiteException("Bazel publish binaries pipeline configuration is empty.")
2583
Florian Weikert5f5d3cb2019-04-15 15:36:27 +02002584 pipeline_steps = []
2585 task_configs = filter_tasks_that_should_be_skipped(task_configs, pipeline_steps)
2586
Florian Weikert843d7a02019-02-03 17:24:50 +01002587 platforms = [get_platform_for_task(t, tc) for t, tc in task_configs.items()]
Philipp Wollermann783d1672019-06-06 13:35:30 +02002588
2589 # These are the platforms that the bazel_publish_binaries.yml config is actually building.
2590 configured_platforms = set(filter(should_publish_binaries_for_platform, platforms))
Philipp Wollermanna2ea5d82018-08-27 14:12:10 +02002591
Philipp Wollermann783d1672019-06-06 13:35:30 +02002592 # These are the platforms that we want to build and publish according to this script.
2593 expected_platforms = set(filter(should_publish_binaries_for_platform, PLATFORMS))
Florian Weikert843d7a02019-02-03 17:24:50 +01002594
Philipp Wollermann30f314d2021-06-11 10:51:39 +02002595 # We can skip this check if we're not on the main branch, because then we're probably
2596 # building a one-off custom debugging binary anyway.
2597 if current_branch_is_main_branch() and not expected_platforms.issubset(configured_platforms):
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002598 raise BuildkiteException(
2599 "Bazel publish binaries pipeline needs to build Bazel for every commit on all publish_binary-enabled platforms."
2600 )
Jakob Buchgraber08e8e402018-03-20 19:22:07 +01002601
Yun Pengd352b6d2018-10-17 13:28:39 +02002602 # Build Bazel
Florian Weikert843d7a02019-02-03 17:24:50 +01002603 for task, task_config in task_configs.items():
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002604 pipeline_steps.append(
Florian Weikert843d7a02019-02-03 17:24:50 +01002605 bazel_build_step(
2606 task,
2607 get_platform_for_task(task, task_config),
2608 "Bazel",
2609 http_config,
2610 file_config,
2611 build_only=True,
2612 )
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002613 )
Jakob Buchgraber4631a032018-03-22 17:12:46 +01002614
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002615 pipeline_steps.append("wait")
Jakob Buchgraber9d6ca8a2018-03-22 17:30:09 +01002616
Yun Pengc2dd6522018-10-17 12:58:35 +02002617 # If all builds succeed, publish the Bazel binaries to GCS.
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002618 pipeline_steps.append(
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002619 create_step(
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002620 label="Publish Bazel Binaries",
Philipp Wollermann57b32682019-05-18 22:09:27 +02002621 commands=[
2622 fetch_bazelcipy_command(),
2623 PLATFORMS[DEFAULT_PLATFORM]["python"] + " bazelci.py publish_binaries",
2624 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002625 platform=DEFAULT_PLATFORM,
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002626 )
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002627 )
Jakob Buchgraber08e8e402018-03-20 19:22:07 +01002628
Florian Weikert13215a82019-05-10 12:42:21 +02002629 print_pipeline_steps(pipeline_steps)
Jakob Buchgraber08e8e402018-03-20 19:22:07 +01002630
2631
Florian Weikert843d7a02019-02-03 17:24:50 +01002632def should_publish_binaries_for_platform(platform):
2633 if platform not in PLATFORMS:
2634 raise BuildkiteException("Unknown platform '{}'".format(platform))
2635
2636 return PLATFORMS[platform]["publish_binary"]
2637
2638
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01002639def print_disabled_projects_info_box_step():
2640 info_text = ["Downstream testing is disabled for the following projects :sadpanda:"]
2641 for project, config in DOWNSTREAM_PROJECTS.items():
2642 disabled_reason = config.get("disabled_reason", None)
2643 if disabled_reason:
2644 info_text.append("* **%s**: %s" % (project, disabled_reason))
2645
2646 if len(info_text) == 1:
2647 return None
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002648 return create_step(
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002649 label=":sadpanda:",
2650 commands=[
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002651 'buildkite-agent annotate --append --style=info "\n' + "\n".join(info_text) + '\n"'
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01002652 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002653 platform=DEFAULT_PLATFORM,
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002654 )
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01002655
Yun Peng6528e652019-01-02 14:41:07 +01002656
2657def print_incompatible_flags_info_box_step(incompatible_flags_map):
2658 info_text = ["Build and test with the following incompatible flags:"]
2659
2660 for flag in incompatible_flags_map:
2661 info_text.append("* **%s**: %s" % (flag, incompatible_flags_map[flag]))
2662
2663 if len(info_text) == 1:
2664 return None
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002665 return create_step(
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002666 label="Incompatible flags info",
2667 commands=[
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002668 'buildkite-agent annotate --append --style=info "\n' + "\n".join(info_text) + '\n"'
Yun Peng6528e652019-01-02 14:41:07 +01002669 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002670 platform=DEFAULT_PLATFORM,
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002671 )
Yun Peng6528e652019-01-02 14:41:07 +01002672
2673
Yun Peng7d302f62019-01-10 16:56:15 +01002674def fetch_incompatible_flags():
Yun Peng6528e652019-01-02 14:41:07 +01002675 """
2676 Return a list of incompatible flags to be tested in downstream with the current release Bazel
2677 """
Yun Peng7d302f62019-01-10 16:56:15 +01002678 incompatible_flags = {}
2679
2680 # If INCOMPATIBLE_FLAGS environment variable is set, we get incompatible flags from it.
2681 if "INCOMPATIBLE_FLAGS" in os.environ:
2682 for flag in os.environ["INCOMPATIBLE_FLAGS"].split():
2683 # We are not able to get the github link for this flag from INCOMPATIBLE_FLAGS,
2684 # so just assign the url to empty string.
2685 incompatible_flags[flag] = ""
2686 return incompatible_flags
2687
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002688 output = subprocess.check_output(
2689 [
2690 "curl",
Florian Weikert78467862021-09-23 13:39:00 +02002691 "https://api.github.com/search/issues?per_page=100&q=repo:bazelbuild/bazel+label:incompatible-change+state:open"
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002692 ]
2693 ).decode("utf-8")
Yun Peng6528e652019-01-02 14:41:07 +01002694 issue_info = json.loads(output)
2695
Yun Peng6528e652019-01-02 14:41:07 +01002696 for issue in issue_info["items"]:
Yun Peng6528e652019-01-02 14:41:07 +01002697 # Every incompatible flags issue should start with "<incompatible flag name (without --)>:"
2698 name = "--" + issue["title"].split(":")[0]
2699 url = issue["html_url"]
2700 if name.startswith("--incompatible_"):
2701 incompatible_flags[name] = url
2702 else:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002703 eprint(
Philipp Wollermann639c0452019-01-03 11:23:54 +01002704 f"{name} is not recognized as an incompatible flag, please modify the issue title "
2705 f'of {url} to "<incompatible flag name (without --)>:..."'
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002706 )
Yun Peng6528e652019-01-02 14:41:07 +01002707
2708 return incompatible_flags
2709
2710
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002711def print_bazel_downstream_pipeline(
Florian Weikert60661912019-12-18 15:17:10 +01002712 task_configs, http_config, file_config, test_incompatible_flags, test_disabled_projects, notify
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002713):
Florian Weikert843d7a02019-02-03 17:24:50 +01002714 if not task_configs:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002715 raise BuildkiteException("Bazel downstream pipeline configuration is empty.")
2716
Florian Weikert5f5d3cb2019-04-15 15:36:27 +02002717 pipeline_steps = []
2718 task_configs = filter_tasks_that_should_be_skipped(task_configs, pipeline_steps)
2719
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002720 pipeline_steps = []
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002721
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01002722 info_box_step = print_disabled_projects_info_box_step()
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01002723 if info_box_step is not None:
2724 pipeline_steps.append(info_box_step)
2725
Yun Peng5599ca22019-01-16 12:32:41 +01002726 if not test_incompatible_flags:
Florian Weikert843d7a02019-02-03 17:24:50 +01002727 for task, task_config in task_configs.items():
Yun Peng5599ca22019-01-16 12:32:41 +01002728 pipeline_steps.append(
Florian Weikert843d7a02019-02-03 17:24:50 +01002729 bazel_build_step(
2730 task,
2731 get_platform_for_task(task, task_config),
2732 "Bazel",
2733 http_config,
2734 file_config,
2735 build_only=True,
2736 )
Yun Peng5599ca22019-01-16 12:32:41 +01002737 )
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002738
Yun Peng5599ca22019-01-16 12:32:41 +01002739 pipeline_steps.append("wait")
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002740
Yun Pengb9998d12018-12-03 10:18:28 +01002741 incompatible_flags = None
Yun Peng7a539ef2018-11-30 15:07:24 +01002742 if test_incompatible_flags:
Yun Peng7d302f62019-01-10 16:56:15 +01002743 incompatible_flags_map = fetch_incompatible_flags()
Yun Peng3c1d7d12020-06-30 14:58:34 +02002744 if not incompatible_flags_map:
Florian Weikert9d5e4c02021-05-27 15:10:20 +02002745 step = create_step(
2746 label="No Incompatible flags info",
2747 commands=[
Florian Weikert42738ac2021-05-27 15:54:14 +02002748 'buildkite-agent annotate --style=error "No incompatible flag issue is found on github for current version of Bazel." --context "noinc"'
Florian Weikert9d5e4c02021-05-27 15:10:20 +02002749 ],
2750 platform=DEFAULT_PLATFORM,
Florian Weikertdb832a02020-11-19 19:14:48 +01002751 )
Florian Weikert9d5e4c02021-05-27 15:10:20 +02002752 pipeline_steps.append(step)
2753 print_pipeline_steps(pipeline_steps)
2754 return
2755
Yun Peng6528e652019-01-02 14:41:07 +01002756 info_box_step = print_incompatible_flags_info_box_step(incompatible_flags_map)
2757 if info_box_step is not None:
2758 pipeline_steps.append(info_box_step)
2759 incompatible_flags = list(incompatible_flags_map.keys())
Yun Peng7a539ef2018-11-30 15:07:24 +01002760
Xùdōng Yáng045c9812021-08-18 01:42:35 +10002761 pipeline_steps.append(create_step(
2762 label="Print skipped tasks annotation",
2763 commands=['buildkite-agent annotate --style=info "The following tasks were skipped since they require specific Bazel versions:\n" --context "ctx-skipped_due_to_bazel_version"'],
2764 platform=DEFAULT_PLATFORM))
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002765 for project, config in DOWNSTREAM_PROJECTS.items():
Yun Peng996efad2018-11-27 17:19:44 +01002766 disabled_reason = config.get("disabled_reason", None)
Yun Pengfb759fa2018-12-13 11:35:39 +01002767 # If test_disabled_projects is true, we add configs for disabled projects.
Florian Weikert7b3f17e2019-03-14 13:52:42 +01002768 # If test_disabled_projects is false, we add configs for not disabled projects.
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002769 if (test_disabled_projects and disabled_reason) or (
2770 not test_disabled_projects and not disabled_reason
2771 ):
Yun Peng996efad2018-11-27 17:19:44 +01002772 pipeline_steps.append(
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002773 upload_project_pipeline_step(
2774 project_name=project,
2775 git_repository=config["git_repository"],
2776 http_config=config.get("http_config", None),
2777 file_config=config.get("file_config", None),
2778 incompatible_flags=incompatible_flags,
2779 )
2780 )
Xùdōng Yáng045c9812021-08-18 01:42:35 +10002781 pipeline_steps.append(create_step(
2782 label="Remove skipped tasks annotation if unneeded",
2783 commands=['buildkite-agent meta-data exists "has-skipped-steps" || buildkite-agent annotation remove --context "ctx-skipped_due_to_bazel_version"'],
2784 platform=DEFAULT_PLATFORM))
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002785
Yun Peng002eab92018-12-17 18:28:14 +01002786 if test_incompatible_flags:
Yun Peng002eab92018-12-17 18:28:14 +01002787 current_build_number = os.environ.get("BUILDKITE_BUILD_NUMBER", None)
2788 if not current_build_number:
2789 raise BuildkiteException("Not running inside Buildkite")
Yun Peng8975c6b2019-02-28 11:55:55 +01002790 if use_bazelisk_migrate():
Florian Weikert09813a02019-10-26 19:34:33 +02002791 pipeline_steps += get_steps_for_aggregating_migration_results(
Florian Weikert60661912019-12-18 15:17:10 +01002792 current_build_number, notify
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002793 )
Yun Peng8975c6b2019-02-28 11:55:55 +01002794 else:
2795 pipeline_steps.append({"wait": "~", "continue_on_failure": "true"})
2796 pipeline_steps.append(
2797 create_step(
2798 label="Test failing jobs with incompatible flag separately",
2799 commands=[
2800 fetch_bazelcipy_command(),
2801 fetch_incompatible_flag_verbose_failures_command(),
Philipp Wollermann57b32682019-05-18 22:09:27 +02002802 PLATFORMS[DEFAULT_PLATFORM]["python"]
Yun Peng8975c6b2019-02-28 11:55:55 +01002803 + " incompatible_flag_verbose_failures.py --build_number=%s | buildkite-agent pipeline upload"
2804 % current_build_number,
2805 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002806 platform=DEFAULT_PLATFORM,
Yun Peng8975c6b2019-02-28 11:55:55 +01002807 )
2808 )
Yun Peng002eab92018-12-17 18:28:14 +01002809
Florian Weikert2896edb2019-04-04 16:12:47 +02002810 if (
2811 not test_disabled_projects
2812 and not test_incompatible_flags
Philipp Wollermann1b5ecdc2021-06-10 21:52:55 +02002813 and current_branch_is_main_branch()
Florian Weikert2896edb2019-04-04 16:12:47 +02002814 ):
Florian Weikert35906542019-04-01 11:53:53 +02002815 # Only update the last green downstream commit in the regular Bazel@HEAD + Downstream pipeline.
2816 pipeline_steps.append("wait")
2817 pipeline_steps.append(
2818 create_step(
2819 label="Try Update Last Green Downstream Commit",
2820 commands=[
2821 fetch_bazelcipy_command(),
Philipp Wollermann57b32682019-05-18 22:09:27 +02002822 PLATFORMS[DEFAULT_PLATFORM]["python"]
2823 + " bazelci.py try_update_last_green_downstream_commit",
Florian Weikert35906542019-04-01 11:53:53 +02002824 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002825 platform=DEFAULT_PLATFORM,
Florian Weikert35906542019-04-01 11:53:53 +02002826 )
2827 )
2828
Florian Weikert13215a82019-05-10 12:42:21 +02002829 print_pipeline_steps(pipeline_steps)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002830
2831
Florian Weikert60661912019-12-18 15:17:10 +01002832def get_steps_for_aggregating_migration_results(current_build_number, notify):
Florian Weikert09813a02019-10-26 19:34:33 +02002833 parts = [
2834 PLATFORMS[DEFAULT_PLATFORM]["python"],
2835 "aggregate_incompatible_flags_test_result.py",
2836 "--build_number=%s" % current_build_number,
Florian Weikert09813a02019-10-26 19:34:33 +02002837 ]
Florian Weikert60661912019-12-18 15:17:10 +01002838 if notify:
2839 parts.append("--notify")
Florian Weikert09813a02019-10-26 19:34:33 +02002840 return [
2841 {"wait": "~", "continue_on_failure": "true"},
2842 create_step(
2843 label="Aggregate incompatible flags test result",
2844 commands=[
2845 fetch_bazelcipy_command(),
2846 fetch_aggregate_incompatible_flags_test_result_command(),
2847 " ".join(parts),
2848 ],
2849 platform=DEFAULT_PLATFORM,
2850 ),
2851 ]
2852
2853
Yun Pengc2dd6522018-10-17 12:58:35 +02002854def bazelci_builds_download_url(platform, git_commit):
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002855 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-builds"
2856 return "https://storage.googleapis.com/{}/artifacts/{}/{}/bazel".format(
2857 bucket_name, platform, git_commit
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002858 )
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002859
2860
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04002861def bazelci_builds_nojdk_download_url(platform, git_commit):
2862 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-builds"
2863 return "https://storage.googleapis.com/{}/artifacts/{}/{}/bazel_nojdk".format(
2864 bucket_name, platform, git_commit
2865 )
2866
2867
Yun Peng20d45602018-10-18 13:27:05 +02002868def bazelci_builds_gs_url(platform, git_commit):
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002869 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-builds"
2870 return "gs://{}/artifacts/{}/{}/bazel".format(bucket_name, platform, git_commit)
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002871
2872
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04002873def bazelci_builds_nojdk_gs_url(platform, git_commit):
2874 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-builds"
2875 return "gs://{}/artifacts/{}/{}/bazel_nojdk".format(bucket_name, platform, git_commit)
2876
2877
mai93f04f9482020-10-20 17:22:30 +02002878def bazelci_latest_build_metadata_url():
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002879 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-builds"
2880 return "gs://{}/metadata/latest.json".format(bucket_name)
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002881
2882
mai93f04f9482020-10-20 17:22:30 +02002883def bazelci_builds_metadata_url(git_commit):
2884 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-builds"
2885 return "gs://{}/metadata/{}.json".format(bucket_name, git_commit)
2886
2887
Yun Peng996efad2018-11-27 17:19:44 +01002888def bazelci_last_green_commit_url(git_repository, pipeline_slug):
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002889 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-untrusted-builds"
2890 return "gs://{}/last_green_commit/{}/{}".format(
2891 bucket_name, git_repository[len("https://") :], pipeline_slug
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002892 )
Yun Pengafe67d42018-11-23 17:06:43 +01002893
2894
Florian Weikert35906542019-04-01 11:53:53 +02002895def bazelci_last_green_downstream_commit_url():
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002896 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-untrusted-builds"
2897 return "gs://{}/last_green_commit/downstream_pipeline".format(bucket_name)
Florian Weikert35906542019-04-01 11:53:53 +02002898
2899
2900def get_last_green_commit(last_green_commit_url):
Yun Peng61a448f2018-11-23 17:11:46 +01002901 try:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002902 return (
2903 subprocess.check_output(
2904 [gsutil_command(), "cat", last_green_commit_url], env=os.environ
2905 )
2906 .decode("utf-8")
2907 .strip()
2908 )
Yun Peng61a448f2018-11-23 17:11:46 +01002909 except subprocess.CalledProcessError:
2910 return None
Yun Peng43239b02018-11-23 13:57:34 +01002911
2912
Yun Peng358cd882018-11-29 10:25:18 +01002913def try_update_last_green_commit():
Florian Weikertde96a6f2019-03-07 14:57:50 +01002914 org_slug = os.getenv("BUILDKITE_ORGANIZATION_SLUG")
Yun Peng358cd882018-11-29 10:25:18 +01002915 pipeline_slug = os.getenv("BUILDKITE_PIPELINE_SLUG")
Florian Weikertde96a6f2019-03-07 14:57:50 +01002916 build_number = os.getenv("BUILDKITE_BUILD_NUMBER")
2917 current_job_id = os.getenv("BUILDKITE_JOB_ID")
2918
2919 client = BuildkiteClient(org=org_slug, pipeline=pipeline_slug)
2920 build_info = client.get_build_info(build_number)
2921
mai9302a609c2021-05-20 10:36:46 +02002922 # Find any failing steps other than Buildifier and steps with soft_fail enabled then "try update last green".
Philipp Wollermannce986af2019-07-18 14:46:05 +02002923 def has_failed(job):
Florian Weikertbd40a272019-03-08 10:20:18 +01002924 state = job.get("state")
2925 # Ignore steps that don't have a state (like "wait").
Florian Weikertde96a6f2019-03-07 14:57:50 +01002926 return (
Florian Weikert35906542019-04-01 11:53:53 +02002927 state is not None
2928 and state != "passed"
mai9302a609c2021-05-20 10:36:46 +02002929 and not job.get("soft_failed")
Florian Weikertde96a6f2019-03-07 14:57:50 +01002930 and job["id"] != current_job_id
2931 and job["name"] != BUILDIFIER_STEP_NAME
2932 )
2933
Philipp Wollermannce986af2019-07-18 14:46:05 +02002934 failing_jobs = [j["name"] for j in build_info["jobs"] if has_failed(j)]
Florian Weikertde96a6f2019-03-07 14:57:50 +01002935 if failing_jobs:
2936 raise BuildkiteException(
2937 "Cannot update last green commit due to {} failing step(s): {}".format(
2938 len(failing_jobs), ", ".join(failing_jobs)
2939 )
2940 )
2941
Yun Peng358cd882018-11-29 10:25:18 +01002942 git_repository = os.getenv("BUILDKITE_REPO")
Florian Weikert35906542019-04-01 11:53:53 +02002943 last_green_commit_url = bazelci_last_green_commit_url(git_repository, pipeline_slug)
2944 update_last_green_commit_if_newer(last_green_commit_url)
2945
2946
2947def update_last_green_commit_if_newer(last_green_commit_url):
2948 last_green_commit = get_last_green_commit(last_green_commit_url)
Yun Peng358cd882018-11-29 10:25:18 +01002949 current_commit = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("utf-8").strip()
2950 if last_green_commit:
Jakob Buchgraber7c7ceee2019-10-28 10:28:58 +01002951 success = False
2952 try:
2953 execute_command(["git", "fetch", "-v", "origin", last_green_commit])
2954 success = True
2955 except subprocess.CalledProcessError:
2956 # If there was an error fetching the commit it typically means
2957 # that the commit does not exist anymore - due to a force push. In
2958 # order to recover from that assume that the current commit is the
2959 # newest commit.
2960 result = [current_commit]
2961 finally:
2962 if success:
2963 result = (
2964 subprocess.check_output(
2965 ["git", "rev-list", "%s..%s" % (last_green_commit, current_commit)]
2966 )
2967 .decode("utf-8")
2968 .strip()
2969 )
Philipp Wollermannce986af2019-07-18 14:46:05 +02002970 else:
2971 result = None
Yun Peng358cd882018-11-29 10:25:18 +01002972
Philipp Wollermann639c0452019-01-03 11:23:54 +01002973 # If current_commit is newer that last_green_commit, `git rev-list A..B` will output a bunch of
2974 # commits, otherwise the output should be empty.
Yun Peng358cd882018-11-29 10:25:18 +01002975 if not last_green_commit or result:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002976 execute_command(
Philipp Wollermann76a7eac2020-02-17 18:29:52 +01002977 [
2978 "echo %s | %s -h 'Cache-Control: no-store' cp - %s"
2979 % (current_commit, gsutil_command(), last_green_commit_url)
2980 ],
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002981 shell=True,
2982 )
Yun Peng358cd882018-11-29 10:25:18 +01002983 else:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002984 eprint(
2985 "Updating abandoned: last green commit (%s) is not older than current commit (%s)."
2986 % (last_green_commit, current_commit)
2987 )
2988
Yun Peng358cd882018-11-29 10:25:18 +01002989
Florian Weikert35906542019-04-01 11:53:53 +02002990def try_update_last_green_downstream_commit():
2991 last_green_commit_url = bazelci_last_green_downstream_commit_url()
2992 update_last_green_commit_if_newer(last_green_commit_url)
2993
2994
Jakob Buchgraber76381e02018-02-19 16:19:56 +01002995def latest_generation_and_build_number():
Philipp Wollermannce986af2019-07-18 14:46:05 +02002996 generation = None
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002997 output = None
Philipp Wollermannce986af2019-07-18 14:46:05 +02002998 for attempt in range(5):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002999 output = subprocess.check_output(
mai93f04f9482020-10-20 17:22:30 +02003000 [gsutil_command(), "stat", bazelci_latest_build_metadata_url()], env=os.environ
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003001 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003002 match = re.search("Generation:[ ]*([0-9]+)", output.decode("utf-8"))
3003 if not match:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02003004 raise BuildkiteException("Couldn't parse generation. gsutil output format changed?")
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003005 generation = match.group(1)
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01003006
Philipp Wollermannff39ef52018-02-21 14:18:52 +01003007 match = re.search(r"Hash \(md5\):[ ]*([^\s]+)", output.decode("utf-8"))
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003008 if not match:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02003009 raise BuildkiteException("Couldn't parse md5 hash. gsutil output format changed?")
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003010 expected_md5hash = base64.b64decode(match.group(1))
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01003011
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003012 output = subprocess.check_output(
mai93f04f9482020-10-20 17:22:30 +02003013 [gsutil_command(), "cat", bazelci_latest_build_metadata_url()], env=os.environ
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003014 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003015 hasher = hashlib.md5()
3016 hasher.update(output)
3017 actual_md5hash = hasher.digest()
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01003018
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003019 if expected_md5hash == actual_md5hash:
3020 break
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003021 info = json.loads(output.decode("utf-8"))
Philipp Wollermannce986af2019-07-18 14:46:05 +02003022 return generation, info["build_number"]
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01003023
Jakob Buchgraber699aece2018-02-19 12:49:30 +01003024
Jakob Buchgraber88083fd2018-02-18 17:23:35 +01003025def sha256_hexdigest(filename):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003026 sha256 = hashlib.sha256()
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02003027 with open(filename, "rb") as f:
3028 for block in iter(lambda: f.read(65536), b""):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003029 sha256.update(block)
3030 return sha256.hexdigest()
Jakob Buchgraber699aece2018-02-19 12:49:30 +01003031
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01003032
Philipp Wollermann02955272019-04-18 18:00:48 +02003033def upload_bazel_binaries():
3034 """
3035 Uploads all Bazel binaries to a deterministic URL based on the current Git commit.
3036
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04003037 Returns maps of platform names to sha256 hashes of the corresponding bazel and bazel_nojdk binaries.
Philipp Wollermann02955272019-04-18 18:00:48 +02003038 """
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04003039 bazel_hashes = {}
3040 bazel_nojdk_hashes = {}
Philipp Wollermannbdd4bf92019-06-06 14:43:50 +02003041 for platform_name, platform in PLATFORMS.items():
Philipp Wollermann783d1672019-06-06 13:35:30 +02003042 if not should_publish_binaries_for_platform(platform_name):
3043 continue
Jakob Buchgraberb13a9a82018-03-27 18:37:09 +02003044 tmpdir = tempfile.mkdtemp()
3045 try:
Philipp Wollermann783d1672019-06-06 13:35:30 +02003046 bazel_binary_path = download_bazel_binary(tmpdir, platform_name)
3047 # One platform that we build on can generate binaries for multiple platforms, e.g.
Philipp Wollermannf4aabb72019-06-25 15:59:00 +02003048 # the centos7 platform generates binaries for the "centos7" platform, but also
Philipp Wollermann783d1672019-06-06 13:35:30 +02003049 # for the generic "linux" platform.
3050 for target_platform_name in platform["publish_binary"]:
3051 execute_command(
3052 [
3053 gsutil_command(),
3054 "cp",
3055 bazel_binary_path,
3056 bazelci_builds_gs_url(target_platform_name, os.environ["BUILDKITE_COMMIT"]),
3057 ]
3058 )
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04003059 bazel_hashes[target_platform_name] = sha256_hexdigest(bazel_binary_path)
3060
3061 # Also publish bazel_nojdk binaries.
3062 bazel_nojdk_binary_path = download_bazel_nojdk_binary(tmpdir, platform_name)
3063 for target_platform_name in platform["publish_binary"]:
3064 execute_command(
3065 [
3066 gsutil_command(),
3067 "cp",
3068 bazel_nojdk_binary_path,
Florian Weikertdb832a02020-11-19 19:14:48 +01003069 bazelci_builds_nojdk_gs_url(
3070 target_platform_name, os.environ["BUILDKITE_COMMIT"]
3071 ),
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04003072 ]
3073 )
3074 bazel_nojdk_hashes[target_platform_name] = sha256_hexdigest(bazel_nojdk_binary_path)
Philipp Wollermann30f314d2021-06-11 10:51:39 +02003075 except subprocess.CalledProcessError as e:
3076 # If we're not on the main branch, we're probably building a custom one-off binary and
3077 # ignore failures for individual platforms (it's possible that we didn't build binaries
3078 # for all platforms).
3079 if not current_branch_is_main_branch():
3080 eprint(
3081 "Ignoring failure to download and publish Bazel binary for platform {}: {}".format(
3082 platform_name, e
3083 )
3084 )
3085 else:
3086 raise e
Jakob Buchgraberb13a9a82018-03-27 18:37:09 +02003087 finally:
3088 shutil.rmtree(tmpdir)
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04003089 return bazel_hashes, bazel_nojdk_hashes
Philipp Wollermann02955272019-04-18 18:00:48 +02003090
3091
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04003092def try_publish_binaries(bazel_hashes, bazel_nojdk_hashes, build_number, expected_generation):
Philipp Wollermann02955272019-04-18 18:00:48 +02003093 """
3094 Uploads the info.json file that contains information about the latest Bazel commit that was
3095 successfully built on CI.
3096 """
3097 now = datetime.datetime.now()
3098 git_commit = os.environ["BUILDKITE_COMMIT"]
3099 info = {
3100 "build_number": build_number,
3101 "build_time": now.strftime("%d-%m-%Y %H:%M"),
3102 "git_commit": git_commit,
3103 "platforms": {},
3104 }
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04003105 for platform, sha256 in bazel_hashes.items():
Philipp Wollermann02955272019-04-18 18:00:48 +02003106 info["platforms"][platform] = {
3107 "url": bazelci_builds_download_url(platform, git_commit),
Philipp Wollermann783d1672019-06-06 13:35:30 +02003108 "sha256": sha256,
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04003109 "nojdk_url": bazelci_builds_nojdk_download_url(platform, git_commit),
3110 "nojdk_sha256": bazel_nojdk_hashes[platform],
Philipp Wollermann02955272019-04-18 18:00:48 +02003111 }
Jakob Buchgraberb13a9a82018-03-27 18:37:09 +02003112 tmpdir = tempfile.mkdtemp()
3113 try:
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003114 info_file = os.path.join(tmpdir, "info.json")
3115 with open(info_file, mode="w", encoding="utf-8") as fp:
Jakob Buchgraber609a20e2018-02-25 17:06:51 +01003116 json.dump(info, fp, indent=2, sort_keys=True)
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003117
3118 try:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003119 execute_command(
3120 [
3121 gsutil_command(),
3122 "-h",
3123 "x-goog-if-generation-match:" + expected_generation,
3124 "-h",
3125 "Content-Type:application/json",
3126 "cp",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003127 info_file,
mai93f04f9482020-10-20 17:22:30 +02003128 bazelci_latest_build_metadata_url(),
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003129 ]
3130 )
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003131 except subprocess.CalledProcessError:
3132 raise BinaryUploadRaceException()
mai93f04f9482020-10-20 17:22:30 +02003133
3134 execute_command(
3135 [
3136 gsutil_command(),
3137 "cp",
3138 bazelci_latest_build_metadata_url(),
3139 bazelci_builds_metadata_url(git_commit),
3140 ]
3141 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003142 finally:
Philipp Wollermann3e1a7712018-02-19 17:34:24 +01003143 shutil.rmtree(tmpdir)
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01003144
3145
Jakob Buchgraber76381e02018-02-19 16:19:56 +01003146def publish_binaries():
Philipp Wollermanndb024862018-02-19 17:16:56 +01003147 """
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003148 Publish Bazel binaries to GCS.
Philipp Wollermanndb024862018-02-19 17:16:56 +01003149 """
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003150 current_build_number = os.environ.get("BUILDKITE_BUILD_NUMBER", None)
3151 if not current_build_number:
3152 raise BuildkiteException("Not running inside Buildkite")
3153 current_build_number = int(current_build_number)
3154
Philipp Wollermann02955272019-04-18 18:00:48 +02003155 # Upload the Bazel binaries for this commit.
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04003156 bazel_hashes, bazel_nojdk_hashes = upload_bazel_binaries()
Philipp Wollermann02955272019-04-18 18:00:48 +02003157
3158 # Try to update the info.json with data about our build. This will fail (expectedly) if we're
Philipp Wollermann1b5ecdc2021-06-10 21:52:55 +02003159 # not the latest build. Only do this if we're building binaries from the main branch to avoid
3160 # accidentally publishing a custom debug build as the "latest" Bazel binary.
3161 if current_branch_is_main_branch():
3162 for _ in range(5):
3163 latest_generation, latest_build_number = latest_generation_and_build_number()
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01003164
Philipp Wollermann1b5ecdc2021-06-10 21:52:55 +02003165 if current_build_number <= latest_build_number:
3166 eprint(
3167 (
3168 "Current build '{0}' is not newer than latest published '{1}'. "
3169 + "Skipping publishing of binaries."
3170 ).format(current_build_number, latest_build_number)
3171 )
3172 break
3173
3174 try:
3175 try_publish_binaries(
3176 bazel_hashes, bazel_nojdk_hashes, current_build_number, latest_generation
3177 )
3178 except BinaryUploadRaceException:
3179 # Retry.
3180 continue
3181
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003182 eprint(
Philipp Wollermann1b5ecdc2021-06-10 21:52:55 +02003183 "Successfully updated '{0}' to binaries from build {1}.".format(
3184 bazelci_latest_build_metadata_url(), current_build_number
3185 )
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003186 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003187 break
Philipp Wollermann1b5ecdc2021-06-10 21:52:55 +02003188 else:
3189 raise BuildkiteException("Could not publish binaries, ran out of attempts.")
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003190
Philipp Wollermann3c8b8512019-07-16 15:28:03 +02003191
Philipp Wollermann639c0452019-01-03 11:23:54 +01003192# This is so that multiline python strings are represented as YAML
3193# block strings.
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01003194def str_presenter(dumper, data):
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003195 if len(data.splitlines()) > 1: # check for multiline string
3196 return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
3197 return dumper.represent_scalar("tag:yaml.org,2002:str", data)
3198
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01003199
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003200def main(argv=None):
3201 if argv is None:
Yun Peng20d45602018-10-18 13:27:05 +02003202 argv = sys.argv[1:]
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003203
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01003204 yaml.add_representer(str, str_presenter)
3205
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02003206 parser = argparse.ArgumentParser(description="Bazel Continuous Integration Script")
Florian Weikert944209b2019-05-10 12:41:48 +02003207 parser.add_argument("--script", type=str)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01003208
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003209 subparsers = parser.add_subparsers(dest="subparsers_name")
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003210
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01003211 bazel_publish_binaries_pipeline = subparsers.add_parser("bazel_publish_binaries_pipeline")
3212 bazel_publish_binaries_pipeline.add_argument("--file_config", type=str)
Jakob Buchgraber08e8e402018-03-20 19:22:07 +01003213 bazel_publish_binaries_pipeline.add_argument("--http_config", type=str)
3214 bazel_publish_binaries_pipeline.add_argument("--git_repository", type=str)
3215
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01003216 bazel_downstream_pipeline = subparsers.add_parser("bazel_downstream_pipeline")
3217 bazel_downstream_pipeline.add_argument("--file_config", type=str)
3218 bazel_downstream_pipeline.add_argument("--http_config", type=str)
3219 bazel_downstream_pipeline.add_argument("--git_repository", type=str)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003220 bazel_downstream_pipeline.add_argument(
3221 "--test_incompatible_flags", type=bool, nargs="?", const=True
3222 )
3223 bazel_downstream_pipeline.add_argument(
3224 "--test_disabled_projects", type=bool, nargs="?", const=True
3225 )
Florian Weikert60661912019-12-18 15:17:10 +01003226 bazel_downstream_pipeline.add_argument("--notify", type=bool, nargs="?", const=True)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01003227
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003228 project_pipeline = subparsers.add_parser("project_pipeline")
3229 project_pipeline.add_argument("--project_name", type=str)
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01003230 project_pipeline.add_argument("--file_config", type=str)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003231 project_pipeline.add_argument("--http_config", type=str)
3232 project_pipeline.add_argument("--git_repository", type=str)
Jakob Buchgraber66ba4fe2018-06-22 15:04:14 +02003233 project_pipeline.add_argument("--monitor_flaky_tests", type=bool, nargs="?", const=True)
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02003234 project_pipeline.add_argument("--use_but", type=bool, nargs="?", const=True)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003235 project_pipeline.add_argument("--incompatible_flag", type=str, action="append")
Florian Weikert60661912019-12-18 15:17:10 +01003236 project_pipeline.add_argument("--notify", type=bool, nargs="?", const=True)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01003237
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003238 runner = subparsers.add_parser("runner")
Florian Weikert843d7a02019-02-03 17:24:50 +01003239 runner.add_argument("--task", action="store", type=str, default="")
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01003240 runner.add_argument("--file_config", type=str)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003241 runner.add_argument("--http_config", type=str)
3242 runner.add_argument("--git_repository", type=str)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003243 runner.add_argument(
3244 "--git_commit", type=str, help="Reset the git repository to this commit after cloning it"
3245 )
3246 runner.add_argument(
Yun Peng5012a862021-09-16 16:35:43 +02003247 "--repo_location",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003248 type=str,
3249 help="Use an existing repository instead of cloning from github",
3250 )
3251 runner.add_argument(
Dan Halperinefda1192019-01-16 00:34:09 -08003252 "--use_bazel_at_commit", type=str, help="Use Bazel binary built at a specific commit"
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003253 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003254 runner.add_argument("--use_but", type=bool, nargs="?", const=True)
3255 runner.add_argument("--save_but", type=bool, nargs="?", const=True)
Yun Peng4d1d6542019-01-17 18:30:33 +01003256 runner.add_argument("--needs_clean", type=bool, nargs="?", const=True)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003257 runner.add_argument("--build_only", type=bool, nargs="?", const=True)
3258 runner.add_argument("--test_only", type=bool, nargs="?", const=True)
Jakob Buchgraber66ba4fe2018-06-22 15:04:14 +02003259 runner.add_argument("--monitor_flaky_tests", type=bool, nargs="?", const=True)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003260 runner.add_argument("--incompatible_flag", type=str, action="append")
Jakob Buchgraberc340f582018-06-22 13:48:33 +02003261
Philipp Wollermannce986af2019-07-18 14:46:05 +02003262 subparsers.add_parser("publish_binaries")
3263 subparsers.add_parser("try_update_last_green_commit")
3264 subparsers.add_parser("try_update_last_green_downstream_commit")
Yun Peng358cd882018-11-29 10:25:18 +01003265
Yun Peng20d45602018-10-18 13:27:05 +02003266 args = parser.parse_args(argv)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01003267
Florian Weikert944209b2019-05-10 12:41:48 +02003268 if args.script:
3269 global SCRIPT_URL
3270 SCRIPT_URL = args.script
3271
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003272 try:
Jakob Buchgraber08e8e402018-03-20 19:22:07 +01003273 if args.subparsers_name == "bazel_publish_binaries_pipeline":
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01003274 configs = fetch_configs(args.http_config, args.file_config)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003275 print_bazel_publish_binaries_pipeline(
Florian Weikert843d7a02019-02-03 17:24:50 +01003276 task_configs=configs.get("tasks", None),
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003277 http_config=args.http_config,
3278 file_config=args.file_config,
3279 )
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01003280 elif args.subparsers_name == "bazel_downstream_pipeline":
3281 configs = fetch_configs(args.http_config, args.file_config)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003282 print_bazel_downstream_pipeline(
Florian Weikert843d7a02019-02-03 17:24:50 +01003283 task_configs=configs.get("tasks", None),
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003284 http_config=args.http_config,
3285 file_config=args.file_config,
3286 test_incompatible_flags=args.test_incompatible_flags,
3287 test_disabled_projects=args.test_disabled_projects,
Florian Weikert60661912019-12-18 15:17:10 +01003288 notify=args.notify,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003289 )
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003290 elif args.subparsers_name == "project_pipeline":
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01003291 configs = fetch_configs(args.http_config, args.file_config)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003292 print_project_pipeline(
Florian Weikertf20ae6f2019-01-16 14:32:09 +01003293 configs=configs,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003294 project_name=args.project_name,
3295 http_config=args.http_config,
3296 file_config=args.file_config,
3297 git_repository=args.git_repository,
3298 monitor_flaky_tests=args.monitor_flaky_tests,
3299 use_but=args.use_but,
3300 incompatible_flags=args.incompatible_flag,
Florian Weikert60661912019-12-18 15:17:10 +01003301 notify=args.notify,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003302 )
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003303 elif args.subparsers_name == "runner":
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01003304 configs = fetch_configs(args.http_config, args.file_config)
Florian Weikert843d7a02019-02-03 17:24:50 +01003305 tasks = configs.get("tasks", {})
3306 task_config = tasks.get(args.task)
3307 if not task_config:
3308 raise BuildkiteException(
3309 "No such task '{}' in configuration. Available: {}".format(
3310 args.task, ", ".join(tasks)
3311 )
3312 )
3313
3314 platform = get_platform_for_task(args.task, task_config)
3315
Yun Pengdb76f842021-08-30 18:39:38 +02003316 # The value of `BUILDKITE_MESSAGE` defaults to the commit message, which can be too large
3317 # on Windows, therefore we truncate the value to 1000 characters.
3318 # See https://github.com/bazelbuild/continuous-integration/issues/1218
3319 if "BUILDKITE_MESSAGE" in os.environ:
3320 os.environ["BUILDKITE_MESSAGE"] = os.environ["BUILDKITE_MESSAGE"][:1000]
3321
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003322 execute_commands(
Florian Weikertc8642af2019-02-03 23:58:51 +01003323 task_config=task_config,
Florian Weikert843d7a02019-02-03 17:24:50 +01003324 platform=platform,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003325 git_repository=args.git_repository,
3326 git_commit=args.git_commit,
Yun Peng5012a862021-09-16 16:35:43 +02003327 repo_location=args.repo_location,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003328 use_bazel_at_commit=args.use_bazel_at_commit,
3329 use_but=args.use_but,
3330 save_but=args.save_but,
Yun Peng4d1d6542019-01-17 18:30:33 +01003331 needs_clean=args.needs_clean,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003332 build_only=args.build_only,
3333 test_only=args.test_only,
3334 monitor_flaky_tests=args.monitor_flaky_tests,
3335 incompatible_flags=args.incompatible_flag,
Florian Weikertc8642af2019-02-03 23:58:51 +01003336 bazel_version=task_config.get("bazel") or configs.get("bazel"),
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003337 )
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003338 elif args.subparsers_name == "publish_binaries":
3339 publish_binaries()
Yun Peng358cd882018-11-29 10:25:18 +01003340 elif args.subparsers_name == "try_update_last_green_commit":
Florian Weikert35906542019-04-01 11:53:53 +02003341 # Update the last green commit of a project pipeline
Yun Peng358cd882018-11-29 10:25:18 +01003342 try_update_last_green_commit()
Florian Weikert35906542019-04-01 11:53:53 +02003343 elif args.subparsers_name == "try_update_last_green_downstream_commit":
3344 # Update the last green commit of the downstream pipeline
3345 try_update_last_green_downstream_commit()
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003346 else:
3347 parser.print_help()
3348 return 2
3349 except BuildkiteException as e:
3350 eprint(str(e))
3351 return 1
3352 return 0
3353
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01003354
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003355if __name__ == "__main__":
3356 sys.exit(main())