blob: 7d88a7a11a428bd2131cb7b8387354709a19f344 [file] [log] [blame]
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01001#!/usr/bin/env python3
2#
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01003# Copyright 2018 The Bazel Authors. All rights reserved.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010017import argparse
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +010018import base64
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010019import codecs
Jakob Buchgraber12807052018-02-25 17:04:56 +010020import datetime
Chi Wang6357efe2020-08-25 16:23:38 +080021import glob
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +010022import hashlib
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010023import json
Jakob Buchgraber6db0f262018-02-17 15:45:54 +010024import multiprocessing
Philipp Wollermann0a04cf32018-02-21 17:07:22 +010025import os
Philipp Wollermanndcaddd92018-02-21 14:13:43 +010026import os.path
Jakob Buchgraber257693b2018-02-20 00:03:56 +010027import random
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010028import re
Yun Peng9337bb32020-02-28 13:31:29 +010029import requests
Philipp Wollermanndcaddd92018-02-21 14:13:43 +010030from shutil import copyfile
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010031import shutil
Philipp Wollermanndcaddd92018-02-21 14:13:43 +010032import stat
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010033import subprocess
34import sys
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010035import tempfile
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +020036import threading
Philipp Wollermanne1318eb2018-08-13 15:08:01 +020037import time
Philipp Wollermannce986af2019-07-18 14:46:05 +020038import urllib.error
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010039import urllib.request
Philipp Wollermanne1318eb2018-08-13 15:08:01 +020040import uuid
Jakob Buchgraber25bb50f2018-02-22 18:06:21 +010041import yaml
Philipp Wollermannc030f2e2018-02-21 17:02:19 +010042from urllib.request import url2pathname
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010043from urllib.parse import urlparse
Philipp Wollermanndcaddd92018-02-21 14:13:43 +010044
45# Initialize the random number generator.
46random.seed()
47
Philipp Wollermanne67eec42019-05-24 15:18:20 +020048BUILDKITE_ORG = os.environ["BUILDKITE_ORGANIZATION_SLUG"]
49THIS_IS_PRODUCTION = BUILDKITE_ORG == "bazel-untrusted"
50THIS_IS_TESTING = BUILDKITE_ORG == "bazel-testing"
51THIS_IS_TRUSTED = BUILDKITE_ORG == "bazel-trusted"
52THIS_IS_SPARTA = True
53
54CLOUD_PROJECT = "bazel-public" if THIS_IS_TRUSTED else "bazel-untrusted"
55
56GITHUB_BRANCH = {"bazel": "master", "bazel-trusted": "master", "bazel-testing": "testing"}[
57 BUILDKITE_ORG
58]
59
60SCRIPT_URL = "https://raw.githubusercontent.com/bazelbuild/continuous-integration/{}/buildkite/bazelci.py?{}".format(
61 GITHUB_BRANCH, int(time.time())
Philipp Wollermannc05ac682019-01-19 12:37:28 +010062)
Jakob Buchgraber95e3d572018-02-21 18:48:49 +010063
Philipp Wollermanne67eec42019-05-24 15:18:20 +020064INCOMPATIBLE_FLAG_VERBOSE_FAILURES_URL = "https://raw.githubusercontent.com/bazelbuild/continuous-integration/{}/buildkite/incompatible_flag_verbose_failures.py?{}".format(
65 GITHUB_BRANCH, int(time.time())
66)
67
68AGGREGATE_INCOMPATIBLE_TEST_RESULT_URL = "https://raw.githubusercontent.com/bazelbuild/continuous-integration/{}/buildkite/aggregate_incompatible_flags_test_result.py?{}".format(
69 GITHUB_BRANCH, int(time.time())
70)
71
72EMERGENCY_FILE_URL = "https://raw.githubusercontent.com/bazelbuild/continuous-integration/{}/buildkite/emergency.yml?{}".format(
73 GITHUB_BRANCH, int(time.time())
74)
75
76FLAKY_TESTS_BUCKET = {
77 "bazel-testing": "gs://bazel-testing-buildkite-stats/flaky-tests-bep/",
78 "bazel-trusted": "gs://bazel-buildkite-stats/flaky-tests-bep/",
79 "bazel": "gs://bazel-buildkite-stats/flaky-tests-bep/",
80}[BUILDKITE_ORG]
81
Chi Wangb2b65682020-08-27 10:36:15 +080082KZIPS_BUCKET = {
83 "bazel-testing": "gs://bazel-kzips-testing/",
84 "bazel-trusted": "gs://bazel-kzips/",
85 "bazel": "gs://bazel-kzips/",
86}[BUILDKITE_ORG]
87
Florian Weikert797787b2019-12-19 15:33:07 +010088# Projects can opt out of receiving GitHub issues from --notify by adding `"do_not_notify": True` to their respective downstream entry.
Philipp Wollermanne67eec42019-05-24 15:18:20 +020089DOWNSTREAM_PROJECTS_PRODUCTION = {
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +010090 "Android Studio Plugin": {
91 "git_repository": "https://github.com/bazelbuild/intellij.git",
92 "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/master/.bazelci/android-studio.yml",
93 "pipeline_slug": "android-studio-plugin",
94 },
Yun Peng996efad2018-11-27 17:19:44 +010095 "Android Testing": {
96 "git_repository": "https://github.com/googlesamples/android-testing.git",
Jingwenbde72602018-12-13 10:57:43 -050097 "http_config": "https://raw.githubusercontent.com/googlesamples/android-testing/master/bazelci/buildkite-pipeline.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +010098 "pipeline_slug": "android-testing",
Yun Peng996efad2018-11-27 17:19:44 +010099 },
Yun Peng8910fa32019-01-03 08:58:16 +0100100 "Bazel": {
101 "git_repository": "https://github.com/bazelbuild/bazel.git",
102 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel/master/.bazelci/postsubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100103 "pipeline_slug": "bazel-bazel",
Yun Peng8910fa32019-01-03 08:58:16 +0100104 },
Tobias Werthd848eca2019-05-14 15:08:35 +0200105 "Bazel Bench": {
106 "git_repository": "https://github.com/bazelbuild/bazel-bench.git",
joeleba92ffec82019-05-22 14:50:15 +0200107 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-bench/master/.bazelci/postsubmit.yml",
Tobias Werthd848eca2019-05-14 15:08:35 +0200108 "pipeline_slug": "bazel-bench",
109 },
Philipp Wollermannfefcbf42019-05-28 14:28:40 +0200110 "Bazel Codelabs": {
111 "git_repository": "https://github.com/bazelbuild/codelabs.git",
112 "http_config": "https://raw.githubusercontent.com/bazelbuild/codelabs/master/.bazelci/presubmit.yml",
113 "pipeline_slug": "bazel-codelabs",
Philipp Wollermannfefcbf42019-05-28 14:28:40 +0200114 },
Jinfce9b302019-08-08 15:18:26 -0400115 "Bazel Examples": {
116 "git_repository": "https://github.com/bazelbuild/examples.git",
117 "http_config": "https://raw.githubusercontent.com/bazelbuild/examples/master/.bazelci/presubmit.yml",
118 "pipeline_slug": "bazel-bazel-examples",
119 },
Florian Weikert4b3ec672019-08-14 19:05:12 +0200120 "Bazel Federation": {
121 "git_repository": "https://github.com/bazelbuild/bazel-federation.git",
122 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-federation/master/.bazelci/presubmit.yml",
123 "pipeline_slug": "bazel-federation",
124 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100125 "Bazel Remote Cache": {
126 "git_repository": "https://github.com/buchgr/bazel-remote.git",
127 "http_config": "https://raw.githubusercontent.com/buchgr/bazel-remote/master/.bazelci/presubmit.yml",
128 "pipeline_slug": "bazel-remote-cache",
Yun Peng996efad2018-11-27 17:19:44 +0100129 },
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200130 "Bazel integration testing": {
131 "git_repository": "https://github.com/bazelbuild/bazel-integration-testing.git",
132 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-integration-testing/master/.bazelci/presubmit.yml",
133 "pipeline_slug": "bazel-integration-testing",
134 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100135 "Bazel skylib": {
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200136 "git_repository": "https://github.com/bazelbuild/bazel-skylib.git",
Yun Peng996efad2018-11-27 17:19:44 +0100137 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-skylib/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100138 "pipeline_slug": "bazel-skylib",
Yun Peng667750b2020-02-20 14:06:43 +0100139 "owned_by_bazel": True,
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200140 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100141 "Bazel toolchains": {
142 "git_repository": "https://github.com/bazelbuild/bazel-toolchains.git",
143 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-toolchains/master/.bazelci/presubmit.yml",
144 "pipeline_slug": "bazel-toolchains",
145 },
146 "Bazel watcher": {
147 "git_repository": "https://github.com/bazelbuild/bazel-watcher.git",
148 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-watcher/master/.bazelci/presubmit.yml",
149 "pipeline_slug": "bazel-watcher",
150 },
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200151 "Bazelisk": {
152 "git_repository": "https://github.com/bazelbuild/bazelisk.git",
153 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazelisk/master/.bazelci/config.yml",
154 "pipeline_slug": "bazelisk",
155 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100156 "Buildfarm": {
157 "git_repository": "https://github.com/bazelbuild/bazel-buildfarm.git",
158 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-buildfarm/master/.bazelci/presubmit.yml",
159 "pipeline_slug": "buildfarm-male-farmer",
160 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100161 "Buildtools": {
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200162 "git_repository": "https://github.com/bazelbuild/buildtools.git",
Yun Peng996efad2018-11-27 17:19:44 +0100163 "http_config": "https://raw.githubusercontent.com/bazelbuild/buildtools/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100164 "pipeline_slug": "buildtools",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200165 },
Yun Peng39a42582018-11-09 10:59:47 +0100166 "CLion Plugin": {
167 "git_repository": "https://github.com/bazelbuild/intellij.git",
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100168 "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/master/.bazelci/clion.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100169 "pipeline_slug": "clion-plugin",
Yun Peng39a42582018-11-09 10:59:47 +0100170 },
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200171 "Cartographer": {
172 "git_repository": "https://github.com/googlecartographer/cartographer.git",
173 "http_config": "https://raw.githubusercontent.com/googlecartographer/cartographer/master/.bazelci/presubmit.yml",
174 "pipeline_slug": "cartographer",
175 },
Philipp Wollermannee850782019-02-05 22:56:04 +0100176 "Cloud Robotics Core": {
Stefan Sauerb4dd3f92019-02-05 22:44:28 +0100177 "git_repository": "https://github.com/googlecloudrobotics/core.git",
178 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/cloud-robotics-postsubmit.yml",
179 "pipeline_slug": "cloud-robotics-core",
180 },
Keith Smiley3b0ba602019-05-15 04:42:19 -0700181 "Envoy": {
182 "git_repository": "https://github.com/envoyproxy/envoy.git",
183 "http_config": "https://raw.githubusercontent.com/envoyproxy/envoy/master/.bazelci/presubmit.yml",
184 "pipeline_slug": "envoy",
185 },
Florian Weikert1fe28b72019-07-02 12:47:55 +0200186 "FlatBuffers": {
187 "git_repository": "https://github.com/google/flatbuffers.git",
188 "http_config": "https://raw.githubusercontent.com/google/flatbuffers/master/.bazelci/presubmit.yml",
189 "pipeline_slug": "flatbuffers",
190 },
Philipp Wollermannf3750fa2019-05-21 17:11:59 +0200191 "Flogger": {
192 "git_repository": "https://github.com/google/flogger.git",
193 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/flogger.yml",
194 "pipeline_slug": "flogger",
195 },
Marcel Hlopkoc8840772018-10-23 12:51:46 +0200196 "Gerrit": {
197 "git_repository": "https://gerrit.googlesource.com/gerrit.git",
Yun Peng996efad2018-11-27 17:19:44 +0100198 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/gerrit-postsubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100199 "pipeline_slug": "gerrit",
Marcel Hlopkoc8840772018-10-23 12:51:46 +0200200 },
Yun Pengd6622022018-11-05 13:10:26 +0100201 "Google Logging": {
202 "git_repository": "https://github.com/google/glog.git",
Yun Peng996efad2018-11-27 17:19:44 +0100203 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/glog-postsubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100204 "pipeline_slug": "google-logging",
Yun Pengd6622022018-11-05 13:10:26 +0100205 },
Yun Peng9586db52018-11-02 10:48:40 +0100206 "IntelliJ Plugin": {
207 "git_repository": "https://github.com/bazelbuild/intellij.git",
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100208 "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/master/.bazelci/intellij.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100209 "pipeline_slug": "intellij-plugin",
Yun Peng9586db52018-11-02 10:48:40 +0100210 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100211 "IntelliJ Plugin Aspect": {
212 "git_repository": "https://github.com/bazelbuild/intellij.git",
213 "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/master/.bazelci/aspect.yml",
214 "pipeline_slug": "intellij-plugin-aspect",
215 },
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200216 "Kythe": {
217 "git_repository": "https://github.com/kythe/kythe.git",
218 "http_config": "https://raw.githubusercontent.com/kythe/kythe/master/.bazelci/presubmit.yml",
219 "pipeline_slug": "kythe",
220 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100221 "Protobuf": {
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200222 "git_repository": "https://github.com/google/protobuf.git",
Yun Peng996efad2018-11-27 17:19:44 +0100223 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/protobuf-postsubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100224 "pipeline_slug": "protobuf",
Yun Peng667750b2020-02-20 14:06:43 +0100225 "owned_by_bazel": True,
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200226 },
Laurent Le Brunf6326d62020-07-28 18:24:10 +0200227 "Stardoc": {
228 "git_repository": "https://github.com/bazelbuild/stardoc.git",
229 "http_config": "https://raw.githubusercontent.com/bazelbuild/stardoc/master/.bazelci/presubmit.yml",
230 "pipeline_slug": "stardoc",
Yun Peng667750b2020-02-20 14:06:43 +0100231 "owned_by_bazel": True,
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200232 },
233 "Subpar": {
234 "git_repository": "https://github.com/google/subpar.git",
235 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/subpar-postsubmit.yml",
236 "pipeline_slug": "subpar",
Yun Peng667750b2020-02-20 14:06:43 +0100237 "owned_by_bazel": True,
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200238 },
239 "TensorFlow": {
240 "git_repository": "https://github.com/tensorflow/tensorflow.git",
241 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/tensorflow-postsubmit.yml",
242 "pipeline_slug": "tensorflow",
243 },
244 "Tulsi": {
245 "git_repository": "https://github.com/bazelbuild/tulsi.git",
246 "http_config": "https://raw.githubusercontent.com/bazelbuild/tulsi/master/.bazelci/presubmit.yml",
247 "pipeline_slug": "tulsi-bazel-darwin",
248 },
249 "re2": {
250 "git_repository": "https://github.com/google/re2.git",
251 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/re2-postsubmit.yml",
252 "pipeline_slug": "re2",
253 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100254 "rules_android": {
255 "git_repository": "https://github.com/bazelbuild/rules_android.git",
256 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_android/master/.bazelci/postsubmit.yml",
257 "pipeline_slug": "rules-android",
Yun Pengc0575c82020-06-03 11:27:45 +0200258 "disabled_reason": "https://github.com/bazelbuild/rules_android/issues/15",
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100259 },
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200260 "rules_appengine": {
261 "git_repository": "https://github.com/bazelbuild/rules_appengine.git",
Yun Peng996efad2018-11-27 17:19:44 +0100262 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_appengine/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100263 "pipeline_slug": "rules-appengine-appengine",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200264 },
Yun Peng809f27b2018-11-13 10:15:39 +0100265 "rules_apple": {
266 "git_repository": "https://github.com/bazelbuild/rules_apple.git",
Yun Peng996efad2018-11-27 17:19:44 +0100267 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_apple/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100268 "pipeline_slug": "rules-apple-darwin",
Yun Peng809f27b2018-11-13 10:15:39 +0100269 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100270 "rules_cc": {
271 "git_repository": "https://github.com/bazelbuild/rules_cc.git",
272 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_cc/master/.bazelci/presubmit.yml",
273 "pipeline_slug": "rules-cc",
Yun Peng667750b2020-02-20 14:06:43 +0100274 "owned_by_bazel": True,
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100275 },
Marcel Hlopko340dfd22018-10-19 11:33:01 +0200276 "rules_closure": {
277 "git_repository": "https://github.com/bazelbuild/rules_closure.git",
Yun Peng996efad2018-11-27 17:19:44 +0100278 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_closure/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100279 "pipeline_slug": "rules-closure-closure-compiler",
Yun Peng667750b2020-02-20 14:06:43 +0100280 "owned_by_bazel": True,
Marcel Hlopko340dfd22018-10-19 11:33:01 +0200281 },
Yun Peng51ce6692019-01-09 14:31:46 +0100282 "rules_d": {
283 "git_repository": "https://github.com/bazelbuild/rules_d.git",
284 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_d/master/.bazelci/presubmit.yml",
285 "pipeline_slug": "rules-d",
286 },
Yun Peng996efad2018-11-27 17:19:44 +0100287 "rules_docker": {
288 "git_repository": "https://github.com/bazelbuild/rules_docker.git",
289 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_docker/master/.bazelci/presubmit.yml",
Jakob Buchgrabera6a8ea82018-12-07 13:51:02 +0100290 "pipeline_slug": "rules-docker-docker",
Yun Peng996efad2018-11-27 17:19:44 +0100291 },
292 "rules_foreign_cc": {
293 "git_repository": "https://github.com/bazelbuild/rules_foreign_cc.git",
294 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_foreign_cc/master/.bazelci/config.yaml",
295 "pipeline_slug": "rules-foreign-cc",
Yun Peng667750b2020-02-20 14:06:43 +0100296 "owned_by_bazel": True,
Yun Peng996efad2018-11-27 17:19:44 +0100297 },
Xindb02c012018-11-07 14:10:54 -0500298 "rules_go": {
299 "git_repository": "https://github.com/bazelbuild/rules_go.git",
Yun Peng996efad2018-11-27 17:19:44 +0100300 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_go/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100301 "pipeline_slug": "rules-go-golang",
Yun Pengb7247ff2018-11-15 13:52:39 +0100302 },
Yun Peng7deea572018-11-05 10:47:45 +0100303 "rules_groovy": {
Yun Peng996efad2018-11-27 17:19:44 +0100304 "git_repository": "https://github.com/bazelbuild/rules_groovy.git",
305 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_groovy/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100306 "pipeline_slug": "rules-groovy",
Yun Peng996efad2018-11-27 17:19:44 +0100307 },
308 "rules_gwt": {
309 "git_repository": "https://github.com/bazelbuild/rules_gwt.git",
310 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_gwt/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100311 "pipeline_slug": "rules-gwt",
Philipp Wollermann2a160432019-09-19 15:57:28 +0200312 },
Florian Weikertff6444e2019-09-16 16:08:57 +0200313 "rules_haskell": {
314 "git_repository": "https://github.com/tweag/rules_haskell.git",
315 "http_config": "https://raw.githubusercontent.com/tweag/rules_haskell/master/.bazelci/presubmit.yml",
316 "pipeline_slug": "rules-haskell-haskell",
Philipp Wollermann2a160432019-09-19 15:57:28 +0200317 },
Yun Peng996efad2018-11-27 17:19:44 +0100318 "rules_jsonnet": {
319 "git_repository": "https://github.com/bazelbuild/rules_jsonnet.git",
320 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_jsonnet/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100321 "pipeline_slug": "rules-jsonnet",
Yun Peng996efad2018-11-27 17:19:44 +0100322 },
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200323 "rules_jvm_external": {
324 "git_repository": "https://github.com/bazelbuild/rules_jvm_external.git",
325 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_jvm_external/master/.bazelci/presubmit.yml",
326 "pipeline_slug": "rules-jvm-external",
Yun Peng667750b2020-02-20 14:06:43 +0100327 "owned_by_bazel": True,
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200328 },
329 "rules_jvm_external - examples": {
330 "git_repository": "https://github.com/bazelbuild/rules_jvm_external.git",
331 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_jvm_external/master/.bazelci/examples.yml",
332 "pipeline_slug": "rules-jvm-external-examples",
Yun Peng667750b2020-02-20 14:06:43 +0100333 "owned_by_bazel": True,
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200334 },
Yun Peng996efad2018-11-27 17:19:44 +0100335 "rules_k8s": {
336 "git_repository": "https://github.com/bazelbuild/rules_k8s.git",
337 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_k8s/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100338 "pipeline_slug": "rules-k8s-k8s",
Yun Peng70160012020-06-29 11:59:28 +0200339 "disabled_reason": "https://github.com/bazelbuild/rules_k8s/pull/580",
Yun Peng996efad2018-11-27 17:19:44 +0100340 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100341 "rules_kotlin": {
342 "git_repository": "https://github.com/bazelbuild/rules_kotlin.git",
343 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_kotlin/master/.bazelci/presubmit.yml",
344 "pipeline_slug": "rules-kotlin-kotlin",
345 },
Yun Penga5650e12018-11-14 10:16:06 +0100346 "rules_nodejs": {
347 "git_repository": "https://github.com/bazelbuild/rules_nodejs.git",
Yun Peng996efad2018-11-27 17:19:44 +0100348 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_nodejs/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100349 "pipeline_slug": "rules-nodejs-nodejs",
Yun Penga5650e12018-11-14 10:16:06 +0100350 },
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200351 "rules_perl": {
352 "git_repository": "https://github.com/bazelbuild/rules_perl.git",
Yun Peng996efad2018-11-27 17:19:44 +0100353 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_perl/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100354 "pipeline_slug": "rules-perl",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200355 },
Yannic6110b3c2019-08-12 15:09:37 +0000356 "rules_proto": {
357 "git_repository": "https://github.com/bazelbuild/rules_proto.git",
358 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_proto/master/.bazelci/presubmit.yml",
359 "pipeline_slug": "rules-proto",
Yun Peng667750b2020-02-20 14:06:43 +0100360 "owned_by_bazel": True,
Yannic6110b3c2019-08-12 15:09:37 +0000361 },
Yun Peng3d5a8a62018-11-19 11:42:01 +0100362 "rules_python": {
363 "git_repository": "https://github.com/bazelbuild/rules_python.git",
Yun Peng996efad2018-11-27 17:19:44 +0100364 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_python/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100365 "pipeline_slug": "rules-python-python",
Yun Peng667750b2020-02-20 14:06:43 +0100366 "owned_by_bazel": True,
Yun Peng3d5a8a62018-11-19 11:42:01 +0100367 },
Xindb02c012018-11-07 14:10:54 -0500368 "rules_rust": {
369 "git_repository": "https://github.com/bazelbuild/rules_rust.git",
Yun Peng996efad2018-11-27 17:19:44 +0100370 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_rust/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100371 "pipeline_slug": "rules-rust-rustlang",
Xindb02c012018-11-07 14:10:54 -0500372 },
Yun Pengca62fff2018-10-31 11:22:03 +0100373 "rules_sass": {
374 "git_repository": "https://github.com/bazelbuild/rules_sass.git",
Yun Peng996efad2018-11-27 17:19:44 +0100375 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_sass/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100376 "pipeline_slug": "rules-sass",
Yun Pengca62fff2018-10-31 11:22:03 +0100377 },
Xindb02c012018-11-07 14:10:54 -0500378 "rules_scala": {
379 "git_repository": "https://github.com/bazelbuild/rules_scala.git",
Yun Peng996efad2018-11-27 17:19:44 +0100380 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_scala/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100381 "pipeline_slug": "rules-scala-scala",
Xindb02c012018-11-07 14:10:54 -0500382 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100383 "rules_swift": {
384 "git_repository": "https://github.com/bazelbuild/rules_swift.git",
385 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_swift/master/.bazelci/presubmit.yml",
386 "pipeline_slug": "rules-swift-swift",
Florian Weikertbaa683f2019-12-27 18:09:58 +0100387 "do_not_notify": "https://github.com/bazelbuild/continuous-integration/issues/915",
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100388 },
Yun Peng996efad2018-11-27 17:19:44 +0100389 "rules_typescript": {
390 "git_repository": "https://github.com/bazelbuild/rules_typescript.git",
391 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_typescript/master/.bazelci/presubmit.yml",
Jakob Buchgrabera6a8ea82018-12-07 13:51:02 +0100392 "pipeline_slug": "rules-typescript-typescript",
Yun Peng996efad2018-11-27 17:19:44 +0100393 },
394 "rules_webtesting": {
395 "git_repository": "https://github.com/bazelbuild/rules_webtesting.git",
Yun Pengc2fab332019-01-04 10:53:49 +0100396 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_webtesting/master/.bazelci/presubmit.yml",
Yun Peng996efad2018-11-27 17:19:44 +0100397 "pipeline_slug": "rules-webtesting-saucelabs",
Yun Peng996efad2018-11-27 17:19:44 +0100398 },
Philipp Wollermann389acd82019-05-21 17:41:48 +0200399 "upb": {
400 "git_repository": "https://github.com/protocolbuffers/upb.git",
401 "http_config": "https://raw.githubusercontent.com/protocolbuffers/upb/master/.bazelci/presubmit.yml",
402 "pipeline_slug": "upb",
403 },
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200404}
405
Philipp Wollermanne67eec42019-05-24 15:18:20 +0200406DOWNSTREAM_PROJECTS_TESTING = {
Philipp Wollermannbed211d2019-06-07 11:38:59 +0200407 "Bazel": DOWNSTREAM_PROJECTS_PRODUCTION["Bazel"],
408 "Bazelisk": DOWNSTREAM_PROJECTS_PRODUCTION["Bazelisk"],
Florian Weikert8fda2a72019-05-31 15:21:54 +0200409 "Federation": {
410 "git_repository": "https://github.com/fweikert/bazel-federation.git",
411 "http_config": "https://raw.githubusercontent.com/fweikert/bazel-federation/master/.bazelci/presubmit.yml",
412 "pipeline_slug": "bazel-federation",
413 },
Philipp Wollermannbed211d2019-06-07 11:38:59 +0200414 "rules_docker": DOWNSTREAM_PROJECTS_PRODUCTION["rules_docker"],
415 "rules_go": DOWNSTREAM_PROJECTS_PRODUCTION["rules_go"],
416 "rules_groovy": DOWNSTREAM_PROJECTS_PRODUCTION["rules_groovy"],
417 "rules_kotlin": DOWNSTREAM_PROJECTS_PRODUCTION["rules_kotlin"],
418 "rules_nodejs": DOWNSTREAM_PROJECTS_PRODUCTION["rules_nodejs"],
419 "rules_rust": DOWNSTREAM_PROJECTS_PRODUCTION["rules_rust"],
420 "rules_scala": DOWNSTREAM_PROJECTS_PRODUCTION["rules_scala"],
Philipp Wollermanne67eec42019-05-24 15:18:20 +0200421}
422
423DOWNSTREAM_PROJECTS = {
424 "bazel-testing": DOWNSTREAM_PROJECTS_TESTING,
425 "bazel-trusted": {},
426 "bazel": DOWNSTREAM_PROJECTS_PRODUCTION,
427}[BUILDKITE_ORG]
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100428
Philipp Wollermann81a88412019-07-12 10:34:33 +0200429DOCKER_REGISTRY_PREFIX = {
430 "bazel-testing": "bazel-public/testing",
431 "bazel-trusted": "bazel-public",
432 "bazel": "bazel-public",
433}[BUILDKITE_ORG]
434
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200435# A map containing all supported platform names as keys, with the values being
436# the platform name in a human readable format, and a the buildkite-agent's
437# working directory.
438PLATFORMS = {
Philipp Wollermanneffcd6e2019-06-21 18:30:34 +0200439 "centos7": {
440 "name": "CentOS 7, Java 8",
441 "emoji-name": ":centos: 7 (Java 8)",
442 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermannf4aabb72019-06-25 15:59:00 +0200443 "publish_binary": ["ubuntu1404", "centos7", "linux"],
Philipp Wollermann5d6765d2020-02-17 17:12:02 +0100444 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/centos7-java8",
Philipp Wollermanneffcd6e2019-06-21 18:30:34 +0200445 "python": "python3.6",
446 },
Philipp Wollermann67fc3712019-06-12 15:39:21 +0200447 "debian10": {
448 "name": "Debian Buster, OpenJDK 11",
449 "emoji-name": ":debian: Buster (OpenJDK 11)",
450 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
451 "publish_binary": [],
Philipp Wollermann5d6765d2020-02-17 17:12:02 +0100452 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/debian10-java11",
Philipp Wollermann67fc3712019-06-12 15:39:21 +0200453 "python": "python3.7",
454 },
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200455 "ubuntu1604": {
Philipp Wollermanndb877332019-04-23 17:58:01 +0200456 "name": "Ubuntu 16.04, OpenJDK 8",
457 "emoji-name": ":ubuntu: 16.04 (OpenJDK 8)",
Philipp Wollermannd551bf62019-05-18 22:04:35 +0200458 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermannf4aabb72019-06-25 15:59:00 +0200459 "publish_binary": ["ubuntu1604"],
Philipp Wollermann5d6765d2020-02-17 17:12:02 +0100460 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu1604-java8",
Philipp Wollermann57b32682019-05-18 22:09:27 +0200461 "python": "python3.6",
Philipp Wollermann438ec242018-09-05 14:39:24 +0200462 },
463 "ubuntu1804": {
Philipp Wollermannf5a2feb2019-04-25 11:13:46 +0200464 "name": "Ubuntu 18.04, OpenJDK 11",
465 "emoji-name": ":ubuntu: 18.04 (OpenJDK 11)",
Philipp Wollermannd551bf62019-05-18 22:04:35 +0200466 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermann783d1672019-06-06 13:35:30 +0200467 "publish_binary": ["ubuntu1804"],
Philipp Wollermann5d6765d2020-02-17 17:12:02 +0100468 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu1804-java11",
Philipp Wollermann57b32682019-05-18 22:09:27 +0200469 "python": "python3.6",
Philipp Wollermann438ec242018-09-05 14:39:24 +0200470 },
471 "ubuntu1804_nojava": {
472 "name": "Ubuntu 18.04, no JDK",
473 "emoji-name": ":ubuntu: 18.04 (no JDK)",
Philipp Wollermannd551bf62019-05-18 22:04:35 +0200474 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermann783d1672019-06-06 13:35:30 +0200475 "publish_binary": [],
Philipp Wollermann5d6765d2020-02-17 17:12:02 +0100476 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu1804-nojava",
Philipp Wollermann57b32682019-05-18 22:09:27 +0200477 "python": "python3.6",
Philipp Wollermann438ec242018-09-05 14:39:24 +0200478 },
Mostyn Bramley-Mooreab4599e2020-06-23 20:31:01 +0200479 "ubuntu2004": {
480 "name": "Ubuntu 20.04, OpenJDK 11",
481 "emoji-name": ":ubuntu: 20.04 (OpenJDK 11)",
482 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermann55f72ac2020-09-21 22:22:05 +0200483 "publish_binary": [],
Mostyn Bramley-Mooreab4599e2020-06-23 20:31:01 +0200484 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu2004-java11",
485 "python": "python3.8",
486 },
487 "ubuntu2004_nojava": {
488 "name": "Ubuntu 20.04, no JDK",
489 "emoji-name": ":ubuntu: 20.04 (no JDK)",
490 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
491 "publish_binary": [],
492 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu2004-nojava",
493 "python": "python3.8",
494 },
Chi Wang6357efe2020-08-25 16:23:38 +0800495 "kythe_ubuntu2004": {
496 "name": "Kythe (Ubuntu 20.04, OpenJDK 11)",
497 "emoji-name": "Kythe (:ubuntu: 20.04, OpenJDK 11)",
498 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
499 "publish_binary": [],
500 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu2004-java11-kythe",
501 "python": "python3.8",
502 },
Philipp Wollermann438ec242018-09-05 14:39:24 +0200503 "macos": {
Philipp Wollermanndb877332019-04-23 17:58:01 +0200504 "name": "macOS, OpenJDK 8",
505 "emoji-name": ":darwin: (OpenJDK 8)",
Philipp Wollermann51147bf2019-05-08 15:50:10 +0200506 "downstream-root": "/Users/buildkite/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermann783d1672019-06-06 13:35:30 +0200507 "publish_binary": ["macos"],
Philipp Wollermann7a185322019-05-18 22:15:48 +0200508 "queue": "macos",
Philipp Wollermann57b32682019-05-18 22:09:27 +0200509 "python": "python3.7",
Philipp Wollermann438ec242018-09-05 14:39:24 +0200510 },
511 "windows": {
Philipp Wollermanndb877332019-04-23 17:58:01 +0200512 "name": "Windows, OpenJDK 8",
513 "emoji-name": ":windows: (OpenJDK 8)",
Philipp Wollermannd5ab3d92020-02-05 16:55:13 +0100514 "downstream-root": "c:/b/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermann783d1672019-06-06 13:35:30 +0200515 "publish_binary": ["windows"],
Philipp Wollermann7a185322019-05-18 22:15:48 +0200516 "queue": "windows",
Philipp Wollermann57b32682019-05-18 22:09:27 +0200517 "python": "python.exe",
Philipp Wollermann438ec242018-09-05 14:39:24 +0200518 },
519 "rbe_ubuntu1604": {
Philipp Wollermanndb877332019-04-23 17:58:01 +0200520 "name": "RBE (Ubuntu 16.04, OpenJDK 8)",
Jakob Buchgraber1f37fbd2019-07-17 17:08:28 +0200521 "emoji-name": "RBE (:ubuntu: 16.04, OpenJDK 8)",
Philipp Wollermannd551bf62019-05-18 22:04:35 +0200522 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermann783d1672019-06-06 13:35:30 +0200523 "publish_binary": [],
Philipp Wollermann5d6765d2020-02-17 17:12:02 +0100524 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu1604-java8",
Philipp Wollermann57b32682019-05-18 22:09:27 +0200525 "python": "python3.6",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100526 },
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200527}
528
Philipp Wollermannfce92bf2019-05-22 15:14:32 +0200529BUILDIFIER_DOCKER_IMAGE = "gcr.io/bazel-public/buildifier"
Florian Weikertf20ae6f2019-01-16 14:32:09 +0100530
Philipp Wollermann1403d2c2019-01-10 13:15:51 +0100531# The platform used for various steps (e.g. stuff that formerly ran on the "pipeline" workers).
532DEFAULT_PLATFORM = "ubuntu1804"
533
Philipp Wollermannf4aabb72019-06-25 15:59:00 +0200534# In order to test that "the one Linux binary" that we build for our official releases actually
535# works on all Linux distributions that we test on, we use the Linux binary built on our official
536# release platform for all Linux downstream tests.
537LINUX_BINARY_PLATFORM = "centos7"
538
Philipp Wollermannbe707ac2020-09-21 19:21:50 +0200539DEFAULT_XCODE_VERSION = "11.7"
Philipp Wollermann380f1e62019-04-12 16:45:27 +0200540XCODE_VERSION_REGEX = re.compile(r"^\d+\.\d+(\.\d+)?$")
Philipp Wollermann06e56972020-01-29 14:46:41 +0100541XCODE_VERSION_OVERRIDES = {
542 "10.2.1": "10.3",
543 "11.2": "11.2.1",
544 "11.3": "11.3.1",
545}
Philipp Wollermann380f1e62019-04-12 16:45:27 +0200546
Philipp Wollermann1403d2c2019-01-10 13:15:51 +0100547ENCRYPTED_SAUCELABS_TOKEN = """
Philipp Wollermanna4722b42019-01-10 16:50:13 +0100548CiQAry63sOlZtTNtuOT5DAOLkum0rGof+DOweppZY1aOWbat8zwSTQAL7Hu+rgHSOr6P4S1cu4YG
549/I1BHsWaOANqUgFt6ip9/CUGGJ1qggsPGXPrmhSbSPqNAIAkpxYzabQ3mfSIObxeBmhKg2dlILA/
550EDql
Philipp Wollermann1403d2c2019-01-10 13:15:51 +0100551""".strip()
552
Florian Weikertc8642af2019-02-03 23:58:51 +0100553BUILD_LABEL_PATTERN = re.compile(r"^Build label: (\S+)$", re.MULTILINE)
554
Florian Weikert29cb7ec2019-03-07 14:52:18 +0100555BUILDIFIER_VERSION_ENV_VAR = "BUILDIFIER_VERSION"
556
Florian Weikert85208912019-03-07 17:08:39 +0100557BUILDIFIER_WARNINGS_ENV_VAR = "BUILDIFIER_WARNINGS"
558
Florian Weikertde96a6f2019-03-07 14:57:50 +0100559BUILDIFIER_STEP_NAME = "Buildifier"
560
Florian Weikert5f5d3cb2019-04-15 15:36:27 +0200561SKIP_TASKS_ENV_VAR = "CI_SKIP_TASKS"
562
Philipp Wollermannce986af2019-07-18 14:46:05 +0200563CONFIG_FILE_EXTENSIONS = {".yml", ".yaml"}
Florian Weikert778251c2019-04-25 15:14:44 +0200564
Chi Wang6357efe2020-08-25 16:23:38 +0800565KYTHE_DIR = "/usr/local/kythe"
566
567INDEX_UPLOAD_POLICY_ALWAYS = "Always"
568
569INDEX_UPLOAD_POLICY_IF_BUILD_SUCCESS = "IfBuildSuccess"
570
571INDEX_UPLOAD_POLICY_NEVER = "Never"
Florian Weikert13215a82019-05-10 12:42:21 +0200572
Philipp Wollermanndcaddd92018-02-21 14:13:43 +0100573class BuildkiteException(Exception):
574 """
575 Raised whenever something goes wrong and we should exit with an error.
576 """
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100577
Philipp Wollermanndcaddd92018-02-21 14:13:43 +0100578 pass
579
580
581class BinaryUploadRaceException(Exception):
582 """
583 Raised when try_publish_binaries wasn't able to publish a set of binaries,
584 because the generation of the current file didn't match the expected value.
585 """
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100586
Philipp Wollermanndcaddd92018-02-21 14:13:43 +0100587 pass
588
589
Florian Weikerta0e74592019-03-07 11:56:12 +0100590class BuildkiteClient(object):
591
592 _ENCRYPTED_BUILDKITE_API_TOKEN = """
593CiQA4DEB9ldzC+E39KomywtqXfaQ86hhulgeDsicds2BuvbCYzsSUAAqwcvXZPh9IMWlwWh94J2F
594exosKKaWB0tSRJiPKnv2NPDfEqGul0ZwVjtWeASpugwxxKeLhFhPMcgHMPfndH6j2GEIY6nkKRbP
595uwoRMCwe
596""".strip()
597
Philipp Wollermanne67eec42019-05-24 15:18:20 +0200598 _ENCRYPTED_BUILDKITE_API_TESTING_TOKEN = """
599CiQAMTBkWjL1C+F5oon3+cC1vmum5+c1y5+96WQY44p0Lxd0PeASUQAy7iU0c6E3W5EOSFYfD5fA
600MWy/SHaMno1NQSUa4xDOl5yc2kizrtxPPVkX4x9pLNuGUY/xwAn2n1DdiUdWZNWlY1bX2C4ex65e
601P9w8kNhEbw==
602""".strip()
603
Florian Weikertde96a6f2019-03-07 14:57:50 +0100604 _BUILD_STATUS_URL_TEMPLATE = (
605 "https://api.buildkite.com/v2/organizations/{}/pipelines/{}/builds/{}"
606 )
Florian Weikerta0e74592019-03-07 11:56:12 +0100607
Yun Peng9337bb32020-02-28 13:31:29 +0100608 _NEW_BUILD_URL_TEMPLATE = (
609 "https://api.buildkite.com/v2/organizations/{}/pipelines/{}/builds"
610 )
611
612 _RETRY_JOB_URL_TEMPLATE = (
613 "https://api.buildkite.com/v2/organizations/{}/pipelines/{}/builds/{}/jobs/{}/retry"
614 )
615
Florian Weikerta0e74592019-03-07 11:56:12 +0100616 def __init__(self, org, pipeline):
617 self._org = org
618 self._pipeline = pipeline
619 self._token = self._get_buildkite_token()
620
Yun Peng9337bb32020-02-28 13:31:29 +0100621
Florian Weikerta0e74592019-03-07 11:56:12 +0100622 def _get_buildkite_token(self):
Florian Weikert849afb22019-12-14 12:22:29 -0800623 return decrypt_token(
624 encrypted_token=self._ENCRYPTED_BUILDKITE_API_TESTING_TOKEN
625 if THIS_IS_TESTING
626 else self._ENCRYPTED_BUILDKITE_API_TOKEN,
627 kms_key="buildkite-testing-api-token"
628 if THIS_IS_TESTING
629 else "buildkite-untrusted-api-token",
Florian Weikerta0e74592019-03-07 11:56:12 +0100630 )
631
Yun Peng9337bb32020-02-28 13:31:29 +0100632
633 def _open_url(self, url, params = []):
Florian Weikert60661912019-12-18 15:17:10 +0100634 try:
Yun Peng9337bb32020-02-28 13:31:29 +0100635 params_str = "".join("&{}={}".format(k, v) for k, v in params)
Florian Weikert60661912019-12-18 15:17:10 +0100636 return (
Yun Peng9337bb32020-02-28 13:31:29 +0100637 urllib.request.urlopen("{}?access_token={}{}".format(url, self._token, params_str))
Florian Weikert60661912019-12-18 15:17:10 +0100638 .read()
Yun Pengdbedc122020-02-28 13:32:04 +0100639 .decode("utf-8", "ignore")
Florian Weikert60661912019-12-18 15:17:10 +0100640 )
641 except urllib.error.HTTPError as ex:
642 raise BuildkiteException("Failed to open {}: {} - {}".format(url, ex.code, ex.reason))
Florian Weikerta0e74592019-03-07 11:56:12 +0100643
Yun Peng9337bb32020-02-28 13:31:29 +0100644
Florian Weikerta0e74592019-03-07 11:56:12 +0100645 def get_build_info(self, build_number):
Yun Peng9337bb32020-02-28 13:31:29 +0100646 """Get build info for a pipeline with a given build number
647 See https://buildkite.com/docs/apis/rest-api/builds#get-a-build
648
649 Parameters
650 ----------
651 build_number : the build number
652
653 Returns
654 -------
655 dict
656 the metadata for the build
657 """
Florian Weikerta0e74592019-03-07 11:56:12 +0100658 url = self._BUILD_STATUS_URL_TEMPLATE.format(self._org, self._pipeline, build_number)
659 output = self._open_url(url)
660 return json.loads(output)
661
Yun Peng9337bb32020-02-28 13:31:29 +0100662
663 def get_build_info_list(self, params):
664 """Get a list of build infos for this pipeline
665 See https://buildkite.com/docs/apis/rest-api/builds#list-builds-for-a-pipeline
666
667 Parameters
668 ----------
669 params : the parameters to filter the result
670
671 Returns
672 -------
673 list of dict
674 the metadata for a list of builds
675 """
676 url = self._BUILD_STATUS_URL_TEMPLATE.format(self._org, self._pipeline, "")
677 output = self._open_url(url, params)
678 return json.loads(output)
679
680
Florian Weikerta0e74592019-03-07 11:56:12 +0100681 def get_build_log(self, job):
682 return self._open_url(job["raw_log_url"])
683
684
Yun Peng9337bb32020-02-28 13:31:29 +0100685 @staticmethod
686 def _check_response(response, expected_status_code):
687 if response.status_code != expected_status_code:
688 eprint("Exit code:", response.status_code)
689 eprint("Response:\n", response.text)
690 response.raise_for_status()
691
692
693 def trigger_new_build(self, commit, message = None, env = {}):
694 """Trigger a new build at a given commit and return the build metadata.
695 See https://buildkite.com/docs/apis/rest-api/builds#create-a-build
696
697 Parameters
698 ----------
699 commit : the commit we want to build at
700 message : the message we should as the build titile
701 env : (optional) the environment variables to set
702
703 Returns
704 -------
705 dict
706 the metadata for the build
707 """
708 url = self._NEW_BUILD_URL_TEMPLATE.format(self._org, self._pipeline)
709 data = {
710 "commit": commit,
711 "branch": "master",
712 "message": message if message else f"Trigger build at {commit}",
713 "env": env,
714 }
715 response = requests.post(url + "?access_token=" + self._token, json = data)
716 BuildkiteClient._check_response(response, requests.codes.created)
717 return json.loads(response.text)
718
719
720 def trigger_job_retry(self, build_number, job_id):
721 """Trigger a job retry and return the job metadata.
722 See https://buildkite.com/docs/apis/rest-api/jobs#retry-a-job
723
724 Parameters
725 ----------
726 build_number : the number of the build we want to retry
727 job_id : the id of the job we want to retry
728
729 Returns
730 -------
731 dict
732 the metadata for the job
733 """
734 url = self._RETRY_JOB_URL_TEMPLATE.format(self._org, self._pipeline, build_number, job_id)
735 response = requests.put(url + "?access_token=" + self._token)
736 BuildkiteClient._check_response(response, requests.codes.ok)
737 return json.loads(response.text)
738
739
740 def wait_job_to_finish(self, build_number, job_id, interval_time=30, logger=None):
741 """Wait a job to finish and return the job metadata
742
743 Parameters
744 ----------
745 build_number : the number of the build we want to wait
746 job_id : the id of the job we want to wait
747 interval_time : (optional) the interval time to check the build status, default to 30s
748 logger : (optional) a logger to report progress
749
750 Returns
751 -------
752 dict
753 the latest metadata for the job
754 """
755 t = 0
756 build_info = self.get_build_info(build_number)
757 while True:
758 for job in build_info["jobs"]:
759 if job["id"] == job_id:
760 state = job["state"]
761 if state != "scheduled" and state != "running" and state != "assigned":
762 return job
763 break
764 else:
765 raise BuildkiteException(f"job id {job_id} doesn't exist in build " + build_info["web_url"])
766 url = build_info["web_url"]
767 if logger:
768 logger.log(f"Waiting for {url}, waited {t} seconds...")
769 time.sleep(interval_time)
770 t += interval_time
771 build_info = self.get_build_info(build_number)
772
773
774 def wait_build_to_finish(self, build_number, interval_time=30, logger=None):
775 """Wait a build to finish and return the build metadata
776
777 Parameters
778 ----------
779 build_number : the number of the build we want to wait
780 interval_time : (optional) the interval time to check the build status, default to 30s
781 logger : (optional) a logger to report progress
782
783 Returns
784 -------
785 dict
786 the latest metadata for the build
787 """
788 t = 0
789 build_info = self.get_build_info(build_number)
790 while build_info["state"] == "scheduled" or build_info["state"] == "running":
791 url = build_info["web_url"]
792 if logger:
793 logger.log(f"Waiting for {url}, waited {t} seconds...")
794 time.sleep(interval_time)
795 t += interval_time
796 build_info = self.get_build_info(build_number)
797 return build_info
798
799
Florian Weikert849afb22019-12-14 12:22:29 -0800800def decrypt_token(encrypted_token, kms_key):
801 return (
802 subprocess.check_output(
803 [
804 gcloud_command(),
805 "kms",
806 "decrypt",
807 "--project",
808 "bazel-untrusted",
809 "--location",
810 "global",
811 "--keyring",
812 "buildkite",
813 "--key",
814 kms_key,
815 "--ciphertext-file",
816 "-",
817 "--plaintext-file",
818 "-",
819 ],
820 input=base64.b64decode(encrypted_token),
821 env=os.environ,
822 )
823 .decode("utf-8")
824 .strip()
825 )
826
827
Philipp Wollermanndcaddd92018-02-21 14:13:43 +0100828def eprint(*args, **kwargs):
829 """
830 Print to stderr and flush (just in case).
831 """
832 print(*args, flush=True, file=sys.stderr, **kwargs)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +0100833
834
Jakob Buchgraber9f153542018-02-27 10:56:04 +0100835def is_windows():
Jakob Buchgraber09048fa2018-02-27 11:39:39 +0100836 return os.name == "nt"
Jakob Buchgraber9f153542018-02-27 10:56:04 +0100837
Jakob Buchgrabere6de16b2018-02-28 12:42:12 +0100838
Jakob Buchgraber9f153542018-02-27 10:56:04 +0100839def gsutil_command():
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200840 return "gsutil.cmd" if is_windows() else "gsutil"
Jakob Buchgraber9f153542018-02-27 10:56:04 +0100841
Jakob Buchgrabere6de16b2018-02-28 12:42:12 +0100842
Jakob Buchgraber9f153542018-02-27 10:56:04 +0100843def gcloud_command():
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200844 return "gcloud.cmd" if is_windows() else "gcloud"
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +0100845
Jakob Buchgrabere6de16b2018-02-28 12:42:12 +0100846
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +0100847def downstream_projects_root(platform):
Philipp Wollermann51147bf2019-05-08 15:50:10 +0200848 downstream_root = os.path.expandvars(PLATFORMS[platform]["downstream-root"])
Philipp Wollermannd5ab3d92020-02-05 16:55:13 +0100849 if platform == "windows" and os.path.exists("d:/b"):
850 # If this is a Windows machine with a local SSD, the build directory is
851 # on drive D.
852 downstream_root = downstream_root.replace("c:/b/", "d:/b/")
Philipp Wollermann51147bf2019-05-08 15:50:10 +0200853 if not os.path.exists(downstream_root):
854 os.makedirs(downstream_root)
855 return downstream_root
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +0100856
857
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +0100858def fetch_configs(http_url, file_config):
Philipp Wollermanndb024862018-02-19 17:16:56 +0100859 """
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +0100860 If specified fetches the build configuration from file_config or http_url, else tries to
Jakob Buchgraber25bb50f2018-02-22 18:06:21 +0100861 read it from .bazelci/presubmit.yml.
Philipp Wollermann598b4a42018-02-19 17:03:36 +0100862 Returns the json configuration as a python data structure.
Philipp Wollermanndb024862018-02-19 17:16:56 +0100863 """
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +0100864 if file_config is not None and http_url is not None:
865 raise BuildkiteException("file_config and http_url cannot be set at the same time")
866
Florian Weikertc8b3ed22019-05-31 16:14:12 +0200867 return load_config(http_url, file_config)
868
869
870def load_config(http_url, file_config, allow_imports=True):
Florian Weikertc8b3ed22019-05-31 16:14:12 +0200871 if http_url:
872 config = load_remote_yaml_file(http_url)
873 else:
874 file_config = file_config or ".bazelci/presubmit.yml"
875 with open(file_config, "r") as fd:
876 config = yaml.safe_load(fd)
877
Florian Weikert843d7a02019-02-03 17:24:50 +0100878 # Legacy mode means that there is exactly one task per platform (e.g. ubuntu1604_nojdk),
879 # which means that we can get away with using the platform name as task ID.
880 # No other updates are needed since get_platform_for_task() falls back to using the
881 # task ID as platform if there is no explicit "platforms" field.
882 if "platforms" in config:
883 config["tasks"] = config.pop("platforms")
884
Florian Weikertc8b3ed22019-05-31 16:14:12 +0200885 if "tasks" not in config:
886 config["tasks"] = {}
887
888 imports = config.pop("imports", None)
889 if imports:
890 if not allow_imports:
891 raise BuildkiteException("Nested imports are not allowed")
892
893 for i in imports:
894 imported_tasks = load_imported_tasks(i, http_url, file_config)
895 config["tasks"].update(imported_tasks)
896
Florian Weikert843d7a02019-02-03 17:24:50 +0100897 return config
898
899
Florian Weikert13215a82019-05-10 12:42:21 +0200900def load_remote_yaml_file(http_url):
901 with urllib.request.urlopen(http_url) as resp:
902 reader = codecs.getreader("utf-8")
Philipp Wollermannd00107e2019-05-18 23:50:59 +0200903 return yaml.safe_load(reader(resp))
Florian Weikert13215a82019-05-10 12:42:21 +0200904
905
Florian Weikertc8b3ed22019-05-31 16:14:12 +0200906def load_imported_tasks(import_name, http_url, file_config):
907 if "/" in import_name:
908 raise BuildkiteException("Invalid import '%s'" % import_name)
909
910 old_path = http_url or file_config
911 new_path = "%s%s" % (old_path[: old_path.rfind("/") + 1], import_name)
912 if http_url:
913 http_url = new_path
914 else:
915 file_config = new_path
916
917 imported_config = load_config(http_url=http_url, file_config=file_config, allow_imports=False)
918
919 namespace = import_name.partition(".")[0]
920 tasks = {}
921 for task_name, task_config in imported_config["tasks"].items():
Florian Weikert61f29b82019-08-12 16:56:56 +0200922 fix_imported_task_platform(task_name, task_config)
923 fix_imported_task_name(namespace, task_config)
924 fix_imported_task_working_directory(namespace, task_config)
Florian Weikertc8b3ed22019-05-31 16:14:12 +0200925 tasks["%s_%s" % (namespace, task_name)] = task_config
926
927 return tasks
928
929
Florian Weikert61f29b82019-08-12 16:56:56 +0200930def fix_imported_task_platform(task_name, task_config):
931 if "platform" not in task_config:
932 task_config["platform"] = task_name
933
934
935def fix_imported_task_name(namespace, task_config):
936 old_name = task_config.get("name")
937 task_config["name"] = "%s (%s)" % (namespace, old_name) if old_name else namespace
938
939
940def fix_imported_task_working_directory(namespace, task_config):
941 old_dir = task_config.get("working_directory")
942 task_config["working_directory"] = os.path.join(namespace, old_dir) if old_dir else namespace
943
944
Jakob Buchgraber3120f7a2018-02-18 13:28:02 +0100945def print_collapsed_group(name):
Jakob Buchgraber5c9b13d2018-02-21 22:28:14 +0100946 eprint("\n\n--- {0}\n\n".format(name))
Jakob Buchgraber3120f7a2018-02-18 13:28:02 +0100947
Jakob Buchgraber9c83de72018-02-18 15:32:44 +0100948
Jakob Buchgraber3120f7a2018-02-18 13:28:02 +0100949def print_expanded_group(name):
Jakob Buchgraber5c9b13d2018-02-21 22:28:14 +0100950 eprint("\n\n+++ {0}\n\n".format(name))
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +0100951
Jakob Buchgraber9c83de72018-02-18 15:32:44 +0100952
Yun Peng8975c6b2019-02-28 11:55:55 +0100953def use_bazelisk_migrate():
954 """
955 If USE_BAZELISK_MIGRATE is set, we use `bazelisk --migrate` to test incompatible flags.
956 """
Florian Weikertde96a6f2019-03-07 14:57:50 +0100957 return bool(os.environ.get("USE_BAZELISK_MIGRATE"))
Yun Peng8975c6b2019-02-28 11:55:55 +0100958
959
960def bazelisk_flags():
961 return ["--migrate"] if use_bazelisk_migrate() else []
962
963
Chi Wang6357efe2020-08-25 16:23:38 +0800964def calculate_flags(task_config, task_config_key, json_profile_key, tmpdir, test_env_vars):
965 include_json_profile = task_config.get("include_json_profile", [])
966
967 json_profile_flags = []
968 json_profile_out = None
969 if json_profile_key in include_json_profile:
970 json_profile_out = os.path.join(tmpdir, "{}.profile.gz".format(json_profile_key))
971 json_profile_flags = get_json_profile_flags(json_profile_out)
972
973 flags = task_config.get(task_config_key) or []
974 flags += json_profile_flags
975 # We have to add --test_env flags to `build`, too, otherwise Bazel
976 # discards its analysis cache between `build` and `test`.
977 if test_env_vars:
978 flags += ["--test_env={}".format(v) for v in test_env_vars]
979
980 return flags, json_profile_out
981
982
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100983def execute_commands(
Florian Weikertc8642af2019-02-03 23:58:51 +0100984 task_config,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100985 platform,
986 git_repository,
987 git_commit,
988 git_repo_location,
989 use_bazel_at_commit,
990 use_but,
991 save_but,
Yun Peng4d1d6542019-01-17 18:30:33 +0100992 needs_clean,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100993 build_only,
994 test_only,
995 monitor_flaky_tests,
996 incompatible_flags,
Florian Weikertc8642af2019-02-03 23:58:51 +0100997 bazel_version=None,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100998):
Yun Pengf50f7b72019-02-28 19:09:52 +0100999 # If we want to test incompatible flags, we ignore bazel_version and always use
1000 # the latest Bazel version through Bazelisk.
1001 if incompatible_flags:
1002 bazel_version = None
Florian Weikert13215a82019-05-10 12:42:21 +02001003 if not bazel_version:
1004 # The last good version of Bazel can be specified in an emergency file.
1005 # However, we only use last_good_bazel for pipelines that do not
1006 # explicitly specify a version of Bazel.
1007 try:
1008 emergency_settings = load_remote_yaml_file(EMERGENCY_FILE_URL)
1009 bazel_version = emergency_settings.get("last_good_bazel")
1010 except urllib.error.HTTPError:
1011 # Ignore this error. The Setup step will have already complained about
1012 # it by showing an error message.
1013 pass
Yun Pengf50f7b72019-02-28 19:09:52 +01001014
Jakob Buchgraberfb95a2f2018-02-22 11:46:25 +01001015 if build_only and test_only:
1016 raise BuildkiteException("build_only and test_only cannot be true at the same time")
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02001017
Yun Peng20d45602018-10-18 13:27:05 +02001018 if use_bazel_at_commit and use_but:
1019 raise BuildkiteException("use_bazel_at_commit cannot be set when use_but is true")
1020
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02001021 tmpdir = tempfile.mkdtemp()
1022 sc_process = None
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001023 try:
Philipp Wollermann380f1e62019-04-12 16:45:27 +02001024 if platform == "macos":
Florian Weikertee84c5c2019-05-28 11:21:51 +02001025 activate_xcode(task_config)
Philipp Wollermann380f1e62019-04-12 16:45:27 +02001026
Florian Weikert4ee0bed2019-02-21 18:03:00 +01001027 # If the CI worker runs Bazelisk, we need to forward all required env variables to the test.
1028 # Otherwise any integration test that invokes Bazel (=Bazelisk in this case) will fail.
Marcel Hlopko198328b2019-02-25 09:19:55 +01001029 test_env_vars = ["LocalAppData"] if platform == "windows" else ["HOME"]
Yun Peng376d2b32018-11-29 10:24:54 +01001030 if git_repo_location:
1031 os.chdir(git_repo_location)
1032 elif git_repository:
1033 clone_git_repository(git_repository, platform, git_commit)
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001034
Philipp Wollermannf4aabb72019-06-25 15:59:00 +02001035 # We use one binary for all Linux platforms (because we also just release one binary for all
1036 # Linux versions and we have to ensure that it works on all of them).
1037 binary_platform = platform if platform in ["macos", "windows"] else LINUX_BINARY_PLATFORM
1038
Yun Peng20d45602018-10-18 13:27:05 +02001039 if use_bazel_at_commit:
1040 print_collapsed_group(":gcloud: Downloading Bazel built at " + use_bazel_at_commit)
Philipp Wollermannf4aabb72019-06-25 15:59:00 +02001041 bazel_binary = download_bazel_binary_at_commit(
1042 tmpdir, binary_platform, use_bazel_at_commit
1043 )
Yun Pengf0a66e22019-10-14 12:45:42 +02001044 os.environ["USE_BAZEL_VERSION"] = bazel_binary
Philipp Wollermann639c0452019-01-03 11:23:54 +01001045 elif use_but:
Jakob Buchgraber92755d72018-02-22 15:33:37 +01001046 print_collapsed_group(":gcloud: Downloading Bazel Under Test")
Philipp Wollermannf4aabb72019-06-25 15:59:00 +02001047 bazel_binary = download_bazel_binary(tmpdir, binary_platform)
Yun Pengf0a66e22019-10-14 12:45:42 +02001048 os.environ["USE_BAZEL_VERSION"] = bazel_binary
Philipp Wollermann639c0452019-01-03 11:23:54 +01001049 else:
1050 bazel_binary = "bazel"
Florian Weikertc8642af2019-02-03 23:58:51 +01001051 if bazel_version:
Florian Weikertc8642af2019-02-03 23:58:51 +01001052 os.environ["USE_BAZEL_VERSION"] = bazel_version
Philipp Wollermann87b45252020-01-22 12:43:42 +01001053 if "USE_BAZEL_VERSION" in os.environ and not task_config.get(
1054 "skip_use_bazel_version_for_test", False
1055 ):
Yun Pengf0a66e22019-10-14 12:45:42 +02001056 # This will only work if the bazel binary in $PATH is actually a bazelisk binary
1057 # (https://github.com/bazelbuild/bazelisk).
1058 test_env_vars.append("USE_BAZEL_VERSION")
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001059
Philipp Wollermann5c7ea412019-05-24 15:26:57 +02001060 for key, value in task_config.get("environment", {}).items():
Philipp Wollermann4ad4aac2019-05-24 15:23:09 +02001061 # We have to explicitly convert the value to a string, because sometimes YAML tries to
1062 # be smart and converts strings like "true" and "false" to booleans.
1063 os.environ[key] = str(value)
Philipp Wollermann213ac9d2019-02-06 11:50:05 +01001064
Yun Peng366f04c2020-08-10 16:55:58 +02001065 # Set BAZELISK_SHUTDOWN to 1 when we use bazelisk --migrate on Windows.
1066 # This is a workaround for https://github.com/bazelbuild/continuous-integration/issues/1012
1067 if use_bazelisk_migrate() and platform == "windows":
1068 os.environ["BAZELISK_SHUTDOWN"] = "1"
1069
Florian Weikerta8c020b2019-08-12 16:56:38 +02001070 cmd_exec_func = execute_batch_commands if platform == "windows" else execute_shell_commands
1071 cmd_exec_func(task_config.get("setup", None))
1072
Philipp Wollermanna5aee2c2019-02-11 16:55:19 +01001073 # Allow the config to override the current working directory.
1074 required_prefix = os.getcwd()
1075 requested_working_dir = os.path.abspath(task_config.get("working_directory", ""))
1076 if os.path.commonpath([required_prefix, requested_working_dir]) != required_prefix:
1077 raise BuildkiteException("working_directory refers to a path outside the workspace")
1078 os.chdir(requested_working_dir)
1079
Florian Weikerte72d68a2019-03-08 18:56:33 +01001080 if platform == "windows":
1081 execute_batch_commands(task_config.get("batch_commands", None))
1082 else:
1083 execute_shell_commands(task_config.get("shell_commands", None))
1084
Florian Weikertc8642af2019-02-03 23:58:51 +01001085 bazel_version = print_bazel_version_info(bazel_binary, platform)
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001086
Yun Penga5a1ee02018-12-05 15:00:58 +01001087 print_environment_variables_info()
1088
Yun Pengd0217ed2018-11-30 14:51:11 +01001089 if incompatible_flags:
1090 print_expanded_group("Build and test with the following incompatible flags:")
1091 for flag in incompatible_flags:
1092 eprint(flag + "\n")
1093
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001094 execute_bazel_run(
Florian Weikertc8642af2019-02-03 23:58:51 +01001095 bazel_binary, platform, task_config.get("run_targets", None), incompatible_flags
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001096 )
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001097
Florian Weikertee84c5c2019-05-28 11:21:51 +02001098 if task_config.get("sauce"):
1099 sc_process = start_sauce_connect_proxy(platform, tmpdir)
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02001100
Yun Peng4d1d6542019-01-17 18:30:33 +01001101 if needs_clean:
Yun Pengea0359e2019-01-17 15:37:47 +01001102 execute_bazel_clean(bazel_binary, platform)
1103
Chi Wang6357efe2020-08-25 16:23:38 +08001104 build_targets, test_targets, index_targets = calculate_targets(
Florian Weikert736d06e2019-05-08 13:16:42 +02001105 task_config, platform, bazel_binary, build_only, test_only
1106 )
Florian Weikert736d06e2019-05-08 13:16:42 +02001107
1108 if build_targets:
Chi Wang6357efe2020-08-25 16:23:38 +08001109 build_flags, json_profile_out_build = calculate_flags(task_config, "build_flags", "build", tmpdir, test_env_vars)
joeleba76887952019-05-16 15:22:17 +02001110 try:
1111 execute_bazel_build(
1112 bazel_version,
1113 bazel_binary,
1114 platform,
Philipp Wollermanna81848e2020-01-30 10:48:48 +01001115 build_flags,
joeleba76887952019-05-16 15:22:17 +02001116 build_targets,
1117 None,
1118 incompatible_flags,
1119 )
1120 if save_but:
1121 upload_bazel_binary(platform)
1122 finally:
Chi Wang6357efe2020-08-25 16:23:38 +08001123 if json_profile_out_build:
Philipp Wollermann92cf51e2019-05-16 15:31:11 +02001124 upload_json_profile(json_profile_out_build, tmpdir)
Philipp Wollermann639c0452019-01-03 11:23:54 +01001125
Florian Weikert736d06e2019-05-08 13:16:42 +02001126 if test_targets:
Chi Wang6357efe2020-08-25 16:23:38 +08001127 test_flags, json_profile_out_test = calculate_flags(task_config, "test_flags", "test", tmpdir, test_env_vars)
Florian Weikert4901c662019-02-26 13:20:11 +01001128 if not is_windows():
1129 # On platforms that support sandboxing (Linux, MacOS) we have
1130 # to allow access to Bazelisk's cache directory.
1131 # However, the flag requires the directory to exist,
1132 # so we create it here in order to not crash when a test
1133 # does not invoke Bazelisk.
1134 bazelisk_cache_dir = get_bazelisk_cache_directory(platform)
1135 os.makedirs(bazelisk_cache_dir, mode=0o755, exist_ok=True)
1136 test_flags.append("--sandbox_writable_path={}".format(bazelisk_cache_dir))
Florian Weikert5b890332019-02-25 14:57:43 +01001137
Philipp Wollermannce986af2019-07-18 14:46:05 +02001138 test_bep_file = os.path.join(tmpdir, "test_bep.json")
1139 stop_request = threading.Event()
1140 upload_thread = threading.Thread(
1141 target=upload_test_logs_from_bep, args=(test_bep_file, tmpdir, stop_request)
1142 )
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01001143 try:
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001144 upload_thread.start()
1145 try:
1146 execute_bazel_test(
1147 bazel_version,
1148 bazel_binary,
1149 platform,
1150 test_flags,
1151 test_targets,
1152 test_bep_file,
1153 monitor_flaky_tests,
1154 incompatible_flags,
1155 )
1156 if monitor_flaky_tests:
1157 upload_bep_logs_for_flaky_tests(test_bep_file)
1158 finally:
Chi Wang6357efe2020-08-25 16:23:38 +08001159 if json_profile_out_test:
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001160 upload_json_profile(json_profile_out_test, tmpdir)
Philipp Wollermann639c0452019-01-03 11:23:54 +01001161 finally:
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001162 stop_request.set()
1163 upload_thread.join()
Chi Wang6357efe2020-08-25 16:23:38 +08001164
1165 if index_targets:
1166 index_flags, json_profile_out_index = calculate_flags(task_config, "index_flags", "index", tmpdir, test_env_vars)
1167 index_upload_policy = task_config.get("index_upload_policy", "IfBuildSuccess")
Chi Wangb2b65682020-08-27 10:36:15 +08001168 index_upload_gcs = task_config.get("index_upload_gcs", False)
Chi Wang6357efe2020-08-25 16:23:38 +08001169
1170 try:
1171 should_upload_kzip = True if index_upload_policy == INDEX_UPLOAD_POLICY_ALWAYS else False
1172 try:
1173 execute_bazel_build_with_kythe(
1174 bazel_version,
1175 bazel_binary,
1176 platform,
1177 index_flags,
1178 index_targets,
1179 None,
1180 incompatible_flags
1181 )
1182
1183 if index_upload_policy == INDEX_UPLOAD_POLICY_IF_BUILD_SUCCESS:
1184 should_upload_kzip = True
1185 except subprocess.CalledProcessError as e:
1186 # If not running with Always policy, raise the build error.
1187 if index_upload_policy != INDEX_UPLOAD_POLICY_ALWAYS:
1188 handle_bazel_failure(e, "build")
1189
1190 if should_upload_kzip:
1191 try:
Chi Wangb2b65682020-08-27 10:36:15 +08001192 merge_and_upload_kythe_kzip(platform, index_upload_gcs)
Chi Wang6357efe2020-08-25 16:23:38 +08001193 except subprocess.CalledProcessError:
1194 raise BuildkiteException("Failed to upload kythe kzip")
1195 finally:
1196 if json_profile_out_index:
1197 upload_json_profile(json_profile_out_index, tmpdir)
1198
1199
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001200 finally:
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001201 terminate_background_process(sc_process)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001202 if tmpdir:
1203 shutil.rmtree(tmpdir)
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01001204
Philipp Wollermann3c8b8512019-07-16 15:28:03 +02001205
Florian Weikertee84c5c2019-05-28 11:21:51 +02001206def activate_xcode(task_config):
1207 # Get the Xcode version from the config.
1208 xcode_version = task_config.get("xcode_version", DEFAULT_XCODE_VERSION)
1209 print_collapsed_group("Activating Xcode {}...".format(xcode_version))
1210
1211 # Ensure it's a valid version number.
1212 if not isinstance(xcode_version, str):
1213 raise BuildkiteException(
1214 "Version number '{}' is not a string. Did you forget to put it in quotes?".format(
1215 xcode_version
1216 )
1217 )
1218 if not XCODE_VERSION_REGEX.match(xcode_version):
1219 raise BuildkiteException(
1220 "Invalid Xcode version format '{}', must match the format X.Y[.Z].".format(
1221 xcode_version
1222 )
1223 )
1224
Philipp Wollermann06e56972020-01-29 14:46:41 +01001225 # This is used to replace e.g. 11.2 with 11.2.1 without having to update all configs.
1226 xcode_version = XCODE_VERSION_OVERRIDES.get(xcode_version, xcode_version)
1227
Florian Weikertee84c5c2019-05-28 11:21:51 +02001228 # Check that the selected Xcode version is actually installed on the host.
1229 xcode_path = "/Applications/Xcode{}.app".format(xcode_version)
1230 if not os.path.exists(xcode_path):
1231 raise BuildkiteException("Xcode not found at '{}'.".format(xcode_path))
1232
1233 # Now activate the specified Xcode version and let it install its required components.
1234 # The CI machines have a sudoers config that allows the 'buildkite' user to run exactly
1235 # these two commands, so don't change them without also modifying the file there.
1236 execute_command(["/usr/bin/sudo", "/usr/bin/xcode-select", "--switch", xcode_path])
1237 execute_command(["/usr/bin/sudo", "/usr/bin/xcodebuild", "-runFirstLaunch"])
1238
1239
Florian Weikert4901c662019-02-26 13:20:11 +01001240def get_bazelisk_cache_directory(platform):
1241 # The path relies on the behavior of Go's os.UserCacheDir()
1242 # and of the Go version of Bazelisk.
Philipp Wollermannce986af2019-07-18 14:46:05 +02001243 cache_dir = "Library/Caches" if platform == "macos" else ".cache"
1244 return os.path.join(os.environ.get("HOME"), cache_dir, "bazelisk")
Florian Weikert4901c662019-02-26 13:20:11 +01001245
Florian Weikert5b890332019-02-25 14:57:43 +01001246
Jakob Buchgraber5d6c7142018-02-21 20:16:51 +01001247def tests_with_status(bep_file, status):
Jakob Buchgraberc874cdf2019-07-16 16:27:41 +02001248 return set(label for label, _ in test_logs_for_status(bep_file, status=[status]))
Jakob Buchgraber257693b2018-02-20 00:03:56 +01001249
Jakob Buchgraber6104a432018-02-21 21:16:53 +01001250
Florian Weikertee84c5c2019-05-28 11:21:51 +02001251def start_sauce_connect_proxy(platform, tmpdir):
1252 print_collapsed_group(":saucelabs: Starting Sauce Connect Proxy")
1253 os.environ["SAUCE_USERNAME"] = "bazel_rules_webtesting"
1254 os.environ["SAUCE_ACCESS_KEY"] = saucelabs_token()
1255 os.environ["TUNNEL_IDENTIFIER"] = str(uuid.uuid4())
1256 os.environ["BUILD_TAG"] = str(uuid.uuid4())
1257 readyfile = os.path.join(tmpdir, "sc_is_ready")
1258 if platform == "windows":
Philipp Wollermann8a8c25a2019-08-23 12:56:36 +02001259 cmd = ["sauce-connect.exe", "-i", os.environ["TUNNEL_IDENTIFIER"], "-f", readyfile]
Florian Weikertee84c5c2019-05-28 11:21:51 +02001260 else:
1261 cmd = ["sc", "-i", os.environ["TUNNEL_IDENTIFIER"], "-f", readyfile]
1262 sc_process = execute_command_background(cmd)
1263 wait_start = time.time()
1264 while not os.path.exists(readyfile):
Philipp Wollermann8a8c25a2019-08-23 12:56:36 +02001265 if time.time() - wait_start > 60:
Florian Weikertee84c5c2019-05-28 11:21:51 +02001266 raise BuildkiteException(
Philipp Wollermann8a8c25a2019-08-23 12:56:36 +02001267 "Sauce Connect Proxy is still not ready after 60 seconds, aborting!"
Florian Weikertee84c5c2019-05-28 11:21:51 +02001268 )
1269 time.sleep(1)
1270 print("Sauce Connect Proxy is ready, continuing...")
1271 return sc_process
1272
1273
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01001274def saucelabs_token():
Florian Weikert60661912019-12-18 15:17:10 +01001275 return decrypt_token(encrypted_token=ENCRYPTED_SAUCELABS_TOKEN, kms_key="saucelabs-access-key")
Yun Pengb6b18862019-01-07 14:31:55 +01001276
1277
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01001278def is_pull_request():
Jakob Buchgraber67761d32018-02-21 19:00:21 +01001279 third_party_repo = os.getenv("BUILDKITE_PULL_REQUEST_REPO", "")
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01001280 return len(third_party_repo) > 0
1281
1282
Jakob Buchgraber02e07222018-02-19 15:05:56 +01001283def has_flaky_tests(bep_file):
Jakob Buchgraberc874cdf2019-07-16 16:27:41 +02001284 return len(test_logs_for_status(bep_file, status=["FLAKY"])) > 0
Jakob Buchgraber02e07222018-02-19 15:05:56 +01001285
1286
Yun Penge3cf12d2018-12-05 15:01:09 +01001287def print_bazel_version_info(bazel_binary, platform):
Jakob Buchgraber99c4bbb2018-02-22 11:59:31 +01001288 print_collapsed_group(":information_source: Bazel Info")
Philipp Wollermannf13804b2019-02-05 21:08:30 +01001289 version_output = execute_command_and_get_output(
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001290 [bazel_binary]
1291 + common_startup_flags(platform)
1292 + ["--nomaster_bazelrc", "--bazelrc=/dev/null", "version"]
1293 )
1294 execute_command(
1295 [bazel_binary]
1296 + common_startup_flags(platform)
1297 + ["--nomaster_bazelrc", "--bazelrc=/dev/null", "info"]
1298 )
Jakob Buchgraber7e690a72018-02-18 13:22:15 +01001299
Florian Weikertc8642af2019-02-03 23:58:51 +01001300 match = BUILD_LABEL_PATTERN.search(version_output)
1301 return match.group(1) if match else "unreleased binary"
1302
Jakob Buchgraber7e690a72018-02-18 13:22:15 +01001303
Yun Penga5a1ee02018-12-05 15:00:58 +01001304def print_environment_variables_info():
1305 print_collapsed_group(":information_source: Environment Variables")
1306 for key, value in os.environ.items():
1307 eprint("%s=(%s)" % (key, value))
1308
1309
Jakob Buchgraber426399e2018-03-20 19:45:46 +01001310def upload_bazel_binary(platform):
Jakob Buchgraber7d1d3bb2018-02-21 22:38:22 +01001311 print_collapsed_group(":gcloud: Uploading Bazel Under Test")
Jakob Buchgraber426399e2018-03-20 19:45:46 +01001312 if platform == "windows":
Philipp Wollermann10183212020-02-04 21:54:14 +01001313 binary_dir = r"bazel-bin\src"
1314 binary_name = r"bazel.exe"
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04001315 binary_nojdk_name = r"bazel_nojdk.exe"
Philipp Wollermann10183212020-02-04 21:54:14 +01001316 else:
1317 binary_dir = "bazel-bin/src"
1318 binary_name = "bazel"
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04001319 binary_nojdk_name = "bazel_nojdk"
Philipp Wollermann10183212020-02-04 21:54:14 +01001320 execute_command(["buildkite-agent", "artifact", "upload", binary_name], cwd=binary_dir)
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04001321 execute_command(["buildkite-agent", "artifact", "upload", binary_nojdk_name], cwd=binary_dir)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001322
1323
Chi Wangb2b65682020-08-27 10:36:15 +08001324def merge_and_upload_kythe_kzip(platform, index_upload_gcs):
Chi Wang6357efe2020-08-25 16:23:38 +08001325 print_collapsed_group(":gcloud: Uploading kythe kzip")
1326
1327 kzips = glob.glob("bazel-out/*/extra_actions/**/*.kzip", recursive=True)
1328
Chi Wang6357efe2020-08-25 16:23:38 +08001329 build_number = os.getenv("BUILDKITE_BUILD_NUMBER")
Chi Wangb2b65682020-08-27 10:36:15 +08001330 git_commit = os.getenv("BUILDKITE_COMMIT")
1331 final_kzip_name = "{}-{}-{}.kzip".format(build_number, platform, git_commit)
Chi Wang6357efe2020-08-25 16:23:38 +08001332
Chi Wangb2b65682020-08-27 10:36:15 +08001333 execute_command([f"{KYTHE_DIR}/tools/kzip", "merge", "--output", final_kzip_name] + kzips)
Chi Wang6357efe2020-08-25 16:23:38 +08001334 execute_command(["buildkite-agent", "artifact", "upload", final_kzip_name])
1335
Chi Wangb2b65682020-08-27 10:36:15 +08001336 if index_upload_gcs:
1337 pipeline = os.getenv("BUILDKITE_PIPELINE_SLUG")
1338 destination = KZIPS_BUCKET + pipeline + "/" + final_kzip_name
1339 print("Uploading to GCS {}".format(destination))
1340 execute_command(
1341 [
1342 gsutil_command(),
1343 "cp",
1344 final_kzip_name,
1345 destination,
1346 ]
1347 )
1348
Chi Wang6357efe2020-08-25 16:23:38 +08001349
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04001350def download_binary(dest_dir, platform, binary_name):
Philipp Wollermannc52e26a2019-05-18 22:10:47 +02001351 source_step = create_label(platform, "Bazel", build_only=True)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001352 execute_command(
Philipp Wollermann10183212020-02-04 21:54:14 +01001353 ["buildkite-agent", "artifact", "download", binary_name, dest_dir, "--step", source_step]
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001354 )
Philipp Wollermann10183212020-02-04 21:54:14 +01001355 bazel_binary_path = os.path.join(dest_dir, binary_name)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001356 st = os.stat(bazel_binary_path)
1357 os.chmod(bazel_binary_path, st.st_mode | stat.S_IEXEC)
1358 return bazel_binary_path
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001359
1360
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04001361def download_bazel_binary(dest_dir, platform):
1362 binary_name = "bazel.exe" if platform == "windows" else "bazel"
1363 return download_binary(dest_dir, platform, binary_name)
1364
1365
1366def download_bazel_nojdk_binary(dest_dir, platform):
1367 binary_name = "bazel_nojdk.exe" if platform == "windows" else "bazel_nojdk"
1368 return download_binary(dest_dir, platform, binary_name)
1369
1370
1371def download_binary_at_commit(dest_dir, platform, bazel_git_commit, bazel_binary_url, bazel_binary_path):
Yun Peng02312732019-01-17 18:17:05 +01001372 try:
1373 execute_command(
1374 [
1375 gsutil_command(),
1376 "cp",
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04001377 bazel_binary_url,
Yun Peng02312732019-01-17 18:17:05 +01001378 bazel_binary_path,
1379 ]
1380 )
1381 except subprocess.CalledProcessError as e:
Philipp Wollermannc05ac682019-01-19 12:37:28 +01001382 raise BuildkiteException(
1383 "Failed to download Bazel binary at %s, error message:\n%s" % (bazel_git_commit, str(e))
1384 )
Yun Peng20d45602018-10-18 13:27:05 +02001385 st = os.stat(bazel_binary_path)
1386 os.chmod(bazel_binary_path, st.st_mode | stat.S_IEXEC)
1387 return bazel_binary_path
1388
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001389
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04001390def download_bazel_binary_at_commit(dest_dir, platform, bazel_git_commit):
1391 url = bazelci_builds_gs_url(platform, bazel_git_commit)
1392 path = os.path.join(dest_dir, "bazel.exe" if platform == "windows" else "bazel")
1393 return download_binary_at_commit(dest_dir, platform, bazel_git_commit, url, path)
1394
1395
1396def download_bazel_nojdk_binary_at_commit(dest_dir, platform, bazel_git_commit):
1397 url = bazelci_builds_nojdk_gs_url(platform, bazel_git_commit)
1398 path = os.path.join(dest_dir, "bazel_nojdk.exe" if platform == "windows" else "bazel_nojdk")
1399 return download_binary_at_commit(dest_dir, platform, bazel_git_commit, url, path)
1400
1401
joeleba7050d842019-05-23 17:03:31 +02001402def get_mirror_path(git_repository, platform):
1403 mirror_root = {
1404 "macos": "/usr/local/var/bazelbuild/",
1405 "windows": "c:\\buildkite\\bazelbuild\\",
1406 }.get(platform, "/var/lib/bazelbuild/")
1407
1408 return mirror_root + re.sub(r"[^0-9A-Za-z]", "-", git_repository)
1409
1410
Yun Peng376d2b32018-11-29 10:24:54 +01001411def clone_git_repository(git_repository, platform, git_commit=None):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001412 root = downstream_projects_root(platform)
Philipp Wollermannff39ef52018-02-21 14:18:52 +01001413 project_name = re.search(r"/([^/]+)\.git$", git_repository).group(1)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001414 clone_path = os.path.join(root, project_name)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001415 print_collapsed_group(
1416 "Fetching %s sources at %s" % (project_name, git_commit if git_commit else "HEAD")
1417 )
Philipp Wollermann438ec242018-09-05 14:39:24 +02001418
joeleba7050d842019-05-23 17:03:31 +02001419 mirror_path = get_mirror_path(git_repository, platform)
Philipp Wollermannea128282019-05-08 11:56:14 +02001420
Philipp Wollermann414703d2018-08-28 16:40:38 +02001421 if not os.path.exists(clone_path):
Philipp Wollermann62f4a032019-05-08 17:44:14 +02001422 if os.path.exists(mirror_path):
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001423 execute_command(
Philipp Wollermann62f4a032019-05-08 17:44:14 +02001424 ["git", "clone", "-v", "--reference", mirror_path, git_repository, clone_path]
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001425 )
Philipp Wollermannd4cd0d82018-05-01 09:56:24 +02001426 else:
Philipp Wollermannea128282019-05-08 11:56:14 +02001427 execute_command(["git", "clone", "-v", git_repository, clone_path])
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001428
Philipp Wollermann414703d2018-08-28 16:40:38 +02001429 os.chdir(clone_path)
1430 execute_command(["git", "remote", "set-url", "origin", git_repository])
1431 execute_command(["git", "clean", "-fdqx"])
Florian Weikertd8f497c2019-06-19 15:44:20 +02001432 execute_command(["git", "submodule", "foreach", "--recursive", "git clean -fdqx"])
Philipp Wollermann414703d2018-08-28 16:40:38 +02001433 execute_command(["git", "fetch", "origin"])
Yun Peng376d2b32018-11-29 10:24:54 +01001434 if git_commit:
1435 # sync to a specific commit of this repository
1436 execute_command(["git", "reset", git_commit, "--hard"])
1437 else:
1438 # sync to the latest commit of HEAD. Unlikely git pull this also works after a force push.
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001439 remote_head = (
1440 subprocess.check_output(["git", "symbolic-ref", "refs/remotes/origin/HEAD"])
1441 .decode("utf-8")
1442 .rstrip()
1443 )
Yun Peng376d2b32018-11-29 10:24:54 +01001444 execute_command(["git", "reset", remote_head, "--hard"])
Philipp Wollermann414703d2018-08-28 16:40:38 +02001445 execute_command(["git", "submodule", "sync", "--recursive"])
1446 execute_command(["git", "submodule", "update", "--init", "--recursive", "--force"])
Florian Weikertd8f497c2019-06-19 15:44:20 +02001447 execute_command(["git", "submodule", "foreach", "--recursive", "git reset --hard"])
Philipp Wollermann414703d2018-08-28 16:40:38 +02001448 execute_command(["git", "clean", "-fdqx"])
Florian Weikertd8f497c2019-06-19 15:44:20 +02001449 execute_command(["git", "submodule", "foreach", "--recursive", "git clean -fdqx"])
Yun Peng20d45602018-10-18 13:27:05 +02001450 return clone_path
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001451
Philipp Wollermann438ec242018-09-05 14:39:24 +02001452
Yun Penga935a542018-05-18 15:08:53 +02001453def execute_batch_commands(commands):
1454 if not commands:
1455 return
1456 print_collapsed_group(":batch: Setup (Batch Commands)")
1457 batch_commands = "&".join(commands)
Jakob Buchgraber4a824412018-06-22 12:56:10 +02001458 return subprocess.run(batch_commands, shell=True, check=True, env=os.environ).returncode
Yun Penga935a542018-05-18 15:08:53 +02001459
Philipp Wollermann414703d2018-08-28 16:40:38 +02001460
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001461def execute_shell_commands(commands):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001462 if not commands:
1463 return
Jakob Buchgraber94d5c212018-02-22 09:57:08 +01001464 print_collapsed_group(":bash: Setup (Shell Commands)")
mostynb14440912020-03-17 17:11:47 +01001465 shell_command = "\n".join(["set -e"] + commands)
Philipp Wollermann3e1a7712018-02-19 17:34:24 +01001466 execute_command([shell_command], shell=True)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001467
1468
Yun Peng0a6a98a2019-03-06 13:07:54 +01001469def handle_bazel_failure(exception, action):
1470 msg = "bazel {0} failed with exit code {1}".format(action, exception.returncode)
1471 if use_bazelisk_migrate():
1472 print_collapsed_group(msg)
1473 else:
1474 raise BuildkiteException(msg)
1475
1476
Yun Peng4be92b32018-11-30 09:48:29 +01001477def execute_bazel_run(bazel_binary, platform, targets, incompatible_flags):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001478 if not targets:
1479 return
1480 print_collapsed_group("Setup (Run Targets)")
Florian Weikert474d7972019-03-01 02:12:01 +01001481 # When using bazelisk --migrate to test incompatible flags,
1482 # incompatible flags set by "INCOMPATIBLE_FLAGS" env var will be ignored.
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001483 incompatible_flags_to_use = (
1484 [] if (use_bazelisk_migrate() or not incompatible_flags) else incompatible_flags
1485 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001486 for target in targets:
Yun Peng0a6a98a2019-03-06 13:07:54 +01001487 try:
1488 execute_command(
1489 [bazel_binary]
1490 + bazelisk_flags()
1491 + common_startup_flags(platform)
1492 + ["run"]
1493 + common_build_flags(None, platform)
1494 + incompatible_flags_to_use
1495 + [target]
1496 )
1497 except subprocess.CalledProcessError as e:
1498 handle_bazel_failure(e, "run")
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001499
1500
Jakob Buchgraber4f1d2712018-02-20 10:22:47 +01001501def remote_caching_flags(platform):
Philipp Wollermanne67eec42019-05-24 15:18:20 +02001502 # Only enable caching for untrusted and testing builds.
Philipp Wollermannfce92bf2019-05-22 15:14:32 +02001503 if CLOUD_PROJECT not in ["bazel-untrusted"]:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001504 return []
Philipp Wollermann02955272019-04-18 18:00:48 +02001505
Philipp Wollermanne67eec42019-05-24 15:18:20 +02001506 platform_cache_key = [BUILDKITE_ORG.encode("utf-8")]
Jakob Buchgraber89df3982019-08-06 13:07:02 +02001507 # Whenever the remote cache was known to have been poisoned increase the number below
Vladimir Moskvaaacef962020-10-19 13:32:06 +02001508 platform_cache_key += ["cache-poisoning-20201011".encode("utf-8")]
Philipp Wollermannfce92bf2019-05-22 15:14:32 +02001509
Philipp Wollermann380f1e62019-04-12 16:45:27 +02001510 if platform == "macos":
Philipp Wollermannfce92bf2019-05-22 15:14:32 +02001511 platform_cache_key += [
Philipp Wollermannef89d2f2019-04-18 15:52:24 +02001512 # macOS version:
1513 subprocess.check_output(["/usr/bin/sw_vers", "-productVersion"]),
1514 # Path to Xcode:
1515 subprocess.check_output(["/usr/bin/xcode-select", "-p"]),
1516 # Xcode version:
1517 subprocess.check_output(["/usr/bin/xcodebuild", "-version"]),
1518 ]
1519 # Use a local cache server for our macOS machines.
Philipp Wollermannb9e96282020-02-18 13:59:27 +01001520 flags = ["--remote_cache=http://100.107.73.148"]
Philipp Wollermannef89d2f2019-04-18 15:52:24 +02001521 else:
Philipp Wollermannfce92bf2019-05-22 15:14:32 +02001522 platform_cache_key += [
Philipp Wollermannef89d2f2019-04-18 15:52:24 +02001523 # Platform name:
1524 platform.encode("utf-8")
1525 ]
Philipp Wollermanne74da4e2019-06-07 11:31:02 +02001526 # Use RBE for caching builds running on GCE.
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001527 flags = [
1528 "--google_default_credentials",
Philipp Wollermanne74da4e2019-06-07 11:31:02 +02001529 "--remote_cache=remotebuildexecution.googleapis.com",
1530 "--remote_instance_name=projects/{}/instances/default_instance".format(CLOUD_PROJECT),
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001531 ]
Philipp Wollermann380f1e62019-04-12 16:45:27 +02001532
1533 platform_cache_digest = hashlib.sha256()
1534 for key in platform_cache_key:
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001535 eprint("Adding to platform cache key: {}".format(key))
Philipp Wollermann380f1e62019-04-12 16:45:27 +02001536 platform_cache_digest.update(key)
1537 platform_cache_digest.update(b":")
1538
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001539 flags += [
Philipp Wollermann94937722019-01-11 14:33:18 +01001540 "--remote_timeout=60",
Philipp Wollermann639c0452019-01-03 11:23:54 +01001541 "--remote_max_connections=200",
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001542 '--remote_default_platform_properties=properties:{name:"cache-silo-key" value:"%s"}'
1543 % platform_cache_digest.hexdigest(),
Philipp Wollermann639c0452019-01-03 11:23:54 +01001544 ]
Jakob Buchgraber4f1d2712018-02-20 10:22:47 +01001545
Philipp Wollermannd96d8fa2019-01-11 14:37:47 +01001546 return flags
1547
Jakob Buchgraber4f1d2712018-02-20 10:22:47 +01001548
Jakob Buchgraberb4342cd2018-02-20 16:35:07 +01001549def remote_enabled(flags):
1550 # Detect if the project configuration enabled its own remote caching / execution.
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001551 remote_flags = ["--remote_executor", "--remote_cache", "--remote_http_cache"]
Jakob Buchgraberb4342cd2018-02-20 16:35:07 +01001552 for flag in flags:
1553 for remote_flag in remote_flags:
1554 if flag.startswith(remote_flag):
1555 return True
1556 return False
1557
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01001558
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001559def concurrent_jobs(platform):
1560 return "75" if platform.startswith("rbe_") else str(multiprocessing.cpu_count())
Jakob Buchgraber51a83662018-02-22 19:49:24 +01001561
1562
Philipp Wollermann3e28d3b2018-02-23 23:19:37 +01001563def concurrent_test_jobs(platform):
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001564 if platform.startswith("rbe_"):
1565 return "75"
1566 elif platform == "windows":
Jakob Buchgrabere3ccda32018-06-22 23:29:48 +02001567 return "8"
Philipp Wollermann111adfb2018-11-22 10:26:03 +01001568 elif platform == "macos":
1569 return "8"
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001570 return "12"
Philipp Wollermann3e28d3b2018-02-23 23:19:37 +01001571
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001572
Yun Peng58977d62018-11-16 12:19:20 +01001573def common_startup_flags(platform):
Philipp Wollermannd5ab3d92020-02-05 16:55:13 +01001574 if platform == "windows":
1575 if os.path.exists("D:/b"):
1576 # This machine has a local SSD mounted as drive D.
1577 return ["--output_user_root=D:/b"]
1578 else:
1579 # This machine uses its PD-SSD as the build directory.
1580 return ["--output_user_root=C:/b"]
1581 return []
Yun Peng58977d62018-11-16 12:19:20 +01001582
1583
1584def common_build_flags(bep_file, platform):
Yun Peng088cc932018-11-16 12:11:46 +01001585 flags = [
Yun Pengf51e7842018-11-16 11:35:43 +01001586 "--show_progress_rate_limit=5",
1587 "--curses=yes",
1588 "--color=yes",
Philipp Wollermannd99414c2019-05-28 17:26:09 +02001589 "--terminal_columns=143",
Philipp Wollermann4c8391e2019-05-28 18:03:35 +02001590 "--show_timestamps",
Yun Pengf51e7842018-11-16 11:35:43 +01001591 "--verbose_failures",
Yun Pengf51e7842018-11-16 11:35:43 +01001592 "--jobs=" + concurrent_jobs(platform),
Yun Pengf51e7842018-11-16 11:35:43 +01001593 "--announce_rc",
Philipp Wollermannb97f9102019-04-16 18:05:56 +02001594 "--experimental_repository_cache_hardlinks",
Philipp Wollermannef89d2f2019-04-18 15:52:24 +02001595 # Some projects set --disk_cache in their project-specific bazelrc, which we never want on
1596 # CI, so let's just disable it explicitly.
1597 "--disk_cache=",
Yun Peng088cc932018-11-16 12:11:46 +01001598 ]
Philipp Wollermann639c0452019-01-03 11:23:54 +01001599
Philipp Wollermannb97f9102019-04-16 18:05:56 +02001600 if platform == "windows":
1601 pass
1602 elif platform == "macos":
1603 flags += [
1604 "--sandbox_writable_path=/var/tmp/_bazel_buildkite/cache/repos/v1",
1605 "--test_env=REPOSITORY_CACHE=/var/tmp/_bazel_buildkite/cache/repos/v1",
1606 ]
1607 else:
Philipp Wollermann639c0452019-01-03 11:23:54 +01001608 flags += ["--sandbox_tmpfs_path=/tmp"]
1609
Yun Peng088cc932018-11-16 12:11:46 +01001610 if bep_file:
Philipp Wollermann639c0452019-01-03 11:23:54 +01001611 flags += [
1612 "--experimental_build_event_json_file_path_conversion=false",
1613 "--build_event_json_file=" + bep_file,
1614 ]
1615
Yun Peng088cc932018-11-16 12:11:46 +01001616 return flags
Philipp Wollermann94bd9e32018-04-30 15:32:28 +02001617
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001618
Yun Pengb7247ff2018-11-15 13:52:39 +01001619def rbe_flags(original_flags, accept_cached):
Philipp Wollermannbcfd9da2018-08-09 15:31:18 +02001620 # Enable remote execution via RBE.
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001621 flags = [
1622 "--remote_executor=remotebuildexecution.googleapis.com",
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01001623 "--remote_instance_name=projects/bazel-untrusted/instances/default_instance",
Philipp Wollermann57dadb82020-02-17 14:32:24 +01001624 "--incompatible_strict_action_env",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001625 "--google_default_credentials",
Philipp Wollermann57dadb82020-02-17 14:32:24 +01001626 "--toolchain_resolution_debug",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001627 ]
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02001628
Philipp Wollermannbcfd9da2018-08-09 15:31:18 +02001629 # Enable BES / Build Results reporting.
1630 flags += [
1631 "--bes_backend=buildeventservice.googleapis.com",
Philipp Wollermannbcfd9da2018-08-09 15:31:18 +02001632 "--bes_timeout=360s",
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01001633 "--project_id=bazel-untrusted",
Philipp Wollermannbcfd9da2018-08-09 15:31:18 +02001634 ]
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02001635
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001636 if not accept_cached:
1637 flags += ["--noremote_accept_cached"]
Philipp Wollermannbcfd9da2018-08-09 15:31:18 +02001638
Nicolas Lopez36996222019-05-28 12:21:28 -04001639 # Adapted from https://github.com/bazelbuild/bazel-toolchains/blob/master/bazelrc/.bazelrc
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001640 flags += [
Nicolas Lopez36996222019-05-28 12:21:28 -04001641 # These should NOT longer need to be modified.
1642 # All that is needed is updating the @bazel_toolchains repo pin
1643 # in projects' WORKSPACE files.
Xindb02c012018-11-07 14:10:54 -05001644 #
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001645 # Toolchain related flags to append at the end of your .bazelrc file.
Nicolas Lopez36996222019-05-28 12:21:28 -04001646 "--host_javabase=@buildkite_config//java:jdk",
1647 "--javabase=@buildkite_config//java:jdk",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001648 "--host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8",
1649 "--java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8",
Nicolas Lopez36996222019-05-28 12:21:28 -04001650 "--crosstool_top=@buildkite_config//cc:toolchain",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001651 "--action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001652 ]
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02001653
Yun Pengb7247ff2018-11-15 13:52:39 +01001654 # Platform flags:
1655 # The toolchain container used for execution is defined in the target indicated
1656 # by "extra_execution_platforms", "host_platform" and "platforms".
Xin03a88ab2019-03-11 19:18:50 -04001657 # If you are using your own toolchain container, you need to create a platform
1658 # target with "constraint_values" that allow for the toolchain specified with
1659 # "extra_toolchains" to be selected (given constraints defined in
1660 # "exec_compatible_with").
Yun Pengb7247ff2018-11-15 13:52:39 +01001661 # More about platforms: https://docs.bazel.build/versions/master/platforms.html
1662 # Don't add platform flags if they are specified already.
1663 platform_flags = {
Nicolas Lopez36996222019-05-28 12:21:28 -04001664 "--extra_toolchains": "@buildkite_config//config:cc-toolchain",
1665 "--extra_execution_platforms": "@buildkite_config//config:platform",
1666 "--host_platform": "@buildkite_config//config:platform",
1667 "--platforms": "@buildkite_config//config:platform",
Yun Pengb7247ff2018-11-15 13:52:39 +01001668 }
Yun Peng67ab5062018-11-15 13:58:15 +01001669 for platform_flag, value in list(platform_flags.items()):
Yun Pengb7247ff2018-11-15 13:52:39 +01001670 found = False
1671 for original_flag in original_flags:
1672 if original_flag.startswith(platform_flag):
1673 found = True
1674 break
1675 if not found:
1676 flags += [platform_flag + "=" + value]
1677
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001678 return flags
1679
1680
Philipp Wollermann87b45252020-01-22 12:43:42 +01001681def compute_flags(
1682 platform, flags, incompatible_flags, bep_file, bazel_binary, enable_remote_cache=False
1683):
Yun Peng58977d62018-11-16 12:19:20 +01001684 aggregated_flags = common_build_flags(bep_file, platform)
Yun Peng32dbec32018-11-02 12:47:41 +01001685 if not remote_enabled(flags):
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001686 if platform.startswith("rbe_"):
Florian Weikert24a4b482018-11-30 19:05:38 +01001687 aggregated_flags += rbe_flags(flags, accept_cached=enable_remote_cache)
1688 elif enable_remote_cache:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001689 aggregated_flags += remote_caching_flags(platform)
Yun Peng75786552018-11-13 15:23:30 +01001690 aggregated_flags += flags
Yun Peng4be92b32018-11-30 09:48:29 +01001691 if incompatible_flags:
1692 aggregated_flags += incompatible_flags
Yun Peng53598002018-12-03 10:42:02 +01001693
Philipp Wollermann87b45252020-01-22 12:43:42 +01001694 for i, flag in enumerate(aggregated_flags):
1695 if "$HOME" in flag:
Philipp Wollermannd5ab3d92020-02-05 16:55:13 +01001696 if platform == "windows":
1697 if os.path.exists("D:/"):
1698 home = "D:"
1699 else:
1700 home = "C:/b"
1701 elif platform == "macos":
1702 home = "/Users/buildkite"
1703 else:
1704 home = "/var/lib/buildkite-agent"
Philipp Wollermann87b45252020-01-22 12:43:42 +01001705 aggregated_flags[i] = flag.replace("$HOME", home)
1706 if "$OUTPUT_BASE" in flag:
1707 output_base = execute_command_and_get_output(
Philipp Wollermann0e394412020-01-26 15:52:32 +01001708 [bazel_binary] + common_startup_flags(platform) + ["info", "output_base"],
1709 print_output=False,
Philipp Wollermann87b45252020-01-22 12:43:42 +01001710 ).strip()
1711 aggregated_flags[i] = flag.replace("$OUTPUT_BASE", output_base)
1712
Florian Weikert24a4b482018-11-30 19:05:38 +01001713 return aggregated_flags
Yun Peng53598002018-12-03 10:42:02 +01001714
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001715
Yun Pengea0359e2019-01-17 15:37:47 +01001716def execute_bazel_clean(bazel_binary, platform):
1717 print_expanded_group(":bazel: Clean")
1718
1719 try:
Philipp Wollermannc05ac682019-01-19 12:37:28 +01001720 execute_command([bazel_binary] + common_startup_flags(platform) + ["clean", "--expunge"])
Yun Pengea0359e2019-01-17 15:37:47 +01001721 except subprocess.CalledProcessError as e:
1722 raise BuildkiteException("bazel clean failed with exit code {}".format(e.returncode))
1723
1724
Chi Wang6357efe2020-08-25 16:23:38 +08001725def kythe_startup_flags():
1726 return [f"--bazelrc={KYTHE_DIR}/extractors.bazelrc"]
1727
1728
1729def kythe_build_flags():
1730 return [f"--override_repository=kythe_release={KYTHE_DIR}"]
1731
1732
Florian Weikertc8642af2019-02-03 23:58:51 +01001733def execute_bazel_build(
1734 bazel_version, bazel_binary, platform, flags, targets, bep_file, incompatible_flags
1735):
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001736 print_collapsed_group(":bazel: Computing flags for build step")
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001737 aggregated_flags = compute_flags(
Yun Peng8975c6b2019-02-28 11:55:55 +01001738 platform,
1739 flags,
1740 # When using bazelisk --migrate to test incompatible flags,
1741 # incompatible flags set by "INCOMPATIBLE_FLAGS" env var will be ignored.
1742 [] if (use_bazelisk_migrate() or not incompatible_flags) else incompatible_flags,
1743 bep_file,
Philipp Wollermann87b45252020-01-22 12:43:42 +01001744 bazel_binary,
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001745 enable_remote_cache=True,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001746 )
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001747
1748 print_expanded_group(":bazel: Build ({})".format(bazel_version))
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01001749 try:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001750 execute_command(
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001751 [bazel_binary]
1752 + bazelisk_flags()
1753 + common_startup_flags(platform)
1754 + ["build"]
1755 + aggregated_flags
Philipp Wollermann2a160432019-09-19 15:57:28 +02001756 + ["--"]
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001757 + targets
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001758 )
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01001759 except subprocess.CalledProcessError as e:
Yun Peng0a6a98a2019-03-06 13:07:54 +01001760 handle_bazel_failure(e, "build")
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001761
1762
Chi Wang6357efe2020-08-25 16:23:38 +08001763def execute_bazel_build_with_kythe(
1764 bazel_version, bazel_binary, platform, flags, targets, bep_file, incompatible_flags
1765):
1766 print_collapsed_group(":bazel: Computing flags for build step")
1767 aggregated_flags = compute_flags(
1768 platform,
1769 flags,
1770 # When using bazelisk --migrate to test incompatible flags,
1771 # incompatible flags set by "INCOMPATIBLE_FLAGS" env var will be ignored.
1772 [] if (use_bazelisk_migrate() or not incompatible_flags) else incompatible_flags,
1773 bep_file,
1774 bazel_binary,
1775 enable_remote_cache=False,
1776 )
1777
1778 print_expanded_group(":bazel: Build ({})".format(bazel_version))
1779
1780 execute_command(
1781 [bazel_binary]
1782 + bazelisk_flags()
1783 + common_startup_flags(platform)
1784 + kythe_startup_flags()
1785 + ["build"]
1786 + kythe_build_flags()
1787 + aggregated_flags
1788 + ["--"]
1789 + targets
1790 )
1791
1792
Florian Weikert736d06e2019-05-08 13:16:42 +02001793def calculate_targets(task_config, platform, bazel_binary, build_only, test_only):
1794 build_targets = [] if test_only else task_config.get("build_targets", [])
1795 test_targets = [] if build_only else task_config.get("test_targets", [])
Chi Wang6357efe2020-08-25 16:23:38 +08001796 index_targets = [] if (build_only or test_only) else task_config.get("index_targets", [])
1797
1798 index_targets_query = None if (build_only or test_only) else task_config.get("index_targets_query", None)
1799 if index_targets_query:
1800 output = execute_command_and_get_output(
1801 [bazel_binary]
1802 + common_startup_flags(platform)
1803 + [
1804 "--nomaster_bazelrc",
1805 "--bazelrc=/dev/null",
1806 "query",
1807 index_targets_query,
1808 ],
1809 print_output=False,
1810 )
1811 index_targets += output.strip().split("\n")
Florian Weikert736d06e2019-05-08 13:16:42 +02001812
Philipp Wollermann2a160432019-09-19 15:57:28 +02001813 # Remove the "--" argument splitter from the list that some configs explicitly
1814 # include. We'll add it back again later where needed.
1815 build_targets = [x.strip() for x in build_targets if x.strip() != "--"]
1816 test_targets = [x.strip() for x in test_targets if x.strip() != "--"]
Chi Wang6357efe2020-08-25 16:23:38 +08001817 index_targets = [x.strip() for x in index_targets if x.strip() != "--"]
Philipp Wollermann2a160432019-09-19 15:57:28 +02001818
Florian Weikert736d06e2019-05-08 13:16:42 +02001819 shard_id = int(os.getenv("BUILDKITE_PARALLEL_JOB", "-1"))
1820 shard_count = int(os.getenv("BUILDKITE_PARALLEL_JOB_COUNT", "-1"))
1821 if shard_id > -1 and shard_count > -1:
1822 print_collapsed_group(
1823 ":female-detective: Calculating targets for shard {}/{}".format(
1824 shard_id + 1, shard_count
1825 )
1826 )
1827 expanded_test_targets = expand_test_target_patterns(bazel_binary, platform, test_targets)
Philipp Wollermann87b45252020-01-22 12:43:42 +01001828 test_targets = get_targets_for_shard(expanded_test_targets, shard_id, shard_count)
Florian Weikert736d06e2019-05-08 13:16:42 +02001829
Chi Wang6357efe2020-08-25 16:23:38 +08001830 return build_targets, test_targets, index_targets
Florian Weikert736d06e2019-05-08 13:16:42 +02001831
1832
1833def expand_test_target_patterns(bazel_binary, platform, test_targets):
Philipp Wollermann2a160432019-09-19 15:57:28 +02001834 included_targets, excluded_targets = partition_targets(test_targets)
Florian Weikert736d06e2019-05-08 13:16:42 +02001835 excluded_string = (
1836 " except tests(set({}))".format(" ".join("'{}'".format(t) for t in excluded_targets))
1837 if excluded_targets
1838 else ""
1839 )
1840
Philipp Wollermann2a160432019-09-19 15:57:28 +02001841 exclude_manual = ' except tests(attr("tags", "manual", set({})))'.format(
1842 " ".join("'{}'".format(t) for t in included_targets)
1843 )
1844
Florian Weikert736d06e2019-05-08 13:16:42 +02001845 eprint("Resolving test targets via bazel query")
1846 output = execute_command_and_get_output(
1847 [bazel_binary]
1848 + common_startup_flags(platform)
1849 + [
1850 "--nomaster_bazelrc",
1851 "--bazelrc=/dev/null",
1852 "query",
Philipp Wollermann2a160432019-09-19 15:57:28 +02001853 "tests(set({})){}{}".format(
1854 " ".join("'{}'".format(t) for t in included_targets),
1855 excluded_string,
1856 exclude_manual,
Florian Weikert736d06e2019-05-08 13:16:42 +02001857 ),
1858 ],
1859 print_output=False,
Florian Weikert736d06e2019-05-08 13:16:42 +02001860 )
Philipp Wollermann2a160432019-09-19 15:57:28 +02001861 return output.strip().split("\n")
Florian Weikert736d06e2019-05-08 13:16:42 +02001862
1863
Philipp Wollermann2a160432019-09-19 15:57:28 +02001864def partition_targets(targets):
Florian Weikert736d06e2019-05-08 13:16:42 +02001865 included_targets, excluded_targets = [], []
Philipp Wollermann2a160432019-09-19 15:57:28 +02001866 for target in targets:
1867 if target.startswith("-"):
Florian Weikert736d06e2019-05-08 13:16:42 +02001868 excluded_targets.append(target[1:])
1869 else:
1870 included_targets.append(target)
1871
1872 return included_targets, excluded_targets
1873
1874
Philipp Wollermann87b45252020-01-22 12:43:42 +01001875def get_targets_for_shard(test_targets, shard_id, shard_count):
Florian Weikert736d06e2019-05-08 13:16:42 +02001876 # TODO(fweikert): implement a more sophisticated algorithm
Philipp Wollermann87b45252020-01-22 12:43:42 +01001877 return sorted(test_targets)[shard_id::shard_count]
Florian Weikert736d06e2019-05-08 13:16:42 +02001878
1879
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001880def execute_bazel_test(
Florian Weikertc8642af2019-02-03 23:58:51 +01001881 bazel_version,
1882 bazel_binary,
1883 platform,
1884 flags,
1885 targets,
1886 bep_file,
1887 monitor_flaky_tests,
1888 incompatible_flags,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001889):
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001890 aggregated_flags = [
1891 "--flaky_test_attempts=3",
1892 "--build_tests_only",
1893 "--local_test_jobs=" + concurrent_test_jobs(platform),
1894 ]
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001895 # Don't enable remote caching if the user enabled remote execution / caching themselves
Jakob Buchgraberc340f582018-06-22 13:48:33 +02001896 # or flaky test monitoring is enabled, as remote caching makes tests look less flaky than
1897 # they are.
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001898 print_collapsed_group(":bazel: Computing flags for test step")
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001899 aggregated_flags += compute_flags(
Yun Peng8975c6b2019-02-28 11:55:55 +01001900 platform,
1901 flags,
1902 # When using bazelisk --migrate to test incompatible flags,
1903 # incompatible flags set by "INCOMPATIBLE_FLAGS" env var will be ignored.
1904 [] if (use_bazelisk_migrate() or not incompatible_flags) else incompatible_flags,
1905 bep_file,
Philipp Wollermann87b45252020-01-22 12:43:42 +01001906 bazel_binary,
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001907 enable_remote_cache=not monitor_flaky_tests,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001908 )
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001909
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001910 print_expanded_group(":bazel: Test ({})".format(bazel_version))
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01001911 try:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001912 execute_command(
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001913 [bazel_binary]
1914 + bazelisk_flags()
1915 + common_startup_flags(platform)
1916 + ["test"]
1917 + aggregated_flags
Philipp Wollermann2a160432019-09-19 15:57:28 +02001918 + ["--"]
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001919 + targets
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001920 )
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01001921 except subprocess.CalledProcessError as e:
Yun Peng0a6a98a2019-03-06 13:07:54 +01001922 handle_bazel_failure(e, "test")
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001923
1924
joeleba76887952019-05-16 15:22:17 +02001925def get_json_profile_flags(out_file):
1926 return [
1927 "--experimental_generate_json_trace_profile",
1928 "--experimental_profile_cpu_usage",
1929 "--experimental_json_trace_compression",
Philipp Wollermann92cf51e2019-05-16 15:31:11 +02001930 "--profile={}".format(out_file),
joeleba76887952019-05-16 15:22:17 +02001931 ]
1932
1933
Florian Weikertee84c5c2019-05-28 11:21:51 +02001934def upload_bep_logs_for_flaky_tests(test_bep_file):
1935 if has_flaky_tests(test_bep_file):
1936 build_number = os.getenv("BUILDKITE_BUILD_NUMBER")
1937 pipeline_slug = os.getenv("BUILDKITE_PIPELINE_SLUG")
1938 execute_command(
1939 [
1940 gsutil_command(),
1941 "cp",
1942 test_bep_file,
1943 FLAKY_TESTS_BUCKET + pipeline_slug + "/" + build_number + ".json",
1944 ]
1945 )
1946
1947
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001948def upload_test_logs_from_bep(bep_file, tmpdir, stop_request):
1949 uploaded_targets = set()
1950 while True:
1951 done = stop_request.isSet()
1952 if os.path.exists(bep_file):
Jakob Buchgraberc874cdf2019-07-16 16:27:41 +02001953 all_test_logs = test_logs_for_status(bep_file, status=["FAILED", "TIMEOUT", "FLAKY"])
Philipp Wollermann3c8b8512019-07-16 15:28:03 +02001954 test_logs_to_upload = [
1955 (target, files) for target, files in all_test_logs if target not in uploaded_targets
1956 ]
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001957
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001958 if test_logs_to_upload:
1959 files_to_upload = rename_test_logs_for_upload(test_logs_to_upload, tmpdir)
1960 cwd = os.getcwd()
1961 try:
1962 os.chdir(tmpdir)
1963 test_logs = [os.path.relpath(file, tmpdir) for file in files_to_upload]
1964 test_logs = sorted(test_logs)
1965 execute_command(["buildkite-agent", "artifact", "upload", ";".join(test_logs)])
1966 finally:
1967 uploaded_targets.update([target for target, _ in test_logs_to_upload])
1968 os.chdir(cwd)
1969 if done:
1970 break
Philipp Wollermann628c26d2020-02-05 14:29:52 +01001971 time.sleep(5)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001972
Philipp Wollermann3c8b8512019-07-16 15:28:03 +02001973
joeleba76887952019-05-16 15:22:17 +02001974def upload_json_profile(json_profile_path, tmpdir):
1975 if not os.path.exists(json_profile_path):
1976 return
1977 print_collapsed_group(":gcloud: Uploading JSON Profile")
Philipp Wollermann92cf51e2019-05-16 15:31:11 +02001978 execute_command(["buildkite-agent", "artifact", "upload", json_profile_path], cwd=tmpdir)
joeleba76887952019-05-16 15:22:17 +02001979
Philipp Wollermann5b00a702019-07-18 11:21:38 +02001980
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001981def rename_test_logs_for_upload(test_logs, tmpdir):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001982 # Rename the test.log files to the target that created them
1983 # so that it's easy to associate test.log and target.
1984 new_paths = []
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001985 for label, files in test_logs:
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001986 attempt = 0
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001987 if len(files) > 1:
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001988 attempt = 1
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001989 for test_log in files:
Jakob Buchgraber070ef5f2018-02-20 17:57:14 +01001990 try:
1991 new_path = test_label_to_path(tmpdir, label, attempt)
Jakob Buchgraber070ef5f2018-02-20 17:57:14 +01001992 os.makedirs(os.path.dirname(new_path), exist_ok=True)
Jakob Buchgraber070ef5f2018-02-20 17:57:14 +01001993 copyfile(test_log, new_path)
Jakob Buchgraber070ef5f2018-02-20 17:57:14 +01001994 new_paths.append(new_path)
Philipp Wollermannc030f2e2018-02-21 17:02:19 +01001995 attempt += 1
Jakob Buchgraber070ef5f2018-02-20 17:57:14 +01001996 except IOError as err:
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01001997 # Log error and ignore.
1998 eprint(err)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001999 return new_paths
Jakob Buchgraber699aece2018-02-19 12:49:30 +01002000
2001
Jakob Buchgraber02e07222018-02-19 15:05:56 +01002002def test_label_to_path(tmpdir, label, attempt):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002003 # remove leading //
2004 path = label[2:]
Philipp Wollermannc030f2e2018-02-21 17:02:19 +01002005 path = path.replace("/", os.sep)
2006 path = path.replace(":", os.sep)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002007 if attempt == 0:
2008 path = os.path.join(path, "test.log")
2009 else:
2010 path = os.path.join(path, "attempt_" + str(attempt) + ".log")
2011 return os.path.join(tmpdir, path)
Jakob Buchgraber699aece2018-02-19 12:49:30 +01002012
2013
Jakob Buchgraber02e07222018-02-19 15:05:56 +01002014def test_logs_for_status(bep_file, status):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002015 targets = []
Jakob Buchgraber94d5c212018-02-22 09:57:08 +01002016 with open(bep_file, encoding="utf-8") as f:
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002017 raw_data = f.read()
2018 decoder = json.JSONDecoder()
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002019
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002020 pos = 0
2021 while pos < len(raw_data):
Jakob Buchgraber0aabefc2019-07-16 16:18:36 +02002022 try:
Philipp Wollermann5b00a702019-07-18 11:21:38 +02002023 bep_obj, size = decoder.raw_decode(raw_data[pos:])
Jakob Buchgraber0aabefc2019-07-16 16:18:36 +02002024 except ValueError as e:
Philipp Wollermannce986af2019-07-18 14:46:05 +02002025 eprint("JSON decoding error: " + str(e))
Philipp Wollermann5b00a702019-07-18 11:21:38 +02002026 return targets
Jakob Buchgraber45e38632018-02-19 17:27:08 +01002027 if "testSummary" in bep_obj:
2028 test_target = bep_obj["id"]["testSummary"]["label"]
2029 test_status = bep_obj["testSummary"]["overallStatus"]
Jakob Buchgraberc874cdf2019-07-16 16:27:41 +02002030 if test_status in status:
Jakob Buchgraber45e38632018-02-19 17:27:08 +01002031 outputs = bep_obj["testSummary"]["failed"]
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002032 test_logs = []
2033 for output in outputs:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002034 test_logs.append(url2pathname(urlparse(output["uri"]).path))
Jakob Buchgraber45e38632018-02-19 17:27:08 +01002035 targets.append((test_target, test_logs))
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002036 pos += size + 1
2037 return targets
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002038
2039
Philipp Wollermannaf35abf2019-05-22 17:52:01 +02002040def execute_command_and_get_output(args, shell=False, fail_if_nonzero=True, print_output=True):
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01002041 eprint(" ".join(args))
Florian Weikertc8642af2019-02-03 23:58:51 +01002042 process = subprocess.run(
2043 args,
2044 shell=shell,
2045 check=fail_if_nonzero,
2046 env=os.environ,
2047 stdout=subprocess.PIPE,
Philipp Wollermannf13804b2019-02-05 21:08:30 +01002048 errors="replace",
Florian Weikertc8642af2019-02-03 23:58:51 +01002049 universal_newlines=True,
2050 )
Florian Weikert736d06e2019-05-08 13:16:42 +02002051 if print_output:
2052 eprint(process.stdout)
2053
Florian Weikertc8642af2019-02-03 23:58:51 +01002054 return process.stdout
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002055
2056
Yun Peng9337bb32020-02-28 13:31:29 +01002057def execute_command(args, shell=False, fail_if_nonzero=True, cwd=None, print_output=True):
2058 if print_output:
2059 eprint(" ".join(args))
Philipp Wollermann92cf51e2019-05-16 15:31:11 +02002060 return subprocess.run(
2061 args, shell=shell, check=fail_if_nonzero, env=os.environ, cwd=cwd
2062 ).returncode
Philipp Wollermannf13804b2019-02-05 21:08:30 +01002063
2064
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02002065def execute_command_background(args):
2066 eprint(" ".join(args))
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02002067 return subprocess.Popen(args, env=os.environ)
2068
Philipp Wollermann3c8b8512019-07-16 15:28:03 +02002069
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02002070def terminate_background_process(process):
2071 if process:
2072 process.terminate()
2073 try:
2074 process.wait(timeout=10)
2075 except subprocess.TimeoutExpired:
2076 process.kill()
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02002077
Philipp Wollermann3c8b8512019-07-16 15:28:03 +02002078
Philipp Wollermann7a185322019-05-18 22:15:48 +02002079def create_step(label, commands, platform, shards=1):
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002080 if "docker-image" in PLATFORMS[platform]:
Florian Weikert736d06e2019-05-08 13:16:42 +02002081 step = create_docker_step(
Philipp Wollermannc05ac682019-01-19 12:37:28 +01002082 label, image=PLATFORMS[platform]["docker-image"], commands=commands
2083 )
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002084 else:
Philipp Wollermannf3750fa2019-05-21 17:11:59 +02002085 step = {
2086 "label": label,
2087 "command": commands,
2088 "agents": {"queue": PLATFORMS[platform]["queue"]},
2089 }
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002090
Florian Weikert736d06e2019-05-08 13:16:42 +02002091 if shards > 1:
2092 step["label"] += " (shard %n)"
2093 step["parallelism"] = shards
2094
Philipp Wollermann5b2f3fc2019-05-18 22:36:17 +02002095 # Enforce a global 8 hour job timeout.
2096 step["timeout_in_minutes"] = 8 * 60
2097
2098 # Automatically retry when an agent got lost (usually due to an infra flake).
Philipp Wollermannf22bba32019-07-18 11:22:50 +02002099 step["retry"] = {
2100 "automatic": [
2101 {"exit_status": -1, "limit": 3}, # Buildkite internal "agent lost" exit code
2102 {"exit_status": 137, "limit": 3}, # SIGKILL
2103 {"exit_status": 143, "limit": 3}, # SIGTERM
2104 ]
2105 }
Philipp Wollermann5b2f3fc2019-05-18 22:36:17 +02002106
Florian Weikert736d06e2019-05-08 13:16:42 +02002107 return step
2108
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002109
Florian Weikert29cb7ec2019-03-07 14:52:18 +01002110def create_docker_step(label, image, commands=None, additional_env_vars=None):
Philipp Wollermann0e051dd2019-05-16 11:37:52 +02002111 env = ["ANDROID_HOME", "ANDROID_NDK_HOME", "BUILDKITE_ARTIFACT_UPLOAD_DESTINATION"]
Florian Weikert29cb7ec2019-03-07 14:52:18 +01002112 if additional_env_vars:
2113 env += ["{}={}".format(k, v) for k, v in additional_env_vars.items()]
2114
Philipp Wollermannc05ac682019-01-19 12:37:28 +01002115 step = {
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002116 "label": label,
2117 "command": commands,
Philipp Wollermannb2fa2f62019-05-18 23:33:23 +02002118 "agents": {"queue": "default"},
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002119 "plugins": {
Philipp Wollermanna9815942020-02-11 17:08:35 +01002120 "docker#v3.5.0": {
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002121 "always-pull": True,
Florian Weikert29cb7ec2019-03-07 14:52:18 +01002122 "environment": env,
Philipp Wollermannc05ac682019-01-19 12:37:28 +01002123 "image": image,
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002124 "network": "host",
2125 "privileged": True,
2126 "propagate-environment": True,
Philipp Wollermann7aa95492019-05-18 22:03:24 +02002127 "propagate-uid-gid": True,
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002128 "volumes": [
Philipp Wollermann7aa95492019-05-18 22:03:24 +02002129 "/etc/group:/etc/group:ro",
2130 "/etc/passwd:/etc/passwd:ro",
Philipp Wollermann338db4a2019-05-18 11:21:04 +02002131 "/opt:/opt:ro",
Philipp Wollermann7aa95492019-05-18 22:03:24 +02002132 "/var/lib/buildkite-agent:/var/lib/buildkite-agent",
Philipp Wollermann338db4a2019-05-18 11:21:04 +02002133 "/var/lib/gitmirrors:/var/lib/gitmirrors:ro",
Philipp Wollermanna65944a2020-02-03 12:45:22 +01002134 "/var/run/docker.sock:/var/run/docker.sock",
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002135 ],
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002136 }
2137 },
2138 }
Philipp Wollermannc05ac682019-01-19 12:37:28 +01002139 if not step["command"]:
2140 del step["command"]
2141 return step
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002142
2143
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002144def print_project_pipeline(
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002145 configs,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002146 project_name,
2147 http_config,
2148 file_config,
2149 git_repository,
2150 monitor_flaky_tests,
2151 use_but,
2152 incompatible_flags,
Florian Weikert60661912019-12-18 15:17:10 +01002153 notify,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002154):
Florian Weikert843d7a02019-02-03 17:24:50 +01002155 task_configs = configs.get("tasks", None)
2156 if not task_configs:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002157 raise BuildkiteException("{0} pipeline configuration is empty.".format(project_name))
2158
Jakob Buchgraberaa2af382018-02-21 19:56:54 +01002159 pipeline_steps = []
mai93f2e116c2020-10-19 09:33:14 +02002160 # If the repository is hosted on Git-on-borg, we show the link to the commit Gerrit review
2161 buildkite_repo = os.getenv("BUILDKITE_REPO")
2162 if is_git_on_borg_repo(buildkite_repo):
2163 show_gerrit_review_link(buildkite_repo, pipeline_steps)
2164
Florian Weikert5f5d3cb2019-04-15 15:36:27 +02002165 task_configs = filter_tasks_that_should_be_skipped(task_configs, pipeline_steps)
Jakob Buchgraberff2bdad2018-02-25 13:06:30 +01002166
Philipp Wollermanndac65512019-02-05 22:14:10 +01002167 # In Bazel Downstream Project pipelines, git_repository and project_name must be specified.
2168 is_downstream_project = (use_but or incompatible_flags) and git_repository and project_name
2169
Florian Weikert85208912019-03-07 17:08:39 +01002170 buildifier_config = configs.get("buildifier")
Philipp Wollermanndac65512019-02-05 22:14:10 +01002171 # Skip Buildifier when we test downstream projects.
Florian Weikert85208912019-03-07 17:08:39 +01002172 if buildifier_config and not is_downstream_project:
2173 buildifier_env_vars = {}
2174 if isinstance(buildifier_config, str):
2175 # Simple format:
2176 # ---
2177 # buildifier: latest
2178 buildifier_env_vars[BUILDIFIER_VERSION_ENV_VAR] = buildifier_config
2179 else:
2180 # Advanced format:
2181 # ---
2182 # buildifier:
2183 # version: latest
2184 # warnings: all
2185
Philipp Wollermannce986af2019-07-18 14:46:05 +02002186 def set_env_var(config_key, env_var_name):
Florian Weikert85208912019-03-07 17:08:39 +01002187 if config_key in buildifier_config:
2188 buildifier_env_vars[env_var_name] = buildifier_config[config_key]
2189
Philipp Wollermannce986af2019-07-18 14:46:05 +02002190 set_env_var("version", BUILDIFIER_VERSION_ENV_VAR)
2191 set_env_var("warnings", BUILDIFIER_WARNINGS_ENV_VAR)
Florian Weikert85208912019-03-07 17:08:39 +01002192
2193 if not buildifier_env_vars:
2194 raise BuildkiteException(
2195 'Invalid buildifier configuration entry "{}"'.format(buildifier_config)
2196 )
2197
Florian Weikert29cb7ec2019-03-07 14:52:18 +01002198 pipeline_steps.append(
2199 create_docker_step(
Florian Weikertde96a6f2019-03-07 14:57:50 +01002200 BUILDIFIER_STEP_NAME,
Florian Weikert29cb7ec2019-03-07 14:52:18 +01002201 image=BUILDIFIER_DOCKER_IMAGE,
Florian Weikert85208912019-03-07 17:08:39 +01002202 additional_env_vars=buildifier_env_vars,
Florian Weikert29cb7ec2019-03-07 14:52:18 +01002203 )
2204 )
Philipp Wollermannc05ac682019-01-19 12:37:28 +01002205
Philipp Wollermanndac65512019-02-05 22:14:10 +01002206 # In Bazel Downstream Project pipelines, we should test the project at the last green commit.
Yun Peng376d2b32018-11-29 10:24:54 +01002207 git_commit = None
Philipp Wollermanndac65512019-02-05 22:14:10 +01002208 if is_downstream_project:
Florian Weikert35906542019-04-01 11:53:53 +02002209 last_green_commit_url = bazelci_last_green_commit_url(
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002210 git_repository, DOWNSTREAM_PROJECTS[project_name]["pipeline_slug"]
2211 )
Florian Weikert35906542019-04-01 11:53:53 +02002212 git_commit = get_last_green_commit(last_green_commit_url)
Philipp Wollermanndac65512019-02-05 22:14:10 +01002213
Florian Weikert854fd852019-06-04 16:44:19 +02002214 config_hashes = set()
Florian Weikert843d7a02019-02-03 17:24:50 +01002215 for task, task_config in task_configs.items():
Florian Weikert854fd852019-06-04 16:44:19 +02002216 # We override the Bazel version in downstream pipelines. This means that two tasks that
2217 # only differ in the value of their explicit "bazel" field will be identical in the
2218 # downstream pipeline, thus leading to duplicate work.
2219 # Consequently, we filter those duplicate tasks here.
2220 if is_downstream_project:
2221 h = hash_task_config(task, task_config)
2222 if h in config_hashes:
2223 continue
Florian Weikert854fd852019-06-04 16:44:19 +02002224 config_hashes.add(h)
2225
Florian Weikert736d06e2019-05-08 13:16:42 +02002226 shards = task_config.get("shards", "1")
2227 try:
2228 shards = int(shards)
2229 except ValueError:
2230 raise BuildkiteException("Task {} has invalid shard value '{}'".format(task, shards))
2231
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002232 step = runner_step(
Florian Weikert843d7a02019-02-03 17:24:50 +01002233 platform=get_platform_for_task(task, task_config),
2234 task=task,
2235 task_name=task_config.get("name"),
2236 project_name=project_name,
2237 http_config=http_config,
2238 file_config=file_config,
2239 git_repository=git_repository,
2240 git_commit=git_commit,
2241 monitor_flaky_tests=monitor_flaky_tests,
2242 use_but=use_but,
2243 incompatible_flags=incompatible_flags,
Florian Weikert736d06e2019-05-08 13:16:42 +02002244 shards=shards,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002245 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002246 pipeline_steps.append(step)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002247
Yun Peng996efad2018-11-27 17:19:44 +01002248 pipeline_slug = os.getenv("BUILDKITE_PIPELINE_SLUG")
2249 all_downstream_pipeline_slugs = []
2250 for _, config in DOWNSTREAM_PROJECTS.items():
2251 all_downstream_pipeline_slugs.append(config["pipeline_slug"])
2252 # We don't need to update last green commit in the following cases:
Philipp Wollermannfce92bf2019-05-22 15:14:32 +02002253 # 1. This job is a GitHub pull request
2254 # 2. This job uses a custom built Bazel binary (in Bazel Downstream Projects pipeline)
2255 # 3. This job doesn't run on master branch (could be a custom build launched manually)
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002256 # 4. We don't intend to run the same job in downstream with Bazel@HEAD (eg. google-bazel-presubmit)
2257 # 5. We are testing incompatible flags
Florian Weikert84249312019-10-28 14:48:46 +01002258 # 6. We are running `bazelisk --migrate` in a non-downstream pipeline
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002259 if not (
2260 is_pull_request()
2261 or use_but
2262 or os.getenv("BUILDKITE_BRANCH") != "master"
2263 or pipeline_slug not in all_downstream_pipeline_slugs
2264 or incompatible_flags
Florian Weikert84249312019-10-28 14:48:46 +01002265 or use_bazelisk_migrate()
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002266 ):
Florian Weikertde96a6f2019-03-07 14:57:50 +01002267 # We need to call "Try Update Last Green Commit" even if there are failures,
2268 # since we don't want a failing Buildifier step to block the update of
2269 # the last green commit for this project.
2270 # try_update_last_green_commit() ensures that we don't update the commit
2271 # if any build or test steps fail.
2272 pipeline_steps.append({"wait": None, "continue_on_failure": True})
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002273 pipeline_steps.append(
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002274 create_step(
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002275 label="Try Update Last Green Commit",
2276 commands=[
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002277 fetch_bazelcipy_command(),
Philipp Wollermann57b32682019-05-18 22:09:27 +02002278 PLATFORMS[DEFAULT_PLATFORM]["python"]
2279 + " bazelci.py try_update_last_green_commit",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002280 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002281 platform=DEFAULT_PLATFORM,
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002282 )
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002283 )
Yun Peng43239b02018-11-23 13:57:34 +01002284
Florian Weikert778251c2019-04-25 15:14:44 +02002285 if "validate_config" in configs:
Florian Weikertf52f91a2019-05-08 15:19:30 +02002286 pipeline_steps += create_config_validation_steps()
Florian Weikert778251c2019-04-25 15:14:44 +02002287
Florian Weikert09813a02019-10-26 19:34:33 +02002288 if use_bazelisk_migrate() and not is_downstream_project:
2289 # Print results of bazelisk --migrate in project pipelines that explicitly set
2290 # the USE_BAZELISK_MIGRATE env var, but that are not being run as part of a
2291 # downstream pipeline.
2292 number = os.getenv("BUILDKITE_BUILD_NUMBER")
Florian Weikert60661912019-12-18 15:17:10 +01002293 pipeline_steps += get_steps_for_aggregating_migration_results(number, notify)
Florian Weikert09813a02019-10-26 19:34:33 +02002294
Florian Weikertd79dc502019-05-13 09:51:05 +02002295 print_pipeline_steps(pipeline_steps, handle_emergencies=not is_downstream_project)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002296
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002297
mai93f2e116c2020-10-19 09:33:14 +02002298def show_gerrit_review_link(git_repository, pipeline_steps):
2299 host = re.search(r"https://(.+?)\.googlesource", git_repository).group(1)
2300 if not host:
2301 raise BuildkiteException("Couldn't get host name from %s" % git_repository)
2302 text = "The transformed code used in this pipeline can be found under https://{}-review.googlesource.com/q/{}". \
2303 format(host, os.getenv("BUILDKITE_COMMIT"))
2304 commands = ["buildkite-agent annotate --style=info '{}'".format(text)]
2305 pipeline_steps.append(
2306 create_step(
2307 label=":pipeline: Print information about Gerrit Review Link",
2308 commands=commands,
2309 platform=DEFAULT_PLATFORM,
2310 )
2311 )
2312
2313
2314def is_git_on_borg_repo(git_repository):
2315 return git_repository and "googlesource.com" in git_repository
2316
2317
Florian Weikert854fd852019-06-04 16:44:19 +02002318def hash_task_config(task_name, task_config):
2319 # Two task configs c1 and c2 have the same hash iff they lead to two functionally identical jobs
2320 # in the downstream pipeline. This function discards the "bazel" field (since it's being
Philipp Wollermannce986af2019-07-18 14:46:05 +02002321 # overridden) and the "name" field (since it has no effect on the actual work).
Florian Weikert854fd852019-06-04 16:44:19 +02002322 # Moreover, it adds an explicit "platform" field if that's missing.
2323 cpy = task_config.copy()
2324 cpy.pop("bazel", None)
2325 cpy.pop("name", None)
2326 if "platform" not in cpy:
2327 cpy["platform"] = task_name
2328
2329 m = hashlib.md5()
2330 for key in sorted(cpy):
Florian Weikert8186c392019-06-05 12:53:39 +02002331 value = "%s:%s;" % (key, cpy[key])
2332 m.update(value.encode("utf-8"))
Florian Weikert854fd852019-06-04 16:44:19 +02002333
2334 return m.digest()
2335
2336
Florian Weikert843d7a02019-02-03 17:24:50 +01002337def get_platform_for_task(task, task_config):
2338 # Most pipeline configurations have exactly one task per platform, which makes it
2339 # convenient to use the platform name as task ID. Consequently, we use the
2340 # task ID as platform if there is no explicit "platform" field.
2341 return task_config.get("platform", task)
2342
2343
Florian Weikertf52f91a2019-05-08 15:19:30 +02002344def create_config_validation_steps():
2345 output = execute_command_and_get_output(
2346 ["git", "diff-tree", "--no-commit-id", "--name-only", "-r", os.getenv("BUILDKITE_COMMIT")]
2347 )
2348 config_files = [
2349 l
2350 for l in output.split("\n")
2351 if l.startswith(".bazelci/") and os.path.splitext(l)[1] in CONFIG_FILE_EXTENSIONS
2352 ]
Florian Weikertf52f91a2019-05-08 15:19:30 +02002353 return [
2354 create_step(
2355 label=":cop: Validate {}".format(f),
2356 commands=[
2357 fetch_bazelcipy_command(),
2358 "{} bazelci.py project_pipeline --file_config={}".format(
Philipp Wollermann57b32682019-05-18 22:09:27 +02002359 PLATFORMS[DEFAULT_PLATFORM]["python"], f
Florian Weikertf52f91a2019-05-08 15:19:30 +02002360 ),
2361 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002362 platform=DEFAULT_PLATFORM,
Florian Weikertf52f91a2019-05-08 15:19:30 +02002363 )
2364 for f in config_files
2365 ]
2366
2367
Florian Weikertd79dc502019-05-13 09:51:05 +02002368def print_pipeline_steps(pipeline_steps, handle_emergencies=True):
2369 if handle_emergencies:
2370 emergency_step = create_emergency_announcement_step_if_necessary()
2371 if emergency_step:
2372 pipeline_steps.insert(0, emergency_step)
Florian Weikert13215a82019-05-10 12:42:21 +02002373
2374 print(yaml.dump({"steps": pipeline_steps}))
2375
2376
2377def create_emergency_announcement_step_if_necessary():
2378 style = "error"
2379 message, issue_url, last_good_bazel = None, None, None
2380 try:
2381 emergency_settings = load_remote_yaml_file(EMERGENCY_FILE_URL)
2382 message = emergency_settings.get("message")
2383 issue_url = emergency_settings.get("issue_url")
2384 last_good_bazel = emergency_settings.get("last_good_bazel")
2385 except urllib.error.HTTPError as ex:
2386 message = str(ex)
2387 style = "warning"
2388
2389 if not any([message, issue_url, last_good_bazel]):
2390 return
2391
2392 text = '<span class="h1">:rotating_light: Emergency :rotating_light:</span>\n'
2393 if message:
2394 text += "- {}\n".format(message)
2395 if issue_url:
2396 text += '- Please check this <a href="{}">issue</a> for more details.\n'.format(issue_url)
2397 if last_good_bazel:
2398 text += (
2399 "- Default Bazel version is *{}*, "
2400 "unless the pipeline configuration specifies an explicit version."
2401 ).format(last_good_bazel)
2402
2403 return create_step(
2404 label=":rotating_light: Emergency :rotating_light:",
Philipp Wollermann7590b962019-05-16 11:35:03 +02002405 commands=[
2406 'buildkite-agent annotate --append --style={} --context "omg" "{}"'.format(style, text)
2407 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002408 platform=DEFAULT_PLATFORM,
Florian Weikert13215a82019-05-10 12:42:21 +02002409 )
2410
2411
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002412def runner_step(
2413 platform,
Florian Weikert843d7a02019-02-03 17:24:50 +01002414 task,
2415 task_name=None,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002416 project_name=None,
2417 http_config=None,
2418 file_config=None,
2419 git_repository=None,
2420 git_commit=None,
2421 monitor_flaky_tests=False,
2422 use_but=False,
2423 incompatible_flags=None,
Florian Weikert736d06e2019-05-08 13:16:42 +02002424 shards=1,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002425):
Philipp Wollermann57b32682019-05-18 22:09:27 +02002426 command = PLATFORMS[platform]["python"] + " bazelci.py runner --task=" + task
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002427 if http_config:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002428 command += " --http_config=" + http_config
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01002429 if file_config:
2430 command += " --file_config=" + file_config
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002431 if git_repository:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002432 command += " --git_repository=" + git_repository
Yun Peng376d2b32018-11-29 10:24:54 +01002433 if git_commit:
2434 command += " --git_commit=" + git_commit
Jakob Buchgraberc340f582018-06-22 13:48:33 +02002435 if monitor_flaky_tests:
2436 command += " --monitor_flaky_tests"
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002437 if use_but:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002438 command += " --use_but"
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002439 for flag in incompatible_flags or []:
Yun Peng4be92b32018-11-30 09:48:29 +01002440 command += " --incompatible_flag=" + flag
Florian Weikert843d7a02019-02-03 17:24:50 +01002441 label = create_label(platform, project_name, task_name=task_name)
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002442 return create_step(
Florian Weikert736d06e2019-05-08 13:16:42 +02002443 label=label, commands=[fetch_bazelcipy_command(), command], platform=platform, shards=shards
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002444 )
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002445
2446
2447def fetch_bazelcipy_command():
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002448 return "curl -sS {0} -o bazelci.py".format(SCRIPT_URL)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002449
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002450
Yun Peng002eab92018-12-17 18:28:14 +01002451def fetch_incompatible_flag_verbose_failures_command():
Philipp Wollermannfe145a52019-01-11 13:16:48 +01002452 return "curl -sS {0} -o incompatible_flag_verbose_failures.py".format(
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002453 INCOMPATIBLE_FLAG_VERBOSE_FAILURES_URL
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002454 )
Yun Peng002eab92018-12-17 18:28:14 +01002455
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002456
Yun Peng8975c6b2019-02-28 11:55:55 +01002457def fetch_aggregate_incompatible_flags_test_result_command():
2458 return "curl -sS {0} -o aggregate_incompatible_flags_test_result.py".format(
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002459 AGGREGATE_INCOMPATIBLE_TEST_RESULT_URL
Yun Peng8975c6b2019-02-28 11:55:55 +01002460 )
2461
2462
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002463def upload_project_pipeline_step(
2464 project_name, git_repository, http_config, file_config, incompatible_flags
2465):
2466 pipeline_command = (
2467 '{0} bazelci.py project_pipeline --project_name="{1}" ' + "--git_repository={2}"
Philipp Wollermann57b32682019-05-18 22:09:27 +02002468 ).format(PLATFORMS[DEFAULT_PLATFORM]["python"], project_name, git_repository)
Philipp Wollermann639c0452019-01-03 11:23:54 +01002469 if incompatible_flags is None:
Yun Peng4be92b32018-11-30 09:48:29 +01002470 pipeline_command += " --use_but"
Yun Peng95908792018-11-30 15:03:55 +01002471 else:
2472 for flag in incompatible_flags:
2473 pipeline_command += " --incompatible_flag=" + flag
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002474 if http_config:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002475 pipeline_command += " --http_config=" + http_config
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01002476 if file_config:
2477 pipeline_command += " --file_config=" + file_config
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002478 pipeline_command += " | buildkite-agent pipeline upload"
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002479
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002480 return create_step(
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002481 label="Setup {0}".format(project_name),
2482 commands=[fetch_bazelcipy_command(), pipeline_command],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002483 platform=DEFAULT_PLATFORM,
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002484 )
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002485
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002486
Florian Weikert843d7a02019-02-03 17:24:50 +01002487def create_label(platform, project_name, build_only=False, test_only=False, task_name=None):
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002488 if build_only and test_only:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002489 raise BuildkiteException("build_only and test_only cannot be true at the same time")
Florian Weikert843d7a02019-02-03 17:24:50 +01002490 platform_display_name = PLATFORMS[platform]["emoji-name"]
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002491
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002492 if build_only:
2493 label = "Build "
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002494 elif test_only:
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002495 label = "Test "
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002496 else:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002497 label = ""
2498
Florian Weikert843d7a02019-02-03 17:24:50 +01002499 platform_label = (
2500 "{0} on {1}".format(task_name, platform_display_name)
2501 if task_name
2502 else platform_display_name
2503 )
2504
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002505 if project_name:
Florian Weikert843d7a02019-02-03 17:24:50 +01002506 label += "{0} ({1})".format(project_name, platform_label)
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002507 else:
Florian Weikert843d7a02019-02-03 17:24:50 +01002508 label += platform_label
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002509
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002510 return label
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002511
2512
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002513def bazel_build_step(
Florian Weikert843d7a02019-02-03 17:24:50 +01002514 task,
2515 platform,
2516 project_name,
2517 http_config=None,
2518 file_config=None,
2519 build_only=False,
2520 test_only=False,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002521):
Philipp Wollermann57b32682019-05-18 22:09:27 +02002522 pipeline_command = PLATFORMS[platform]["python"] + " bazelci.py runner"
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002523 if build_only:
Philipp Wollermannc52e26a2019-05-18 22:10:47 +02002524 pipeline_command += " --build_only --save_but"
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002525 if test_only:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002526 pipeline_command += " --test_only"
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002527 if http_config:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002528 pipeline_command += " --http_config=" + http_config
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01002529 if file_config:
2530 pipeline_command += " --file_config=" + file_config
Florian Weikert843d7a02019-02-03 17:24:50 +01002531 pipeline_command += " --task=" + task
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002532
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002533 return create_step(
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002534 label=create_label(platform, project_name, build_only, test_only),
2535 commands=[fetch_bazelcipy_command(), pipeline_command],
2536 platform=platform,
2537 )
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002538
2539
Florian Weikert5f5d3cb2019-04-15 15:36:27 +02002540def filter_tasks_that_should_be_skipped(task_configs, pipeline_steps):
2541 skip_tasks = get_skip_tasks()
2542 if not skip_tasks:
Florian Weikert5f5d3cb2019-04-15 15:36:27 +02002543 return task_configs
2544
2545 actually_skipped = []
2546 skip_tasks = set(skip_tasks)
2547 for task in list(task_configs.keys()):
2548 if task in skip_tasks:
2549 actually_skipped.append(task)
2550 del task_configs[task]
2551 skip_tasks.remove(task)
2552
2553 if not task_configs:
2554 raise BuildkiteException(
2555 "Nothing to do since all tasks in the configuration should be skipped."
2556 )
2557
2558 annotations = []
2559 if actually_skipped:
2560 annotations.append(
2561 ("info", "Skipping the following task(s): {}".format(", ".join(actually_skipped)))
2562 )
2563
2564 if skip_tasks:
2565 annotations.append(
2566 (
2567 "warning",
2568 (
2569 "The following tasks should have been skipped, "
2570 "but were not part of the configuration: {}"
2571 ).format(", ".join(skip_tasks)),
2572 )
2573 )
2574
2575 if annotations:
2576 print_skip_task_annotations(annotations, pipeline_steps)
2577
2578 return task_configs
2579
2580
2581def get_skip_tasks():
2582 value = os.getenv(SKIP_TASKS_ENV_VAR, "")
2583 return [v for v in value.split(",") if v]
2584
2585
2586def print_skip_task_annotations(annotations, pipeline_steps):
2587 commands = [
2588 "buildkite-agent annotate --style={} '{}' --context 'ctx-{}'".format(s, t, hash(t))
2589 for s, t in annotations
2590 ]
2591 pipeline_steps.append(
Philipp Wollermann7a185322019-05-18 22:15:48 +02002592 create_step(
2593 label=":pipeline: Print information about skipped tasks",
2594 commands=commands,
2595 platform=DEFAULT_PLATFORM,
2596 )
Florian Weikert5f5d3cb2019-04-15 15:36:27 +02002597 )
2598
2599
Florian Weikert843d7a02019-02-03 17:24:50 +01002600def print_bazel_publish_binaries_pipeline(task_configs, http_config, file_config):
2601 if not task_configs:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002602 raise BuildkiteException("Bazel publish binaries pipeline configuration is empty.")
2603
Florian Weikert5f5d3cb2019-04-15 15:36:27 +02002604 pipeline_steps = []
2605 task_configs = filter_tasks_that_should_be_skipped(task_configs, pipeline_steps)
2606
Florian Weikert843d7a02019-02-03 17:24:50 +01002607 platforms = [get_platform_for_task(t, tc) for t, tc in task_configs.items()]
Philipp Wollermann783d1672019-06-06 13:35:30 +02002608
2609 # These are the platforms that the bazel_publish_binaries.yml config is actually building.
2610 configured_platforms = set(filter(should_publish_binaries_for_platform, platforms))
Philipp Wollermanna2ea5d82018-08-27 14:12:10 +02002611
Philipp Wollermann783d1672019-06-06 13:35:30 +02002612 # These are the platforms that we want to build and publish according to this script.
2613 expected_platforms = set(filter(should_publish_binaries_for_platform, PLATFORMS))
Florian Weikert843d7a02019-02-03 17:24:50 +01002614
Philipp Wollermannf64bf942019-06-06 14:34:41 +02002615 if not expected_platforms.issubset(configured_platforms):
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002616 raise BuildkiteException(
2617 "Bazel publish binaries pipeline needs to build Bazel for every commit on all publish_binary-enabled platforms."
2618 )
Jakob Buchgraber08e8e402018-03-20 19:22:07 +01002619
Yun Pengd352b6d2018-10-17 13:28:39 +02002620 # Build Bazel
Florian Weikert843d7a02019-02-03 17:24:50 +01002621 for task, task_config in task_configs.items():
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002622 pipeline_steps.append(
Florian Weikert843d7a02019-02-03 17:24:50 +01002623 bazel_build_step(
2624 task,
2625 get_platform_for_task(task, task_config),
2626 "Bazel",
2627 http_config,
2628 file_config,
2629 build_only=True,
2630 )
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002631 )
Jakob Buchgraber4631a032018-03-22 17:12:46 +01002632
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002633 pipeline_steps.append("wait")
Jakob Buchgraber9d6ca8a2018-03-22 17:30:09 +01002634
Yun Pengc2dd6522018-10-17 12:58:35 +02002635 # If all builds succeed, publish the Bazel binaries to GCS.
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002636 pipeline_steps.append(
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002637 create_step(
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002638 label="Publish Bazel Binaries",
Philipp Wollermann57b32682019-05-18 22:09:27 +02002639 commands=[
2640 fetch_bazelcipy_command(),
2641 PLATFORMS[DEFAULT_PLATFORM]["python"] + " bazelci.py publish_binaries",
2642 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002643 platform=DEFAULT_PLATFORM,
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002644 )
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002645 )
Jakob Buchgraber08e8e402018-03-20 19:22:07 +01002646
Florian Weikert13215a82019-05-10 12:42:21 +02002647 print_pipeline_steps(pipeline_steps)
Jakob Buchgraber08e8e402018-03-20 19:22:07 +01002648
2649
Florian Weikert843d7a02019-02-03 17:24:50 +01002650def should_publish_binaries_for_platform(platform):
2651 if platform not in PLATFORMS:
2652 raise BuildkiteException("Unknown platform '{}'".format(platform))
2653
2654 return PLATFORMS[platform]["publish_binary"]
2655
2656
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01002657def print_disabled_projects_info_box_step():
2658 info_text = ["Downstream testing is disabled for the following projects :sadpanda:"]
2659 for project, config in DOWNSTREAM_PROJECTS.items():
2660 disabled_reason = config.get("disabled_reason", None)
2661 if disabled_reason:
2662 info_text.append("* **%s**: %s" % (project, disabled_reason))
2663
2664 if len(info_text) == 1:
2665 return None
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002666 return create_step(
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002667 label=":sadpanda:",
2668 commands=[
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002669 'buildkite-agent annotate --append --style=info "\n' + "\n".join(info_text) + '\n"'
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01002670 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002671 platform=DEFAULT_PLATFORM,
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002672 )
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01002673
Yun Peng6528e652019-01-02 14:41:07 +01002674
2675def print_incompatible_flags_info_box_step(incompatible_flags_map):
2676 info_text = ["Build and test with the following incompatible flags:"]
2677
2678 for flag in incompatible_flags_map:
2679 info_text.append("* **%s**: %s" % (flag, incompatible_flags_map[flag]))
2680
2681 if len(info_text) == 1:
2682 return None
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002683 return create_step(
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002684 label="Incompatible flags info",
2685 commands=[
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002686 'buildkite-agent annotate --append --style=info "\n' + "\n".join(info_text) + '\n"'
Yun Peng6528e652019-01-02 14:41:07 +01002687 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002688 platform=DEFAULT_PLATFORM,
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002689 )
Yun Peng6528e652019-01-02 14:41:07 +01002690
2691
Yun Peng7d302f62019-01-10 16:56:15 +01002692def fetch_incompatible_flags():
Yun Peng6528e652019-01-02 14:41:07 +01002693 """
2694 Return a list of incompatible flags to be tested in downstream with the current release Bazel
2695 """
Yun Peng7d302f62019-01-10 16:56:15 +01002696 incompatible_flags = {}
2697
2698 # If INCOMPATIBLE_FLAGS environment variable is set, we get incompatible flags from it.
2699 if "INCOMPATIBLE_FLAGS" in os.environ:
2700 for flag in os.environ["INCOMPATIBLE_FLAGS"].split():
2701 # We are not able to get the github link for this flag from INCOMPATIBLE_FLAGS,
2702 # so just assign the url to empty string.
2703 incompatible_flags[flag] = ""
2704 return incompatible_flags
2705
Florian Weikertf4302192019-12-23 16:13:57 +01002706 bazel_major_version = get_bazel_major_version()
Yun Peng6528e652019-01-02 14:41:07 +01002707
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002708 output = subprocess.check_output(
2709 [
2710 "curl",
Philipp Wollermann6f4058f2019-08-21 13:58:50 +02002711 "https://api.github.com/search/issues?per_page=100&q=repo:bazelbuild/bazel+label:migration-%s"
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002712 % bazel_major_version,
2713 ]
2714 ).decode("utf-8")
Yun Peng6528e652019-01-02 14:41:07 +01002715 issue_info = json.loads(output)
2716
Yun Peng6528e652019-01-02 14:41:07 +01002717 for issue in issue_info["items"]:
Yun Peng6528e652019-01-02 14:41:07 +01002718 # Every incompatible flags issue should start with "<incompatible flag name (without --)>:"
2719 name = "--" + issue["title"].split(":")[0]
2720 url = issue["html_url"]
2721 if name.startswith("--incompatible_"):
2722 incompatible_flags[name] = url
2723 else:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002724 eprint(
Philipp Wollermann639c0452019-01-03 11:23:54 +01002725 f"{name} is not recognized as an incompatible flag, please modify the issue title "
2726 f'of {url} to "<incompatible flag name (without --)>:..."'
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002727 )
Yun Peng6528e652019-01-02 14:41:07 +01002728
2729 return incompatible_flags
2730
2731
Florian Weikertf4302192019-12-23 16:13:57 +01002732def get_bazel_major_version():
2733 # Get bazel major version on CI, eg. 0.21 from "Build label: 0.21.0\n..."
2734 output = subprocess.check_output(
2735 ["bazel", "--nomaster_bazelrc", "--bazelrc=/dev/null", "version"]
2736 ).decode("utf-8")
2737 return output.split()[2].rsplit(".", 1)[0]
2738
2739
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002740def print_bazel_downstream_pipeline(
Florian Weikert60661912019-12-18 15:17:10 +01002741 task_configs, http_config, file_config, test_incompatible_flags, test_disabled_projects, notify
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002742):
Florian Weikert843d7a02019-02-03 17:24:50 +01002743 if not task_configs:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002744 raise BuildkiteException("Bazel downstream pipeline configuration is empty.")
2745
Florian Weikert5f5d3cb2019-04-15 15:36:27 +02002746 pipeline_steps = []
2747 task_configs = filter_tasks_that_should_be_skipped(task_configs, pipeline_steps)
2748
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002749 pipeline_steps = []
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002750
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01002751 info_box_step = print_disabled_projects_info_box_step()
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01002752 if info_box_step is not None:
2753 pipeline_steps.append(info_box_step)
2754
Yun Peng5599ca22019-01-16 12:32:41 +01002755 if not test_incompatible_flags:
Florian Weikert843d7a02019-02-03 17:24:50 +01002756 for task, task_config in task_configs.items():
Yun Peng5599ca22019-01-16 12:32:41 +01002757 pipeline_steps.append(
Florian Weikert843d7a02019-02-03 17:24:50 +01002758 bazel_build_step(
2759 task,
2760 get_platform_for_task(task, task_config),
2761 "Bazel",
2762 http_config,
2763 file_config,
2764 build_only=True,
2765 )
Yun Peng5599ca22019-01-16 12:32:41 +01002766 )
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002767
Yun Peng5599ca22019-01-16 12:32:41 +01002768 pipeline_steps.append("wait")
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002769
Yun Pengb9998d12018-12-03 10:18:28 +01002770 incompatible_flags = None
Yun Peng7a539ef2018-11-30 15:07:24 +01002771 if test_incompatible_flags:
Yun Peng7d302f62019-01-10 16:56:15 +01002772 incompatible_flags_map = fetch_incompatible_flags()
Yun Peng3c1d7d12020-06-30 14:58:34 +02002773 if not incompatible_flags_map:
2774 raise BuildkiteException("No incompatible flag issue is found on github for current version of Bazel.")
Yun Peng6528e652019-01-02 14:41:07 +01002775 info_box_step = print_incompatible_flags_info_box_step(incompatible_flags_map)
2776 if info_box_step is not None:
2777 pipeline_steps.append(info_box_step)
2778 incompatible_flags = list(incompatible_flags_map.keys())
Yun Peng7a539ef2018-11-30 15:07:24 +01002779
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002780 for project, config in DOWNSTREAM_PROJECTS.items():
Yun Peng996efad2018-11-27 17:19:44 +01002781 disabled_reason = config.get("disabled_reason", None)
Yun Pengfb759fa2018-12-13 11:35:39 +01002782 # If test_disabled_projects is true, we add configs for disabled projects.
Florian Weikert7b3f17e2019-03-14 13:52:42 +01002783 # If test_disabled_projects is false, we add configs for not disabled projects.
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002784 if (test_disabled_projects and disabled_reason) or (
2785 not test_disabled_projects and not disabled_reason
2786 ):
Yun Peng996efad2018-11-27 17:19:44 +01002787 pipeline_steps.append(
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002788 upload_project_pipeline_step(
2789 project_name=project,
2790 git_repository=config["git_repository"],
2791 http_config=config.get("http_config", None),
2792 file_config=config.get("file_config", None),
2793 incompatible_flags=incompatible_flags,
2794 )
2795 )
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002796
Yun Peng002eab92018-12-17 18:28:14 +01002797 if test_incompatible_flags:
Yun Peng002eab92018-12-17 18:28:14 +01002798 current_build_number = os.environ.get("BUILDKITE_BUILD_NUMBER", None)
2799 if not current_build_number:
2800 raise BuildkiteException("Not running inside Buildkite")
Yun Peng8975c6b2019-02-28 11:55:55 +01002801 if use_bazelisk_migrate():
Florian Weikert09813a02019-10-26 19:34:33 +02002802 pipeline_steps += get_steps_for_aggregating_migration_results(
Florian Weikert60661912019-12-18 15:17:10 +01002803 current_build_number, notify
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002804 )
Yun Peng8975c6b2019-02-28 11:55:55 +01002805 else:
2806 pipeline_steps.append({"wait": "~", "continue_on_failure": "true"})
2807 pipeline_steps.append(
2808 create_step(
2809 label="Test failing jobs with incompatible flag separately",
2810 commands=[
2811 fetch_bazelcipy_command(),
2812 fetch_incompatible_flag_verbose_failures_command(),
Philipp Wollermann57b32682019-05-18 22:09:27 +02002813 PLATFORMS[DEFAULT_PLATFORM]["python"]
Yun Peng8975c6b2019-02-28 11:55:55 +01002814 + " incompatible_flag_verbose_failures.py --build_number=%s | buildkite-agent pipeline upload"
2815 % current_build_number,
2816 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002817 platform=DEFAULT_PLATFORM,
Yun Peng8975c6b2019-02-28 11:55:55 +01002818 )
2819 )
Yun Peng002eab92018-12-17 18:28:14 +01002820
Florian Weikert2896edb2019-04-04 16:12:47 +02002821 if (
2822 not test_disabled_projects
2823 and not test_incompatible_flags
2824 and os.getenv("BUILDKITE_BRANCH") == "master"
2825 ):
Florian Weikert35906542019-04-01 11:53:53 +02002826 # Only update the last green downstream commit in the regular Bazel@HEAD + Downstream pipeline.
2827 pipeline_steps.append("wait")
2828 pipeline_steps.append(
2829 create_step(
2830 label="Try Update Last Green Downstream Commit",
2831 commands=[
2832 fetch_bazelcipy_command(),
Philipp Wollermann57b32682019-05-18 22:09:27 +02002833 PLATFORMS[DEFAULT_PLATFORM]["python"]
2834 + " bazelci.py try_update_last_green_downstream_commit",
Florian Weikert35906542019-04-01 11:53:53 +02002835 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002836 platform=DEFAULT_PLATFORM,
Florian Weikert35906542019-04-01 11:53:53 +02002837 )
2838 )
2839
Florian Weikert13215a82019-05-10 12:42:21 +02002840 print_pipeline_steps(pipeline_steps)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002841
2842
Florian Weikert60661912019-12-18 15:17:10 +01002843def get_steps_for_aggregating_migration_results(current_build_number, notify):
Florian Weikert09813a02019-10-26 19:34:33 +02002844 parts = [
2845 PLATFORMS[DEFAULT_PLATFORM]["python"],
2846 "aggregate_incompatible_flags_test_result.py",
2847 "--build_number=%s" % current_build_number,
Florian Weikert09813a02019-10-26 19:34:33 +02002848 ]
Florian Weikert60661912019-12-18 15:17:10 +01002849 if notify:
2850 parts.append("--notify")
Florian Weikert09813a02019-10-26 19:34:33 +02002851 return [
2852 {"wait": "~", "continue_on_failure": "true"},
2853 create_step(
2854 label="Aggregate incompatible flags test result",
2855 commands=[
2856 fetch_bazelcipy_command(),
2857 fetch_aggregate_incompatible_flags_test_result_command(),
2858 " ".join(parts),
2859 ],
2860 platform=DEFAULT_PLATFORM,
2861 ),
2862 ]
2863
2864
Yun Pengc2dd6522018-10-17 12:58:35 +02002865def bazelci_builds_download_url(platform, git_commit):
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002866 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-builds"
2867 return "https://storage.googleapis.com/{}/artifacts/{}/{}/bazel".format(
2868 bucket_name, platform, git_commit
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002869 )
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002870
2871
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04002872def bazelci_builds_nojdk_download_url(platform, git_commit):
2873 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-builds"
2874 return "https://storage.googleapis.com/{}/artifacts/{}/{}/bazel_nojdk".format(
2875 bucket_name, platform, git_commit
2876 )
2877
2878
Yun Peng20d45602018-10-18 13:27:05 +02002879def bazelci_builds_gs_url(platform, git_commit):
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002880 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-builds"
2881 return "gs://{}/artifacts/{}/{}/bazel".format(bucket_name, platform, git_commit)
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002882
2883
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04002884def bazelci_builds_nojdk_gs_url(platform, git_commit):
2885 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-builds"
2886 return "gs://{}/artifacts/{}/{}/bazel_nojdk".format(bucket_name, platform, git_commit)
2887
2888
mai93f04f9482020-10-20 17:22:30 +02002889def bazelci_latest_build_metadata_url():
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002890 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-builds"
2891 return "gs://{}/metadata/latest.json".format(bucket_name)
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002892
2893
mai93f04f9482020-10-20 17:22:30 +02002894def bazelci_builds_metadata_url(git_commit):
2895 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-builds"
2896 return "gs://{}/metadata/{}.json".format(bucket_name, git_commit)
2897
2898
Yun Peng996efad2018-11-27 17:19:44 +01002899def bazelci_last_green_commit_url(git_repository, pipeline_slug):
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002900 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-untrusted-builds"
2901 return "gs://{}/last_green_commit/{}/{}".format(
2902 bucket_name, git_repository[len("https://") :], pipeline_slug
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002903 )
Yun Pengafe67d42018-11-23 17:06:43 +01002904
2905
Florian Weikert35906542019-04-01 11:53:53 +02002906def bazelci_last_green_downstream_commit_url():
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002907 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-untrusted-builds"
2908 return "gs://{}/last_green_commit/downstream_pipeline".format(bucket_name)
Florian Weikert35906542019-04-01 11:53:53 +02002909
2910
2911def get_last_green_commit(last_green_commit_url):
Yun Peng61a448f2018-11-23 17:11:46 +01002912 try:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002913 return (
2914 subprocess.check_output(
2915 [gsutil_command(), "cat", last_green_commit_url], env=os.environ
2916 )
2917 .decode("utf-8")
2918 .strip()
2919 )
Yun Peng61a448f2018-11-23 17:11:46 +01002920 except subprocess.CalledProcessError:
2921 return None
Yun Peng43239b02018-11-23 13:57:34 +01002922
2923
Yun Peng358cd882018-11-29 10:25:18 +01002924def try_update_last_green_commit():
Florian Weikertde96a6f2019-03-07 14:57:50 +01002925 org_slug = os.getenv("BUILDKITE_ORGANIZATION_SLUG")
Yun Peng358cd882018-11-29 10:25:18 +01002926 pipeline_slug = os.getenv("BUILDKITE_PIPELINE_SLUG")
Florian Weikertde96a6f2019-03-07 14:57:50 +01002927 build_number = os.getenv("BUILDKITE_BUILD_NUMBER")
2928 current_job_id = os.getenv("BUILDKITE_JOB_ID")
2929
2930 client = BuildkiteClient(org=org_slug, pipeline=pipeline_slug)
2931 build_info = client.get_build_info(build_number)
2932
2933 # Find any failing steps other than Buildifier and "try update last green".
Philipp Wollermannce986af2019-07-18 14:46:05 +02002934 def has_failed(job):
Florian Weikertbd40a272019-03-08 10:20:18 +01002935 state = job.get("state")
2936 # Ignore steps that don't have a state (like "wait").
Florian Weikertde96a6f2019-03-07 14:57:50 +01002937 return (
Florian Weikert35906542019-04-01 11:53:53 +02002938 state is not None
2939 and state != "passed"
Florian Weikertde96a6f2019-03-07 14:57:50 +01002940 and job["id"] != current_job_id
2941 and job["name"] != BUILDIFIER_STEP_NAME
2942 )
2943
Philipp Wollermannce986af2019-07-18 14:46:05 +02002944 failing_jobs = [j["name"] for j in build_info["jobs"] if has_failed(j)]
Florian Weikertde96a6f2019-03-07 14:57:50 +01002945 if failing_jobs:
2946 raise BuildkiteException(
2947 "Cannot update last green commit due to {} failing step(s): {}".format(
2948 len(failing_jobs), ", ".join(failing_jobs)
2949 )
2950 )
2951
Yun Peng358cd882018-11-29 10:25:18 +01002952 git_repository = os.getenv("BUILDKITE_REPO")
Florian Weikert35906542019-04-01 11:53:53 +02002953 last_green_commit_url = bazelci_last_green_commit_url(git_repository, pipeline_slug)
2954 update_last_green_commit_if_newer(last_green_commit_url)
2955
2956
2957def update_last_green_commit_if_newer(last_green_commit_url):
2958 last_green_commit = get_last_green_commit(last_green_commit_url)
Yun Peng358cd882018-11-29 10:25:18 +01002959 current_commit = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("utf-8").strip()
2960 if last_green_commit:
Jakob Buchgraber7c7ceee2019-10-28 10:28:58 +01002961 success = False
2962 try:
2963 execute_command(["git", "fetch", "-v", "origin", last_green_commit])
2964 success = True
2965 except subprocess.CalledProcessError:
2966 # If there was an error fetching the commit it typically means
2967 # that the commit does not exist anymore - due to a force push. In
2968 # order to recover from that assume that the current commit is the
2969 # newest commit.
2970 result = [current_commit]
2971 finally:
2972 if success:
2973 result = (
2974 subprocess.check_output(
2975 ["git", "rev-list", "%s..%s" % (last_green_commit, current_commit)]
2976 )
2977 .decode("utf-8")
2978 .strip()
2979 )
Philipp Wollermannce986af2019-07-18 14:46:05 +02002980 else:
2981 result = None
Yun Peng358cd882018-11-29 10:25:18 +01002982
Philipp Wollermann639c0452019-01-03 11:23:54 +01002983 # If current_commit is newer that last_green_commit, `git rev-list A..B` will output a bunch of
2984 # commits, otherwise the output should be empty.
Yun Peng358cd882018-11-29 10:25:18 +01002985 if not last_green_commit or result:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002986 execute_command(
Philipp Wollermann76a7eac2020-02-17 18:29:52 +01002987 [
2988 "echo %s | %s -h 'Cache-Control: no-store' cp - %s"
2989 % (current_commit, gsutil_command(), last_green_commit_url)
2990 ],
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002991 shell=True,
2992 )
Yun Peng358cd882018-11-29 10:25:18 +01002993 else:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002994 eprint(
2995 "Updating abandoned: last green commit (%s) is not older than current commit (%s)."
2996 % (last_green_commit, current_commit)
2997 )
2998
Yun Peng358cd882018-11-29 10:25:18 +01002999
Florian Weikert35906542019-04-01 11:53:53 +02003000def try_update_last_green_downstream_commit():
3001 last_green_commit_url = bazelci_last_green_downstream_commit_url()
3002 update_last_green_commit_if_newer(last_green_commit_url)
3003
3004
Jakob Buchgraber76381e02018-02-19 16:19:56 +01003005def latest_generation_and_build_number():
Philipp Wollermannce986af2019-07-18 14:46:05 +02003006 generation = None
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003007 output = None
Philipp Wollermannce986af2019-07-18 14:46:05 +02003008 for attempt in range(5):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003009 output = subprocess.check_output(
mai93f04f9482020-10-20 17:22:30 +02003010 [gsutil_command(), "stat", bazelci_latest_build_metadata_url()], env=os.environ
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003011 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003012 match = re.search("Generation:[ ]*([0-9]+)", output.decode("utf-8"))
3013 if not match:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02003014 raise BuildkiteException("Couldn't parse generation. gsutil output format changed?")
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003015 generation = match.group(1)
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01003016
Philipp Wollermannff39ef52018-02-21 14:18:52 +01003017 match = re.search(r"Hash \(md5\):[ ]*([^\s]+)", output.decode("utf-8"))
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003018 if not match:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02003019 raise BuildkiteException("Couldn't parse md5 hash. gsutil output format changed?")
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003020 expected_md5hash = base64.b64decode(match.group(1))
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01003021
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003022 output = subprocess.check_output(
mai93f04f9482020-10-20 17:22:30 +02003023 [gsutil_command(), "cat", bazelci_latest_build_metadata_url()], env=os.environ
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003024 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003025 hasher = hashlib.md5()
3026 hasher.update(output)
3027 actual_md5hash = hasher.digest()
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01003028
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003029 if expected_md5hash == actual_md5hash:
3030 break
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003031 info = json.loads(output.decode("utf-8"))
Philipp Wollermannce986af2019-07-18 14:46:05 +02003032 return generation, info["build_number"]
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01003033
Jakob Buchgraber699aece2018-02-19 12:49:30 +01003034
Jakob Buchgraber88083fd2018-02-18 17:23:35 +01003035def sha256_hexdigest(filename):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003036 sha256 = hashlib.sha256()
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02003037 with open(filename, "rb") as f:
3038 for block in iter(lambda: f.read(65536), b""):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003039 sha256.update(block)
3040 return sha256.hexdigest()
Jakob Buchgraber699aece2018-02-19 12:49:30 +01003041
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01003042
Philipp Wollermann02955272019-04-18 18:00:48 +02003043def upload_bazel_binaries():
3044 """
3045 Uploads all Bazel binaries to a deterministic URL based on the current Git commit.
3046
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04003047 Returns maps of platform names to sha256 hashes of the corresponding bazel and bazel_nojdk binaries.
Philipp Wollermann02955272019-04-18 18:00:48 +02003048 """
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04003049 bazel_hashes = {}
3050 bazel_nojdk_hashes = {}
Philipp Wollermannbdd4bf92019-06-06 14:43:50 +02003051 for platform_name, platform in PLATFORMS.items():
Philipp Wollermann783d1672019-06-06 13:35:30 +02003052 if not should_publish_binaries_for_platform(platform_name):
3053 continue
Jakob Buchgraberb13a9a82018-03-27 18:37:09 +02003054 tmpdir = tempfile.mkdtemp()
3055 try:
Philipp Wollermann783d1672019-06-06 13:35:30 +02003056 bazel_binary_path = download_bazel_binary(tmpdir, platform_name)
3057 # One platform that we build on can generate binaries for multiple platforms, e.g.
Philipp Wollermannf4aabb72019-06-25 15:59:00 +02003058 # the centos7 platform generates binaries for the "centos7" platform, but also
Philipp Wollermann783d1672019-06-06 13:35:30 +02003059 # for the generic "linux" platform.
3060 for target_platform_name in platform["publish_binary"]:
3061 execute_command(
3062 [
3063 gsutil_command(),
3064 "cp",
3065 bazel_binary_path,
3066 bazelci_builds_gs_url(target_platform_name, os.environ["BUILDKITE_COMMIT"]),
3067 ]
3068 )
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04003069 bazel_hashes[target_platform_name] = sha256_hexdigest(bazel_binary_path)
3070
3071 # Also publish bazel_nojdk binaries.
3072 bazel_nojdk_binary_path = download_bazel_nojdk_binary(tmpdir, platform_name)
3073 for target_platform_name in platform["publish_binary"]:
3074 execute_command(
3075 [
3076 gsutil_command(),
3077 "cp",
3078 bazel_nojdk_binary_path,
3079 bazelci_builds_nojdk_gs_url(target_platform_name, os.environ["BUILDKITE_COMMIT"]),
3080 ]
3081 )
3082 bazel_nojdk_hashes[target_platform_name] = sha256_hexdigest(bazel_nojdk_binary_path)
Jakob Buchgraberb13a9a82018-03-27 18:37:09 +02003083 finally:
3084 shutil.rmtree(tmpdir)
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04003085 return bazel_hashes, bazel_nojdk_hashes
Philipp Wollermann02955272019-04-18 18:00:48 +02003086
3087
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04003088def try_publish_binaries(bazel_hashes, bazel_nojdk_hashes, build_number, expected_generation):
Philipp Wollermann02955272019-04-18 18:00:48 +02003089 """
3090 Uploads the info.json file that contains information about the latest Bazel commit that was
3091 successfully built on CI.
3092 """
3093 now = datetime.datetime.now()
3094 git_commit = os.environ["BUILDKITE_COMMIT"]
3095 info = {
3096 "build_number": build_number,
3097 "build_time": now.strftime("%d-%m-%Y %H:%M"),
3098 "git_commit": git_commit,
3099 "platforms": {},
3100 }
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04003101 for platform, sha256 in bazel_hashes.items():
Philipp Wollermann02955272019-04-18 18:00:48 +02003102 info["platforms"][platform] = {
3103 "url": bazelci_builds_download_url(platform, git_commit),
Philipp Wollermann783d1672019-06-06 13:35:30 +02003104 "sha256": sha256,
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04003105 "nojdk_url": bazelci_builds_nojdk_download_url(platform, git_commit),
3106 "nojdk_sha256": bazel_nojdk_hashes[platform],
Philipp Wollermann02955272019-04-18 18:00:48 +02003107 }
Jakob Buchgraberb13a9a82018-03-27 18:37:09 +02003108 tmpdir = tempfile.mkdtemp()
3109 try:
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003110 info_file = os.path.join(tmpdir, "info.json")
3111 with open(info_file, mode="w", encoding="utf-8") as fp:
Jakob Buchgraber609a20e2018-02-25 17:06:51 +01003112 json.dump(info, fp, indent=2, sort_keys=True)
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003113
3114 try:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003115 execute_command(
3116 [
3117 gsutil_command(),
3118 "-h",
3119 "x-goog-if-generation-match:" + expected_generation,
3120 "-h",
3121 "Content-Type:application/json",
3122 "cp",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003123 info_file,
mai93f04f9482020-10-20 17:22:30 +02003124 bazelci_latest_build_metadata_url(),
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003125 ]
3126 )
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003127 except subprocess.CalledProcessError:
3128 raise BinaryUploadRaceException()
mai93f04f9482020-10-20 17:22:30 +02003129
3130 execute_command(
3131 [
3132 gsutil_command(),
3133 "cp",
3134 bazelci_latest_build_metadata_url(),
3135 bazelci_builds_metadata_url(git_commit),
3136 ]
3137 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003138 finally:
Philipp Wollermann3e1a7712018-02-19 17:34:24 +01003139 shutil.rmtree(tmpdir)
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01003140
3141
Jakob Buchgraber76381e02018-02-19 16:19:56 +01003142def publish_binaries():
Philipp Wollermanndb024862018-02-19 17:16:56 +01003143 """
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003144 Publish Bazel binaries to GCS.
Philipp Wollermanndb024862018-02-19 17:16:56 +01003145 """
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003146 current_build_number = os.environ.get("BUILDKITE_BUILD_NUMBER", None)
3147 if not current_build_number:
3148 raise BuildkiteException("Not running inside Buildkite")
3149 current_build_number = int(current_build_number)
3150
Philipp Wollermann02955272019-04-18 18:00:48 +02003151 # Upload the Bazel binaries for this commit.
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04003152 bazel_hashes, bazel_nojdk_hashes = upload_bazel_binaries()
Philipp Wollermann02955272019-04-18 18:00:48 +02003153
3154 # Try to update the info.json with data about our build. This will fail (expectedly) if we're
3155 # not the latest build.
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003156 for _ in range(5):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003157 latest_generation, latest_build_number = latest_generation_and_build_number()
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01003158
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003159 if current_build_number <= latest_build_number:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003160 eprint(
3161 (
3162 "Current build '{0}' is not newer than latest published '{1}'. "
3163 + "Skipping publishing of binaries."
3164 ).format(current_build_number, latest_build_number)
3165 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003166 break
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01003167
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003168 try:
Rupert Shuttleworth7f3a91e2020-08-22 07:31:47 -04003169 try_publish_binaries(bazel_hashes, bazel_nojdk_hashes, current_build_number, latest_generation)
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003170 except BinaryUploadRaceException:
3171 # Retry.
3172 continue
3173
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003174 eprint(
3175 "Successfully updated '{0}' to binaries from build {1}.".format(
mai93f04f9482020-10-20 17:22:30 +02003176 bazelci_latest_build_metadata_url(), current_build_number
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003177 )
3178 )
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003179 break
3180 else:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003181 raise BuildkiteException("Could not publish binaries, ran out of attempts.")
3182
Philipp Wollermann3c8b8512019-07-16 15:28:03 +02003183
Philipp Wollermann639c0452019-01-03 11:23:54 +01003184# This is so that multiline python strings are represented as YAML
3185# block strings.
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01003186def str_presenter(dumper, data):
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003187 if len(data.splitlines()) > 1: # check for multiline string
3188 return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
3189 return dumper.represent_scalar("tag:yaml.org,2002:str", data)
3190
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01003191
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003192def main(argv=None):
3193 if argv is None:
Yun Peng20d45602018-10-18 13:27:05 +02003194 argv = sys.argv[1:]
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003195
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01003196 yaml.add_representer(str, str_presenter)
3197
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02003198 parser = argparse.ArgumentParser(description="Bazel Continuous Integration Script")
Florian Weikert944209b2019-05-10 12:41:48 +02003199 parser.add_argument("--script", type=str)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01003200
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003201 subparsers = parser.add_subparsers(dest="subparsers_name")
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003202
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01003203 bazel_publish_binaries_pipeline = subparsers.add_parser("bazel_publish_binaries_pipeline")
3204 bazel_publish_binaries_pipeline.add_argument("--file_config", type=str)
Jakob Buchgraber08e8e402018-03-20 19:22:07 +01003205 bazel_publish_binaries_pipeline.add_argument("--http_config", type=str)
3206 bazel_publish_binaries_pipeline.add_argument("--git_repository", type=str)
3207
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01003208 bazel_downstream_pipeline = subparsers.add_parser("bazel_downstream_pipeline")
3209 bazel_downstream_pipeline.add_argument("--file_config", type=str)
3210 bazel_downstream_pipeline.add_argument("--http_config", type=str)
3211 bazel_downstream_pipeline.add_argument("--git_repository", type=str)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003212 bazel_downstream_pipeline.add_argument(
3213 "--test_incompatible_flags", type=bool, nargs="?", const=True
3214 )
3215 bazel_downstream_pipeline.add_argument(
3216 "--test_disabled_projects", type=bool, nargs="?", const=True
3217 )
Florian Weikert60661912019-12-18 15:17:10 +01003218 bazel_downstream_pipeline.add_argument("--notify", type=bool, nargs="?", const=True)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01003219
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003220 project_pipeline = subparsers.add_parser("project_pipeline")
3221 project_pipeline.add_argument("--project_name", type=str)
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01003222 project_pipeline.add_argument("--file_config", type=str)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003223 project_pipeline.add_argument("--http_config", type=str)
3224 project_pipeline.add_argument("--git_repository", type=str)
Jakob Buchgraber66ba4fe2018-06-22 15:04:14 +02003225 project_pipeline.add_argument("--monitor_flaky_tests", type=bool, nargs="?", const=True)
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02003226 project_pipeline.add_argument("--use_but", type=bool, nargs="?", const=True)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003227 project_pipeline.add_argument("--incompatible_flag", type=str, action="append")
Florian Weikert60661912019-12-18 15:17:10 +01003228 project_pipeline.add_argument("--notify", type=bool, nargs="?", const=True)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01003229
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003230 runner = subparsers.add_parser("runner")
Florian Weikert843d7a02019-02-03 17:24:50 +01003231 runner.add_argument("--task", action="store", type=str, default="")
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01003232 runner.add_argument("--file_config", type=str)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003233 runner.add_argument("--http_config", type=str)
3234 runner.add_argument("--git_repository", type=str)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003235 runner.add_argument(
3236 "--git_commit", type=str, help="Reset the git repository to this commit after cloning it"
3237 )
3238 runner.add_argument(
3239 "--git_repo_location",
3240 type=str,
3241 help="Use an existing repository instead of cloning from github",
3242 )
3243 runner.add_argument(
Dan Halperinefda1192019-01-16 00:34:09 -08003244 "--use_bazel_at_commit", type=str, help="Use Bazel binary built at a specific commit"
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003245 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003246 runner.add_argument("--use_but", type=bool, nargs="?", const=True)
3247 runner.add_argument("--save_but", type=bool, nargs="?", const=True)
Yun Peng4d1d6542019-01-17 18:30:33 +01003248 runner.add_argument("--needs_clean", type=bool, nargs="?", const=True)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01003249 runner.add_argument("--build_only", type=bool, nargs="?", const=True)
3250 runner.add_argument("--test_only", type=bool, nargs="?", const=True)
Jakob Buchgraber66ba4fe2018-06-22 15:04:14 +02003251 runner.add_argument("--monitor_flaky_tests", type=bool, nargs="?", const=True)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003252 runner.add_argument("--incompatible_flag", type=str, action="append")
Jakob Buchgraberc340f582018-06-22 13:48:33 +02003253
Philipp Wollermannce986af2019-07-18 14:46:05 +02003254 subparsers.add_parser("publish_binaries")
3255 subparsers.add_parser("try_update_last_green_commit")
3256 subparsers.add_parser("try_update_last_green_downstream_commit")
Yun Peng358cd882018-11-29 10:25:18 +01003257
Yun Peng20d45602018-10-18 13:27:05 +02003258 args = parser.parse_args(argv)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01003259
Florian Weikert944209b2019-05-10 12:41:48 +02003260 if args.script:
3261 global SCRIPT_URL
3262 SCRIPT_URL = args.script
3263
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003264 try:
Jakob Buchgraber08e8e402018-03-20 19:22:07 +01003265 if args.subparsers_name == "bazel_publish_binaries_pipeline":
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01003266 configs = fetch_configs(args.http_config, args.file_config)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003267 print_bazel_publish_binaries_pipeline(
Florian Weikert843d7a02019-02-03 17:24:50 +01003268 task_configs=configs.get("tasks", None),
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003269 http_config=args.http_config,
3270 file_config=args.file_config,
3271 )
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01003272 elif args.subparsers_name == "bazel_downstream_pipeline":
3273 configs = fetch_configs(args.http_config, args.file_config)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003274 print_bazel_downstream_pipeline(
Florian Weikert843d7a02019-02-03 17:24:50 +01003275 task_configs=configs.get("tasks", None),
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003276 http_config=args.http_config,
3277 file_config=args.file_config,
3278 test_incompatible_flags=args.test_incompatible_flags,
3279 test_disabled_projects=args.test_disabled_projects,
Florian Weikert60661912019-12-18 15:17:10 +01003280 notify=args.notify,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003281 )
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003282 elif args.subparsers_name == "project_pipeline":
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01003283 configs = fetch_configs(args.http_config, args.file_config)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003284 print_project_pipeline(
Florian Weikertf20ae6f2019-01-16 14:32:09 +01003285 configs=configs,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003286 project_name=args.project_name,
3287 http_config=args.http_config,
3288 file_config=args.file_config,
3289 git_repository=args.git_repository,
3290 monitor_flaky_tests=args.monitor_flaky_tests,
3291 use_but=args.use_but,
3292 incompatible_flags=args.incompatible_flag,
Florian Weikert60661912019-12-18 15:17:10 +01003293 notify=args.notify,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003294 )
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003295 elif args.subparsers_name == "runner":
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01003296 configs = fetch_configs(args.http_config, args.file_config)
Florian Weikert843d7a02019-02-03 17:24:50 +01003297 tasks = configs.get("tasks", {})
3298 task_config = tasks.get(args.task)
3299 if not task_config:
3300 raise BuildkiteException(
3301 "No such task '{}' in configuration. Available: {}".format(
3302 args.task, ", ".join(tasks)
3303 )
3304 )
3305
3306 platform = get_platform_for_task(args.task, task_config)
3307
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003308 execute_commands(
Florian Weikertc8642af2019-02-03 23:58:51 +01003309 task_config=task_config,
Florian Weikert843d7a02019-02-03 17:24:50 +01003310 platform=platform,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003311 git_repository=args.git_repository,
3312 git_commit=args.git_commit,
3313 git_repo_location=args.git_repo_location,
3314 use_bazel_at_commit=args.use_bazel_at_commit,
3315 use_but=args.use_but,
3316 save_but=args.save_but,
Yun Peng4d1d6542019-01-17 18:30:33 +01003317 needs_clean=args.needs_clean,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003318 build_only=args.build_only,
3319 test_only=args.test_only,
3320 monitor_flaky_tests=args.monitor_flaky_tests,
3321 incompatible_flags=args.incompatible_flag,
Florian Weikertc8642af2019-02-03 23:58:51 +01003322 bazel_version=task_config.get("bazel") or configs.get("bazel"),
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01003323 )
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003324 elif args.subparsers_name == "publish_binaries":
3325 publish_binaries()
Yun Peng358cd882018-11-29 10:25:18 +01003326 elif args.subparsers_name == "try_update_last_green_commit":
Florian Weikert35906542019-04-01 11:53:53 +02003327 # Update the last green commit of a project pipeline
Yun Peng358cd882018-11-29 10:25:18 +01003328 try_update_last_green_commit()
Florian Weikert35906542019-04-01 11:53:53 +02003329 elif args.subparsers_name == "try_update_last_green_downstream_commit":
3330 # Update the last green commit of the downstream pipeline
3331 try_update_last_green_downstream_commit()
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003332 else:
3333 parser.print_help()
3334 return 2
3335 except BuildkiteException as e:
3336 eprint(str(e))
3337 return 1
3338 return 0
3339
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01003340
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01003341if __name__ == "__main__":
3342 sys.exit(main())