blob: 6fe41ff9b0c32c40f6de25511d1d4c18a61ec892 [file] [log] [blame]
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01001#!/usr/bin/env python3
2#
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01003# Copyright 2018 The Bazel Authors. All rights reserved.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010017import argparse
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +010018import base64
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010019import codecs
Jakob Buchgraber12807052018-02-25 17:04:56 +010020import datetime
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +010021import hashlib
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010022import json
Jakob Buchgraber6db0f262018-02-17 15:45:54 +010023import multiprocessing
Philipp Wollermann0a04cf32018-02-21 17:07:22 +010024import os
Philipp Wollermanndcaddd92018-02-21 14:13:43 +010025import os.path
Jakob Buchgraber257693b2018-02-20 00:03:56 +010026import random
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010027import re
Philipp Wollermanndcaddd92018-02-21 14:13:43 +010028from shutil import copyfile
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010029import shutil
Philipp Wollermanndcaddd92018-02-21 14:13:43 +010030import stat
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010031import subprocess
32import sys
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010033import tempfile
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +020034import threading
Philipp Wollermanne1318eb2018-08-13 15:08:01 +020035import time
Philipp Wollermannce986af2019-07-18 14:46:05 +020036import urllib.error
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010037import urllib.request
Philipp Wollermanne1318eb2018-08-13 15:08:01 +020038import uuid
Jakob Buchgraber25bb50f2018-02-22 18:06:21 +010039import yaml
Philipp Wollermannc030f2e2018-02-21 17:02:19 +010040from urllib.request import url2pathname
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +010041from urllib.parse import urlparse
Philipp Wollermanndcaddd92018-02-21 14:13:43 +010042
43# Initialize the random number generator.
44random.seed()
45
Philipp Wollermanne67eec42019-05-24 15:18:20 +020046BUILDKITE_ORG = os.environ["BUILDKITE_ORGANIZATION_SLUG"]
47THIS_IS_PRODUCTION = BUILDKITE_ORG == "bazel-untrusted"
48THIS_IS_TESTING = BUILDKITE_ORG == "bazel-testing"
49THIS_IS_TRUSTED = BUILDKITE_ORG == "bazel-trusted"
50THIS_IS_SPARTA = True
51
52CLOUD_PROJECT = "bazel-public" if THIS_IS_TRUSTED else "bazel-untrusted"
53
54GITHUB_BRANCH = {"bazel": "master", "bazel-trusted": "master", "bazel-testing": "testing"}[
55 BUILDKITE_ORG
56]
57
58SCRIPT_URL = "https://raw.githubusercontent.com/bazelbuild/continuous-integration/{}/buildkite/bazelci.py?{}".format(
59 GITHUB_BRANCH, int(time.time())
Philipp Wollermannc05ac682019-01-19 12:37:28 +010060)
Jakob Buchgraber95e3d572018-02-21 18:48:49 +010061
Philipp Wollermanne67eec42019-05-24 15:18:20 +020062INCOMPATIBLE_FLAG_VERBOSE_FAILURES_URL = "https://raw.githubusercontent.com/bazelbuild/continuous-integration/{}/buildkite/incompatible_flag_verbose_failures.py?{}".format(
63 GITHUB_BRANCH, int(time.time())
64)
65
66AGGREGATE_INCOMPATIBLE_TEST_RESULT_URL = "https://raw.githubusercontent.com/bazelbuild/continuous-integration/{}/buildkite/aggregate_incompatible_flags_test_result.py?{}".format(
67 GITHUB_BRANCH, int(time.time())
68)
69
70EMERGENCY_FILE_URL = "https://raw.githubusercontent.com/bazelbuild/continuous-integration/{}/buildkite/emergency.yml?{}".format(
71 GITHUB_BRANCH, int(time.time())
72)
73
74FLAKY_TESTS_BUCKET = {
75 "bazel-testing": "gs://bazel-testing-buildkite-stats/flaky-tests-bep/",
76 "bazel-trusted": "gs://bazel-buildkite-stats/flaky-tests-bep/",
77 "bazel": "gs://bazel-buildkite-stats/flaky-tests-bep/",
78}[BUILDKITE_ORG]
79
80DOWNSTREAM_PROJECTS_PRODUCTION = {
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +010081 "Android Studio Plugin": {
82 "git_repository": "https://github.com/bazelbuild/intellij.git",
83 "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/master/.bazelci/android-studio.yml",
84 "pipeline_slug": "android-studio-plugin",
85 },
Yun Peng996efad2018-11-27 17:19:44 +010086 "Android Testing": {
87 "git_repository": "https://github.com/googlesamples/android-testing.git",
Jingwenbde72602018-12-13 10:57:43 -050088 "http_config": "https://raw.githubusercontent.com/googlesamples/android-testing/master/bazelci/buildkite-pipeline.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +010089 "pipeline_slug": "android-testing",
Yun Peng996efad2018-11-27 17:19:44 +010090 },
Yun Peng8910fa32019-01-03 08:58:16 +010091 "Bazel": {
92 "git_repository": "https://github.com/bazelbuild/bazel.git",
93 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel/master/.bazelci/postsubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +010094 "pipeline_slug": "bazel-bazel",
Yun Peng8910fa32019-01-03 08:58:16 +010095 },
Tobias Werthd848eca2019-05-14 15:08:35 +020096 "Bazel Bench": {
97 "git_repository": "https://github.com/bazelbuild/bazel-bench.git",
joeleba92ffec82019-05-22 14:50:15 +020098 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-bench/master/.bazelci/postsubmit.yml",
Tobias Werthd848eca2019-05-14 15:08:35 +020099 "pipeline_slug": "bazel-bench",
100 },
Philipp Wollermannfefcbf42019-05-28 14:28:40 +0200101 "Bazel Codelabs": {
102 "git_repository": "https://github.com/bazelbuild/codelabs.git",
103 "http_config": "https://raw.githubusercontent.com/bazelbuild/codelabs/master/.bazelci/presubmit.yml",
104 "pipeline_slug": "bazel-codelabs",
Philipp Wollermannfefcbf42019-05-28 14:28:40 +0200105 },
Jinfce9b302019-08-08 15:18:26 -0400106 "Bazel Examples": {
107 "git_repository": "https://github.com/bazelbuild/examples.git",
108 "http_config": "https://raw.githubusercontent.com/bazelbuild/examples/master/.bazelci/presubmit.yml",
109 "pipeline_slug": "bazel-bazel-examples",
110 },
Florian Weikert4b3ec672019-08-14 19:05:12 +0200111 "Bazel Federation": {
112 "git_repository": "https://github.com/bazelbuild/bazel-federation.git",
113 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-federation/master/.bazelci/presubmit.yml",
114 "pipeline_slug": "bazel-federation",
115 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100116 "Bazel Remote Cache": {
117 "git_repository": "https://github.com/buchgr/bazel-remote.git",
118 "http_config": "https://raw.githubusercontent.com/buchgr/bazel-remote/master/.bazelci/presubmit.yml",
119 "pipeline_slug": "bazel-remote-cache",
Yun Peng996efad2018-11-27 17:19:44 +0100120 },
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200121 "Bazel integration testing": {
122 "git_repository": "https://github.com/bazelbuild/bazel-integration-testing.git",
123 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-integration-testing/master/.bazelci/presubmit.yml",
124 "pipeline_slug": "bazel-integration-testing",
125 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100126 "Bazel skylib": {
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200127 "git_repository": "https://github.com/bazelbuild/bazel-skylib.git",
Yun Peng996efad2018-11-27 17:19:44 +0100128 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-skylib/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100129 "pipeline_slug": "bazel-skylib",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200130 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100131 "Bazel toolchains": {
132 "git_repository": "https://github.com/bazelbuild/bazel-toolchains.git",
133 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-toolchains/master/.bazelci/presubmit.yml",
134 "pipeline_slug": "bazel-toolchains",
135 },
136 "Bazel watcher": {
137 "git_repository": "https://github.com/bazelbuild/bazel-watcher.git",
138 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-watcher/master/.bazelci/presubmit.yml",
139 "pipeline_slug": "bazel-watcher",
140 },
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200141 "Bazelisk": {
142 "git_repository": "https://github.com/bazelbuild/bazelisk.git",
143 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazelisk/master/.bazelci/config.yml",
144 "pipeline_slug": "bazelisk",
145 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100146 "Buildfarm": {
147 "git_repository": "https://github.com/bazelbuild/bazel-buildfarm.git",
148 "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-buildfarm/master/.bazelci/presubmit.yml",
149 "pipeline_slug": "buildfarm-male-farmer",
150 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100151 "Buildtools": {
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200152 "git_repository": "https://github.com/bazelbuild/buildtools.git",
Yun Peng996efad2018-11-27 17:19:44 +0100153 "http_config": "https://raw.githubusercontent.com/bazelbuild/buildtools/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100154 "pipeline_slug": "buildtools",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200155 },
Yun Peng39a42582018-11-09 10:59:47 +0100156 "CLion Plugin": {
157 "git_repository": "https://github.com/bazelbuild/intellij.git",
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100158 "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/master/.bazelci/clion.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100159 "pipeline_slug": "clion-plugin",
Yun Peng39a42582018-11-09 10:59:47 +0100160 },
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200161 "Cartographer": {
162 "git_repository": "https://github.com/googlecartographer/cartographer.git",
163 "http_config": "https://raw.githubusercontent.com/googlecartographer/cartographer/master/.bazelci/presubmit.yml",
164 "pipeline_slug": "cartographer",
165 },
Philipp Wollermannee850782019-02-05 22:56:04 +0100166 "Cloud Robotics Core": {
Stefan Sauerb4dd3f92019-02-05 22:44:28 +0100167 "git_repository": "https://github.com/googlecloudrobotics/core.git",
168 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/cloud-robotics-postsubmit.yml",
169 "pipeline_slug": "cloud-robotics-core",
170 },
Keith Smiley3b0ba602019-05-15 04:42:19 -0700171 "Envoy": {
172 "git_repository": "https://github.com/envoyproxy/envoy.git",
173 "http_config": "https://raw.githubusercontent.com/envoyproxy/envoy/master/.bazelci/presubmit.yml",
174 "pipeline_slug": "envoy",
175 },
Florian Weikert1fe28b72019-07-02 12:47:55 +0200176 "FlatBuffers": {
177 "git_repository": "https://github.com/google/flatbuffers.git",
178 "http_config": "https://raw.githubusercontent.com/google/flatbuffers/master/.bazelci/presubmit.yml",
179 "pipeline_slug": "flatbuffers",
180 },
Philipp Wollermannf3750fa2019-05-21 17:11:59 +0200181 "Flogger": {
182 "git_repository": "https://github.com/google/flogger.git",
183 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/flogger.yml",
184 "pipeline_slug": "flogger",
185 },
Marcel Hlopkoc8840772018-10-23 12:51:46 +0200186 "Gerrit": {
187 "git_repository": "https://gerrit.googlesource.com/gerrit.git",
Yun Peng996efad2018-11-27 17:19:44 +0100188 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/gerrit-postsubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100189 "pipeline_slug": "gerrit",
Marcel Hlopkoc8840772018-10-23 12:51:46 +0200190 },
Yun Pengd6622022018-11-05 13:10:26 +0100191 "Google Logging": {
192 "git_repository": "https://github.com/google/glog.git",
Yun Peng996efad2018-11-27 17:19:44 +0100193 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/glog-postsubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100194 "pipeline_slug": "google-logging",
Yun Pengd6622022018-11-05 13:10:26 +0100195 },
Yun Peng9586db52018-11-02 10:48:40 +0100196 "IntelliJ Plugin": {
197 "git_repository": "https://github.com/bazelbuild/intellij.git",
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100198 "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/master/.bazelci/intellij.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100199 "pipeline_slug": "intellij-plugin",
Yun Peng9586db52018-11-02 10:48:40 +0100200 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100201 "IntelliJ Plugin Aspect": {
202 "git_repository": "https://github.com/bazelbuild/intellij.git",
203 "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/master/.bazelci/aspect.yml",
204 "pipeline_slug": "intellij-plugin-aspect",
205 },
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200206 "Kythe": {
207 "git_repository": "https://github.com/kythe/kythe.git",
208 "http_config": "https://raw.githubusercontent.com/kythe/kythe/master/.bazelci/presubmit.yml",
209 "pipeline_slug": "kythe",
210 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100211 "Protobuf": {
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200212 "git_repository": "https://github.com/google/protobuf.git",
Yun Peng996efad2018-11-27 17:19:44 +0100213 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/protobuf-postsubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100214 "pipeline_slug": "protobuf",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200215 },
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200216 "Skydoc": {
217 "git_repository": "https://github.com/bazelbuild/skydoc.git",
218 "http_config": "https://raw.githubusercontent.com/bazelbuild/skydoc/master/.bazelci/presubmit.yml",
219 "pipeline_slug": "skydoc",
220 },
221 "Subpar": {
222 "git_repository": "https://github.com/google/subpar.git",
223 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/subpar-postsubmit.yml",
224 "pipeline_slug": "subpar",
225 },
226 "TensorFlow": {
227 "git_repository": "https://github.com/tensorflow/tensorflow.git",
228 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/tensorflow-postsubmit.yml",
229 "pipeline_slug": "tensorflow",
230 },
231 "Tulsi": {
232 "git_repository": "https://github.com/bazelbuild/tulsi.git",
233 "http_config": "https://raw.githubusercontent.com/bazelbuild/tulsi/master/.bazelci/presubmit.yml",
234 "pipeline_slug": "tulsi-bazel-darwin",
235 },
236 "re2": {
237 "git_repository": "https://github.com/google/re2.git",
238 "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/pipelines/re2-postsubmit.yml",
239 "pipeline_slug": "re2",
240 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100241 "rules_android": {
242 "git_repository": "https://github.com/bazelbuild/rules_android.git",
243 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_android/master/.bazelci/postsubmit.yml",
244 "pipeline_slug": "rules-android",
245 },
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200246 "rules_appengine": {
247 "git_repository": "https://github.com/bazelbuild/rules_appengine.git",
Yun Peng996efad2018-11-27 17:19:44 +0100248 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_appengine/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100249 "pipeline_slug": "rules-appengine-appengine",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200250 },
Yun Peng809f27b2018-11-13 10:15:39 +0100251 "rules_apple": {
252 "git_repository": "https://github.com/bazelbuild/rules_apple.git",
Yun Peng996efad2018-11-27 17:19:44 +0100253 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_apple/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100254 "pipeline_slug": "rules-apple-darwin",
Yun Peng809f27b2018-11-13 10:15:39 +0100255 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100256 "rules_cc": {
257 "git_repository": "https://github.com/bazelbuild/rules_cc.git",
258 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_cc/master/.bazelci/presubmit.yml",
259 "pipeline_slug": "rules-cc",
260 },
Marcel Hlopko340dfd22018-10-19 11:33:01 +0200261 "rules_closure": {
262 "git_repository": "https://github.com/bazelbuild/rules_closure.git",
Yun Peng996efad2018-11-27 17:19:44 +0100263 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_closure/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100264 "pipeline_slug": "rules-closure-closure-compiler",
Marcel Hlopko340dfd22018-10-19 11:33:01 +0200265 },
Yun Peng51ce6692019-01-09 14:31:46 +0100266 "rules_d": {
267 "git_repository": "https://github.com/bazelbuild/rules_d.git",
268 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_d/master/.bazelci/presubmit.yml",
269 "pipeline_slug": "rules-d",
270 },
Yun Peng996efad2018-11-27 17:19:44 +0100271 "rules_docker": {
272 "git_repository": "https://github.com/bazelbuild/rules_docker.git",
273 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_docker/master/.bazelci/presubmit.yml",
Jakob Buchgrabera6a8ea82018-12-07 13:51:02 +0100274 "pipeline_slug": "rules-docker-docker",
Yun Peng996efad2018-11-27 17:19:44 +0100275 },
276 "rules_foreign_cc": {
277 "git_repository": "https://github.com/bazelbuild/rules_foreign_cc.git",
278 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_foreign_cc/master/.bazelci/config.yaml",
279 "pipeline_slug": "rules-foreign-cc",
Yun Peng996efad2018-11-27 17:19:44 +0100280 },
Xindb02c012018-11-07 14:10:54 -0500281 "rules_go": {
282 "git_repository": "https://github.com/bazelbuild/rules_go.git",
Yun Peng996efad2018-11-27 17:19:44 +0100283 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_go/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100284 "pipeline_slug": "rules-go-golang",
Yun Pengb7247ff2018-11-15 13:52:39 +0100285 },
Yun Peng7deea572018-11-05 10:47:45 +0100286 "rules_groovy": {
Yun Peng996efad2018-11-27 17:19:44 +0100287 "git_repository": "https://github.com/bazelbuild/rules_groovy.git",
288 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_groovy/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100289 "pipeline_slug": "rules-groovy",
Yun Peng996efad2018-11-27 17:19:44 +0100290 },
291 "rules_gwt": {
292 "git_repository": "https://github.com/bazelbuild/rules_gwt.git",
293 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_gwt/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100294 "pipeline_slug": "rules-gwt",
Philipp Wollermann2a160432019-09-19 15:57:28 +0200295 },
Florian Weikertff6444e2019-09-16 16:08:57 +0200296 "rules_haskell": {
297 "git_repository": "https://github.com/tweag/rules_haskell.git",
298 "http_config": "https://raw.githubusercontent.com/tweag/rules_haskell/master/.bazelci/presubmit.yml",
299 "pipeline_slug": "rules-haskell-haskell",
Philipp Wollermann2a160432019-09-19 15:57:28 +0200300 },
Yun Peng996efad2018-11-27 17:19:44 +0100301 "rules_jsonnet": {
302 "git_repository": "https://github.com/bazelbuild/rules_jsonnet.git",
303 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_jsonnet/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100304 "pipeline_slug": "rules-jsonnet",
Yun Peng996efad2018-11-27 17:19:44 +0100305 },
Philipp Wollermann1dc76992019-05-28 16:42:51 +0200306 "rules_jvm_external": {
307 "git_repository": "https://github.com/bazelbuild/rules_jvm_external.git",
308 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_jvm_external/master/.bazelci/presubmit.yml",
309 "pipeline_slug": "rules-jvm-external",
310 },
311 "rules_jvm_external - examples": {
312 "git_repository": "https://github.com/bazelbuild/rules_jvm_external.git",
313 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_jvm_external/master/.bazelci/examples.yml",
314 "pipeline_slug": "rules-jvm-external-examples",
315 },
Yun Peng996efad2018-11-27 17:19:44 +0100316 "rules_k8s": {
317 "git_repository": "https://github.com/bazelbuild/rules_k8s.git",
318 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_k8s/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100319 "pipeline_slug": "rules-k8s-k8s",
Yun Peng996efad2018-11-27 17:19:44 +0100320 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100321 "rules_kotlin": {
322 "git_repository": "https://github.com/bazelbuild/rules_kotlin.git",
323 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_kotlin/master/.bazelci/presubmit.yml",
324 "pipeline_slug": "rules-kotlin-kotlin",
325 },
Yun Penga5650e12018-11-14 10:16:06 +0100326 "rules_nodejs": {
327 "git_repository": "https://github.com/bazelbuild/rules_nodejs.git",
Yun Peng996efad2018-11-27 17:19:44 +0100328 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_nodejs/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100329 "pipeline_slug": "rules-nodejs-nodejs",
Yun Penga5650e12018-11-14 10:16:06 +0100330 },
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200331 "rules_perl": {
332 "git_repository": "https://github.com/bazelbuild/rules_perl.git",
Yun Peng996efad2018-11-27 17:19:44 +0100333 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_perl/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100334 "pipeline_slug": "rules-perl",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200335 },
Yannic6110b3c2019-08-12 15:09:37 +0000336 "rules_proto": {
337 "git_repository": "https://github.com/bazelbuild/rules_proto.git",
338 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_proto/master/.bazelci/presubmit.yml",
339 "pipeline_slug": "rules-proto",
340 },
Yun Peng3d5a8a62018-11-19 11:42:01 +0100341 "rules_python": {
342 "git_repository": "https://github.com/bazelbuild/rules_python.git",
Yun Peng996efad2018-11-27 17:19:44 +0100343 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_python/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100344 "pipeline_slug": "rules-python-python",
Yun Peng3d5a8a62018-11-19 11:42:01 +0100345 },
Xindb02c012018-11-07 14:10:54 -0500346 "rules_rust": {
347 "git_repository": "https://github.com/bazelbuild/rules_rust.git",
Yun Peng996efad2018-11-27 17:19:44 +0100348 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_rust/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100349 "pipeline_slug": "rules-rust-rustlang",
Xindb02c012018-11-07 14:10:54 -0500350 },
Yun Pengca62fff2018-10-31 11:22:03 +0100351 "rules_sass": {
352 "git_repository": "https://github.com/bazelbuild/rules_sass.git",
Yun Peng996efad2018-11-27 17:19:44 +0100353 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_sass/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100354 "pipeline_slug": "rules-sass",
Yun Pengca62fff2018-10-31 11:22:03 +0100355 },
Xindb02c012018-11-07 14:10:54 -0500356 "rules_scala": {
357 "git_repository": "https://github.com/bazelbuild/rules_scala.git",
Yun Peng996efad2018-11-27 17:19:44 +0100358 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_scala/master/.bazelci/presubmit.yml",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100359 "pipeline_slug": "rules-scala-scala",
Xindb02c012018-11-07 14:10:54 -0500360 },
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100361 "rules_swift": {
362 "git_repository": "https://github.com/bazelbuild/rules_swift.git",
363 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_swift/master/.bazelci/presubmit.yml",
364 "pipeline_slug": "rules-swift-swift",
365 },
Yun Peng996efad2018-11-27 17:19:44 +0100366 "rules_typescript": {
367 "git_repository": "https://github.com/bazelbuild/rules_typescript.git",
368 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_typescript/master/.bazelci/presubmit.yml",
Jakob Buchgrabera6a8ea82018-12-07 13:51:02 +0100369 "pipeline_slug": "rules-typescript-typescript",
Yun Peng996efad2018-11-27 17:19:44 +0100370 },
371 "rules_webtesting": {
372 "git_repository": "https://github.com/bazelbuild/rules_webtesting.git",
Yun Pengc2fab332019-01-04 10:53:49 +0100373 "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_webtesting/master/.bazelci/presubmit.yml",
Yun Peng996efad2018-11-27 17:19:44 +0100374 "pipeline_slug": "rules-webtesting-saucelabs",
Yun Peng996efad2018-11-27 17:19:44 +0100375 },
Philipp Wollermann389acd82019-05-21 17:41:48 +0200376 "upb": {
377 "git_repository": "https://github.com/protocolbuffers/upb.git",
378 "http_config": "https://raw.githubusercontent.com/protocolbuffers/upb/master/.bazelci/presubmit.yml",
379 "pipeline_slug": "upb",
Marcel Hlopko0bc60892019-05-24 14:37:29 +0200380 "disabled_reason": "https://github.com/protocolbuffers/upb/issues/172",
Philipp Wollermann389acd82019-05-21 17:41:48 +0200381 },
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200382}
383
Philipp Wollermanne67eec42019-05-24 15:18:20 +0200384DOWNSTREAM_PROJECTS_TESTING = {
Philipp Wollermannbed211d2019-06-07 11:38:59 +0200385 "Bazel": DOWNSTREAM_PROJECTS_PRODUCTION["Bazel"],
386 "Bazelisk": DOWNSTREAM_PROJECTS_PRODUCTION["Bazelisk"],
Florian Weikert8fda2a72019-05-31 15:21:54 +0200387 "Federation": {
388 "git_repository": "https://github.com/fweikert/bazel-federation.git",
389 "http_config": "https://raw.githubusercontent.com/fweikert/bazel-federation/master/.bazelci/presubmit.yml",
390 "pipeline_slug": "bazel-federation",
391 },
Philipp Wollermannbed211d2019-06-07 11:38:59 +0200392 "rules_docker": DOWNSTREAM_PROJECTS_PRODUCTION["rules_docker"],
393 "rules_go": DOWNSTREAM_PROJECTS_PRODUCTION["rules_go"],
394 "rules_groovy": DOWNSTREAM_PROJECTS_PRODUCTION["rules_groovy"],
395 "rules_kotlin": DOWNSTREAM_PROJECTS_PRODUCTION["rules_kotlin"],
396 "rules_nodejs": DOWNSTREAM_PROJECTS_PRODUCTION["rules_nodejs"],
397 "rules_rust": DOWNSTREAM_PROJECTS_PRODUCTION["rules_rust"],
398 "rules_scala": DOWNSTREAM_PROJECTS_PRODUCTION["rules_scala"],
Philipp Wollermanne67eec42019-05-24 15:18:20 +0200399}
400
401DOWNSTREAM_PROJECTS = {
402 "bazel-testing": DOWNSTREAM_PROJECTS_TESTING,
403 "bazel-trusted": {},
404 "bazel": DOWNSTREAM_PROJECTS_PRODUCTION,
405}[BUILDKITE_ORG]
Philipp Wollermann6dd7aa32019-02-05 22:42:15 +0100406
Philipp Wollermann81a88412019-07-12 10:34:33 +0200407DOCKER_REGISTRY_PREFIX = {
408 "bazel-testing": "bazel-public/testing",
409 "bazel-trusted": "bazel-public",
410 "bazel": "bazel-public",
411}[BUILDKITE_ORG]
412
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200413# A map containing all supported platform names as keys, with the values being
414# the platform name in a human readable format, and a the buildkite-agent's
415# working directory.
416PLATFORMS = {
Philipp Wollermanneffcd6e2019-06-21 18:30:34 +0200417 "centos7": {
418 "name": "CentOS 7, Java 8",
419 "emoji-name": ":centos: 7 (Java 8)",
420 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermannf4aabb72019-06-25 15:59:00 +0200421 "publish_binary": ["ubuntu1404", "centos7", "linux"],
Philipp Wollermann81a88412019-07-12 10:34:33 +0200422 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/centos7:java8",
Philipp Wollermanneffcd6e2019-06-21 18:30:34 +0200423 "python": "python3.6",
424 },
Philipp Wollermann67fc3712019-06-12 15:39:21 +0200425 "debian10": {
426 "name": "Debian Buster, OpenJDK 11",
427 "emoji-name": ":debian: Buster (OpenJDK 11)",
428 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
429 "publish_binary": [],
430 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/debian10:java11",
431 "python": "python3.7",
432 },
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200433 "ubuntu1604": {
Philipp Wollermanndb877332019-04-23 17:58:01 +0200434 "name": "Ubuntu 16.04, OpenJDK 8",
435 "emoji-name": ":ubuntu: 16.04 (OpenJDK 8)",
Philipp Wollermannd551bf62019-05-18 22:04:35 +0200436 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermannf4aabb72019-06-25 15:59:00 +0200437 "publish_binary": ["ubuntu1604"],
Philipp Wollermann81a88412019-07-12 10:34:33 +0200438 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu1604:java8",
Philipp Wollermann57b32682019-05-18 22:09:27 +0200439 "python": "python3.6",
Philipp Wollermann438ec242018-09-05 14:39:24 +0200440 },
441 "ubuntu1804": {
Philipp Wollermannf5a2feb2019-04-25 11:13:46 +0200442 "name": "Ubuntu 18.04, OpenJDK 11",
443 "emoji-name": ":ubuntu: 18.04 (OpenJDK 11)",
Philipp Wollermannd551bf62019-05-18 22:04:35 +0200444 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermann783d1672019-06-06 13:35:30 +0200445 "publish_binary": ["ubuntu1804"],
Philipp Wollermann81a88412019-07-12 10:34:33 +0200446 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu1804:java11",
Philipp Wollermann57b32682019-05-18 22:09:27 +0200447 "python": "python3.6",
Philipp Wollermann438ec242018-09-05 14:39:24 +0200448 },
449 "ubuntu1804_nojava": {
450 "name": "Ubuntu 18.04, no JDK",
451 "emoji-name": ":ubuntu: 18.04 (no JDK)",
Philipp Wollermannd551bf62019-05-18 22:04:35 +0200452 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermann783d1672019-06-06 13:35:30 +0200453 "publish_binary": [],
Philipp Wollermann81a88412019-07-12 10:34:33 +0200454 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu1804:nojava",
Philipp Wollermann57b32682019-05-18 22:09:27 +0200455 "python": "python3.6",
Philipp Wollermann438ec242018-09-05 14:39:24 +0200456 },
Philipp Wollermann438ec242018-09-05 14:39:24 +0200457 "macos": {
Philipp Wollermanndb877332019-04-23 17:58:01 +0200458 "name": "macOS, OpenJDK 8",
459 "emoji-name": ":darwin: (OpenJDK 8)",
Philipp Wollermann51147bf2019-05-08 15:50:10 +0200460 "downstream-root": "/Users/buildkite/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermann783d1672019-06-06 13:35:30 +0200461 "publish_binary": ["macos"],
Philipp Wollermann7a185322019-05-18 22:15:48 +0200462 "queue": "macos",
Philipp Wollermann57b32682019-05-18 22:09:27 +0200463 "python": "python3.7",
Philipp Wollermann438ec242018-09-05 14:39:24 +0200464 },
465 "windows": {
Philipp Wollermanndb877332019-04-23 17:58:01 +0200466 "name": "Windows, OpenJDK 8",
467 "emoji-name": ":windows: (OpenJDK 8)",
Philipp Wollermann51147bf2019-05-08 15:50:10 +0200468 "downstream-root": "d:/b/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermann783d1672019-06-06 13:35:30 +0200469 "publish_binary": ["windows"],
Philipp Wollermann7a185322019-05-18 22:15:48 +0200470 "queue": "windows",
Philipp Wollermann57b32682019-05-18 22:09:27 +0200471 "python": "python.exe",
Philipp Wollermann438ec242018-09-05 14:39:24 +0200472 },
473 "rbe_ubuntu1604": {
Philipp Wollermanndb877332019-04-23 17:58:01 +0200474 "name": "RBE (Ubuntu 16.04, OpenJDK 8)",
Jakob Buchgraber1f37fbd2019-07-17 17:08:28 +0200475 "emoji-name": "RBE (:ubuntu: 16.04, OpenJDK 8)",
Philipp Wollermannd551bf62019-05-18 22:04:35 +0200476 "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects",
Philipp Wollermann783d1672019-06-06 13:35:30 +0200477 "publish_binary": [],
Philipp Wollermann81a88412019-07-12 10:34:33 +0200478 "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu1604:java8",
Philipp Wollermann57b32682019-05-18 22:09:27 +0200479 "python": "python3.6",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100480 },
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200481}
482
Philipp Wollermannfce92bf2019-05-22 15:14:32 +0200483BUILDIFIER_DOCKER_IMAGE = "gcr.io/bazel-public/buildifier"
Florian Weikertf20ae6f2019-01-16 14:32:09 +0100484
Philipp Wollermann1403d2c2019-01-10 13:15:51 +0100485# The platform used for various steps (e.g. stuff that formerly ran on the "pipeline" workers).
486DEFAULT_PLATFORM = "ubuntu1804"
487
Philipp Wollermannf4aabb72019-06-25 15:59:00 +0200488# In order to test that "the one Linux binary" that we build for our official releases actually
489# works on all Linux distributions that we test on, we use the Linux binary built on our official
490# release platform for all Linux downstream tests.
491LINUX_BINARY_PLATFORM = "centos7"
492
Philipp Wollermann0ecf9922019-05-13 13:24:52 +0200493DEFAULT_XCODE_VERSION = "10.2.1"
Philipp Wollermann380f1e62019-04-12 16:45:27 +0200494XCODE_VERSION_REGEX = re.compile(r"^\d+\.\d+(\.\d+)?$")
495
Philipp Wollermann1403d2c2019-01-10 13:15:51 +0100496ENCRYPTED_SAUCELABS_TOKEN = """
Philipp Wollermanna4722b42019-01-10 16:50:13 +0100497CiQAry63sOlZtTNtuOT5DAOLkum0rGof+DOweppZY1aOWbat8zwSTQAL7Hu+rgHSOr6P4S1cu4YG
498/I1BHsWaOANqUgFt6ip9/CUGGJ1qggsPGXPrmhSbSPqNAIAkpxYzabQ3mfSIObxeBmhKg2dlILA/
499EDql
Philipp Wollermann1403d2c2019-01-10 13:15:51 +0100500""".strip()
501
Florian Weikertc8642af2019-02-03 23:58:51 +0100502BUILD_LABEL_PATTERN = re.compile(r"^Build label: (\S+)$", re.MULTILINE)
503
Florian Weikert29cb7ec2019-03-07 14:52:18 +0100504BUILDIFIER_VERSION_ENV_VAR = "BUILDIFIER_VERSION"
505
Florian Weikert85208912019-03-07 17:08:39 +0100506BUILDIFIER_WARNINGS_ENV_VAR = "BUILDIFIER_WARNINGS"
507
Florian Weikertde96a6f2019-03-07 14:57:50 +0100508BUILDIFIER_STEP_NAME = "Buildifier"
509
Florian Weikert5f5d3cb2019-04-15 15:36:27 +0200510SKIP_TASKS_ENV_VAR = "CI_SKIP_TASKS"
511
Philipp Wollermannce986af2019-07-18 14:46:05 +0200512CONFIG_FILE_EXTENSIONS = {".yml", ".yaml"}
Florian Weikert778251c2019-04-25 15:14:44 +0200513
Florian Weikert13215a82019-05-10 12:42:21 +0200514
Philipp Wollermanndcaddd92018-02-21 14:13:43 +0100515class BuildkiteException(Exception):
516 """
517 Raised whenever something goes wrong and we should exit with an error.
518 """
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100519
Philipp Wollermanndcaddd92018-02-21 14:13:43 +0100520 pass
521
522
523class BinaryUploadRaceException(Exception):
524 """
525 Raised when try_publish_binaries wasn't able to publish a set of binaries,
526 because the generation of the current file didn't match the expected value.
527 """
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100528
Philipp Wollermanndcaddd92018-02-21 14:13:43 +0100529 pass
530
531
Florian Weikerta0e74592019-03-07 11:56:12 +0100532class BuildkiteClient(object):
533
534 _ENCRYPTED_BUILDKITE_API_TOKEN = """
535CiQA4DEB9ldzC+E39KomywtqXfaQ86hhulgeDsicds2BuvbCYzsSUAAqwcvXZPh9IMWlwWh94J2F
536exosKKaWB0tSRJiPKnv2NPDfEqGul0ZwVjtWeASpugwxxKeLhFhPMcgHMPfndH6j2GEIY6nkKRbP
537uwoRMCwe
538""".strip()
539
Philipp Wollermanne67eec42019-05-24 15:18:20 +0200540 _ENCRYPTED_BUILDKITE_API_TESTING_TOKEN = """
541CiQAMTBkWjL1C+F5oon3+cC1vmum5+c1y5+96WQY44p0Lxd0PeASUQAy7iU0c6E3W5EOSFYfD5fA
542MWy/SHaMno1NQSUa4xDOl5yc2kizrtxPPVkX4x9pLNuGUY/xwAn2n1DdiUdWZNWlY1bX2C4ex65e
543P9w8kNhEbw==
544""".strip()
545
Florian Weikertde96a6f2019-03-07 14:57:50 +0100546 _BUILD_STATUS_URL_TEMPLATE = (
547 "https://api.buildkite.com/v2/organizations/{}/pipelines/{}/builds/{}"
548 )
Florian Weikerta0e74592019-03-07 11:56:12 +0100549
550 def __init__(self, org, pipeline):
551 self._org = org
552 self._pipeline = pipeline
553 self._token = self._get_buildkite_token()
554
555 def _get_buildkite_token(self):
556 return (
557 subprocess.check_output(
558 [
559 gcloud_command(),
560 "kms",
561 "decrypt",
562 "--project",
563 "bazel-untrusted",
564 "--location",
565 "global",
566 "--keyring",
567 "buildkite",
568 "--key",
Philipp Wollermanne67eec42019-05-24 15:18:20 +0200569 "buildkite-testing-api-token"
570 if THIS_IS_TESTING
571 else "buildkite-untrusted-api-token",
Florian Weikerta0e74592019-03-07 11:56:12 +0100572 "--ciphertext-file",
573 "-",
574 "--plaintext-file",
575 "-",
576 ],
Philipp Wollermanne67eec42019-05-24 15:18:20 +0200577 input=base64.b64decode(
578 self._ENCRYPTED_BUILDKITE_API_TESTING_TOKEN
579 if THIS_IS_TESTING
580 else self._ENCRYPTED_BUILDKITE_API_TOKEN
581 ),
Florian Weikerta0e74592019-03-07 11:56:12 +0100582 env=os.environ,
583 )
584 .decode("utf-8")
585 .strip()
586 )
587
588 def _open_url(self, url):
Florian Weikertde96a6f2019-03-07 14:57:50 +0100589 return (
590 urllib.request.urlopen("{}?access_token={}".format(url, self._token))
591 .read()
592 .decode("utf-8")
593 )
Florian Weikerta0e74592019-03-07 11:56:12 +0100594
595 def get_build_info(self, build_number):
596 url = self._BUILD_STATUS_URL_TEMPLATE.format(self._org, self._pipeline, build_number)
597 output = self._open_url(url)
598 return json.loads(output)
599
600 def get_build_log(self, job):
601 return self._open_url(job["raw_log_url"])
602
603
Philipp Wollermanndcaddd92018-02-21 14:13:43 +0100604def eprint(*args, **kwargs):
605 """
606 Print to stderr and flush (just in case).
607 """
608 print(*args, flush=True, file=sys.stderr, **kwargs)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +0100609
610
Jakob Buchgraber9f153542018-02-27 10:56:04 +0100611def is_windows():
Jakob Buchgraber09048fa2018-02-27 11:39:39 +0100612 return os.name == "nt"
Jakob Buchgraber9f153542018-02-27 10:56:04 +0100613
Jakob Buchgrabere6de16b2018-02-28 12:42:12 +0100614
Jakob Buchgraber9f153542018-02-27 10:56:04 +0100615def gsutil_command():
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200616 return "gsutil.cmd" if is_windows() else "gsutil"
Jakob Buchgraber9f153542018-02-27 10:56:04 +0100617
Jakob Buchgrabere6de16b2018-02-28 12:42:12 +0100618
Jakob Buchgraber9f153542018-02-27 10:56:04 +0100619def gcloud_command():
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200620 return "gcloud.cmd" if is_windows() else "gcloud"
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +0100621
Jakob Buchgrabere6de16b2018-02-28 12:42:12 +0100622
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +0100623def downstream_projects_root(platform):
Philipp Wollermann51147bf2019-05-08 15:50:10 +0200624 downstream_root = os.path.expandvars(PLATFORMS[platform]["downstream-root"])
625 if not os.path.exists(downstream_root):
626 os.makedirs(downstream_root)
627 return downstream_root
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +0100628
629
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +0100630def fetch_configs(http_url, file_config):
Philipp Wollermanndb024862018-02-19 17:16:56 +0100631 """
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +0100632 If specified fetches the build configuration from file_config or http_url, else tries to
Jakob Buchgraber25bb50f2018-02-22 18:06:21 +0100633 read it from .bazelci/presubmit.yml.
Philipp Wollermann598b4a42018-02-19 17:03:36 +0100634 Returns the json configuration as a python data structure.
Philipp Wollermanndb024862018-02-19 17:16:56 +0100635 """
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +0100636 if file_config is not None and http_url is not None:
637 raise BuildkiteException("file_config and http_url cannot be set at the same time")
638
Florian Weikertc8b3ed22019-05-31 16:14:12 +0200639 return load_config(http_url, file_config)
640
641
642def load_config(http_url, file_config, allow_imports=True):
Florian Weikertc8b3ed22019-05-31 16:14:12 +0200643 if http_url:
644 config = load_remote_yaml_file(http_url)
645 else:
646 file_config = file_config or ".bazelci/presubmit.yml"
647 with open(file_config, "r") as fd:
648 config = yaml.safe_load(fd)
649
Florian Weikert843d7a02019-02-03 17:24:50 +0100650 # Legacy mode means that there is exactly one task per platform (e.g. ubuntu1604_nojdk),
651 # which means that we can get away with using the platform name as task ID.
652 # No other updates are needed since get_platform_for_task() falls back to using the
653 # task ID as platform if there is no explicit "platforms" field.
654 if "platforms" in config:
655 config["tasks"] = config.pop("platforms")
656
Florian Weikertc8b3ed22019-05-31 16:14:12 +0200657 if "tasks" not in config:
658 config["tasks"] = {}
659
660 imports = config.pop("imports", None)
661 if imports:
662 if not allow_imports:
663 raise BuildkiteException("Nested imports are not allowed")
664
665 for i in imports:
666 imported_tasks = load_imported_tasks(i, http_url, file_config)
667 config["tasks"].update(imported_tasks)
668
Florian Weikert843d7a02019-02-03 17:24:50 +0100669 return config
670
671
Florian Weikert13215a82019-05-10 12:42:21 +0200672def load_remote_yaml_file(http_url):
673 with urllib.request.urlopen(http_url) as resp:
674 reader = codecs.getreader("utf-8")
Philipp Wollermannd00107e2019-05-18 23:50:59 +0200675 return yaml.safe_load(reader(resp))
Florian Weikert13215a82019-05-10 12:42:21 +0200676
677
Florian Weikertc8b3ed22019-05-31 16:14:12 +0200678def load_imported_tasks(import_name, http_url, file_config):
679 if "/" in import_name:
680 raise BuildkiteException("Invalid import '%s'" % import_name)
681
682 old_path = http_url or file_config
683 new_path = "%s%s" % (old_path[: old_path.rfind("/") + 1], import_name)
684 if http_url:
685 http_url = new_path
686 else:
687 file_config = new_path
688
689 imported_config = load_config(http_url=http_url, file_config=file_config, allow_imports=False)
690
691 namespace = import_name.partition(".")[0]
692 tasks = {}
693 for task_name, task_config in imported_config["tasks"].items():
Florian Weikert61f29b82019-08-12 16:56:56 +0200694 fix_imported_task_platform(task_name, task_config)
695 fix_imported_task_name(namespace, task_config)
696 fix_imported_task_working_directory(namespace, task_config)
Florian Weikertc8b3ed22019-05-31 16:14:12 +0200697 tasks["%s_%s" % (namespace, task_name)] = task_config
698
699 return tasks
700
701
Florian Weikert61f29b82019-08-12 16:56:56 +0200702def fix_imported_task_platform(task_name, task_config):
703 if "platform" not in task_config:
704 task_config["platform"] = task_name
705
706
707def fix_imported_task_name(namespace, task_config):
708 old_name = task_config.get("name")
709 task_config["name"] = "%s (%s)" % (namespace, old_name) if old_name else namespace
710
711
712def fix_imported_task_working_directory(namespace, task_config):
713 old_dir = task_config.get("working_directory")
714 task_config["working_directory"] = os.path.join(namespace, old_dir) if old_dir else namespace
715
716
Jakob Buchgraber3120f7a2018-02-18 13:28:02 +0100717def print_collapsed_group(name):
Jakob Buchgraber5c9b13d2018-02-21 22:28:14 +0100718 eprint("\n\n--- {0}\n\n".format(name))
Jakob Buchgraber3120f7a2018-02-18 13:28:02 +0100719
Jakob Buchgraber9c83de72018-02-18 15:32:44 +0100720
Jakob Buchgraber3120f7a2018-02-18 13:28:02 +0100721def print_expanded_group(name):
Jakob Buchgraber5c9b13d2018-02-21 22:28:14 +0100722 eprint("\n\n+++ {0}\n\n".format(name))
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +0100723
Jakob Buchgraber9c83de72018-02-18 15:32:44 +0100724
Yun Peng8975c6b2019-02-28 11:55:55 +0100725def use_bazelisk_migrate():
726 """
727 If USE_BAZELISK_MIGRATE is set, we use `bazelisk --migrate` to test incompatible flags.
728 """
Florian Weikertde96a6f2019-03-07 14:57:50 +0100729 return bool(os.environ.get("USE_BAZELISK_MIGRATE"))
Yun Peng8975c6b2019-02-28 11:55:55 +0100730
731
732def bazelisk_flags():
733 return ["--migrate"] if use_bazelisk_migrate() else []
734
735
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100736def execute_commands(
Florian Weikertc8642af2019-02-03 23:58:51 +0100737 task_config,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100738 platform,
739 git_repository,
740 git_commit,
741 git_repo_location,
742 use_bazel_at_commit,
743 use_but,
744 save_but,
Yun Peng4d1d6542019-01-17 18:30:33 +0100745 needs_clean,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100746 build_only,
747 test_only,
748 monitor_flaky_tests,
749 incompatible_flags,
Florian Weikertc8642af2019-02-03 23:58:51 +0100750 bazel_version=None,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100751):
Yun Pengf50f7b72019-02-28 19:09:52 +0100752 # If we want to test incompatible flags, we ignore bazel_version and always use
753 # the latest Bazel version through Bazelisk.
754 if incompatible_flags:
755 bazel_version = None
Florian Weikert13215a82019-05-10 12:42:21 +0200756 if not bazel_version:
757 # The last good version of Bazel can be specified in an emergency file.
758 # However, we only use last_good_bazel for pipelines that do not
759 # explicitly specify a version of Bazel.
760 try:
761 emergency_settings = load_remote_yaml_file(EMERGENCY_FILE_URL)
762 bazel_version = emergency_settings.get("last_good_bazel")
763 except urllib.error.HTTPError:
764 # Ignore this error. The Setup step will have already complained about
765 # it by showing an error message.
766 pass
Yun Pengf50f7b72019-02-28 19:09:52 +0100767
Jakob Buchgraberfb95a2f2018-02-22 11:46:25 +0100768 if build_only and test_only:
769 raise BuildkiteException("build_only and test_only cannot be true at the same time")
Philipp Wollermanne1318eb2018-08-13 15:08:01 +0200770
Yun Peng20d45602018-10-18 13:27:05 +0200771 if use_bazel_at_commit and use_but:
772 raise BuildkiteException("use_bazel_at_commit cannot be set when use_but is true")
773
Philipp Wollermanne1318eb2018-08-13 15:08:01 +0200774 tmpdir = tempfile.mkdtemp()
775 sc_process = None
Philipp Wollermann598b4a42018-02-19 17:03:36 +0100776 try:
Philipp Wollermann380f1e62019-04-12 16:45:27 +0200777 if platform == "macos":
Florian Weikertee84c5c2019-05-28 11:21:51 +0200778 activate_xcode(task_config)
Philipp Wollermann380f1e62019-04-12 16:45:27 +0200779
Florian Weikert4ee0bed2019-02-21 18:03:00 +0100780 # If the CI worker runs Bazelisk, we need to forward all required env variables to the test.
781 # Otherwise any integration test that invokes Bazel (=Bazelisk in this case) will fail.
Marcel Hlopko198328b2019-02-25 09:19:55 +0100782 test_env_vars = ["LocalAppData"] if platform == "windows" else ["HOME"]
Yun Peng376d2b32018-11-29 10:24:54 +0100783 if git_repo_location:
784 os.chdir(git_repo_location)
785 elif git_repository:
786 clone_git_repository(git_repository, platform, git_commit)
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200787
Philipp Wollermannf4aabb72019-06-25 15:59:00 +0200788 # We use one binary for all Linux platforms (because we also just release one binary for all
789 # Linux versions and we have to ensure that it works on all of them).
790 binary_platform = platform if platform in ["macos", "windows"] else LINUX_BINARY_PLATFORM
791
Yun Peng20d45602018-10-18 13:27:05 +0200792 if use_bazel_at_commit:
793 print_collapsed_group(":gcloud: Downloading Bazel built at " + use_bazel_at_commit)
Philipp Wollermannf4aabb72019-06-25 15:59:00 +0200794 bazel_binary = download_bazel_binary_at_commit(
795 tmpdir, binary_platform, use_bazel_at_commit
796 )
Philipp Wollermann639c0452019-01-03 11:23:54 +0100797 elif use_but:
Jakob Buchgraber92755d72018-02-22 15:33:37 +0100798 print_collapsed_group(":gcloud: Downloading Bazel Under Test")
Philipp Wollermannf4aabb72019-06-25 15:59:00 +0200799 bazel_binary = download_bazel_binary(tmpdir, binary_platform)
Philipp Wollermann639c0452019-01-03 11:23:54 +0100800 else:
801 bazel_binary = "bazel"
Florian Weikertc8642af2019-02-03 23:58:51 +0100802 if bazel_version:
803 # This will only work if the bazel binary in $PATH is actually a bazelisk binary
Philipp Wollermann55ff3572019-05-14 13:30:12 +0200804 # (https://github.com/bazelbuild/bazelisk).
Florian Weikertc8642af2019-02-03 23:58:51 +0100805 os.environ["USE_BAZEL_VERSION"] = bazel_version
Florian Weikert4ee0bed2019-02-21 18:03:00 +0100806 test_env_vars.append("USE_BAZEL_VERSION")
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200807
Philipp Wollermann5c7ea412019-05-24 15:26:57 +0200808 for key, value in task_config.get("environment", {}).items():
Philipp Wollermann4ad4aac2019-05-24 15:23:09 +0200809 # We have to explicitly convert the value to a string, because sometimes YAML tries to
810 # be smart and converts strings like "true" and "false" to booleans.
811 os.environ[key] = str(value)
Philipp Wollermann213ac9d2019-02-06 11:50:05 +0100812
Florian Weikerta8c020b2019-08-12 16:56:38 +0200813 cmd_exec_func = execute_batch_commands if platform == "windows" else execute_shell_commands
814 cmd_exec_func(task_config.get("setup", None))
815
Philipp Wollermanna5aee2c2019-02-11 16:55:19 +0100816 # Allow the config to override the current working directory.
817 required_prefix = os.getcwd()
818 requested_working_dir = os.path.abspath(task_config.get("working_directory", ""))
819 if os.path.commonpath([required_prefix, requested_working_dir]) != required_prefix:
820 raise BuildkiteException("working_directory refers to a path outside the workspace")
821 os.chdir(requested_working_dir)
822
Florian Weikerte72d68a2019-03-08 18:56:33 +0100823 if platform == "windows":
824 execute_batch_commands(task_config.get("batch_commands", None))
825 else:
826 execute_shell_commands(task_config.get("shell_commands", None))
827
Florian Weikertc8642af2019-02-03 23:58:51 +0100828 bazel_version = print_bazel_version_info(bazel_binary, platform)
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200829
Yun Penga5a1ee02018-12-05 15:00:58 +0100830 print_environment_variables_info()
831
Yun Pengd0217ed2018-11-30 14:51:11 +0100832 if incompatible_flags:
833 print_expanded_group("Build and test with the following incompatible flags:")
834 for flag in incompatible_flags:
835 eprint(flag + "\n")
836
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100837 execute_bazel_run(
Florian Weikertc8642af2019-02-03 23:58:51 +0100838 bazel_binary, platform, task_config.get("run_targets", None), incompatible_flags
Philipp Wollermanncd5694c2019-01-03 14:53:04 +0100839 )
Philipp Wollermann2409c6e2018-08-07 07:37:54 +0200840
Florian Weikertee84c5c2019-05-28 11:21:51 +0200841 if task_config.get("sauce"):
842 sc_process = start_sauce_connect_proxy(platform, tmpdir)
Philipp Wollermanne1318eb2018-08-13 15:08:01 +0200843
Yun Peng4d1d6542019-01-17 18:30:33 +0100844 if needs_clean:
Yun Pengea0359e2019-01-17 15:37:47 +0100845 execute_bazel_clean(bazel_binary, platform)
846
Florian Weikert736d06e2019-05-08 13:16:42 +0200847 build_targets, test_targets = calculate_targets(
848 task_config, platform, bazel_binary, build_only, test_only
849 )
Florian Weikert736d06e2019-05-08 13:16:42 +0200850
joeleba76887952019-05-16 15:22:17 +0200851 include_json_profile = task_config.get("include_json_profile", [])
852
Florian Weikert736d06e2019-05-08 13:16:42 +0200853 if build_targets:
joeleba76887952019-05-16 15:22:17 +0200854 json_profile_flags = []
Philipp Wollermann92cf51e2019-05-16 15:31:11 +0200855 include_json_profile_build = "build" in include_json_profile
Philipp Wollermannce986af2019-07-18 14:46:05 +0200856 json_profile_out_build = None
joeleba76887952019-05-16 15:22:17 +0200857 if include_json_profile_build:
joeleba05dbc8a2019-05-16 19:27:46 +0200858 json_profile_out_build = os.path.join(tmpdir, "build.profile.gz")
Philipp Wollermann92cf51e2019-05-16 15:31:11 +0200859 json_profile_flags = get_json_profile_flags(json_profile_out_build)
joeleba76887952019-05-16 15:22:17 +0200860
Florian Weikerte2affab2019-06-04 13:44:34 +0200861 build_flags = task_config.get("build_flags") or []
joeleba76887952019-05-16 15:22:17 +0200862 try:
863 execute_bazel_build(
864 bazel_version,
865 bazel_binary,
866 platform,
Florian Weikerte2affab2019-06-04 13:44:34 +0200867 build_flags + json_profile_flags,
joeleba76887952019-05-16 15:22:17 +0200868 build_targets,
869 None,
870 incompatible_flags,
871 )
872 if save_but:
873 upload_bazel_binary(platform)
874 finally:
875 if include_json_profile_build:
Philipp Wollermann92cf51e2019-05-16 15:31:11 +0200876 upload_json_profile(json_profile_out_build, tmpdir)
Philipp Wollermann639c0452019-01-03 11:23:54 +0100877
Florian Weikert736d06e2019-05-08 13:16:42 +0200878 if test_targets:
joeleba76887952019-05-16 15:22:17 +0200879 json_profile_flags = []
Philipp Wollermann92cf51e2019-05-16 15:31:11 +0200880 include_json_profile_test = "test" in include_json_profile
Philipp Wollermannce986af2019-07-18 14:46:05 +0200881 json_profile_out_test = None
joeleba76887952019-05-16 15:22:17 +0200882 if include_json_profile_test:
joeleba05dbc8a2019-05-16 19:27:46 +0200883 json_profile_out_test = os.path.join(tmpdir, "test.profile.gz")
Philipp Wollermann92cf51e2019-05-16 15:31:11 +0200884 json_profile_flags = get_json_profile_flags(json_profile_out_test)
joeleba76887952019-05-16 15:22:17 +0200885
Florian Weikerte2affab2019-06-04 13:44:34 +0200886 test_flags = task_config.get("test_flags") or []
887 test_flags += json_profile_flags
Florian Weikert4ee0bed2019-02-21 18:03:00 +0100888 if test_env_vars:
889 test_flags += ["--test_env={}".format(v) for v in test_env_vars]
890
Florian Weikert4901c662019-02-26 13:20:11 +0100891 if not is_windows():
892 # On platforms that support sandboxing (Linux, MacOS) we have
893 # to allow access to Bazelisk's cache directory.
894 # However, the flag requires the directory to exist,
895 # so we create it here in order to not crash when a test
896 # does not invoke Bazelisk.
897 bazelisk_cache_dir = get_bazelisk_cache_directory(platform)
898 os.makedirs(bazelisk_cache_dir, mode=0o755, exist_ok=True)
899 test_flags.append("--sandbox_writable_path={}".format(bazelisk_cache_dir))
Florian Weikert5b890332019-02-25 14:57:43 +0100900
Philipp Wollermannce986af2019-07-18 14:46:05 +0200901 test_bep_file = os.path.join(tmpdir, "test_bep.json")
902 stop_request = threading.Event()
903 upload_thread = threading.Thread(
904 target=upload_test_logs_from_bep, args=(test_bep_file, tmpdir, stop_request)
905 )
Jakob Buchgraber95e3d572018-02-21 18:48:49 +0100906 try:
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +0200907 upload_thread.start()
908 try:
909 execute_bazel_test(
910 bazel_version,
911 bazel_binary,
912 platform,
913 test_flags,
914 test_targets,
915 test_bep_file,
916 monitor_flaky_tests,
917 incompatible_flags,
918 )
919 if monitor_flaky_tests:
920 upload_bep_logs_for_flaky_tests(test_bep_file)
921 finally:
922 if include_json_profile_test:
923 upload_json_profile(json_profile_out_test, tmpdir)
Philipp Wollermann639c0452019-01-03 11:23:54 +0100924 finally:
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +0200925 stop_request.set()
926 upload_thread.join()
Philipp Wollermann598b4a42018-02-19 17:03:36 +0100927 finally:
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +0200928 terminate_background_process(sc_process)
Philipp Wollermann598b4a42018-02-19 17:03:36 +0100929 if tmpdir:
930 shutil.rmtree(tmpdir)
Philipp Wollermanndcaddd92018-02-21 14:13:43 +0100931
Philipp Wollermann3c8b8512019-07-16 15:28:03 +0200932
Florian Weikertee84c5c2019-05-28 11:21:51 +0200933def activate_xcode(task_config):
934 # Get the Xcode version from the config.
935 xcode_version = task_config.get("xcode_version", DEFAULT_XCODE_VERSION)
936 print_collapsed_group("Activating Xcode {}...".format(xcode_version))
937
938 # Ensure it's a valid version number.
939 if not isinstance(xcode_version, str):
940 raise BuildkiteException(
941 "Version number '{}' is not a string. Did you forget to put it in quotes?".format(
942 xcode_version
943 )
944 )
945 if not XCODE_VERSION_REGEX.match(xcode_version):
946 raise BuildkiteException(
947 "Invalid Xcode version format '{}', must match the format X.Y[.Z].".format(
948 xcode_version
949 )
950 )
951
952 # Check that the selected Xcode version is actually installed on the host.
953 xcode_path = "/Applications/Xcode{}.app".format(xcode_version)
954 if not os.path.exists(xcode_path):
955 raise BuildkiteException("Xcode not found at '{}'.".format(xcode_path))
956
957 # Now activate the specified Xcode version and let it install its required components.
958 # The CI machines have a sudoers config that allows the 'buildkite' user to run exactly
959 # these two commands, so don't change them without also modifying the file there.
960 execute_command(["/usr/bin/sudo", "/usr/bin/xcode-select", "--switch", xcode_path])
961 execute_command(["/usr/bin/sudo", "/usr/bin/xcodebuild", "-runFirstLaunch"])
962
963
Florian Weikert4901c662019-02-26 13:20:11 +0100964def get_bazelisk_cache_directory(platform):
965 # The path relies on the behavior of Go's os.UserCacheDir()
966 # and of the Go version of Bazelisk.
Philipp Wollermannce986af2019-07-18 14:46:05 +0200967 cache_dir = "Library/Caches" if platform == "macos" else ".cache"
968 return os.path.join(os.environ.get("HOME"), cache_dir, "bazelisk")
Florian Weikert4901c662019-02-26 13:20:11 +0100969
Florian Weikert5b890332019-02-25 14:57:43 +0100970
Jakob Buchgraber5d6c7142018-02-21 20:16:51 +0100971def tests_with_status(bep_file, status):
Jakob Buchgraberc874cdf2019-07-16 16:27:41 +0200972 return set(label for label, _ in test_logs_for_status(bep_file, status=[status]))
Jakob Buchgraber257693b2018-02-20 00:03:56 +0100973
Jakob Buchgraber6104a432018-02-21 21:16:53 +0100974
Florian Weikertee84c5c2019-05-28 11:21:51 +0200975def start_sauce_connect_proxy(platform, tmpdir):
976 print_collapsed_group(":saucelabs: Starting Sauce Connect Proxy")
977 os.environ["SAUCE_USERNAME"] = "bazel_rules_webtesting"
978 os.environ["SAUCE_ACCESS_KEY"] = saucelabs_token()
979 os.environ["TUNNEL_IDENTIFIER"] = str(uuid.uuid4())
980 os.environ["BUILD_TAG"] = str(uuid.uuid4())
981 readyfile = os.path.join(tmpdir, "sc_is_ready")
982 if platform == "windows":
Philipp Wollermann8a8c25a2019-08-23 12:56:36 +0200983 cmd = ["sauce-connect.exe", "-i", os.environ["TUNNEL_IDENTIFIER"], "-f", readyfile]
Florian Weikertee84c5c2019-05-28 11:21:51 +0200984 else:
985 cmd = ["sc", "-i", os.environ["TUNNEL_IDENTIFIER"], "-f", readyfile]
986 sc_process = execute_command_background(cmd)
987 wait_start = time.time()
988 while not os.path.exists(readyfile):
Philipp Wollermann8a8c25a2019-08-23 12:56:36 +0200989 if time.time() - wait_start > 60:
Florian Weikertee84c5c2019-05-28 11:21:51 +0200990 raise BuildkiteException(
Philipp Wollermann8a8c25a2019-08-23 12:56:36 +0200991 "Sauce Connect Proxy is still not ready after 60 seconds, aborting!"
Florian Weikertee84c5c2019-05-28 11:21:51 +0200992 )
993 time.sleep(1)
994 print("Sauce Connect Proxy is ready, continuing...")
995 return sc_process
996
997
Philipp Wollermann1403d2c2019-01-10 13:15:51 +0100998def saucelabs_token():
999 return (
1000 subprocess.check_output(
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001001 [
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01001002 gcloud_command(),
1003 "kms",
1004 "decrypt",
Philipp Wollermanna4722b42019-01-10 16:50:13 +01001005 "--project",
1006 "bazel-untrusted",
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01001007 "--location",
1008 "global",
1009 "--keyring",
1010 "buildkite",
1011 "--key",
1012 "saucelabs-access-key",
1013 "--ciphertext-file",
1014 "-",
1015 "--plaintext-file",
1016 "-",
1017 ],
1018 input=base64.b64decode(ENCRYPTED_SAUCELABS_TOKEN),
1019 env=os.environ,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001020 )
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01001021 .decode("utf-8")
1022 .strip()
1023 )
Yun Pengb6b18862019-01-07 14:31:55 +01001024
1025
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01001026def is_pull_request():
Jakob Buchgraber67761d32018-02-21 19:00:21 +01001027 third_party_repo = os.getenv("BUILDKITE_PULL_REQUEST_REPO", "")
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01001028 return len(third_party_repo) > 0
1029
1030
Jakob Buchgraber02e07222018-02-19 15:05:56 +01001031def has_flaky_tests(bep_file):
Jakob Buchgraberc874cdf2019-07-16 16:27:41 +02001032 return len(test_logs_for_status(bep_file, status=["FLAKY"])) > 0
Jakob Buchgraber02e07222018-02-19 15:05:56 +01001033
1034
Yun Penge3cf12d2018-12-05 15:01:09 +01001035def print_bazel_version_info(bazel_binary, platform):
Jakob Buchgraber99c4bbb2018-02-22 11:59:31 +01001036 print_collapsed_group(":information_source: Bazel Info")
Philipp Wollermannf13804b2019-02-05 21:08:30 +01001037 version_output = execute_command_and_get_output(
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001038 [bazel_binary]
1039 + common_startup_flags(platform)
1040 + ["--nomaster_bazelrc", "--bazelrc=/dev/null", "version"]
1041 )
1042 execute_command(
1043 [bazel_binary]
1044 + common_startup_flags(platform)
1045 + ["--nomaster_bazelrc", "--bazelrc=/dev/null", "info"]
1046 )
Jakob Buchgraber7e690a72018-02-18 13:22:15 +01001047
Florian Weikertc8642af2019-02-03 23:58:51 +01001048 match = BUILD_LABEL_PATTERN.search(version_output)
1049 return match.group(1) if match else "unreleased binary"
1050
Jakob Buchgraber7e690a72018-02-18 13:22:15 +01001051
Yun Penga5a1ee02018-12-05 15:00:58 +01001052def print_environment_variables_info():
1053 print_collapsed_group(":information_source: Environment Variables")
1054 for key, value in os.environ.items():
1055 eprint("%s=(%s)" % (key, value))
1056
1057
Jakob Buchgraber426399e2018-03-20 19:45:46 +01001058def upload_bazel_binary(platform):
Jakob Buchgraber7d1d3bb2018-02-21 22:38:22 +01001059 print_collapsed_group(":gcloud: Uploading Bazel Under Test")
Tobias Werth322aa742018-12-20 11:44:16 +01001060 binary_path = "bazel-bin/src/bazel"
Jakob Buchgraber426399e2018-03-20 19:45:46 +01001061 if platform == "windows":
Tobias Werth322aa742018-12-20 11:44:16 +01001062 binary_path = r"bazel-bin\src\bazel"
Philipp Wollermann94bd9e32018-04-30 15:32:28 +02001063 execute_command(["buildkite-agent", "artifact", "upload", binary_path])
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001064
1065
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01001066def download_bazel_binary(dest_dir, platform):
Tobias Werth322aa742018-12-20 11:44:16 +01001067 binary_path = "bazel-bin/src/bazel"
Jakob Buchgraber426399e2018-03-20 19:45:46 +01001068 if platform == "windows":
Tobias Werth322aa742018-12-20 11:44:16 +01001069 binary_path = r"bazel-bin\src\bazel"
Jakob Buchgraber426399e2018-03-20 19:45:46 +01001070
Philipp Wollermannc52e26a2019-05-18 22:10:47 +02001071 source_step = create_label(platform, "Bazel", build_only=True)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001072 execute_command(
1073 ["buildkite-agent", "artifact", "download", binary_path, dest_dir, "--step", source_step]
1074 )
Jakob Buchgraber00046232018-03-27 20:15:26 +02001075 bazel_binary_path = os.path.join(dest_dir, binary_path)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001076 st = os.stat(bazel_binary_path)
1077 os.chmod(bazel_binary_path, st.st_mode | stat.S_IEXEC)
1078 return bazel_binary_path
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001079
1080
Yun Peng20d45602018-10-18 13:27:05 +02001081def download_bazel_binary_at_commit(dest_dir, platform, bazel_git_commit):
Yun Peng20d45602018-10-18 13:27:05 +02001082 bazel_binary_path = os.path.join(dest_dir, "bazel.exe" if platform == "windows" else "bazel")
Yun Peng02312732019-01-17 18:17:05 +01001083 try:
1084 execute_command(
1085 [
1086 gsutil_command(),
1087 "cp",
1088 bazelci_builds_gs_url(platform, bazel_git_commit),
1089 bazel_binary_path,
1090 ]
1091 )
1092 except subprocess.CalledProcessError as e:
Philipp Wollermannc05ac682019-01-19 12:37:28 +01001093 raise BuildkiteException(
1094 "Failed to download Bazel binary at %s, error message:\n%s" % (bazel_git_commit, str(e))
1095 )
Yun Peng20d45602018-10-18 13:27:05 +02001096 st = os.stat(bazel_binary_path)
1097 os.chmod(bazel_binary_path, st.st_mode | stat.S_IEXEC)
1098 return bazel_binary_path
1099
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001100
joeleba7050d842019-05-23 17:03:31 +02001101def get_mirror_path(git_repository, platform):
1102 mirror_root = {
1103 "macos": "/usr/local/var/bazelbuild/",
1104 "windows": "c:\\buildkite\\bazelbuild\\",
1105 }.get(platform, "/var/lib/bazelbuild/")
1106
1107 return mirror_root + re.sub(r"[^0-9A-Za-z]", "-", git_repository)
1108
1109
Yun Peng376d2b32018-11-29 10:24:54 +01001110def clone_git_repository(git_repository, platform, git_commit=None):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001111 root = downstream_projects_root(platform)
Philipp Wollermannff39ef52018-02-21 14:18:52 +01001112 project_name = re.search(r"/([^/]+)\.git$", git_repository).group(1)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001113 clone_path = os.path.join(root, project_name)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001114 print_collapsed_group(
1115 "Fetching %s sources at %s" % (project_name, git_commit if git_commit else "HEAD")
1116 )
Philipp Wollermann438ec242018-09-05 14:39:24 +02001117
joeleba7050d842019-05-23 17:03:31 +02001118 mirror_path = get_mirror_path(git_repository, platform)
Philipp Wollermannea128282019-05-08 11:56:14 +02001119
Philipp Wollermann414703d2018-08-28 16:40:38 +02001120 if not os.path.exists(clone_path):
Philipp Wollermann62f4a032019-05-08 17:44:14 +02001121 if os.path.exists(mirror_path):
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001122 execute_command(
Philipp Wollermann62f4a032019-05-08 17:44:14 +02001123 ["git", "clone", "-v", "--reference", mirror_path, git_repository, clone_path]
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001124 )
Philipp Wollermannd4cd0d82018-05-01 09:56:24 +02001125 else:
Philipp Wollermannea128282019-05-08 11:56:14 +02001126 execute_command(["git", "clone", "-v", git_repository, clone_path])
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001127
Philipp Wollermann414703d2018-08-28 16:40:38 +02001128 os.chdir(clone_path)
1129 execute_command(["git", "remote", "set-url", "origin", git_repository])
1130 execute_command(["git", "clean", "-fdqx"])
Florian Weikertd8f497c2019-06-19 15:44:20 +02001131 execute_command(["git", "submodule", "foreach", "--recursive", "git clean -fdqx"])
Philipp Wollermann414703d2018-08-28 16:40:38 +02001132 execute_command(["git", "fetch", "origin"])
Yun Peng376d2b32018-11-29 10:24:54 +01001133 if git_commit:
1134 # sync to a specific commit of this repository
1135 execute_command(["git", "reset", git_commit, "--hard"])
1136 else:
1137 # sync to the latest commit of HEAD. Unlikely git pull this also works after a force push.
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001138 remote_head = (
1139 subprocess.check_output(["git", "symbolic-ref", "refs/remotes/origin/HEAD"])
1140 .decode("utf-8")
1141 .rstrip()
1142 )
Yun Peng376d2b32018-11-29 10:24:54 +01001143 execute_command(["git", "reset", remote_head, "--hard"])
Philipp Wollermann414703d2018-08-28 16:40:38 +02001144 execute_command(["git", "submodule", "sync", "--recursive"])
1145 execute_command(["git", "submodule", "update", "--init", "--recursive", "--force"])
Florian Weikertd8f497c2019-06-19 15:44:20 +02001146 execute_command(["git", "submodule", "foreach", "--recursive", "git reset --hard"])
Philipp Wollermann414703d2018-08-28 16:40:38 +02001147 execute_command(["git", "clean", "-fdqx"])
Florian Weikertd8f497c2019-06-19 15:44:20 +02001148 execute_command(["git", "submodule", "foreach", "--recursive", "git clean -fdqx"])
Yun Peng20d45602018-10-18 13:27:05 +02001149 return clone_path
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001150
Philipp Wollermann438ec242018-09-05 14:39:24 +02001151
Yun Penga935a542018-05-18 15:08:53 +02001152def execute_batch_commands(commands):
1153 if not commands:
1154 return
1155 print_collapsed_group(":batch: Setup (Batch Commands)")
1156 batch_commands = "&".join(commands)
Jakob Buchgraber4a824412018-06-22 12:56:10 +02001157 return subprocess.run(batch_commands, shell=True, check=True, env=os.environ).returncode
Yun Penga935a542018-05-18 15:08:53 +02001158
Philipp Wollermann414703d2018-08-28 16:40:38 +02001159
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001160def execute_shell_commands(commands):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001161 if not commands:
1162 return
Jakob Buchgraber94d5c212018-02-22 09:57:08 +01001163 print_collapsed_group(":bash: Setup (Shell Commands)")
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001164 shell_command = "\n".join(commands)
Philipp Wollermann3e1a7712018-02-19 17:34:24 +01001165 execute_command([shell_command], shell=True)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001166
1167
Yun Peng0a6a98a2019-03-06 13:07:54 +01001168def handle_bazel_failure(exception, action):
1169 msg = "bazel {0} failed with exit code {1}".format(action, exception.returncode)
1170 if use_bazelisk_migrate():
1171 print_collapsed_group(msg)
1172 else:
1173 raise BuildkiteException(msg)
1174
1175
Yun Peng4be92b32018-11-30 09:48:29 +01001176def execute_bazel_run(bazel_binary, platform, targets, incompatible_flags):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001177 if not targets:
1178 return
1179 print_collapsed_group("Setup (Run Targets)")
Florian Weikert474d7972019-03-01 02:12:01 +01001180 # When using bazelisk --migrate to test incompatible flags,
1181 # incompatible flags set by "INCOMPATIBLE_FLAGS" env var will be ignored.
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001182 incompatible_flags_to_use = (
1183 [] if (use_bazelisk_migrate() or not incompatible_flags) else incompatible_flags
1184 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001185 for target in targets:
Yun Peng0a6a98a2019-03-06 13:07:54 +01001186 try:
1187 execute_command(
1188 [bazel_binary]
1189 + bazelisk_flags()
1190 + common_startup_flags(platform)
1191 + ["run"]
1192 + common_build_flags(None, platform)
1193 + incompatible_flags_to_use
1194 + [target]
1195 )
1196 except subprocess.CalledProcessError as e:
1197 handle_bazel_failure(e, "run")
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001198
1199
Jakob Buchgraber4f1d2712018-02-20 10:22:47 +01001200def remote_caching_flags(platform):
Philipp Wollermanne67eec42019-05-24 15:18:20 +02001201 # Only enable caching for untrusted and testing builds.
Philipp Wollermannfce92bf2019-05-22 15:14:32 +02001202 if CLOUD_PROJECT not in ["bazel-untrusted"]:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001203 return []
Philipp Wollermann02955272019-04-18 18:00:48 +02001204
Philipp Wollermanne67eec42019-05-24 15:18:20 +02001205 platform_cache_key = [BUILDKITE_ORG.encode("utf-8")]
Jakob Buchgraber89df3982019-08-06 13:07:02 +02001206 # Whenever the remote cache was known to have been poisoned increase the number below
Philipp Wollermann0a1a7092019-08-30 11:07:55 +02001207 platform_cache_key += ["cache-poisoning-20190830".encode("utf-8")]
Philipp Wollermannfce92bf2019-05-22 15:14:32 +02001208
Philipp Wollermann380f1e62019-04-12 16:45:27 +02001209 if platform == "macos":
Philipp Wollermannfce92bf2019-05-22 15:14:32 +02001210 platform_cache_key += [
Philipp Wollermannef89d2f2019-04-18 15:52:24 +02001211 # macOS version:
1212 subprocess.check_output(["/usr/bin/sw_vers", "-productVersion"]),
1213 # Path to Xcode:
1214 subprocess.check_output(["/usr/bin/xcode-select", "-p"]),
1215 # Xcode version:
1216 subprocess.check_output(["/usr/bin/xcodebuild", "-version"]),
1217 ]
1218 # Use a local cache server for our macOS machines.
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001219 flags = ["--remote_cache=http://100.107.73.186"]
Philipp Wollermannef89d2f2019-04-18 15:52:24 +02001220 else:
Philipp Wollermannfce92bf2019-05-22 15:14:32 +02001221 platform_cache_key += [
Philipp Wollermannef89d2f2019-04-18 15:52:24 +02001222 # Platform name:
1223 platform.encode("utf-8")
1224 ]
Philipp Wollermanne74da4e2019-06-07 11:31:02 +02001225 # Use RBE for caching builds running on GCE.
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001226 flags = [
1227 "--google_default_credentials",
Philipp Wollermanne74da4e2019-06-07 11:31:02 +02001228 "--remote_cache=remotebuildexecution.googleapis.com",
1229 "--remote_instance_name=projects/{}/instances/default_instance".format(CLOUD_PROJECT),
1230 "--tls_enabled=true",
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001231 ]
Philipp Wollermann380f1e62019-04-12 16:45:27 +02001232
1233 platform_cache_digest = hashlib.sha256()
1234 for key in platform_cache_key:
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001235 eprint("Adding to platform cache key: {}".format(key))
Philipp Wollermann380f1e62019-04-12 16:45:27 +02001236 platform_cache_digest.update(key)
1237 platform_cache_digest.update(b":")
1238
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001239 flags += [
Philipp Wollermann94937722019-01-11 14:33:18 +01001240 "--remote_timeout=60",
Philipp Wollermann639c0452019-01-03 11:23:54 +01001241 "--remote_max_connections=200",
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001242 '--remote_default_platform_properties=properties:{name:"cache-silo-key" value:"%s"}'
1243 % platform_cache_digest.hexdigest(),
Philipp Wollermann639c0452019-01-03 11:23:54 +01001244 ]
Jakob Buchgraber4f1d2712018-02-20 10:22:47 +01001245
Philipp Wollermannd96d8fa2019-01-11 14:37:47 +01001246 return flags
1247
Jakob Buchgraber4f1d2712018-02-20 10:22:47 +01001248
Jakob Buchgraberb4342cd2018-02-20 16:35:07 +01001249def remote_enabled(flags):
1250 # Detect if the project configuration enabled its own remote caching / execution.
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001251 remote_flags = ["--remote_executor", "--remote_cache", "--remote_http_cache"]
Jakob Buchgraberb4342cd2018-02-20 16:35:07 +01001252 for flag in flags:
1253 for remote_flag in remote_flags:
1254 if flag.startswith(remote_flag):
1255 return True
1256 return False
1257
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01001258
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001259def concurrent_jobs(platform):
1260 return "75" if platform.startswith("rbe_") else str(multiprocessing.cpu_count())
Jakob Buchgraber51a83662018-02-22 19:49:24 +01001261
1262
Philipp Wollermann3e28d3b2018-02-23 23:19:37 +01001263def concurrent_test_jobs(platform):
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001264 if platform.startswith("rbe_"):
1265 return "75"
1266 elif platform == "windows":
Jakob Buchgrabere3ccda32018-06-22 23:29:48 +02001267 return "8"
Philipp Wollermann111adfb2018-11-22 10:26:03 +01001268 elif platform == "macos":
1269 return "8"
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001270 return "12"
Philipp Wollermann3e28d3b2018-02-23 23:19:37 +01001271
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001272
Yun Peng58977d62018-11-16 12:19:20 +01001273def common_startup_flags(platform):
Philipp Wollermann639c0452019-01-03 11:23:54 +01001274 return ["--output_user_root=D:/b"] if platform == "windows" else []
Yun Peng58977d62018-11-16 12:19:20 +01001275
1276
1277def common_build_flags(bep_file, platform):
Yun Peng088cc932018-11-16 12:11:46 +01001278 flags = [
Yun Pengf51e7842018-11-16 11:35:43 +01001279 "--show_progress_rate_limit=5",
1280 "--curses=yes",
1281 "--color=yes",
Philipp Wollermannd99414c2019-05-28 17:26:09 +02001282 "--terminal_columns=143",
Philipp Wollermann4c8391e2019-05-28 18:03:35 +02001283 "--show_timestamps",
Yun Pengf51e7842018-11-16 11:35:43 +01001284 "--verbose_failures",
1285 "--keep_going",
1286 "--jobs=" + concurrent_jobs(platform),
Yun Pengf51e7842018-11-16 11:35:43 +01001287 "--announce_rc",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001288 "--experimental_multi_threaded_digest",
Philipp Wollermannb97f9102019-04-16 18:05:56 +02001289 "--experimental_repository_cache_hardlinks",
Philipp Wollermannef89d2f2019-04-18 15:52:24 +02001290 # Some projects set --disk_cache in their project-specific bazelrc, which we never want on
1291 # CI, so let's just disable it explicitly.
1292 "--disk_cache=",
Yun Peng088cc932018-11-16 12:11:46 +01001293 ]
Philipp Wollermann639c0452019-01-03 11:23:54 +01001294
Philipp Wollermannb97f9102019-04-16 18:05:56 +02001295 if platform == "windows":
1296 pass
1297 elif platform == "macos":
1298 flags += [
1299 "--sandbox_writable_path=/var/tmp/_bazel_buildkite/cache/repos/v1",
1300 "--test_env=REPOSITORY_CACHE=/var/tmp/_bazel_buildkite/cache/repos/v1",
1301 ]
1302 else:
Philipp Wollermann639c0452019-01-03 11:23:54 +01001303 flags += ["--sandbox_tmpfs_path=/tmp"]
1304
Yun Peng088cc932018-11-16 12:11:46 +01001305 if bep_file:
Philipp Wollermann639c0452019-01-03 11:23:54 +01001306 flags += [
1307 "--experimental_build_event_json_file_path_conversion=false",
1308 "--build_event_json_file=" + bep_file,
1309 ]
1310
Yun Peng088cc932018-11-16 12:11:46 +01001311 return flags
Philipp Wollermann94bd9e32018-04-30 15:32:28 +02001312
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001313
Yun Pengb7247ff2018-11-15 13:52:39 +01001314def rbe_flags(original_flags, accept_cached):
Philipp Wollermannbcfd9da2018-08-09 15:31:18 +02001315 # Enable remote execution via RBE.
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001316 flags = [
1317 "--remote_executor=remotebuildexecution.googleapis.com",
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01001318 "--remote_instance_name=projects/bazel-untrusted/instances/default_instance",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001319 "--remote_timeout=3600",
Yun Peng97d69c52019-05-21 14:02:53 +02001320 # TODO(pcloudy): Remove this flag after upgrading Bazel to 0.27.0
1321 "--incompatible_list_based_execution_strategy_selection",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001322 "--experimental_strict_action_env",
1323 "--tls_enabled=true",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001324 "--google_default_credentials",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001325 ]
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02001326
Philipp Wollermannbcfd9da2018-08-09 15:31:18 +02001327 # Enable BES / Build Results reporting.
1328 flags += [
1329 "--bes_backend=buildeventservice.googleapis.com",
Philipp Wollermannbcfd9da2018-08-09 15:31:18 +02001330 "--bes_timeout=360s",
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01001331 "--project_id=bazel-untrusted",
Philipp Wollermannbcfd9da2018-08-09 15:31:18 +02001332 ]
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02001333
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001334 if not accept_cached:
1335 flags += ["--noremote_accept_cached"]
Philipp Wollermannbcfd9da2018-08-09 15:31:18 +02001336
Nicolas Lopez36996222019-05-28 12:21:28 -04001337 # Adapted from https://github.com/bazelbuild/bazel-toolchains/blob/master/bazelrc/.bazelrc
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001338 flags += [
Nicolas Lopez36996222019-05-28 12:21:28 -04001339 # These should NOT longer need to be modified.
1340 # All that is needed is updating the @bazel_toolchains repo pin
1341 # in projects' WORKSPACE files.
Xindb02c012018-11-07 14:10:54 -05001342 #
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001343 # Toolchain related flags to append at the end of your .bazelrc file.
Nicolas Lopez36996222019-05-28 12:21:28 -04001344 "--host_javabase=@buildkite_config//java:jdk",
1345 "--javabase=@buildkite_config//java:jdk",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001346 "--host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8",
1347 "--java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8",
Nicolas Lopez36996222019-05-28 12:21:28 -04001348 "--crosstool_top=@buildkite_config//cc:toolchain",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001349 "--action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1",
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001350 ]
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02001351
Yun Pengb7247ff2018-11-15 13:52:39 +01001352 # Platform flags:
1353 # The toolchain container used for execution is defined in the target indicated
1354 # by "extra_execution_platforms", "host_platform" and "platforms".
Xin03a88ab2019-03-11 19:18:50 -04001355 # If you are using your own toolchain container, you need to create a platform
1356 # target with "constraint_values" that allow for the toolchain specified with
1357 # "extra_toolchains" to be selected (given constraints defined in
1358 # "exec_compatible_with").
Yun Pengb7247ff2018-11-15 13:52:39 +01001359 # More about platforms: https://docs.bazel.build/versions/master/platforms.html
1360 # Don't add platform flags if they are specified already.
1361 platform_flags = {
Nicolas Lopez36996222019-05-28 12:21:28 -04001362 "--extra_toolchains": "@buildkite_config//config:cc-toolchain",
1363 "--extra_execution_platforms": "@buildkite_config//config:platform",
1364 "--host_platform": "@buildkite_config//config:platform",
1365 "--platforms": "@buildkite_config//config:platform",
Yun Pengb7247ff2018-11-15 13:52:39 +01001366 }
Yun Peng67ab5062018-11-15 13:58:15 +01001367 for platform_flag, value in list(platform_flags.items()):
Yun Pengb7247ff2018-11-15 13:52:39 +01001368 found = False
1369 for original_flag in original_flags:
1370 if original_flag.startswith(platform_flag):
1371 found = True
1372 break
1373 if not found:
1374 flags += [platform_flag + "=" + value]
1375
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001376 return flags
1377
1378
Florian Weikert24a4b482018-11-30 19:05:38 +01001379def compute_flags(platform, flags, incompatible_flags, bep_file, enable_remote_cache=False):
Yun Peng58977d62018-11-16 12:19:20 +01001380 aggregated_flags = common_build_flags(bep_file, platform)
Yun Peng32dbec32018-11-02 12:47:41 +01001381 if not remote_enabled(flags):
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001382 if platform.startswith("rbe_"):
Florian Weikert24a4b482018-11-30 19:05:38 +01001383 aggregated_flags += rbe_flags(flags, accept_cached=enable_remote_cache)
1384 elif enable_remote_cache:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001385 aggregated_flags += remote_caching_flags(platform)
Yun Peng75786552018-11-13 15:23:30 +01001386 aggregated_flags += flags
Yun Peng4be92b32018-11-30 09:48:29 +01001387 if incompatible_flags:
1388 aggregated_flags += incompatible_flags
Yun Peng53598002018-12-03 10:42:02 +01001389
Florian Weikert24a4b482018-11-30 19:05:38 +01001390 return aggregated_flags
Yun Peng53598002018-12-03 10:42:02 +01001391
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001392
Yun Pengea0359e2019-01-17 15:37:47 +01001393def execute_bazel_clean(bazel_binary, platform):
1394 print_expanded_group(":bazel: Clean")
1395
1396 try:
Philipp Wollermannc05ac682019-01-19 12:37:28 +01001397 execute_command([bazel_binary] + common_startup_flags(platform) + ["clean", "--expunge"])
Yun Pengea0359e2019-01-17 15:37:47 +01001398 except subprocess.CalledProcessError as e:
1399 raise BuildkiteException("bazel clean failed with exit code {}".format(e.returncode))
1400
1401
Florian Weikertc8642af2019-02-03 23:58:51 +01001402def execute_bazel_build(
1403 bazel_version, bazel_binary, platform, flags, targets, bep_file, incompatible_flags
1404):
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001405 print_collapsed_group(":bazel: Computing flags for build step")
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001406 aggregated_flags = compute_flags(
Yun Peng8975c6b2019-02-28 11:55:55 +01001407 platform,
1408 flags,
1409 # When using bazelisk --migrate to test incompatible flags,
1410 # incompatible flags set by "INCOMPATIBLE_FLAGS" env var will be ignored.
1411 [] if (use_bazelisk_migrate() or not incompatible_flags) else incompatible_flags,
1412 bep_file,
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001413 enable_remote_cache=True,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001414 )
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001415
1416 print_expanded_group(":bazel: Build ({})".format(bazel_version))
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01001417 try:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001418 execute_command(
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001419 [bazel_binary]
1420 + bazelisk_flags()
1421 + common_startup_flags(platform)
1422 + ["build"]
1423 + aggregated_flags
Philipp Wollermann2a160432019-09-19 15:57:28 +02001424 + ["--"]
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001425 + targets
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001426 )
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01001427 except subprocess.CalledProcessError as e:
Yun Peng0a6a98a2019-03-06 13:07:54 +01001428 handle_bazel_failure(e, "build")
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001429
1430
Florian Weikert736d06e2019-05-08 13:16:42 +02001431def calculate_targets(task_config, platform, bazel_binary, build_only, test_only):
1432 build_targets = [] if test_only else task_config.get("build_targets", [])
1433 test_targets = [] if build_only else task_config.get("test_targets", [])
1434
Philipp Wollermann2a160432019-09-19 15:57:28 +02001435 # Remove the "--" argument splitter from the list that some configs explicitly
1436 # include. We'll add it back again later where needed.
1437 build_targets = [x.strip() for x in build_targets if x.strip() != "--"]
1438 test_targets = [x.strip() for x in test_targets if x.strip() != "--"]
1439
Florian Weikert736d06e2019-05-08 13:16:42 +02001440 shard_id = int(os.getenv("BUILDKITE_PARALLEL_JOB", "-1"))
1441 shard_count = int(os.getenv("BUILDKITE_PARALLEL_JOB_COUNT", "-1"))
1442 if shard_id > -1 and shard_count > -1:
1443 print_collapsed_group(
1444 ":female-detective: Calculating targets for shard {}/{}".format(
1445 shard_id + 1, shard_count
1446 )
1447 )
1448 expanded_test_targets = expand_test_target_patterns(bazel_binary, platform, test_targets)
1449 build_targets, test_targets = get_targets_for_shard(
1450 build_targets, expanded_test_targets, shard_id, shard_count
1451 )
1452
1453 return build_targets, test_targets
1454
1455
1456def expand_test_target_patterns(bazel_binary, platform, test_targets):
Philipp Wollermann2a160432019-09-19 15:57:28 +02001457 included_targets, excluded_targets = partition_targets(test_targets)
Florian Weikert736d06e2019-05-08 13:16:42 +02001458 excluded_string = (
1459 " except tests(set({}))".format(" ".join("'{}'".format(t) for t in excluded_targets))
1460 if excluded_targets
1461 else ""
1462 )
1463
Philipp Wollermann2a160432019-09-19 15:57:28 +02001464 exclude_manual = ' except tests(attr("tags", "manual", set({})))'.format(
1465 " ".join("'{}'".format(t) for t in included_targets)
1466 )
1467
Florian Weikert736d06e2019-05-08 13:16:42 +02001468 eprint("Resolving test targets via bazel query")
1469 output = execute_command_and_get_output(
1470 [bazel_binary]
1471 + common_startup_flags(platform)
1472 + [
1473 "--nomaster_bazelrc",
1474 "--bazelrc=/dev/null",
1475 "query",
Philipp Wollermann2a160432019-09-19 15:57:28 +02001476 "tests(set({})){}{}".format(
1477 " ".join("'{}'".format(t) for t in included_targets),
1478 excluded_string,
1479 exclude_manual,
Florian Weikert736d06e2019-05-08 13:16:42 +02001480 ),
1481 ],
1482 print_output=False,
Florian Weikert736d06e2019-05-08 13:16:42 +02001483 )
Philipp Wollermann2a160432019-09-19 15:57:28 +02001484 return output.strip().split("\n")
Florian Weikert736d06e2019-05-08 13:16:42 +02001485
1486
Philipp Wollermann2a160432019-09-19 15:57:28 +02001487def partition_targets(targets):
Florian Weikert736d06e2019-05-08 13:16:42 +02001488 included_targets, excluded_targets = [], []
Philipp Wollermann2a160432019-09-19 15:57:28 +02001489 for target in targets:
1490 if target.startswith("-"):
Florian Weikert736d06e2019-05-08 13:16:42 +02001491 excluded_targets.append(target[1:])
1492 else:
1493 included_targets.append(target)
1494
1495 return included_targets, excluded_targets
1496
1497
1498def get_targets_for_shard(build_targets, test_targets, shard_id, shard_count):
1499 # TODO(fweikert): implement a more sophisticated algorithm
Philipp Wollermann2a160432019-09-19 15:57:28 +02001500 included_build_targets, excluded_build_targets = partition_targets(build_targets)
1501 build_targets_for_this_shard = sorted(included_build_targets)[shard_id::shard_count] + [
1502 "-" + x for x in excluded_build_targets
1503 ]
Florian Weikert736d06e2019-05-08 13:16:42 +02001504 test_targets_for_this_shard = sorted(test_targets)[shard_id::shard_count]
1505
1506 return build_targets_for_this_shard, test_targets_for_this_shard
1507
1508
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001509def execute_bazel_test(
Florian Weikertc8642af2019-02-03 23:58:51 +01001510 bazel_version,
1511 bazel_binary,
1512 platform,
1513 flags,
1514 targets,
1515 bep_file,
1516 monitor_flaky_tests,
1517 incompatible_flags,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001518):
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001519 aggregated_flags = [
1520 "--flaky_test_attempts=3",
1521 "--build_tests_only",
1522 "--local_test_jobs=" + concurrent_test_jobs(platform),
1523 ]
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001524 # Don't enable remote caching if the user enabled remote execution / caching themselves
Jakob Buchgraberc340f582018-06-22 13:48:33 +02001525 # or flaky test monitoring is enabled, as remote caching makes tests look less flaky than
1526 # they are.
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001527 print_collapsed_group(":bazel: Computing flags for test step")
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001528 aggregated_flags += compute_flags(
Yun Peng8975c6b2019-02-28 11:55:55 +01001529 platform,
1530 flags,
1531 # When using bazelisk --migrate to test incompatible flags,
1532 # incompatible flags set by "INCOMPATIBLE_FLAGS" env var will be ignored.
1533 [] if (use_bazelisk_migrate() or not incompatible_flags) else incompatible_flags,
1534 bep_file,
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001535 enable_remote_cache=not monitor_flaky_tests,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001536 )
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001537
Philipp Wollermannbda4b7d2019-05-16 20:04:17 +02001538 print_expanded_group(":bazel: Test ({})".format(bazel_version))
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01001539 try:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001540 execute_command(
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001541 [bazel_binary]
1542 + bazelisk_flags()
1543 + common_startup_flags(platform)
1544 + ["test"]
1545 + aggregated_flags
Philipp Wollermann2a160432019-09-19 15:57:28 +02001546 + ["--"]
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001547 + targets
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001548 )
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01001549 except subprocess.CalledProcessError as e:
Yun Peng0a6a98a2019-03-06 13:07:54 +01001550 handle_bazel_failure(e, "test")
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001551
1552
joeleba76887952019-05-16 15:22:17 +02001553def get_json_profile_flags(out_file):
1554 return [
1555 "--experimental_generate_json_trace_profile",
1556 "--experimental_profile_cpu_usage",
1557 "--experimental_json_trace_compression",
Philipp Wollermann92cf51e2019-05-16 15:31:11 +02001558 "--profile={}".format(out_file),
joeleba76887952019-05-16 15:22:17 +02001559 ]
1560
1561
Florian Weikertee84c5c2019-05-28 11:21:51 +02001562def upload_bep_logs_for_flaky_tests(test_bep_file):
1563 if has_flaky_tests(test_bep_file):
1564 build_number = os.getenv("BUILDKITE_BUILD_NUMBER")
1565 pipeline_slug = os.getenv("BUILDKITE_PIPELINE_SLUG")
1566 execute_command(
1567 [
1568 gsutil_command(),
1569 "cp",
1570 test_bep_file,
1571 FLAKY_TESTS_BUCKET + pipeline_slug + "/" + build_number + ".json",
1572 ]
1573 )
1574
1575
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001576def upload_test_logs_from_bep(bep_file, tmpdir, stop_request):
1577 uploaded_targets = set()
1578 while True:
1579 done = stop_request.isSet()
1580 if os.path.exists(bep_file):
Jakob Buchgraberc874cdf2019-07-16 16:27:41 +02001581 all_test_logs = test_logs_for_status(bep_file, status=["FAILED", "TIMEOUT", "FLAKY"])
Philipp Wollermann3c8b8512019-07-16 15:28:03 +02001582 test_logs_to_upload = [
1583 (target, files) for target, files in all_test_logs if target not in uploaded_targets
1584 ]
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001585
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001586 if test_logs_to_upload:
1587 files_to_upload = rename_test_logs_for_upload(test_logs_to_upload, tmpdir)
1588 cwd = os.getcwd()
1589 try:
1590 os.chdir(tmpdir)
1591 test_logs = [os.path.relpath(file, tmpdir) for file in files_to_upload]
1592 test_logs = sorted(test_logs)
1593 execute_command(["buildkite-agent", "artifact", "upload", ";".join(test_logs)])
1594 finally:
1595 uploaded_targets.update([target for target, _ in test_logs_to_upload])
1596 os.chdir(cwd)
1597 if done:
1598 break
1599 time.sleep(0.2)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001600
Philipp Wollermann3c8b8512019-07-16 15:28:03 +02001601
joeleba76887952019-05-16 15:22:17 +02001602def upload_json_profile(json_profile_path, tmpdir):
1603 if not os.path.exists(json_profile_path):
1604 return
1605 print_collapsed_group(":gcloud: Uploading JSON Profile")
Philipp Wollermann92cf51e2019-05-16 15:31:11 +02001606 execute_command(["buildkite-agent", "artifact", "upload", json_profile_path], cwd=tmpdir)
joeleba76887952019-05-16 15:22:17 +02001607
Philipp Wollermann5b00a702019-07-18 11:21:38 +02001608
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001609def rename_test_logs_for_upload(test_logs, tmpdir):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001610 # Rename the test.log files to the target that created them
1611 # so that it's easy to associate test.log and target.
1612 new_paths = []
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001613 for label, files in test_logs:
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001614 attempt = 0
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001615 if len(files) > 1:
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001616 attempt = 1
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001617 for test_log in files:
Jakob Buchgraber070ef5f2018-02-20 17:57:14 +01001618 try:
1619 new_path = test_label_to_path(tmpdir, label, attempt)
Jakob Buchgraber070ef5f2018-02-20 17:57:14 +01001620 os.makedirs(os.path.dirname(new_path), exist_ok=True)
Jakob Buchgraber070ef5f2018-02-20 17:57:14 +01001621 copyfile(test_log, new_path)
Jakob Buchgraber070ef5f2018-02-20 17:57:14 +01001622 new_paths.append(new_path)
Philipp Wollermannc030f2e2018-02-21 17:02:19 +01001623 attempt += 1
Jakob Buchgraber070ef5f2018-02-20 17:57:14 +01001624 except IOError as err:
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01001625 # Log error and ignore.
1626 eprint(err)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001627 return new_paths
Jakob Buchgraber699aece2018-02-19 12:49:30 +01001628
1629
Jakob Buchgraber02e07222018-02-19 15:05:56 +01001630def test_label_to_path(tmpdir, label, attempt):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001631 # remove leading //
1632 path = label[2:]
Philipp Wollermannc030f2e2018-02-21 17:02:19 +01001633 path = path.replace("/", os.sep)
1634 path = path.replace(":", os.sep)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001635 if attempt == 0:
1636 path = os.path.join(path, "test.log")
1637 else:
1638 path = os.path.join(path, "attempt_" + str(attempt) + ".log")
1639 return os.path.join(tmpdir, path)
Jakob Buchgraber699aece2018-02-19 12:49:30 +01001640
1641
Jakob Buchgraber02e07222018-02-19 15:05:56 +01001642def test_logs_for_status(bep_file, status):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001643 targets = []
Jakob Buchgraber94d5c212018-02-22 09:57:08 +01001644 with open(bep_file, encoding="utf-8") as f:
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001645 raw_data = f.read()
1646 decoder = json.JSONDecoder()
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001647
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001648 pos = 0
1649 while pos < len(raw_data):
Jakob Buchgraber0aabefc2019-07-16 16:18:36 +02001650 try:
Philipp Wollermann5b00a702019-07-18 11:21:38 +02001651 bep_obj, size = decoder.raw_decode(raw_data[pos:])
Jakob Buchgraber0aabefc2019-07-16 16:18:36 +02001652 except ValueError as e:
Philipp Wollermannce986af2019-07-18 14:46:05 +02001653 eprint("JSON decoding error: " + str(e))
Philipp Wollermann5b00a702019-07-18 11:21:38 +02001654 return targets
Jakob Buchgraber45e38632018-02-19 17:27:08 +01001655 if "testSummary" in bep_obj:
1656 test_target = bep_obj["id"]["testSummary"]["label"]
1657 test_status = bep_obj["testSummary"]["overallStatus"]
Jakob Buchgraberc874cdf2019-07-16 16:27:41 +02001658 if test_status in status:
Jakob Buchgraber45e38632018-02-19 17:27:08 +01001659 outputs = bep_obj["testSummary"]["failed"]
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001660 test_logs = []
1661 for output in outputs:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001662 test_logs.append(url2pathname(urlparse(output["uri"]).path))
Jakob Buchgraber45e38632018-02-19 17:27:08 +01001663 targets.append((test_target, test_logs))
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001664 pos += size + 1
1665 return targets
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001666
1667
Philipp Wollermannaf35abf2019-05-22 17:52:01 +02001668def execute_command_and_get_output(args, shell=False, fail_if_nonzero=True, print_output=True):
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01001669 eprint(" ".join(args))
Florian Weikertc8642af2019-02-03 23:58:51 +01001670 process = subprocess.run(
1671 args,
1672 shell=shell,
1673 check=fail_if_nonzero,
1674 env=os.environ,
1675 stdout=subprocess.PIPE,
Philipp Wollermannf13804b2019-02-05 21:08:30 +01001676 errors="replace",
Florian Weikertc8642af2019-02-03 23:58:51 +01001677 universal_newlines=True,
1678 )
Florian Weikert736d06e2019-05-08 13:16:42 +02001679 if print_output:
1680 eprint(process.stdout)
1681
Florian Weikertc8642af2019-02-03 23:58:51 +01001682 return process.stdout
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001683
1684
joeleba76887952019-05-16 15:22:17 +02001685def execute_command(args, shell=False, fail_if_nonzero=True, cwd=None):
Philipp Wollermannf13804b2019-02-05 21:08:30 +01001686 eprint(" ".join(args))
Philipp Wollermann92cf51e2019-05-16 15:31:11 +02001687 return subprocess.run(
1688 args, shell=shell, check=fail_if_nonzero, env=os.environ, cwd=cwd
1689 ).returncode
Philipp Wollermannf13804b2019-02-05 21:08:30 +01001690
1691
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02001692def execute_command_background(args):
1693 eprint(" ".join(args))
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02001694 return subprocess.Popen(args, env=os.environ)
1695
Philipp Wollermann3c8b8512019-07-16 15:28:03 +02001696
Jakob Buchgraber2b63a7f2019-07-16 15:27:34 +02001697def terminate_background_process(process):
1698 if process:
1699 process.terminate()
1700 try:
1701 process.wait(timeout=10)
1702 except subprocess.TimeoutExpired:
1703 process.kill()
Philipp Wollermanne1318eb2018-08-13 15:08:01 +02001704
Philipp Wollermann3c8b8512019-07-16 15:28:03 +02001705
Philipp Wollermann7a185322019-05-18 22:15:48 +02001706def create_step(label, commands, platform, shards=1):
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01001707 if "docker-image" in PLATFORMS[platform]:
Florian Weikert736d06e2019-05-08 13:16:42 +02001708 step = create_docker_step(
Philipp Wollermannc05ac682019-01-19 12:37:28 +01001709 label, image=PLATFORMS[platform]["docker-image"], commands=commands
1710 )
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01001711 else:
Philipp Wollermannf3750fa2019-05-21 17:11:59 +02001712 step = {
1713 "label": label,
1714 "command": commands,
1715 "agents": {"queue": PLATFORMS[platform]["queue"]},
1716 }
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01001717
Florian Weikert736d06e2019-05-08 13:16:42 +02001718 if shards > 1:
1719 step["label"] += " (shard %n)"
1720 step["parallelism"] = shards
1721
Philipp Wollermann5b2f3fc2019-05-18 22:36:17 +02001722 # Enforce a global 8 hour job timeout.
1723 step["timeout_in_minutes"] = 8 * 60
1724
1725 # Automatically retry when an agent got lost (usually due to an infra flake).
Philipp Wollermannf22bba32019-07-18 11:22:50 +02001726 step["retry"] = {
1727 "automatic": [
1728 {"exit_status": -1, "limit": 3}, # Buildkite internal "agent lost" exit code
1729 {"exit_status": 137, "limit": 3}, # SIGKILL
1730 {"exit_status": 143, "limit": 3}, # SIGTERM
1731 ]
1732 }
Philipp Wollermann5b2f3fc2019-05-18 22:36:17 +02001733
Florian Weikert736d06e2019-05-08 13:16:42 +02001734 return step
1735
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01001736
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001737def create_docker_step(label, image, commands=None, additional_env_vars=None):
Philipp Wollermann0e051dd2019-05-16 11:37:52 +02001738 env = ["ANDROID_HOME", "ANDROID_NDK_HOME", "BUILDKITE_ARTIFACT_UPLOAD_DESTINATION"]
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001739 if additional_env_vars:
1740 env += ["{}={}".format(k, v) for k, v in additional_env_vars.items()]
1741
Philipp Wollermannc05ac682019-01-19 12:37:28 +01001742 step = {
Florian Weikertf20ae6f2019-01-16 14:32:09 +01001743 "label": label,
1744 "command": commands,
Philipp Wollermannb2fa2f62019-05-18 23:33:23 +02001745 "agents": {"queue": "default"},
Florian Weikertf20ae6f2019-01-16 14:32:09 +01001746 "plugins": {
Philipp Wollermannea128282019-05-08 11:56:14 +02001747 "docker#v3.2.0": {
Florian Weikertf20ae6f2019-01-16 14:32:09 +01001748 "always-pull": True,
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001749 "environment": env,
Philipp Wollermannc05ac682019-01-19 12:37:28 +01001750 "image": image,
Florian Weikertf20ae6f2019-01-16 14:32:09 +01001751 "network": "host",
1752 "privileged": True,
1753 "propagate-environment": True,
Philipp Wollermann7aa95492019-05-18 22:03:24 +02001754 "propagate-uid-gid": True,
Florian Weikertf20ae6f2019-01-16 14:32:09 +01001755 "volumes": [
Philipp Wollermann7aa95492019-05-18 22:03:24 +02001756 "/etc/group:/etc/group:ro",
1757 "/etc/passwd:/etc/passwd:ro",
Philipp Wollermann338db4a2019-05-18 11:21:04 +02001758 "/opt:/opt:ro",
Philipp Wollermann7aa95492019-05-18 22:03:24 +02001759 "/var/lib/buildkite-agent:/var/lib/buildkite-agent",
Philipp Wollermann338db4a2019-05-18 11:21:04 +02001760 "/var/lib/gitmirrors:/var/lib/gitmirrors:ro",
1761 "/var/run/docker.sock:/var/run/docker.sock",
Florian Weikertf20ae6f2019-01-16 14:32:09 +01001762 ],
Florian Weikertf20ae6f2019-01-16 14:32:09 +01001763 }
1764 },
1765 }
Philipp Wollermannc05ac682019-01-19 12:37:28 +01001766 if not step["command"]:
1767 del step["command"]
1768 return step
Florian Weikertf20ae6f2019-01-16 14:32:09 +01001769
1770
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001771def print_project_pipeline(
Florian Weikertf20ae6f2019-01-16 14:32:09 +01001772 configs,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001773 project_name,
1774 http_config,
1775 file_config,
1776 git_repository,
1777 monitor_flaky_tests,
1778 use_but,
1779 incompatible_flags,
1780):
Florian Weikert843d7a02019-02-03 17:24:50 +01001781 task_configs = configs.get("tasks", None)
1782 if not task_configs:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02001783 raise BuildkiteException("{0} pipeline configuration is empty.".format(project_name))
1784
Jakob Buchgraberaa2af382018-02-21 19:56:54 +01001785 pipeline_steps = []
Florian Weikert5f5d3cb2019-04-15 15:36:27 +02001786 task_configs = filter_tasks_that_should_be_skipped(task_configs, pipeline_steps)
Jakob Buchgraberff2bdad2018-02-25 13:06:30 +01001787
Philipp Wollermanndac65512019-02-05 22:14:10 +01001788 # In Bazel Downstream Project pipelines, git_repository and project_name must be specified.
1789 is_downstream_project = (use_but or incompatible_flags) and git_repository and project_name
1790
Florian Weikert85208912019-03-07 17:08:39 +01001791 buildifier_config = configs.get("buildifier")
Philipp Wollermanndac65512019-02-05 22:14:10 +01001792 # Skip Buildifier when we test downstream projects.
Florian Weikert85208912019-03-07 17:08:39 +01001793 if buildifier_config and not is_downstream_project:
1794 buildifier_env_vars = {}
1795 if isinstance(buildifier_config, str):
1796 # Simple format:
1797 # ---
1798 # buildifier: latest
1799 buildifier_env_vars[BUILDIFIER_VERSION_ENV_VAR] = buildifier_config
1800 else:
1801 # Advanced format:
1802 # ---
1803 # buildifier:
1804 # version: latest
1805 # warnings: all
1806
Philipp Wollermannce986af2019-07-18 14:46:05 +02001807 def set_env_var(config_key, env_var_name):
Florian Weikert85208912019-03-07 17:08:39 +01001808 if config_key in buildifier_config:
1809 buildifier_env_vars[env_var_name] = buildifier_config[config_key]
1810
Philipp Wollermannce986af2019-07-18 14:46:05 +02001811 set_env_var("version", BUILDIFIER_VERSION_ENV_VAR)
1812 set_env_var("warnings", BUILDIFIER_WARNINGS_ENV_VAR)
Florian Weikert85208912019-03-07 17:08:39 +01001813
1814 if not buildifier_env_vars:
1815 raise BuildkiteException(
1816 'Invalid buildifier configuration entry "{}"'.format(buildifier_config)
1817 )
1818
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001819 pipeline_steps.append(
1820 create_docker_step(
Florian Weikertde96a6f2019-03-07 14:57:50 +01001821 BUILDIFIER_STEP_NAME,
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001822 image=BUILDIFIER_DOCKER_IMAGE,
Florian Weikert85208912019-03-07 17:08:39 +01001823 additional_env_vars=buildifier_env_vars,
Florian Weikert29cb7ec2019-03-07 14:52:18 +01001824 )
1825 )
Philipp Wollermannc05ac682019-01-19 12:37:28 +01001826
Philipp Wollermanndac65512019-02-05 22:14:10 +01001827 # In Bazel Downstream Project pipelines, we should test the project at the last green commit.
Yun Peng376d2b32018-11-29 10:24:54 +01001828 git_commit = None
Philipp Wollermanndac65512019-02-05 22:14:10 +01001829 if is_downstream_project:
Florian Weikert35906542019-04-01 11:53:53 +02001830 last_green_commit_url = bazelci_last_green_commit_url(
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001831 git_repository, DOWNSTREAM_PROJECTS[project_name]["pipeline_slug"]
1832 )
Florian Weikert35906542019-04-01 11:53:53 +02001833 git_commit = get_last_green_commit(last_green_commit_url)
Philipp Wollermanndac65512019-02-05 22:14:10 +01001834
Florian Weikert854fd852019-06-04 16:44:19 +02001835 config_hashes = set()
Florian Weikert843d7a02019-02-03 17:24:50 +01001836 for task, task_config in task_configs.items():
Florian Weikert854fd852019-06-04 16:44:19 +02001837 # We override the Bazel version in downstream pipelines. This means that two tasks that
1838 # only differ in the value of their explicit "bazel" field will be identical in the
1839 # downstream pipeline, thus leading to duplicate work.
1840 # Consequently, we filter those duplicate tasks here.
1841 if is_downstream_project:
1842 h = hash_task_config(task, task_config)
1843 if h in config_hashes:
1844 continue
Florian Weikert854fd852019-06-04 16:44:19 +02001845 config_hashes.add(h)
1846
Florian Weikert736d06e2019-05-08 13:16:42 +02001847 shards = task_config.get("shards", "1")
1848 try:
1849 shards = int(shards)
1850 except ValueError:
1851 raise BuildkiteException("Task {} has invalid shard value '{}'".format(task, shards))
1852
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001853 step = runner_step(
Florian Weikert843d7a02019-02-03 17:24:50 +01001854 platform=get_platform_for_task(task, task_config),
1855 task=task,
1856 task_name=task_config.get("name"),
1857 project_name=project_name,
1858 http_config=http_config,
1859 file_config=file_config,
1860 git_repository=git_repository,
1861 git_commit=git_commit,
1862 monitor_flaky_tests=monitor_flaky_tests,
1863 use_but=use_but,
1864 incompatible_flags=incompatible_flags,
Florian Weikert736d06e2019-05-08 13:16:42 +02001865 shards=shards,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001866 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01001867 pipeline_steps.append(step)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001868
Yun Peng996efad2018-11-27 17:19:44 +01001869 pipeline_slug = os.getenv("BUILDKITE_PIPELINE_SLUG")
1870 all_downstream_pipeline_slugs = []
1871 for _, config in DOWNSTREAM_PROJECTS.items():
1872 all_downstream_pipeline_slugs.append(config["pipeline_slug"])
1873 # We don't need to update last green commit in the following cases:
Philipp Wollermannfce92bf2019-05-22 15:14:32 +02001874 # 1. This job is a GitHub pull request
1875 # 2. This job uses a custom built Bazel binary (in Bazel Downstream Projects pipeline)
1876 # 3. This job doesn't run on master branch (could be a custom build launched manually)
Philipp Wollermanne67eec42019-05-24 15:18:20 +02001877 # 4. We don't intend to run the same job in downstream with Bazel@HEAD (eg. google-bazel-presubmit)
1878 # 5. We are testing incompatible flags
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001879 if not (
1880 is_pull_request()
1881 or use_but
1882 or os.getenv("BUILDKITE_BRANCH") != "master"
1883 or pipeline_slug not in all_downstream_pipeline_slugs
1884 or incompatible_flags
1885 ):
Florian Weikertde96a6f2019-03-07 14:57:50 +01001886 # We need to call "Try Update Last Green Commit" even if there are failures,
1887 # since we don't want a failing Buildifier step to block the update of
1888 # the last green commit for this project.
1889 # try_update_last_green_commit() ensures that we don't update the commit
1890 # if any build or test steps fail.
1891 pipeline_steps.append({"wait": None, "continue_on_failure": True})
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001892 pipeline_steps.append(
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01001893 create_step(
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01001894 label="Try Update Last Green Commit",
1895 commands=[
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001896 fetch_bazelcipy_command(),
Philipp Wollermann57b32682019-05-18 22:09:27 +02001897 PLATFORMS[DEFAULT_PLATFORM]["python"]
1898 + " bazelci.py try_update_last_green_commit",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001899 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02001900 platform=DEFAULT_PLATFORM,
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01001901 )
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001902 )
Yun Peng43239b02018-11-23 13:57:34 +01001903
Florian Weikert778251c2019-04-25 15:14:44 +02001904 if "validate_config" in configs:
Florian Weikertf52f91a2019-05-08 15:19:30 +02001905 pipeline_steps += create_config_validation_steps()
Florian Weikert778251c2019-04-25 15:14:44 +02001906
Florian Weikertd79dc502019-05-13 09:51:05 +02001907 print_pipeline_steps(pipeline_steps, handle_emergencies=not is_downstream_project)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01001908
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01001909
Florian Weikert854fd852019-06-04 16:44:19 +02001910def hash_task_config(task_name, task_config):
1911 # Two task configs c1 and c2 have the same hash iff they lead to two functionally identical jobs
1912 # in the downstream pipeline. This function discards the "bazel" field (since it's being
Philipp Wollermannce986af2019-07-18 14:46:05 +02001913 # overridden) and the "name" field (since it has no effect on the actual work).
Florian Weikert854fd852019-06-04 16:44:19 +02001914 # Moreover, it adds an explicit "platform" field if that's missing.
1915 cpy = task_config.copy()
1916 cpy.pop("bazel", None)
1917 cpy.pop("name", None)
1918 if "platform" not in cpy:
1919 cpy["platform"] = task_name
1920
1921 m = hashlib.md5()
1922 for key in sorted(cpy):
Florian Weikert8186c392019-06-05 12:53:39 +02001923 value = "%s:%s;" % (key, cpy[key])
1924 m.update(value.encode("utf-8"))
Florian Weikert854fd852019-06-04 16:44:19 +02001925
1926 return m.digest()
1927
1928
Florian Weikert843d7a02019-02-03 17:24:50 +01001929def get_platform_for_task(task, task_config):
1930 # Most pipeline configurations have exactly one task per platform, which makes it
1931 # convenient to use the platform name as task ID. Consequently, we use the
1932 # task ID as platform if there is no explicit "platform" field.
1933 return task_config.get("platform", task)
1934
1935
Florian Weikertf52f91a2019-05-08 15:19:30 +02001936def create_config_validation_steps():
1937 output = execute_command_and_get_output(
1938 ["git", "diff-tree", "--no-commit-id", "--name-only", "-r", os.getenv("BUILDKITE_COMMIT")]
1939 )
1940 config_files = [
1941 l
1942 for l in output.split("\n")
1943 if l.startswith(".bazelci/") and os.path.splitext(l)[1] in CONFIG_FILE_EXTENSIONS
1944 ]
Florian Weikertf52f91a2019-05-08 15:19:30 +02001945 return [
1946 create_step(
1947 label=":cop: Validate {}".format(f),
1948 commands=[
1949 fetch_bazelcipy_command(),
1950 "{} bazelci.py project_pipeline --file_config={}".format(
Philipp Wollermann57b32682019-05-18 22:09:27 +02001951 PLATFORMS[DEFAULT_PLATFORM]["python"], f
Florian Weikertf52f91a2019-05-08 15:19:30 +02001952 ),
1953 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02001954 platform=DEFAULT_PLATFORM,
Florian Weikertf52f91a2019-05-08 15:19:30 +02001955 )
1956 for f in config_files
1957 ]
1958
1959
Florian Weikertd79dc502019-05-13 09:51:05 +02001960def print_pipeline_steps(pipeline_steps, handle_emergencies=True):
1961 if handle_emergencies:
1962 emergency_step = create_emergency_announcement_step_if_necessary()
1963 if emergency_step:
1964 pipeline_steps.insert(0, emergency_step)
Florian Weikert13215a82019-05-10 12:42:21 +02001965
1966 print(yaml.dump({"steps": pipeline_steps}))
1967
1968
1969def create_emergency_announcement_step_if_necessary():
1970 style = "error"
1971 message, issue_url, last_good_bazel = None, None, None
1972 try:
1973 emergency_settings = load_remote_yaml_file(EMERGENCY_FILE_URL)
1974 message = emergency_settings.get("message")
1975 issue_url = emergency_settings.get("issue_url")
1976 last_good_bazel = emergency_settings.get("last_good_bazel")
1977 except urllib.error.HTTPError as ex:
1978 message = str(ex)
1979 style = "warning"
1980
1981 if not any([message, issue_url, last_good_bazel]):
1982 return
1983
1984 text = '<span class="h1">:rotating_light: Emergency :rotating_light:</span>\n'
1985 if message:
1986 text += "- {}\n".format(message)
1987 if issue_url:
1988 text += '- Please check this <a href="{}">issue</a> for more details.\n'.format(issue_url)
1989 if last_good_bazel:
1990 text += (
1991 "- Default Bazel version is *{}*, "
1992 "unless the pipeline configuration specifies an explicit version."
1993 ).format(last_good_bazel)
1994
1995 return create_step(
1996 label=":rotating_light: Emergency :rotating_light:",
Philipp Wollermann7590b962019-05-16 11:35:03 +02001997 commands=[
1998 'buildkite-agent annotate --append --style={} --context "omg" "{}"'.format(style, text)
1999 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002000 platform=DEFAULT_PLATFORM,
Florian Weikert13215a82019-05-10 12:42:21 +02002001 )
2002
2003
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002004def runner_step(
2005 platform,
Florian Weikert843d7a02019-02-03 17:24:50 +01002006 task,
2007 task_name=None,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002008 project_name=None,
2009 http_config=None,
2010 file_config=None,
2011 git_repository=None,
2012 git_commit=None,
2013 monitor_flaky_tests=False,
2014 use_but=False,
2015 incompatible_flags=None,
Florian Weikert736d06e2019-05-08 13:16:42 +02002016 shards=1,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002017):
Philipp Wollermann57b32682019-05-18 22:09:27 +02002018 command = PLATFORMS[platform]["python"] + " bazelci.py runner --task=" + task
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002019 if http_config:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002020 command += " --http_config=" + http_config
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01002021 if file_config:
2022 command += " --file_config=" + file_config
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002023 if git_repository:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002024 command += " --git_repository=" + git_repository
Yun Peng376d2b32018-11-29 10:24:54 +01002025 if git_commit:
2026 command += " --git_commit=" + git_commit
Jakob Buchgraberc340f582018-06-22 13:48:33 +02002027 if monitor_flaky_tests:
2028 command += " --monitor_flaky_tests"
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002029 if use_but:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002030 command += " --use_but"
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002031 for flag in incompatible_flags or []:
Yun Peng4be92b32018-11-30 09:48:29 +01002032 command += " --incompatible_flag=" + flag
Florian Weikert843d7a02019-02-03 17:24:50 +01002033 label = create_label(platform, project_name, task_name=task_name)
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002034 return create_step(
Florian Weikert736d06e2019-05-08 13:16:42 +02002035 label=label, commands=[fetch_bazelcipy_command(), command], platform=platform, shards=shards
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002036 )
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002037
2038
2039def fetch_bazelcipy_command():
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002040 return "curl -sS {0} -o bazelci.py".format(SCRIPT_URL)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002041
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002042
Yun Peng002eab92018-12-17 18:28:14 +01002043def fetch_incompatible_flag_verbose_failures_command():
Philipp Wollermannfe145a52019-01-11 13:16:48 +01002044 return "curl -sS {0} -o incompatible_flag_verbose_failures.py".format(
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002045 INCOMPATIBLE_FLAG_VERBOSE_FAILURES_URL
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002046 )
Yun Peng002eab92018-12-17 18:28:14 +01002047
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002048
Yun Peng8975c6b2019-02-28 11:55:55 +01002049def fetch_aggregate_incompatible_flags_test_result_command():
2050 return "curl -sS {0} -o aggregate_incompatible_flags_test_result.py".format(
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002051 AGGREGATE_INCOMPATIBLE_TEST_RESULT_URL
Yun Peng8975c6b2019-02-28 11:55:55 +01002052 )
2053
2054
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002055def upload_project_pipeline_step(
2056 project_name, git_repository, http_config, file_config, incompatible_flags
2057):
2058 pipeline_command = (
2059 '{0} bazelci.py project_pipeline --project_name="{1}" ' + "--git_repository={2}"
Philipp Wollermann57b32682019-05-18 22:09:27 +02002060 ).format(PLATFORMS[DEFAULT_PLATFORM]["python"], project_name, git_repository)
Philipp Wollermann639c0452019-01-03 11:23:54 +01002061 if incompatible_flags is None:
Yun Peng4be92b32018-11-30 09:48:29 +01002062 pipeline_command += " --use_but"
Yun Peng95908792018-11-30 15:03:55 +01002063 else:
2064 for flag in incompatible_flags:
2065 pipeline_command += " --incompatible_flag=" + flag
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002066 if http_config:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002067 pipeline_command += " --http_config=" + http_config
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01002068 if file_config:
2069 pipeline_command += " --file_config=" + file_config
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002070 pipeline_command += " | buildkite-agent pipeline upload"
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002071
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002072 return create_step(
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002073 label="Setup {0}".format(project_name),
2074 commands=[fetch_bazelcipy_command(), pipeline_command],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002075 platform=DEFAULT_PLATFORM,
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002076 )
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002077
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002078
Florian Weikert843d7a02019-02-03 17:24:50 +01002079def create_label(platform, project_name, build_only=False, test_only=False, task_name=None):
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002080 if build_only and test_only:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002081 raise BuildkiteException("build_only and test_only cannot be true at the same time")
Florian Weikert843d7a02019-02-03 17:24:50 +01002082 platform_display_name = PLATFORMS[platform]["emoji-name"]
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002083
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002084 if build_only:
2085 label = "Build "
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002086 elif test_only:
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002087 label = "Test "
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002088 else:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002089 label = ""
2090
Florian Weikert843d7a02019-02-03 17:24:50 +01002091 platform_label = (
2092 "{0} on {1}".format(task_name, platform_display_name)
2093 if task_name
2094 else platform_display_name
2095 )
2096
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002097 if project_name:
Florian Weikert843d7a02019-02-03 17:24:50 +01002098 label += "{0} ({1})".format(project_name, platform_label)
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002099 else:
Florian Weikert843d7a02019-02-03 17:24:50 +01002100 label += platform_label
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002101
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002102 return label
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002103
2104
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002105def bazel_build_step(
Florian Weikert843d7a02019-02-03 17:24:50 +01002106 task,
2107 platform,
2108 project_name,
2109 http_config=None,
2110 file_config=None,
2111 build_only=False,
2112 test_only=False,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002113):
Philipp Wollermann57b32682019-05-18 22:09:27 +02002114 pipeline_command = PLATFORMS[platform]["python"] + " bazelci.py runner"
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002115 if build_only:
Philipp Wollermannc52e26a2019-05-18 22:10:47 +02002116 pipeline_command += " --build_only --save_but"
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002117 if test_only:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002118 pipeline_command += " --test_only"
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002119 if http_config:
Philipp Wollermannf6be4662018-02-21 14:48:28 +01002120 pipeline_command += " --http_config=" + http_config
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01002121 if file_config:
2122 pipeline_command += " --file_config=" + file_config
Florian Weikert843d7a02019-02-03 17:24:50 +01002123 pipeline_command += " --task=" + task
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002124
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002125 return create_step(
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002126 label=create_label(platform, project_name, build_only, test_only),
2127 commands=[fetch_bazelcipy_command(), pipeline_command],
2128 platform=platform,
2129 )
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002130
2131
Florian Weikert5f5d3cb2019-04-15 15:36:27 +02002132def filter_tasks_that_should_be_skipped(task_configs, pipeline_steps):
2133 skip_tasks = get_skip_tasks()
2134 if not skip_tasks:
Florian Weikert5f5d3cb2019-04-15 15:36:27 +02002135 return task_configs
2136
2137 actually_skipped = []
2138 skip_tasks = set(skip_tasks)
2139 for task in list(task_configs.keys()):
2140 if task in skip_tasks:
2141 actually_skipped.append(task)
2142 del task_configs[task]
2143 skip_tasks.remove(task)
2144
2145 if not task_configs:
2146 raise BuildkiteException(
2147 "Nothing to do since all tasks in the configuration should be skipped."
2148 )
2149
2150 annotations = []
2151 if actually_skipped:
2152 annotations.append(
2153 ("info", "Skipping the following task(s): {}".format(", ".join(actually_skipped)))
2154 )
2155
2156 if skip_tasks:
2157 annotations.append(
2158 (
2159 "warning",
2160 (
2161 "The following tasks should have been skipped, "
2162 "but were not part of the configuration: {}"
2163 ).format(", ".join(skip_tasks)),
2164 )
2165 )
2166
2167 if annotations:
2168 print_skip_task_annotations(annotations, pipeline_steps)
2169
2170 return task_configs
2171
2172
2173def get_skip_tasks():
2174 value = os.getenv(SKIP_TASKS_ENV_VAR, "")
2175 return [v for v in value.split(",") if v]
2176
2177
2178def print_skip_task_annotations(annotations, pipeline_steps):
2179 commands = [
2180 "buildkite-agent annotate --style={} '{}' --context 'ctx-{}'".format(s, t, hash(t))
2181 for s, t in annotations
2182 ]
2183 pipeline_steps.append(
Philipp Wollermann7a185322019-05-18 22:15:48 +02002184 create_step(
2185 label=":pipeline: Print information about skipped tasks",
2186 commands=commands,
2187 platform=DEFAULT_PLATFORM,
2188 )
Florian Weikert5f5d3cb2019-04-15 15:36:27 +02002189 )
2190
2191
Florian Weikert843d7a02019-02-03 17:24:50 +01002192def print_bazel_publish_binaries_pipeline(task_configs, http_config, file_config):
2193 if not task_configs:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002194 raise BuildkiteException("Bazel publish binaries pipeline configuration is empty.")
2195
Florian Weikert5f5d3cb2019-04-15 15:36:27 +02002196 pipeline_steps = []
2197 task_configs = filter_tasks_that_should_be_skipped(task_configs, pipeline_steps)
2198
Florian Weikert843d7a02019-02-03 17:24:50 +01002199 platforms = [get_platform_for_task(t, tc) for t, tc in task_configs.items()]
Philipp Wollermann783d1672019-06-06 13:35:30 +02002200
2201 # These are the platforms that the bazel_publish_binaries.yml config is actually building.
2202 configured_platforms = set(filter(should_publish_binaries_for_platform, platforms))
Philipp Wollermanna2ea5d82018-08-27 14:12:10 +02002203
Philipp Wollermann783d1672019-06-06 13:35:30 +02002204 # These are the platforms that we want to build and publish according to this script.
2205 expected_platforms = set(filter(should_publish_binaries_for_platform, PLATFORMS))
Florian Weikert843d7a02019-02-03 17:24:50 +01002206
Philipp Wollermannf64bf942019-06-06 14:34:41 +02002207 if not expected_platforms.issubset(configured_platforms):
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002208 raise BuildkiteException(
2209 "Bazel publish binaries pipeline needs to build Bazel for every commit on all publish_binary-enabled platforms."
2210 )
Jakob Buchgraber08e8e402018-03-20 19:22:07 +01002211
Yun Pengd352b6d2018-10-17 13:28:39 +02002212 # Build Bazel
Florian Weikert843d7a02019-02-03 17:24:50 +01002213 for task, task_config in task_configs.items():
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002214 pipeline_steps.append(
Florian Weikert843d7a02019-02-03 17:24:50 +01002215 bazel_build_step(
2216 task,
2217 get_platform_for_task(task, task_config),
2218 "Bazel",
2219 http_config,
2220 file_config,
2221 build_only=True,
2222 )
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002223 )
Jakob Buchgraber4631a032018-03-22 17:12:46 +01002224
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002225 pipeline_steps.append("wait")
Jakob Buchgraber9d6ca8a2018-03-22 17:30:09 +01002226
Yun Pengc2dd6522018-10-17 12:58:35 +02002227 # If all builds succeed, publish the Bazel binaries to GCS.
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002228 pipeline_steps.append(
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002229 create_step(
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002230 label="Publish Bazel Binaries",
Philipp Wollermann57b32682019-05-18 22:09:27 +02002231 commands=[
2232 fetch_bazelcipy_command(),
2233 PLATFORMS[DEFAULT_PLATFORM]["python"] + " bazelci.py publish_binaries",
2234 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002235 platform=DEFAULT_PLATFORM,
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002236 )
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002237 )
Jakob Buchgraber08e8e402018-03-20 19:22:07 +01002238
Florian Weikert13215a82019-05-10 12:42:21 +02002239 print_pipeline_steps(pipeline_steps)
Jakob Buchgraber08e8e402018-03-20 19:22:07 +01002240
2241
Florian Weikert843d7a02019-02-03 17:24:50 +01002242def should_publish_binaries_for_platform(platform):
2243 if platform not in PLATFORMS:
2244 raise BuildkiteException("Unknown platform '{}'".format(platform))
2245
2246 return PLATFORMS[platform]["publish_binary"]
2247
2248
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01002249def print_disabled_projects_info_box_step():
2250 info_text = ["Downstream testing is disabled for the following projects :sadpanda:"]
2251 for project, config in DOWNSTREAM_PROJECTS.items():
2252 disabled_reason = config.get("disabled_reason", None)
2253 if disabled_reason:
2254 info_text.append("* **%s**: %s" % (project, disabled_reason))
2255
2256 if len(info_text) == 1:
2257 return None
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002258 return create_step(
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002259 label=":sadpanda:",
2260 commands=[
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002261 'buildkite-agent annotate --append --style=info "\n' + "\n".join(info_text) + '\n"'
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01002262 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002263 platform=DEFAULT_PLATFORM,
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002264 )
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01002265
Yun Peng6528e652019-01-02 14:41:07 +01002266
2267def print_incompatible_flags_info_box_step(incompatible_flags_map):
2268 info_text = ["Build and test with the following incompatible flags:"]
2269
2270 for flag in incompatible_flags_map:
2271 info_text.append("* **%s**: %s" % (flag, incompatible_flags_map[flag]))
2272
2273 if len(info_text) == 1:
2274 return None
Philipp Wollermannc43d3cf2019-01-10 13:24:15 +01002275 return create_step(
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002276 label="Incompatible flags info",
2277 commands=[
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002278 'buildkite-agent annotate --append --style=info "\n' + "\n".join(info_text) + '\n"'
Yun Peng6528e652019-01-02 14:41:07 +01002279 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002280 platform=DEFAULT_PLATFORM,
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002281 )
Yun Peng6528e652019-01-02 14:41:07 +01002282
2283
Yun Peng7d302f62019-01-10 16:56:15 +01002284def fetch_incompatible_flags():
Yun Peng6528e652019-01-02 14:41:07 +01002285 """
2286 Return a list of incompatible flags to be tested in downstream with the current release Bazel
2287 """
Yun Peng7d302f62019-01-10 16:56:15 +01002288 incompatible_flags = {}
2289
2290 # If INCOMPATIBLE_FLAGS environment variable is set, we get incompatible flags from it.
2291 if "INCOMPATIBLE_FLAGS" in os.environ:
2292 for flag in os.environ["INCOMPATIBLE_FLAGS"].split():
2293 # We are not able to get the github link for this flag from INCOMPATIBLE_FLAGS,
2294 # so just assign the url to empty string.
2295 incompatible_flags[flag] = ""
2296 return incompatible_flags
2297
Yun Peng6528e652019-01-02 14:41:07 +01002298 # Get bazel major version on CI, eg. 0.21 from "Build label: 0.21.0\n..."
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002299 output = subprocess.check_output(
2300 ["bazel", "--nomaster_bazelrc", "--bazelrc=/dev/null", "version"]
2301 ).decode("utf-8")
Yun Peng6528e652019-01-02 14:41:07 +01002302 bazel_major_version = output.split()[2].rsplit(".", 1)[0]
2303
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002304 output = subprocess.check_output(
2305 [
2306 "curl",
Philipp Wollermann6f4058f2019-08-21 13:58:50 +02002307 "https://api.github.com/search/issues?per_page=100&q=repo:bazelbuild/bazel+label:migration-%s"
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002308 % bazel_major_version,
2309 ]
2310 ).decode("utf-8")
Yun Peng6528e652019-01-02 14:41:07 +01002311 issue_info = json.loads(output)
2312
Yun Peng6528e652019-01-02 14:41:07 +01002313 for issue in issue_info["items"]:
Yun Peng6528e652019-01-02 14:41:07 +01002314 # Every incompatible flags issue should start with "<incompatible flag name (without --)>:"
2315 name = "--" + issue["title"].split(":")[0]
2316 url = issue["html_url"]
2317 if name.startswith("--incompatible_"):
2318 incompatible_flags[name] = url
2319 else:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002320 eprint(
Philipp Wollermann639c0452019-01-03 11:23:54 +01002321 f"{name} is not recognized as an incompatible flag, please modify the issue title "
2322 f'of {url} to "<incompatible flag name (without --)>:..."'
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002323 )
Yun Peng6528e652019-01-02 14:41:07 +01002324
2325 return incompatible_flags
2326
2327
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002328def print_bazel_downstream_pipeline(
Florian Weikert843d7a02019-02-03 17:24:50 +01002329 task_configs, http_config, file_config, test_incompatible_flags, test_disabled_projects
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002330):
Florian Weikert843d7a02019-02-03 17:24:50 +01002331 if not task_configs:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002332 raise BuildkiteException("Bazel downstream pipeline configuration is empty.")
2333
Florian Weikert5f5d3cb2019-04-15 15:36:27 +02002334 pipeline_steps = []
2335 task_configs = filter_tasks_that_should_be_skipped(task_configs, pipeline_steps)
2336
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002337 pipeline_steps = []
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002338
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01002339 info_box_step = print_disabled_projects_info_box_step()
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01002340 if info_box_step is not None:
2341 pipeline_steps.append(info_box_step)
2342
Yun Peng5599ca22019-01-16 12:32:41 +01002343 if not test_incompatible_flags:
Florian Weikert843d7a02019-02-03 17:24:50 +01002344 for task, task_config in task_configs.items():
Yun Peng5599ca22019-01-16 12:32:41 +01002345 pipeline_steps.append(
Florian Weikert843d7a02019-02-03 17:24:50 +01002346 bazel_build_step(
2347 task,
2348 get_platform_for_task(task, task_config),
2349 "Bazel",
2350 http_config,
2351 file_config,
2352 build_only=True,
2353 )
Yun Peng5599ca22019-01-16 12:32:41 +01002354 )
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002355
Yun Peng5599ca22019-01-16 12:32:41 +01002356 pipeline_steps.append("wait")
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002357
Yun Pengb9998d12018-12-03 10:18:28 +01002358 incompatible_flags = None
Yun Peng7a539ef2018-11-30 15:07:24 +01002359 if test_incompatible_flags:
Yun Peng7d302f62019-01-10 16:56:15 +01002360 incompatible_flags_map = fetch_incompatible_flags()
Yun Peng6528e652019-01-02 14:41:07 +01002361 info_box_step = print_incompatible_flags_info_box_step(incompatible_flags_map)
2362 if info_box_step is not None:
2363 pipeline_steps.append(info_box_step)
2364 incompatible_flags = list(incompatible_flags_map.keys())
Yun Peng7a539ef2018-11-30 15:07:24 +01002365
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002366 for project, config in DOWNSTREAM_PROJECTS.items():
Yun Peng996efad2018-11-27 17:19:44 +01002367 disabled_reason = config.get("disabled_reason", None)
Yun Pengfb759fa2018-12-13 11:35:39 +01002368 # If test_disabled_projects is true, we add configs for disabled projects.
Florian Weikert7b3f17e2019-03-14 13:52:42 +01002369 # If test_disabled_projects is false, we add configs for not disabled projects.
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002370 if (test_disabled_projects and disabled_reason) or (
2371 not test_disabled_projects and not disabled_reason
2372 ):
Yun Peng996efad2018-11-27 17:19:44 +01002373 pipeline_steps.append(
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002374 upload_project_pipeline_step(
2375 project_name=project,
2376 git_repository=config["git_repository"],
2377 http_config=config.get("http_config", None),
2378 file_config=config.get("file_config", None),
2379 incompatible_flags=incompatible_flags,
2380 )
2381 )
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002382
Yun Peng002eab92018-12-17 18:28:14 +01002383 if test_incompatible_flags:
Yun Peng002eab92018-12-17 18:28:14 +01002384 current_build_number = os.environ.get("BUILDKITE_BUILD_NUMBER", None)
2385 if not current_build_number:
2386 raise BuildkiteException("Not running inside Buildkite")
Yun Peng8975c6b2019-02-28 11:55:55 +01002387 if use_bazelisk_migrate():
2388 pipeline_steps.append({"wait": "~", "continue_on_failure": "true"})
2389 pipeline_steps.append(
2390 create_step(
2391 label="Aggregate incompatible flags test result",
2392 commands=[
2393 fetch_bazelcipy_command(),
2394 fetch_aggregate_incompatible_flags_test_result_command(),
Philipp Wollermann57b32682019-05-18 22:09:27 +02002395 PLATFORMS[DEFAULT_PLATFORM]["python"]
Yun Penge1679032019-02-28 17:04:08 +01002396 + " aggregate_incompatible_flags_test_result.py --build_number=%s"
Yun Peng8975c6b2019-02-28 11:55:55 +01002397 % current_build_number,
2398 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002399 platform=DEFAULT_PLATFORM,
Yun Peng8975c6b2019-02-28 11:55:55 +01002400 )
Philipp Wollermann1403d2c2019-01-10 13:15:51 +01002401 )
Yun Peng8975c6b2019-02-28 11:55:55 +01002402 else:
2403 pipeline_steps.append({"wait": "~", "continue_on_failure": "true"})
2404 pipeline_steps.append(
2405 create_step(
2406 label="Test failing jobs with incompatible flag separately",
2407 commands=[
2408 fetch_bazelcipy_command(),
2409 fetch_incompatible_flag_verbose_failures_command(),
Philipp Wollermann57b32682019-05-18 22:09:27 +02002410 PLATFORMS[DEFAULT_PLATFORM]["python"]
Yun Peng8975c6b2019-02-28 11:55:55 +01002411 + " incompatible_flag_verbose_failures.py --build_number=%s | buildkite-agent pipeline upload"
2412 % current_build_number,
2413 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002414 platform=DEFAULT_PLATFORM,
Yun Peng8975c6b2019-02-28 11:55:55 +01002415 )
2416 )
Yun Peng002eab92018-12-17 18:28:14 +01002417
Florian Weikert2896edb2019-04-04 16:12:47 +02002418 if (
2419 not test_disabled_projects
2420 and not test_incompatible_flags
2421 and os.getenv("BUILDKITE_BRANCH") == "master"
2422 ):
Florian Weikert35906542019-04-01 11:53:53 +02002423 # Only update the last green downstream commit in the regular Bazel@HEAD + Downstream pipeline.
2424 pipeline_steps.append("wait")
2425 pipeline_steps.append(
2426 create_step(
2427 label="Try Update Last Green Downstream Commit",
2428 commands=[
2429 fetch_bazelcipy_command(),
Philipp Wollermann57b32682019-05-18 22:09:27 +02002430 PLATFORMS[DEFAULT_PLATFORM]["python"]
2431 + " bazelci.py try_update_last_green_downstream_commit",
Florian Weikert35906542019-04-01 11:53:53 +02002432 ],
Philipp Wollermann7a185322019-05-18 22:15:48 +02002433 platform=DEFAULT_PLATFORM,
Florian Weikert35906542019-04-01 11:53:53 +02002434 )
2435 )
2436
Florian Weikert13215a82019-05-10 12:42:21 +02002437 print_pipeline_steps(pipeline_steps)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002438
2439
Yun Pengc2dd6522018-10-17 12:58:35 +02002440def bazelci_builds_download_url(platform, git_commit):
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002441 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-builds"
2442 return "https://storage.googleapis.com/{}/artifacts/{}/{}/bazel".format(
2443 bucket_name, platform, git_commit
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002444 )
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002445
2446
Yun Peng20d45602018-10-18 13:27:05 +02002447def bazelci_builds_gs_url(platform, git_commit):
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002448 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-builds"
2449 return "gs://{}/artifacts/{}/{}/bazel".format(bucket_name, platform, git_commit)
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002450
2451
Jakob Buchgraber76381e02018-02-19 16:19:56 +01002452def bazelci_builds_metadata_url():
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002453 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-builds"
2454 return "gs://{}/metadata/latest.json".format(bucket_name)
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002455
2456
Yun Peng996efad2018-11-27 17:19:44 +01002457def bazelci_last_green_commit_url(git_repository, pipeline_slug):
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002458 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-untrusted-builds"
2459 return "gs://{}/last_green_commit/{}/{}".format(
2460 bucket_name, git_repository[len("https://") :], pipeline_slug
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002461 )
Yun Pengafe67d42018-11-23 17:06:43 +01002462
2463
Florian Weikert35906542019-04-01 11:53:53 +02002464def bazelci_last_green_downstream_commit_url():
Philipp Wollermanne67eec42019-05-24 15:18:20 +02002465 bucket_name = "bazel-testing-builds" if THIS_IS_TESTING else "bazel-untrusted-builds"
2466 return "gs://{}/last_green_commit/downstream_pipeline".format(bucket_name)
Florian Weikert35906542019-04-01 11:53:53 +02002467
2468
2469def get_last_green_commit(last_green_commit_url):
Yun Peng61a448f2018-11-23 17:11:46 +01002470 try:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002471 return (
2472 subprocess.check_output(
2473 [gsutil_command(), "cat", last_green_commit_url], env=os.environ
2474 )
2475 .decode("utf-8")
2476 .strip()
2477 )
Yun Peng61a448f2018-11-23 17:11:46 +01002478 except subprocess.CalledProcessError:
2479 return None
Yun Peng43239b02018-11-23 13:57:34 +01002480
2481
Yun Peng358cd882018-11-29 10:25:18 +01002482def try_update_last_green_commit():
Florian Weikertde96a6f2019-03-07 14:57:50 +01002483 org_slug = os.getenv("BUILDKITE_ORGANIZATION_SLUG")
Yun Peng358cd882018-11-29 10:25:18 +01002484 pipeline_slug = os.getenv("BUILDKITE_PIPELINE_SLUG")
Florian Weikertde96a6f2019-03-07 14:57:50 +01002485 build_number = os.getenv("BUILDKITE_BUILD_NUMBER")
2486 current_job_id = os.getenv("BUILDKITE_JOB_ID")
2487
2488 client = BuildkiteClient(org=org_slug, pipeline=pipeline_slug)
2489 build_info = client.get_build_info(build_number)
2490
2491 # Find any failing steps other than Buildifier and "try update last green".
Philipp Wollermannce986af2019-07-18 14:46:05 +02002492 def has_failed(job):
Florian Weikertbd40a272019-03-08 10:20:18 +01002493 state = job.get("state")
2494 # Ignore steps that don't have a state (like "wait").
Florian Weikertde96a6f2019-03-07 14:57:50 +01002495 return (
Florian Weikert35906542019-04-01 11:53:53 +02002496 state is not None
2497 and state != "passed"
Florian Weikertde96a6f2019-03-07 14:57:50 +01002498 and job["id"] != current_job_id
2499 and job["name"] != BUILDIFIER_STEP_NAME
2500 )
2501
Philipp Wollermannce986af2019-07-18 14:46:05 +02002502 failing_jobs = [j["name"] for j in build_info["jobs"] if has_failed(j)]
Florian Weikertde96a6f2019-03-07 14:57:50 +01002503 if failing_jobs:
2504 raise BuildkiteException(
2505 "Cannot update last green commit due to {} failing step(s): {}".format(
2506 len(failing_jobs), ", ".join(failing_jobs)
2507 )
2508 )
2509
Yun Peng358cd882018-11-29 10:25:18 +01002510 git_repository = os.getenv("BUILDKITE_REPO")
Florian Weikert35906542019-04-01 11:53:53 +02002511 last_green_commit_url = bazelci_last_green_commit_url(git_repository, pipeline_slug)
2512 update_last_green_commit_if_newer(last_green_commit_url)
2513
2514
2515def update_last_green_commit_if_newer(last_green_commit_url):
2516 last_green_commit = get_last_green_commit(last_green_commit_url)
Yun Peng358cd882018-11-29 10:25:18 +01002517 current_commit = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("utf-8").strip()
2518 if last_green_commit:
Yun Peng384058a2019-01-15 13:26:35 +01002519 execute_command(["git", "fetch", "-v", "origin", last_green_commit])
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002520 result = (
2521 subprocess.check_output(
2522 ["git", "rev-list", "%s..%s" % (last_green_commit, current_commit)]
2523 )
2524 .decode("utf-8")
2525 .strip()
2526 )
Philipp Wollermannce986af2019-07-18 14:46:05 +02002527 else:
2528 result = None
Yun Peng358cd882018-11-29 10:25:18 +01002529
Philipp Wollermann639c0452019-01-03 11:23:54 +01002530 # If current_commit is newer that last_green_commit, `git rev-list A..B` will output a bunch of
2531 # commits, otherwise the output should be empty.
Yun Peng358cd882018-11-29 10:25:18 +01002532 if not last_green_commit or result:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002533 execute_command(
Florian Weikert35906542019-04-01 11:53:53 +02002534 ["echo %s | %s cp - %s" % (current_commit, gsutil_command(), last_green_commit_url)],
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002535 shell=True,
2536 )
Yun Peng358cd882018-11-29 10:25:18 +01002537 else:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002538 eprint(
2539 "Updating abandoned: last green commit (%s) is not older than current commit (%s)."
2540 % (last_green_commit, current_commit)
2541 )
2542
Yun Peng358cd882018-11-29 10:25:18 +01002543
Florian Weikert35906542019-04-01 11:53:53 +02002544def try_update_last_green_downstream_commit():
2545 last_green_commit_url = bazelci_last_green_downstream_commit_url()
2546 update_last_green_commit_if_newer(last_green_commit_url)
2547
2548
Jakob Buchgraber76381e02018-02-19 16:19:56 +01002549def latest_generation_and_build_number():
Philipp Wollermannce986af2019-07-18 14:46:05 +02002550 generation = None
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002551 output = None
Philipp Wollermannce986af2019-07-18 14:46:05 +02002552 for attempt in range(5):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002553 output = subprocess.check_output(
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002554 [gsutil_command(), "stat", bazelci_builds_metadata_url()], env=os.environ
2555 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002556 match = re.search("Generation:[ ]*([0-9]+)", output.decode("utf-8"))
2557 if not match:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002558 raise BuildkiteException("Couldn't parse generation. gsutil output format changed?")
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002559 generation = match.group(1)
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002560
Philipp Wollermannff39ef52018-02-21 14:18:52 +01002561 match = re.search(r"Hash \(md5\):[ ]*([^\s]+)", output.decode("utf-8"))
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002562 if not match:
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002563 raise BuildkiteException("Couldn't parse md5 hash. gsutil output format changed?")
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002564 expected_md5hash = base64.b64decode(match.group(1))
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002565
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002566 output = subprocess.check_output(
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002567 [gsutil_command(), "cat", bazelci_builds_metadata_url()], env=os.environ
2568 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002569 hasher = hashlib.md5()
2570 hasher.update(output)
2571 actual_md5hash = hasher.digest()
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002572
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002573 if expected_md5hash == actual_md5hash:
2574 break
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002575 info = json.loads(output.decode("utf-8"))
Philipp Wollermannce986af2019-07-18 14:46:05 +02002576 return generation, info["build_number"]
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002577
Jakob Buchgraber699aece2018-02-19 12:49:30 +01002578
Jakob Buchgraber88083fd2018-02-18 17:23:35 +01002579def sha256_hexdigest(filename):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002580 sha256 = hashlib.sha256()
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002581 with open(filename, "rb") as f:
2582 for block in iter(lambda: f.read(65536), b""):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002583 sha256.update(block)
2584 return sha256.hexdigest()
Jakob Buchgraber699aece2018-02-19 12:49:30 +01002585
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002586
Philipp Wollermann02955272019-04-18 18:00:48 +02002587def upload_bazel_binaries():
2588 """
2589 Uploads all Bazel binaries to a deterministic URL based on the current Git commit.
2590
2591 Returns a map of platform names to sha256 hashes of the corresponding Bazel binary.
2592 """
2593 hashes = {}
Philipp Wollermannbdd4bf92019-06-06 14:43:50 +02002594 for platform_name, platform in PLATFORMS.items():
Philipp Wollermann783d1672019-06-06 13:35:30 +02002595 if not should_publish_binaries_for_platform(platform_name):
2596 continue
Jakob Buchgraberb13a9a82018-03-27 18:37:09 +02002597 tmpdir = tempfile.mkdtemp()
2598 try:
Philipp Wollermann783d1672019-06-06 13:35:30 +02002599 bazel_binary_path = download_bazel_binary(tmpdir, platform_name)
2600 # One platform that we build on can generate binaries for multiple platforms, e.g.
Philipp Wollermannf4aabb72019-06-25 15:59:00 +02002601 # the centos7 platform generates binaries for the "centos7" platform, but also
Philipp Wollermann783d1672019-06-06 13:35:30 +02002602 # for the generic "linux" platform.
2603 for target_platform_name in platform["publish_binary"]:
2604 execute_command(
2605 [
2606 gsutil_command(),
2607 "cp",
2608 bazel_binary_path,
2609 bazelci_builds_gs_url(target_platform_name, os.environ["BUILDKITE_COMMIT"]),
2610 ]
2611 )
2612 hashes[target_platform_name] = sha256_hexdigest(bazel_binary_path)
Jakob Buchgraberb13a9a82018-03-27 18:37:09 +02002613 finally:
2614 shutil.rmtree(tmpdir)
Philipp Wollermann02955272019-04-18 18:00:48 +02002615 return hashes
2616
2617
2618def try_publish_binaries(hashes, build_number, expected_generation):
2619 """
2620 Uploads the info.json file that contains information about the latest Bazel commit that was
2621 successfully built on CI.
2622 """
2623 now = datetime.datetime.now()
2624 git_commit = os.environ["BUILDKITE_COMMIT"]
2625 info = {
2626 "build_number": build_number,
2627 "build_time": now.strftime("%d-%m-%Y %H:%M"),
2628 "git_commit": git_commit,
2629 "platforms": {},
2630 }
Philipp Wollermann783d1672019-06-06 13:35:30 +02002631 for platform, sha256 in hashes.items():
Philipp Wollermann02955272019-04-18 18:00:48 +02002632 info["platforms"][platform] = {
2633 "url": bazelci_builds_download_url(platform, git_commit),
Philipp Wollermann783d1672019-06-06 13:35:30 +02002634 "sha256": sha256,
Philipp Wollermann02955272019-04-18 18:00:48 +02002635 }
Jakob Buchgraberb13a9a82018-03-27 18:37:09 +02002636 tmpdir = tempfile.mkdtemp()
2637 try:
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002638 info_file = os.path.join(tmpdir, "info.json")
2639 with open(info_file, mode="w", encoding="utf-8") as fp:
Jakob Buchgraber609a20e2018-02-25 17:06:51 +01002640 json.dump(info, fp, indent=2, sort_keys=True)
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01002641
2642 try:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002643 execute_command(
2644 [
2645 gsutil_command(),
2646 "-h",
2647 "x-goog-if-generation-match:" + expected_generation,
2648 "-h",
2649 "Content-Type:application/json",
2650 "cp",
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002651 info_file,
2652 bazelci_builds_metadata_url(),
2653 ]
2654 )
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01002655 except subprocess.CalledProcessError:
2656 raise BinaryUploadRaceException()
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002657 finally:
Philipp Wollermann3e1a7712018-02-19 17:34:24 +01002658 shutil.rmtree(tmpdir)
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002659
2660
Jakob Buchgraber76381e02018-02-19 16:19:56 +01002661def publish_binaries():
Philipp Wollermanndb024862018-02-19 17:16:56 +01002662 """
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002663 Publish Bazel binaries to GCS.
Philipp Wollermanndb024862018-02-19 17:16:56 +01002664 """
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01002665 current_build_number = os.environ.get("BUILDKITE_BUILD_NUMBER", None)
2666 if not current_build_number:
2667 raise BuildkiteException("Not running inside Buildkite")
2668 current_build_number = int(current_build_number)
2669
Philipp Wollermann02955272019-04-18 18:00:48 +02002670 # Upload the Bazel binaries for this commit.
2671 hashes = upload_bazel_binaries()
2672
2673 # Try to update the info.json with data about our build. This will fail (expectedly) if we're
2674 # not the latest build.
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01002675 for _ in range(5):
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002676 latest_generation, latest_build_number = latest_generation_and_build_number()
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002677
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002678 if current_build_number <= latest_build_number:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002679 eprint(
2680 (
2681 "Current build '{0}' is not newer than latest published '{1}'. "
2682 + "Skipping publishing of binaries."
2683 ).format(current_build_number, latest_build_number)
2684 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002685 break
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002686
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01002687 try:
Philipp Wollermann02955272019-04-18 18:00:48 +02002688 try_publish_binaries(hashes, current_build_number, latest_generation)
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01002689 except BinaryUploadRaceException:
2690 # Retry.
2691 continue
2692
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002693 eprint(
2694 "Successfully updated '{0}' to binaries from build {1}.".format(
2695 bazelci_builds_metadata_url(), current_build_number
2696 )
2697 )
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01002698 break
2699 else:
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002700 raise BuildkiteException("Could not publish binaries, ran out of attempts.")
2701
Philipp Wollermann3c8b8512019-07-16 15:28:03 +02002702
Philipp Wollermann639c0452019-01-03 11:23:54 +01002703# This is so that multiline python strings are represented as YAML
2704# block strings.
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01002705def str_presenter(dumper, data):
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002706 if len(data.splitlines()) > 1: # check for multiline string
2707 return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
2708 return dumper.represent_scalar("tag:yaml.org,2002:str", data)
2709
Jakob Buchgraberd20ffeb2018-02-18 03:16:43 +01002710
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01002711def main(argv=None):
2712 if argv is None:
Yun Peng20d45602018-10-18 13:27:05 +02002713 argv = sys.argv[1:]
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01002714
Jakob Buchgraber9952a3b2018-12-06 15:38:51 +01002715 yaml.add_representer(str, str_presenter)
2716
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002717 parser = argparse.ArgumentParser(description="Bazel Continuous Integration Script")
Florian Weikert944209b2019-05-10 12:41:48 +02002718 parser.add_argument("--script", type=str)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002719
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002720 subparsers = parser.add_subparsers(dest="subparsers_name")
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01002721
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01002722 bazel_publish_binaries_pipeline = subparsers.add_parser("bazel_publish_binaries_pipeline")
2723 bazel_publish_binaries_pipeline.add_argument("--file_config", type=str)
Jakob Buchgraber08e8e402018-03-20 19:22:07 +01002724 bazel_publish_binaries_pipeline.add_argument("--http_config", type=str)
2725 bazel_publish_binaries_pipeline.add_argument("--git_repository", type=str)
2726
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01002727 bazel_downstream_pipeline = subparsers.add_parser("bazel_downstream_pipeline")
2728 bazel_downstream_pipeline.add_argument("--file_config", type=str)
2729 bazel_downstream_pipeline.add_argument("--http_config", type=str)
2730 bazel_downstream_pipeline.add_argument("--git_repository", type=str)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002731 bazel_downstream_pipeline.add_argument(
2732 "--test_incompatible_flags", type=bool, nargs="?", const=True
2733 )
2734 bazel_downstream_pipeline.add_argument(
2735 "--test_disabled_projects", type=bool, nargs="?", const=True
2736 )
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002737
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002738 project_pipeline = subparsers.add_parser("project_pipeline")
2739 project_pipeline.add_argument("--project_name", type=str)
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01002740 project_pipeline.add_argument("--file_config", type=str)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002741 project_pipeline.add_argument("--http_config", type=str)
2742 project_pipeline.add_argument("--git_repository", type=str)
Jakob Buchgraber66ba4fe2018-06-22 15:04:14 +02002743 project_pipeline.add_argument("--monitor_flaky_tests", type=bool, nargs="?", const=True)
Philipp Wollermann2409c6e2018-08-07 07:37:54 +02002744 project_pipeline.add_argument("--use_but", type=bool, nargs="?", const=True)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002745 project_pipeline.add_argument("--incompatible_flag", type=str, action="append")
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002746
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002747 runner = subparsers.add_parser("runner")
Florian Weikert843d7a02019-02-03 17:24:50 +01002748 runner.add_argument("--task", action="store", type=str, default="")
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01002749 runner.add_argument("--file_config", type=str)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002750 runner.add_argument("--http_config", type=str)
2751 runner.add_argument("--git_repository", type=str)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002752 runner.add_argument(
2753 "--git_commit", type=str, help="Reset the git repository to this commit after cloning it"
2754 )
2755 runner.add_argument(
2756 "--git_repo_location",
2757 type=str,
2758 help="Use an existing repository instead of cloning from github",
2759 )
2760 runner.add_argument(
Dan Halperinefda1192019-01-16 00:34:09 -08002761 "--use_bazel_at_commit", type=str, help="Use Bazel binary built at a specific commit"
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002762 )
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002763 runner.add_argument("--use_but", type=bool, nargs="?", const=True)
2764 runner.add_argument("--save_but", type=bool, nargs="?", const=True)
Yun Peng4d1d6542019-01-17 18:30:33 +01002765 runner.add_argument("--needs_clean", type=bool, nargs="?", const=True)
Philipp Wollermann598b4a42018-02-19 17:03:36 +01002766 runner.add_argument("--build_only", type=bool, nargs="?", const=True)
2767 runner.add_argument("--test_only", type=bool, nargs="?", const=True)
Jakob Buchgraber66ba4fe2018-06-22 15:04:14 +02002768 runner.add_argument("--monitor_flaky_tests", type=bool, nargs="?", const=True)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002769 runner.add_argument("--incompatible_flag", type=str, action="append")
Jakob Buchgraberc340f582018-06-22 13:48:33 +02002770
Philipp Wollermannce986af2019-07-18 14:46:05 +02002771 subparsers.add_parser("publish_binaries")
2772 subparsers.add_parser("try_update_last_green_commit")
2773 subparsers.add_parser("try_update_last_green_downstream_commit")
Yun Peng358cd882018-11-29 10:25:18 +01002774
Yun Peng20d45602018-10-18 13:27:05 +02002775 args = parser.parse_args(argv)
Jakob Buchgraber0b6a39d2018-02-17 00:45:14 +01002776
Florian Weikert944209b2019-05-10 12:41:48 +02002777 if args.script:
2778 global SCRIPT_URL
2779 SCRIPT_URL = args.script
2780
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01002781 try:
Jakob Buchgraber08e8e402018-03-20 19:22:07 +01002782 if args.subparsers_name == "bazel_publish_binaries_pipeline":
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01002783 configs = fetch_configs(args.http_config, args.file_config)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002784 print_bazel_publish_binaries_pipeline(
Florian Weikert843d7a02019-02-03 17:24:50 +01002785 task_configs=configs.get("tasks", None),
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002786 http_config=args.http_config,
2787 file_config=args.file_config,
2788 )
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01002789 elif args.subparsers_name == "bazel_downstream_pipeline":
2790 configs = fetch_configs(args.http_config, args.file_config)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002791 print_bazel_downstream_pipeline(
Florian Weikert843d7a02019-02-03 17:24:50 +01002792 task_configs=configs.get("tasks", None),
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002793 http_config=args.http_config,
2794 file_config=args.file_config,
2795 test_incompatible_flags=args.test_incompatible_flags,
2796 test_disabled_projects=args.test_disabled_projects,
2797 )
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01002798 elif args.subparsers_name == "project_pipeline":
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01002799 configs = fetch_configs(args.http_config, args.file_config)
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002800 print_project_pipeline(
Florian Weikertf20ae6f2019-01-16 14:32:09 +01002801 configs=configs,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002802 project_name=args.project_name,
2803 http_config=args.http_config,
2804 file_config=args.file_config,
2805 git_repository=args.git_repository,
2806 monitor_flaky_tests=args.monitor_flaky_tests,
2807 use_but=args.use_but,
2808 incompatible_flags=args.incompatible_flag,
2809 )
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01002810 elif args.subparsers_name == "runner":
Jakob Buchgraberc57d4ad2018-03-22 12:33:17 +01002811 configs = fetch_configs(args.http_config, args.file_config)
Florian Weikert843d7a02019-02-03 17:24:50 +01002812 tasks = configs.get("tasks", {})
2813 task_config = tasks.get(args.task)
2814 if not task_config:
2815 raise BuildkiteException(
2816 "No such task '{}' in configuration. Available: {}".format(
2817 args.task, ", ".join(tasks)
2818 )
2819 )
2820
2821 platform = get_platform_for_task(args.task, task_config)
2822
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002823 execute_commands(
Florian Weikertc8642af2019-02-03 23:58:51 +01002824 task_config=task_config,
Florian Weikert843d7a02019-02-03 17:24:50 +01002825 platform=platform,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002826 git_repository=args.git_repository,
2827 git_commit=args.git_commit,
2828 git_repo_location=args.git_repo_location,
2829 use_bazel_at_commit=args.use_bazel_at_commit,
2830 use_but=args.use_but,
2831 save_but=args.save_but,
Yun Peng4d1d6542019-01-17 18:30:33 +01002832 needs_clean=args.needs_clean,
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002833 build_only=args.build_only,
2834 test_only=args.test_only,
2835 monitor_flaky_tests=args.monitor_flaky_tests,
2836 incompatible_flags=args.incompatible_flag,
Florian Weikertc8642af2019-02-03 23:58:51 +01002837 bazel_version=task_config.get("bazel") or configs.get("bazel"),
Philipp Wollermanncd5694c2019-01-03 14:53:04 +01002838 )
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01002839 elif args.subparsers_name == "publish_binaries":
2840 publish_binaries()
Yun Peng358cd882018-11-29 10:25:18 +01002841 elif args.subparsers_name == "try_update_last_green_commit":
Florian Weikert35906542019-04-01 11:53:53 +02002842 # Update the last green commit of a project pipeline
Yun Peng358cd882018-11-29 10:25:18 +01002843 try_update_last_green_commit()
Florian Weikert35906542019-04-01 11:53:53 +02002844 elif args.subparsers_name == "try_update_last_green_downstream_commit":
2845 # Update the last green commit of the downstream pipeline
2846 try_update_last_green_downstream_commit()
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01002847 else:
2848 parser.print_help()
2849 return 2
2850 except BuildkiteException as e:
2851 eprint(str(e))
2852 return 1
2853 return 0
2854
Jakob Buchgraber95e3d572018-02-21 18:48:49 +01002855
Philipp Wollermanndcaddd92018-02-21 14:13:43 +01002856if __name__ == "__main__":
2857 sys.exit(main())