Add option to include JSON profile in builds (#655)
diff --git a/buildkite/bazelci.py b/buildkite/bazelci.py
index f7d3ace..2383011 100755
--- a/buildkite/bazelci.py
+++ b/buildkite/bazelci.py
@@ -764,21 +764,41 @@
if not build_targets and not test_targets:
raise BuildkiteException("There are neither build nor test targets")
+ include_json_profile = task_config.get("include_json_profile", [])
+
if build_targets:
- execute_bazel_build(
- bazel_version,
- bazel_binary,
- platform,
- task_config.get("build_flags", []),
- build_targets,
- None,
- incompatible_flags,
- )
- if save_but:
- upload_bazel_binary(platform)
+ json_profile_flags = []
+ include_json_profile_build = 'build' in include_json_profile
+ if include_json_profile_build:
+ json_profile_out_build = os.path.join(tmpdir, "build.profile")
+ json_profile_flags = get_json_profile_flags(
+ json_profile_out_build)
+
+ try:
+ execute_bazel_build(
+ bazel_version,
+ bazel_binary,
+ platform,
+ task_config.get("build_flags", []) + json_profile_flags,
+ build_targets,
+ None,
+ incompatible_flags,
+ )
+ if save_but:
+ upload_bazel_binary(platform)
+ finally:
+ if include_json_profile_build:
+ upload_json_profile(json_profile_out_build, tmpdir)
if test_targets:
- test_flags = task_config.get("test_flags", [])
+ json_profile_flags = []
+ include_json_profile_test = 'test' in include_json_profile
+ if include_json_profile_test:
+ json_profile_out_test = os.path.join(tmpdir, "test.profile")
+ json_profile_flags = get_json_profile_flags(
+ json_profile_out_test)
+
+ test_flags = task_config.get("test_flags", []) + json_profile_path
if test_env_vars:
test_flags += ["--test_env={}".format(v) for v in test_env_vars]
@@ -823,6 +843,8 @@
)
finally:
upload_test_logs(test_bep_file, tmpdir)
+ if include_json_profile_test:
+ upload_json_profile(json_profile_out_test, tmpdir)
finally:
if sc_process:
sc_process.terminate()
@@ -1379,6 +1401,15 @@
handle_bazel_failure(e, "test")
+def get_json_profile_flags(out_file):
+ return [
+ "--experimental_generate_json_trace_profile",
+ "--experimental_profile_cpu_usage",
+ "--experimental_json_trace_compression",
+ "--profile={}".format(out_file)
+ ]
+
+
def upload_test_logs(bep_file, tmpdir):
if not os.path.exists(bep_file):
return
@@ -1395,6 +1426,15 @@
os.chdir(cwd)
+def upload_json_profile(json_profile_path, tmpdir):
+ if not os.path.exists(json_profile_path):
+ return
+ print_collapsed_group(":gcloud: Uploading JSON Profile")
+ execute_command(
+ ["buildkite-agent", "artifact", "upload", json_profile_path],
+ cwd=tmpdir)
+
+
def test_logs_to_upload(bep_file, tmpdir):
failed = test_logs_for_status(bep_file, status="FAILED")
timed_out = test_logs_for_status(bep_file, status="TIMEOUT")
@@ -1474,9 +1514,9 @@
return process.stdout
-def execute_command(args, shell=False, fail_if_nonzero=True):
+def execute_command(args, shell=False, fail_if_nonzero=True, cwd=None):
eprint(" ".join(args))
- return subprocess.run(args, shell=shell, check=fail_if_nonzero, env=os.environ).returncode
+ return subprocess.run(args, shell=shell, check=fail_if_nonzero, env=os.environ, cwd=cwd).returncode
def execute_command_background(args):
