| #!/usr/bin/env python3 | |
| # | |
| # Copyright 2020 The Bazel Authors. All rights reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| import sys | |
| import threading | |
| import bazelci | |
| BUILDKITE_ORG = "bazel" | |
| DOWNSTREAM_PIPELINE = "bazel-at-head-plus-downstream" | |
| CULPRIT_FINDER_PIPELINE = "culprit-finder" | |
| COLORS = { | |
| "SERIOUS" : '\033[95m', | |
| "HEADER" : '\033[34m', | |
| "PASSED" : '\033[92m', | |
| "WARNING" : '\033[93m', | |
| "FAIL" : '\033[91m', | |
| "ENDC" : '\033[0m', | |
| "INFO" : '\033[0m', | |
| "BOLD" : '\033[1m', | |
| } | |
| DOWNSTREAM_PIPELINE_CLIENT = bazelci.BuildkiteClient(BUILDKITE_ORG, DOWNSTREAM_PIPELINE) | |
| CULPRIT_FINDER_PIPELINE_CLIENT = bazelci.BuildkiteClient(BUILDKITE_ORG, CULPRIT_FINDER_PIPELINE) | |
| def print_info(context, style, info, append=True): | |
| info_str = "\n".join(info) | |
| bazelci.execute_command( | |
| args = [ | |
| "buildkite-agent", | |
| "annotate", | |
| ] + (["--append"] if append else []) + [ | |
| f"--context={context}", | |
| f"--style={style}", | |
| f"\n{info_str}\n", | |
| ], | |
| print_output = False, | |
| ) | |
| # The class to fetch build result and do the analysis | |
| # The build result are stored in main_result and downstream_result, their structures are like: | |
| # 1. An example of downstream _result | |
| # { | |
| # "build_number": 1396, | |
| # "bazel_commit": "83c5bb00b59137effd976e4f510f74613d85ee1c", | |
| # "commit": "e2b6e2cffc79b89cf0840499c30e3818aaa9ee4b", | |
| # "state": "passed", | |
| # "tasks": { | |
| # "ubuntu1604": { | |
| # "id": "ba3b744c-536d-4f7b-ad9a-c2edd8839ba1", | |
| # "name": "Android Studio Plugin (:ubuntu: 16.04 (OpenJDK 8))", | |
| # "state": "passed", | |
| # "web_url": "https://buildkite.com/bazel/bazel-at-head-plus-downstream/builds/1396#ba3b744c-536d-4f7b-ad9a-c2edd8839ba1" | |
| # } | |
| # } | |
| # } | |
| # 2. An example of main_result | |
| # { | |
| # "build_number": 6299, | |
| # "commit": "e2b6e2cffc79b89cf0840499c30e3818aaa9ee4b", | |
| # "last_green_commit": "e2b6e2cffc79b89cf0840499c30e3818aaa9ee4b", | |
| # "state": "passed", | |
| # "tasks": { | |
| # "ubuntu1604": { | |
| # "id": "1333fa60-c26f-45dd-9169-eeb9a7cc9add", | |
| # "name": ":ubuntu: 16.04 (OpenJDK 8)", | |
| # "state": "passed", | |
| # "web_url": "https://buildkite.com/bazel/android-studio-plugin/builds/6299#1333fa60-c26f-45dd-9169-eeb9a7cc9add" | |
| # } | |
| # } | |
| # } | |
| class BuildInfoAnalyzer(threading.Thread): | |
| success_log_lock = threading.Lock() | |
| success_log = [] | |
| def __init__(self, project, pipeline, downstream_result): | |
| super().__init__() | |
| self.project = project | |
| self.pipeline = pipeline | |
| self.downstream_result = downstream_result | |
| self.main_result = None | |
| self.client = bazelci.BuildkiteClient(BUILDKITE_ORG, pipeline) | |
| self.analyze_log = [f"{COLORS['HEADER']}Analyzing {self.project}: {COLORS['ENDC']}"] | |
| self.broken_by_infra = False | |
| def _get_main_build_result(self): | |
| pipeline_info = self.client.get_pipeline_info() | |
| if not pipeline_info: | |
| error = f"Cannot find pipeline info for pipeline {self.pipeline}." | |
| self._log("SERIOUS", error) | |
| raise bazelci.BuildkiteException(error) | |
| build_info_list = self.client.get_build_info_list([ | |
| ("branch", pipeline_info.get("default_branch") or "master"), | |
| ("page", "1"), | |
| ("per_page", "1"), | |
| ("state[]", "failed"), | |
| ("state[]", "passed"), | |
| ]) | |
| if not build_info_list: | |
| error = f"Cannot find finished build for pipeline {self.pipeline}, please try to rerun the pipeline first." | |
| self._log("SERIOUS", error) | |
| raise bazelci.BuildkiteException(error) | |
| main_build_info = build_info_list[0] | |
| self.main_result = {} | |
| self.main_result["commit"] = main_build_info["commit"] | |
| self.main_result["build_number"] = main_build_info["number"] | |
| job_infos = filter(lambda x: bool(x), (extract_job_info_by_key(job) for job in main_build_info["jobs"] if not ("soft_failed" in job and job["soft_failed"]))) | |
| self.main_result["tasks"] = group_job_info_by_task(job_infos) | |
| self.main_result["state"] = get_project_state(self.main_result["tasks"]) | |
| self.main_result["last_green_commit"] = bazelci.get_last_green_commit(self.project) | |
| # Log all succeeded projects in the same annotate block | |
| def _log_success(self, text): | |
| with BuildInfoAnalyzer.success_log_lock: | |
| BuildInfoAnalyzer.success_log.append(f"{COLORS['HEADER']}Analyzing {self.project}: {COLORS['PASSED']}{text}{COLORS['ENDC']}") | |
| info = [ | |
| "<details><summary><strong>:bk-status-passed: Success</strong></summary><p>", | |
| "", | |
| "```term", | |
| ] + BuildInfoAnalyzer.success_log + [ | |
| "```", | |
| "", | |
| "</p></details>" | |
| ] | |
| print_info("success-info", "success", info, append = False) | |
| # Log broken projects in their separate annotate block | |
| def _log(self, c, text): | |
| self.analyze_log.append(f"{COLORS[c]}{text}{COLORS['ENDC']}") | |
| info = [ | |
| f"<details><summary><strong>:bk-status-failed: {self.project}</strong></summary><p>", | |
| "", | |
| "```term", | |
| ] + self.analyze_log + [ | |
| "```", | |
| "", | |
| "</p></details>" | |
| ] | |
| print_info(self.pipeline, "warning", info, append = False) | |
| # Implement the log function so that it can be called from BuildkiteClient | |
| def log(self, text): | |
| # Be smart here to reduce log length. | |
| if self.analyze_log[-1].startswith(COLORS["INFO"] + "Waiting for "): | |
| self.analyze_log.pop(-1) | |
| self._log("INFO", text) | |
| def _trigger_bisect(self, tasks): | |
| env = { | |
| "PROJECT_NAME": self.project, | |
| "TASK_NAME_LIST": ",".join(tasks) if tasks else "", | |
| } | |
| return CULPRIT_FINDER_PIPELINE_CLIENT.trigger_new_build("HEAD", f"Bisecting {self.project}", env) | |
| # Search for certain message in the log to determine the bisect result. | |
| # Return values are | |
| # 1. A bisect result message | |
| # 2. The commit of the culprit if found, otherwise None | |
| def _determine_bisect_result(self, job): | |
| bisect_log = CULPRIT_FINDER_PIPELINE_CLIENT.get_build_log(job) | |
| pos = bisect_log.rfind("first bad commit is ") | |
| if pos != -1: | |
| start = pos + len("first bad commit is ") | |
| # The length of a full commit hash is 40 | |
| culprit_commit = bisect_log[start:start + 40] | |
| return "\n".join([ | |
| "Culprit found!", | |
| bisect_log[pos:].replace("\r", ""), | |
| "Bisect URL: " + job["web_url"], | |
| ]), culprit_commit | |
| pos = bisect_log.rfind("Cannot find a good bazel commit, abort bisecting.") | |
| if pos != -1: | |
| self.broken_by_infra = True | |
| return "\n".join([ | |
| "Given good commit is now failing. This is probably caused by remote cache issue or infra change.", | |
| "Please ping philwo@ or pcloudy@ for investigation.", | |
| "Bisect URL: " + job["web_url"], | |
| ]), None | |
| pos = bisect_log.rfind("first bad commit not found, every commit succeeded.") | |
| if pos != -1: | |
| return "\n".join([ | |
| "Bisect didn't manage to reproduce the failure, all builds succeeded.", | |
| "Maybe the build are cached from previous build with a different Bazel version or it could be flaky.", | |
| "Please try to rerun the bisect with NEEDS_CLEAN=1 and REPEAT_TIMES=3.", | |
| "Bisect URL: " + job["web_url"], | |
| ]), None | |
| return "Bisect failed due to unknown reason, please check " + job["web_url"], None | |
| def _retry_failed_jobs(self, build_result, buildkite_client): | |
| retry_per_failed_task = {} | |
| for task, info in build_result["tasks"].items(): | |
| if info["state"] != "passed": | |
| retry_per_failed_task[task] = buildkite_client.trigger_job_retry(build_result["build_number"], info["id"]) | |
| for task, job_info in retry_per_failed_task.items(): | |
| retry_per_failed_task[task] = buildkite_client.wait_job_to_finish(build_number = build_result["build_number"], job_id = job_info["id"], logger = self) | |
| return retry_per_failed_task | |
| def _print_job_list(self, jobs): | |
| for job in jobs: | |
| self._log("INFO", f" {job['name']}: {job['web_url']}") | |
| self._log("INFO", "") | |
| def _analyze_main_pipeline_result(self): | |
| self._log("INFO", "") | |
| self._log("PASSED", "***Analyze failures in main pipeline***") | |
| # Report failed tasks | |
| self._log("WARNING", "The following tasks are failing in main pipeline") | |
| self._print_job_list([info for _, info in self.main_result["tasks"].items() if info["state"] != "passed"]) | |
| # Retry all failed tasks | |
| self._log("PASSED", "Retry failed main pipeline tasks...") | |
| retry_per_failed_task = self._retry_failed_jobs(self.main_result, self.client) | |
| # Report tasks that succeeded after retry | |
| succeeded_tasks = [] | |
| for task, info in retry_per_failed_task.items(): | |
| if info["state"] == "passed": | |
| succeeded_tasks.append(info) | |
| self.main_result["tasks"][task]["flaky"] = True | |
| if succeeded_tasks: | |
| self._log("WARNING", "The following tasks succeeded after retry, they might be flaky") | |
| self._print_job_list(succeeded_tasks) | |
| # Report tasks that are still failing after retry | |
| still_failing_tasks = [] | |
| for task, info in retry_per_failed_task.items(): | |
| if info["state"] != "passed": | |
| still_failing_tasks.append(info) | |
| self.main_result["tasks"][task]["broken"] = True | |
| if still_failing_tasks: | |
| last_green_commit = self.main_result["last_green_commit"] | |
| self._log("FAIL", f"The following tasks are still failing after retry, they are probably broken due to changes from the project itself.") | |
| self._log("FAIL", f"The last recorded green commit is {last_green_commit}. Please file bug for the repository.") | |
| self._print_job_list(still_failing_tasks) | |
| def _analyze_for_downstream_pipeline_result(self): | |
| self._log("INFO", "") | |
| self._log("PASSED", "***Analyze failures in downstream pipeline***") | |
| # Report failed tasks | |
| self._log("WARNING", "The following tasks are failing in downstream pipeline") | |
| self._print_job_list([info for _, info in self.downstream_result["tasks"].items() if info["state"] != "passed"]) | |
| # Retry all failed tasks | |
| self._log("PASSED", "Retry failed downstream pipeline tasks...") | |
| retry_per_failed_task = self._retry_failed_jobs(self.downstream_result, DOWNSTREAM_PIPELINE_CLIENT) | |
| # Report tasks that succeeded after retry | |
| succeeded_tasks = [] | |
| for task, info in retry_per_failed_task.items(): | |
| if info["state"] == "passed": | |
| succeeded_tasks.append(info) | |
| self.downstream_result["tasks"][task]["flaky"] = True | |
| if succeeded_tasks: | |
| self._log("WARNING", "The following tasks succeeded after retry, they might be flaky") | |
| self._print_job_list(succeeded_tasks) | |
| # Report tasks that are still failing after retry | |
| still_failing_tasks = [] | |
| failing_task_names = [] | |
| for task, info in retry_per_failed_task.items(): | |
| if info["state"] != "passed": | |
| still_failing_tasks.append(info) | |
| failing_task_names.append(task) | |
| self.downstream_result["tasks"][task]["broken"] = True | |
| if not still_failing_tasks: | |
| return | |
| self._log("FAIL", f"The following tasks are still failing after retry, they are probably broken due to recent Bazel changes.") | |
| self._print_job_list(still_failing_tasks) | |
| # Do bisect for still failing jobs | |
| self._log("PASSED", f"Bisect for still failing tasks...") | |
| bisect_build = self._trigger_bisect(failing_task_names) | |
| bisect_build = CULPRIT_FINDER_PIPELINE_CLIENT.wait_build_to_finish(build_number = bisect_build["number"], logger = self) | |
| bisect_result_by_task = {} | |
| for task in failing_task_names: | |
| for job in bisect_build["jobs"]: | |
| if ("--task_name=" + task) in job["command"]: | |
| bisect_result_by_task[task], culprit = self._determine_bisect_result(job) | |
| if culprit: | |
| self.downstream_result["tasks"][task]["culprit"] = culprit | |
| if task not in bisect_result_by_task: | |
| error = f"Bisect job for task {task} is missing in " + bisect_build["web_url"] | |
| self._log("SERIOUS", error) | |
| raise bazelci.BuildkiteException(error) | |
| # Report bisect result | |
| for task, result in bisect_result_by_task.items(): | |
| self._log("WARNING", "Bisect result for " + self.downstream_result["tasks"][task]["name"]) | |
| self._log("INFO", result) | |
| def _analyze(self): | |
| # Main build: PASSED; Downstream build: PASSED | |
| if self.main_result["state"] == "passed" and self.downstream_result["state"] == "passed": | |
| self._log_success("Main build: PASSED; Downstream build: PASSED") | |
| return | |
| # Main build: FAILED; Downstream build: PASSED | |
| if self.main_result["state"] == "failed" and self.downstream_result["state"] == "passed": | |
| self._log("FAIL", "Main build: FAILED") | |
| self._log("PASSED", "Downstream build: PASSED") | |
| self._analyze_main_pipeline_result() | |
| self._log("HEADER", "Analyzing finished.") | |
| return | |
| # Main build: PASSED; Downstream build: FAILED | |
| if self.main_result["state"] == "passed" and self.downstream_result["state"] == "failed": | |
| self._log("PASSED", "Main build: PASSED") | |
| self._log("FAIL", "Downstream build: FAILED") | |
| self._analyze_for_downstream_pipeline_result() | |
| self._log("HEADER", "Analyzing finished.") | |
| return | |
| # Main build: FAILED; Downstream build: FAILED | |
| if self.main_result["state"] == "failed" and self.downstream_result["state"] == "failed": | |
| self._log("FAIL", "Main build: FAILED") | |
| self._log("FAIL", "Downstream build: FAILED") | |
| last_green_commit = self.main_result["last_green_commit"] | |
| # If the lastest build is the last green commit, that means some infra change has caused the breakage. | |
| if last_green_commit == self.main_result["commit"]: | |
| self.broken_by_infra = True | |
| self._log("SERIOUS", f"Project failed at last green commit. This is probably caused by an infra change, please ping philwo@ or pcloudy@.") | |
| self._log("HEADER", "Analyzing finished.") | |
| return | |
| # Rebuild the project at last green commit, check if the failure is caused by infra change. | |
| self._log("PASSED", f"Rebuild at last green commit {last_green_commit}...") | |
| build_info = self.client.trigger_new_build(last_green_commit, "Trigger build at last green commit.") | |
| build_info = self.client.wait_build_to_finish(build_number = build_info["number"], logger = self) | |
| if build_info["state"] == "failed": | |
| self.broken_by_infra = True | |
| self._log("SERIOUS", f"Project failed at last green commit. This is probably caused by an infra change, please ping philwo@ or pcloudy@.") | |
| elif build_info["state"] == "passed": | |
| self._log("PASSED", f"Project succeeded at last green commit. Maybe main pipeline and downstream pipeline are broken for different reasons.") | |
| self._analyze_main_pipeline_result() | |
| self._analyze_for_downstream_pipeline_result() | |
| else: | |
| self._log("SERIOUS", f"Rebuilding project at last green commit failed with unknown reason. Please check " + build_info["web_url"]) | |
| self._log("HEADER", "Analyzing finished.") | |
| return | |
| def run(self): | |
| self._get_main_build_result() | |
| self._analyze() | |
| def get_html_link_text(content, link): | |
| return f'<a href="{link}" target="_blank">{content}</a>' | |
| def add_tasks_info_text(tasks_per_project, info_text): | |
| for project, task_list in tasks_per_project.items(): | |
| html_link_text = ", ".join([get_html_link_text(name, url) for name, url in task_list]) | |
| info_text.append(f"* **{project}**: {html_link_text}") | |
| def collect_tasks_by_key(build_result, project_name, tasks_per_project, key): | |
| for task_info in build_result["tasks"].values(): | |
| if task_info.get(key): | |
| if project_name not in tasks_per_project: | |
| tasks_per_project[project_name] = [] | |
| tasks_per_project[project_name].append((task_info["name"], task_info["web_url"])) | |
| def get_bazel_commit_url(commit): | |
| return f"https://github.com/bazelbuild/bazel/commit/{commit}" | |
| def get_buildkite_pipeline_url(pipeline): | |
| return f"https://buildkite.com/bazel/{pipeline}" | |
| def report_infra_breakages(analyzers): | |
| projects_broken_by_infra = [(analyzer.project, analyzer.pipeline) for analyzer in analyzers if analyzer.broken_by_infra] | |
| if not projects_broken_by_infra: | |
| return | |
| info_text = [ | |
| "#### The following projects are probably broken by infra change", | |
| "Check the analyze log for more details.", | |
| ] | |
| html_link_text = ", ".join([get_html_link_text(project, get_buildkite_pipeline_url(pipeline)) for project, pipeline in projects_broken_by_infra]) | |
| info_text.append(f"* {html_link_text}") | |
| print_info("broken_tasks_by_infra", "error", info_text) | |
| def report_downstream_breakages(analyzers): | |
| broken_downstream_tasks_per_project = {} | |
| culprits_per_project = {} | |
| for analyzer in analyzers: | |
| collect_tasks_by_key(analyzer.downstream_result, analyzer.project, broken_downstream_tasks_per_project, "broken") | |
| culprits = set() | |
| for task_info in analyzer.downstream_result["tasks"].values(): | |
| if task_info.get("culprit"): | |
| culprits.add(task_info["culprit"]) | |
| if culprits: | |
| culprits_per_project[analyzer.project] = culprits | |
| if not broken_downstream_tasks_per_project: | |
| return | |
| info_text = [ | |
| "#### Broken projects with Bazel at HEAD (Downstream Build)", | |
| "These projects are probably broken by recent Bazel changes.", | |
| ] | |
| for project, task_list in broken_downstream_tasks_per_project.items(): | |
| html_link_text = ", ".join([get_html_link_text(name, url) for name, url in task_list]) | |
| info_text.append(f"* **{project}**: {html_link_text}") | |
| if project in culprits_per_project: | |
| culprit_text = ", ".join([get_html_link_text(commit, get_bazel_commit_url(commit)) for commit in culprits_per_project[project]]) | |
| info_text.append(f" - Culprit Found: {culprit_text}") | |
| else: | |
| info_text.append(" - Culprit Not Found: Please check the analyze log for more details") | |
| print_info("broken_downstream_tasks", "error", info_text) | |
| def report_main_breakages(analyzers): | |
| broken_main_tasks_per_project = {} | |
| for analyzer in analyzers: | |
| collect_tasks_by_key(analyzer.main_result, analyzer.project, broken_main_tasks_per_project, "broken") | |
| if not broken_main_tasks_per_project: | |
| return | |
| info_text = [ | |
| "#### Broken projects with Bazel at latest release (Main Build)", | |
| "These projects are probably broken by their own changes.", | |
| ] | |
| add_tasks_info_text(broken_main_tasks_per_project, info_text) | |
| print_info("broken_main_tasks", "warning", info_text) | |
| def report_flaky_tasks(analyzers): | |
| flaky_main_tasks_per_project = {} | |
| flaky_downstream_tasks_per_project = {} | |
| for analyzer in analyzers: | |
| collect_tasks_by_key(analyzer.main_result, analyzer.project, flaky_main_tasks_per_project, "flaky") | |
| collect_tasks_by_key(analyzer.downstream_result, analyzer.project, flaky_downstream_tasks_per_project, "flaky") | |
| if not flaky_main_tasks_per_project and not flaky_downstream_tasks_per_project: | |
| return | |
| info_text = ["#### Flaky Projects"] | |
| if flaky_main_tasks_per_project: | |
| info_text.append("##### Main Build") | |
| add_tasks_info_text(flaky_main_tasks_per_project, info_text) | |
| if flaky_downstream_tasks_per_project: | |
| info_text.append("##### Downstream Build") | |
| add_tasks_info_text(flaky_downstream_tasks_per_project, info_text) | |
| print_info("flaky_tasks", "warning", info_text) | |
| def report(analyzers): | |
| print_info("analyze_log", "info", ["#### Analyze log"]) | |
| report_flaky_tasks(analyzers) | |
| report_main_breakages(analyzers) | |
| report_downstream_breakages(analyzers) | |
| report_infra_breakages(analyzers) | |
| # Get the raw downstream build result from the lastest finished build | |
| def get_latest_downstream_build_info(): | |
| downstream_build_list = DOWNSTREAM_PIPELINE_CLIENT.get_build_info_list([ | |
| ("branch", "master"), | |
| ("page", "1"), | |
| ("per_page", "1"), | |
| ("state[]", "failed"), | |
| ("state[]", "passed"), | |
| ]) | |
| if len(downstream_build_list) == 0: | |
| raise bazelci.BuildkiteException("Cannot find finished downstream build, please try to rerun downstream pipeline first.") | |
| return downstream_build_list[0] | |
| # Parse infos from command and extract info from original job infos. | |
| # Result is like: | |
| # [ | |
| # {"task": "A", "state": "passed", "web_url": "http://foo/bar/A"}, | |
| # {"task": "B", "state": "failed", "web_url": "http://foo/bar/B"}, | |
| # ] | |
| def extract_job_info_by_key(job, info_from_command = [], info_from_job = ["name", "state", "web_url", "id"]): | |
| if "command" not in job or not job["command"] or "bazelci.py runner" not in job["command"]: | |
| return None | |
| # We have to know which task this job info belongs to | |
| if "task" not in info_from_command: | |
| info_from_command.append("task") | |
| job_info = {} | |
| # Assume there is no space in each argument | |
| args = job["command"].split(" ") | |
| for info in info_from_command: | |
| prefix = "--" + info + "=" | |
| for arg in args: | |
| if arg.startswith(prefix): | |
| job_info[info] = arg[len(prefix):] | |
| break | |
| if info not in job_info: | |
| return None | |
| for info in info_from_job: | |
| if info not in job: | |
| return None | |
| job_info[info] = job[info] | |
| return job_info | |
| # Turn a list of job infos | |
| # [ | |
| # {"task": "windows", "state": "passed", "web_url": "http://foo/bar/A"}, | |
| # {"task": "macos", "state": "failed", "web_url": "http://foo/bar/B"}, | |
| # ] | |
| # into a map of task name to job info | |
| # { | |
| # "windows": {"state": "passed", "web_url": "http://foo/bar/A"}, | |
| # "macos": {"state": "failed", "web_url": "http://foo/bar/B"}, | |
| # } | |
| def group_job_info_by_task(job_infos): | |
| job_info_by_task = {} | |
| for job_info in job_infos: | |
| if "task" not in job_info: | |
| raise bazelci.BuildkiteException(f"'task' must be a key of job_info: {job_info}") | |
| task_name = job_info["task"] | |
| del job_info["task"] | |
| job_info_by_task[task_name] = job_info | |
| return job_info_by_task | |
| # If any of the tasks didn't pass, we condsider the state of this project failed. | |
| def get_project_state(tasks): | |
| for _, infos in tasks.items(): | |
| if infos["state"] != "passed" and infos["state"] != "soft_failed": | |
| return "failed" | |
| return "passed" | |
| # Get the downstream build result in a structure like: | |
| # { | |
| # "project_x" : { | |
| # "commit": "XXXXXXXX", | |
| # "bazel_commit": "XXXXXXXX", | |
| # "tasks" : { | |
| # "A": {"state": "passed", "web_url": "http://foo/bar/A"}, | |
| # "B": {"state": "failed", "web_url": "http://foo/bar/B"}, | |
| # } | |
| # } | |
| # "project_y" : { | |
| # "commit": "XXXXXXXX", | |
| # "bazel_commit": "XXXXXXXX", | |
| # "tasks" : { | |
| # "C": {"state": "passed", "web_url": "http://foo/bar/C"}, | |
| # "D": {"state": "failed", "web_url": "http://foo/bar/D"}, | |
| # } | |
| # } | |
| # } | |
| def get_downstream_result_by_project(downstream_build_info): | |
| downstream_result = {} | |
| jobs_per_project = {} | |
| for job in downstream_build_info["jobs"]: | |
| # Skip all soft failed jobs | |
| if "soft_failed" in job and job["soft_failed"]: | |
| continue | |
| job_info = extract_job_info_by_key(job = job, info_from_command = ["git_commit"]) | |
| if job_info: | |
| project_name = get_project_name_from_job(job_info) | |
| if project_name not in downstream_result: | |
| jobs_per_project[project_name] = [] | |
| downstream_result[project_name] = { | |
| "bazel_commit": downstream_build_info["commit"], | |
| "build_number": downstream_build_info["number"], | |
| "commit": job_info["git_commit"], | |
| } | |
| jobs_per_project[project_name].append(job_info) | |
| for project_name in jobs_per_project: | |
| tasks = group_job_info_by_task(jobs_per_project[project_name]) | |
| downstream_result[project_name]["tasks"] = tasks | |
| downstream_result[project_name]["state"] = get_project_state(tasks) | |
| return downstream_result | |
| def get_project_name_from_job(job_info): | |
| # This is a bit of a hack that depends on how bazelci.create_label() | |
| # formats job names in downstream pipelines. | |
| return job_info.get("name").partition(" (")[0] | |
| def main(argv=None): | |
| downstream_build_info = get_latest_downstream_build_info() | |
| downstream_result = get_downstream_result_by_project(downstream_build_info) | |
| analyzers = [] | |
| for project_name, project_info in bazelci.DOWNSTREAM_PROJECTS.items(): | |
| if project_name in downstream_result: | |
| analyzer = BuildInfoAnalyzer(project_name, project_info["pipeline_slug"], downstream_result[project_name]) | |
| analyzers.append(analyzer) | |
| analyzer.start() | |
| for analyzer in analyzers: | |
| analyzer.join() | |
| report(analyzers) | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) |