#!/usr/bin/env python3
#
# Copyright 2018 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import collections
import os
import re
import requests
import sys
import threading

import bazelci

BUILDKITE_ORG = os.environ["BUILDKITE_ORGANIZATION_SLUG"]

PIPELINE = os.environ["BUILDKITE_PIPELINE_SLUG"]

FAIL_IF_MIGRATION_REQUIRED = os.environ.get("USE_BAZELISK_MIGRATE", "").upper() == "FAIL"

REPO_PATTERN = re.compile(r"https?://github.com/(?P<owner>[^/]+)/(?P<repo>[^/]+).git")

EMOJI_PATTERN = re.compile(r":([\w+-]+):")

EMOJI_IMAGE_TEMPLATE = '<img src="https://raw.githubusercontent.com/buildkite/emojis/master/img-buildkite-64/{}.png" height="16"/>'

INCOMPATIBLE_FLAG_LINE_PATTERN = re.compile(
    r"\s*(?P<flag>--incompatible_\S+)\s*(\(Bazel (?P<version>.+?): (?P<url>.+?)\))?"
)

ISSUE_TEMPLATE = """Incompatible flag {flag} will be enabled by default in {version}, thus breaking {project}.

The flag is documented here: {issue_url}

Please check the following CI builds for build and test results:

{links}

Never heard of incompatible flags before? We have [documentation](https://docs.bazel.build/versions/master/backward-compatibility.html) that explains everything.

If you don't want to receive any future issues for {project} or if you have any questions,
please file an issue in https://github.com/bazelbuild/continuous-integration

**Important**: Please do NOT modify the issue title since that might break our tools.
{version_footnote}
"""

GITHUB_ISSUE_REPORTER = "bazel-flag-bot"

GITHUB_TOKEN_KMS_KEY = "github-api-token"

ENCRYPTED_GITHUB_API_TOKEN = """
CiQA6OLsm0n0R4F/5qdkav2pVIJ+SJnDwcW0+aMgmE0m2UfAtgESUQBsAAJAzHhCcAOfDkOiO0VI7hdPac
vKgsR3LRgJhvwAhomic1ijEXFwUSOcCgvPYXcQK04YCKMJf+/DaExdtRmslvvCBkGI4tjUlMRhJ+8RLQ==
""".strip()


FlagDetails = collections.namedtuple("FlagDetails", ["bazel_version", "issue_url"])


class GitHubError(Exception):
    def __init__(self, code, message):
        super(GitHubError, self).__init__("{}: {}".format(code, message))
        self.code = code
        self.message = message


class GitHubIssueClient(object):

    LINK_PATTERN = re.compile(r'<(?P<url>.*?)>; rel="(?P<type>\w+)"')

    def __init__(self, reporter, oauth_token):
        self._reporter = reporter
        self._session = requests.Session()
        self._session.headers.update(
            {
                "Accept": "application/vnd.github.v3+json",
                "Authorization": "token {}".format(oauth_token),
                "Content-Type": "application/json",
            }
        )

    def get_issue(self, repo_owner, repo_name, title):
        # Returns an arbitrary matching issue if multiple matching issues exist.
        generator = self._send_request(repo_owner, repo_name, params={"creator": self._reporter})
        for issue_subset in generator:
            for i in issue_subset:
                if i["title"] == title:
                    return i["number"]

    def create_issue(self, repo_owner, repo_name, title, body):
        generator = self._send_request(
            repo_owner,
            repo_name,
            verb="post",
            json={"title": title, "body": body, "assignee": None, "labels": [], "milestone": None},
        )
        return next(generator).get("number", "")

    def update_title(self, repo_owner, repo_name, issue_number, title):
        self._send_request(
            repo_owner, repo_name, issue=issue_number, verb="patch", json={"title": title}
        )

    def _send_request(self, repo_owner, repo_name, issue=None, verb="get", **kwargs):
        url = "https://api.github.com/repos/{}/{}/issues".format(repo_owner, repo_name)
        if issue:
            url = os.path.join(url, str(issue))

        method = getattr(self._session, verb)

        while url:
            response = method(url, **kwargs)
            if response.status_code // 100 != 2:
                raise GitHubError(response.status_code, response.content)

            url = self.get_next_page_url(response.headers)
            yield response.json()

    def get_next_page_url(self, headers):
        link = headers.get("Link", "")
        for part in link.split(","):
            match = self.LINK_PATTERN.match(part.strip())
            if match and match.group("type") == "next":
                return match.group("url")


class LogFetcher(threading.Thread):
    def __init__(self, job, client):
        threading.Thread.__init__(self)
        self.job = job
        self.client = client
        self.log = None

    def run(self):
        self.log = self.client.get_build_log(self.job)


def process_build_log(failed_jobs_per_flag, already_failing_jobs, log, job, details_per_flag):
    if "Failure: Command failed, even without incompatible flags." in log:
        already_failing_jobs.append(job)

    def handle_failing_flags(line, details_per_flag):
        flag = extract_flag_details(line, details_per_flag)
        if flag:
            failed_jobs_per_flag[flag][job["id"]] = job

    # bazelisk --migrate might run for multiple times for run / build / test,
    # so there could be several "+++ Result" sections.
    while "+++ Result" in log:
        index_success = log.rfind("Command was successful with the following flags:")
        index_failure = log.rfind("Migration is needed for the following flags:")
        if index_success == -1 or index_failure == -1:
            raise bazelci.BuildkiteException("Cannot recognize log of " + job["web_url"])

        extract_all_flags(log[index_success:index_failure], extract_flag_details, details_per_flag)
        extract_all_flags(log[index_failure:], handle_failing_flags, details_per_flag)
        log = log[0 : log.rfind("+++ Result")]

    # If the job failed for other reasons, we add it into already failing jobs.
    if job["state"] == "failed":
        already_failing_jobs.append(job)


def extract_all_flags(log, line_callback, details_per_flag):
    for line in log.split("\n"):
        line_callback(line, details_per_flag)


def extract_flag_details(line, details_per_flag):
    match = INCOMPATIBLE_FLAG_LINE_PATTERN.match(line)
    if match:
        flag = match.group("flag")
        if details_per_flag.get(flag, (None, None)) == (None, None):
            details_per_flag[flag] = FlagDetails(
                bazel_version=match.group("version"), issue_url=match.group("url")
            )

        return flag


def get_html_link_text(content, link):
    return f'<a href="{link}" target="_blank">{content}</a>'


# Check if any of the given jobs needs to be migrated by the Bazel team
def needs_bazel_team_migrate(jobs):
    for job in jobs:
        pipeline, _ = get_pipeline_and_platform(job)
        if pipeline in bazelci.DOWNSTREAM_PROJECTS and bazelci.DOWNSTREAM_PROJECTS[pipeline].get("owned_by_bazel"):
            return True
    return False


def print_flags_ready_to_flip(failed_jobs_per_flag, details_per_flag):
    info_text1 = ["#### The following flags didn't break any passing projects"]
    for flag in sorted(list(details_per_flag.keys())):
        if flag not in failed_jobs_per_flag:
            html_link_text = get_html_link_text(":github:", details_per_flag[flag].issue_url)
            info_text1.append(f"* **{flag}** {html_link_text}")

    if len(info_text1) == 1:
        info_text1 = []

    info_text2 = ["#### The following flags didn't break any passing Bazel team owned/co-owned projects"]
    for flag, jobs in failed_jobs_per_flag.items():
        if not needs_bazel_team_migrate(jobs.values()):
            failed_cnt = len(jobs)
            s1 = "" if failed_cnt == 1 else "s"
            s2 = "s" if failed_cnt == 1 else ""
            html_link_text = get_html_link_text(":github:", details_per_flag[flag].issue_url)
            info_text2.append(f"* **{flag}** {html_link_text}  ({failed_cnt} other job{s1} need{s2} migration)")

    if len(info_text2) == 1:
        info_text2 = []

    print_info("flags_ready_to_flip", "success", info_text1 + info_text2)


def print_already_fail_jobs(already_failing_jobs):
    info_text = ["#### The following jobs already fail without incompatible flags"]
    info_text += merge_and_format_jobs(already_failing_jobs, "* **{}**: {}")
    if len(info_text) == 1:
        return
    print_info("already_fail_jobs", "warning", info_text)


def print_projects_need_to_migrate(failed_jobs_per_flag):
    info_text = ["#### The following projects need migration"]
    jobs_need_migration = {}
    for jobs in failed_jobs_per_flag.values():
        for job in jobs.values():
            jobs_need_migration[job["name"]] = job

    job_list = jobs_need_migration.values()
    job_num = len(job_list)
    if job_num == 0:
        return

    projects = set()
    for job in job_list:
        project, _ = get_pipeline_and_platform(job)
        projects.add(project)
    project_num = len(projects)

    s1 = "" if project_num == 1 else "s"
    s2 = "s" if project_num == 1 else ""
    info_text.append(
        f"<details><summary>{project_num} project{s1} need{s2} migration, click to see details</summary><ul>"
    )

    entries = merge_and_format_jobs(job_list, "    <li><strong>{}</strong>: {}</li>")
    info_text += entries
    info_text.append("</ul></details>")

    info_str = "\n".join(info_text)
    bazelci.execute_command(
        [
            "buildkite-agent",
            "annotate",
            "--append",
            f"--context=projects_need_migration",
            f"--style=error",
            f"\n{info_str}\n",
        ]
    )


def print_flags_need_to_migrate(failed_jobs_per_flag, details_per_flag):
    # The info box printed later is above info box printed before,
    # so reverse the flag list to maintain the same order.
    printed_flag_boxes = False
    for flag in sorted(list(failed_jobs_per_flag.keys()), reverse=True):
        jobs = failed_jobs_per_flag[flag]
        if jobs:
            github_url = details_per_flag[flag].issue_url
            info_text = [f"* **{flag}** " + get_html_link_text(":github:", github_url)]
            jobs_per_pipeline = merge_jobs(jobs.values())
            for pipeline, platforms in jobs_per_pipeline.items():
                bazel_mark = ""
                if pipeline in bazelci.DOWNSTREAM_PROJECTS and bazelci.DOWNSTREAM_PROJECTS[pipeline].get("owned_by_bazel"):
                    bazel_mark = ":bazel:"
                platforms_text = ", ".join(platforms)
                info_text.append(f"  - {bazel_mark}**{pipeline}**: {platforms_text}")
            # Use flag as the context so that each flag gets a different info box.
            print_info(flag, "error", info_text)
            printed_flag_boxes = True
    if not printed_flag_boxes:
        return
    info_text = [
        "#### Downstream projects need to migrate for the following flags:",
        "Projects marked with :bazel: need to be migrated by the Bazel team.",
    ]
    print_info("flags_need_to_migrate", "error", info_text)


def merge_jobs(jobs):
    jobs_per_pipeline = collections.defaultdict(list)
    for job in sorted(jobs, key=lambda s: s["name"].lower()):
        pipeline, platform = get_pipeline_and_platform(job)
        jobs_per_pipeline[pipeline].append(get_html_link_text(platform, job["web_url"]))
    return jobs_per_pipeline


def merge_and_format_jobs(jobs, line_pattern):
    # Merges all jobs for a single pipeline into one line.
    # Example:
    #   pipeline (platform1)
    #   pipeline (platform2)
    #   pipeline (platform3)
    # with line_pattern ">> {}: {}" becomes
    #   >> pipeline: platform1, platform2, platform3
    jobs_per_pipeline = merge_jobs(jobs)
    return [
        line_pattern.format(pipeline, ", ".join(platforms))
        for pipeline, platforms in jobs_per_pipeline.items()
    ]


def get_pipeline_and_platform(job):
    name = job["name"]
    platform = ""
    for p in bazelci.PLATFORMS.values():
        platform_label = p.get("emoji-name")
        if platform_label in name:
            platform = platform_label
            name = name.replace(platform_label, "")
            break

    name = name.partition("-")[0].partition("(")[0].strip()
    return name, platform


def print_info(context, style, info):
    # CHUNK_SIZE is to prevent buildkite-agent "argument list too long" error
    CHUNK_SIZE = 20
    for i in range(0, len(info), CHUNK_SIZE):
        info_str = "\n".join(info[i : i + CHUNK_SIZE])
        bazelci.execute_command(
            [
                "buildkite-agent",
                "annotate",
                "--append",
                f"--context={context}",
                f"--style={style}",
                f"\n{info_str}\n",
            ]
        )


def analyze_logs(build_number, client):
    build_info = client.get_build_info(build_number)

    already_failing_jobs = []

    # dict(flag name -> dict(job id -> job))
    failed_jobs_per_flag = collections.defaultdict(dict)
    # dict(flag name -> (Bazel version where it's flipped, GitHub issue URL))
    details_per_flag = {}

    threads = []
    for job in build_info["jobs"]:
        # Some irrelevant job has no "state" field
        if "state" in job:
            thread = LogFetcher(job, client)
            threads.append(thread)
            thread.start()

    for thread in threads:
        thread.join()
        process_build_log(
            failed_jobs_per_flag, already_failing_jobs, thread.log, thread.job, details_per_flag
        )

    return already_failing_jobs, failed_jobs_per_flag, details_per_flag


def handle_already_flipped_flags(failed_jobs_per_flag, details_per_flag):
    # Process and remove all flags that have already been flipped.
    # Bazelisk may return already flipped flags if a project uses an old Bazel version
    # via its .bazelversion file.
    current_major_version = bazelci.get_bazel_major_version()
    failed_jobs_for_new_flags = {}
    details_for_new_flags = {}

    for flag, details in details_per_flag.items():
        if details.bazel_version < current_major_version:
            # TOOD(fweikert): maybe display a Buildkite annotation
            bazelci.eprint(
                "Ignoring {} since it has already been flipped in Bazel {} (latest is {}).".format(
                    flag, details.bazel_version, current_major_version
                )
            )
            continue

        details_for_new_flags[flag] = details
        if flag in failed_jobs_per_flag:
            failed_jobs_for_new_flags[flag] = failed_jobs_per_flag[flag]

    return failed_jobs_for_new_flags, details_for_new_flags


def print_result_info(already_failing_jobs, failed_jobs_per_flag, details_per_flag):
    print_flags_need_to_migrate(failed_jobs_per_flag, details_per_flag)

    print_projects_need_to_migrate(failed_jobs_per_flag)

    print_already_fail_jobs(already_failing_jobs)

    print_flags_ready_to_flip(failed_jobs_per_flag, details_per_flag)

    return bool(failed_jobs_per_flag)


def notify_projects(failed_jobs_per_flag, details_per_flag):
    links_per_project_and_flag = collect_notification_links(failed_jobs_per_flag)
    create_all_issues(details_per_flag, links_per_project_and_flag)


def collect_notification_links(failed_jobs_per_flag):
    links_per_project_and_flag = collections.defaultdict(set)
    for flag, job_data in failed_jobs_per_flag.items():
        for job in job_data.values():
            project_label, platform = get_pipeline_and_platform(job)
            link = get_link_for_build(platform, job["web_url"])
            links_per_project_and_flag[(project_label, flag)].add(link)

    return links_per_project_and_flag


def get_link_for_build(platform, url):
    names = [v["name"] for v in bazelci.PLATFORMS.values() if v["emoji-name"] == platform]
    display_name = names[0] if names else ""

    match = EMOJI_PATTERN.search(platform)
    img = ""
    if match:
        # darwin emoji has image mac.png
        emoji = match.group(1).replace("darwin", "mac")
        img = EMOJI_IMAGE_TEMPLATE.format(emoji)

    text = (img + display_name) or platform
    return get_html_link_text(text, url)


def create_all_issues(details_per_flag, links_per_project_and_flag):
    errors = set()
    issue_client = get_github_client()
    for (project_label, flag), links in links_per_project_and_flag.items():
        try:
            details = details_per_flag.get(flag, (None, None))
            if details.bazel_version in (None, "unreleased binary"):
                raise bazelci.BuildkiteException(
                    "Notifications: Invalid Bazel version '{}' for flag {}".format(
                        details.bazel_version or "", flag
                    )
                )

            if not details.issue_url:
                raise bazelci.BuildkiteException(
                    "Notifications: Missing GitHub issue URL for flag {}".format(flag)
                )

            repo_owner, repo_name, do_not_notify = get_project_details(project_label)
            if do_not_notify:
                bazelci.eprint("{} has opted out of notifications.".format(project_label))
                continue

            temporary_title = get_temporary_issue_title(project_label, flag)
            final_title = get_final_issue_title(project_label, details.bazel_version, flag)
            has_target_release = details.bazel_version != "TBD"

            # Three possible scenarios:
            # 1. There is already an issue with the target release in the title -> do nothing
            # 2. There is an issue, but without the target release, and we now know the target release -> update title
            # 3. There is no issue -> create one
            if issue_client.get_issue(repo_owner, repo_name, final_title):
                bazelci.eprint(
                    "There is already an issue in {}/{} for project {}, flag {} and Bazel {}".format(
                        repo_owner, repo_name, project_label, flag, details.bazel_version
                    )
                )
            else:
                number = issue_client.get_issue(repo_owner, repo_name, temporary_title)
                if number:
                    if has_target_release:
                        issue_client.update_title(repo_owner, repo_name, number, final_title)
                else:
                    body = create_issue_body(project_label, flag, details, links)
                    title = final_title if has_target_release else temporary_title
                    issue_client.create_issue(repo_owner, repo_name, title, body)
        except (bazelci.BuildkiteException, GitHubError) as ex:
            errors.add("Could not notify project '{}': {}".format(project_label, ex))

    if errors:
        print_info("notify_errors", "error", list(errors))


def get_github_client():
    try:
        github_token = bazelci.decrypt_token(
            encrypted_token=ENCRYPTED_GITHUB_API_TOKEN, kms_key=GITHUB_TOKEN_KMS_KEY
        )
    except Exception as ex:
        raise bazelci.BuildkiteException("Failed to decrypt GitHub API token: {}".format(ex))

    return GitHubIssueClient(reporter=GITHUB_ISSUE_REPORTER, oauth_token=github_token)


def get_project_details(project_label):
    entry = bazelci.DOWNSTREAM_PROJECTS.get(project_label, {})
    full_repo = entry.get("git_repository", "")
    if not full_repo:
        raise bazelci.BuildkiteException(
            "Could not retrieve Git repository for project '{}'".format(project_label)
        )
    match = REPO_PATTERN.match(full_repo)
    if not match:
        raise bazelci.BuildkiteException(
            "Hosts other than GitHub are currently not supported ({})".format(full_repo)
        )

    return match.group("owner"), match.group("repo"), entry.get("do_not_notify", False)


def get_temporary_issue_title(project_label, flag):
    return "Flag {} will break {} in a future Bazel release".format(flag, project_label)


def get_final_issue_title(project_label, bazel_version, flag):
    return "Flag {} will break {} in Bazel {}".format(flag, project_label, bazel_version)


def create_issue_body(project_label, flag, details, links):
    if details.bazel_version == "TBD":
        version = "a future Bazel release [1]"
        version_footnote = (
            "\n[1] The target release hasn't been determined yet. "
            "Our tool will update the issue title once the flag flip has been scheduled."
        )
    else:
        version = "Bazel {}".format(details.bazel_version)
        version_footnote = ""

    return ISSUE_TEMPLATE.format(
        project=project_label,
        version=version,
        version_footnote=version_footnote,
        issue_url=details.issue_url,
        flag=flag,
        links="\n".join("* {}".format(l) for l in links),
    )


def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]

    parser = argparse.ArgumentParser(
        description="Script to aggregate `bazelisk --migrate` test result for incompatible flags and generate pretty Buildkite info messages."
    )
    parser.add_argument("--build_number", type=str)
    parser.add_argument("--notify", type=bool, nargs="?", const=True)

    args = parser.parse_args(argv)
    try:
        if args.build_number:
            client = bazelci.BuildkiteClient(org=BUILDKITE_ORG, pipeline=PIPELINE)
            already_failing_jobs, failed_jobs_per_flag, details_per_flag = analyze_logs(
                args.build_number, client
            )
            failed_jobs_per_flag, details_per_flag = handle_already_flipped_flags(
                failed_jobs_per_flag, details_per_flag
            )
            migration_required = print_result_info(
                already_failing_jobs, failed_jobs_per_flag, details_per_flag
            )

            if args.notify:
                notify_projects(failed_jobs_per_flag, details_per_flag)

            if migration_required and FAIL_IF_MIGRATION_REQUIRED:
                bazelci.eprint("Exiting with code 3 since a migration is required.")
                return 3
        else:
            parser.print_help()
            return 2

    except bazelci.BuildkiteException as e:
        bazelci.eprint(str(e))
        return 1
    return 0


if __name__ == "__main__":
    sys.exit(main())