// Copyright 2019 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.runtime;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.eventbus.EventBus;
import com.google.devtools.build.lib.actions.ActionOwner;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.analysis.AliasProvider;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.test.TestProvider;
import com.google.devtools.build.lib.analysis.test.TestProvider.TestParams;
import com.google.devtools.build.lib.analysis.test.TestResult;
import com.google.devtools.build.lib.concurrent.ThreadSafety;
import com.google.devtools.build.lib.exec.TestAttempt;
import com.google.devtools.build.lib.packages.TestSize;
import com.google.devtools.build.lib.packages.TestTimeout;
import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

/** This class aggregates and reports target-wide test statuses in real-time. */
@ThreadSafety.ThreadSafe
final class TestResultAggregator {
  /**
   * Settings for the aggregator; there are usually many aggregator instances with the same set of
   * settings, so we move them to a separate object.
   */
  static final class AggregationPolicy {
    private final EventBus eventBus;
    private final boolean testCheckUpToDate;
    private final boolean testVerboseTimeoutWarnings;

    AggregationPolicy(
        EventBus eventBus, boolean testCheckUpToDate, boolean testVerboseTimeoutWarnings) {
      this.eventBus = eventBus;
      this.testCheckUpToDate = testCheckUpToDate;
      this.testVerboseTimeoutWarnings = testVerboseTimeoutWarnings;
    }
  }

  private final AggregationPolicy policy;
  private final ConfiguredTarget testTarget;
  private final TestSummary.Builder summary;
  private final Set<Artifact> remainingRuns;
  private final Map<Artifact, TestResult> statusMap = new HashMap<>();

  public TestResultAggregator(
      ConfiguredTarget target, BuildConfiguration configuration, AggregationPolicy policy) {
    this.testTarget = target;
    this.policy = policy;

    // And create an empty summary suitable for incremental analysis.
    // Also has the nice side effect of mapping labels to RuleConfiguredTargets.
    this.summary = TestSummary.newBuilder();
    this.summary.setTarget(target);
    if (configuration != null) {
      // This can be null for testing.
      this.summary.setConfiguration(configuration);
    }
    this.summary.setStatus(BlazeTestStatus.NO_STATUS);
    this.remainingRuns = new HashSet<>(TestProvider.getTestStatusArtifacts(target));
  }

  /**
   * Records a new test run result and incrementally updates the target status. This event is sent
   * upon completion of executed test runs.
   */
  synchronized void testEvent(TestResult result) {
    ActionOwner testOwner = result.getTestAction().getOwner();
    ConfiguredTargetKey targetLabel =
        ConfiguredTargetKey.of(testOwner.getLabel(), result.getTestAction().getConfiguration());
    Preconditions.checkArgument(targetLabel.equals(asKey(testTarget)));

    TestResult previousResult = statusMap.put(result.getTestStatusArtifact(), result);
    if (previousResult != null) {
      throw new IllegalStateException(
          String.format(
              "Duplicate result reported for an individual test shard %s.\nNew: %s\nPrevious: %s",
              result.getTestStatusArtifact(), result.getData(), previousResult.getData()));
    }

    // If a test result was cached, then post the cached attempts to the event bus.
    if (result.isCached()) {
      for (TestAttempt attempt : result.getCachedTestAttempts()) {
        policy.eventBus.post(attempt);
      }
    }

    TestSummary finalTestSummary = null;
    Preconditions.checkNotNull(summary);
    if (!remainingRuns.remove(result.getTestStatusArtifact())) {
      // This can happen if a buildCompleteEvent() was processed before this event reached us.
      // This situation is likely to happen if --notest_keep_going is set with multiple targets.
      return;
    }

    incrementalAnalyze(result);

    // If all runs are processed, the target is finished and ready to report.
    if (remainingRuns.isEmpty()) {
      finalTestSummary = summary.build();
    }

    // Report finished targets.
    if (finalTestSummary != null) {
      policy.eventBus.post(finalTestSummary);
    }
  }

  synchronized void targetFailure(boolean blazeHalted, boolean skipTargetsOnFailure) {
    if (remainingRuns.isEmpty()) {
      // Blaze does not guarantee that BuildResult.getSuccessfulTargets() and posted TestResult
      // events are in sync. Thus, it is possible that a test event was posted, but the target is
      // not present in the set of successful targets.
      return;
    }

    markUnbuilt(blazeHalted, skipTargetsOnFailure);

    // These are never going to run; removing them marks the target complete.
    remainingRuns.clear();
    policy.eventBus.post(summary.build());
  }

  /** Returns the known aggregate results for the given target at the current moment. */
  synchronized TestSummary.Builder getCurrentSummaryForTesting() {
    return summary;
  }

  /**
   * Returns all test status artifacts associated with a given target whose runs have yet to finish.
   */
  synchronized Collection<Artifact> getIncompleteRunsForTesting() {
    return ImmutableSet.copyOf(remainingRuns);
  }

  synchronized Map<Artifact, TestResult> getStatusMapForTesting() {
    return ImmutableMap.copyOf(statusMap);
  }

  private static ConfiguredTargetKey asKey(ConfiguredTarget target) {
    return ConfiguredTargetKey.of(
        // A test is never in the host configuration.
        AliasProvider.getDependencyLabel(target),
        target.getConfigurationKey(),
        /*isHostConfiguration=*/ false);
  }

  private static BlazeTestStatus aggregateStatus(BlazeTestStatus status, BlazeTestStatus other) {
    return status.getNumber() > other.getNumber() ? status : other;
  }

  /**
   * Helper for differential analysis which aggregates the TestSummary for an individual target,
   * reporting runs on the EventBus if necessary.
   */
  synchronized TestSummary aggregateAndReportSummary(boolean skipTargetsOnFailure) {
    // If already reported by the listener, no work remains for this target.
    if (remainingRuns.isEmpty()) {
      return summary.build();
    }

    // We will get back multiple TestResult instances if test had to be retried several
    // times before passing. Sharding and multiple runs of the same test without retries
    // will be represented by separate artifacts and will produce exactly one TestResult.
    for (Artifact testStatus : TestProvider.getTestStatusArtifacts(testTarget)) {
      // When a build is interrupted ( eg. a broken target with --nokeep_going ) runResult could
      // be null for an unrelated test because we were not able to even try to execute the test.
      // In that case, for tests that were previously passing we return null ( == NO STATUS),
      // because checking if the cached test target is up-to-date would require running the
      // dependency checker transitively.
      TestResult runResult = statusMap.get(testStatus);
      boolean isIncompleteRun = remainingRuns.contains(testStatus);
      if (runResult == null) {
        markIncomplete(skipTargetsOnFailure);
      } else if (isIncompleteRun) {
        incrementalAnalyze(runResult);
      }
    }

    // The target was not posted by the listener and must be posted now.
    TestSummary result = summary.build();
    policy.eventBus.post(result);
    return result;
  }

  /**
   * Incrementally updates a TestSummary given an existing summary and a new TestResult. Only call
   * on built targets.
   *
   * @param result New test result to aggregate into the summary.
   */
  synchronized void incrementalAnalyze(TestResult result) {
    // Cache retrieval should have been performed already.
    Preconditions.checkNotNull(result);
    TestSummary existingSummary = Preconditions.checkNotNull(summary.peek());

    BlazeTestStatus status = existingSummary.getStatus();
    int numCached = existingSummary.numCached();
    int numLocalActionCached = existingSummary.numLocalActionCached();

    // If a test was neither cached locally nor remotely we say action was taken.
    if (!(result.isCached() || result.getData().getRemotelyCached())) {
      summary.setActionRan(true);
    } else {
      numCached++;
    }

    if (result.isCached()) {
      numLocalActionCached++;
    }

    Path coverageData = result.getCoverageData();
    if (coverageData != null) {
      summary.addCoverageFiles(ImmutableList.of(coverageData));
    }

    TransitiveInfoCollection target = existingSummary.getTarget();
    Preconditions.checkNotNull(target, "The existing TestSummary must be associated with a target");
    TestParams testParams = target.getProvider(TestProvider.class).getTestParams();

    if (!testParams.runsDetectsFlakes()) {
      status = aggregateStatus(status, result.getData().getStatus());
    } else {
      int shardNumber = result.getShardNum();
      int runsPerTestForLabel = testParams.getRuns();
      List<BlazeTestStatus> singleShardStatuses =
          summary.addShardStatus(shardNumber, result.getData().getStatus());
      if (singleShardStatuses.size() == runsPerTestForLabel) {
        // Aggregation is based on the order of status enums where larger values take precedence
        // over smaller ones (NO_STATUS = 0, PASSED = 1, etc.). However, there are some special
        // cases:
        // 1. Tests that have some passing, some not passing shard are marked as flaky
        // 2. The INCOMPLETE status is ignored - it is used for tests runs that are cancelled by
        //    Bazel if --cancel_concurrent_tests is set, otherwise INCOMPLETE is not used
        // 3. Individual test shards can be FLAKY if the test is marked flaky and
        //    --flaky_test_attempts is not zero for this test
        BlazeTestStatus shardStatus = BlazeTestStatus.NO_STATUS;
        int passes = 0;
        int cancelled = 0;
        for (BlazeTestStatus runStatusForShard : singleShardStatuses) {
          if (runStatusForShard == BlazeTestStatus.INCOMPLETE) {
            // If runs_per_test_detects_flakes is enabled, then INCOMPLETE status indicates
            // cancelled test runs. We count them separately so that they don't result in a
            // flaky status below.
            cancelled++;
          } else {
            shardStatus = aggregateStatus(shardStatus, runStatusForShard);
            if (TestResult.isBlazeTestStatusPassed(runStatusForShard)) {
              passes++;
            }
          }
        }
        // Under the RunsPerTestDetectsFlakes option, return flaky if 0 < p < (n-cancelled) shards
        // pass. Otherwise, we aggregate the shardStatus.
        if (passes == 0 || (passes + cancelled) == runsPerTestForLabel) {
          status = aggregateStatus(status, shardStatus);
        } else {
          status = aggregateStatus(status, BlazeTestStatus.FLAKY);
        }
      }
    }

    if (result.getData().hasPassedLog()) {
      summary.addPassedLog(result.getTestLogPath().getRelative(result.getData().getPassedLog()));
    }
    for (String path : result.getData().getFailedLogsList()) {
      summary.addFailedLog(result.getTestLogPath().getRelative(path));
    }

    summary
        .addTestTimes(result.getData().getTestTimesList())
        .mergeTiming(
            result.getData().getStartTimeMillisEpoch(), result.getData().getRunDurationMillis())
        .addWarnings(result.getData().getWarningList())
        .collectTestCases(result.getData().hasTestCase() ? result.getData().getTestCase() : null)
        .setRanRemotely(result.getData().getIsRemoteStrategy());

    List<String> warnings = new ArrayList<>();
    if (status == BlazeTestStatus.PASSED
        && shouldEmitTestSizeWarningInSummary(
            policy.testVerboseTimeoutWarnings,
            warnings,
            result.getData().getTestProcessTimesList(),
            target)) {
      summary.setWasUnreportedWrongSize(true);
    }

    summary
        .setStatus(status)
        .setNumCached(numCached)
        .setNumLocalActionCached(numLocalActionCached)
        .addWarnings(warnings);
  }

  private void markIncomplete(boolean skipTargetsOnFailure) {
    // TODO(bazel-team): (2010) Make NotRunTestResult support both tests failed to built and
    // tests with no status and post it here.
    TestSummary peekSummary = summary.peek();
    BlazeTestStatus status = peekSummary.getStatus();
    if (skipTargetsOnFailure) {
      status = BlazeTestStatus.NO_STATUS;
    } else if (status != BlazeTestStatus.NO_STATUS) {
      status = aggregateStatus(status, BlazeTestStatus.INCOMPLETE);
    }

    summary.setStatus(status);
  }

  private void markUnbuilt(boolean blazeHalted, boolean skipTargetsOnFailure) {
    BlazeTestStatus runStatus =
        blazeHalted
            ? BlazeTestStatus.BLAZE_HALTED_BEFORE_TESTING
            : (policy.testCheckUpToDate || skipTargetsOnFailure
                ? BlazeTestStatus.NO_STATUS
                : BlazeTestStatus.FAILED_TO_BUILD);

    summary.setStatus(runStatus);
  }

  /**
   * Checks whether the specified test timeout could have been smaller or is too small and adds a
   * warning message if verbose is true.
   *
   * <p>Returns true if there was a test with the wrong timeout, but if was not reported.
   */
  private static boolean shouldEmitTestSizeWarningInSummary(
      boolean verbose,
      List<String> warnings,
      List<Long> testTimes,
      TransitiveInfoCollection target) {

    TestTimeout specifiedTimeout =
        target.getProvider(TestProvider.class).getTestParams().getTimeout();
    long maxTimeOfShard = 0;

    for (Long shardTime : testTimes) {
      if (shardTime != null) {
        maxTimeOfShard = Math.max(maxTimeOfShard, shardTime);
      }
    }

    int maxTimeInSeconds = (int) (maxTimeOfShard / 1000);

    if (!specifiedTimeout.isInRangeFuzzy(maxTimeInSeconds)) {
      TestTimeout expectedTimeout = TestTimeout.getSuggestedTestTimeout(maxTimeInSeconds);
      TestSize expectedSize = TestSize.getTestSize(expectedTimeout);
      if (verbose) {
        StringBuilder builder =
            new StringBuilder(
                String.format(
                    "%s: Test execution time (%.1fs excluding execution overhead) outside of "
                        + "range for %s tests. Consider setting timeout=\"%s\"",
                    AliasProvider.getDependencyLabel(target),
                    maxTimeOfShard / 1000.0,
                    specifiedTimeout.prettyPrint(),
                    expectedTimeout));
        if (expectedSize != null) {
          builder.append(" or size=\"").append(expectedSize).append("\"");
        }
        builder.append(".");
        warnings.add(builder.toString());
        return false;
      }
      return true;
    } else {
      return false;
    }
  }
}