src/main/java/com/google/devtools/build/lib/runtime/TestResultAggregator.java - bazel - Git at Google

 // Copyright 2019 The Bazel Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package com.google.devtools.build.lib.runtime;

 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.eventbus.EventBus;
 import com.google.devtools.build.lib.actions.ActionOwner;
 import com.google.devtools.build.lib.actions.Artifact;
 import com.google.devtools.build.lib.analysis.AliasProvider;
 import com.google.devtools.build.lib.analysis.ConfiguredTarget;
 import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
 import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
 import com.google.devtools.build.lib.analysis.test.TestProvider;
 import com.google.devtools.build.lib.analysis.test.TestProvider.TestParams;
 import com.google.devtools.build.lib.analysis.test.TestResult;
 import com.google.devtools.build.lib.concurrent.ThreadSafety;
 import com.google.devtools.build.lib.exec.TestAttempt;
 import com.google.devtools.build.lib.packages.TestSize;
 import com.google.devtools.build.lib.packages.TestTimeout;
 import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;

 /** This class aggregates and reports target-wide test statuses in real-time. */
 @ThreadSafety.ThreadSafe
 final class TestResultAggregator {
   /**
    * Settings for the aggregator; there are usually many aggregator instances with the same set of
    * settings, so we move them to a separate object.
    */
   static final class AggregationPolicy {
     private final EventBus eventBus;
     private final boolean testCheckUpToDate;
     private final boolean testVerboseTimeoutWarnings;

     AggregationPolicy(
         EventBus eventBus, boolean testCheckUpToDate, boolean testVerboseTimeoutWarnings) {
       this.eventBus = eventBus;
       this.testCheckUpToDate = testCheckUpToDate;
       this.testVerboseTimeoutWarnings = testVerboseTimeoutWarnings;
     }
   }

   private final AggregationPolicy policy;
   private final ConfiguredTarget testTarget;
   private final TestSummary.Builder summary;
   private final Set<Artifact> remainingRuns;
   private final Map<Artifact, TestResult> statusMap = new HashMap<>();

   public TestResultAggregator(
       ConfiguredTarget target, BuildConfiguration configuration, AggregationPolicy policy) {
     this.testTarget = target;
     this.policy = policy;

     // And create an empty summary suitable for incremental analysis.
     // Also has the nice side effect of mapping labels to RuleConfiguredTargets.
     this.summary = TestSummary.newBuilder();
     this.summary.setTarget(target);
     if (configuration != null) {
       // This can be null for testing.
       this.summary.setConfiguration(configuration);
     }
     this.summary.setStatus(BlazeTestStatus.NO_STATUS);
     this.remainingRuns = new HashSet<>(TestProvider.getTestStatusArtifacts(target));
   }

   /**
    * Records a new test run result and incrementally updates the target status. This event is sent
    * upon completion of executed test runs.
    */
   synchronized void testEvent(TestResult result) {
     ActionOwner testOwner = result.getTestAction().getOwner();
     ConfiguredTargetKey targetLabel =
         ConfiguredTargetKey.of(testOwner.getLabel(), result.getTestAction().getConfiguration());
     Preconditions.checkArgument(targetLabel.equals(asKey(testTarget)));

     TestResult previousResult = statusMap.put(result.getTestStatusArtifact(), result);
     if (previousResult != null) {
       throw new IllegalStateException(
           String.format(
               "Duplicate result reported for an individual test shard %s.\nNew: %s\nPrevious: %s",
               result.getTestStatusArtifact(), result.getData(), previousResult.getData()));
     }

     // If a test result was cached, then post the cached attempts to the event bus.
     if (result.isCached()) {
       for (TestAttempt attempt : result.getCachedTestAttempts()) {
         policy.eventBus.post(attempt);
       }
     }

     TestSummary finalTestSummary = null;
     Preconditions.checkNotNull(summary);
     if (!remainingRuns.remove(result.getTestStatusArtifact())) {
       // This can happen if a buildCompleteEvent() was processed before this event reached us.
       // This situation is likely to happen if --notest_keep_going is set with multiple targets.
       return;
     }

     incrementalAnalyze(result);

     // If all runs are processed, the target is finished and ready to report.
     if (remainingRuns.isEmpty()) {
       finalTestSummary = summary.build();
     }

     // Report finished targets.
     if (finalTestSummary != null) {
       policy.eventBus.post(finalTestSummary);
     }
   }

   synchronized void targetFailure(boolean blazeHalted, boolean skipTargetsOnFailure) {
     if (remainingRuns.isEmpty()) {
       // Blaze does not guarantee that BuildResult.getSuccessfulTargets() and posted TestResult
       // events are in sync. Thus, it is possible that a test event was posted, but the target is
       // not present in the set of successful targets.
       return;
     }

     markUnbuilt(blazeHalted, skipTargetsOnFailure);

     // These are never going to run; removing them marks the target complete.
     remainingRuns.clear();
     policy.eventBus.post(summary.build());
   }

   /** Returns the known aggregate results for the given target at the current moment. */
   synchronized TestSummary.Builder getCurrentSummaryForTesting() {
     return summary;
   }

   /**
    * Returns all test status artifacts associated with a given target whose runs have yet to finish.
    */
   synchronized Collection<Artifact> getIncompleteRunsForTesting() {
     return ImmutableSet.copyOf(remainingRuns);
   }

   synchronized Map<Artifact, TestResult> getStatusMapForTesting() {
     return ImmutableMap.copyOf(statusMap);
   }

   private static ConfiguredTargetKey asKey(ConfiguredTarget target) {
     return ConfiguredTargetKey.of(
         // A test is never in the host configuration.
         AliasProvider.getDependencyLabel(target),
         target.getConfigurationKey(),
         /*isHostConfiguration=*/ false);
   }

   private static BlazeTestStatus aggregateStatus(BlazeTestStatus status, BlazeTestStatus other) {
     return status.getNumber() > other.getNumber() ? status : other;
   }

   /**
    * Helper for differential analysis which aggregates the TestSummary for an individual target,
    * reporting runs on the EventBus if necessary.
    */
   synchronized TestSummary aggregateAndReportSummary(boolean skipTargetsOnFailure) {
     // If already reported by the listener, no work remains for this target.
     if (remainingRuns.isEmpty()) {
       return summary.build();
     }

     // We will get back multiple TestResult instances if test had to be retried several
     // times before passing. Sharding and multiple runs of the same test without retries
     // will be represented by separate artifacts and will produce exactly one TestResult.
     for (Artifact testStatus : TestProvider.getTestStatusArtifacts(testTarget)) {
       // When a build is interrupted ( eg. a broken target with --nokeep_going ) runResult could
       // be null for an unrelated test because we were not able to even try to execute the test.
       // In that case, for tests that were previously passing we return null ( == NO STATUS),
       // because checking if the cached test target is up-to-date would require running the
       // dependency checker transitively.
       TestResult runResult = statusMap.get(testStatus);
       boolean isIncompleteRun = remainingRuns.contains(testStatus);
       if (runResult == null) {
         markIncomplete(skipTargetsOnFailure);
       } else if (isIncompleteRun) {
         incrementalAnalyze(runResult);
       }
     }

     // The target was not posted by the listener and must be posted now.
     TestSummary result = summary.build();
     policy.eventBus.post(result);
     return result;
   }

   /**
    * Incrementally updates a TestSummary given an existing summary and a new TestResult. Only call
    * on built targets.
    *
    * @param result New test result to aggregate into the summary.
    */
   synchronized void incrementalAnalyze(TestResult result) {
     // Cache retrieval should have been performed already.
     Preconditions.checkNotNull(result);
     TestSummary existingSummary = Preconditions.checkNotNull(summary.peek());

     BlazeTestStatus status = existingSummary.getStatus();
     int numCached = existingSummary.numCached();
     int numLocalActionCached = existingSummary.numLocalActionCached();

     // If a test was neither cached locally nor remotely we say action was taken.
     if (!(result.isCached() || result.getData().getRemotelyCached())) {
       summary.setActionRan(true);
     } else {
       numCached++;
     }

     if (result.isCached()) {
       numLocalActionCached++;
     }

     Path coverageData = result.getCoverageData();
     if (coverageData != null) {
       summary.addCoverageFiles(ImmutableList.of(coverageData));
     }

     TransitiveInfoCollection target = existingSummary.getTarget();
     Preconditions.checkNotNull(target, "The existing TestSummary must be associated with a target");
     TestParams testParams = target.getProvider(TestProvider.class).getTestParams();

     if (!testParams.runsDetectsFlakes()) {
       status = aggregateStatus(status, result.getData().getStatus());
     } else {
       int shardNumber = result.getShardNum();
       int runsPerTestForLabel = testParams.getRuns();
       List<BlazeTestStatus> singleShardStatuses =
           summary.addShardStatus(shardNumber, result.getData().getStatus());
       if (singleShardStatuses.size() == runsPerTestForLabel) {
         // Aggregation is based on the order of status enums where larger values take precedence
         // over smaller ones (NO_STATUS = 0, PASSED = 1, etc.). However, there are some special
         // cases:
         // 1. Tests that have some passing, some not passing shard are marked as flaky
         // 2. The INCOMPLETE status is ignored - it is used for tests runs that are cancelled by
         //    Bazel if --cancel_concurrent_tests is set, otherwise INCOMPLETE is not used
         // 3. Individual test shards can be FLAKY if the test is marked flaky and
         //    --flaky_test_attempts is not zero for this test
         BlazeTestStatus shardStatus = BlazeTestStatus.NO_STATUS;
         int passes = 0;
         int cancelled = 0;
         for (BlazeTestStatus runStatusForShard : singleShardStatuses) {
           if (runStatusForShard == BlazeTestStatus.INCOMPLETE) {
             // If runs_per_test_detects_flakes is enabled, then INCOMPLETE status indicates
             // cancelled test runs. We count them separately so that they don't result in a
             // flaky status below.
             cancelled++;
           } else {
             shardStatus = aggregateStatus(shardStatus, runStatusForShard);
             if (TestResult.isBlazeTestStatusPassed(runStatusForShard)) {
               passes++;
             }
           }
         }
         // Under the RunsPerTestDetectsFlakes option, return flaky if 0 < p < (n-cancelled) shards
         // pass. Otherwise, we aggregate the shardStatus.
         if (passes == 0 || (passes + cancelled) == runsPerTestForLabel) {
           status = aggregateStatus(status, shardStatus);
         } else {
           status = aggregateStatus(status, BlazeTestStatus.FLAKY);
         }
       }
     }

     if (result.getData().hasPassedLog()) {
       summary.addPassedLog(result.getTestLogPath().getRelative(result.getData().getPassedLog()));
     }
     for (String path : result.getData().getFailedLogsList()) {
       summary.addFailedLog(result.getTestLogPath().getRelative(path));
     }

     summary
         .addTestTimes(result.getData().getTestTimesList())
         .mergeTiming(
             result.getData().getStartTimeMillisEpoch(), result.getData().getRunDurationMillis())
         .addWarnings(result.getData().getWarningList())
         .collectTestCases(result.getData().hasTestCase() ? result.getData().getTestCase() : null)
         .setRanRemotely(result.getData().getIsRemoteStrategy());

     List<String> warnings = new ArrayList<>();
     if (status == BlazeTestStatus.PASSED
         && shouldEmitTestSizeWarningInSummary(
             policy.testVerboseTimeoutWarnings,
             warnings,
             result.getData().getTestProcessTimesList(),
             target)) {
       summary.setWasUnreportedWrongSize(true);
     }

     summary
         .setStatus(status)
         .setNumCached(numCached)
         .setNumLocalActionCached(numLocalActionCached)
         .addWarnings(warnings);
   }

   private void markIncomplete(boolean skipTargetsOnFailure) {
     // TODO(bazel-team): (2010) Make NotRunTestResult support both tests failed to built and
     // tests with no status and post it here.
     TestSummary peekSummary = summary.peek();
     BlazeTestStatus status = peekSummary.getStatus();
     if (skipTargetsOnFailure) {
       status = BlazeTestStatus.NO_STATUS;
     } else if (status != BlazeTestStatus.NO_STATUS) {
       status = aggregateStatus(status, BlazeTestStatus.INCOMPLETE);
     }

     summary.setStatus(status);
   }

   private void markUnbuilt(boolean blazeHalted, boolean skipTargetsOnFailure) {
     BlazeTestStatus runStatus =
         blazeHalted
             ? BlazeTestStatus.BLAZE_HALTED_BEFORE_TESTING
             : (policy.testCheckUpToDate || skipTargetsOnFailure
                 ? BlazeTestStatus.NO_STATUS
                 : BlazeTestStatus.FAILED_TO_BUILD);

     summary.setStatus(runStatus);
   }

   /**
    * Checks whether the specified test timeout could have been smaller or is too small and adds a
    * warning message if verbose is true.
    *
    * <p>Returns true if there was a test with the wrong timeout, but if was not reported.
    */
   private static boolean shouldEmitTestSizeWarningInSummary(
       boolean verbose,
       List<String> warnings,
       List<Long> testTimes,
       TransitiveInfoCollection target) {

     TestTimeout specifiedTimeout =
         target.getProvider(TestProvider.class).getTestParams().getTimeout();
     long maxTimeOfShard = 0;

     for (Long shardTime : testTimes) {
       if (shardTime != null) {
         maxTimeOfShard = Math.max(maxTimeOfShard, shardTime);
       }
     }

     int maxTimeInSeconds = (int) (maxTimeOfShard / 1000);

     if (!specifiedTimeout.isInRangeFuzzy(maxTimeInSeconds)) {
       TestTimeout expectedTimeout = TestTimeout.getSuggestedTestTimeout(maxTimeInSeconds);
       TestSize expectedSize = TestSize.getTestSize(expectedTimeout);
       if (verbose) {
         StringBuilder builder =
             new StringBuilder(
                 String.format(
                     "%s: Test execution time (%.1fs excluding execution overhead) outside of "
                         + "range for %s tests. Consider setting timeout=\"%s\"",
                     AliasProvider.getDependencyLabel(target),
                     maxTimeOfShard / 1000.0,
                     specifiedTimeout.prettyPrint(),
                     expectedTimeout));
         if (expectedSize != null) {
           builder.append(" or size=\"").append(expectedSize).append("\"");
         }
         builder.append(".");
         warnings.add(builder.toString());
         return false;
       }
       return true;
     } else {
       return false;
     }
   }
 }
	// Copyright 2019 The Bazel Authors. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package com.google.devtools.build.lib.runtime;

	import com.google.common.base.Preconditions;
	import com.google.common.collect.ImmutableList;
	import com.google.common.collect.ImmutableMap;
	import com.google.common.collect.ImmutableSet;
	import com.google.common.eventbus.EventBus;
	import com.google.devtools.build.lib.actions.ActionOwner;
	import com.google.devtools.build.lib.actions.Artifact;
	import com.google.devtools.build.lib.analysis.AliasProvider;
	import com.google.devtools.build.lib.analysis.ConfiguredTarget;
	import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
	import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
	import com.google.devtools.build.lib.analysis.test.TestProvider;
	import com.google.devtools.build.lib.analysis.test.TestProvider.TestParams;
	import com.google.devtools.build.lib.analysis.test.TestResult;
	import com.google.devtools.build.lib.concurrent.ThreadSafety;
	import com.google.devtools.build.lib.exec.TestAttempt;
	import com.google.devtools.build.lib.packages.TestSize;
	import com.google.devtools.build.lib.packages.TestTimeout;
	import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey;
	import com.google.devtools.build.lib.vfs.Path;
	import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
	import java.util.ArrayList;
	import java.util.Collection;
	import java.util.HashMap;
	import java.util.HashSet;
	import java.util.List;
	import java.util.Map;
	import java.util.Set;

	/** This class aggregates and reports target-wide test statuses in real-time. */
	@ThreadSafety.ThreadSafe
	final class TestResultAggregator {
	/**
	* Settings for the aggregator; there are usually many aggregator instances with the same set of
	* settings, so we move them to a separate object.
	*/
	static final class AggregationPolicy {
	private final EventBus eventBus;
	private final boolean testCheckUpToDate;
	private final boolean testVerboseTimeoutWarnings;

	AggregationPolicy(
	EventBus eventBus, boolean testCheckUpToDate, boolean testVerboseTimeoutWarnings) {
	this.eventBus = eventBus;
	this.testCheckUpToDate = testCheckUpToDate;
	this.testVerboseTimeoutWarnings = testVerboseTimeoutWarnings;
	}
	}

	private final AggregationPolicy policy;
	private final ConfiguredTarget testTarget;
	private final TestSummary.Builder summary;
	private final Set<Artifact> remainingRuns;
	private final Map<Artifact, TestResult> statusMap = new HashMap<>();

	public TestResultAggregator(
	ConfiguredTarget target, BuildConfiguration configuration, AggregationPolicy policy) {
	this.testTarget = target;
	this.policy = policy;

	// And create an empty summary suitable for incremental analysis.
	// Also has the nice side effect of mapping labels to RuleConfiguredTargets.
	this.summary = TestSummary.newBuilder();
	this.summary.setTarget(target);
	if (configuration != null) {
	// This can be null for testing.
	this.summary.setConfiguration(configuration);
	}
	this.summary.setStatus(BlazeTestStatus.NO_STATUS);
	this.remainingRuns = new HashSet<>(TestProvider.getTestStatusArtifacts(target));
	}

	/**
	* Records a new test run result and incrementally updates the target status. This event is sent
	* upon completion of executed test runs.
	*/
	synchronized void testEvent(TestResult result) {
	ActionOwner testOwner = result.getTestAction().getOwner();
	ConfiguredTargetKey targetLabel =
	ConfiguredTargetKey.of(testOwner.getLabel(), result.getTestAction().getConfiguration());
	Preconditions.checkArgument(targetLabel.equals(asKey(testTarget)));

	TestResult previousResult = statusMap.put(result.getTestStatusArtifact(), result);
	if (previousResult != null) {
	throw new IllegalStateException(
	String.format(
	"Duplicate result reported for an individual test shard %s.\nNew: %s\nPrevious: %s",
	result.getTestStatusArtifact(), result.getData(), previousResult.getData()));
	}

	// If a test result was cached, then post the cached attempts to the event bus.
	if (result.isCached()) {
	for (TestAttempt attempt : result.getCachedTestAttempts()) {
	policy.eventBus.post(attempt);
	}
	}

	TestSummary finalTestSummary = null;
	Preconditions.checkNotNull(summary);
	if (!remainingRuns.remove(result.getTestStatusArtifact())) {
	// This can happen if a buildCompleteEvent() was processed before this event reached us.
	// This situation is likely to happen if --notest_keep_going is set with multiple targets.
	return;
	}

	incrementalAnalyze(result);

	// If all runs are processed, the target is finished and ready to report.
	if (remainingRuns.isEmpty()) {
	finalTestSummary = summary.build();
	}

	// Report finished targets.
	if (finalTestSummary != null) {
	policy.eventBus.post(finalTestSummary);
	}
	}

	synchronized void targetFailure(boolean blazeHalted, boolean skipTargetsOnFailure) {
	if (remainingRuns.isEmpty()) {
	// Blaze does not guarantee that BuildResult.getSuccessfulTargets() and posted TestResult
	// events are in sync. Thus, it is possible that a test event was posted, but the target is
	// not present in the set of successful targets.
	return;
	}

	markUnbuilt(blazeHalted, skipTargetsOnFailure);

	// These are never going to run; removing them marks the target complete.
	remainingRuns.clear();
	policy.eventBus.post(summary.build());
	}

	/** Returns the known aggregate results for the given target at the current moment. */
	synchronized TestSummary.Builder getCurrentSummaryForTesting() {
	return summary;
	}

	/**
	* Returns all test status artifacts associated with a given target whose runs have yet to finish.
	*/
	synchronized Collection<Artifact> getIncompleteRunsForTesting() {
	return ImmutableSet.copyOf(remainingRuns);
	}

	synchronized Map<Artifact, TestResult> getStatusMapForTesting() {
	return ImmutableMap.copyOf(statusMap);
	}

	private static ConfiguredTargetKey asKey(ConfiguredTarget target) {
	return ConfiguredTargetKey.of(
	// A test is never in the host configuration.
	AliasProvider.getDependencyLabel(target),
	target.getConfigurationKey(),
	/isHostConfiguration=/ false);
	}

	private static BlazeTestStatus aggregateStatus(BlazeTestStatus status, BlazeTestStatus other) {
	return status.getNumber() > other.getNumber() ? status : other;
	}

	/**
	* Helper for differential analysis which aggregates the TestSummary for an individual target,
	* reporting runs on the EventBus if necessary.
	*/
	synchronized TestSummary aggregateAndReportSummary(boolean skipTargetsOnFailure) {
	// If already reported by the listener, no work remains for this target.
	if (remainingRuns.isEmpty()) {
	return summary.build();
	}

	// We will get back multiple TestResult instances if test had to be retried several
	// times before passing. Sharding and multiple runs of the same test without retries
	// will be represented by separate artifacts and will produce exactly one TestResult.
	for (Artifact testStatus : TestProvider.getTestStatusArtifacts(testTarget)) {
	// When a build is interrupted ( eg. a broken target with --nokeep_going ) runResult could
	// be null for an unrelated test because we were not able to even try to execute the test.
	// In that case, for tests that were previously passing we return null ( == NO STATUS),
	// because checking if the cached test target is up-to-date would require running the
	// dependency checker transitively.
	TestResult runResult = statusMap.get(testStatus);
	boolean isIncompleteRun = remainingRuns.contains(testStatus);
	if (runResult == null) {
	markIncomplete(skipTargetsOnFailure);
	} else if (isIncompleteRun) {
	incrementalAnalyze(runResult);
	}
	}

	// The target was not posted by the listener and must be posted now.
	TestSummary result = summary.build();
	policy.eventBus.post(result);
	return result;
	}

	/**
	* Incrementally updates a TestSummary given an existing summary and a new TestResult. Only call
	* on built targets.
	*
	* @param result New test result to aggregate into the summary.
	*/
	synchronized void incrementalAnalyze(TestResult result) {
	// Cache retrieval should have been performed already.
	Preconditions.checkNotNull(result);
	TestSummary existingSummary = Preconditions.checkNotNull(summary.peek());

	BlazeTestStatus status = existingSummary.getStatus();
	int numCached = existingSummary.numCached();
	int numLocalActionCached = existingSummary.numLocalActionCached();

	// If a test was neither cached locally nor remotely we say action was taken.
	if (!(result.isCached() \|\| result.getData().getRemotelyCached())) {
	summary.setActionRan(true);
	} else {
	numCached++;
	}

	if (result.isCached()) {
	numLocalActionCached++;
	}

	Path coverageData = result.getCoverageData();
	if (coverageData != null) {
	summary.addCoverageFiles(ImmutableList.of(coverageData));
	}

	TransitiveInfoCollection target = existingSummary.getTarget();
	Preconditions.checkNotNull(target, "The existing TestSummary must be associated with a target");
	TestParams testParams = target.getProvider(TestProvider.class).getTestParams();

	if (!testParams.runsDetectsFlakes()) {
	status = aggregateStatus(status, result.getData().getStatus());
	} else {
	int shardNumber = result.getShardNum();
	int runsPerTestForLabel = testParams.getRuns();
	List<BlazeTestStatus> singleShardStatuses =
	summary.addShardStatus(shardNumber, result.getData().getStatus());
	if (singleShardStatuses.size() == runsPerTestForLabel) {
	// Aggregation is based on the order of status enums where larger values take precedence
	// over smaller ones (NO_STATUS = 0, PASSED = 1, etc.). However, there are some special
	// cases:
	// 1. Tests that have some passing, some not passing shard are marked as flaky
	// 2. The INCOMPLETE status is ignored - it is used for tests runs that are cancelled by
	// Bazel if --cancel_concurrent_tests is set, otherwise INCOMPLETE is not used
	// 3. Individual test shards can be FLAKY if the test is marked flaky and
	// --flaky_test_attempts is not zero for this test
	BlazeTestStatus shardStatus = BlazeTestStatus.NO_STATUS;
	int passes = 0;
	int cancelled = 0;
	for (BlazeTestStatus runStatusForShard : singleShardStatuses) {
	if (runStatusForShard == BlazeTestStatus.INCOMPLETE) {
	// If runs_per_test_detects_flakes is enabled, then INCOMPLETE status indicates
	// cancelled test runs. We count them separately so that they don't result in a
	// flaky status below.
	cancelled++;
	} else {
	shardStatus = aggregateStatus(shardStatus, runStatusForShard);
	if (TestResult.isBlazeTestStatusPassed(runStatusForShard)) {
	passes++;
	}
	}
	}
	// Under the RunsPerTestDetectsFlakes option, return flaky if 0 < p < (n-cancelled) shards
	// pass. Otherwise, we aggregate the shardStatus.
	if (passes == 0 \|\| (passes + cancelled) == runsPerTestForLabel) {
	status = aggregateStatus(status, shardStatus);
	} else {
	status = aggregateStatus(status, BlazeTestStatus.FLAKY);
	}
	}
	}

	if (result.getData().hasPassedLog()) {
	summary.addPassedLog(result.getTestLogPath().getRelative(result.getData().getPassedLog()));
	}
	for (String path : result.getData().getFailedLogsList()) {
	summary.addFailedLog(result.getTestLogPath().getRelative(path));
	}

	summary
	.addTestTimes(result.getData().getTestTimesList())
	.mergeTiming(
	result.getData().getStartTimeMillisEpoch(), result.getData().getRunDurationMillis())
	.addWarnings(result.getData().getWarningList())
	.collectTestCases(result.getData().hasTestCase() ? result.getData().getTestCase() : null)
	.setRanRemotely(result.getData().getIsRemoteStrategy());

	List<String> warnings = new ArrayList<>();
	if (status == BlazeTestStatus.PASSED
	&& shouldEmitTestSizeWarningInSummary(
	policy.testVerboseTimeoutWarnings,
	warnings,
	result.getData().getTestProcessTimesList(),
	target)) {
	summary.setWasUnreportedWrongSize(true);
	}

	summary
	.setStatus(status)
	.setNumCached(numCached)
	.setNumLocalActionCached(numLocalActionCached)
	.addWarnings(warnings);
	}

	private void markIncomplete(boolean skipTargetsOnFailure) {
	// TODO(bazel-team): (2010) Make NotRunTestResult support both tests failed to built and
	// tests with no status and post it here.
	TestSummary peekSummary = summary.peek();
	BlazeTestStatus status = peekSummary.getStatus();
	if (skipTargetsOnFailure) {
	status = BlazeTestStatus.NO_STATUS;
	} else if (status != BlazeTestStatus.NO_STATUS) {
	status = aggregateStatus(status, BlazeTestStatus.INCOMPLETE);
	}

	summary.setStatus(status);
	}

	private void markUnbuilt(boolean blazeHalted, boolean skipTargetsOnFailure) {
	BlazeTestStatus runStatus =
	blazeHalted
	? BlazeTestStatus.BLAZE_HALTED_BEFORE_TESTING
	: (policy.testCheckUpToDate \|\| skipTargetsOnFailure
	? BlazeTestStatus.NO_STATUS
	: BlazeTestStatus.FAILED_TO_BUILD);

	summary.setStatus(runStatus);
	}

	/**
	* Checks whether the specified test timeout could have been smaller or is too small and adds a
	* warning message if verbose is true.
	*
	* <p>Returns true if there was a test with the wrong timeout, but if was not reported.
	*/
	private static boolean shouldEmitTestSizeWarningInSummary(
	boolean verbose,
	List<String> warnings,
	List<Long> testTimes,
	TransitiveInfoCollection target) {

	TestTimeout specifiedTimeout =
	target.getProvider(TestProvider.class).getTestParams().getTimeout();
	long maxTimeOfShard = 0;

	for (Long shardTime : testTimes) {
	if (shardTime != null) {
	maxTimeOfShard = Math.max(maxTimeOfShard, shardTime);
	}
	}

	int maxTimeInSeconds = (int) (maxTimeOfShard / 1000);

	if (!specifiedTimeout.isInRangeFuzzy(maxTimeInSeconds)) {
	TestTimeout expectedTimeout = TestTimeout.getSuggestedTestTimeout(maxTimeInSeconds);
	TestSize expectedSize = TestSize.getTestSize(expectedTimeout);
	if (verbose) {
	StringBuilder builder =
	new StringBuilder(
	String.format(
	"%s: Test execution time (%.1fs excluding execution overhead) outside of "
	+ "range for %s tests. Consider setting timeout=\"%s\"",
	AliasProvider.getDependencyLabel(target),
	maxTimeOfShard / 1000.0,
	specifiedTimeout.prettyPrint(),
	expectedTimeout));
	if (expectedSize != null) {
	builder.append(" or size=\"").append(expectedSize).append("\"");
	}
	builder.append(".");
	warnings.add(builder.toString());
	return false;
	}
	return true;
	} else {
	return false;
	}
	}
	}