src/main/java/com/google/devtools/build/lib/runtime/TestResultAnalyzer.java - bazel - Git at Google

 // Copyright 2014 The Bazel Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 package com.google.devtools.build.lib.runtime;

 import com.google.common.base.Preconditions;
 import com.google.common.collect.Sets;
 import com.google.common.eventbus.EventBus;
 import com.google.common.eventbus.Subscribe;
 import com.google.devtools.build.lib.actions.Artifact;
 import com.google.devtools.build.lib.analysis.AliasProvider;
 import com.google.devtools.build.lib.analysis.ConfiguredTarget;
 import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
 import com.google.devtools.build.lib.analysis.test.TestProvider;
 import com.google.devtools.build.lib.analysis.test.TestResult;
 import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent;
 import com.google.devtools.build.lib.cmdline.Label;
 import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
 import com.google.devtools.build.lib.exec.ExecutionOptions;
 import com.google.devtools.build.lib.packages.TestSize;
 import com.google.devtools.build.lib.packages.TestTimeout;
 import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier.TestSummaryOptions;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;

 /**
  * Prints results to the terminal, showing the results of each test target.
  */
 @ThreadCompatible
 public class TestResultAnalyzer {
   private final TestSummaryOptions summaryOptions;
   private final ExecutionOptions executionOptions;
   private final EventBus eventBus;

   // Store information about potential failures in the presence of --nokeep_going or
   // --notest_keep_going.
   private boolean skipTargetsOnFailure;

   /**
    * @param summaryOptions Parsed test summarization options.
    * @param executionOptions Parsed build/test execution options.
    * @param eventBus For reporting failed to build and cached tests.
    */
   public TestResultAnalyzer(TestSummaryOptions summaryOptions,
                             ExecutionOptions executionOptions,
                             EventBus eventBus) {
     this.summaryOptions = summaryOptions;
     this.executionOptions = executionOptions;
     this.eventBus = eventBus;
     eventBus.register(this);
   }

   @Subscribe
   public void doneBuild(BuildCompleteEvent event) {
     skipTargetsOnFailure = event.getResult().getStopOnFirstFailure();
   }

   /**
    * Prints out the results of the given tests, and returns true if they all passed.
    * Posts any targets which weren't already completed by the listener to the EventBus.
    * Reports all targets on the console via the given notifier.
    * Run at the end of the build, run only once.
    *
    * @param testTargets The list of targets being run
    * @param listener An aggregating listener with intermediate results
    * @param notifier A console notifier to echo results to.
    * @return true if all the tests passed, else false
    */
   public boolean differentialAnalyzeAndReport(
       Collection<ConfiguredTarget> testTargets,
       Collection<ConfiguredTarget> skippedTargets,
       AggregatingTestListener listener,
       TestResultNotifier notifier) {

     Preconditions.checkNotNull(testTargets);
     Preconditions.checkNotNull(listener);
     Preconditions.checkNotNull(notifier);

     // The natural ordering of the summaries defines their output order.
     Set<TestSummary> summaries = Sets.newTreeSet();

     int totalRun = 0; // Number of targets running at least one non-cached test.
     int passCount = 0;

     for (ConfiguredTarget testTarget : testTargets) {
       TestSummary summary = aggregateAndReportSummary(testTarget, listener);
       summaries.add(summary);

       // Finished aggregating; build the final console output.
       if (summary.actionRan()) {
         totalRun++;
       }

       if (TestResult.isBlazeTestStatusPassed(summary.getStatus())) {
         passCount++;
       }
     }

     int summarySize = summaries.size();
     int testTargetsSize = testTargets.size();
     Preconditions.checkState(
         summarySize == testTargetsSize,
         "Unequal sizes: %s vs %s (%s and %s)",
         summarySize,
         testTargetsSize,
         summaries,
         testTargets);

     notifier.notify(summaries, totalRun);
     // skipped targets are not in passCount since they have NO_STATUS
     Set<ConfiguredTarget> testTargetsSet = new HashSet<>(testTargets);
     Set<ConfiguredTarget> skippedTargetsSet = new HashSet<>(skippedTargets);
     return passCount == Sets.difference(testTargetsSet, skippedTargetsSet).size();
   }

   private static BlazeTestStatus aggregateStatus(BlazeTestStatus status, BlazeTestStatus other) {
     return status.getNumber() > other.getNumber() ? status : other;
   }

   /**
    * Helper for differential analysis which aggregates the TestSummary for an individual target,
    * reporting runs on the EventBus if necessary.
    */
   private TestSummary aggregateAndReportSummary(
       ConfiguredTarget testTarget, AggregatingTestListener listener) {

     // If already reported by the listener, no work remains for this target.
     TestSummary.Builder summary = listener.getCurrentSummary(testTarget);
     Label testLabel = AliasProvider.getDependencyLabel(testTarget);
     Preconditions.checkNotNull(summary,
         "%s did not complete test filtering, but has a test result", testLabel);
     if (listener.targetReported(testTarget)) {
       return summary.build();
     }

     Collection<Artifact> incompleteRuns = listener.getIncompleteRuns(testTarget);
     Map<Artifact, TestResult> statusMap = listener.getStatusMap();

     // We will get back multiple TestResult instances if test had to be retried several
     // times before passing. Sharding and multiple runs of the same test without retries
     // will be represented by separate artifacts and will produce exactly one TestResult.
     for (Artifact testStatus : TestProvider.getTestStatusArtifacts(testTarget)) {
       // When a build is interrupted ( eg. a broken target with --nokeep_going ) runResult could
       // be null for an unrelated test because we were not able to even try to execute the test.
       // In that case, for tests that were previously passing we return null ( == NO STATUS),
       // because checking if the cached test target is up-to-date would require running the
       // dependency checker transitively.
       TestResult runResult = statusMap.get(testStatus);
       boolean isIncompleteRun = incompleteRuns.contains(testStatus);
       if (runResult == null) {
         summary = markIncomplete(summary);
       } else if (isIncompleteRun) {
         // Only process results which were not recorded by the listener.

         boolean newlyFetched = !statusMap.containsKey(testStatus);
         summary = incrementalAnalyze(summary, runResult);
         if (newlyFetched) {
           eventBus.post(runResult);
         }
         Preconditions.checkState(
             listener.getIncompleteRuns(testTarget).contains(testStatus) == isIncompleteRun,
             "TestListener changed in differential analysis. Ensure it isn't still registered.");
       }
     }

     // The target was not posted by the listener and must be posted now.
     TestSummary result = summary.build();
     eventBus.post(result);
     return result;
   }

   /**
    * Incrementally updates a TestSummary given an existing summary
    * and a new TestResult. Only call on built targets.
    *
    * @param summaryBuilder Existing unbuilt test summary associated with a target.
    * @param result New test result to aggregate into the summary.
    * @return The updated TestSummary.
    */
   public TestSummary.Builder incrementalAnalyze(TestSummary.Builder summaryBuilder,
                                                 TestResult result) {
     // Cache retrieval should have been performed already.
     Preconditions.checkNotNull(result);
     Preconditions.checkNotNull(summaryBuilder);
     TestSummary existingSummary = Preconditions.checkNotNull(summaryBuilder.peek());

     BlazeTestStatus status = existingSummary.getStatus();
     int numCached = existingSummary.numCached();
     int numLocalActionCached = existingSummary.numLocalActionCached();

     // If a test was neither cached locally nor remotely we say action was taken.
     if (!(result.isCached() || result.getData().getRemotelyCached())) {
       summaryBuilder.setActionRan(true);
     } else {
       numCached++;
     }

     if (result.isCached()) {
       numLocalActionCached++;
     }

     Path coverageData = result.getCoverageData();
     if (coverageData != null) {
       summaryBuilder.addCoverageFiles(Collections.singletonList(coverageData));
     }

     TransitiveInfoCollection target = existingSummary.getTarget();
     Preconditions.checkNotNull(target, "The existing TestSummary must be associated with a target");

     if (!executionOptions.runsPerTestDetectsFlakes) {
       status = aggregateStatus(status, result.getData().getStatus());
     } else {
       int shardNumber = result.getShardNum();
       int runsPerTestForLabel = target.getProvider(TestProvider.class).getTestParams().getRuns();
       List<BlazeTestStatus> singleShardStatuses = summaryBuilder.addShardStatus(
           shardNumber, result.getData().getStatus());
       if (singleShardStatuses.size() == runsPerTestForLabel) {
         BlazeTestStatus shardStatus = BlazeTestStatus.NO_STATUS;
         int passes = 0;
         for (BlazeTestStatus runStatusForShard : singleShardStatuses) {
           shardStatus = aggregateStatus(shardStatus, runStatusForShard);
           if (TestResult.isBlazeTestStatusPassed(runStatusForShard)) {
             passes++;
           }
         }
         // Under the RunsPerTestDetectsFlakes option, return flaky if 1 <= p < n shards pass.
         // If all results pass or fail, aggregate the passing/failing shardStatus.
         if (passes == 0 || passes == runsPerTestForLabel) {
           status = aggregateStatus(status, shardStatus);
         } else {
           status = aggregateStatus(status, BlazeTestStatus.FLAKY);
         }
       }
     }

     if (result.getData().hasPassedLog()) {
       summaryBuilder.addPassedLog(
           result.getTestLogPath().getRelative(result.getData().getPassedLog()));
     }
     for (String path : result.getData().getFailedLogsList()) {
       summaryBuilder.addFailedLog(result.getTestLogPath().getRelative(path));
     }

     summaryBuilder
         .addTestTimes(result.getData().getTestTimesList())
         .addWarnings(result.getData().getWarningList())
         .collectFailedTests(result.getData().getTestCase())
         .countTotalTestCases(result.getData().getTestCase())
         .setRanRemotely(result.getData().getIsRemoteStrategy());

     List<String> warnings = new ArrayList<>();
     if (status == BlazeTestStatus.PASSED
         && shouldEmitTestSizeWarningInSummary(
             summaryOptions.testVerboseTimeoutWarnings,
             warnings,
             result.getData().getTestProcessTimesList(),
             target)) {
       summaryBuilder.setWasUnreportedWrongSize(true);
     }

     return summaryBuilder
         .setStatus(status)
         .setNumCached(numCached)
         .setNumLocalActionCached(numLocalActionCached)
         .addWarnings(warnings);
   }

   private TestSummary.Builder markIncomplete(TestSummary.Builder summaryBuilder) {
     // TODO(bazel-team): (2010) Make NotRunTestResult support both tests failed to built and
     // tests with no status and post it here.
     TestSummary summary = summaryBuilder.peek();
     BlazeTestStatus status = summary.getStatus();
     if (skipTargetsOnFailure) {
       status = BlazeTestStatus.NO_STATUS;
     } else if (status != BlazeTestStatus.NO_STATUS) {
       status = aggregateStatus(status, BlazeTestStatus.INCOMPLETE);
     }

     return summaryBuilder.setStatus(status);
   }

   TestSummary.Builder markUnbuilt(TestSummary.Builder summary, boolean blazeHalted) {
     BlazeTestStatus runStatus =
         blazeHalted
             ? BlazeTestStatus.BLAZE_HALTED_BEFORE_TESTING
             : (executionOptions.testCheckUpToDate || skipTargetsOnFailure
                 ? BlazeTestStatus.NO_STATUS
                 : BlazeTestStatus.FAILED_TO_BUILD);

     return summary.setStatus(runStatus);
   }

   /**
    * Checks whether the specified test timeout could have been smaller or is too small and adds a
    * warning message if verbose is true.
    *
    * <p>Returns true if there was a test with the wrong timeout, but if was not reported.
    */
   private static boolean shouldEmitTestSizeWarningInSummary(
       boolean verbose,
       List<String> warnings,
       List<Long> testTimes,
       TransitiveInfoCollection target) {

     TestTimeout specifiedTimeout =
         target.getProvider(TestProvider.class).getTestParams().getTimeout();
     long maxTimeOfShard = 0;

     for (Long shardTime : testTimes) {
       if (shardTime != null) {
         maxTimeOfShard = Math.max(maxTimeOfShard, shardTime);
       }
     }

     int maxTimeInSeconds = (int) (maxTimeOfShard / 1000);

     if (!specifiedTimeout.isInRangeFuzzy(maxTimeInSeconds)) {
       TestTimeout expectedTimeout = TestTimeout.getSuggestedTestTimeout(maxTimeInSeconds);
       TestSize expectedSize = TestSize.getTestSize(expectedTimeout);
       if (verbose) {
         StringBuilder builder = new StringBuilder(String.format(
             "%s: Test execution time (%.1fs excluding execution overhead) outside of "
             + "range for %s tests. Consider setting timeout=\"%s\"",
             AliasProvider.getDependencyLabel(target),
             maxTimeOfShard / 1000.0,
             specifiedTimeout.prettyPrint(),
             expectedTimeout));
         if (expectedSize != null) {
           builder.append(" or size=\"").append(expectedSize).append("\"");
         }
         builder.append(".");
         warnings.add(builder.toString());
         return false;
       }
       return true;
     } else {
       return false;
     }
   }
 }
	// Copyright 2014 The Bazel Authors. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.
	package com.google.devtools.build.lib.runtime;

	import com.google.common.base.Preconditions;
	import com.google.common.collect.Sets;
	import com.google.common.eventbus.EventBus;
	import com.google.common.eventbus.Subscribe;
	import com.google.devtools.build.lib.actions.Artifact;
	import com.google.devtools.build.lib.analysis.AliasProvider;
	import com.google.devtools.build.lib.analysis.ConfiguredTarget;
	import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
	import com.google.devtools.build.lib.analysis.test.TestProvider;
	import com.google.devtools.build.lib.analysis.test.TestResult;
	import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent;
	import com.google.devtools.build.lib.cmdline.Label;
	import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
	import com.google.devtools.build.lib.exec.ExecutionOptions;
	import com.google.devtools.build.lib.packages.TestSize;
	import com.google.devtools.build.lib.packages.TestTimeout;
	import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier.TestSummaryOptions;
	import com.google.devtools.build.lib.vfs.Path;
	import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
	import java.util.ArrayList;
	import java.util.Collection;
	import java.util.Collections;
	import java.util.HashSet;
	import java.util.List;
	import java.util.Map;
	import java.util.Set;

	/**
	* Prints results to the terminal, showing the results of each test target.
	*/
	@ThreadCompatible
	public class TestResultAnalyzer {
	private final TestSummaryOptions summaryOptions;
	private final ExecutionOptions executionOptions;
	private final EventBus eventBus;

	// Store information about potential failures in the presence of --nokeep_going or
	// --notest_keep_going.
	private boolean skipTargetsOnFailure;

	/**
	* @param summaryOptions Parsed test summarization options.
	* @param executionOptions Parsed build/test execution options.
	* @param eventBus For reporting failed to build and cached tests.
	*/
	public TestResultAnalyzer(TestSummaryOptions summaryOptions,
	ExecutionOptions executionOptions,
	EventBus eventBus) {
	this.summaryOptions = summaryOptions;
	this.executionOptions = executionOptions;
	this.eventBus = eventBus;
	eventBus.register(this);
	}

	@Subscribe
	public void doneBuild(BuildCompleteEvent event) {
	skipTargetsOnFailure = event.getResult().getStopOnFirstFailure();
	}

	/**
	* Prints out the results of the given tests, and returns true if they all passed.
	* Posts any targets which weren't already completed by the listener to the EventBus.
	* Reports all targets on the console via the given notifier.
	* Run at the end of the build, run only once.
	*
	* @param testTargets The list of targets being run
	* @param listener An aggregating listener with intermediate results
	* @param notifier A console notifier to echo results to.
	* @return true if all the tests passed, else false
	*/
	public boolean differentialAnalyzeAndReport(
	Collection<ConfiguredTarget> testTargets,
	Collection<ConfiguredTarget> skippedTargets,
	AggregatingTestListener listener,
	TestResultNotifier notifier) {

	Preconditions.checkNotNull(testTargets);
	Preconditions.checkNotNull(listener);
	Preconditions.checkNotNull(notifier);

	// The natural ordering of the summaries defines their output order.
	Set<TestSummary> summaries = Sets.newTreeSet();

	int totalRun = 0; // Number of targets running at least one non-cached test.
	int passCount = 0;

	for (ConfiguredTarget testTarget : testTargets) {
	TestSummary summary = aggregateAndReportSummary(testTarget, listener);
	summaries.add(summary);

	// Finished aggregating; build the final console output.
	if (summary.actionRan()) {
	totalRun++;
	}

	if (TestResult.isBlazeTestStatusPassed(summary.getStatus())) {
	passCount++;
	}
	}

	int summarySize = summaries.size();
	int testTargetsSize = testTargets.size();
	Preconditions.checkState(
	summarySize == testTargetsSize,
	"Unequal sizes: %s vs %s (%s and %s)",
	summarySize,
	testTargetsSize,
	summaries,
	testTargets);

	notifier.notify(summaries, totalRun);
	// skipped targets are not in passCount since they have NO_STATUS
	Set<ConfiguredTarget> testTargetsSet = new HashSet<>(testTargets);
	Set<ConfiguredTarget> skippedTargetsSet = new HashSet<>(skippedTargets);
	return passCount == Sets.difference(testTargetsSet, skippedTargetsSet).size();
	}

	private static BlazeTestStatus aggregateStatus(BlazeTestStatus status, BlazeTestStatus other) {
	return status.getNumber() > other.getNumber() ? status : other;
	}

	/**
	* Helper for differential analysis which aggregates the TestSummary for an individual target,
	* reporting runs on the EventBus if necessary.
	*/
	private TestSummary aggregateAndReportSummary(
	ConfiguredTarget testTarget, AggregatingTestListener listener) {

	// If already reported by the listener, no work remains for this target.
	TestSummary.Builder summary = listener.getCurrentSummary(testTarget);
	Label testLabel = AliasProvider.getDependencyLabel(testTarget);
	Preconditions.checkNotNull(summary,
	"%s did not complete test filtering, but has a test result", testLabel);
	if (listener.targetReported(testTarget)) {
	return summary.build();
	}

	Collection<Artifact> incompleteRuns = listener.getIncompleteRuns(testTarget);
	Map<Artifact, TestResult> statusMap = listener.getStatusMap();

	// We will get back multiple TestResult instances if test had to be retried several
	// times before passing. Sharding and multiple runs of the same test without retries
	// will be represented by separate artifacts and will produce exactly one TestResult.
	for (Artifact testStatus : TestProvider.getTestStatusArtifacts(testTarget)) {
	// When a build is interrupted ( eg. a broken target with --nokeep_going ) runResult could
	// be null for an unrelated test because we were not able to even try to execute the test.
	// In that case, for tests that were previously passing we return null ( == NO STATUS),
	// because checking if the cached test target is up-to-date would require running the
	// dependency checker transitively.
	TestResult runResult = statusMap.get(testStatus);
	boolean isIncompleteRun = incompleteRuns.contains(testStatus);
	if (runResult == null) {
	summary = markIncomplete(summary);
	} else if (isIncompleteRun) {
	// Only process results which were not recorded by the listener.

	boolean newlyFetched = !statusMap.containsKey(testStatus);
	summary = incrementalAnalyze(summary, runResult);
	if (newlyFetched) {
	eventBus.post(runResult);
	}
	Preconditions.checkState(
	listener.getIncompleteRuns(testTarget).contains(testStatus) == isIncompleteRun,
	"TestListener changed in differential analysis. Ensure it isn't still registered.");
	}
	}

	// The target was not posted by the listener and must be posted now.
	TestSummary result = summary.build();
	eventBus.post(result);
	return result;
	}

	/**
	* Incrementally updates a TestSummary given an existing summary
	* and a new TestResult. Only call on built targets.
	*
	* @param summaryBuilder Existing unbuilt test summary associated with a target.
	* @param result New test result to aggregate into the summary.
	* @return The updated TestSummary.
	*/
	public TestSummary.Builder incrementalAnalyze(TestSummary.Builder summaryBuilder,
	TestResult result) {
	// Cache retrieval should have been performed already.
	Preconditions.checkNotNull(result);
	Preconditions.checkNotNull(summaryBuilder);
	TestSummary existingSummary = Preconditions.checkNotNull(summaryBuilder.peek());

	BlazeTestStatus status = existingSummary.getStatus();
	int numCached = existingSummary.numCached();
	int numLocalActionCached = existingSummary.numLocalActionCached();

	// If a test was neither cached locally nor remotely we say action was taken.
	if (!(result.isCached() \|\| result.getData().getRemotelyCached())) {
	summaryBuilder.setActionRan(true);
	} else {
	numCached++;
	}

	if (result.isCached()) {
	numLocalActionCached++;
	}

	Path coverageData = result.getCoverageData();
	if (coverageData != null) {
	summaryBuilder.addCoverageFiles(Collections.singletonList(coverageData));
	}

	TransitiveInfoCollection target = existingSummary.getTarget();
	Preconditions.checkNotNull(target, "The existing TestSummary must be associated with a target");

	if (!executionOptions.runsPerTestDetectsFlakes) {
	status = aggregateStatus(status, result.getData().getStatus());
	} else {
	int shardNumber = result.getShardNum();
	int runsPerTestForLabel = target.getProvider(TestProvider.class).getTestParams().getRuns();
	List<BlazeTestStatus> singleShardStatuses = summaryBuilder.addShardStatus(
	shardNumber, result.getData().getStatus());
	if (singleShardStatuses.size() == runsPerTestForLabel) {
	BlazeTestStatus shardStatus = BlazeTestStatus.NO_STATUS;
	int passes = 0;
	for (BlazeTestStatus runStatusForShard : singleShardStatuses) {
	shardStatus = aggregateStatus(shardStatus, runStatusForShard);
	if (TestResult.isBlazeTestStatusPassed(runStatusForShard)) {
	passes++;
	}
	}
	// Under the RunsPerTestDetectsFlakes option, return flaky if 1 <= p < n shards pass.
	// If all results pass or fail, aggregate the passing/failing shardStatus.
	if (passes == 0 \|\| passes == runsPerTestForLabel) {
	status = aggregateStatus(status, shardStatus);
	} else {
	status = aggregateStatus(status, BlazeTestStatus.FLAKY);
	}
	}
	}

	if (result.getData().hasPassedLog()) {
	summaryBuilder.addPassedLog(
	result.getTestLogPath().getRelative(result.getData().getPassedLog()));
	}
	for (String path : result.getData().getFailedLogsList()) {
	summaryBuilder.addFailedLog(result.getTestLogPath().getRelative(path));
	}

	summaryBuilder
	.addTestTimes(result.getData().getTestTimesList())
	.addWarnings(result.getData().getWarningList())
	.collectFailedTests(result.getData().getTestCase())
	.countTotalTestCases(result.getData().getTestCase())
	.setRanRemotely(result.getData().getIsRemoteStrategy());

	List<String> warnings = new ArrayList<>();
	if (status == BlazeTestStatus.PASSED
	&& shouldEmitTestSizeWarningInSummary(
	summaryOptions.testVerboseTimeoutWarnings,
	warnings,
	result.getData().getTestProcessTimesList(),
	target)) {
	summaryBuilder.setWasUnreportedWrongSize(true);
	}

	return summaryBuilder
	.setStatus(status)
	.setNumCached(numCached)
	.setNumLocalActionCached(numLocalActionCached)
	.addWarnings(warnings);
	}

	private TestSummary.Builder markIncomplete(TestSummary.Builder summaryBuilder) {
	// TODO(bazel-team): (2010) Make NotRunTestResult support both tests failed to built and
	// tests with no status and post it here.
	TestSummary summary = summaryBuilder.peek();
	BlazeTestStatus status = summary.getStatus();
	if (skipTargetsOnFailure) {
	status = BlazeTestStatus.NO_STATUS;
	} else if (status != BlazeTestStatus.NO_STATUS) {
	status = aggregateStatus(status, BlazeTestStatus.INCOMPLETE);
	}

	return summaryBuilder.setStatus(status);
	}

	TestSummary.Builder markUnbuilt(TestSummary.Builder summary, boolean blazeHalted) {
	BlazeTestStatus runStatus =
	blazeHalted
	? BlazeTestStatus.BLAZE_HALTED_BEFORE_TESTING
	: (executionOptions.testCheckUpToDate \|\| skipTargetsOnFailure
	? BlazeTestStatus.NO_STATUS
	: BlazeTestStatus.FAILED_TO_BUILD);

	return summary.setStatus(runStatus);
	}

	/**
	* Checks whether the specified test timeout could have been smaller or is too small and adds a
	* warning message if verbose is true.
	*
	* <p>Returns true if there was a test with the wrong timeout, but if was not reported.
	*/
	private static boolean shouldEmitTestSizeWarningInSummary(
	boolean verbose,
	List<String> warnings,
	List<Long> testTimes,
	TransitiveInfoCollection target) {

	TestTimeout specifiedTimeout =
	target.getProvider(TestProvider.class).getTestParams().getTimeout();
	long maxTimeOfShard = 0;

	for (Long shardTime : testTimes) {
	if (shardTime != null) {
	maxTimeOfShard = Math.max(maxTimeOfShard, shardTime);
	}
	}

	int maxTimeInSeconds = (int) (maxTimeOfShard / 1000);

	if (!specifiedTimeout.isInRangeFuzzy(maxTimeInSeconds)) {
	TestTimeout expectedTimeout = TestTimeout.getSuggestedTestTimeout(maxTimeInSeconds);
	TestSize expectedSize = TestSize.getTestSize(expectedTimeout);
	if (verbose) {
	StringBuilder builder = new StringBuilder(String.format(
	"%s: Test execution time (%.1fs excluding execution overhead) outside of "
	+ "range for %s tests. Consider setting timeout=\"%s\"",
	AliasProvider.getDependencyLabel(target),
	maxTimeOfShard / 1000.0,
	specifiedTimeout.prettyPrint(),
	expectedTimeout));
	if (expectedSize != null) {
	builder.append(" or size=\"").append(expectedSize).append("\"");
	}
	builder.append(".");
	warnings.add(builder.toString());
	return false;
	}
	return true;
	} else {
	return false;
	}
	}
	}