src/main/java/com/google/devtools/build/lib/runtime/TestResultAggregator.java - bazel - Git at Google

 // Copyright 2019 The Bazel Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package com.google.devtools.build.lib.runtime;

 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.eventbus.EventBus;
 import com.google.devtools.build.lib.analysis.AliasProvider;
 import com.google.devtools.build.lib.analysis.ConfiguredTarget;
 import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
 import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue;
 import com.google.devtools.build.lib.analysis.test.TestAttempt;
 import com.google.devtools.build.lib.analysis.test.TestProvider;
 import com.google.devtools.build.lib.analysis.test.TestProvider.TestParams;
 import com.google.devtools.build.lib.analysis.test.TestResult;
 import com.google.devtools.build.lib.concurrent.ThreadSafety;
 import com.google.devtools.build.lib.packages.TestSize;
 import com.google.devtools.build.lib.packages.TestTimeout;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
 import java.util.ArrayList;
 import java.util.List;

 /** This class aggregates and reports target-wide test statuses in real-time. */
 @ThreadSafety.ThreadSafe
 final class TestResultAggregator {
   /**
    * Settings for the aggregator; there are usually many aggregator instances with the same set of
    * settings, so we move them to a separate object.
    */
   static final class AggregationPolicy {
     private final EventBus eventBus;
     private final boolean testCheckUpToDate;
     private final boolean testVerboseTimeoutWarnings;

     AggregationPolicy(
         EventBus eventBus, boolean testCheckUpToDate, boolean testVerboseTimeoutWarnings) {
       this.eventBus = eventBus;
       this.testCheckUpToDate = testCheckUpToDate;
       this.testVerboseTimeoutWarnings = testVerboseTimeoutWarnings;
     }
   }

   private final AggregationPolicy policy;
   private final TestSummary.Builder summary;
   private int remainingRuns;
   private boolean summaryPosted = false;

   TestResultAggregator(
       ConfiguredTarget target,
       BuildConfigurationValue configuration,
       AggregationPolicy policy,
       boolean skippedThisTest) {
     this.policy = policy;
     this.summary =
         TestSummary.newBuilder(target)
             .setConfiguration(configuration)
             .setStatus(BlazeTestStatus.NO_STATUS)
             .setSkipped(skippedThisTest);
     this.remainingRuns = TestProvider.getTestStatusArtifacts(target).size();
   }

   /**
    * Records a new test run result and incrementally updates the target status. This event is sent
    * upon completion of executed test runs.
    */
   synchronized void testEvent(TestResult result) {
     // If a test result was cached, then post the cached attempts to the event bus.
     if (result.isCached()) {
       for (TestAttempt attempt : result.getCachedTestAttempts()) {
         policy.eventBus.post(attempt);
       }
     }

     remainingRuns--;
     if (summaryPosted) {
       // This can happen if a buildCompleteEvent() was processed before this event reached us.
       // This situation is likely to happen if --notest_keep_going is set with multiple targets.
       return;
     }

     incrementalAnalyze(result);

     // If all runs are processed, the target is finished and ready to report.
     if (remainingRuns == 0) {
       postSummary();
     }
   }

   private TestSummary postSummary() {
     TestSummary result = summary.build();
     policy.eventBus.post(result);
     summaryPosted = true;
     return result;
   }

   synchronized void targetFailure(boolean blazeHalted, boolean skipTargetsOnFailure) {
     if (summaryPosted) {
       // Blaze does not guarantee that BuildResult.getSuccessfulTargets() and posted TestResult
       // events are in sync. Thus, it is possible that a test event was posted, but the target is
       // not present in the set of successful targets.
       return;
     }

     markUnbuilt(blazeHalted, skipTargetsOnFailure);

     // These are never going to run; removing them marks the target complete.
     postSummary();
   }

   synchronized void targetSkipped() {
     if (summaryPosted) {
       // Blaze does not guarantee that BuildResult.getSuccessfulTargets() and posted TestResult
       // events are in sync. Thus, it is possible that a test event was posted, but the target is
       // not present in the set of successful targets.
       return;
     }

     summary.setStatus(BlazeTestStatus.NO_STATUS);

     // These are never going to run; removing them marks the target complete.
     postSummary();
   }

   static BlazeTestStatus aggregateStatus(BlazeTestStatus status, BlazeTestStatus other) {
     return status.getNumber() > other.getNumber() ? status : other;
   }

   /**
    * Helper for differential analysis which aggregates the TestSummary for an individual target,
    * reporting runs on the EventBus if necessary.
    */
   synchronized TestSummary aggregateAndReportSummary(boolean skipTargetsOnFailure) {
     // If already reported by the listener, no work remains for this target.
     if (summaryPosted) {
       return summary.build(); // Reuses the same summary if nothing has changed.
     }

     // Build may have been interrupted.
     if (remainingRuns > 0) {
       markIncomplete(skipTargetsOnFailure);
     }

     return postSummary();
   }

   /**
    * Incrementally updates a TestSummary given an existing summary and a new TestResult. Only call
    * on built targets.
    *
    * @param result New test result to aggregate into the summary.
    */
   private void incrementalAnalyze(TestResult result) {
     // Cache retrieval should have been performed already.
     TestSummary existingSummary = Preconditions.checkNotNull(summary.peek());

     BlazeTestStatus status = existingSummary.getStatus();
     int numCached = existingSummary.numCached();
     int numLocalActionCached = existingSummary.numLocalActionCached();

     // If a test was neither cached locally nor remotely we say action was taken.
     if (!(result.isCached() || result.getData().getRemotelyCached())) {
       summary.setActionRan(true);
     } else {
       numCached++;
     }

     if (result.isCached()) {
       numLocalActionCached++;
     }

     Path coverageData = result.getCoverageData();
     if (coverageData != null) {
       summary.addCoverageFiles(ImmutableList.of(coverageData));
     }

     TransitiveInfoCollection target = existingSummary.getTarget();
     Preconditions.checkNotNull(target, "The existing TestSummary must be associated with a target");
     TestParams testParams = target.getProvider(TestProvider.class).getTestParams();

     int shardNumber = result.getShardNum();
     summary.addShardAttempts(shardNumber, result.getData().getTestTimesCount());

     if (!testParams.runsDetectsFlakes()) {
       status = aggregateStatus(status, result.getData().getStatus());
     } else {
       int runsPerTestForLabel = testParams.getRuns();
       List<BlazeTestStatus> singleShardStatuses =
           summary.addShardStatus(shardNumber, result.getData().getStatus());
       if (singleShardStatuses.size() == runsPerTestForLabel) {
         // Aggregation is based on the order of status enums where larger values take precedence
         // over smaller ones (NO_STATUS = 0, PASSED = 1, etc.). However, there are some special
         // cases:
         // 1. Tests that have some passing, some not passing shard are marked as flaky
         // 2. The INCOMPLETE status is ignored - it is used for tests runs that are cancelled by
         //    Bazel if --cancel_concurrent_tests is set, otherwise INCOMPLETE is not used
         // 3. Individual test shards can be FLAKY if the test is marked flaky and
         //    --flaky_test_attempts is not zero for this test
         BlazeTestStatus shardStatus = BlazeTestStatus.NO_STATUS;
         int passes = 0;
         int cancelled = 0;
         for (BlazeTestStatus runStatusForShard : singleShardStatuses) {
           if (runStatusForShard == BlazeTestStatus.INCOMPLETE) {
             // If runs_per_test_detects_flakes is enabled, then INCOMPLETE status indicates
             // cancelled test runs. We count them separately so that they don't result in a
             // flaky status below.
             cancelled++;
           } else {
             shardStatus = aggregateStatus(shardStatus, runStatusForShard);
             if (TestResult.isBlazeTestStatusPassed(runStatusForShard)) {
               passes++;
             }
           }
         }
         // Under the RunsPerTestDetectsFlakes option, return flaky if 0 < p < (n-cancelled) shards
         // pass. Otherwise, we aggregate the shardStatus.
         if (passes == 0 || (passes + cancelled) == runsPerTestForLabel) {
           status = aggregateStatus(status, shardStatus);
         } else {
           status = aggregateStatus(status, BlazeTestStatus.FLAKY);
         }
       }
     }

     if (result.getData().hasPassedLog()) {
       summary.addPassedLog(result.getTestLogPath().getRelative(result.getData().getPassedLog()));
     }
     for (String path : result.getData().getFailedLogsList()) {
       summary.addFailedLog(result.getTestLogPath().getRelative(path));
     }

     summary
         .addTestTimes(result.getData().getTestTimesList())
         .mergeTiming(
             result.getData().getStartTimeMillisEpoch(), result.getData().getRunDurationMillis())
         .addWarnings(result.getData().getWarningList())
         .collectTestCases(result.getData().hasTestCase() ? result.getData().getTestCase() : null)
         .setRanRemotely(result.getData().getIsRemoteStrategy());

     List<String> warnings = new ArrayList<>();
     if (status == BlazeTestStatus.PASSED
         && shouldEmitTestSizeWarningInSummary(
             policy.testVerboseTimeoutWarnings,
             warnings,
             result.getData().getTestProcessTimesList(),
             target)) {
       summary.setWasUnreportedWrongSize(true);
     }

     summary
         .mergeSystemFailure(result.getSystemFailure())
         .setStatus(status)
         .setNumCached(numCached)
         .setNumLocalActionCached(numLocalActionCached)
         .addWarnings(warnings);
   }

   private void markIncomplete(boolean skipTargetsOnFailure) {
     // TODO(bazel-team): (2010) Make NotRunTestResult support both tests failed to built and
     // tests with no status and post it here.
     TestSummary peekSummary = summary.peek();
     BlazeTestStatus status = peekSummary.getStatus();
     if (skipTargetsOnFailure) {
       status = BlazeTestStatus.NO_STATUS;
     } else if (status != BlazeTestStatus.NO_STATUS) {
       status = aggregateStatus(status, BlazeTestStatus.INCOMPLETE);
     }

     summary.setStatus(status);
   }

   private void markUnbuilt(boolean blazeHalted, boolean skipTargetsOnFailure) {
     BlazeTestStatus runStatus =
         blazeHalted
             ? BlazeTestStatus.BLAZE_HALTED_BEFORE_TESTING
             : (policy.testCheckUpToDate || skipTargetsOnFailure
                 ? BlazeTestStatus.NO_STATUS
                 : BlazeTestStatus.FAILED_TO_BUILD);

     summary.setStatus(runStatus);
   }

   /**
    * Checks whether the specified test timeout could have been smaller or is too small and adds a
    * warning message if verbose is true.
    *
    * <p>Returns true if there was a test with the wrong timeout, but if was not reported.
    */
   private static boolean shouldEmitTestSizeWarningInSummary(
       boolean verbose,
       List<String> warnings,
       List<Long> testTimes,
       TransitiveInfoCollection target) {

     TestTimeout specifiedTimeout =
         target.getProvider(TestProvider.class).getTestParams().getTimeout();
     long maxTimeOfShard = 0;

     for (Long shardTime : testTimes) {
       if (shardTime != null) {
         maxTimeOfShard = Math.max(maxTimeOfShard, shardTime);
       }
     }

     int maxTimeInSeconds = (int) (maxTimeOfShard / 1000);

     if (!specifiedTimeout.isInRangeFuzzy(maxTimeInSeconds)) {
       TestTimeout expectedTimeout = TestTimeout.getSuggestedTestTimeout(maxTimeInSeconds);
       TestSize expectedSize = TestSize.getTestSize(expectedTimeout);
       if (verbose) {
         StringBuilder builder =
             new StringBuilder(
                 String.format(
                     "%s: Test execution time (%.1fs excluding execution overhead) outside of "
                         + "range for %s tests. Consider setting timeout=\"%s\"",
                     AliasProvider.getDependencyLabel(target),
                     maxTimeOfShard / 1000.0,
                     specifiedTimeout.prettyPrint(),
                     expectedTimeout));
         if (expectedSize != null) {
           builder.append(" or size=\"").append(expectedSize).append("\"");
         }
         builder.append(".");
         warnings.add(builder.toString());
         return false;
       }
       return true;
     } else {
       return false;
     }
   }
 }
	// Copyright 2019 The Bazel Authors. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package com.google.devtools.build.lib.runtime;

	import com.google.common.base.Preconditions;
	import com.google.common.collect.ImmutableList;
	import com.google.common.eventbus.EventBus;
	import com.google.devtools.build.lib.analysis.AliasProvider;
	import com.google.devtools.build.lib.analysis.ConfiguredTarget;
	import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
	import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue;
	import com.google.devtools.build.lib.analysis.test.TestAttempt;
	import com.google.devtools.build.lib.analysis.test.TestProvider;
	import com.google.devtools.build.lib.analysis.test.TestProvider.TestParams;
	import com.google.devtools.build.lib.analysis.test.TestResult;
	import com.google.devtools.build.lib.concurrent.ThreadSafety;
	import com.google.devtools.build.lib.packages.TestSize;
	import com.google.devtools.build.lib.packages.TestTimeout;
	import com.google.devtools.build.lib.vfs.Path;
	import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
	import java.util.ArrayList;
	import java.util.List;

	/** This class aggregates and reports target-wide test statuses in real-time. */
	@ThreadSafety.ThreadSafe
	final class TestResultAggregator {
	/**
	* Settings for the aggregator; there are usually many aggregator instances with the same set of
	* settings, so we move them to a separate object.
	*/
	static final class AggregationPolicy {
	private final EventBus eventBus;
	private final boolean testCheckUpToDate;
	private final boolean testVerboseTimeoutWarnings;

	AggregationPolicy(
	EventBus eventBus, boolean testCheckUpToDate, boolean testVerboseTimeoutWarnings) {
	this.eventBus = eventBus;
	this.testCheckUpToDate = testCheckUpToDate;
	this.testVerboseTimeoutWarnings = testVerboseTimeoutWarnings;
	}
	}

	private final AggregationPolicy policy;
	private final TestSummary.Builder summary;
	private int remainingRuns;
	private boolean summaryPosted = false;

	TestResultAggregator(
	ConfiguredTarget target,
	BuildConfigurationValue configuration,
	AggregationPolicy policy,
	boolean skippedThisTest) {
	this.policy = policy;
	this.summary =
	TestSummary.newBuilder(target)
	.setConfiguration(configuration)
	.setStatus(BlazeTestStatus.NO_STATUS)
	.setSkipped(skippedThisTest);
	this.remainingRuns = TestProvider.getTestStatusArtifacts(target).size();
	}

	/**
	* Records a new test run result and incrementally updates the target status. This event is sent
	* upon completion of executed test runs.
	*/
	synchronized void testEvent(TestResult result) {
	// If a test result was cached, then post the cached attempts to the event bus.
	if (result.isCached()) {
	for (TestAttempt attempt : result.getCachedTestAttempts()) {
	policy.eventBus.post(attempt);
	}
	}

	remainingRuns--;
	if (summaryPosted) {
	// This can happen if a buildCompleteEvent() was processed before this event reached us.
	// This situation is likely to happen if --notest_keep_going is set with multiple targets.
	return;
	}

	incrementalAnalyze(result);

	// If all runs are processed, the target is finished and ready to report.
	if (remainingRuns == 0) {
	postSummary();
	}
	}

	private TestSummary postSummary() {
	TestSummary result = summary.build();
	policy.eventBus.post(result);
	summaryPosted = true;
	return result;
	}

	synchronized void targetFailure(boolean blazeHalted, boolean skipTargetsOnFailure) {
	if (summaryPosted) {
	// Blaze does not guarantee that BuildResult.getSuccessfulTargets() and posted TestResult
	// events are in sync. Thus, it is possible that a test event was posted, but the target is
	// not present in the set of successful targets.
	return;
	}

	markUnbuilt(blazeHalted, skipTargetsOnFailure);

	// These are never going to run; removing them marks the target complete.
	postSummary();
	}

	synchronized void targetSkipped() {
	if (summaryPosted) {
	// Blaze does not guarantee that BuildResult.getSuccessfulTargets() and posted TestResult
	// events are in sync. Thus, it is possible that a test event was posted, but the target is
	// not present in the set of successful targets.
	return;
	}

	summary.setStatus(BlazeTestStatus.NO_STATUS);

	// These are never going to run; removing them marks the target complete.
	postSummary();
	}

	static BlazeTestStatus aggregateStatus(BlazeTestStatus status, BlazeTestStatus other) {
	return status.getNumber() > other.getNumber() ? status : other;
	}

	/**
	* Helper for differential analysis which aggregates the TestSummary for an individual target,
	* reporting runs on the EventBus if necessary.
	*/
	synchronized TestSummary aggregateAndReportSummary(boolean skipTargetsOnFailure) {
	// If already reported by the listener, no work remains for this target.
	if (summaryPosted) {
	return summary.build(); // Reuses the same summary if nothing has changed.
	}

	// Build may have been interrupted.
	if (remainingRuns > 0) {
	markIncomplete(skipTargetsOnFailure);
	}

	return postSummary();
	}

	/**
	* Incrementally updates a TestSummary given an existing summary and a new TestResult. Only call
	* on built targets.
	*
	* @param result New test result to aggregate into the summary.
	*/
	private void incrementalAnalyze(TestResult result) {
	// Cache retrieval should have been performed already.
	TestSummary existingSummary = Preconditions.checkNotNull(summary.peek());

	BlazeTestStatus status = existingSummary.getStatus();
	int numCached = existingSummary.numCached();
	int numLocalActionCached = existingSummary.numLocalActionCached();

	// If a test was neither cached locally nor remotely we say action was taken.
	if (!(result.isCached() \|\| result.getData().getRemotelyCached())) {
	summary.setActionRan(true);
	} else {
	numCached++;
	}

	if (result.isCached()) {
	numLocalActionCached++;
	}

	Path coverageData = result.getCoverageData();
	if (coverageData != null) {
	summary.addCoverageFiles(ImmutableList.of(coverageData));
	}

	TransitiveInfoCollection target = existingSummary.getTarget();
	Preconditions.checkNotNull(target, "The existing TestSummary must be associated with a target");
	TestParams testParams = target.getProvider(TestProvider.class).getTestParams();

	int shardNumber = result.getShardNum();
	summary.addShardAttempts(shardNumber, result.getData().getTestTimesCount());

	if (!testParams.runsDetectsFlakes()) {
	status = aggregateStatus(status, result.getData().getStatus());
	} else {
	int runsPerTestForLabel = testParams.getRuns();
	List<BlazeTestStatus> singleShardStatuses =
	summary.addShardStatus(shardNumber, result.getData().getStatus());
	if (singleShardStatuses.size() == runsPerTestForLabel) {
	// Aggregation is based on the order of status enums where larger values take precedence
	// over smaller ones (NO_STATUS = 0, PASSED = 1, etc.). However, there are some special
	// cases:
	// 1. Tests that have some passing, some not passing shard are marked as flaky
	// 2. The INCOMPLETE status is ignored - it is used for tests runs that are cancelled by
	// Bazel if --cancel_concurrent_tests is set, otherwise INCOMPLETE is not used
	// 3. Individual test shards can be FLAKY if the test is marked flaky and
	// --flaky_test_attempts is not zero for this test
	BlazeTestStatus shardStatus = BlazeTestStatus.NO_STATUS;
	int passes = 0;
	int cancelled = 0;
	for (BlazeTestStatus runStatusForShard : singleShardStatuses) {
	if (runStatusForShard == BlazeTestStatus.INCOMPLETE) {
	// If runs_per_test_detects_flakes is enabled, then INCOMPLETE status indicates
	// cancelled test runs. We count them separately so that they don't result in a
	// flaky status below.
	cancelled++;
	} else {
	shardStatus = aggregateStatus(shardStatus, runStatusForShard);
	if (TestResult.isBlazeTestStatusPassed(runStatusForShard)) {
	passes++;
	}
	}
	}
	// Under the RunsPerTestDetectsFlakes option, return flaky if 0 < p < (n-cancelled) shards
	// pass. Otherwise, we aggregate the shardStatus.
	if (passes == 0 \|\| (passes + cancelled) == runsPerTestForLabel) {
	status = aggregateStatus(status, shardStatus);
	} else {
	status = aggregateStatus(status, BlazeTestStatus.FLAKY);
	}
	}
	}

	if (result.getData().hasPassedLog()) {
	summary.addPassedLog(result.getTestLogPath().getRelative(result.getData().getPassedLog()));
	}
	for (String path : result.getData().getFailedLogsList()) {
	summary.addFailedLog(result.getTestLogPath().getRelative(path));
	}

	summary
	.addTestTimes(result.getData().getTestTimesList())
	.mergeTiming(
	result.getData().getStartTimeMillisEpoch(), result.getData().getRunDurationMillis())
	.addWarnings(result.getData().getWarningList())
	.collectTestCases(result.getData().hasTestCase() ? result.getData().getTestCase() : null)
	.setRanRemotely(result.getData().getIsRemoteStrategy());

	List<String> warnings = new ArrayList<>();
	if (status == BlazeTestStatus.PASSED
	&& shouldEmitTestSizeWarningInSummary(
	policy.testVerboseTimeoutWarnings,
	warnings,
	result.getData().getTestProcessTimesList(),
	target)) {
	summary.setWasUnreportedWrongSize(true);
	}

	summary
	.mergeSystemFailure(result.getSystemFailure())
	.setStatus(status)
	.setNumCached(numCached)
	.setNumLocalActionCached(numLocalActionCached)
	.addWarnings(warnings);
	}

	private void markIncomplete(boolean skipTargetsOnFailure) {
	// TODO(bazel-team): (2010) Make NotRunTestResult support both tests failed to built and
	// tests with no status and post it here.
	TestSummary peekSummary = summary.peek();
	BlazeTestStatus status = peekSummary.getStatus();
	if (skipTargetsOnFailure) {
	status = BlazeTestStatus.NO_STATUS;
	} else if (status != BlazeTestStatus.NO_STATUS) {
	status = aggregateStatus(status, BlazeTestStatus.INCOMPLETE);
	}

	summary.setStatus(status);
	}

	private void markUnbuilt(boolean blazeHalted, boolean skipTargetsOnFailure) {
	BlazeTestStatus runStatus =
	blazeHalted
	? BlazeTestStatus.BLAZE_HALTED_BEFORE_TESTING
	: (policy.testCheckUpToDate \|\| skipTargetsOnFailure
	? BlazeTestStatus.NO_STATUS
	: BlazeTestStatus.FAILED_TO_BUILD);

	summary.setStatus(runStatus);
	}

	/**
	* Checks whether the specified test timeout could have been smaller or is too small and adds a
	* warning message if verbose is true.
	*
	* <p>Returns true if there was a test with the wrong timeout, but if was not reported.
	*/
	private static boolean shouldEmitTestSizeWarningInSummary(
	boolean verbose,
	List<String> warnings,
	List<Long> testTimes,
	TransitiveInfoCollection target) {

	TestTimeout specifiedTimeout =
	target.getProvider(TestProvider.class).getTestParams().getTimeout();
	long maxTimeOfShard = 0;

	for (Long shardTime : testTimes) {
	if (shardTime != null) {
	maxTimeOfShard = Math.max(maxTimeOfShard, shardTime);
	}
	}

	int maxTimeInSeconds = (int) (maxTimeOfShard / 1000);

	if (!specifiedTimeout.isInRangeFuzzy(maxTimeInSeconds)) {
	TestTimeout expectedTimeout = TestTimeout.getSuggestedTestTimeout(maxTimeInSeconds);
	TestSize expectedSize = TestSize.getTestSize(expectedTimeout);
	if (verbose) {
	StringBuilder builder =
	new StringBuilder(
	String.format(
	"%s: Test execution time (%.1fs excluding execution overhead) outside of "
	+ "range for %s tests. Consider setting timeout=\"%s\"",
	AliasProvider.getDependencyLabel(target),
	maxTimeOfShard / 1000.0,
	specifiedTimeout.prettyPrint(),
	expectedTimeout));
	if (expectedSize != null) {
	builder.append(" or size=\"").append(expectedSize).append("\"");
	}
	builder.append(".");
	warnings.add(builder.toString());
	return false;
	}
	return true;
	} else {
	return false;
	}
	}
	}