blob: 66ecb01b2e6ef02a198b6ade2cea33810486fcfb [file] [log] [blame]
// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.exec;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Verify;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.io.ByteStreams;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionInputHelper;
import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
import com.google.devtools.build.lib.actions.ArtifactPathResolver;
import com.google.devtools.build.lib.actions.EnvironmentalExecException;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.ExecutionRequirements;
import com.google.devtools.build.lib.actions.SimpleSpawn;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.SpawnMetrics;
import com.google.devtools.build.lib.actions.SpawnResult;
import com.google.devtools.build.lib.actions.TestExecException;
import com.google.devtools.build.lib.analysis.actions.SpawnAction;
import com.google.devtools.build.lib.analysis.test.TestAttempt;
import com.google.devtools.build.lib.analysis.test.TestResult;
import com.google.devtools.build.lib.analysis.test.TestRunnerAction;
import com.google.devtools.build.lib.analysis.test.TestRunnerAction.ResolvedPaths;
import com.google.devtools.build.lib.analysis.test.TestStrategy;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.TestResult.ExecutionInfo;
import com.google.devtools.build.lib.buildeventstream.TestFileNameConstants;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.collect.nestedset.Order;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.server.FailureDetails.Execution.Code;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.server.FailureDetails.TestAction;
import com.google.devtools.build.lib.util.Pair;
import com.google.devtools.build.lib.util.io.FileOutErr;
import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
import com.google.devtools.build.lib.view.test.TestStatus.TestCase;
import com.google.devtools.build.lib.view.test.TestStatus.TestResultData;
import com.google.protobuf.util.Durations;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
/** Runs TestRunnerAction actions. */
// TODO(bazel-team): add tests for this strategy.
public class StandaloneTestStrategy extends TestStrategy {
private static final ImmutableMap<String, String> ENV_VARS =
ImmutableMap.<String, String>builder()
.put("TZ", "UTC")
.put("TEST_SRCDIR", TestPolicy.RUNFILES_DIR)
// TODO(lberki): Remove JAVA_RUNFILES and PYTHON_RUNFILES.
.put("JAVA_RUNFILES", TestPolicy.RUNFILES_DIR)
.put("PYTHON_RUNFILES", TestPolicy.RUNFILES_DIR)
.put("RUNFILES_DIR", TestPolicy.RUNFILES_DIR)
.put("TEST_TMPDIR", TestPolicy.TEST_TMP_DIR)
.put("RUN_UNDER_RUNFILES", "1")
.build();
public static final TestPolicy DEFAULT_LOCAL_POLICY = new TestPolicy(ENV_VARS);
protected final Path tmpDirRoot;
public StandaloneTestStrategy(
ExecutionOptions executionOptions, BinTools binTools, Path tmpDirRoot) {
super(executionOptions, binTools);
this.tmpDirRoot = tmpDirRoot;
}
@Override
public TestRunnerSpawn createTestRunnerSpawn(
TestRunnerAction action, ActionExecutionContext actionExecutionContext)
throws ExecException, InterruptedException {
if (action.getExecutionSettings().getInputManifest() == null) {
throw createTestExecException(
TestAction.Code.LOCAL_TEST_PREREQ_UNMET,
"cannot run local tests with --nobuild_runfile_manifests");
}
Map<String, String> testEnvironment =
createEnvironment(
actionExecutionContext, action, tmpDirRoot, executionOptions.splitXmlGeneration);
Map<String, String> executionInfo =
new TreeMap<>(action.getTestProperties().getExecutionInfo());
if (!action.shouldCacheResult()) {
executionInfo.put(ExecutionRequirements.NO_CACHE, "");
}
executionInfo.put(ExecutionRequirements.TIMEOUT, "" + getTimeout(action).getSeconds());
SimpleSpawn.LocalResourcesSupplier localResourcesSupplier =
() ->
action
.getTestProperties()
.getLocalResourceUsage(
action.getOwner().getLabel(), executionOptions.usingLocalTestJobs());
Spawn spawn =
new SimpleSpawn(
action,
getArgs(action),
ImmutableMap.copyOf(testEnvironment),
ImmutableMap.copyOf(executionInfo),
action.getRunfilesSupplier(),
ImmutableMap.of(),
/*inputs=*/ action.getInputs(),
NestedSetBuilder.emptySet(Order.STABLE_ORDER),
ImmutableSet.copyOf(action.getSpawnOutputs()),
/*mandatoryOutputs=*/ ImmutableSet.of(),
localResourcesSupplier);
Path execRoot = actionExecutionContext.getExecRoot();
ArtifactPathResolver pathResolver = actionExecutionContext.getPathResolver();
Path runfilesDir = pathResolver.convertPath(action.getExecutionSettings().getRunfilesDir());
Path tmpDir = pathResolver.convertPath(tmpDirRoot.getChild(TestStrategy.getTmpDirName(action)));
Path workingDirectory = runfilesDir.getRelative(action.getRunfilesPrefix());
return new StandaloneTestRunnerSpawn(
action, actionExecutionContext, spawn, tmpDir, workingDirectory, execRoot);
}
private static ImmutableList<Pair<String, Path>> renameOutputs(
ActionExecutionContext actionExecutionContext,
TestRunnerAction action,
ImmutableList<Pair<String, Path>> testOutputs,
int attemptId)
throws IOException {
// Rename outputs
String namePrefix =
FileSystemUtils.removeExtension(action.getTestLog().getExecPath().getBaseName());
Path testRoot = actionExecutionContext.getInputPath(action.getTestLog()).getParentDirectory();
Path attemptsDir = testRoot.getChild(namePrefix + "_attempts");
attemptsDir.createDirectory();
String attemptPrefix = "attempt_" + attemptId;
Path testLog = attemptsDir.getChild(attemptPrefix + ".log");
// Get the normal test output paths, and then update them to use "attempt_N" names, and
// attemptDir, before adding them to the outputs.
ImmutableList.Builder<Pair<String, Path>> testOutputsBuilder = new ImmutableList.Builder<>();
for (Pair<String, Path> testOutput : testOutputs) {
// e.g. /testRoot/test.dir/file, an example we follow throughout this loop's comments.
Path testOutputPath = testOutput.getSecond();
Path destinationPath;
if (testOutput.getFirst().equals(TestFileNameConstants.TEST_LOG)) {
// The rename rules for the test log are different than for all the other files.
destinationPath = testLog;
} else {
// e.g. test.dir/file
PathFragment relativeToTestDirectory = testOutputPath.relativeTo(testRoot);
// e.g. attempt_1.dir/file
String destinationPathFragmentStr =
relativeToTestDirectory.getSafePathString().replaceFirst("test", attemptPrefix);
PathFragment destinationPathFragment = PathFragment.create(destinationPathFragmentStr);
// e.g. /attemptsDir/attempt_1.dir/file
destinationPath = attemptsDir.getRelative(destinationPathFragment);
destinationPath.getParentDirectory().createDirectory();
}
// Move to the destination.
testOutputPath.renameTo(destinationPath);
testOutputsBuilder.add(Pair.of(testOutput.getFirst(), destinationPath));
}
return testOutputsBuilder.build();
}
private StandaloneFailedAttemptResult processFailedTestAttempt(
int attemptId,
ActionExecutionContext actionExecutionContext,
TestRunnerAction action,
StandaloneTestResult result)
throws IOException {
return processTestAttempt(
attemptId, /*isLastAttempt=*/ false, actionExecutionContext, action, result);
}
private void finalizeTest(
TestRunnerAction action,
ActionExecutionContext actionExecutionContext,
StandaloneTestResult standaloneTestResult,
List<FailedAttemptResult> failedAttempts)
throws IOException {
processTestAttempt(
failedAttempts.size() + 1,
/*isLastAttempt=*/ true,
actionExecutionContext,
action,
standaloneTestResult);
TestResultData.Builder dataBuilder = standaloneTestResult.testResultDataBuilder();
for (FailedAttemptResult failedAttempt : failedAttempts) {
TestResultData failedAttemptData =
((StandaloneFailedAttemptResult) failedAttempt).testResultData;
dataBuilder.addAllFailedLogs(failedAttemptData.getFailedLogsList());
dataBuilder.addTestTimes(failedAttemptData.getTestTimes(0));
dataBuilder.addAllTestProcessTimes(failedAttemptData.getTestProcessTimesList());
}
if (dataBuilder.getStatus() == BlazeTestStatus.PASSED && !failedAttempts.isEmpty()) {
dataBuilder.setStatus(BlazeTestStatus.FLAKY);
}
TestResultData data = dataBuilder.build();
TestResult result =
new TestResult(action, data, false, standaloneTestResult.primarySystemFailure());
postTestResult(actionExecutionContext, result);
}
private StandaloneFailedAttemptResult processTestAttempt(
int attemptId,
boolean isLastAttempt,
ActionExecutionContext actionExecutionContext,
TestRunnerAction action,
StandaloneTestResult result)
throws IOException {
ImmutableList<Pair<String, Path>> testOutputs =
action.getTestOutputsMapping(
actionExecutionContext.getPathResolver(), actionExecutionContext.getExecRoot());
if (!isLastAttempt) {
testOutputs = renameOutputs(actionExecutionContext, action, testOutputs, attemptId);
}
// Recover the test log path, which may have been renamed, and add it to the data builder.
Path renamedTestLog = null;
for (Pair<String, Path> pair : testOutputs) {
if (TestFileNameConstants.TEST_LOG.equals(pair.getFirst())) {
Preconditions.checkState(renamedTestLog == null, "multiple test_log matches");
renamedTestLog = pair.getSecond();
}
}
TestResultData.Builder dataBuilder = result.testResultDataBuilder();
// If the test log path does not exist, mark the test as incomplete
if (renamedTestLog == null) {
dataBuilder.setStatus(BlazeTestStatus.INCOMPLETE);
}
if (dataBuilder.getStatus() == BlazeTestStatus.PASSED) {
dataBuilder.setPassedLog(renamedTestLog.toString());
} else if (dataBuilder.getStatus() == BlazeTestStatus.INCOMPLETE) {
// Incomplete (cancelled) test runs don't have a log.
Preconditions.checkState(renamedTestLog == null);
} else {
dataBuilder.addFailedLogs(renamedTestLog.toString());
}
// Add the test log to the output
TestResultData data = dataBuilder.build();
actionExecutionContext
.getEventHandler()
.post(
TestAttempt.forExecutedTestResult(
action, data, attemptId, testOutputs, result.executionInfo(), isLastAttempt));
processTestOutput(actionExecutionContext, data, action.getTestName(), renamedTestLog);
return new StandaloneFailedAttemptResult(data);
}
private static Map<String, String> setupEnvironment(
TestRunnerAction action,
Map<String, String> clientEnv,
Path execRoot,
Path runfilesDir,
Path tmpDir) {
PathFragment relativeTmpDir;
if (tmpDir.startsWith(execRoot)) {
relativeTmpDir = tmpDir.relativeTo(execRoot);
} else {
relativeTmpDir = tmpDir.asFragment();
}
return DEFAULT_LOCAL_POLICY.computeTestEnvironment(
action, clientEnv, getTimeout(action), runfilesDir.relativeTo(execRoot), relativeTmpDir);
}
private TestAttemptResult beginTestAttempt(
TestRunnerAction testAction,
Spawn spawn,
ActionExecutionContext actionExecutionContext,
Path execRoot)
throws ExecException, IOException, InterruptedException {
ResolvedPaths resolvedPaths = testAction.resolve(execRoot);
Path out = actionExecutionContext.getInputPath(testAction.getTestLog());
Path err = resolvedPaths.getTestStderr();
FileOutErr testOutErr = new FileOutErr(out, err);
Closeable streamed = null;
if (executionOptions.testOutput.equals(ExecutionOptions.TestOutputFormat.STREAMED)) {
streamed =
createStreamedTestOutput(
Reporter.outErrForReporter(actionExecutionContext.getEventHandler()), out);
}
long startTimeMillis = actionExecutionContext.getClock().currentTimeMillis();
SpawnStrategyResolver resolver = actionExecutionContext.getContext(SpawnStrategyResolver.class);
return runTestAttempt(
testAction,
actionExecutionContext,
spawn,
resolver,
resolvedPaths,
testOutErr,
streamed,
startTimeMillis);
}
private static void appendCoverageLog(FileOutErr coverageOutErr, FileOutErr outErr)
throws IOException {
writeOutFile(coverageOutErr.getErrorPath(), outErr.getOutputPath());
writeOutFile(coverageOutErr.getOutputPath(), outErr.getOutputPath());
}
private static void writeOutFile(Path inFilePath, Path outFilePath) throws IOException {
FileStatus stat = inFilePath.statNullable();
if (stat != null) {
try {
if (stat.getSize() > 0) {
if (outFilePath.exists()) {
outFilePath.setWritable(true);
}
try (OutputStream out = outFilePath.getOutputStream(true);
InputStream in = inFilePath.getInputStream()) {
ByteStreams.copy(in, out);
}
}
} finally {
inFilePath.delete();
}
}
}
private static BuildEventStreamProtos.TestResult.ExecutionInfo extractExecutionInfo(
SpawnResult spawnResult, TestResultData.Builder result) {
BuildEventStreamProtos.TestResult.ExecutionInfo.Builder executionInfo =
BuildEventStreamProtos.TestResult.ExecutionInfo.newBuilder();
if (spawnResult.isCacheHit()) {
result.setRemotelyCached(true);
executionInfo.setCachedRemotely(true);
}
String strategy = spawnResult.getRunnerName();
if (strategy != null) {
executionInfo.setStrategy(strategy);
result.setIsRemoteStrategy(strategy.equals("remote"));
}
if (spawnResult.getExecutorHostName() != null) {
executionInfo.setHostname(spawnResult.getExecutorHostName());
}
SpawnMetrics sm = spawnResult.getMetrics();
executionInfo.setTimingBreakdown(
BuildEventStreamProtos.TestResult.ExecutionInfo.TimingBreakdown.newBuilder()
.setName("totalTime")
.setTime(toProtoDuration(sm.totalTime()))
.addChild(
BuildEventStreamProtos.TestResult.ExecutionInfo.TimingBreakdown.newBuilder()
.setName("parseTime")
.setTime(toProtoDuration(sm.parseTime()))
.build())
.addChild(
BuildEventStreamProtos.TestResult.ExecutionInfo.TimingBreakdown.newBuilder()
.setName("fetchTime")
.setTime(toProtoDuration(sm.fetchTime()))
.build())
.addChild(
BuildEventStreamProtos.TestResult.ExecutionInfo.TimingBreakdown.newBuilder()
.setName("queueTime")
.setTime(toProtoDuration(sm.queueTime()))
.build())
.addChild(
BuildEventStreamProtos.TestResult.ExecutionInfo.TimingBreakdown.newBuilder()
.setName("uploadTime")
.setTime(toProtoDuration(sm.uploadTime()))
.build())
.addChild(
BuildEventStreamProtos.TestResult.ExecutionInfo.TimingBreakdown.newBuilder()
.setName("setupTime")
.setTime(toProtoDuration(sm.setupTime()))
.build())
.addChild(
BuildEventStreamProtos.TestResult.ExecutionInfo.TimingBreakdown.newBuilder()
.setName("executionWallTime")
.setTime(toProtoDuration(sm.executionWallTime()))
.build())
.addChild(
BuildEventStreamProtos.TestResult.ExecutionInfo.TimingBreakdown.newBuilder()
.setName("processOutputsTime")
.setTime(toProtoDuration(sm.processOutputsTime()))
.build())
.addChild(
BuildEventStreamProtos.TestResult.ExecutionInfo.TimingBreakdown.newBuilder()
.setName("networkTime")
.setTime(toProtoDuration(sm.networkTime()))
.build())
.build());
return executionInfo.build();
}
private static com.google.protobuf.Duration toProtoDuration(Duration d) {
return Durations.fromNanos(d.toNanos());
}
/**
* A spawn to generate a test.xml file from the test log. This is only used if the test does not
* generate a test.xml file itself.
*/
private static Spawn createXmlGeneratingSpawn(
TestRunnerAction action, ImmutableMap<String, String> testEnv, SpawnResult result) {
ImmutableList<String> args =
ImmutableList.of(
action.getTestXmlGeneratorScript().getExecPath().getCallablePathString(),
action.getTestLog().getExecPathString(),
action.getXmlOutputPath().getPathString(),
Long.toString(result.getWallTime().orElse(Duration.ZERO).getSeconds()),
Integer.toString(result.exitCode()));
ImmutableMap.Builder<String, String> envBuilder = ImmutableMap.builder();
// "PATH" and "TEST_BINARY" are also required, they should always be set in testEnv.
Preconditions.checkArgument(testEnv.containsKey("PATH"));
Preconditions.checkArgument(testEnv.containsKey("TEST_BINARY"));
envBuilder.putAll(testEnv).put("TEST_NAME", action.getTestName());
// testEnv only contains TEST_SHARD_INDEX and TEST_TOTAL_SHARDS if the test action is sharded,
// we need to set the default value when the action isn't sharded.
if (!action.isSharded()) {
envBuilder.put("TEST_SHARD_INDEX", "0");
envBuilder.put("TEST_TOTAL_SHARDS", "0");
}
Map<String, String> executionInfo =
Maps.newHashMapWithExpectedSize(action.getExecutionInfo().size() + 1);
executionInfo.putAll(action.getExecutionInfo());
if (result.exitCode() != 0) {
// If the test is failed, the spawn shouldn't use remote cache since the test.xml file is
// renamed immediately after the spawn execution. If there is another test attempt, the async
// upload will fail because it cannot read the file at original position.
executionInfo.put(ExecutionRequirements.NO_REMOTE_CACHE, "");
}
return new SimpleSpawn(
action,
args,
envBuilder.buildOrThrow(),
// Pass the execution info of the action which is identical to the supported tags set on the
// test target. In particular, this does not set the test timeout on the spawn.
ImmutableMap.copyOf(executionInfo),
null,
ImmutableMap.of(),
/*inputs=*/ NestedSetBuilder.create(
Order.STABLE_ORDER, action.getTestXmlGeneratorScript(), action.getTestLog()),
/*tools=*/ NestedSetBuilder.emptySet(Order.STABLE_ORDER),
/*outputs=*/ ImmutableSet.of(ActionInputHelper.fromPath(action.getXmlOutputPath())),
/*mandatoryOutputs=*/ null,
SpawnAction.DEFAULT_RESOURCE_SET);
}
private static Spawn createCoveragePostProcessingSpawn(
ActionExecutionContext actionExecutionContext,
TestRunnerAction action,
List<ActionInput> expandedCoverageDir,
Path tmpDirRoot,
boolean splitXmlGeneration) {
ImmutableList<String> args =
ImmutableList.of(action.getCollectCoverageScript().getExecPathString());
Map<String, String> testEnvironment =
createEnvironment(actionExecutionContext, action, tmpDirRoot, splitXmlGeneration);
testEnvironment.put("TEST_SHARD_INDEX", Integer.toString(action.getShardNum()));
testEnvironment.put(
"TEST_TOTAL_SHARDS", Integer.toString(action.getExecutionSettings().getTotalShards()));
testEnvironment.put("TEST_NAME", action.getTestName());
testEnvironment.put("IS_COVERAGE_SPAWN", "1");
return new SimpleSpawn(
action,
args,
ImmutableMap.copyOf(testEnvironment),
action.getExecutionInfo(),
action.getLcovMergerRunfilesSupplier(),
/*filesetMappings=*/ ImmutableMap.of(),
/*inputs=*/ NestedSetBuilder.<ActionInput>compileOrder()
.addTransitive(action.getInputs())
.addAll(expandedCoverageDir)
.add(action.getCollectCoverageScript())
.add(action.getCoverageManifest())
.addTransitive(action.getLcovMergerFilesToRun().build())
.build(),
/*tools=*/ NestedSetBuilder.emptySet(Order.STABLE_ORDER),
/*outputs=*/ ImmutableSet.of(
ActionInputHelper.fromPath(action.getCoverageData().getExecPath())),
/*mandatoryOutputs=*/ null,
SpawnAction.DEFAULT_RESOURCE_SET);
}
private static Map<String, String> createEnvironment(
ActionExecutionContext actionExecutionContext,
TestRunnerAction action,
Path tmpDirRoot,
boolean splitXmlGeneration) {
Path execRoot = actionExecutionContext.getExecRoot();
ArtifactPathResolver pathResolver = actionExecutionContext.getPathResolver();
Path runfilesDir = pathResolver.convertPath(action.getExecutionSettings().getRunfilesDir());
Path tmpDir = pathResolver.convertPath(tmpDirRoot.getChild(TestStrategy.getTmpDirName(action)));
Map<String, String> testEnvironment =
setupEnvironment(
action, actionExecutionContext.getClientEnv(), execRoot, runfilesDir, tmpDir);
if (splitXmlGeneration) {
testEnvironment.put("EXPERIMENTAL_SPLIT_XML_GENERATION", "1");
}
return testEnvironment;
}
@Override
public TestResult newCachedTestResult(
Path execRoot, TestRunnerAction action, TestResultData data) {
return new TestResult(action, data, /*cached*/ true, execRoot, /*systemFailure=*/ null);
}
@VisibleForTesting
static final class StandaloneFailedAttemptResult implements FailedAttemptResult {
private final TestResultData testResultData;
StandaloneFailedAttemptResult(TestResultData testResultData) {
this.testResultData = testResultData;
}
TestResultData testResultData() {
return testResultData;
}
}
private final class StandaloneTestRunnerSpawn implements TestRunnerSpawn {
private final TestRunnerAction testAction;
private final ActionExecutionContext actionExecutionContext;
private final Spawn spawn;
private final Path tmpDir;
private final Path workingDirectory;
private final Path execRoot;
StandaloneTestRunnerSpawn(
TestRunnerAction testAction,
ActionExecutionContext actionExecutionContext,
Spawn spawn,
Path tmpDir,
Path workingDirectory,
Path execRoot) {
this.testAction = testAction;
this.actionExecutionContext = actionExecutionContext;
this.spawn = spawn;
this.tmpDir = tmpDir;
this.workingDirectory = workingDirectory;
this.execRoot = execRoot;
}
@Override
public ActionExecutionContext getActionExecutionContext() {
return actionExecutionContext;
}
@Override
public TestAttemptResult execute() throws InterruptedException, IOException, ExecException {
prepareFileSystem(testAction, execRoot, tmpDir, workingDirectory);
return beginTestAttempt(testAction, spawn, actionExecutionContext, execRoot);
}
@Override
public int getMaxAttempts(TestAttemptResult firstTestAttemptResult) {
return getTestAttempts(testAction);
}
@Override
public FailedAttemptResult finalizeFailedTestAttempt(
TestAttemptResult testAttemptResult, int attempt) throws IOException {
return processFailedTestAttempt(
attempt, actionExecutionContext, testAction, (StandaloneTestResult) testAttemptResult);
}
@Override
public void finalizeTest(
TestAttemptResult finalResult, List<FailedAttemptResult> failedAttempts)
throws IOException {
StandaloneTestStrategy.this.finalizeTest(
testAction, actionExecutionContext, (StandaloneTestResult) finalResult, failedAttempts);
}
@Override
public void finalizeCancelledTest(List<FailedAttemptResult> failedAttempts) throws IOException {
TestResultData.Builder builder =
TestResultData.newBuilder()
.setCachable(false)
.setTestPassed(false)
.setStatus(BlazeTestStatus.INCOMPLETE);
StandaloneTestResult standaloneTestResult =
StandaloneTestResult.builder()
.setSpawnResults(ImmutableList.of())
.setTestResultDataBuilder(builder)
.setExecutionInfo(ExecutionInfo.getDefaultInstance())
.build();
finalizeTest(standaloneTestResult, failedAttempts);
}
}
private static TestExecException createTestExecException(
TestAction.Code errorCode, String errorMessage) {
return new TestExecException(
errorMessage,
FailureDetail.newBuilder()
.setTestAction(TestAction.newBuilder().setCode(errorCode))
.setMessage(errorMessage)
.build());
}
private TestAttemptResult runTestAttempt(
TestRunnerAction testAction,
ActionExecutionContext actionExecutionContext,
Spawn spawn,
SpawnStrategyResolver resolver,
ResolvedPaths resolvedPaths,
FileOutErr fileOutErr,
Closeable streamed,
long startTimeMillis)
throws InterruptedException, ExecException, IOException {
ImmutableList<SpawnResult> spawnResults;
// We have two protos to represent test attempts:
// 1. com.google.devtools.build.lib.view.test.TestStatus.TestResultData represents both
// failed attempts and finished tests. Bazel stores this to disk to persist cached test
// result information across server restarts.
// 2. com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.TestResult
// represents only individual attempts (failed or not). Bazel reports this as an event to
// the Build Event Protocol, but never saves it to disk.
//
// The TestResult proto is always constructed from a TestResultData instance, either one
// that is created right here, or one that is read back from disk.
TestResultData.Builder testResultDataBuilder;
try {
spawnResults = resolver.exec(spawn, actionExecutionContext.withFileOutErr(fileOutErr));
testResultDataBuilder = TestResultData.newBuilder();
testResultDataBuilder.setCachable(true).setTestPassed(true).setStatus(BlazeTestStatus.PASSED);
} catch (SpawnExecException e) {
if (e.isCatastrophic()) {
closeSuppressed(e, streamed);
closeSuppressed(e, fileOutErr);
throw e;
}
if (!e.getSpawnResult().setupSuccess()) {
closeSuppressed(e, streamed);
closeSuppressed(e, fileOutErr);
// Rethrow as the test could not be run and thus there's no point in retrying.
throw e;
}
spawnResults = ImmutableList.of(e.getSpawnResult());
testResultDataBuilder = TestResultData.newBuilder();
testResultDataBuilder
.setCachable(e.getSpawnResult().status().isConsideredUserError())
.setTestPassed(false)
.setStatus(e.hasTimedOut() ? BlazeTestStatus.TIMEOUT : BlazeTestStatus.FAILED);
} catch (InterruptedException e) {
closeSuppressed(e, streamed);
closeSuppressed(e, fileOutErr);
throw e;
}
long endTimeMillis = actionExecutionContext.getClock().currentTimeMillis();
// SpawnActionContext guarantees the first entry to correspond to the spawn passed in (there
// may be additional entries due to tree artifact handling).
SpawnResult primaryResult = spawnResults.get(0);
// The SpawnResult of a remotely cached or remotely executed action may not have walltime
// set. We fall back to the time measured here for backwards compatibility.
long durationMillis = endTimeMillis - startTimeMillis;
durationMillis =
primaryResult.getWallTime().orElse(Duration.ofMillis(durationMillis)).toMillis();
testResultDataBuilder
.setStartTimeMillisEpoch(startTimeMillis)
.addTestTimes(durationMillis)
.addTestProcessTimes(durationMillis)
.setRunDurationMillis(durationMillis)
.setHasCoverage(testAction.isCoverageMode());
if (testAction.isCoverageMode() && testAction.getSplitCoveragePostProcessing()) {
if (testAction.getCoverageDirectoryTreeArtifact() == null) {
// Otherwise we'll get a NPE https://github.com/bazelbuild/bazel/issues/13185
TestExecException e =
createTestExecException(
TestAction.Code.LOCAL_TEST_PREREQ_UNMET,
"coverageDirectoryTreeArtifact is null:"
+ " --experimental_split_coverage_postprocessing depends on"
+ " --experimental_fetch_all_coverage_outputs being enabled");
closeSuppressed(e, streamed);
closeSuppressed(e, fileOutErr);
throw e;
}
actionExecutionContext
.getMetadataHandler()
.getMetadata(testAction.getCoverageDirectoryTreeArtifact());
ImmutableSet<? extends ActionInput> expandedCoverageDir =
actionExecutionContext
.getMetadataHandler()
.getTreeArtifactChildren(
(SpecialArtifact) testAction.getCoverageDirectoryTreeArtifact());
Spawn coveragePostProcessingSpawn =
createCoveragePostProcessingSpawn(
actionExecutionContext,
testAction,
ImmutableList.copyOf(expandedCoverageDir),
tmpDirRoot,
executionOptions.splitXmlGeneration);
SpawnStrategyResolver spawnStrategyResolver =
actionExecutionContext.getContext(SpawnStrategyResolver.class);
Path testRoot =
actionExecutionContext.getInputPath(testAction.getTestLog()).getParentDirectory();
Path out = testRoot.getChild("coverage.log");
Path err = testRoot.getChild("coverage.err");
FileOutErr coverageOutErr = new FileOutErr(out, err);
ActionExecutionContext actionExecutionContextWithCoverageFileOutErr =
actionExecutionContext.withFileOutErr(coverageOutErr);
writeOutFile(coverageOutErr.getErrorPath(), coverageOutErr.getOutputPath());
appendCoverageLog(coverageOutErr, fileOutErr);
try {
spawnStrategyResolver.exec(
coveragePostProcessingSpawn, actionExecutionContextWithCoverageFileOutErr);
} catch (SpawnExecException e) {
if (e.isCatastrophic()) {
closeSuppressed(e, streamed);
closeSuppressed(e, fileOutErr);
throw e;
}
if (!e.getSpawnResult().setupSuccess()) {
closeSuppressed(e, streamed);
closeSuppressed(e, fileOutErr);
// Rethrow as the test could not be run and thus there's no point in retrying.
throw e;
}
testResultDataBuilder
.setCachable(e.getSpawnResult().status().isConsideredUserError())
.setTestPassed(false)
.setStatus(e.hasTimedOut() ? BlazeTestStatus.TIMEOUT : BlazeTestStatus.FAILED);
} catch (ExecException | InterruptedException e) {
closeSuppressed(e, streamed);
closeSuppressed(e, fileOutErr);
throw e;
}
}
Verify.verify(
!(testAction.isCoverageMode() && testAction.getSplitCoveragePostProcessing())
|| testAction.getCoverageData().getPath().exists());
Verify.verifyNotNull(spawnResults);
Verify.verifyNotNull(testResultDataBuilder);
try {
if (!fileOutErr.hasRecordedOutput()) {
// Make sure that the test.log exists.Spaw
FileSystemUtils.touchFile(fileOutErr.getOutputPath());
}
// Append any error output to the test.log. This is very rare.
writeOutFile(fileOutErr.getErrorPath(), fileOutErr.getOutputPath());
fileOutErr.close();
if (streamed != null) {
streamed.close();
}
} catch (IOException e) {
throw new EnvironmentalExecException(e, Code.TEST_OUT_ERR_IO_EXCEPTION);
}
Path xmlOutputPath = resolvedPaths.getXmlOutputPath();
// If the test did not create a test.xml, and --experimental_split_xml_generation is enabled,
// then we run a separate action to create a test.xml from test.log. We do this as a spawn
// rather than doing it locally in-process, as the test.log file may only exist remotely (when
// remote execution is enabled), and we do not want to have to download it.
if (executionOptions.splitXmlGeneration
&& fileOutErr.getOutputPath().exists()
&& !xmlOutputPath.exists()) {
Spawn xmlGeneratingSpawn =
createXmlGeneratingSpawn(testAction, spawn.getEnvironment(), spawnResults.get(0));
SpawnStrategyResolver spawnStrategyResolver =
actionExecutionContext.getContext(SpawnStrategyResolver.class);
// We treat all failures to generate the test.xml here as catastrophic, and won't rerun
// the test if this fails. We redirect the output to a temporary file.
FileOutErr xmlSpawnOutErr = actionExecutionContext.getFileOutErr().childOutErr();
try {
ImmutableList<SpawnResult> xmlSpawnResults =
spawnStrategyResolver.exec(
xmlGeneratingSpawn, actionExecutionContext.withFileOutErr(xmlSpawnOutErr));
spawnResults =
ImmutableList.<SpawnResult>builder()
.addAll(spawnResults)
.addAll(xmlSpawnResults)
.build();
} catch (InterruptedException | ExecException e) {
closeSuppressed(e, xmlSpawnOutErr);
throw e;
}
}
TestCase details = parseTestResult(xmlOutputPath);
if (details != null) {
testResultDataBuilder.setTestCase(details);
}
BuildEventStreamProtos.TestResult.ExecutionInfo executionInfo =
extractExecutionInfo(spawnResults.get(0), testResultDataBuilder);
return StandaloneTestResult.builder()
.setSpawnResults(spawnResults)
// We return the TestResultData.Builder rather than the finished TestResultData
// instance, as we may have to rename the output files in case the test needs to be
// rerun (if it failed here _and_ is marked flaky _and_ the number of flaky attempts
// is larger than 1).
.setTestResultDataBuilder(testResultDataBuilder)
.setExecutionInfo(executionInfo)
.build();
}
}