src/main/java/com/google/devtools/build/lib/exec/StandaloneTestStrategy.java

// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.exec;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Verify;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.io.ByteStreams;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionInputHelper;
import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
import com.google.devtools.build.lib.actions.ArtifactPathResolver;
import com.google.devtools.build.lib.actions.EnvironmentalExecException;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.ExecutionRequirements;
import com.google.devtools.build.lib.actions.SimpleSpawn;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.SpawnMetrics;
import com.google.devtools.build.lib.actions.SpawnResult;
import com.google.devtools.build.lib.actions.TestExecException;
import com.google.devtools.build.lib.analysis.actions.SpawnAction;
import com.google.devtools.build.lib.analysis.test.TestAttempt;
import com.google.devtools.build.lib.analysis.test.TestResult;
import com.google.devtools.build.lib.analysis.test.TestRunnerAction;
import com.google.devtools.build.lib.analysis.test.TestRunnerAction.ResolvedPaths;
import com.google.devtools.build.lib.analysis.test.TestStrategy;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.TestResult.ExecutionInfo;
import com.google.devtools.build.lib.buildeventstream.TestFileNameConstants;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.collect.nestedset.Order;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.server.FailureDetails.Execution.Code;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.server.FailureDetails.TestAction;
import com.google.devtools.build.lib.util.Pair;
import com.google.devtools.build.lib.util.io.FileOutErr;
import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
import com.google.devtools.build.lib.view.test.TestStatus.TestCase;
import com.google.devtools.build.lib.view.test.TestStatus.TestResultData;
import com.google.protobuf.Duration;
import com.google.protobuf.util.Durations;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

/** Runs TestRunnerAction actions. */
// TODO(bazel-team): add tests for this strategy.
public class StandaloneTestStrategy extends TestStrategy {
  private static final ImmutableMap<String, String> ENV_VARS =
      ImmutableMap.<String, String>builder()
          .put("TZ", "UTC")
          .put("TEST_SRCDIR", TestPolicy.RUNFILES_DIR)
          // TODO(lberki): Remove JAVA_RUNFILES and PYTHON_RUNFILES.
          .put("JAVA_RUNFILES", TestPolicy.RUNFILES_DIR)
          .put("PYTHON_RUNFILES", TestPolicy.RUNFILES_DIR)
          .put("RUNFILES_DIR", TestPolicy.RUNFILES_DIR)
          .put("TEST_TMPDIR", TestPolicy.TEST_TMP_DIR)
          .put("RUN_UNDER_RUNFILES", "1")
          .build();

  public static final TestPolicy DEFAULT_LOCAL_POLICY = new TestPolicy(ENV_VARS);

  protected final Path tmpDirRoot;

  public StandaloneTestStrategy(
      ExecutionOptions executionOptions, BinTools binTools, Path tmpDirRoot) {
    super(executionOptions, binTools);
    this.tmpDirRoot = tmpDirRoot;
  }

  @Override
  public TestRunnerSpawn createTestRunnerSpawn(
      TestRunnerAction action, ActionExecutionContext actionExecutionContext)
      throws ExecException, InterruptedException {
    if (action.getExecutionSettings().getInputManifest() == null) {
      throw createTestExecException(
          TestAction.Code.LOCAL_TEST_PREREQ_UNMET,
          "cannot run local tests with --nobuild_runfile_manifests");
    }
    Map<String, String> testEnvironment =
        createEnvironment(
            actionExecutionContext, action, tmpDirRoot, executionOptions.splitXmlGeneration);

    Map<String, String> executionInfo =
        new TreeMap<>(action.getTestProperties().getExecutionInfo());
    if (!action.shouldCacheResult()) {
      executionInfo.put(ExecutionRequirements.NO_CACHE, "");
    }
    executionInfo.put(ExecutionRequirements.TIMEOUT, "" + getTimeout(action).getSeconds());

    SimpleSpawn.LocalResourcesSupplier localResourcesSupplier =
        () ->
            action
                .getTestProperties()
                .getLocalResourceUsage(
                    action.getOwner().getLabel(), executionOptions.usingLocalTestJobs());

    Spawn spawn =
        new SimpleSpawn(
            action,
            getArgs(action),
            ImmutableMap.copyOf(testEnvironment),
            ImmutableMap.copyOf(executionInfo),
            action.getRunfilesSupplier(),
            ImmutableMap.of(),
            /*inputs=*/ action.getInputs(),
            NestedSetBuilder.emptySet(Order.STABLE_ORDER),
            ImmutableSet.copyOf(action.getSpawnOutputs()),
            /*mandatoryOutputs=*/ ImmutableSet.of(),
            localResourcesSupplier);
    Path execRoot = actionExecutionContext.getExecRoot();
    ArtifactPathResolver pathResolver = actionExecutionContext.getPathResolver();
    Path runfilesDir = pathResolver.convertPath(action.getExecutionSettings().getRunfilesDir());
    Path tmpDir = pathResolver.convertPath(tmpDirRoot.getChild(TestStrategy.getTmpDirName(action)));
    Path workingDirectory = runfilesDir.getRelative(action.getRunfilesPrefix());
    return new StandaloneTestRunnerSpawn(
        action, actionExecutionContext, spawn, tmpDir, workingDirectory, execRoot);
  }

  private static ImmutableList<Pair<String, Path>> renameOutputs(
      ActionExecutionContext actionExecutionContext,
      TestRunnerAction action,
      ImmutableList<Pair<String, Path>> testOutputs,
      int attemptId)
      throws IOException {
    // Rename outputs
    String namePrefix =
        FileSystemUtils.removeExtension(action.getTestLog().getExecPath().getBaseName());
    Path testRoot = actionExecutionContext.getInputPath(action.getTestLog()).getParentDirectory();
    Path attemptsDir = testRoot.getChild(namePrefix + "_attempts");
    attemptsDir.createDirectory();
    String attemptPrefix = "attempt_" + attemptId;
    Path testLog = attemptsDir.getChild(attemptPrefix + ".log");

    // Get the normal test output paths, and then update them to use "attempt_N" names, and
    // attemptDir, before adding them to the outputs.
    ImmutableList.Builder<Pair<String, Path>> testOutputsBuilder = new ImmutableList.Builder<>();
    for (Pair<String, Path> testOutput : testOutputs) {
      // e.g. /testRoot/test.dir/file, an example we follow throughout this loop's comments.
      Path testOutputPath = testOutput.getSecond();
      Path destinationPath;
      if (testOutput.getFirst().equals(TestFileNameConstants.TEST_LOG)) {
        // The rename rules for the test log are different than for all the other files.
        destinationPath = testLog;
      } else {
        // e.g. test.dir/file
        PathFragment relativeToTestDirectory = testOutputPath.relativeTo(testRoot);

        // e.g. attempt_1.dir/file
        String destinationPathFragmentStr =
            relativeToTestDirectory.getSafePathString().replaceFirst("test", attemptPrefix);
        PathFragment destinationPathFragment = PathFragment.create(destinationPathFragmentStr);

        // e.g. /attemptsDir/attempt_1.dir/file
        destinationPath = attemptsDir.getRelative(destinationPathFragment);
        destinationPath.getParentDirectory().createDirectory();
      }

      // Move to the destination.
      testOutputPath.renameTo(destinationPath);

      testOutputsBuilder.add(Pair.of(testOutput.getFirst(), destinationPath));
    }
    return testOutputsBuilder.build();
  }

  private StandaloneFailedAttemptResult processFailedTestAttempt(
      int attemptId,
      ActionExecutionContext actionExecutionContext,
      TestRunnerAction action,
      StandaloneTestResult result)
      throws IOException {
    return processTestAttempt(
        attemptId, /*isLastAttempt=*/ false, actionExecutionContext, action, result);
  }

  private void finalizeTest(
      TestRunnerAction action,
      ActionExecutionContext actionExecutionContext,
      StandaloneTestResult standaloneTestResult,
      List<FailedAttemptResult> failedAttempts)
      throws IOException {
    processTestAttempt(
        failedAttempts.size() + 1,
        /*isLastAttempt=*/ true,
        actionExecutionContext,
        action,
        standaloneTestResult);

    TestResultData.Builder dataBuilder = standaloneTestResult.testResultDataBuilder();
    for (FailedAttemptResult failedAttempt : failedAttempts) {
      TestResultData failedAttemptData =
          ((StandaloneFailedAttemptResult) failedAttempt).testResultData;
      dataBuilder.addAllFailedLogs(failedAttemptData.getFailedLogsList());
      dataBuilder.addTestTimes(failedAttemptData.getTestTimes(0));
      dataBuilder.addAllTestProcessTimes(failedAttemptData.getTestProcessTimesList());
    }
    if (dataBuilder.getStatus() == BlazeTestStatus.PASSED && !failedAttempts.isEmpty()) {
      dataBuilder.setStatus(BlazeTestStatus.FLAKY);
    }
    TestResultData data = dataBuilder.build();
    TestResult result =
        new TestResult(action, data, false, standaloneTestResult.primarySystemFailure());
    postTestResult(actionExecutionContext, result);
  }

  private StandaloneFailedAttemptResult processTestAttempt(
      int attemptId,
      boolean isLastAttempt,
      ActionExecutionContext actionExecutionContext,
      TestRunnerAction action,
      StandaloneTestResult result)
      throws IOException {
    ImmutableList<Pair<String, Path>> testOutputs =
        action.getTestOutputsMapping(
            actionExecutionContext.getPathResolver(), actionExecutionContext.getExecRoot());
    if (!isLastAttempt) {
      testOutputs = renameOutputs(actionExecutionContext, action, testOutputs, attemptId);
    }

    // Recover the test log path, which may have been renamed, and add it to the data builder.
    Path renamedTestLog = null;
    for (Pair<String, Path> pair : testOutputs) {
      if (TestFileNameConstants.TEST_LOG.equals(pair.getFirst())) {
        Preconditions.checkState(renamedTestLog == null, "multiple test_log matches");
        renamedTestLog = pair.getSecond();
      }
    }

    TestResultData.Builder dataBuilder = result.testResultDataBuilder();
    // If the test log path does not exist, mark the test as incomplete
    if (renamedTestLog == null) {
      dataBuilder.setStatus(BlazeTestStatus.INCOMPLETE);
    }

    if (dataBuilder.getStatus() == BlazeTestStatus.PASSED) {
      dataBuilder.setPassedLog(renamedTestLog.toString());
    } else if (dataBuilder.getStatus() == BlazeTestStatus.INCOMPLETE) {
      // Incomplete (cancelled) test runs don't have a log.
      Preconditions.checkState(renamedTestLog == null);
    } else {
      dataBuilder.addFailedLogs(renamedTestLog.toString());
    }

    // Add the test log to the output
    TestResultData data = dataBuilder.build();
    actionExecutionContext
        .getEventHandler()
        .post(
            TestAttempt.forExecutedTestResult(
                action, data, attemptId, testOutputs, result.executionInfo(), isLastAttempt));
    processTestOutput(actionExecutionContext, data, action.getTestName(), renamedTestLog);
    return new StandaloneFailedAttemptResult(data);
  }

  private static Map<String, String> setupEnvironment(
      TestRunnerAction action,
      Map<String, String> clientEnv,
      Path execRoot,
      Path runfilesDir,
      Path tmpDir) {
    PathFragment relativeTmpDir;
    if (tmpDir.startsWith(execRoot)) {
      relativeTmpDir = tmpDir.relativeTo(execRoot);
    } else {
      relativeTmpDir = tmpDir.asFragment();
    }
    return DEFAULT_LOCAL_POLICY.computeTestEnvironment(
        action, clientEnv, getTimeout(action), runfilesDir.relativeTo(execRoot), relativeTmpDir);
  }

  private TestAttemptResult beginTestAttempt(
      TestRunnerAction testAction,
      Spawn spawn,
      ActionExecutionContext actionExecutionContext,
      Path execRoot)
      throws ExecException, IOException, InterruptedException {
    ResolvedPaths resolvedPaths = testAction.resolve(execRoot);
    Path out = actionExecutionContext.getInputPath(testAction.getTestLog());
    Path err = resolvedPaths.getTestStderr();
    FileOutErr testOutErr = new FileOutErr(out, err);
    Closeable streamed = null;
    if (executionOptions.testOutput.equals(ExecutionOptions.TestOutputFormat.STREAMED)) {
      streamed =
          createStreamedTestOutput(
              Reporter.outErrForReporter(actionExecutionContext.getEventHandler()), out);
    }

    long startTimeMillis = actionExecutionContext.getClock().currentTimeMillis();
    SpawnStrategyResolver resolver = actionExecutionContext.getContext(SpawnStrategyResolver.class);

    return runTestAttempt(
        testAction,
        actionExecutionContext,
        spawn,
        resolver,
        resolvedPaths,
        testOutErr,
        streamed,
        startTimeMillis);
  }

  private static void appendCoverageLog(FileOutErr coverageOutErr, FileOutErr outErr)
      throws IOException {
    writeOutFile(coverageOutErr.getErrorPath(), outErr.getOutputPath());
    writeOutFile(coverageOutErr.getOutputPath(), outErr.getOutputPath());
  }

  private static void writeOutFile(Path inFilePath, Path outFilePath) throws IOException {
    FileStatus stat = inFilePath.statNullable();
    if (stat != null) {
      try {
        if (stat.getSize() > 0) {
          if (outFilePath.exists()) {
            outFilePath.setWritable(true);
          }
          try (OutputStream out = outFilePath.getOutputStream(true);
              InputStream in = inFilePath.getInputStream()) {
            ByteStreams.copy(in, out);
          }
        }
      } finally {
        inFilePath.delete();
      }
    }
  }

  private static BuildEventStreamProtos.TestResult.ExecutionInfo extractExecutionInfo(
      SpawnResult spawnResult, TestResultData.Builder result) {
    BuildEventStreamProtos.TestResult.ExecutionInfo.Builder executionInfo =
        BuildEventStreamProtos.TestResult.ExecutionInfo.newBuilder();

    if (spawnResult.isCacheHit()) {
      result.setRemotelyCached(true);
      executionInfo.setCachedRemotely(true);
    }

    String strategy = spawnResult.getRunnerName();
    if (strategy != null) {
      executionInfo.setStrategy(strategy);
      result.setIsRemoteStrategy(strategy.equals("remote"));
    }

    if (spawnResult.getExecutorHostName() != null) {
      executionInfo.setHostname(spawnResult.getExecutorHostName());
    }

    SpawnMetrics sm = spawnResult.getMetrics();
    executionInfo.setTimingBreakdown(
        BuildEventStreamProtos.TestResult.ExecutionInfo.TimingBreakdown.newBuilder()
            .setName("totalTime")
            .setTime(toProtoDuration(sm.totalTimeInMs()))
            .addChild(
                BuildEventStreamProtos.TestResult.ExecutionInfo.TimingBreakdown.newBuilder()
                    .setName("parseTime")
                    .setTime(toProtoDuration(sm.parseTimeInMs()))
                    .build())
            .addChild(
                BuildEventStreamProtos.TestResult.ExecutionInfo.TimingBreakdown.newBuilder()
                    .setName("fetchTime")
                    .setTime(toProtoDuration(sm.fetchTimeInMs()))
                    .build())
            .addChild(
                BuildEventStreamProtos.TestResult.ExecutionInfo.TimingBreakdown.newBuilder()
                    .setName("queueTime")
                    .setTime(toProtoDuration(sm.queueTimeInMs()))
                    .build())
            .addChild(
                BuildEventStreamProtos.TestResult.ExecutionInfo.TimingBreakdown.newBuilder()
                    .setName("uploadTime")
                    .setTime(toProtoDuration(sm.uploadTimeInMs()))
                    .build())
            .addChild(
                BuildEventStreamProtos.TestResult.ExecutionInfo.TimingBreakdown.newBuilder()
                    .setName("setupTime")
                    .setTime(toProtoDuration(sm.setupTimeInMs()))
                    .build())
            .addChild(
                BuildEventStreamProtos.TestResult.ExecutionInfo.TimingBreakdown.newBuilder()
                    .setName("executionWallTime")
                    .setTime(toProtoDuration(sm.executionWallTimeInMs()))
                    .build())
            .addChild(
                BuildEventStreamProtos.TestResult.ExecutionInfo.TimingBreakdown.newBuilder()
                    .setName("processOutputsTime")
                    .setTime(toProtoDuration(sm.processOutputsTimeInMs()))
                    .build())
            .addChild(
                BuildEventStreamProtos.TestResult.ExecutionInfo.TimingBreakdown.newBuilder()
                    .setName("networkTime")
                    .setTime(toProtoDuration(sm.networkTimeInMs()))
                    .build())
            .build());

    return executionInfo.build();
  }

  private static Duration toProtoDuration(int timeInMs) {
    return Durations.fromMillis(timeInMs);
  }

  /**
   * A spawn to generate a test.xml file from the test log. This is only used if the test does not
   * generate a test.xml file itself.
   */
  private static Spawn createXmlGeneratingSpawn(
      TestRunnerAction action, ImmutableMap<String, String> testEnv, SpawnResult result) {
    ImmutableList<String> args =
        ImmutableList.of(
            action.getTestXmlGeneratorScript().getExecPath().getCallablePathString(),
            action.getTestLog().getExecPathString(),
            action.getXmlOutputPath().getPathString(),
            Integer.toString(result.getWallTimeInMs() / 1000),
            Integer.toString(result.exitCode()));
    ImmutableMap.Builder<String, String> envBuilder = ImmutableMap.builder();
    // "PATH" and "TEST_BINARY" are also required, they should always be set in testEnv.
    Preconditions.checkArgument(testEnv.containsKey("PATH"));
    Preconditions.checkArgument(testEnv.containsKey("TEST_BINARY"));
    envBuilder.putAll(testEnv).put("TEST_NAME", action.getTestName());
    // testEnv only contains TEST_SHARD_INDEX and TEST_TOTAL_SHARDS if the test action is sharded,
    // we need to set the default value when the action isn't sharded.
    if (!action.isSharded()) {
      envBuilder.put("TEST_SHARD_INDEX", "0");
      envBuilder.put("TEST_TOTAL_SHARDS", "0");
    }
    Map<String, String> executionInfo =
        Maps.newHashMapWithExpectedSize(action.getExecutionInfo().size() + 1);
    executionInfo.putAll(action.getExecutionInfo());
    if (result.exitCode() != 0) {
      // If the test is failed, the spawn shouldn't use remote cache since the test.xml file is
      // renamed immediately after the spawn execution. If there is another test attempt, the async
      // upload will fail because it cannot read the file at original position.
      executionInfo.put(ExecutionRequirements.NO_REMOTE_CACHE, "");
    }
    return new SimpleSpawn(
        action,
        args,
        envBuilder.buildOrThrow(),
        // Pass the execution info of the action which is identical to the supported tags set on the
        // test target. In particular, this does not set the test timeout on the spawn.
        ImmutableMap.copyOf(executionInfo),
        null,
        ImmutableMap.of(),
        /*inputs=*/ NestedSetBuilder.create(
            Order.STABLE_ORDER, action.getTestXmlGeneratorScript(), action.getTestLog()),
        /*tools=*/ NestedSetBuilder.emptySet(Order.STABLE_ORDER),
        /*outputs=*/ ImmutableSet.of(ActionInputHelper.fromPath(action.getXmlOutputPath())),
        /*mandatoryOutputs=*/ null,
        SpawnAction.DEFAULT_RESOURCE_SET);
  }

  private static Spawn createCoveragePostProcessingSpawn(
      ActionExecutionContext actionExecutionContext,
      TestRunnerAction action,
      List<ActionInput> expandedCoverageDir,
      Path tmpDirRoot,
      boolean splitXmlGeneration) {
    ImmutableList<String> args =
        ImmutableList.of(action.getCollectCoverageScript().getExecPathString());

    Map<String, String> testEnvironment =
        createEnvironment(actionExecutionContext, action, tmpDirRoot, splitXmlGeneration);

    testEnvironment.put("TEST_SHARD_INDEX", Integer.toString(action.getShardNum()));
    testEnvironment.put(
        "TEST_TOTAL_SHARDS", Integer.toString(action.getExecutionSettings().getTotalShards()));
    testEnvironment.put("TEST_NAME", action.getTestName());
    testEnvironment.put("IS_COVERAGE_SPAWN", "1");
    return new SimpleSpawn(
        action,
        args,
        ImmutableMap.copyOf(testEnvironment),
        action.getExecutionInfo(),
        action.getLcovMergerRunfilesSupplier(),
        /*filesetMappings=*/ ImmutableMap.of(),
        /*inputs=*/ NestedSetBuilder.<ActionInput>compileOrder()
            .addTransitive(action.getInputs())
            .addAll(expandedCoverageDir)
            .add(action.getCollectCoverageScript())
            .add(action.getCoverageManifest())
            .addTransitive(action.getLcovMergerFilesToRun().build())
            .build(),
        /*tools=*/ NestedSetBuilder.emptySet(Order.STABLE_ORDER),
        /*outputs=*/ ImmutableSet.of(
            ActionInputHelper.fromPath(action.getCoverageData().getExecPath())),
        /*mandatoryOutputs=*/ null,
        SpawnAction.DEFAULT_RESOURCE_SET);
  }

  private static Map<String, String> createEnvironment(
      ActionExecutionContext actionExecutionContext,
      TestRunnerAction action,
      Path tmpDirRoot,
      boolean splitXmlGeneration) {
    Path execRoot = actionExecutionContext.getExecRoot();
    ArtifactPathResolver pathResolver = actionExecutionContext.getPathResolver();
    Path runfilesDir = pathResolver.convertPath(action.getExecutionSettings().getRunfilesDir());
    Path tmpDir = pathResolver.convertPath(tmpDirRoot.getChild(TestStrategy.getTmpDirName(action)));
    Map<String, String> testEnvironment =
        setupEnvironment(
            action, actionExecutionContext.getClientEnv(), execRoot, runfilesDir, tmpDir);
    if (splitXmlGeneration) {
      testEnvironment.put("EXPERIMENTAL_SPLIT_XML_GENERATION", "1");
    }
    return testEnvironment;
  }

  @Override
  public TestResult newCachedTestResult(
      Path execRoot, TestRunnerAction action, TestResultData data) {
    return new TestResult(action, data, /*cached*/ true, execRoot, /*systemFailure=*/ null);
  }

  @VisibleForTesting
  static final class StandaloneFailedAttemptResult implements FailedAttemptResult {
    private final TestResultData testResultData;

    StandaloneFailedAttemptResult(TestResultData testResultData) {
      this.testResultData = testResultData;
    }

    TestResultData testResultData() {
      return testResultData;
    }
  }

  private final class StandaloneTestRunnerSpawn implements TestRunnerSpawn {
    private final TestRunnerAction testAction;
    private final ActionExecutionContext actionExecutionContext;
    private final Spawn spawn;
    private final Path tmpDir;
    private final Path workingDirectory;
    private final Path execRoot;

    StandaloneTestRunnerSpawn(
        TestRunnerAction testAction,
        ActionExecutionContext actionExecutionContext,
        Spawn spawn,
        Path tmpDir,
        Path workingDirectory,
        Path execRoot) {
      this.testAction = testAction;
      this.actionExecutionContext = actionExecutionContext;
      this.spawn = spawn;
      this.tmpDir = tmpDir;
      this.workingDirectory = workingDirectory;
      this.execRoot = execRoot;
    }

    @Override
    public ActionExecutionContext getActionExecutionContext() {
      return actionExecutionContext;
    }

    @Override
    public TestAttemptResult execute() throws InterruptedException, IOException, ExecException {
      prepareFileSystem(testAction, execRoot, tmpDir, workingDirectory);
      return beginTestAttempt(testAction, spawn, actionExecutionContext, execRoot);
    }

    @Override
    public int getMaxAttempts(TestAttemptResult firstTestAttemptResult) {
      return getTestAttempts(testAction);
    }

    @Override
    public FailedAttemptResult finalizeFailedTestAttempt(
        TestAttemptResult testAttemptResult, int attempt) throws IOException {
      return processFailedTestAttempt(
          attempt, actionExecutionContext, testAction, (StandaloneTestResult) testAttemptResult);
    }

    @Override
    public void finalizeTest(
        TestAttemptResult finalResult, List<FailedAttemptResult> failedAttempts)
        throws IOException {
      StandaloneTestStrategy.this.finalizeTest(
          testAction, actionExecutionContext, (StandaloneTestResult) finalResult, failedAttempts);
    }

    @Override
    public void finalizeCancelledTest(List<FailedAttemptResult> failedAttempts) throws IOException {
      TestResultData.Builder builder =
          TestResultData.newBuilder()
              .setCachable(false)
              .setTestPassed(false)
              .setStatus(BlazeTestStatus.INCOMPLETE);
      StandaloneTestResult standaloneTestResult =
          StandaloneTestResult.builder()
              .setSpawnResults(ImmutableList.of())
              .setTestResultDataBuilder(builder)
              .setExecutionInfo(ExecutionInfo.getDefaultInstance())
              .build();
      finalizeTest(standaloneTestResult, failedAttempts);
    }
  }

  private static TestExecException createTestExecException(
      TestAction.Code errorCode, String errorMessage) {
    return new TestExecException(
        errorMessage,
        FailureDetail.newBuilder()
            .setTestAction(TestAction.newBuilder().setCode(errorCode))
            .setMessage(errorMessage)
            .build());
  }

  private TestAttemptResult runTestAttempt(
      TestRunnerAction testAction,
      ActionExecutionContext actionExecutionContext,
      Spawn spawn,
      SpawnStrategyResolver resolver,
      ResolvedPaths resolvedPaths,
      FileOutErr fileOutErr,
      Closeable streamed,
      long startTimeMillis)
      throws InterruptedException, ExecException, IOException {

    ImmutableList<SpawnResult> spawnResults;

    // We have two protos to represent test attempts:
    // 1. com.google.devtools.build.lib.view.test.TestStatus.TestResultData represents both
    //    failed attempts and finished tests. Bazel stores this to disk to persist cached test
    //    result information across server restarts.
    // 2. com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.TestResult
    //    represents only individual attempts (failed or not). Bazel reports this as an event to
    //    the Build Event Protocol, but never saves it to disk.
    //
    // The TestResult proto is always constructed from a TestResultData instance, either one
    // that is created right here, or one that is read back from disk.
    TestResultData.Builder testResultDataBuilder;
    try {
      spawnResults = resolver.exec(spawn, actionExecutionContext.withFileOutErr(fileOutErr));
      testResultDataBuilder = TestResultData.newBuilder();
      testResultDataBuilder.setCachable(true).setTestPassed(true).setStatus(BlazeTestStatus.PASSED);
    } catch (SpawnExecException e) {
      if (e.isCatastrophic()) {
        closeSuppressed(e, streamed);
        closeSuppressed(e, fileOutErr);
        throw e;
      }
      if (!e.getSpawnResult().setupSuccess()) {
        closeSuppressed(e, streamed);
        closeSuppressed(e, fileOutErr);
        // Rethrow as the test could not be run and thus there's no point in retrying.
        throw e;
      }
      spawnResults = ImmutableList.of(e.getSpawnResult());
      testResultDataBuilder = TestResultData.newBuilder();
      testResultDataBuilder
          .setCachable(e.getSpawnResult().status().isConsideredUserError())
          .setTestPassed(false)
          .setStatus(e.hasTimedOut() ? BlazeTestStatus.TIMEOUT : BlazeTestStatus.FAILED);
    } catch (InterruptedException e) {
      closeSuppressed(e, streamed);
      closeSuppressed(e, fileOutErr);
      throw e;
    }
    long endTimeMillis = actionExecutionContext.getClock().currentTimeMillis();

    if (testAction.isSharded()) {
      if (testAction.checkShardingSupport()
          && !actionExecutionContext
              .getPathResolver()
              .convertPath(resolvedPaths.getTestShard())
              .exists()) {
        TestExecException e =
            createTestExecException(
                TestAction.Code.LOCAL_TEST_PREREQ_UNMET,
                "Sharding requested, but the test runner did not advertise support for it by "
                    + "touching TEST_SHARD_STATUS_FILE. Either remove the 'shard_count' attribute, "
                    + "use a test runner that supports sharding or temporarily disable this check "
                    + "via --noincompatible_check_sharding_support.");
        closeSuppressed(e, streamed);
        closeSuppressed(e, fileOutErr);
        throw e;
      }
    }

    // SpawnActionContext guarantees the first entry to correspond to the spawn passed in (there
    // may be additional entries due to tree artifact handling).
    SpawnResult primaryResult = spawnResults.get(0);

    // The SpawnResult of a remotely cached or remotely executed action may not have walltime
    // set. We fall back to the time measured here for backwards compatibility.
    long durationMillis = endTimeMillis - startTimeMillis;
    durationMillis =
        (primaryResult.getWallTimeInMs() != 0 ? primaryResult.getWallTimeInMs() : durationMillis);

    testResultDataBuilder
        .setStartTimeMillisEpoch(startTimeMillis)
        .addTestTimes(durationMillis)
        .addTestProcessTimes(durationMillis)
        .setRunDurationMillis(durationMillis)
        .setHasCoverage(testAction.isCoverageMode());

    if (testAction.isCoverageMode() && testAction.getSplitCoveragePostProcessing()) {
      if (testAction.getCoverageDirectoryTreeArtifact() == null) {
        // Otherwise we'll get a NPE https://github.com/bazelbuild/bazel/issues/13185
        TestExecException e =
            createTestExecException(
                TestAction.Code.LOCAL_TEST_PREREQ_UNMET,
                "coverageDirectoryTreeArtifact is null:"
                    + " --experimental_split_coverage_postprocessing depends on"
                    + " --experimental_fetch_all_coverage_outputs being enabled");
        closeSuppressed(e, streamed);
        closeSuppressed(e, fileOutErr);
        throw e;
      }
      var unused =
          actionExecutionContext
              .getOutputMetadataStore()
              .getOutputMetadata(testAction.getCoverageDirectoryTreeArtifact());

      ImmutableSet<? extends ActionInput> expandedCoverageDir =
          actionExecutionContext
              .getOutputMetadataStore()
              .getTreeArtifactChildren(
                  (SpecialArtifact) testAction.getCoverageDirectoryTreeArtifact());
      Spawn coveragePostProcessingSpawn =
          createCoveragePostProcessingSpawn(
              actionExecutionContext,
              testAction,
              ImmutableList.copyOf(expandedCoverageDir),
              tmpDirRoot,
              executionOptions.splitXmlGeneration);
      SpawnStrategyResolver spawnStrategyResolver =
          actionExecutionContext.getContext(SpawnStrategyResolver.class);

      Path testRoot =
          actionExecutionContext.getInputPath(testAction.getTestLog()).getParentDirectory();

      Path out = testRoot.getChild("coverage.log");
      Path err = testRoot.getChild("coverage.err");
      FileOutErr coverageOutErr = new FileOutErr(out, err);
      ActionExecutionContext coverageActionExecutionContext =
          actionExecutionContext
              .withFileOutErr(coverageOutErr)
              .withOutputsAsInputs(expandedCoverageDir);

      writeOutFile(coverageOutErr.getErrorPath(), coverageOutErr.getOutputPath());
      appendCoverageLog(coverageOutErr, fileOutErr);
      try {
        spawnStrategyResolver.exec(coveragePostProcessingSpawn, coverageActionExecutionContext);
      } catch (SpawnExecException e) {
        if (e.isCatastrophic()) {
          closeSuppressed(e, streamed);
          closeSuppressed(e, fileOutErr);
          throw e;
        }
        if (!e.getSpawnResult().setupSuccess()) {
          closeSuppressed(e, streamed);
          closeSuppressed(e, fileOutErr);
          // Rethrow as the test could not be run and thus there's no point in retrying.
          throw e;
        }
        testResultDataBuilder
            .setCachable(e.getSpawnResult().status().isConsideredUserError())
            .setTestPassed(false)
            .setStatus(e.hasTimedOut() ? BlazeTestStatus.TIMEOUT : BlazeTestStatus.FAILED);
      } catch (ExecException | InterruptedException e) {
        closeSuppressed(e, streamed);
        closeSuppressed(e, fileOutErr);
        throw e;
      }
    }

    Verify.verify(
        !(testAction.isCoverageMode() && testAction.getSplitCoveragePostProcessing())
            || actionExecutionContext
                .getPathResolver()
                .convertPath(testAction.getCoverageData().getPath())
                .exists());
    Verify.verifyNotNull(spawnResults);
    Verify.verifyNotNull(testResultDataBuilder);

    try {
      if (!fileOutErr.hasRecordedOutput()) {
        // Make sure that the test.log exists.Spaw
        FileSystemUtils.touchFile(fileOutErr.getOutputPath());
      }
      // Append any error output to the test.log. This is very rare.
      writeOutFile(fileOutErr.getErrorPath(), fileOutErr.getOutputPath());
      fileOutErr.close();
      if (streamed != null) {
        streamed.close();
      }
    } catch (IOException e) {
      throw new EnvironmentalExecException(e, Code.TEST_OUT_ERR_IO_EXCEPTION);
    }

    Path xmlOutputPath = resolvedPaths.getXmlOutputPath();

    // If the test did not create a test.xml, and --experimental_split_xml_generation is enabled,
    // then we run a separate action to create a test.xml from test.log. We do this as a spawn
    // rather than doing it locally in-process, as the test.log file may only exist remotely (when
    // remote execution is enabled), and we do not want to have to download it.
    if (executionOptions.splitXmlGeneration
        && fileOutErr.getOutputPath().exists()
        && !xmlOutputPath.exists()) {
      Spawn xmlGeneratingSpawn =
          createXmlGeneratingSpawn(testAction, spawn.getEnvironment(), spawnResults.get(0));
      SpawnStrategyResolver spawnStrategyResolver =
          actionExecutionContext.getContext(SpawnStrategyResolver.class);
      // We treat all failures to generate the test.xml here as catastrophic, and won't rerun
      // the test if this fails. We redirect the output to a temporary file.
      FileOutErr xmlSpawnOutErr = actionExecutionContext.getFileOutErr().childOutErr();

      ActionExecutionContext xmlActionExecutionContext =
          actionExecutionContext
              .withFileOutErr(xmlSpawnOutErr)
              .withOutputsAsInputs(ImmutableList.of(testAction.getTestLog()));
      try {

        ImmutableList<SpawnResult> xmlSpawnResults =
            spawnStrategyResolver.exec(xmlGeneratingSpawn, xmlActionExecutionContext);
        spawnResults =
            ImmutableList.<SpawnResult>builder()
                .addAll(spawnResults)
                .addAll(xmlSpawnResults)
                .build();
      } catch (InterruptedException | ExecException e) {
        closeSuppressed(e, xmlSpawnOutErr);
        throw e;
      }
    }

    TestCase details = parseTestResult(xmlOutputPath);
    if (details != null) {
      testResultDataBuilder.setTestCase(details);
    }

    BuildEventStreamProtos.TestResult.ExecutionInfo executionInfo =
        extractExecutionInfo(spawnResults.get(0), testResultDataBuilder);
    return StandaloneTestResult.builder()
        .setSpawnResults(spawnResults)
        // We return the TestResultData.Builder rather than the finished TestResultData
        // instance, as we may have to rename the output files in case the test needs to be
        // rerun (if it failed here _and_ is marked flaky _and_ the number of flaky attempts
        // is larger than 1).
        .setTestResultDataBuilder(testResultDataBuilder)
        .setExecutionInfo(executionInfo)
        .build();
  }
}