src/main/java/com/google/devtools/build/lib/exec/TestStrategy.java

// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.exec;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.io.ByteStreams;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.CommandLineExpansionException;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.TestExecException;
import com.google.devtools.build.lib.actions.UserExecException;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.config.PerLabelOptions;
import com.google.devtools.build.lib.analysis.test.TestActionContext;
import com.google.devtools.build.lib.analysis.test.TestConfiguration;
import com.google.devtools.build.lib.analysis.test.TestResult;
import com.google.devtools.build.lib.analysis.test.TestRunnerAction;
import com.google.devtools.build.lib.analysis.test.TestRunnerAction.ResolvedPaths;
import com.google.devtools.build.lib.analysis.test.TestTargetExecutionSettings;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.EventKind;
import com.google.devtools.build.lib.profiler.Profiler;
import com.google.devtools.build.lib.profiler.ProfilerTask;
import com.google.devtools.build.lib.util.Fingerprint;
import com.google.devtools.build.lib.util.OS;
import com.google.devtools.build.lib.util.io.FileWatcher;
import com.google.devtools.build.lib.util.io.OutErr;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
import com.google.devtools.build.lib.view.test.TestStatus.TestCase;
import com.google.devtools.build.lib.view.test.TestStatus.TestResultData;
import com.google.devtools.common.options.EnumConverter;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.time.Duration;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;

/** A strategy for executing a {@link TestRunnerAction}. */
public abstract class TestStrategy implements TestActionContext {

  /**
   * Ensures that all directories used to run test are in the correct state and their content will
   * not result in stale files.
   */
  protected void prepareFileSystem(
      TestRunnerAction testAction, Path execRoot, Path tmpDir, Path workingDirectory)
      throws IOException {
    if (tmpDir != null) {
      recreateDirectory(tmpDir);
    }
    if (workingDirectory != null) {
      workingDirectory.createDirectoryAndParents();
    }

    ResolvedPaths resolvedPaths = testAction.resolve(execRoot);
    if (testAction.isCoverageMode()) {
      recreateDirectory(resolvedPaths.getCoverageDirectory());
    }

    resolvedPaths.getBaseDir().createDirectoryAndParents();
    resolvedPaths.getUndeclaredOutputsDir().createDirectoryAndParents();
    resolvedPaths.getUndeclaredOutputsAnnotationsDir().createDirectoryAndParents();
    resolvedPaths.getSplitLogsDir().createDirectoryAndParents();
  }

  /**
   * Ensures that all directories used to run test are in the correct state and their content will
   * not result in stale files. Only use this if no local tmp and working directory are required.
   */
  protected void prepareFileSystem(TestRunnerAction testAction, Path execRoot) throws IOException {
    prepareFileSystem(testAction, execRoot, null, null);
  }

  /** Removes directory if it exists and recreates it. */
  private void recreateDirectory(Path directory) throws IOException {
    directory.deleteTree();
    directory.createDirectoryAndParents();
  }

  /** An enum for specifying different formats of test output. */
  public enum TestOutputFormat {
    SUMMARY, // Provide summary output only.
    ERRORS, // Print output from failed tests to the stderr after the test failure.
    ALL, // Print output from all tests to the stderr after the test completion.
    STREAMED; // Stream output for each test.

    /** Converts to {@link TestOutputFormat}. */
    public static class Converter extends EnumConverter<TestOutputFormat> {
      public Converter() {
        super(TestOutputFormat.class, "test output");
      }
    }
  }

  /** An enum for specifying different formatting styles of test summaries. */
  public enum TestSummaryFormat {
    SHORT, // Print information only about tests.
    TERSE, // Like "SHORT", but even shorter: Do not print PASSED and NO STATUS tests.
    DETAILED, // Print information only about failed test cases.
    NONE, // Do not print summary.
    TESTCASE; // Print summary in test case resolution, do not print detailed information about
    // failed test cases.

    /** Converts to {@link TestSummaryFormat}. */
    public static class Converter extends EnumConverter<TestSummaryFormat> {
      public Converter() {
        super(TestSummaryFormat.class, "test summary");
      }
    }
  }

  public static final PathFragment TEST_TMP_ROOT = PathFragment.create("_tmp");

  // Used for generating unique temporary directory names. Contains the next numeric index for every
  // executable base name.
  private final Map<String, Integer> tmpIndex = new HashMap<>();
  protected final ExecutionOptions executionOptions;
  protected final BinTools binTools;

  public TestStrategy(ExecutionOptions executionOptions, BinTools binTools) {
    this.executionOptions = executionOptions;
    this.binTools = binTools;
  }

  @Override
  public final boolean isTestKeepGoing() {
    return executionOptions.testKeepGoing;
  }

  /**
   * Generates a command line to run for the test action, taking into account coverage and {@code
   * --run_under} settings.
   *
   * @param testAction The test action.
   * @return the command line as string list.
   * @throws ExecException
   */
  public static ImmutableList<String> getArgs(TestRunnerAction testAction) throws ExecException {
    List<String> args = Lists.newArrayList();
    // TODO(ulfjack): `executedOnWindows` is incorrect for remote execution, where we need to
    // consider the target configuration, not the machine Bazel happens to run on. Change this to
    // something like: testAction.getConfiguration().getTargetOS() == OS.WINDOWS
    final boolean executedOnWindows = (OS.getCurrent() == OS.WINDOWS);
    final boolean useTestWrapper = testAction.isUsingTestWrapperInsteadOfTestSetupScript();

    if (executedOnWindows && !useTestWrapper) {
      // TestActionBuilder constructs TestRunnerAction with a 'null' shell path only when we use the
      // native test wrapper. Something clearly went wrong.
      Preconditions.checkNotNull(testAction.getShExecutableMaybe(), "%s", testAction);
      args.add(testAction.getShExecutableMaybe().getPathString());
      args.add("-c");
      args.add("$0 \"$@\"");
    }

    Artifact testSetup = testAction.getTestSetupScript();
    args.add(testSetup.getExecPath().getCallablePathString());

    if (testAction.isCoverageMode()) {
      args.add(testAction.getCollectCoverageScript().getExecPathString());
    }

    TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();

    // Insert the command prefix specified by the "--run_under=<command-prefix>" option, if any.
    if (execSettings.getRunUnder() != null) {
      addRunUnderArgs(testAction, args, executedOnWindows);
    }

    // Execute the test using the alias in the runfiles tree, as mandated by the Test Encyclopedia.
    args.add(execSettings.getExecutable().getRootRelativePath().getCallablePathString());
    try {
      Iterables.addAll(args, execSettings.getArgs().arguments());
    } catch (CommandLineExpansionException e) {
      throw new UserExecException(e);
    }
    return ImmutableList.copyOf(args);
  }

  private static void addRunUnderArgs(
      TestRunnerAction testAction, List<String> args, boolean executedOnWindows) {
    TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();
    if (execSettings.getRunUnderExecutable() != null) {
      args.add(execSettings.getRunUnderExecutable().getRootRelativePath().getCallablePathString());
    } else {
      if (execSettings.needsShell(executedOnWindows)) {
        // TestActionBuilder constructs TestRunnerAction with a 'null' shell only when none is
        // required. Something clearly went wrong.
        Preconditions.checkNotNull(testAction.getShExecutableMaybe(), "%s", testAction);
        String shellExecutable = testAction.getShExecutableMaybe().getPathString();
        args.add(shellExecutable);
        args.add("-c");
        args.add("\"$@\"");
        args.add(shellExecutable); // Sets $0.
      }
      args.add(execSettings.getRunUnder().getCommand());
    }
    args.addAll(testAction.getExecutionSettings().getRunUnder().getOptions());
  }

  /**
   * Returns the number of attempts specific test action can be retried.
   *
   * <p>For rules with "flaky = 1" attribute, this method will return 3 unless --flaky_test_attempts
   * option is given and specifies another value.
   */
  @VisibleForTesting /* protected */
  public int getTestAttempts(TestRunnerAction action) {
    return action.getTestProperties().isFlaky()
        ? getTestAttemptsForFlakyTest(action)
        : getTestAttempts(action, /*defaultTestAttempts=*/ 1);
  }

  public int getTestAttemptsForFlakyTest(TestRunnerAction action) {
    return getTestAttempts(action, /*defaultTestAttempts=*/ 3);
  }

  private int getTestAttempts(TestRunnerAction action, int defaultTestAttempts) {
    Label testLabel = action.getOwner().getLabel();
    return getTestAttemptsPerLabel(executionOptions, testLabel, defaultTestAttempts);
  }

  private static int getTestAttemptsPerLabel(
      ExecutionOptions options, Label label, int defaultTestAttempts) {
    // Check from the last provided, so that the last option provided takes precedence.
    for (PerLabelOptions perLabelAttempts : Lists.reverse(options.testAttempts)) {
      if (perLabelAttempts.isIncluded(label)) {
        String attempts = Iterables.getOnlyElement(perLabelAttempts.getOptions());
        if ("default".equals(attempts)) {
          return defaultTestAttempts;
        }
        return Integer.parseInt(attempts);
      }
    }
    return defaultTestAttempts;
  }

  /**
   * Returns timeout value in seconds that should be used for the given test action. We always use
   * the "categorical timeouts" which are based on the --test_timeout flag. A rule picks its timeout
   * but ends up with the same effective value as all other rules in that bucket.
   */
  protected final Duration getTimeout(TestRunnerAction testAction) {
    BuildConfiguration configuration = testAction.getConfiguration();
    return configuration
        .getFragment(TestConfiguration.class)
        .getTestTimeout()
        .get(testAction.getTestProperties().getTimeout());
  }

  /*
   * Finalize test run: persist the result, and post on the event bus.
   */
  protected void postTestResult(ActionExecutionContext actionExecutionContext, TestResult result)
      throws IOException {
    result.getTestAction().saveCacheStatus(actionExecutionContext, result.getData());
    actionExecutionContext.getEventHandler().post(result);
  }

  /**
   * Returns a unique name for a temporary directory a test could use.
   *
   * <p>Since each test within single Blaze run must have a unique TEST_TMPDIR, we will use rule
   * name and a unique (within single Blaze request) number to generate directory name.
   *
   * <p>This does not create the directory.
   */
  protected String getTmpDirName(PathFragment execPath) {
    String basename = execPath.getBaseName();

    synchronized (tmpIndex) {
      int index = tmpIndex.containsKey(basename) ? tmpIndex.get(basename) : 1;
      tmpIndex.put(basename, index + 1);
      return basename + "_" + index;
    }
  }

  public static String getTmpDirName(TestRunnerAction action) {
    Fingerprint digest = new Fingerprint();
    digest.addPath(action.getExecutionSettings().getExecutable().getExecPath());
    digest.addInt(action.getShardNum());
    digest.addInt(action.getRunNumber());
    return digest.hexDigestAndReset();
  }

  /** Parse a test result XML file into a {@link TestCase}. */
  @Nullable
  protected TestCase parseTestResult(Path resultFile) {
    /* xml files. We avoid parsing it unnecessarily, since test results can potentially consume
    a large amount of memory. */
    if ((executionOptions.testSummary != TestSummaryFormat.DETAILED)
        && (executionOptions.testSummary != TestSummaryFormat.TESTCASE)) {
      return null;
    }

    try (InputStream fileStream = resultFile.getInputStream()) {
      return new TestXmlOutputParser().parseXmlIntoTestResult(fileStream);
    } catch (IOException | TestXmlOutputParserException e) {
      return null;
    }
  }

  /**
   * Outputs test result to the stdout after test has finished (e.g. for --test_output=all or
   * --test_output=errors). Will also try to group output lines together (up to 10000 lines) so
   * parallel test outputs will not get interleaved.
   */
  protected void processTestOutput(
      ActionExecutionContext actionExecutionContext,
      TestResultData testResultData,
      String testName,
      Path testLog)
      throws IOException {
    boolean isPassed = testResultData.getTestPassed();
    try {
      if (TestLogHelper.shouldOutputTestLog(executionOptions.testOutput, isPassed)) {
        TestLogHelper.writeTestLog(
            testLog, testName, actionExecutionContext.getFileOutErr().getOutputStream());
      }
    } finally {
      if (isPassed) {
        actionExecutionContext.getEventHandler().handle(Event.of(EventKind.PASS, null, testName));
      } else {
        if (testResultData.hasStatusDetails()) {
          actionExecutionContext
              .getEventHandler()
              .handle(Event.error(testName + ": " + testResultData.getStatusDetails()));
        }
        if (testResultData.getStatus() == BlazeTestStatus.TIMEOUT) {
          actionExecutionContext
              .getEventHandler()
              .handle(Event.of(EventKind.TIMEOUT, null, testName + " (see " + testLog + ")"));
        } else {
          actionExecutionContext
              .getEventHandler()
              .handle(Event.of(EventKind.FAIL, null, testName + " (see " + testLog + ")"));
        }
      }
    }
  }

  /**
   * Returns a temporary directory for all tests in a workspace to use. Individual tests should
   * create child directories to actually use.
   *
   * <p>This either dynamically generates a directory name or uses the directory specified by
   * --test_tmpdir. This does not create the directory.
   */
  public static Path getTmpRoot(Path workspace, Path execRoot, ExecutionOptions executionOptions) {
    return executionOptions.testTmpDir != null
        ? workspace.getRelative(executionOptions.testTmpDir).getRelative(TEST_TMP_ROOT)
        : execRoot.getRelative(TEST_TMP_ROOT);
  }

  /**
   * For an given environment, returns a subset containing all variables in the given list if they
   * are defined in the given environment.
   */
  @VisibleForTesting
  public static Map<String, String> getMapping(
      Iterable<String> variables, Map<String, String> environment) {
    Map<String, String> result = new HashMap<>();
    for (String var : variables) {
      if (environment.containsKey(var)) {
        result.put(var, environment.get(var));
      }
    }
    return result;
  }

  /**
   * Returns the runfiles directory associated with the test executable, creating/updating it if
   * necessary and --build_runfile_links is specified.
   */
  protected static Path getLocalRunfilesDirectory(
      TestRunnerAction testAction,
      ActionExecutionContext actionExecutionContext,
      BinTools binTools,
      ImmutableMap<String, String> shellEnvironment,
      boolean enableRunfiles)
      throws ExecException, InterruptedException {
    TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();

    if (execSettings.getInputManifest() == null) {
      throw new TestExecException("cannot run local tests with --nobuild_runfile_manifests");
    }

    Path runfilesDir =
        actionExecutionContext.getPathResolver().convertPath(execSettings.getRunfilesDir());

    // If the symlink farm is already created then return the existing directory. If not we
    // need to explicitly build it. This can happen when --nobuild_runfile_links is supplied
    // as a flag to the build.
    if (execSettings.getRunfilesSymlinksCreated()) {
      return runfilesDir;
    }

    // Synchronize runfiles tree generation on the runfiles manifest artifact.
    // This is necessary, because we might end up with multiple test runner actions
    // trying to generate same runfiles tree in case of --runs_per_test > 1 or
    // local test sharding.
    long startTime = Profiler.nanoTimeMaybe();
    synchronized (execSettings.getInputManifest()) {
      Profiler.instance().logSimpleTask(startTime, ProfilerTask.WAIT, testAction.describe());
      updateLocalRunfilesDirectory(
          testAction,
          runfilesDir,
          actionExecutionContext,
          binTools,
          shellEnvironment,
          enableRunfiles);
    }

    return runfilesDir;
  }

  /**
   * Ensure the runfiles tree exists and is consistent with the TestAction's manifest
   * ($0.runfiles_manifest), bringing it into consistency if not. The contents of the output file
   * $0.runfiles/MANIFEST, if it exists, are used a proxy for the set of existing symlinks, to avoid
   * the need for recursion.
   */
  private static void updateLocalRunfilesDirectory(
      TestRunnerAction testAction,
      Path runfilesDir,
      ActionExecutionContext actionExecutionContext,
      BinTools binTools,
      ImmutableMap<String, String> shellEnvironment,
      boolean enableRunfiles)
      throws ExecException, InterruptedException {
    TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();
    Path outputManifest = runfilesDir.getRelative("MANIFEST");
    try {
      // Avoid rebuilding the runfiles directory if the manifest in it matches the input manifest,
      // implying the symlinks exist and are already up to date. If the output manifest is a
      // symbolic link, it is likely a symbolic link to the input manifest, so we cannot trust it as
      // an up-to-date check.
      if (!outputManifest.isSymbolicLink()
          && Arrays.equals(
              outputManifest.getDigest(),
              actionExecutionContext.getInputPath(execSettings.getInputManifest()).getDigest())) {
        return;
      }
    } catch (IOException e1) {
      // Ignore it - we will just try to create runfiles directory.
    }

    actionExecutionContext
        .getEventHandler()
        .handle(
            Event.progress(
                "Building runfiles directory for '"
                    + execSettings.getExecutable().prettyPrint()
                    + "'."));

    new SymlinkTreeHelper(
            actionExecutionContext.getInputPath(execSettings.getInputManifest()),
            runfilesDir,
            false)
        .createSymlinks(actionExecutionContext, binTools, shellEnvironment, enableRunfiles);

    actionExecutionContext.getEventHandler()
        .handle(Event.progress(testAction.getProgressMessage()));
  }

  protected static void closeSuppressed(Throwable e, @Nullable Closeable c) {
    if (c == null) {
      return;
    }
    try {
      c.close();
    } catch (IOException e2) {
      e.addSuppressed(e2);
    }
  }

  /** Implements the --test_output=streamed option. */
  protected static class StreamedTestOutput implements Closeable {
    private final TestLogHelper.FilterTestHeaderOutputStream headerFilter;
    private final FileWatcher watcher;
    private final Path testLogPath;
    private final OutErr outErr;

    public StreamedTestOutput(OutErr outErr, Path testLogPath) throws IOException {
      this.testLogPath = testLogPath;
      this.outErr = outErr;
      this.headerFilter = TestLogHelper.getHeaderFilteringOutputStream(outErr.getOutputStream());
      this.watcher = new FileWatcher(testLogPath, OutErr.create(headerFilter, headerFilter), false);
      watcher.start();
    }

    @Override
    public void close() throws IOException {
      watcher.stopPumping();
      try {
        // The watcher thread might leak if the following call is interrupted.
        // This is a relatively minor issue since the worst it could do is
        // write one additional line from the test.log to the console later on
        // in the build.
        watcher.join();
      } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
      }
      if (!headerFilter.foundHeader()) {
        try (InputStream input = testLogPath.getInputStream()) {
          ByteStreams.copy(input, outErr.getOutputStream());
        }
      }
    }
  }
}