src/main/java/com/google/devtools/build/lib/exec/TestStrategy.java - bazel - Git at Google

 // Copyright 2014 The Bazel Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 // http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package com.google.devtools.build.lib.exec;

 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.io.ByteStreams;
 import com.google.common.io.Closeables;
 import com.google.devtools.build.lib.actions.ActionExecutionContext;
 import com.google.devtools.build.lib.actions.ExecException;
 import com.google.devtools.build.lib.actions.Executor;
 import com.google.devtools.build.lib.analysis.config.BinTools;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.profiler.Profiler;
 import com.google.devtools.build.lib.profiler.ProfilerTask;
 import com.google.devtools.build.lib.rules.test.TestActionContext;
 import com.google.devtools.build.lib.rules.test.TestResult;
 import com.google.devtools.build.lib.rules.test.TestRunnerAction;
 import com.google.devtools.build.lib.rules.test.TestTargetExecutionSettings;
 import com.google.devtools.build.lib.util.OS;
 import com.google.devtools.build.lib.util.ShellEscaper;
 import com.google.devtools.build.lib.util.io.FileWatcher;
 import com.google.devtools.build.lib.util.io.OutErr;
 import com.google.devtools.build.lib.vfs.FileStatus;
 import com.google.devtools.build.lib.vfs.FileSystemUtils;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import com.google.devtools.build.lib.vfs.SearchPath;
 import com.google.devtools.build.lib.view.test.TestStatus.TestCase;
 import com.google.devtools.common.options.Converters.RangeConverter;
 import com.google.devtools.common.options.EnumConverter;
 import com.google.devtools.common.options.OptionsClassProvider;
 import com.google.devtools.common.options.OptionsParsingException;
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import javax.annotation.Nullable;

 /** A strategy for executing a {@link TestRunnerAction}. */
 public abstract class TestStrategy implements TestActionContext {
   public static final PathFragment COVERAGE_TMP_ROOT = new PathFragment("_coverage");

   public static final String TEST_SETUP_BASENAME = "test-setup.sh";

   /** Returns true if coverage data should be gathered. */
   protected static boolean isCoverageMode(TestRunnerAction action) {
     return action.getCoverageData() != null;
   }

   /**
    * Ensures that all directories used to run test are in the correct state and their content will
    * not result in stale files.
    */
   protected void prepareFileSystem(
       TestRunnerAction testAction, Path tmpDir, Path coverageDir, Path workingDirectory)
       throws IOException {
     if (isCoverageMode(testAction)) {
       try {
         recreateDirectory(coverageDir);
       } catch (IOException e) {
         throw new IOException("Failed to recreate coverage directory " + coverageDir, e);
       }
     }
     try {
       recreateDirectory(tmpDir);
     } catch (IOException e) {
       throw new IOException("Failed to recreate temporary directory " + tmpDir, e);
     }
     FileSystemUtils.createDirectoryAndParents(workingDirectory);
   }

   /** Removes directory if it exists and recreates it. */
   protected void recreateDirectory(Path directory) throws IOException {
     FileSystemUtils.deleteTree(directory);
     FileSystemUtils.createDirectoryAndParents(directory);
   }

   /** Converter for the --flaky_test_attempts option. */
   public static class TestAttemptsConverter extends RangeConverter {
     public TestAttemptsConverter() {
       super(1, 10);
     }

     @Override
     public Integer convert(String input) throws OptionsParsingException {
       if ("default".equals(input)) {
         return -1;
       } else {
         return super.convert(input);
       }
     }

     @Override
     public String getTypeDescription() {
       return super.getTypeDescription() + " or the string \"default\"";
     }
   }

   public enum TestOutputFormat {
     SUMMARY, // Provide summary output only.
     ERRORS, // Print output from failed tests to the stderr after the test failure.
     ALL, // Print output from all tests to the stderr after the test completion.
     STREAMED; // Stream output for each test.

     /** Converts to {@link TestOutputFormat}. */
     public static class Converter extends EnumConverter<TestOutputFormat> {
       public Converter() {
         super(TestOutputFormat.class, "test output");
       }
     }
   }

   public enum TestSummaryFormat {
     SHORT, // Print information only about tests.
     TERSE, // Like "SHORT", but even shorter: Do not print PASSED tests.
     DETAILED, // Print information only about failed test cases.
     NONE; // Do not print summary.

     /** Converts to {@link TestSummaryFormat}. */
     public static class Converter extends EnumConverter<TestSummaryFormat> {
       public Converter() {
         super(TestSummaryFormat.class, "test summary");
       }
     }
   }

   public static final PathFragment TEST_TMP_ROOT = new PathFragment("_tmp");

   // Used for selecting subset of testcase / testmethods.
   private static final String TEST_BRIDGE_TEST_FILTER_ENV = "TESTBRIDGE_TEST_ONLY";

   // Used for generating unique temporary directory names. Contains the next numeric index for every
   // executable base name.
   private final Map<String, Integer> tmpIndex = new HashMap<>();
   protected final ImmutableMap<String, String> clientEnv;
   protected final ExecutionOptions executionOptions;
   protected final BinTools binTools;

   public TestStrategy(
       OptionsClassProvider requestOptionsProvider,
       BinTools binTools,
       Map<String, String> clientEnv) {
     this.executionOptions = requestOptionsProvider.getOptions(ExecutionOptions.class);
     this.binTools = binTools;
     this.clientEnv = ImmutableMap.copyOf(clientEnv);
   }

   @Override
   public abstract void exec(TestRunnerAction action, ActionExecutionContext actionExecutionContext)
       throws ExecException, InterruptedException;

   /** Returns true if coverage data should be gathered. */
   protected static boolean isMicroCoverageMode(TestRunnerAction action) {
     return action.getMicroCoverageData() != null;
   }

   /**
    * Returns directory to store coverage results for the given action relative to the execution
    * root. This directory is used to store all coverage results related to the test execution with
    * exception of the locally generated *.gcda files. Those are stored separately using relative
    * path within coverage directory.
    *
    * <p>Coverage directory name for the given test runner action is constructed as: {@code $(blaze
    * info execution_root)/_coverage/target_path/test_log_name} where {@code test_log_name} is
    * usually a target name but potentially can include extra suffix, such as a shard number (if test
    * execution was sharded).
    */
   protected static PathFragment getCoverageDirectory(TestRunnerAction action) {
     return COVERAGE_TMP_ROOT.getRelative(
         FileSystemUtils.removeExtension(action.getTestLog().getRootRelativePath()));
   }

   /**
    * Returns mutable map of default testing shell environment. By itself it is incomplete and is
    * modified further by the specific test strategy implementations (mostly due to the fact that
    * environments used locally and remotely are different).
    */
   protected Map<String, String> getDefaultTestEnvironment(TestRunnerAction action) {
     Map<String, String> env = new HashMap<>();

     env.putAll(action.getConfiguration().getLocalShellEnvironment());
     env.remove("LANG");
     env.put("TZ", "UTC");
     env.put("TEST_SIZE", action.getTestProperties().getSize().toString());
     env.put("TEST_TIMEOUT", Integer.toString(getTimeout(action)));

     if (action.isSharded()) {
       env.put("TEST_SHARD_INDEX", Integer.toString(action.getShardNum()));
       env.put(
           "TEST_TOTAL_SHARDS", Integer.toString(action.getExecutionSettings().getTotalShards()));
     }

     // When we run test multiple times, set different TEST_RANDOM_SEED values for each run.
     if (action.getConfiguration().getRunsPerTestForLabel(action.getOwner().getLabel()) > 1) {
       env.put("TEST_RANDOM_SEED", Integer.toString(action.getRunNumber() + 1));
     }

     String testFilter = action.getExecutionSettings().getTestFilter();
     if (testFilter != null) {
       env.put(TEST_BRIDGE_TEST_FILTER_ENV, testFilter);
     }

     if (isCoverageMode(action)) {
       env.put(
           "COVERAGE_MANIFEST",
           action.getExecutionSettings().getInstrumentedFileManifest().getExecPathString());
       // Instruct remote-runtest.sh/local-runtest.sh not to cd into the runfiles directory.
       env.put("RUNTEST_PRESERVE_CWD", "1");
       env.put("MICROCOVERAGE_REQUESTED", isMicroCoverageMode(action) ? "true" : "false");
     }

     return env;
   }

   /**
    * Generates a command line to run for the test action, taking into account coverage and {@code
    * --run_under} settings.
    *
    * @param testScript the setup script that invokes the test
    * @param coverageScript a script interjected between setup script and rest of command line to
    *     collect coverage data. If this is an empty string, it is ignored.
    * @param testAction The test action.
    * @return the command line as string list.
    */
   protected List<String> getArgs(
       String testScript, String coverageScript, TestRunnerAction testAction) {
     List<String> args = Lists.newArrayList();
     if (OS.getCurrent() == OS.WINDOWS) {
       args.add(testAction.getShExecutable().getPathString());
       args.add("-c");
       args.add("$0 $*");
     }
     args.add(testScript);
     TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();

     List<String> execArgs = new ArrayList<>();
     if (!coverageScript.isEmpty() && isCoverageMode(testAction)) {
       execArgs.add(coverageScript);
     }

     // Execute the test using the alias in the runfiles tree, as mandated by
     // the Test Encyclopedia.
     execArgs.add(execSettings.getExecutable().getRootRelativePath().getPathString());
     execArgs.addAll(execSettings.getArgs());

     // Insert the command prefix specified by the "--run_under=<command-prefix>" option,
     // if any.
     if (execSettings.getRunUnder() == null) {
       args.addAll(execArgs);
     } else if (execSettings.getRunUnderExecutable() != null) {
       args.add(execSettings.getRunUnderExecutable().getRootRelativePath().getPathString());
       args.addAll(execSettings.getRunUnder().getOptions());
       args.addAll(execArgs);
     } else {
       args.add(testAction.getConfiguration().getShellExecutable().getPathString());
       args.add("-c");

       String runUnderCommand = ShellEscaper.escapeString(execSettings.getRunUnder().getCommand());

       Path fullySpecified =
           SearchPath.which(
               SearchPath.parse(
                   testAction.getTestLog().getPath().getFileSystem(), clientEnv.get("PATH")),
               runUnderCommand);

       if (fullySpecified != null) {
         runUnderCommand = fullySpecified.toString();
       }

       args.add(
           runUnderCommand
               + ' '
               + ShellEscaper.escapeJoinAll(
                   Iterables.concat(execSettings.getRunUnder().getOptions(), execArgs)));
     }
     return args;
   }

   /**
    * Returns the number of attempts specific test action can be retried.
    *
    * <p>For rules with "flaky = 1" attribute, this method will return 3 unless --flaky_test_attempts
    * option is given and specifies another value.
    */
   @VisibleForTesting /* protected */
   public int getTestAttempts(TestRunnerAction action) {
     if (executionOptions.testAttempts == -1) {
       return action.getTestProperties().isFlaky() ? 3 : 1;
     } else {
       return executionOptions.testAttempts;
     }
   }

   /**
    * Returns timeout value in seconds that should be used for the given test action. We always use
    * the "categorical timeouts" which are based on the --test_timeout flag. A rule picks its timeout
    * but ends up with the same effective value as all other rules in that bucket.
    */
   protected final int getTimeout(TestRunnerAction testAction) {
     return executionOptions.testTimeout.get(testAction.getTestProperties().getTimeout());
   }

   /**
    * Returns a subset of the environment from the current shell.
    *
    * <p>Warning: Since these variables are not part of the configuration's fingerprint, they MUST
    * NOT be used by any rule or action in such a way as to affect the semantics of that build step.
    */
   public Map<String, String> getAdmissibleShellEnvironment(Iterable<String> variables) {
     return getMapping(variables, clientEnv);
   }

   /*
    * Finalize test run: persist the result, and post on the event bus.
    */
   protected void postTestResult(Executor executor, TestResult result) throws IOException {
     result.getTestAction().saveCacheStatus(result.getData());
     executor.getEventBus().post(result);
   }

   /**
    * Returns a unique name for a temporary directory a test could use.
    *
    * <p>Since each test within single Blaze run must have a unique TEST_TMPDIR, we will use rule
    * name and a unique (within single Blaze request) number to generate directory name.
    *
    * <p>This does not create the directory.
    */
   protected String getTmpDirName(PathFragment execPath) {
     String basename = execPath.getBaseName();

     synchronized (tmpIndex) {
       int index = tmpIndex.containsKey(basename) ? tmpIndex.get(basename) : 1;
       tmpIndex.put(basename, index + 1);
       return basename + "_" + index;
     }
   }

   /** Parse a test result XML file into a {@link TestCase}. */
   @Nullable
   protected TestCase parseTestResult(Path resultFile) {
     /* xml files. We avoid parsing it unnecessarily, since test results can potentially consume
     a large amount of memory. */
     if (executionOptions.testSummary != TestSummaryFormat.DETAILED) {
       return null;
     }

     try (InputStream fileStream = resultFile.getInputStream()) {
       return new TestXmlOutputParser().parseXmlIntoTestResult(fileStream);
     } catch (IOException | TestXmlOutputParserException e) {
       return null;
     }
   }

   /**
    * Returns a temporary directory for all tests in a workspace to use. Individual tests should
    * create child directories to actually use.
    *
    * <p>This either dynamically generates a directory name or uses the directory specified by
    * --test_tmpdir. This does not create the directory.
    */
   public static Path getTmpRoot(Path workspace, Path execRoot, ExecutionOptions executionOptions) {
     return executionOptions.testTmpDir != null
         ? workspace.getRelative(executionOptions.testTmpDir).getRelative(TEST_TMP_ROOT)
         : execRoot.getRelative(TEST_TMP_ROOT);
   }

   /**
    * For an given environment, returns a subset containing all variables in the given list if they
    * are defined in the given environment.
    */
   @VisibleForTesting
   public static Map<String, String> getMapping(
       Iterable<String> variables, Map<String, String> environment) {
     Map<String, String> result = new HashMap<>();
     for (String var : variables) {
       if (environment.containsKey(var)) {
         result.put(var, environment.get(var));
       }
     }
     return result;
   }

   /**
    * Returns the runfiles directory associated with the test executable, creating/updating it if
    * necessary and --build_runfile_links is specified.
    */
   protected static Path getLocalRunfilesDirectory(
       TestRunnerAction testAction,
       ActionExecutionContext actionExecutionContext,
       BinTools binTools,
       ImmutableMap<String, String> shellEnvironment,
       boolean enableRunfiles)
       throws ExecException, InterruptedException {
     TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();

     // If the symlink farm is already created then return the existing directory. If not we
     // need to explicitly build it. This can happen when --nobuild_runfile_links is supplied
     // as a flag to the build.
     if (execSettings.getRunfilesSymlinksCreated()) {
       return execSettings.getRunfilesDir();
     }

     // TODO(bazel-team): Should we be using TestTargetExecutionSettings#getRunfilesDir() here over
     // generating the directory ourselves?
     Path program = execSettings.getExecutable().getPath();
     Path runfilesDir = program.getParentDirectory().getChild(program.getBaseName() + ".runfiles");

     // Synchronize runfiles tree generation on the runfiles manifest artifact.
     // This is necessary, because we might end up with multiple test runner actions
     // trying to generate same runfiles tree in case of --runs_per_test > 1 or
     // local test sharding.
     long startTime = Profiler.nanoTimeMaybe();
     synchronized (execSettings.getInputManifest()) {
       Profiler.instance().logSimpleTask(startTime, ProfilerTask.WAIT, testAction);
       updateLocalRunfilesDirectory(
           testAction,
           runfilesDir,
           actionExecutionContext,
           binTools,
           shellEnvironment,
           enableRunfiles);
     }

     return runfilesDir;
   }

   /**
    * Ensure the runfiles tree exists and is consistent with the TestAction's manifest
    * ($0.runfiles_manifest), bringing it into consistency if not. The contents of the output file
    * $0.runfiles/MANIFEST, if it exists, are used a proxy for the set of existing symlinks, to avoid
    * the need for recursion.
    */
   private static void updateLocalRunfilesDirectory(
       TestRunnerAction testAction,
       Path runfilesDir,
       ActionExecutionContext actionExecutionContext,
       BinTools binTools,
       ImmutableMap<String, String> shellEnvironment,
       boolean enableRunfiles)
       throws ExecException, InterruptedException {
     Executor executor = actionExecutionContext.getExecutor();

     TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();
     try {
       // Avoid rebuilding the runfiles directory if the manifest in it matches the input manifest,
       // implying the symlinks exist and are already up to date.
       if (Arrays.equals(
           runfilesDir.getRelative("MANIFEST").getDigest(),
           execSettings.getInputManifest().getPath().getDigest())) {
         return;
       }
     } catch (IOException e1) {
       // Ignore it - we will just try to create runfiles directory.
     }

     executor
         .getEventHandler()
         .handle(
             Event.progress(
                 "Building runfiles directory for '"
                     + execSettings.getExecutable().prettyPrint()
                     + "'."));

     new SymlinkTreeHelper(execSettings.getInputManifest().getPath(), runfilesDir, false)
         .createSymlinks(
             testAction, actionExecutionContext, binTools, shellEnvironment, enableRunfiles);

     executor.getEventHandler().handle(Event.progress(testAction.getProgressMessage()));
   }

   /** In rare cases, we might write something to stderr. Append it to the real test.log. */
   protected static void appendStderr(Path stdOut, Path stdErr) throws IOException {
     FileStatus stat = stdErr.statNullable();
     OutputStream out = null;
     InputStream in = null;
     if (stat != null) {
       try {
         if (stat.getSize() > 0) {
           if (stdOut.exists()) {
             stdOut.setWritable(true);
           }
           out = stdOut.getOutputStream(true);
           in = stdErr.getInputStream();
           ByteStreams.copy(in, out);
         }
       } finally {
         Closeables.close(out, true);
         Closeables.close(in, true);
         stdErr.delete();
       }
     }
   }

   /** Implements the --test_output=streamed option. */
   protected static class StreamedTestOutput implements Closeable {
     private final TestLogHelper.FilterTestHeaderOutputStream headerFilter;
     private final FileWatcher watcher;
     private final Path testLogPath;
     private final OutErr outErr;

     public StreamedTestOutput(OutErr outErr, Path testLogPath) throws IOException {
       this.testLogPath = testLogPath;
       this.outErr = outErr;
       this.headerFilter = TestLogHelper.getHeaderFilteringOutputStream(outErr.getOutputStream());
       this.watcher = new FileWatcher(testLogPath, OutErr.create(headerFilter, headerFilter), false);
       watcher.start();
     }

     @Override
     public void close() throws IOException {
       watcher.stopPumping();
       try {
         // The watcher thread might leak if the following call is interrupted.
         // This is a relatively minor issue since the worst it could do is
         // write one additional line from the test.log to the console later on
         // in the build.
         watcher.join();
       } catch (InterruptedException e) {
         Thread.currentThread().interrupt();
       }
       if (!headerFilter.foundHeader()) {
         try (InputStream input = testLogPath.getInputStream()) {
           ByteStreams.copy(input, outErr.getOutputStream());
         }
       }
     }
   }
 }
	// Copyright 2014 The Bazel Authors. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package com.google.devtools.build.lib.exec;

	import com.google.common.annotations.VisibleForTesting;
	import com.google.common.collect.ImmutableMap;
	import com.google.common.collect.Iterables;
	import com.google.common.collect.Lists;
	import com.google.common.io.ByteStreams;
	import com.google.common.io.Closeables;
	import com.google.devtools.build.lib.actions.ActionExecutionContext;
	import com.google.devtools.build.lib.actions.ExecException;
	import com.google.devtools.build.lib.actions.Executor;
	import com.google.devtools.build.lib.analysis.config.BinTools;
	import com.google.devtools.build.lib.events.Event;
	import com.google.devtools.build.lib.profiler.Profiler;
	import com.google.devtools.build.lib.profiler.ProfilerTask;
	import com.google.devtools.build.lib.rules.test.TestActionContext;
	import com.google.devtools.build.lib.rules.test.TestResult;
	import com.google.devtools.build.lib.rules.test.TestRunnerAction;
	import com.google.devtools.build.lib.rules.test.TestTargetExecutionSettings;
	import com.google.devtools.build.lib.util.OS;
	import com.google.devtools.build.lib.util.ShellEscaper;
	import com.google.devtools.build.lib.util.io.FileWatcher;
	import com.google.devtools.build.lib.util.io.OutErr;
	import com.google.devtools.build.lib.vfs.FileStatus;
	import com.google.devtools.build.lib.vfs.FileSystemUtils;
	import com.google.devtools.build.lib.vfs.Path;
	import com.google.devtools.build.lib.vfs.PathFragment;
	import com.google.devtools.build.lib.vfs.SearchPath;
	import com.google.devtools.build.lib.view.test.TestStatus.TestCase;
	import com.google.devtools.common.options.Converters.RangeConverter;
	import com.google.devtools.common.options.EnumConverter;
	import com.google.devtools.common.options.OptionsClassProvider;
	import com.google.devtools.common.options.OptionsParsingException;
	import java.io.Closeable;
	import java.io.IOException;
	import java.io.InputStream;
	import java.io.OutputStream;
	import java.util.ArrayList;
	import java.util.Arrays;
	import java.util.HashMap;
	import java.util.List;
	import java.util.Map;
	import javax.annotation.Nullable;

	/** A strategy for executing a {@link TestRunnerAction}. */
	public abstract class TestStrategy implements TestActionContext {
	public static final PathFragment COVERAGE_TMP_ROOT = new PathFragment("_coverage");

	public static final String TEST_SETUP_BASENAME = "test-setup.sh";

	/** Returns true if coverage data should be gathered. */
	protected static boolean isCoverageMode(TestRunnerAction action) {
	return action.getCoverageData() != null;
	}

	/**
	* Ensures that all directories used to run test are in the correct state and their content will
	* not result in stale files.
	*/
	protected void prepareFileSystem(
	TestRunnerAction testAction, Path tmpDir, Path coverageDir, Path workingDirectory)
	throws IOException {
	if (isCoverageMode(testAction)) {
	try {
	recreateDirectory(coverageDir);
	} catch (IOException e) {
	throw new IOException("Failed to recreate coverage directory " + coverageDir, e);
	}
	}
	try {
	recreateDirectory(tmpDir);
	} catch (IOException e) {
	throw new IOException("Failed to recreate temporary directory " + tmpDir, e);
	}
	FileSystemUtils.createDirectoryAndParents(workingDirectory);
	}

	/** Removes directory if it exists and recreates it. */
	protected void recreateDirectory(Path directory) throws IOException {
	FileSystemUtils.deleteTree(directory);
	FileSystemUtils.createDirectoryAndParents(directory);
	}

	/** Converter for the --flaky_test_attempts option. */
	public static class TestAttemptsConverter extends RangeConverter {
	public TestAttemptsConverter() {
	super(1, 10);
	}

	@Override
	public Integer convert(String input) throws OptionsParsingException {
	if ("default".equals(input)) {
	return -1;
	} else {
	return super.convert(input);
	}
	}

	@Override
	public String getTypeDescription() {
	return super.getTypeDescription() + " or the string \"default\"";
	}
	}

	public enum TestOutputFormat {
	SUMMARY, // Provide summary output only.
	ERRORS, // Print output from failed tests to the stderr after the test failure.
	ALL, // Print output from all tests to the stderr after the test completion.
	STREAMED; // Stream output for each test.

	/** Converts to {@link TestOutputFormat}. */
	public static class Converter extends EnumConverter<TestOutputFormat> {
	public Converter() {
	super(TestOutputFormat.class, "test output");
	}
	}
	}

	public enum TestSummaryFormat {
	SHORT, // Print information only about tests.
	TERSE, // Like "SHORT", but even shorter: Do not print PASSED tests.
	DETAILED, // Print information only about failed test cases.
	NONE; // Do not print summary.

	/** Converts to {@link TestSummaryFormat}. */
	public static class Converter extends EnumConverter<TestSummaryFormat> {
	public Converter() {
	super(TestSummaryFormat.class, "test summary");
	}
	}
	}

	public static final PathFragment TEST_TMP_ROOT = new PathFragment("_tmp");

	// Used for selecting subset of testcase / testmethods.
	private static final String TEST_BRIDGE_TEST_FILTER_ENV = "TESTBRIDGE_TEST_ONLY";

	// Used for generating unique temporary directory names. Contains the next numeric index for every
	// executable base name.
	private final Map<String, Integer> tmpIndex = new HashMap<>();
	protected final ImmutableMap<String, String> clientEnv;
	protected final ExecutionOptions executionOptions;
	protected final BinTools binTools;

	public TestStrategy(
	OptionsClassProvider requestOptionsProvider,
	BinTools binTools,
	Map<String, String> clientEnv) {
	this.executionOptions = requestOptionsProvider.getOptions(ExecutionOptions.class);
	this.binTools = binTools;
	this.clientEnv = ImmutableMap.copyOf(clientEnv);
	}

	@Override
	public abstract void exec(TestRunnerAction action, ActionExecutionContext actionExecutionContext)
	throws ExecException, InterruptedException;

	/** Returns true if coverage data should be gathered. */
	protected static boolean isMicroCoverageMode(TestRunnerAction action) {
	return action.getMicroCoverageData() != null;
	}

	/**
	* Returns directory to store coverage results for the given action relative to the execution
	* root. This directory is used to store all coverage results related to the test execution with
	* exception of the locally generated *.gcda files. Those are stored separately using relative
	* path within coverage directory.
	*
	* <p>Coverage directory name for the given test runner action is constructed as: {@code $(blaze
	* info execution_root)/_coverage/target_path/test_log_name} where {@code test_log_name} is
	* usually a target name but potentially can include extra suffix, such as a shard number (if test
	* execution was sharded).
	*/
	protected static PathFragment getCoverageDirectory(TestRunnerAction action) {
	return COVERAGE_TMP_ROOT.getRelative(
	FileSystemUtils.removeExtension(action.getTestLog().getRootRelativePath()));
	}

	/**
	* Returns mutable map of default testing shell environment. By itself it is incomplete and is
	* modified further by the specific test strategy implementations (mostly due to the fact that
	* environments used locally and remotely are different).
	*/
	protected Map<String, String> getDefaultTestEnvironment(TestRunnerAction action) {
	Map<String, String> env = new HashMap<>();

	env.putAll(action.getConfiguration().getLocalShellEnvironment());
	env.remove("LANG");
	env.put("TZ", "UTC");
	env.put("TEST_SIZE", action.getTestProperties().getSize().toString());
	env.put("TEST_TIMEOUT", Integer.toString(getTimeout(action)));

	if (action.isSharded()) {
	env.put("TEST_SHARD_INDEX", Integer.toString(action.getShardNum()));
	env.put(
	"TEST_TOTAL_SHARDS", Integer.toString(action.getExecutionSettings().getTotalShards()));
	}

	// When we run test multiple times, set different TEST_RANDOM_SEED values for each run.
	if (action.getConfiguration().getRunsPerTestForLabel(action.getOwner().getLabel()) > 1) {
	env.put("TEST_RANDOM_SEED", Integer.toString(action.getRunNumber() + 1));
	}

	String testFilter = action.getExecutionSettings().getTestFilter();
	if (testFilter != null) {
	env.put(TEST_BRIDGE_TEST_FILTER_ENV, testFilter);
	}

	if (isCoverageMode(action)) {
	env.put(
	"COVERAGE_MANIFEST",
	action.getExecutionSettings().getInstrumentedFileManifest().getExecPathString());
	// Instruct remote-runtest.sh/local-runtest.sh not to cd into the runfiles directory.
	env.put("RUNTEST_PRESERVE_CWD", "1");
	env.put("MICROCOVERAGE_REQUESTED", isMicroCoverageMode(action) ? "true" : "false");
	}

	return env;
	}

	/**
	* Generates a command line to run for the test action, taking into account coverage and {@code
	* --run_under} settings.
	*
	* @param testScript the setup script that invokes the test
	* @param coverageScript a script interjected between setup script and rest of command line to
	* collect coverage data. If this is an empty string, it is ignored.
	* @param testAction The test action.
	* @return the command line as string list.
	*/
	protected List<String> getArgs(
	String testScript, String coverageScript, TestRunnerAction testAction) {
	List<String> args = Lists.newArrayList();
	if (OS.getCurrent() == OS.WINDOWS) {
	args.add(testAction.getShExecutable().getPathString());
	args.add("-c");
	args.add("$0 $*");
	}
	args.add(testScript);
	TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();

	List<String> execArgs = new ArrayList<>();
	if (!coverageScript.isEmpty() && isCoverageMode(testAction)) {
	execArgs.add(coverageScript);
	}

	// Execute the test using the alias in the runfiles tree, as mandated by
	// the Test Encyclopedia.
	execArgs.add(execSettings.getExecutable().getRootRelativePath().getPathString());
	execArgs.addAll(execSettings.getArgs());

	// Insert the command prefix specified by the "--run_under=<command-prefix>" option,
	// if any.
	if (execSettings.getRunUnder() == null) {
	args.addAll(execArgs);
	} else if (execSettings.getRunUnderExecutable() != null) {
	args.add(execSettings.getRunUnderExecutable().getRootRelativePath().getPathString());
	args.addAll(execSettings.getRunUnder().getOptions());
	args.addAll(execArgs);
	} else {
	args.add(testAction.getConfiguration().getShellExecutable().getPathString());
	args.add("-c");

	String runUnderCommand = ShellEscaper.escapeString(execSettings.getRunUnder().getCommand());

	Path fullySpecified =
	SearchPath.which(
	SearchPath.parse(
	testAction.getTestLog().getPath().getFileSystem(), clientEnv.get("PATH")),
	runUnderCommand);

	if (fullySpecified != null) {
	runUnderCommand = fullySpecified.toString();
	}

	args.add(
	runUnderCommand
	+ ' '
	+ ShellEscaper.escapeJoinAll(
	Iterables.concat(execSettings.getRunUnder().getOptions(), execArgs)));
	}
	return args;
	}

	/**
	* Returns the number of attempts specific test action can be retried.
	*
	* <p>For rules with "flaky = 1" attribute, this method will return 3 unless --flaky_test_attempts
	* option is given and specifies another value.
	*/
	@VisibleForTesting /* protected */
	public int getTestAttempts(TestRunnerAction action) {
	if (executionOptions.testAttempts == -1) {
	return action.getTestProperties().isFlaky() ? 3 : 1;
	} else {
	return executionOptions.testAttempts;
	}
	}

	/**
	* Returns timeout value in seconds that should be used for the given test action. We always use
	* the "categorical timeouts" which are based on the --test_timeout flag. A rule picks its timeout
	* but ends up with the same effective value as all other rules in that bucket.
	*/
	protected final int getTimeout(TestRunnerAction testAction) {
	return executionOptions.testTimeout.get(testAction.getTestProperties().getTimeout());
	}

	/**
	* Returns a subset of the environment from the current shell.
	*
	* <p>Warning: Since these variables are not part of the configuration's fingerprint, they MUST
	* NOT be used by any rule or action in such a way as to affect the semantics of that build step.
	*/
	public Map<String, String> getAdmissibleShellEnvironment(Iterable<String> variables) {
	return getMapping(variables, clientEnv);
	}

	/*
	* Finalize test run: persist the result, and post on the event bus.
	*/
	protected void postTestResult(Executor executor, TestResult result) throws IOException {
	result.getTestAction().saveCacheStatus(result.getData());
	executor.getEventBus().post(result);
	}

	/**
	* Returns a unique name for a temporary directory a test could use.
	*
	* <p>Since each test within single Blaze run must have a unique TEST_TMPDIR, we will use rule
	* name and a unique (within single Blaze request) number to generate directory name.
	*
	* <p>This does not create the directory.
	*/
	protected String getTmpDirName(PathFragment execPath) {
	String basename = execPath.getBaseName();

	synchronized (tmpIndex) {
	int index = tmpIndex.containsKey(basename) ? tmpIndex.get(basename) : 1;
	tmpIndex.put(basename, index + 1);
	return basename + "_" + index;
	}
	}

	/** Parse a test result XML file into a {@link TestCase}. */
	@Nullable
	protected TestCase parseTestResult(Path resultFile) {
	/* xml files. We avoid parsing it unnecessarily, since test results can potentially consume
	a large amount of memory. */
	if (executionOptions.testSummary != TestSummaryFormat.DETAILED) {
	return null;
	}

	try (InputStream fileStream = resultFile.getInputStream()) {
	return new TestXmlOutputParser().parseXmlIntoTestResult(fileStream);
	} catch (IOException \| TestXmlOutputParserException e) {
	return null;
	}
	}

	/**
	* Returns a temporary directory for all tests in a workspace to use. Individual tests should
	* create child directories to actually use.
	*
	* <p>This either dynamically generates a directory name or uses the directory specified by
	* --test_tmpdir. This does not create the directory.
	*/
	public static Path getTmpRoot(Path workspace, Path execRoot, ExecutionOptions executionOptions) {
	return executionOptions.testTmpDir != null
	? workspace.getRelative(executionOptions.testTmpDir).getRelative(TEST_TMP_ROOT)
	: execRoot.getRelative(TEST_TMP_ROOT);
	}

	/**
	* For an given environment, returns a subset containing all variables in the given list if they
	* are defined in the given environment.
	*/
	@VisibleForTesting
	public static Map<String, String> getMapping(
	Iterable<String> variables, Map<String, String> environment) {
	Map<String, String> result = new HashMap<>();
	for (String var : variables) {
	if (environment.containsKey(var)) {
	result.put(var, environment.get(var));
	}
	}
	return result;
	}

	/**
	* Returns the runfiles directory associated with the test executable, creating/updating it if
	* necessary and --build_runfile_links is specified.
	*/
	protected static Path getLocalRunfilesDirectory(
	TestRunnerAction testAction,
	ActionExecutionContext actionExecutionContext,
	BinTools binTools,
	ImmutableMap<String, String> shellEnvironment,
	boolean enableRunfiles)
	throws ExecException, InterruptedException {
	TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();

	// If the symlink farm is already created then return the existing directory. If not we
	// need to explicitly build it. This can happen when --nobuild_runfile_links is supplied
	// as a flag to the build.
	if (execSettings.getRunfilesSymlinksCreated()) {
	return execSettings.getRunfilesDir();
	}

	// TODO(bazel-team): Should we be using TestTargetExecutionSettings#getRunfilesDir() here over
	// generating the directory ourselves?
	Path program = execSettings.getExecutable().getPath();
	Path runfilesDir = program.getParentDirectory().getChild(program.getBaseName() + ".runfiles");

	// Synchronize runfiles tree generation on the runfiles manifest artifact.
	// This is necessary, because we might end up with multiple test runner actions
	// trying to generate same runfiles tree in case of --runs_per_test > 1 or
	// local test sharding.
	long startTime = Profiler.nanoTimeMaybe();
	synchronized (execSettings.getInputManifest()) {
	Profiler.instance().logSimpleTask(startTime, ProfilerTask.WAIT, testAction);
	updateLocalRunfilesDirectory(
	testAction,
	runfilesDir,
	actionExecutionContext,
	binTools,
	shellEnvironment,
	enableRunfiles);
	}

	return runfilesDir;
	}

	/**
	* Ensure the runfiles tree exists and is consistent with the TestAction's manifest
	* ($0.runfiles_manifest), bringing it into consistency if not. The contents of the output file
	* $0.runfiles/MANIFEST, if it exists, are used a proxy for the set of existing symlinks, to avoid
	* the need for recursion.
	*/
	private static void updateLocalRunfilesDirectory(
	TestRunnerAction testAction,
	Path runfilesDir,
	ActionExecutionContext actionExecutionContext,
	BinTools binTools,
	ImmutableMap<String, String> shellEnvironment,
	boolean enableRunfiles)
	throws ExecException, InterruptedException {
	Executor executor = actionExecutionContext.getExecutor();

	TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();
	try {
	// Avoid rebuilding the runfiles directory if the manifest in it matches the input manifest,
	// implying the symlinks exist and are already up to date.
	if (Arrays.equals(
	runfilesDir.getRelative("MANIFEST").getDigest(),
	execSettings.getInputManifest().getPath().getDigest())) {
	return;
	}
	} catch (IOException e1) {
	// Ignore it - we will just try to create runfiles directory.
	}

	executor
	.getEventHandler()
	.handle(
	Event.progress(
	"Building runfiles directory for '"
	+ execSettings.getExecutable().prettyPrint()
	+ "'."));

	new SymlinkTreeHelper(execSettings.getInputManifest().getPath(), runfilesDir, false)
	.createSymlinks(
	testAction, actionExecutionContext, binTools, shellEnvironment, enableRunfiles);

	executor.getEventHandler().handle(Event.progress(testAction.getProgressMessage()));
	}

	/** In rare cases, we might write something to stderr. Append it to the real test.log. */
	protected static void appendStderr(Path stdOut, Path stdErr) throws IOException {
	FileStatus stat = stdErr.statNullable();
	OutputStream out = null;
	InputStream in = null;
	if (stat != null) {
	try {
	if (stat.getSize() > 0) {
	if (stdOut.exists()) {
	stdOut.setWritable(true);
	}
	out = stdOut.getOutputStream(true);
	in = stdErr.getInputStream();
	ByteStreams.copy(in, out);
	}
	} finally {
	Closeables.close(out, true);
	Closeables.close(in, true);
	stdErr.delete();
	}
	}
	}

	/** Implements the --test_output=streamed option. */
	protected static class StreamedTestOutput implements Closeable {
	private final TestLogHelper.FilterTestHeaderOutputStream headerFilter;
	private final FileWatcher watcher;
	private final Path testLogPath;
	private final OutErr outErr;

	public StreamedTestOutput(OutErr outErr, Path testLogPath) throws IOException {
	this.testLogPath = testLogPath;
	this.outErr = outErr;
	this.headerFilter = TestLogHelper.getHeaderFilteringOutputStream(outErr.getOutputStream());
	this.watcher = new FileWatcher(testLogPath, OutErr.create(headerFilter, headerFilter), false);
	watcher.start();
	}

	@Override
	public void close() throws IOException {
	watcher.stopPumping();
	try {
	// The watcher thread might leak if the following call is interrupted.
	// This is a relatively minor issue since the worst it could do is
	// write one additional line from the test.log to the console later on
	// in the build.
	watcher.join();
	} catch (InterruptedException e) {
	Thread.currentThread().interrupt();
	}
	if (!headerFilter.foundHeader()) {
	try (InputStream input = testLogPath.getInputStream()) {
	ByteStreams.copy(input, outErr.getOutputStream());
	}
	}
	}
	}
	}