blob: 3430486829e266d6dd663344185ad110c700c67d [file] [log] [blame]
// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.exec;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.io.ByteStreams;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionOwner;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.CommandLineExpansionException;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.UserExecException;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.config.PerLabelOptions;
import com.google.devtools.build.lib.analysis.test.TestActionContext;
import com.google.devtools.build.lib.analysis.test.TestConfiguration;
import com.google.devtools.build.lib.analysis.test.TestResult;
import com.google.devtools.build.lib.analysis.test.TestRunnerAction;
import com.google.devtools.build.lib.analysis.test.TestRunnerAction.ResolvedPaths;
import com.google.devtools.build.lib.analysis.test.TestTargetExecutionSettings;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.EventKind;
import com.google.devtools.build.lib.util.Fingerprint;
import com.google.devtools.build.lib.util.OS;
import com.google.devtools.build.lib.util.io.FileWatcher;
import com.google.devtools.build.lib.util.io.OutErr;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
import com.google.devtools.build.lib.view.test.TestStatus.TestCase;
import com.google.devtools.build.lib.view.test.TestStatus.TestResultData;
import com.google.devtools.common.options.EnumConverter;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.time.Duration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import javax.annotation.Nullable;
/** A strategy for executing a {@link TestRunnerAction}. */
public abstract class TestStrategy implements TestActionContext {
private final ConcurrentHashMap<ShardKey, ListenableFuture<Void>> futures =
new ConcurrentHashMap<>();
/**
* Ensures that all directories used to run test are in the correct state and their content will
* not result in stale files.
*/
protected void prepareFileSystem(
TestRunnerAction testAction, Path execRoot, Path tmpDir, Path workingDirectory)
throws IOException {
if (tmpDir != null) {
recreateDirectory(tmpDir);
}
if (workingDirectory != null) {
workingDirectory.createDirectoryAndParents();
}
ResolvedPaths resolvedPaths = testAction.resolve(execRoot);
if (testAction.isCoverageMode()) {
recreateDirectory(resolvedPaths.getCoverageDirectory());
}
resolvedPaths.getBaseDir().createDirectoryAndParents();
resolvedPaths.getUndeclaredOutputsDir().createDirectoryAndParents();
resolvedPaths.getUndeclaredOutputsAnnotationsDir().createDirectoryAndParents();
resolvedPaths.getSplitLogsDir().createDirectoryAndParents();
}
/**
* Ensures that all directories used to run test are in the correct state and their content will
* not result in stale files. Only use this if no local tmp and working directory are required.
*/
protected void prepareFileSystem(TestRunnerAction testAction, Path execRoot) throws IOException {
prepareFileSystem(testAction, execRoot, null, null);
}
/** Removes directory if it exists and recreates it. */
private void recreateDirectory(Path directory) throws IOException {
directory.deleteTree();
directory.createDirectoryAndParents();
}
/** An enum for specifying different formats of test output. */
public enum TestOutputFormat {
SUMMARY, // Provide summary output only.
ERRORS, // Print output from failed tests to the stderr after the test failure.
ALL, // Print output from all tests to the stderr after the test completion.
STREAMED; // Stream output for each test.
/** Converts to {@link TestOutputFormat}. */
public static class Converter extends EnumConverter<TestOutputFormat> {
public Converter() {
super(TestOutputFormat.class, "test output");
}
}
}
/** An enum for specifying different formatting styles of test summaries. */
public enum TestSummaryFormat {
SHORT, // Print information only about tests.
TERSE, // Like "SHORT", but even shorter: Do not print PASSED and NO STATUS tests.
DETAILED, // Print information only about failed test cases.
NONE, // Do not print summary.
TESTCASE; // Print summary in test case resolution, do not print detailed information about
// failed test cases.
/** Converts to {@link TestSummaryFormat}. */
public static class Converter extends EnumConverter<TestSummaryFormat> {
public Converter() {
super(TestSummaryFormat.class, "test summary");
}
}
}
public static final PathFragment TEST_TMP_ROOT = PathFragment.create("_tmp");
// Used for generating unique temporary directory names. Contains the next numeric index for every
// executable base name.
private final Map<String, Integer> tmpIndex = new HashMap<>();
protected final ExecutionOptions executionOptions;
protected final BinTools binTools;
public TestStrategy(ExecutionOptions executionOptions, BinTools binTools) {
this.executionOptions = executionOptions;
this.binTools = binTools;
}
@Override
public final boolean isTestKeepGoing() {
return executionOptions.testKeepGoing;
}
@Override
public final ListenableFuture<Void> getTestCancelFuture(ActionOwner owner, int shardNum) {
ShardKey key = new ShardKey(owner, shardNum);
return futures.computeIfAbsent(key, (k) -> SettableFuture.<Void>create());
}
/**
* Generates a command line to run for the test action, taking into account coverage and {@code
* --run_under} settings.
*
* @param testAction The test action.
* @return the command line as string list.
* @throws ExecException
*/
public static ImmutableList<String> getArgs(TestRunnerAction testAction) throws ExecException {
List<String> args = Lists.newArrayList();
// TODO(ulfjack): `executedOnWindows` is incorrect for remote execution, where we need to
// consider the target configuration, not the machine Bazel happens to run on. Change this to
// something like: testAction.getConfiguration().getTargetOS() == OS.WINDOWS
final boolean executedOnWindows = (OS.getCurrent() == OS.WINDOWS);
final boolean useTestWrapper = testAction.isUsingTestWrapperInsteadOfTestSetupScript();
if (executedOnWindows && !useTestWrapper) {
// TestActionBuilder constructs TestRunnerAction with a 'null' shell path only when we use the
// native test wrapper. Something clearly went wrong.
Preconditions.checkNotNull(testAction.getShExecutableMaybe(), "%s", testAction);
args.add(testAction.getShExecutableMaybe().getPathString());
args.add("-c");
args.add("$0 \"$@\"");
}
Artifact testSetup = testAction.getTestSetupScript();
args.add(testSetup.getExecPath().getCallablePathString());
if (testAction.isCoverageMode()) {
args.add(testAction.getCollectCoverageScript().getExecPathString());
}
TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();
// Insert the command prefix specified by the "--run_under=<command-prefix>" option, if any.
if (execSettings.getRunUnder() != null) {
addRunUnderArgs(testAction, args, executedOnWindows);
}
// Execute the test using the alias in the runfiles tree, as mandated by the Test Encyclopedia.
args.add(execSettings.getExecutable().getRootRelativePath().getCallablePathString());
try {
Iterables.addAll(args, execSettings.getArgs().arguments());
} catch (CommandLineExpansionException e) {
throw new UserExecException(e);
}
return ImmutableList.copyOf(args);
}
private static void addRunUnderArgs(
TestRunnerAction testAction, List<String> args, boolean executedOnWindows) {
TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();
if (execSettings.getRunUnderExecutable() != null) {
args.add(execSettings.getRunUnderExecutable().getRootRelativePath().getCallablePathString());
} else {
if (execSettings.needsShell(executedOnWindows)) {
// TestActionBuilder constructs TestRunnerAction with a 'null' shell only when none is
// required. Something clearly went wrong.
Preconditions.checkNotNull(testAction.getShExecutableMaybe(), "%s", testAction);
String shellExecutable = testAction.getShExecutableMaybe().getPathString();
args.add(shellExecutable);
args.add("-c");
args.add("\"$@\"");
args.add(shellExecutable); // Sets $0.
}
args.add(execSettings.getRunUnder().getCommand());
}
args.addAll(testAction.getExecutionSettings().getRunUnder().getOptions());
}
/**
* Returns the number of attempts specific test action can be retried.
*
* <p>For rules with "flaky = 1" attribute, this method will return 3 unless --flaky_test_attempts
* option is given and specifies another value.
*/
@VisibleForTesting /* protected */
public int getTestAttempts(TestRunnerAction action) {
return action.getTestProperties().isFlaky()
? getTestAttemptsForFlakyTest(action)
: getTestAttempts(action, /*defaultTestAttempts=*/ 1);
}
public int getTestAttemptsForFlakyTest(TestRunnerAction action) {
return getTestAttempts(action, /*defaultTestAttempts=*/ 3);
}
private int getTestAttempts(TestRunnerAction action, int defaultTestAttempts) {
Label testLabel = action.getOwner().getLabel();
return getTestAttemptsPerLabel(executionOptions, testLabel, defaultTestAttempts);
}
private static int getTestAttemptsPerLabel(
ExecutionOptions options, Label label, int defaultTestAttempts) {
// Check from the last provided, so that the last option provided takes precedence.
for (PerLabelOptions perLabelAttempts : Lists.reverse(options.testAttempts)) {
if (perLabelAttempts.isIncluded(label)) {
String attempts = Iterables.getOnlyElement(perLabelAttempts.getOptions());
if ("default".equals(attempts)) {
return defaultTestAttempts;
}
return Integer.parseInt(attempts);
}
}
return defaultTestAttempts;
}
/**
* Returns timeout value in seconds that should be used for the given test action. We always use
* the "categorical timeouts" which are based on the --test_timeout flag. A rule picks its timeout
* but ends up with the same effective value as all other rules in that bucket.
*/
protected final Duration getTimeout(TestRunnerAction testAction) {
BuildConfiguration configuration = testAction.getConfiguration();
return configuration
.getFragment(TestConfiguration.class)
.getTestTimeout()
.get(testAction.getTestProperties().getTimeout());
}
/*
* Finalize test run: persist the result, and post on the event bus.
*/
protected void postTestResult(ActionExecutionContext actionExecutionContext, TestResult result)
throws IOException {
result.getTestAction().saveCacheStatus(actionExecutionContext, result.getData());
actionExecutionContext.getEventHandler().post(result);
}
/**
* Returns a unique name for a temporary directory a test could use.
*
* <p>Since each test within single Blaze run must have a unique TEST_TMPDIR, we will use rule
* name and a unique (within single Blaze request) number to generate directory name.
*
* <p>This does not create the directory.
*/
protected String getTmpDirName(PathFragment execPath) {
String basename = execPath.getBaseName();
synchronized (tmpIndex) {
int index = tmpIndex.containsKey(basename) ? tmpIndex.get(basename) : 1;
tmpIndex.put(basename, index + 1);
return basename + "_" + index;
}
}
public static String getTmpDirName(TestRunnerAction action) {
Fingerprint digest = new Fingerprint();
digest.addPath(action.getExecutionSettings().getExecutable().getExecPath());
digest.addInt(action.getShardNum());
digest.addInt(action.getRunNumber());
// Truncate the string to 32 character to avoid exceeding path length limit on Windows and macOS
return digest.hexDigestAndReset().substring(0, 32);
}
/** Parse a test result XML file into a {@link TestCase}. */
@Nullable
protected TestCase parseTestResult(Path resultFile) {
/* xml files. We avoid parsing it unnecessarily, since test results can potentially consume
a large amount of memory. */
if ((executionOptions.testSummary != TestSummaryFormat.DETAILED)
&& (executionOptions.testSummary != TestSummaryFormat.TESTCASE)) {
return null;
}
try (InputStream fileStream = resultFile.getInputStream()) {
return new TestXmlOutputParser().parseXmlIntoTestResult(fileStream);
} catch (IOException | TestXmlOutputParserException e) {
return null;
}
}
/**
* Outputs test result to the stdout after test has finished (e.g. for --test_output=all or
* --test_output=errors). Will also try to group output lines together (up to 10000 lines) so
* parallel test outputs will not get interleaved.
*/
protected void processTestOutput(
ActionExecutionContext actionExecutionContext,
TestResultData testResultData,
String testName,
Path testLog)
throws IOException {
boolean isPassed = testResultData.getTestPassed();
try {
if (testResultData.getStatus() != BlazeTestStatus.INCOMPLETE
&& TestLogHelper.shouldOutputTestLog(executionOptions.testOutput, isPassed)) {
TestLogHelper.writeTestLog(
testLog, testName, actionExecutionContext.getFileOutErr().getOutputStream());
}
} finally {
if (isPassed) {
actionExecutionContext.getEventHandler().handle(Event.of(EventKind.PASS, null, testName));
} else {
if (testResultData.hasStatusDetails()) {
actionExecutionContext
.getEventHandler()
.handle(Event.error(testName + ": " + testResultData.getStatusDetails()));
}
if (testResultData.getStatus() == BlazeTestStatus.TIMEOUT) {
actionExecutionContext
.getEventHandler()
.handle(Event.of(EventKind.TIMEOUT, null, testName + " (see " + testLog + ")"));
} else if (testResultData.getStatus() == BlazeTestStatus.INCOMPLETE) {
actionExecutionContext
.getEventHandler()
.handle(Event.of(EventKind.CANCELLED, null, testName));
} else {
actionExecutionContext
.getEventHandler()
.handle(Event.of(EventKind.FAIL, null, testName + " (see " + testLog + ")"));
}
}
}
}
/**
* Returns a temporary directory for all tests in a workspace to use. Individual tests should
* create child directories to actually use.
*
* <p>This either dynamically generates a directory name or uses the directory specified by
* --test_tmpdir. This does not create the directory.
*/
public static Path getTmpRoot(Path workspace, Path execRoot, ExecutionOptions executionOptions) {
return executionOptions.testTmpDir != null
? workspace.getRelative(executionOptions.testTmpDir).getRelative(TEST_TMP_ROOT)
: execRoot.getRelative(TEST_TMP_ROOT);
}
/**
* For an given environment, returns a subset containing all variables in the given list if they
* are defined in the given environment.
*/
@VisibleForTesting
public static Map<String, String> getMapping(
Iterable<String> variables, Map<String, String> environment) {
Map<String, String> result = new HashMap<>();
for (String var : variables) {
if (environment.containsKey(var)) {
result.put(var, environment.get(var));
}
}
return result;
}
protected static void closeSuppressed(Throwable e, @Nullable Closeable c) {
if (c == null) {
return;
}
try {
c.close();
} catch (IOException e2) {
e.addSuppressed(e2);
}
}
/** Implements the --test_output=streamed option. */
protected static class StreamedTestOutput implements Closeable {
private final TestLogHelper.FilterTestHeaderOutputStream headerFilter;
private final FileWatcher watcher;
private final Path testLogPath;
private final OutErr outErr;
public StreamedTestOutput(OutErr outErr, Path testLogPath) throws IOException {
this.testLogPath = testLogPath;
this.outErr = outErr;
this.headerFilter = TestLogHelper.getHeaderFilteringOutputStream(outErr.getOutputStream());
this.watcher = new FileWatcher(testLogPath, OutErr.create(headerFilter, headerFilter), false);
watcher.start();
}
@Override
public void close() throws IOException {
watcher.stopPumping();
try {
// The watcher thread might leak if the following call is interrupted.
// This is a relatively minor issue since the worst it could do is
// write one additional line from the test.log to the console later on
// in the build.
watcher.join();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
if (!headerFilter.foundHeader()) {
try (InputStream input = testLogPath.getInputStream()) {
ByteStreams.copy(input, outErr.getOutputStream());
}
}
}
}
private static final class ShardKey {
private final ActionOwner owner;
private final int shard;
ShardKey(ActionOwner owner, int shard) {
this.owner = Preconditions.checkNotNull(owner);
this.shard = shard;
}
@Override
public int hashCode() {
return Objects.hash(owner, shard);
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof ShardKey)) {
return false;
}
ShardKey s = (ShardKey) o;
return owner.equals(s.owner) && shard == s.shard;
}
}
}