| // Copyright 2014 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package com.google.devtools.build.lib.exec; |
| |
| import com.google.common.annotations.VisibleForTesting; |
| import com.google.common.collect.ImmutableMap; |
| import com.google.common.collect.Iterables; |
| import com.google.common.collect.Lists; |
| import com.google.common.io.ByteStreams; |
| import com.google.common.io.Closeables; |
| import com.google.devtools.build.lib.actions.ActionExecutionContext; |
| import com.google.devtools.build.lib.actions.ExecException; |
| import com.google.devtools.build.lib.actions.Executor; |
| import com.google.devtools.build.lib.analysis.config.BinTools; |
| import com.google.devtools.build.lib.events.Event; |
| import com.google.devtools.build.lib.profiler.Profiler; |
| import com.google.devtools.build.lib.profiler.ProfilerTask; |
| import com.google.devtools.build.lib.rules.test.TestActionContext; |
| import com.google.devtools.build.lib.rules.test.TestResult; |
| import com.google.devtools.build.lib.rules.test.TestRunnerAction; |
| import com.google.devtools.build.lib.rules.test.TestTargetExecutionSettings; |
| import com.google.devtools.build.lib.util.OS; |
| import com.google.devtools.build.lib.util.ShellEscaper; |
| import com.google.devtools.build.lib.util.io.FileWatcher; |
| import com.google.devtools.build.lib.util.io.OutErr; |
| import com.google.devtools.build.lib.vfs.FileStatus; |
| import com.google.devtools.build.lib.vfs.FileSystemUtils; |
| import com.google.devtools.build.lib.vfs.Path; |
| import com.google.devtools.build.lib.vfs.PathFragment; |
| import com.google.devtools.build.lib.vfs.SearchPath; |
| import com.google.devtools.build.lib.view.test.TestStatus.TestCase; |
| import com.google.devtools.common.options.Converters.RangeConverter; |
| import com.google.devtools.common.options.EnumConverter; |
| import com.google.devtools.common.options.OptionsClassProvider; |
| import com.google.devtools.common.options.OptionsParsingException; |
| import java.io.Closeable; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.io.OutputStream; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.HashMap; |
| import java.util.List; |
| import java.util.Map; |
| import javax.annotation.Nullable; |
| |
| /** A strategy for executing a {@link TestRunnerAction}. */ |
| public abstract class TestStrategy implements TestActionContext { |
| public static final PathFragment COVERAGE_TMP_ROOT = new PathFragment("_coverage"); |
| |
| public static final String TEST_SETUP_BASENAME = "test-setup.sh"; |
| |
| /** Returns true if coverage data should be gathered. */ |
| protected static boolean isCoverageMode(TestRunnerAction action) { |
| return action.getCoverageData() != null; |
| } |
| |
| /** |
| * Ensures that all directories used to run test are in the correct state and their content will |
| * not result in stale files. |
| */ |
| protected void prepareFileSystem( |
| TestRunnerAction testAction, Path tmpDir, Path coverageDir, Path workingDirectory) |
| throws IOException { |
| if (isCoverageMode(testAction)) { |
| try { |
| recreateDirectory(coverageDir); |
| } catch (IOException e) { |
| throw new IOException("Failed to recreate coverage directory " + coverageDir, e); |
| } |
| } |
| try { |
| recreateDirectory(tmpDir); |
| } catch (IOException e) { |
| throw new IOException("Failed to recreate temporary directory " + tmpDir, e); |
| } |
| FileSystemUtils.createDirectoryAndParents(workingDirectory); |
| } |
| |
| /** Removes directory if it exists and recreates it. */ |
| protected void recreateDirectory(Path directory) throws IOException { |
| FileSystemUtils.deleteTree(directory); |
| FileSystemUtils.createDirectoryAndParents(directory); |
| } |
| |
| /** Converter for the --flaky_test_attempts option. */ |
| public static class TestAttemptsConverter extends RangeConverter { |
| public TestAttemptsConverter() { |
| super(1, 10); |
| } |
| |
| @Override |
| public Integer convert(String input) throws OptionsParsingException { |
| if ("default".equals(input)) { |
| return -1; |
| } else { |
| return super.convert(input); |
| } |
| } |
| |
| @Override |
| public String getTypeDescription() { |
| return super.getTypeDescription() + " or the string \"default\""; |
| } |
| } |
| |
| public enum TestOutputFormat { |
| SUMMARY, // Provide summary output only. |
| ERRORS, // Print output from failed tests to the stderr after the test failure. |
| ALL, // Print output from all tests to the stderr after the test completion. |
| STREAMED; // Stream output for each test. |
| |
| /** Converts to {@link TestOutputFormat}. */ |
| public static class Converter extends EnumConverter<TestOutputFormat> { |
| public Converter() { |
| super(TestOutputFormat.class, "test output"); |
| } |
| } |
| } |
| |
| public enum TestSummaryFormat { |
| SHORT, // Print information only about tests. |
| TERSE, // Like "SHORT", but even shorter: Do not print PASSED tests. |
| DETAILED, // Print information only about failed test cases. |
| NONE; // Do not print summary. |
| |
| /** Converts to {@link TestSummaryFormat}. */ |
| public static class Converter extends EnumConverter<TestSummaryFormat> { |
| public Converter() { |
| super(TestSummaryFormat.class, "test summary"); |
| } |
| } |
| } |
| |
| public static final PathFragment TEST_TMP_ROOT = new PathFragment("_tmp"); |
| |
| // Used for selecting subset of testcase / testmethods. |
| private static final String TEST_BRIDGE_TEST_FILTER_ENV = "TESTBRIDGE_TEST_ONLY"; |
| |
| // Used for generating unique temporary directory names. Contains the next numeric index for every |
| // executable base name. |
| private final Map<String, Integer> tmpIndex = new HashMap<>(); |
| protected final ImmutableMap<String, String> clientEnv; |
| protected final ExecutionOptions executionOptions; |
| protected final BinTools binTools; |
| |
| public TestStrategy( |
| OptionsClassProvider requestOptionsProvider, |
| BinTools binTools, |
| Map<String, String> clientEnv) { |
| this.executionOptions = requestOptionsProvider.getOptions(ExecutionOptions.class); |
| this.binTools = binTools; |
| this.clientEnv = ImmutableMap.copyOf(clientEnv); |
| } |
| |
| @Override |
| public abstract void exec(TestRunnerAction action, ActionExecutionContext actionExecutionContext) |
| throws ExecException, InterruptedException; |
| |
| /** Returns true if coverage data should be gathered. */ |
| protected static boolean isMicroCoverageMode(TestRunnerAction action) { |
| return action.getMicroCoverageData() != null; |
| } |
| |
| /** |
| * Returns directory to store coverage results for the given action relative to the execution |
| * root. This directory is used to store all coverage results related to the test execution with |
| * exception of the locally generated *.gcda files. Those are stored separately using relative |
| * path within coverage directory. |
| * |
| * <p>Coverage directory name for the given test runner action is constructed as: {@code $(blaze |
| * info execution_root)/_coverage/target_path/test_log_name} where {@code test_log_name} is |
| * usually a target name but potentially can include extra suffix, such as a shard number (if test |
| * execution was sharded). |
| */ |
| protected static PathFragment getCoverageDirectory(TestRunnerAction action) { |
| return COVERAGE_TMP_ROOT.getRelative( |
| FileSystemUtils.removeExtension(action.getTestLog().getRootRelativePath())); |
| } |
| |
| /** |
| * Returns mutable map of default testing shell environment. By itself it is incomplete and is |
| * modified further by the specific test strategy implementations (mostly due to the fact that |
| * environments used locally and remotely are different). |
| */ |
| protected Map<String, String> getDefaultTestEnvironment(TestRunnerAction action) { |
| Map<String, String> env = new HashMap<>(); |
| |
| env.putAll(action.getConfiguration().getLocalShellEnvironment()); |
| env.remove("LANG"); |
| env.put("TZ", "UTC"); |
| env.put("TEST_SIZE", action.getTestProperties().getSize().toString()); |
| env.put("TEST_TIMEOUT", Integer.toString(getTimeout(action))); |
| |
| if (action.isSharded()) { |
| env.put("TEST_SHARD_INDEX", Integer.toString(action.getShardNum())); |
| env.put( |
| "TEST_TOTAL_SHARDS", Integer.toString(action.getExecutionSettings().getTotalShards())); |
| } |
| |
| // When we run test multiple times, set different TEST_RANDOM_SEED values for each run. |
| if (action.getConfiguration().getRunsPerTestForLabel(action.getOwner().getLabel()) > 1) { |
| env.put("TEST_RANDOM_SEED", Integer.toString(action.getRunNumber() + 1)); |
| } |
| |
| String testFilter = action.getExecutionSettings().getTestFilter(); |
| if (testFilter != null) { |
| env.put(TEST_BRIDGE_TEST_FILTER_ENV, testFilter); |
| } |
| |
| if (isCoverageMode(action)) { |
| env.put( |
| "COVERAGE_MANIFEST", |
| action.getExecutionSettings().getInstrumentedFileManifest().getExecPathString()); |
| // Instruct remote-runtest.sh/local-runtest.sh not to cd into the runfiles directory. |
| env.put("RUNTEST_PRESERVE_CWD", "1"); |
| env.put("MICROCOVERAGE_REQUESTED", isMicroCoverageMode(action) ? "true" : "false"); |
| } |
| |
| return env; |
| } |
| |
| /** |
| * Generates a command line to run for the test action, taking into account coverage and {@code |
| * --run_under} settings. |
| * |
| * @param testScript the setup script that invokes the test |
| * @param coverageScript a script interjected between setup script and rest of command line to |
| * collect coverage data. If this is an empty string, it is ignored. |
| * @param testAction The test action. |
| * @return the command line as string list. |
| */ |
| protected List<String> getArgs( |
| String testScript, String coverageScript, TestRunnerAction testAction) { |
| List<String> args = Lists.newArrayList(); |
| if (OS.getCurrent() == OS.WINDOWS) { |
| args.add(testAction.getShExecutable().getPathString()); |
| args.add("-c"); |
| args.add("$0 $*"); |
| } |
| args.add(testScript); |
| TestTargetExecutionSettings execSettings = testAction.getExecutionSettings(); |
| |
| List<String> execArgs = new ArrayList<>(); |
| if (!coverageScript.isEmpty() && isCoverageMode(testAction)) { |
| execArgs.add(coverageScript); |
| } |
| |
| // Execute the test using the alias in the runfiles tree, as mandated by |
| // the Test Encyclopedia. |
| execArgs.add(execSettings.getExecutable().getRootRelativePath().getPathString()); |
| execArgs.addAll(execSettings.getArgs()); |
| |
| // Insert the command prefix specified by the "--run_under=<command-prefix>" option, |
| // if any. |
| if (execSettings.getRunUnder() == null) { |
| args.addAll(execArgs); |
| } else if (execSettings.getRunUnderExecutable() != null) { |
| args.add(execSettings.getRunUnderExecutable().getRootRelativePath().getPathString()); |
| args.addAll(execSettings.getRunUnder().getOptions()); |
| args.addAll(execArgs); |
| } else { |
| args.add(testAction.getConfiguration().getShellExecutable().getPathString()); |
| args.add("-c"); |
| |
| String runUnderCommand = ShellEscaper.escapeString(execSettings.getRunUnder().getCommand()); |
| |
| Path fullySpecified = |
| SearchPath.which( |
| SearchPath.parse( |
| testAction.getTestLog().getPath().getFileSystem(), clientEnv.get("PATH")), |
| runUnderCommand); |
| |
| if (fullySpecified != null) { |
| runUnderCommand = fullySpecified.toString(); |
| } |
| |
| args.add( |
| runUnderCommand |
| + ' ' |
| + ShellEscaper.escapeJoinAll( |
| Iterables.concat(execSettings.getRunUnder().getOptions(), execArgs))); |
| } |
| return args; |
| } |
| |
| /** |
| * Returns the number of attempts specific test action can be retried. |
| * |
| * <p>For rules with "flaky = 1" attribute, this method will return 3 unless --flaky_test_attempts |
| * option is given and specifies another value. |
| */ |
| @VisibleForTesting /* protected */ |
| public int getTestAttempts(TestRunnerAction action) { |
| if (executionOptions.testAttempts == -1) { |
| return action.getTestProperties().isFlaky() ? 3 : 1; |
| } else { |
| return executionOptions.testAttempts; |
| } |
| } |
| |
| /** |
| * Returns timeout value in seconds that should be used for the given test action. We always use |
| * the "categorical timeouts" which are based on the --test_timeout flag. A rule picks its timeout |
| * but ends up with the same effective value as all other rules in that bucket. |
| */ |
| protected final int getTimeout(TestRunnerAction testAction) { |
| return executionOptions.testTimeout.get(testAction.getTestProperties().getTimeout()); |
| } |
| |
| /** |
| * Returns a subset of the environment from the current shell. |
| * |
| * <p>Warning: Since these variables are not part of the configuration's fingerprint, they MUST |
| * NOT be used by any rule or action in such a way as to affect the semantics of that build step. |
| */ |
| public Map<String, String> getAdmissibleShellEnvironment(Iterable<String> variables) { |
| return getMapping(variables, clientEnv); |
| } |
| |
| /* |
| * Finalize test run: persist the result, and post on the event bus. |
| */ |
| protected void postTestResult(Executor executor, TestResult result) throws IOException { |
| result.getTestAction().saveCacheStatus(result.getData()); |
| executor.getEventBus().post(result); |
| } |
| |
| /** |
| * Returns a unique name for a temporary directory a test could use. |
| * |
| * <p>Since each test within single Blaze run must have a unique TEST_TMPDIR, we will use rule |
| * name and a unique (within single Blaze request) number to generate directory name. |
| * |
| * <p>This does not create the directory. |
| */ |
| protected String getTmpDirName(PathFragment execPath) { |
| String basename = execPath.getBaseName(); |
| |
| synchronized (tmpIndex) { |
| int index = tmpIndex.containsKey(basename) ? tmpIndex.get(basename) : 1; |
| tmpIndex.put(basename, index + 1); |
| return basename + "_" + index; |
| } |
| } |
| |
| /** Parse a test result XML file into a {@link TestCase}. */ |
| @Nullable |
| protected TestCase parseTestResult(Path resultFile) { |
| /* xml files. We avoid parsing it unnecessarily, since test results can potentially consume |
| a large amount of memory. */ |
| if (executionOptions.testSummary != TestSummaryFormat.DETAILED) { |
| return null; |
| } |
| |
| try (InputStream fileStream = resultFile.getInputStream()) { |
| return new TestXmlOutputParser().parseXmlIntoTestResult(fileStream); |
| } catch (IOException | TestXmlOutputParserException e) { |
| return null; |
| } |
| } |
| |
| /** |
| * Returns a temporary directory for all tests in a workspace to use. Individual tests should |
| * create child directories to actually use. |
| * |
| * <p>This either dynamically generates a directory name or uses the directory specified by |
| * --test_tmpdir. This does not create the directory. |
| */ |
| public static Path getTmpRoot(Path workspace, Path execRoot, ExecutionOptions executionOptions) { |
| return executionOptions.testTmpDir != null |
| ? workspace.getRelative(executionOptions.testTmpDir).getRelative(TEST_TMP_ROOT) |
| : execRoot.getRelative(TEST_TMP_ROOT); |
| } |
| |
| /** |
| * For an given environment, returns a subset containing all variables in the given list if they |
| * are defined in the given environment. |
| */ |
| @VisibleForTesting |
| public static Map<String, String> getMapping( |
| Iterable<String> variables, Map<String, String> environment) { |
| Map<String, String> result = new HashMap<>(); |
| for (String var : variables) { |
| if (environment.containsKey(var)) { |
| result.put(var, environment.get(var)); |
| } |
| } |
| return result; |
| } |
| |
| /** |
| * Returns the runfiles directory associated with the test executable, creating/updating it if |
| * necessary and --build_runfile_links is specified. |
| */ |
| protected static Path getLocalRunfilesDirectory( |
| TestRunnerAction testAction, |
| ActionExecutionContext actionExecutionContext, |
| BinTools binTools, |
| ImmutableMap<String, String> shellEnvironment, |
| boolean enableRunfiles) |
| throws ExecException, InterruptedException { |
| TestTargetExecutionSettings execSettings = testAction.getExecutionSettings(); |
| |
| // If the symlink farm is already created then return the existing directory. If not we |
| // need to explicitly build it. This can happen when --nobuild_runfile_links is supplied |
| // as a flag to the build. |
| if (execSettings.getRunfilesSymlinksCreated()) { |
| return execSettings.getRunfilesDir(); |
| } |
| |
| // TODO(bazel-team): Should we be using TestTargetExecutionSettings#getRunfilesDir() here over |
| // generating the directory ourselves? |
| Path program = execSettings.getExecutable().getPath(); |
| Path runfilesDir = program.getParentDirectory().getChild(program.getBaseName() + ".runfiles"); |
| |
| // Synchronize runfiles tree generation on the runfiles manifest artifact. |
| // This is necessary, because we might end up with multiple test runner actions |
| // trying to generate same runfiles tree in case of --runs_per_test > 1 or |
| // local test sharding. |
| long startTime = Profiler.nanoTimeMaybe(); |
| synchronized (execSettings.getInputManifest()) { |
| Profiler.instance().logSimpleTask(startTime, ProfilerTask.WAIT, testAction); |
| updateLocalRunfilesDirectory( |
| testAction, |
| runfilesDir, |
| actionExecutionContext, |
| binTools, |
| shellEnvironment, |
| enableRunfiles); |
| } |
| |
| return runfilesDir; |
| } |
| |
| /** |
| * Ensure the runfiles tree exists and is consistent with the TestAction's manifest |
| * ($0.runfiles_manifest), bringing it into consistency if not. The contents of the output file |
| * $0.runfiles/MANIFEST, if it exists, are used a proxy for the set of existing symlinks, to avoid |
| * the need for recursion. |
| */ |
| private static void updateLocalRunfilesDirectory( |
| TestRunnerAction testAction, |
| Path runfilesDir, |
| ActionExecutionContext actionExecutionContext, |
| BinTools binTools, |
| ImmutableMap<String, String> shellEnvironment, |
| boolean enableRunfiles) |
| throws ExecException, InterruptedException { |
| Executor executor = actionExecutionContext.getExecutor(); |
| |
| TestTargetExecutionSettings execSettings = testAction.getExecutionSettings(); |
| try { |
| // Avoid rebuilding the runfiles directory if the manifest in it matches the input manifest, |
| // implying the symlinks exist and are already up to date. |
| if (Arrays.equals( |
| runfilesDir.getRelative("MANIFEST").getDigest(), |
| execSettings.getInputManifest().getPath().getDigest())) { |
| return; |
| } |
| } catch (IOException e1) { |
| // Ignore it - we will just try to create runfiles directory. |
| } |
| |
| executor |
| .getEventHandler() |
| .handle( |
| Event.progress( |
| "Building runfiles directory for '" |
| + execSettings.getExecutable().prettyPrint() |
| + "'.")); |
| |
| new SymlinkTreeHelper(execSettings.getInputManifest().getPath(), runfilesDir, false) |
| .createSymlinks( |
| testAction, actionExecutionContext, binTools, shellEnvironment, enableRunfiles); |
| |
| executor.getEventHandler().handle(Event.progress(testAction.getProgressMessage())); |
| } |
| |
| /** In rare cases, we might write something to stderr. Append it to the real test.log. */ |
| protected static void appendStderr(Path stdOut, Path stdErr) throws IOException { |
| FileStatus stat = stdErr.statNullable(); |
| OutputStream out = null; |
| InputStream in = null; |
| if (stat != null) { |
| try { |
| if (stat.getSize() > 0) { |
| if (stdOut.exists()) { |
| stdOut.setWritable(true); |
| } |
| out = stdOut.getOutputStream(true); |
| in = stdErr.getInputStream(); |
| ByteStreams.copy(in, out); |
| } |
| } finally { |
| Closeables.close(out, true); |
| Closeables.close(in, true); |
| stdErr.delete(); |
| } |
| } |
| } |
| |
| /** Implements the --test_output=streamed option. */ |
| protected static class StreamedTestOutput implements Closeable { |
| private final TestLogHelper.FilterTestHeaderOutputStream headerFilter; |
| private final FileWatcher watcher; |
| private final Path testLogPath; |
| private final OutErr outErr; |
| |
| public StreamedTestOutput(OutErr outErr, Path testLogPath) throws IOException { |
| this.testLogPath = testLogPath; |
| this.outErr = outErr; |
| this.headerFilter = TestLogHelper.getHeaderFilteringOutputStream(outErr.getOutputStream()); |
| this.watcher = new FileWatcher(testLogPath, OutErr.create(headerFilter, headerFilter), false); |
| watcher.start(); |
| } |
| |
| @Override |
| public void close() throws IOException { |
| watcher.stopPumping(); |
| try { |
| // The watcher thread might leak if the following call is interrupted. |
| // This is a relatively minor issue since the worst it could do is |
| // write one additional line from the test.log to the console later on |
| // in the build. |
| watcher.join(); |
| } catch (InterruptedException e) { |
| Thread.currentThread().interrupt(); |
| } |
| if (!headerFilter.foundHeader()) { |
| try (InputStream input = testLogPath.getInputStream()) { |
| ByteStreams.copy(input, outErr.getOutputStream()); |
| } |
| } |
| } |
| } |
| } |