process runner for junit integration test framework Closes #5435. PiperOrigin-RevId: 202100672
diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD index 1322ac9..a5944fa 100644 --- a/src/test/java/com/google/devtools/build/lib/BUILD +++ b/src/test/java/com/google/devtools/build/lib/BUILD
@@ -1544,6 +1544,24 @@ ], ) +java_test( + name = "java-integration-tests", + srcs = glob(["integration/blackbox/**/*.java"]), + test_class = "com.google.devtools.build.lib.AllTests", + deps = [ + ":guava_junit_truth", + ":test_runner", + ":testutil", + "//src/main/java/com/google/devtools/build/lib:io", + "//src/main/java/com/google/devtools/build/lib:os_util", + "//src/main/java/com/google/devtools/build/lib:util", + "//third_party:auto_value", + "//third_party:guava", + "//third_party:jsr305", + "//third_party:junit4", + ], +) + java_library( name = "guava_junit_truth", testonly = 1,
diff --git a/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessParameters.java b/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessParameters.java new file mode 100644 index 0000000..d9f728a --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessParameters.java
@@ -0,0 +1,90 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.integration.blackbox.framework; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** {@link ProcessRunner} parameters */ +@AutoValue +public abstract class ProcessParameters { + abstract String name(); + + abstract ImmutableList<String> arguments(); + + abstract File workingDirectory(); + + abstract int expectedExitCode(); + + abstract boolean expectedEmptyError(); + + abstract Optional<ImmutableMap<String, String>> environment(); + + abstract long timeoutMillis(); + + abstract Optional<Path> redirectOutput(); + + abstract Optional<Path> redirectError(); + + public static Builder builder() { + return new AutoValue_ProcessParameters.Builder() + .setExpectedExitCode(0) + .setExpectedEmptyError(true) + .setTimeoutMillis(30 * 1000) + .setArguments(); + } + + /** Builder class */ + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setName(String value); + + public abstract Builder setArguments(String... args); + + public abstract Builder setArguments(ImmutableList<String> args); + + public Builder setArguments(List<String> args) { + setArguments(ImmutableList.copyOf(args)); + return this; + } + + public abstract Builder setWorkingDirectory(File value); + + public abstract Builder setExpectedExitCode(int value); + + public abstract Builder setExpectedEmptyError(boolean value); + + public abstract Builder setEnvironment(ImmutableMap<String, String> map); + + public Builder setEnvironment(Map<String, String> map) { + setEnvironment(ImmutableMap.copyOf(map)); + return this; + } + + public abstract Builder setTimeoutMillis(long millis); + + public abstract Builder setRedirectOutput(Path path); + + public abstract Builder setRedirectError(Path path); + + public abstract ProcessParameters build(); + } +}
diff --git a/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessResult.java b/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessResult.java new file mode 100644 index 0000000..9fccf9f --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessResult.java
@@ -0,0 +1,42 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.integration.blackbox.framework; + +import com.google.auto.value.AutoValue; +import com.google.devtools.build.lib.util.StringUtilities; +import java.util.List; + +/** Result of the external process execution, see {@link ProcessRunner} */ +@AutoValue +public abstract class ProcessResult { + + static ProcessResult create(int exitCode, List<String> out, List<String> err) { + return new AutoValue_ProcessResult(exitCode, out, err); + } + + abstract int exitCode(); + + abstract List<String> out(); + + abstract List<String> err(); + + public String outString() { + return StringUtilities.joinLines(out()); + } + + public String errString() { + return StringUtilities.joinLines(err()); + } +}
diff --git a/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessRunner.java b/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessRunner.java new file mode 100644 index 0000000..2db7717 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessRunner.java
@@ -0,0 +1,194 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.integration.blackbox.framework; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.io.LineReader; +import com.google.devtools.build.lib.util.StringUtilities; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import javax.annotation.Nullable; + +/** + * Helper class for running Bazel process as external process from JUnit tests Can be used to run + * arbitrary external process and explore the results + */ +public final class ProcessRunner { + private static final Logger logger = Logger.getLogger(ProcessRunner.class.getName()); + private final ProcessParameters parameters; + private final ExecutorService executorService; + + /** + * Creates ProcessRunner + * + * @param parameters process parameters like executable name, arguments, timeout etc + * @param executorService to use for process output/error streams reading; intentionally passed as + * a parameter so we can use the thread pool to speed up. Should be multi-threaded, as two + * separate tasks are submitted, to read from output and error streams. + * <p>SuppressWarnings: WeakerAccess - suppress the warning about constructor being public: + * the class is intended to be used outside the package. (IDE currently marks the possibility + * for the constructor to be package-private because the current usages are only inside the + * package, but it is going to change) + */ + @SuppressWarnings("WeakerAccess") + public ProcessRunner(ProcessParameters parameters, ExecutorService executorService) { + this.parameters = parameters; + this.executorService = executorService; + } + + public ProcessResult runSynchronously() throws Exception { + ImmutableList<String> args = parameters.arguments(); + final List<String> commandParts = new ArrayList<>(args.size() + 1); + commandParts.add(parameters.name()); + commandParts.addAll(args); + + logger.info("Running: " + commandParts.stream().collect(Collectors.joining(" "))); + + ProcessBuilder processBuilder = new ProcessBuilder(commandParts); + processBuilder.directory(parameters.workingDirectory()); + parameters.environment().ifPresent(map -> processBuilder.environment().putAll(map)); + + parameters.redirectOutput().ifPresent(path -> processBuilder.redirectOutput(path.toFile())); + parameters.redirectError().ifPresent(path -> processBuilder.redirectError(path.toFile())); + + Process process = processBuilder.start(); + + try (ProcessStreamReader outReader = + parameters.redirectOutput().isPresent() + ? null + : createReader(process.getInputStream(), ">> "); + ProcessStreamReader errReader = + parameters.redirectError().isPresent() + ? null + : createReader(process.getErrorStream(), "ERROR: ")) { + + long timeoutMillis = parameters.timeoutMillis(); + if (!process.waitFor(timeoutMillis, TimeUnit.MILLISECONDS)) { + throw new TimeoutException( + String.format( + "%s timed out after %d seconds (%d millis)", + parameters.name(), timeoutMillis / 1000, timeoutMillis)); + } + + List<String> err = + errReader != null + ? errReader.get() + : Files.readAllLines(parameters.redirectError().get()); + List<String> out = + outReader != null + ? outReader.get() + : Files.readAllLines(parameters.redirectOutput().get()); + + if (parameters.expectedExitCode() != process.exitValue()) { + throw new ProcessRunnerException( + String.format( + "Expected exit code %d, but found %d.\nError: %s\nOutput: %s", + parameters.expectedExitCode(), + process.exitValue(), + StringUtilities.joinLines(err), + StringUtilities.joinLines(out))); + } + + if (parameters.expectedEmptyError()) { + if (!err.isEmpty()) { + throw new ProcessRunnerException( + "Expected empty error stream, but found: " + StringUtilities.joinLines(err)); + } + } + return ProcessResult.create(parameters.expectedExitCode(), out, err); + } finally { + process.destroy(); + } + } + + private ProcessStreamReader createReader(InputStream stream, String prefix) { + return new ProcessStreamReader(executorService, stream, s -> logger.fine(prefix + s)); + } + + /** Specific runtime exception for external process errors */ + public static class ProcessRunnerException extends RuntimeException { + ProcessRunnerException(String message) { + super(message); + } + } + + private static class ProcessStreamReader implements AutoCloseable { + + private final InputStream stream; + private final Future<List<String>> future; + private final AtomicReference<IOException> exception = new AtomicReference<>(); + + private ProcessStreamReader( + ExecutorService executorService, + InputStream stream, + @Nullable Consumer<String> logConsumer) { + this.stream = stream; + future = + executorService.submit( + () -> { + final List<String> lines = Lists.newArrayList(); + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + LineReader lineReader = new LineReader(reader); + String line; + while ((line = lineReader.readLine()) != null) { + if (logConsumer != null) { + logConsumer.accept(line); + } + lines.add(line); + } + } catch (IOException e) { + exception.set(e); + } + return lines; + }); + } + + public List<String> get() + throws InterruptedException, ExecutionException, TimeoutException, IOException { + try { + List<String> lines = future.get(15, TimeUnit.SECONDS); + if (exception.get() != null) { + throw exception.get(); + } + return lines; + } finally { + // if future is timed out + stream.close(); + } + } + + @Override + public void close() throws Exception { + stream.close(); + } + } +}
diff --git a/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessRunnerTest.java b/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessRunnerTest.java new file mode 100644 index 0000000..ac38e45 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessRunnerTest.java
@@ -0,0 +1,193 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.integration.blackbox.framework; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.devtools.build.lib.util.OS; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Test of {@link ProcessRunner} */ +@RunWith(JUnit4.class) +public final class ProcessRunnerTest { + private static ExecutorService executorService; + private Path directory; + private Path path; + + @BeforeClass + public static void setUpExecutor() { + // we need only two threads to schedule reading from output and error streams + executorService = + MoreExecutors.getExitingExecutorService( + (ThreadPoolExecutor) Executors.newFixedThreadPool(2), 1, TimeUnit.SECONDS); + } + + @Before + public void setUp() throws Exception { + directory = Files.createTempDirectory(getClass().getSimpleName()); + path = Files.createTempFile(directory, "script", isWindows() ? ".bat" : ""); + assertThat(Files.exists(path)).isTrue(); + assertThat(path.toFile().setExecutable(true)).isTrue(); + path.toFile().deleteOnExit(); + directory.toFile().deleteOnExit(); + } + + @AfterClass + public static void tearDownExecutor() { + MoreExecutors.shutdownAndAwaitTermination(executorService, 5, TimeUnit.SECONDS); + } + + @Test + public void testSuccess() throws Exception { + Files.write(path, createScriptText(/* exit code */ 0, /* output */ "Hello!", /* error */ null)); + + ProcessParameters parameters = createBuilder().build(); + ProcessResult result = new ProcessRunner(parameters, executorService).runSynchronously(); + + assertThat(result.exitCode()).isEqualTo(0); + assertThat(result.outString()).isEqualTo("Hello!"); + assertThat(result.errString()).isEmpty(); + } + + @Test + public void testFailure() throws Exception { + Files.write( + path, createScriptText(/* exit code */ 124, /* output */ null, /* error */ "Failure")); + + ProcessParameters parameters = + createBuilder().setExpectedExitCode(124).setExpectedEmptyError(false).build(); + ProcessResult result = new ProcessRunner(parameters, executorService).runSynchronously(); + + assertThat(result.exitCode()).isEqualTo(124); + assertThat(result.outString()).isEmpty(); + assertThat(result.errString()).isEqualTo("Failure"); + } + + @Test + public void testTimeout() throws Exception { + // Windows script to sleep 5 seconds, so that we can test timeout. + // This script finds PowerShell using %systemroot% variable, which we assume is always + // defined. It passes some standard parameters like input and output formats, + // important part is the Command parameter, which actually calls Sleep from PowerShell. + String windowsScript = + "%systemroot%\\system32\\cmd.exe /C \"start /I /B powershell" + + " -Version 3.0 -NoLogo -Sta -NoProfile -InputFormat Text -OutputFormat Text" + + " -NonInteractive -Command \"\"&PowerShell Sleep 5\""; + Files.write(path, Collections.singleton(isWindows() ? windowsScript : "read smthg")); + + ProcessParameters parameters = + createBuilder() + .setExpectedExitCode(-1) + .setExpectedEmptyError(false) + .setTimeoutMillis(100) + .build(); + try { + new ProcessRunner(parameters, executorService).runSynchronously(); + assertThat(false).isTrue(); + } catch (TimeoutException e) { + // ignore + } + } + + @Test + public void testRedirect() throws Exception { + Files.write( + path, + createScriptText( + /* exit code */ 12, + /* output */ Lists.newArrayList("Info", "Multi", "line"), + /* error */ Collections.singletonList("Failure"))); + + Path out = directory.resolve("out.txt"); + Path err = directory.resolve("err.txt"); + + try { + ProcessParameters parameters = + createBuilder() + .setExpectedExitCode(12) + .setExpectedEmptyError(false) + .setRedirectOutput(out) + .setRedirectError(err) + .build(); + ProcessResult result = new ProcessRunner(parameters, executorService).runSynchronously(); + + assertThat(result.exitCode()).isEqualTo(12); + assertThat(result.outString()).isEqualTo("Info\nMulti\nline"); + assertThat(result.errString()).isEqualTo("Failure"); + } finally { + Files.delete(out); + Files.delete(err); + } + } + + private ProcessParameters.Builder createBuilder() { + return ProcessParameters.builder() + .setWorkingDirectory(directory.toFile()) + .setName(path.toAbsolutePath().toString()); + } + + private static List<String> createScriptText( + final int exitCode, @Nullable final String output, @Nullable final String error) { + return createScriptText( + exitCode, + output != null ? Collections.singletonList(output) : null, + error != null ? Collections.singletonList(error) : null); + } + + private static List<String> createScriptText( + final int exitCode, @Nullable final List<String> output, @Nullable final List<String> error) { + List<String> text = Lists.newArrayList(); + if (isWindows()) { + text.add("@echo off"); + } + text.addAll(echoStrings(output, "")); + text.addAll(echoStrings(error, isWindows() ? ">&2" : " 1>&2")); + text.add((isWindows() ? "exit /b " : "exit ") + exitCode); + return text; + } + + private static List<String> echoStrings(@Nullable List<String> input, String redirect) { + if (input == null) { + return Collections.emptyList(); + } + String quote = isWindows() ? "" : "\""; + return input + .stream() + .map(s -> String.format("echo %s%s%s%s", quote, s, quote, redirect)) + .collect(Collectors.toList()); + } + + private static boolean isWindows() { + return OS.WINDOWS.equals(OS.getCurrent()); + } +}