blob: 3fef6d2db77492fc27e3d491d8195c57c188c706 [file] [log] [blame]
// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.shell;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.flogger.GoogleLogger;
import com.google.common.flogger.LazyArgs;
import com.google.common.io.ByteStreams;
import com.google.devtools.build.lib.shell.Consumers.OutErrConsumers;
import com.google.devtools.build.lib.util.DescribableExecutionUnit;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.time.Duration;
import java.util.Map;
import javax.annotation.Nullable;
/**
* An executable command, including its arguments and runtime environment (environment variables,
* working directory). It lets a caller execute a command, get its results, and optionally forward
* interrupts to the subprocess. This class creates threads to ensure timely reading of subprocess
* outputs.
*
* <p>This class is immutable and thread-safe.
*
* <p>The use of "shell" in the package name of this class is a misnomer. In terms of the way its
* arguments are interpreted, this class is closer to {@code execve(2)} than to {@code system(3)}.
* No shell is executed.
*
* <h4>Examples</h4>
*
* <p>The most basic use-case for this class is as follows:
*
* <pre>
* String[] args = { "/bin/du", "-s", directory };
* BlazeCommandResult result = new Command(args).execute();
* String output = new String(result.getStdout());
* </pre>
*
* which writes the output of the {@code du(1)} command into {@code output}. More complex cases
* might inspect the stderr stream, kill the subprocess asynchronously, feed input to its standard
* input, handle the exceptions thrown if the command fails, or print the termination status (exit
* code or signal name).
*
* <h4>Other Features</h4>
*
* <p>A caller can optionally specify bytes to be written to the process's "stdin". The returned
* {@link CommandResult} object gives the caller access to the exit status, as well as output from
* "stdout" and "stderr". To use this class with processes that generate very large amounts of
* input/output, consider {@link #execute(OutputStream, OutputStream)}, {@link
* #executeAsync(OutputStream, OutputStream)}, or {@link #executeAsync(InputStream, OutputStream,
* OutputStream, boolean)}.
*
* <p>This class ensures that stdout and stderr streams are read promptly, avoiding potential
* deadlock if the output is large. See <a
* href="http://www.javaworld.com/javaworld/jw-12-2000/jw-1229-traps.html">when <code>Runtime.exec()
* </code> won't</a>.
*
* <h4>Caution: Invoking Shell Commands</h4>
*
* <p>Perhaps the most common command invoked programmatically is the UNIX shell, {@code /bin/sh}.
* Because the shell is a general-purpose programming language, care must be taken to ensure that
* variable parts of the shell command (e.g. strings entered by the user) do not contain shell
* metacharacters, as this poses a correctness and/or security risk.
*
* <p>To execute a shell command directly, use the following pattern:
*
* <pre>
* String[] args = { "/bin/sh", "-c", shellCommand };
* BlazeCommandResult result = new Command(args).execute();
* </pre>
*
* {@code shellCommand} is a complete Bourne shell program, possibly containing all kinds of
* unescaped metacharacters. For example, here's a shell command that enumerates the working
* directories of all processes named "foo":
*
* <pre>ps auxx | grep foo | awk '{print $1}' |
* while read pid; do readlink /proc/$pid/cwd; done</pre>
*
* It is the responsibility of the caller to ensure that this string means what they intend.
*
* <p>Consider the risk posed by allowing the "foo" part of the previous command to be some
* arbitrary (untrusted) string called {@code processName}:
*
* <pre>
* // WARNING: unsafe!
* String shellCommand = "ps auxx | grep " + processName + " | awk '{print $1}' | "
* + "while read pid; do readlink /proc/$pid/cwd; done";</pre>
*
* </pre>
*
* Passing this string to {@link Command} is unsafe because if the string {@code processName}
* contains shell metacharacters, the meaning of the command can be arbitrarily changed; consider:
*
* <pre>String processName = ". ; rm -fr $HOME & ";</pre>
*
* <p>To defend against this possibility, it is essential to properly quote the variable portions of
* the shell command so that shell metacharacters are escaped. Use {@link ShellUtils#shellEscape}
* for this purpose:
*
* <pre>
* // Safe.
* String shellCommand = "ps auxx | grep " + ShellUtils.shellEscape(processName)
* + " | awk '{print $1}' | while read pid; do readlink /proc/$pid/cwd; done";
* </pre>
*
* <p>Tip: if you are only invoking a single known command, and no shell features (e.g. $PATH
* lookup, output redirection, pipelines, etc) are needed, call it directly without using a shell,
* as in the {@code du(1)} example above.
*/
public final class Command implements DescribableExecutionUnit {
private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
/** Pass this value to {@link #execute} to indicate that no input should be written to stdin. */
public static final InputStream NO_INPUT = new NullInputStream();
public static final boolean KILL_SUBPROCESS_ON_INTERRUPT = true;
public static final boolean CONTINUE_SUBPROCESS_ON_INTERRUPT = false;
private final SubprocessBuilder subprocessBuilder;
/**
* Creates a new {@link Command} for the given command line. The environment is inherited from the
* current process, as is the working directory. No timeout is enforced. The command line is
* executed exactly as given, without a shell. Subsequent calls to {@link #execute()} will use the
* JVM's working directory and environment.
*
* @param commandLineElements elements of raw command line to execute
* @throws IllegalArgumentException if commandLine is null or empty
*/
public Command(String[] commandLineElements) {
this(commandLineElements, null, null, Duration.ZERO);
}
/**
* Just like {@link #Command(String[], Map, File, Duration)}, but without a timeout.
*/
public Command(
String[] commandLineElements,
@Nullable Map<String, String> environmentVariables,
@Nullable File workingDirectory) {
this(commandLineElements, environmentVariables, workingDirectory, Duration.ZERO);
}
/**
* Creates a new {@link Command} for the given command line elements. The command line is executed
* without a shell.
*
* <p>The given environment variables and working directory are used in subsequent calls to {@link
* #execute()}.
*
* <p>This command treats the 0-th element of {@code commandLineElement} (the name of an
* executable to run) specially.
*
* <ul>
* <li>If it is an absolute path, it is used as it
* <li>If it is a single file name, the PATH lookup is performed
* <li>If it is a relative path that is not a single file name, the command will attempt to
* execute the binary at that path relative to {@code workingDirectory}.
* </ul>
*
* @param commandLineElements elements of raw command line to execute
* @param environmentVariables environment variables to replace JVM's environment variables; may
* be null
* @param workingDirectory working directory for execution; if null, the VM's current working
* directory is used
* @param timeout timeout; a value less than or equal to 0 is treated as no timeout
* @throws IllegalArgumentException if commandLine is null or empty
*/
// TODO(ulfjack): Throw a special exception if there was a timeout.
public Command(
String[] commandLineElements,
@Nullable Map<String, String> environmentVariables,
@Nullable File workingDirectory,
Duration timeout) {
Preconditions.checkNotNull(commandLineElements);
Preconditions.checkArgument(
commandLineElements.length != 0, "cannot run an empty command line");
File executable = new File(commandLineElements[0]);
if (!executable.isAbsolute() && executable.getParent() != null) {
commandLineElements = commandLineElements.clone();
commandLineElements[0] = new File(workingDirectory, commandLineElements[0]).getAbsolutePath();
}
this.subprocessBuilder = new SubprocessBuilder();
subprocessBuilder.setArgv(ImmutableList.copyOf(commandLineElements));
subprocessBuilder.setEnv(environmentVariables);
subprocessBuilder.setWorkingDirectory(workingDirectory);
subprocessBuilder.setTimeoutMillis(timeout.toMillis());
}
/** Returns the raw command line elements to be executed */
@Override
public ImmutableList<String> getArguments() {
return subprocessBuilder.getArgv();
}
/** Returns an (unmodifiable) {@link Map} view of command's environment variables or null. */
@Override
@Nullable
public ImmutableMap<String, String> getEnvironment() {
return subprocessBuilder.getEnv();
}
/** Returns the working directory to be used for execution, or null. */
@Nullable public File getWorkingDirectory() {
return subprocessBuilder.getWorkingDirectory();
}
/**
* Execute this command with no input to stdin, and with the output captured in memory. If the
* current process is interrupted, then the subprocess is also interrupted. This call blocks until
* the subprocess completes or an error occurs.
*
* <p>This method is a convenience wrapper for <code>executeAsync().get()</code>.
*
* @return {@link CommandResult} representing result of the execution
* @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any reason
* @throws AbnormalTerminationException if an {@link IOException} is encountered while reading
* from the process, or the process was terminated due to a signal
* @throws BadExitStatusException if the process exits with a non-zero status
*/
public CommandResult execute() throws CommandException, InterruptedException {
return executeAsync().get();
}
/**
* Execute this command with no input to stdin, and with the output streamed to the given output
* streams, which must be thread-safe. If the current process is interrupted, then the subprocess
* is also interrupted. This call blocks until the subprocess completes or an error occurs.
*
* <p>Note that the given output streams are never closed by this class.
*
* <p>This method is a convenience wrapper for <code>executeAsync(stdOut, stdErr).get()</code>.
*
* @return {@link CommandResult} representing result of the execution
* @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any reason
* @throws AbnormalTerminationException if an {@link IOException} is encountered while reading
* from the process, or the process was terminated due to a signal
* @throws BadExitStatusException if the process exits with a non-zero status
*/
public CommandResult execute(OutputStream stdOut, OutputStream stdErr)
throws CommandException, InterruptedException {
return doExecute(
NO_INPUT, Consumers.createStreamingConsumers(stdOut, stdErr), KILL_SUBPROCESS_ON_INTERRUPT)
.get();
}
/**
* Execute this command with no input to stdin, and with the output captured in memory. If the
* current process is interrupted, then the subprocess is also interrupted. This call blocks until
* the subprocess is started or throws an error if that fails, but does not wait for the
* subprocess to exit.
*
* @return {@link CommandResult} representing result of the execution
* @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any reason
* @throws AbnormalTerminationException if an {@link IOException} is encountered while reading
* from the process, or the process was terminated due to a signal
* @throws BadExitStatusException if the process exits with a non-zero status
*/
public FutureCommandResult executeAsync() throws CommandException {
return doExecute(
NO_INPUT, Consumers.createAccumulatingConsumers(), KILL_SUBPROCESS_ON_INTERRUPT);
}
/**
* Execute this command with no input to stdin, and with the output streamed to the given output
* streams, which must be thread-safe. If the current process is interrupted, then the subprocess
* is also interrupted. This call blocks until the subprocess is started or throws an error if
* that fails, but does not wait for the subprocess to exit.
*
* <p>Note that the given output streams are never closed by this class.
*
* @return {@link CommandResult} representing result of the execution
* @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any reason
* @throws AbnormalTerminationException if an {@link IOException} is encountered while reading
* from the process, or the process was terminated due to a signal
* @throws BadExitStatusException if the process exits with a non-zero status
*/
public FutureCommandResult executeAsync(OutputStream stdOut, OutputStream stdErr)
throws CommandException {
return doExecute(
NO_INPUT, Consumers.createStreamingConsumers(stdOut, stdErr), KILL_SUBPROCESS_ON_INTERRUPT);
}
/**
* Execute this command with no input to stdin, and with the output captured in memory. This call
* blocks until the subprocess is started or throws an error if that fails, but does not wait for
* the subprocess to exit.
*
* @param killSubprocessOnInterrupt whether the subprocess should be killed if the current process
* is interrupted. If this is true, the returned {@link FutureCommandResult} object may throw
* {@link InterruptedException} on {@link FutureCommandResult#get} if the thread is
* interrupted while waiting for the process to complete. Otherwise, it will not.
* @return {@link CommandResult} representing result of the execution
* @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any reason
* @throws AbnormalTerminationException if an {@link IOException} is encountered while reading
* from the process, or the process was terminated due to a signal
* @throws BadExitStatusException if the process exits with a non-zero status
*/
public FutureCommandResult executeAsync(InputStream stdinInput, boolean killSubprocessOnInterrupt)
throws CommandException {
return doExecute(
stdinInput, Consumers.createAccumulatingConsumers(), killSubprocessOnInterrupt);
}
/**
* Execute this command with no input to stdin, and with the output streamed to the given output
* streams, which must be thread-safe. This call blocks until the subprocess is started or throws
* an error if that fails, but does not wait for the subprocess to exit.
*
* <p>Note that the given output streams are never closed by this class.
*
* @param killSubprocessOnInterrupt whether the subprocess should be killed if the current process
* is interrupted
* @return {@link CommandResult} representing result of the execution
* @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any reason
* @throws AbnormalTerminationException if an {@link IOException} is encountered while reading
* from the process, or the process was terminated due to a signal
* @throws BadExitStatusException if the process exits with a non-zero status
*/
public FutureCommandResult executeAsync(
InputStream stdinInput,
OutputStream stdOut,
OutputStream stdErr,
boolean killSubprocessOnInterrupt)
throws CommandException {
return doExecute(
stdinInput, Consumers.createStreamingConsumers(stdOut, stdErr), killSubprocessOnInterrupt);
}
/**
* A string representation of this command object which includes the arguments, the environment,
* and the working directory. Avoid relying on the specifics of this format. Note that the size of
* the result string will reflect the size of the command.
*/
public String toDebugString() {
StringBuilder message = new StringBuilder(128);
message.append("Executing (without brackets):");
for (String arg : subprocessBuilder.getArgv()) {
message.append(" [");
message.append(arg);
message.append(']');
}
message.append("; environment: ");
message.append(subprocessBuilder.getEnv());
message.append("; working dir: ");
File workingDirectory = subprocessBuilder.getWorkingDirectory();
message.append(workingDirectory == null ?
"(current)" :
workingDirectory.toString());
return message.toString();
}
private FutureCommandResult doExecute(
InputStream stdinInput, OutErrConsumers outErrConsumers, boolean killSubprocessOnInterrupt)
throws ExecFailedException {
Preconditions.checkNotNull(stdinInput, "stdinInput");
logCommand();
Subprocess process = startProcess();
outErrConsumers.logConsumptionStrategy();
outErrConsumers.registerInputs(
process.getInputStream(), process.getErrorStream(), /* closeStreams= */ false);
// TODO(ulfjack): This call blocks until all input is written. If stdinInput is large (or
// unbounded), then the async calls can block for a long time, and the timeout is not properly
// enforced.
processInput(stdinInput, process);
return new FutureCommandResult(this, process, outErrConsumers, killSubprocessOnInterrupt);
}
private Subprocess startProcess() throws ExecFailedException {
try {
return subprocessBuilder.start();
} catch (IOException ioe) {
throw new ExecFailedException(this, ioe);
}
}
private static class NullInputStream extends InputStream {
@Override
public int read() {
return -1;
}
@Override
public int available() {
return 0;
}
}
private static void processInput(InputStream stdinInput, Subprocess process) {
logger.atFiner().log("%s", stdinInput);
try (OutputStream out = process.getOutputStream()) {
ByteStreams.copy(stdinInput, out);
} catch (IOException ioe) {
// Note: this is not an error! Perhaps the command just isn't hungry for our input and exited
// with success. Process.waitFor (later) will tell us.
//
// (Unlike out/err streams, which are read asynchronously, the input stream is written
// synchronously, in its entirety, before processInput returns. If the input is infinite, and
// is passed through e.g. "cat" subprocess and back into the ByteArrayOutputStream, that will
// eventually run out of memory, causing the output stream to be closed, "cat" to terminate
// with SIGPIPE, and processInput to receive an IOException.
}
}
private void logCommand() {
logger.atFine().log("%s", LazyArgs.lazy(this::toDebugString));
}
@Override
public String getMnemonic() {
return "<shell command>";
}
}