blob: 1f601471b98bbacb522e90d581c657cd91537199 [file] [log] [blame]
// Copyright 2018 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.actions;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander;
import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType;
import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
import com.google.devtools.build.lib.collect.IterablesChain;
import com.google.devtools.build.lib.util.Fingerprint;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.protobuf.ByteString;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
import javax.annotation.Nullable;
/**
* A class that keeps a list of command lines and optional associated parameter file info.
*
* <p>This class is used by {@link com.google.devtools.build.lib.exec.SpawnRunner} implementations
* to expand the command lines into a master argument list + any param files needed to be written.
*/
public class CommandLines {
// A (hopefully) conservative estimate of how much long each param file arg would be
// eg. the length of '@path/to/param_file'.
private static final int PARAM_FILE_ARG_LENGTH_ESTIMATE = 512;
private static final UUID PARAM_FILE_UUID =
UUID.fromString("106c1389-88d7-4cc1-8f05-f8a61fd8f7b1");
/** Command line OS limitations, such as the max length. */
public static class CommandLineLimits {
/**
* "Unlimited" command line limits.
*
* <p>Use these limits when you want to prohibit param files, or you don't use param files so
* you don't care what the limit is.
*/
public static final CommandLineLimits UNLIMITED = new CommandLineLimits(Integer.MAX_VALUE);
public final int maxLength;
public CommandLineLimits(int maxLength) {
this.maxLength = maxLength;
}
}
/** A simple tuple of a {@link CommandLine} and a {@link ParamFileInfo}. */
public static class CommandLineAndParamFileInfo {
public final CommandLine commandLine;
@Nullable public final ParamFileInfo paramFileInfo;
public CommandLineAndParamFileInfo(
CommandLine commandLine, @Nullable ParamFileInfo paramFileInfo) {
this.commandLine = commandLine;
this.paramFileInfo = paramFileInfo;
}
}
/**
* Memory optimization: Store as Object instead of <code>List<CommandLineAndParamFileInfo></code>.
*
* <p>We store either a single CommandLine or CommandLineAndParamFileInfo, or list of Objects
* where each item is either a CommandLine or CommandLineAndParamFileInfo. This minimizes unneeded
* wrapper objects.
*
* <p>In the case of actions with a single CommandLine, this saves 48 bytes per action.
*/
private final Object commandLines;
private CommandLines(Object commandLines) {
this.commandLines = commandLines;
}
/**
* Expands this object into a single primary command line and (0-N) param files. The spawn runner
* is expected to write these param files prior to execution of an action.
*
* @param artifactExpander The artifact expander to use.
* @param paramFileBasePath Used to derive param file names. Often the first output of an action.
* @param limits The command line limits the host OS can support.
* @return The expanded command line and its param files (if any).
*/
public ExpandedCommandLines expand(
ArtifactExpander artifactExpander, PathFragment paramFileBasePath, CommandLineLimits limits)
throws CommandLineExpansionException {
return expand(artifactExpander, paramFileBasePath, limits, PARAM_FILE_ARG_LENGTH_ESTIMATE);
}
/**
* Returns all arguments, including ones inside of param files.
*
* <p>Suitable for debugging and printing messages to users. This expands all command lines, so it
* is potentially expensive.
*/
public ImmutableList<String> allArguments() throws CommandLineExpansionException {
ImmutableList.Builder<String> arguments = ImmutableList.builder();
for (CommandLineAndParamFileInfo pair : getCommandLines()) {
arguments.addAll(pair.commandLine.arguments());
}
return arguments.build();
}
@VisibleForTesting
ExpandedCommandLines expand(
ArtifactExpander artifactExpander,
PathFragment paramFileBasePath,
CommandLineLimits limits,
int paramFileArgLengthEstimate)
throws CommandLineExpansionException {
// Optimize for simple case of single command line
if (commandLines instanceof CommandLine) {
CommandLine commandLine = (CommandLine) commandLines;
Iterable<String> arguments = commandLine.arguments(artifactExpander);
return new ExpandedCommandLines(arguments, ImmutableList.of());
}
List<CommandLineAndParamFileInfo> commandLines = getCommandLines();
IterablesChain.Builder<String> arguments = IterablesChain.builder();
ArrayList<ParamFileActionInput> paramFiles = new ArrayList<>(commandLines.size());
int conservativeMaxLength = limits.maxLength - commandLines.size() * paramFileArgLengthEstimate;
int cmdLineLength = 0;
// We name based on the output, starting at <output>-0.params and then incrementing
int paramFileNameSuffix = 0;
for (CommandLineAndParamFileInfo pair : commandLines) {
CommandLine commandLine = pair.commandLine;
ParamFileInfo paramFileInfo = pair.paramFileInfo;
if (paramFileInfo == null) {
Iterable<String> args = commandLine.arguments(artifactExpander);
arguments.add(args);
cmdLineLength += totalArgLen(args);
} else {
Preconditions.checkNotNull(paramFileInfo); // If null, we would have just had a CommandLine
Iterable<String> args = commandLine.arguments(artifactExpander);
boolean useParamFile = true;
if (!paramFileInfo.always()) {
int tentativeCmdLineLength = cmdLineLength + totalArgLen(args);
if (tentativeCmdLineLength <= conservativeMaxLength) {
arguments.add(args);
cmdLineLength = tentativeCmdLineLength;
useParamFile = false;
}
}
if (useParamFile) {
PathFragment paramFileExecPath =
ParameterFile.derivePath(paramFileBasePath, Integer.toString(paramFileNameSuffix));
++paramFileNameSuffix;
String paramArg =
SingleStringArgFormatter.format(
paramFileInfo.getFlagFormatString(), paramFileExecPath.getPathString());
arguments.addElement(paramArg);
cmdLineLength += paramArg.length() + 1;
paramFiles.add(
new ParamFileActionInput(
paramFileExecPath,
args,
paramFileInfo.getFileType(),
paramFileInfo.getCharset()));
}
}
}
return new ExpandedCommandLines(arguments.build(), paramFiles);
}
public void addToFingerprint(ActionKeyContext actionKeyContext, Fingerprint fingerprint)
throws CommandLineExpansionException {
// Optimize for simple case of single command line
if (commandLines instanceof CommandLine) {
CommandLine commandLine = (CommandLine) commandLines;
commandLine.addToFingerprint(actionKeyContext, fingerprint);
return;
}
List<CommandLineAndParamFileInfo> commandLines = getCommandLines();
for (CommandLineAndParamFileInfo pair : commandLines) {
CommandLine commandLine = pair.commandLine;
ParamFileInfo paramFileInfo = pair.paramFileInfo;
commandLine.addToFingerprint(actionKeyContext, fingerprint);
if (paramFileInfo != null) {
addParamFileInfoToFingerprint(paramFileInfo, fingerprint);
}
}
}
/**
* Expanded command lines.
*
* <p>The spawn runner implementation is expected to ensure the param files are available once the
* spawn is executed.
*/
public static class ExpandedCommandLines {
private final Iterable<String> arguments;
private final List<ParamFileActionInput> paramFiles;
ExpandedCommandLines(
Iterable<String> arguments,
List<ParamFileActionInput> paramFiles) {
this.arguments = arguments;
this.paramFiles = paramFiles;
}
/** Returns the primary command line of the command. */
public Iterable<String> arguments() {
return arguments;
}
/** Returns the param file action inputs needed to execute the command. */
public List<ParamFileActionInput> getParamFiles() {
return paramFiles;
}
}
/** An in-memory param file virtual action input. */
public static final class ParamFileActionInput implements VirtualActionInput {
final PathFragment paramFileExecPath;
final Iterable<String> arguments;
final ParameterFileType type;
final Charset charset;
public ParamFileActionInput(
PathFragment paramFileExecPath,
Iterable<String> arguments,
ParameterFileType type,
Charset charset) {
this.paramFileExecPath = paramFileExecPath;
this.arguments = arguments;
this.type = type;
this.charset = charset;
}
@Override
public boolean isSymlink() {
return false;
}
@Override
public void writeTo(OutputStream out) throws IOException {
ParameterFile.writeParameterFile(out, arguments, type, charset);
}
@Override
public ByteString getBytes() throws IOException {
ByteString.Output out = ByteString.newOutput();
writeTo(out);
return out.toByteString();
}
@Override
public String getExecPathString() {
return paramFileExecPath.getPathString();
}
@Override
public PathFragment getExecPath() {
return paramFileExecPath;
}
}
// Helper function to unpack the optimized storage format into a list
@SuppressWarnings("unchecked")
public List<CommandLineAndParamFileInfo> getCommandLines() {
if (commandLines instanceof CommandLine) {
return ImmutableList.of(new CommandLineAndParamFileInfo((CommandLine) commandLines, null));
} else if (commandLines instanceof CommandLineAndParamFileInfo) {
return ImmutableList.of((CommandLineAndParamFileInfo) commandLines);
} else {
List<Object> commandLines = Arrays.asList((Object[]) this.commandLines);
ImmutableList.Builder<CommandLineAndParamFileInfo> result =
ImmutableList.builderWithExpectedSize(commandLines.size());
for (Object commandLine : commandLines) {
if (commandLine instanceof CommandLine) {
result.add(new CommandLineAndParamFileInfo((CommandLine) commandLine, null));
} else {
result.add((CommandLineAndParamFileInfo) commandLine);
}
}
return result.build();
}
}
private static int totalArgLen(Iterable<String> args) {
int result = 0;
for (String s : args) {
result += s.length() + 1;
}
return result;
}
private static void addParamFileInfoToFingerprint(
ParamFileInfo paramFileInfo, Fingerprint fingerprint) {
fingerprint.addUUID(PARAM_FILE_UUID);
fingerprint.addString(paramFileInfo.getFlagFormatString());
fingerprint.addString(paramFileInfo.getFileType().toString());
fingerprint.addString(paramFileInfo.getCharset().toString());
}
public static Builder builder() {
return new Builder();
}
public static Builder builder(Builder other) {
return new Builder(other);
}
/** Returns an instance with a single command line. */
public static CommandLines of(CommandLine commandLine) {
return new CommandLines(commandLine);
}
/** Returns an instance with a single trivial command line. */
public static CommandLines of(Iterable<String> args) {
return new CommandLines(CommandLine.of(args));
}
public static CommandLines concat(CommandLine commandLine, CommandLines commandLines) {
Builder builder = builder();
builder.addCommandLine(commandLine);
for (CommandLineAndParamFileInfo pair : commandLines.getCommandLines()) {
builder.addCommandLine(pair);
}
return builder.build();
}
/** Builder for {@link CommandLines}. */
public static class Builder {
private final List<CommandLineAndParamFileInfo> commandLines;
Builder() {
commandLines = new ArrayList<>();
}
Builder(Builder other) {
commandLines = new ArrayList<>(other.commandLines);
}
public Builder addCommandLine(CommandLine commandLine) {
commandLines.add(new CommandLineAndParamFileInfo(commandLine, null));
return this;
}
public Builder addCommandLine(CommandLine commandLine, ParamFileInfo paramFileInfo) {
return addCommandLine(new CommandLineAndParamFileInfo(commandLine, paramFileInfo));
}
public Builder addCommandLine(CommandLineAndParamFileInfo pair) {
commandLines.add(pair);
return this;
}
public CommandLines build() {
final Object commandLines;
if (this.commandLines.size() == 1) {
CommandLineAndParamFileInfo pair = this.commandLines.get(0);
if (pair.paramFileInfo != null) {
commandLines = pair;
} else {
commandLines = pair.commandLine;
}
} else {
Object[] result = new Object[this.commandLines.size()];
for (int i = 0; i < this.commandLines.size(); ++i) {
CommandLineAndParamFileInfo pair = this.commandLines.get(i);
if (pair.paramFileInfo != null) {
result[i] = pair;
} else {
result[i] = pair.commandLine;
}
}
commandLines = result;
}
return new CommandLines(commandLines);
}
}
}