| // Copyright 2018 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| package com.google.devtools.build.lib.actions; |
| |
| import com.google.common.annotations.VisibleForTesting; |
| import com.google.common.base.Preconditions; |
| import com.google.common.collect.ImmutableList; |
| import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander; |
| import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; |
| import com.google.devtools.build.lib.actions.cache.VirtualActionInput; |
| import com.google.devtools.build.lib.collect.IterablesChain; |
| import com.google.devtools.build.lib.util.Fingerprint; |
| import com.google.devtools.build.lib.vfs.PathFragment; |
| import com.google.protobuf.ByteString; |
| import java.io.IOException; |
| import java.io.OutputStream; |
| import java.nio.charset.Charset; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.List; |
| import java.util.UUID; |
| import javax.annotation.Nullable; |
| |
| /** |
| * A class that keeps a list of command lines and optional associated parameter file info. |
| * |
| * <p>This class is used by {@link com.google.devtools.build.lib.exec.SpawnRunner} implementations |
| * to expand the command lines into a master argument list + any param files needed to be written. |
| */ |
| public class CommandLines { |
| |
| // A (hopefully) conservative estimate of how much long each param file arg would be |
| // eg. the length of '@path/to/param_file'. |
| private static final int PARAM_FILE_ARG_LENGTH_ESTIMATE = 512; |
| private static final UUID PARAM_FILE_UUID = |
| UUID.fromString("106c1389-88d7-4cc1-8f05-f8a61fd8f7b1"); |
| |
| /** Command line OS limitations, such as the max length. */ |
| public static class CommandLineLimits { |
| /** |
| * "Unlimited" command line limits. |
| * |
| * <p>Use these limits when you want to prohibit param files, or you don't use param files so |
| * you don't care what the limit is. |
| */ |
| public static final CommandLineLimits UNLIMITED = new CommandLineLimits(Integer.MAX_VALUE); |
| |
| public final int maxLength; |
| |
| public CommandLineLimits(int maxLength) { |
| this.maxLength = maxLength; |
| } |
| } |
| |
| /** A simple tuple of a {@link CommandLine} and a {@link ParamFileInfo}. */ |
| public static class CommandLineAndParamFileInfo { |
| public final CommandLine commandLine; |
| @Nullable public final ParamFileInfo paramFileInfo; |
| |
| public CommandLineAndParamFileInfo( |
| CommandLine commandLine, @Nullable ParamFileInfo paramFileInfo) { |
| this.commandLine = commandLine; |
| this.paramFileInfo = paramFileInfo; |
| } |
| } |
| |
| /** |
| * Memory optimization: Store as Object instead of <code>List<CommandLineAndParamFileInfo></code>. |
| * |
| * <p>We store either a single CommandLine or CommandLineAndParamFileInfo, or list of Objects |
| * where each item is either a CommandLine or CommandLineAndParamFileInfo. This minimizes unneeded |
| * wrapper objects. |
| * |
| * <p>In the case of actions with a single CommandLine, this saves 48 bytes per action. |
| */ |
| private final Object commandLines; |
| |
| private CommandLines(Object commandLines) { |
| this.commandLines = commandLines; |
| } |
| |
| /** |
| * Expands this object into a single primary command line and (0-N) param files. The spawn runner |
| * is expected to write these param files prior to execution of an action. |
| * |
| * @param artifactExpander The artifact expander to use. |
| * @param paramFileBasePath Used to derive param file names. Often the first output of an action. |
| * @param limits The command line limits the host OS can support. |
| * @return The expanded command line and its param files (if any). |
| */ |
| public ExpandedCommandLines expand( |
| ArtifactExpander artifactExpander, PathFragment paramFileBasePath, CommandLineLimits limits) |
| throws CommandLineExpansionException { |
| return expand(artifactExpander, paramFileBasePath, limits, PARAM_FILE_ARG_LENGTH_ESTIMATE); |
| } |
| |
| /** |
| * Returns all arguments, including ones inside of param files. |
| * |
| * <p>Suitable for debugging and printing messages to users. This expands all command lines, so it |
| * is potentially expensive. |
| */ |
| public ImmutableList<String> allArguments() throws CommandLineExpansionException { |
| ImmutableList.Builder<String> arguments = ImmutableList.builder(); |
| for (CommandLineAndParamFileInfo pair : getCommandLines()) { |
| arguments.addAll(pair.commandLine.arguments()); |
| } |
| return arguments.build(); |
| } |
| |
| @VisibleForTesting |
| ExpandedCommandLines expand( |
| ArtifactExpander artifactExpander, |
| PathFragment paramFileBasePath, |
| CommandLineLimits limits, |
| int paramFileArgLengthEstimate) |
| throws CommandLineExpansionException { |
| // Optimize for simple case of single command line |
| if (commandLines instanceof CommandLine) { |
| CommandLine commandLine = (CommandLine) commandLines; |
| Iterable<String> arguments = commandLine.arguments(artifactExpander); |
| return new ExpandedCommandLines(arguments, ImmutableList.of()); |
| } |
| List<CommandLineAndParamFileInfo> commandLines = getCommandLines(); |
| IterablesChain.Builder<String> arguments = IterablesChain.builder(); |
| ArrayList<ParamFileActionInput> paramFiles = new ArrayList<>(commandLines.size()); |
| int conservativeMaxLength = limits.maxLength - commandLines.size() * paramFileArgLengthEstimate; |
| int cmdLineLength = 0; |
| // We name based on the output, starting at <output>-0.params and then incrementing |
| int paramFileNameSuffix = 0; |
| for (CommandLineAndParamFileInfo pair : commandLines) { |
| CommandLine commandLine = pair.commandLine; |
| ParamFileInfo paramFileInfo = pair.paramFileInfo; |
| if (paramFileInfo == null) { |
| Iterable<String> args = commandLine.arguments(artifactExpander); |
| arguments.add(args); |
| cmdLineLength += totalArgLen(args); |
| } else { |
| Preconditions.checkNotNull(paramFileInfo); // If null, we would have just had a CommandLine |
| Iterable<String> args = commandLine.arguments(artifactExpander); |
| boolean useParamFile = true; |
| if (!paramFileInfo.always()) { |
| int tentativeCmdLineLength = cmdLineLength + totalArgLen(args); |
| if (tentativeCmdLineLength <= conservativeMaxLength) { |
| arguments.add(args); |
| cmdLineLength = tentativeCmdLineLength; |
| useParamFile = false; |
| } |
| } |
| if (useParamFile) { |
| PathFragment paramFileExecPath = |
| ParameterFile.derivePath(paramFileBasePath, Integer.toString(paramFileNameSuffix)); |
| ++paramFileNameSuffix; |
| |
| String paramArg = |
| SingleStringArgFormatter.format( |
| paramFileInfo.getFlagFormatString(), paramFileExecPath.getPathString()); |
| arguments.addElement(paramArg); |
| cmdLineLength += paramArg.length() + 1; |
| paramFiles.add( |
| new ParamFileActionInput( |
| paramFileExecPath, |
| args, |
| paramFileInfo.getFileType(), |
| paramFileInfo.getCharset())); |
| } |
| } |
| } |
| return new ExpandedCommandLines(arguments.build(), paramFiles); |
| } |
| |
| public void addToFingerprint(ActionKeyContext actionKeyContext, Fingerprint fingerprint) |
| throws CommandLineExpansionException { |
| // Optimize for simple case of single command line |
| if (commandLines instanceof CommandLine) { |
| CommandLine commandLine = (CommandLine) commandLines; |
| commandLine.addToFingerprint(actionKeyContext, fingerprint); |
| return; |
| } |
| List<CommandLineAndParamFileInfo> commandLines = getCommandLines(); |
| for (CommandLineAndParamFileInfo pair : commandLines) { |
| CommandLine commandLine = pair.commandLine; |
| ParamFileInfo paramFileInfo = pair.paramFileInfo; |
| commandLine.addToFingerprint(actionKeyContext, fingerprint); |
| if (paramFileInfo != null) { |
| addParamFileInfoToFingerprint(paramFileInfo, fingerprint); |
| } |
| } |
| } |
| |
| /** |
| * Expanded command lines. |
| * |
| * <p>The spawn runner implementation is expected to ensure the param files are available once the |
| * spawn is executed. |
| */ |
| public static class ExpandedCommandLines { |
| private final Iterable<String> arguments; |
| private final List<ParamFileActionInput> paramFiles; |
| |
| ExpandedCommandLines( |
| Iterable<String> arguments, |
| List<ParamFileActionInput> paramFiles) { |
| this.arguments = arguments; |
| this.paramFiles = paramFiles; |
| } |
| |
| /** Returns the primary command line of the command. */ |
| public Iterable<String> arguments() { |
| return arguments; |
| } |
| |
| /** Returns the param file action inputs needed to execute the command. */ |
| public List<ParamFileActionInput> getParamFiles() { |
| return paramFiles; |
| } |
| } |
| |
| /** An in-memory param file virtual action input. */ |
| public static final class ParamFileActionInput implements VirtualActionInput { |
| final PathFragment paramFileExecPath; |
| final Iterable<String> arguments; |
| final ParameterFileType type; |
| final Charset charset; |
| |
| public ParamFileActionInput( |
| PathFragment paramFileExecPath, |
| Iterable<String> arguments, |
| ParameterFileType type, |
| Charset charset) { |
| this.paramFileExecPath = paramFileExecPath; |
| this.arguments = arguments; |
| this.type = type; |
| this.charset = charset; |
| } |
| |
| @Override |
| public void writeTo(OutputStream out) throws IOException { |
| ParameterFile.writeParameterFile(out, arguments, type, charset); |
| } |
| |
| @Override |
| public ByteString getBytes() throws IOException { |
| ByteString.Output out = ByteString.newOutput(); |
| writeTo(out); |
| return out.toByteString(); |
| } |
| |
| @Override |
| public String getExecPathString() { |
| return paramFileExecPath.getPathString(); |
| } |
| |
| @Override |
| public PathFragment getExecPath() { |
| return paramFileExecPath; |
| } |
| } |
| |
| // Helper function to unpack the optimized storage format into a list |
| @SuppressWarnings("unchecked") |
| public List<CommandLineAndParamFileInfo> getCommandLines() { |
| if (commandLines instanceof CommandLine) { |
| return ImmutableList.of(new CommandLineAndParamFileInfo((CommandLine) commandLines, null)); |
| } else if (commandLines instanceof CommandLineAndParamFileInfo) { |
| return ImmutableList.of((CommandLineAndParamFileInfo) commandLines); |
| } else { |
| List<Object> commandLines = Arrays.asList((Object[]) this.commandLines); |
| ImmutableList.Builder<CommandLineAndParamFileInfo> result = |
| ImmutableList.builderWithExpectedSize(commandLines.size()); |
| for (Object commandLine : commandLines) { |
| if (commandLine instanceof CommandLine) { |
| result.add(new CommandLineAndParamFileInfo((CommandLine) commandLine, null)); |
| } else { |
| result.add((CommandLineAndParamFileInfo) commandLine); |
| } |
| } |
| return result.build(); |
| } |
| } |
| |
| private static int totalArgLen(Iterable<String> args) { |
| int result = 0; |
| for (String s : args) { |
| result += s.length() + 1; |
| } |
| return result; |
| } |
| |
| private static void addParamFileInfoToFingerprint( |
| ParamFileInfo paramFileInfo, Fingerprint fingerprint) { |
| fingerprint.addUUID(PARAM_FILE_UUID); |
| fingerprint.addString(paramFileInfo.getFlagFormatString()); |
| fingerprint.addString(paramFileInfo.getFileType().toString()); |
| fingerprint.addString(paramFileInfo.getCharset().toString()); |
| } |
| |
| public static Builder builder() { |
| return new Builder(); |
| } |
| |
| public static Builder builder(Builder other) { |
| return new Builder(other); |
| } |
| |
| /** Returns an instance with a single command line. */ |
| public static CommandLines of(CommandLine commandLine) { |
| return new CommandLines(commandLine); |
| } |
| |
| /** Returns an instance with a single trivial command line. */ |
| public static CommandLines of(Iterable<String> args) { |
| return new CommandLines(CommandLine.of(args)); |
| } |
| |
| public static CommandLines concat(CommandLine commandLine, CommandLines commandLines) { |
| Builder builder = builder(); |
| builder.addCommandLine(commandLine); |
| for (CommandLineAndParamFileInfo pair : commandLines.getCommandLines()) { |
| builder.addCommandLine(pair); |
| } |
| return builder.build(); |
| } |
| |
| /** Builder for {@link CommandLines}. */ |
| public static class Builder { |
| private final List<CommandLineAndParamFileInfo> commandLines; |
| |
| Builder() { |
| commandLines = new ArrayList<>(); |
| } |
| |
| Builder(Builder other) { |
| commandLines = new ArrayList<>(other.commandLines); |
| } |
| |
| public Builder addCommandLine(CommandLine commandLine) { |
| commandLines.add(new CommandLineAndParamFileInfo(commandLine, null)); |
| return this; |
| } |
| |
| public Builder addCommandLine(CommandLine commandLine, ParamFileInfo paramFileInfo) { |
| return addCommandLine(new CommandLineAndParamFileInfo(commandLine, paramFileInfo)); |
| } |
| |
| public Builder addCommandLine(CommandLineAndParamFileInfo pair) { |
| commandLines.add(pair); |
| return this; |
| } |
| |
| public CommandLines build() { |
| final Object commandLines; |
| if (this.commandLines.size() == 1) { |
| CommandLineAndParamFileInfo pair = this.commandLines.get(0); |
| if (pair.paramFileInfo != null) { |
| commandLines = pair; |
| } else { |
| commandLines = pair.commandLine; |
| } |
| } else { |
| Object[] result = new Object[this.commandLines.size()]; |
| for (int i = 0; i < this.commandLines.size(); ++i) { |
| CommandLineAndParamFileInfo pair = this.commandLines.get(i); |
| if (pair.paramFileInfo != null) { |
| result[i] = pair; |
| } else { |
| result[i] = pair.commandLine; |
| } |
| } |
| commandLines = result; |
| } |
| return new CommandLines(commandLines); |
| } |
| } |
| } |