| // Copyright 2014 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package com.google.devtools.build.lib.analysis; |
| |
| import com.google.common.annotations.VisibleForTesting; |
| import com.google.common.collect.ImmutableCollection; |
| import com.google.common.collect.ImmutableList; |
| import com.google.common.collect.ImmutableMap; |
| import com.google.common.collect.Iterables; |
| import com.google.common.collect.Sets; |
| import com.google.devtools.build.lib.actions.Artifact; |
| import com.google.devtools.build.lib.actions.ExecutionRequirements; |
| import com.google.devtools.build.lib.actions.RunfilesSupplier; |
| import com.google.devtools.build.lib.analysis.actions.FileWriteAction; |
| import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget.Mode; |
| import com.google.devtools.build.lib.cmdline.Label; |
| import com.google.devtools.build.lib.collect.nestedset.NestedSet; |
| import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; |
| import com.google.devtools.build.lib.syntax.SkylarkList; |
| import com.google.devtools.build.lib.util.OS; |
| import com.google.devtools.build.lib.util.Pair; |
| import com.google.devtools.build.lib.vfs.PathFragment; |
| import java.util.Collection; |
| import java.util.HashMap; |
| import java.util.List; |
| import java.util.Map; |
| import javax.annotation.Nullable; |
| |
| /** |
| * Provides shared functionality for parameterized command-line launching. |
| * Also used by {@link com.google.devtools.build.lib.rules.extra.ExtraActionFactory}. |
| * |
| * Two largely independent separate sets of functionality are provided: |
| * 1- string interpolation for {@code $(location[s] ...)} and {@code $(MakeVariable)} |
| * 2- a utility to build potentially large command lines (presumably made of multiple commands), |
| * that if presumed too large for the kernel's taste can be dumped into a shell script |
| * that will contain the same commands, |
| * at which point the shell script is added to the list of inputs. |
| */ |
| public final class CommandHelper { |
| |
| /** |
| * Returns a new {@link Builder} to create a {@link CommandHelper} based on the given {@link |
| * RuleContext}. |
| */ |
| public static Builder builder(RuleContext ruleContext) { |
| return new Builder(ruleContext); |
| } |
| |
| /** |
| * Builder class to assist with creating an instance of {@link CommandHelper}. The Builder can |
| * optionally add additional tools as dependencies, and a map of labels to be resolved. |
| */ |
| public static final class Builder { |
| private final RuleContext ruleContext; |
| private final ImmutableList.Builder<Iterable<? extends TransitiveInfoCollection>> |
| toolDependencies = ImmutableList.builder(); |
| private final ImmutableMap.Builder<Label, Iterable<Artifact>> labelMap = ImmutableMap.builder(); |
| |
| private Builder(RuleContext ruleContext) { |
| this.ruleContext = ruleContext; |
| } |
| |
| /** |
| * Adds tools, as a set of executable binaries, by fetching them from the given attribute on the |
| * {@code ruleContext}, in HOST mode. Populates manifests, remoteRunfiles and label map where |
| * required. |
| */ |
| public Builder addHostToolDependencies(String toolAttributeName) { |
| List<? extends TransitiveInfoCollection> dependencies = |
| ruleContext.getPrerequisites(toolAttributeName, Mode.HOST); |
| addToolDependencies(dependencies); |
| return this; |
| } |
| |
| /** |
| * Adds tools, as a set of executable binaries. Populates manifests, remoteRunfiles and label |
| * map where required. |
| */ |
| public Builder addToolDependencies( |
| Iterable<? extends TransitiveInfoCollection> toolDependencies) { |
| this.toolDependencies.add(toolDependencies); |
| return this; |
| } |
| |
| /** Adds files to set of known files of label. Used for resolving $(location) variables. */ |
| public Builder addLabelMap(Map<Label, ? extends Iterable<Artifact>> labelMap) { |
| this.labelMap.putAll(labelMap); |
| return this; |
| } |
| |
| /** Returns the built {@link CommandHelper}. */ |
| public CommandHelper build() { |
| return new CommandHelper(ruleContext, toolDependencies.build(), labelMap.build()); |
| } |
| } |
| |
| /** |
| * Maximum total command-line length, in bytes, not counting "/bin/bash -c ". |
| * If the command is very long, then we write the command to a script file, |
| * to avoid overflowing any limits on command-line length. |
| * For short commands, we just use /bin/bash -c command. |
| * |
| * Maximum command line length on Windows is 32767[1], but for cmd.exe it is 8192[2]. |
| * [1] https://msdn.microsoft.com/en-us/library/ms682425(VS.85).aspx |
| * [2] https://support.microsoft.com/en-us/kb/830473. |
| */ |
| @VisibleForTesting |
| public static int maxCommandLength = OS.getCurrent() == OS.WINDOWS ? 8000 : 64000; |
| |
| /** {@link RunfilesSupplier}s for tools used by this rule. */ |
| private final SkylarkList<RunfilesSupplier> toolsRunfilesSuppliers; |
| |
| /** |
| * Use labelMap for heuristically expanding labels (does not include "outs") |
| * This is similar to heuristic location expansion in LocationExpander |
| * and should be kept in sync. |
| */ |
| private final ImmutableMap<Label, ImmutableCollection<Artifact>> labelMap; |
| |
| /** |
| * The ruleContext this helper works on |
| */ |
| private final RuleContext ruleContext; |
| |
| /** |
| * Output executable files from the 'tools' attribute. |
| */ |
| private final NestedSet<Artifact> resolvedTools; |
| |
| /** |
| * Creates a {@link CommandHelper}. |
| * |
| * @param toolsList resolves sets of tools into set of executable binaries. Populates manifests, |
| * remoteRunfiles and label map where required. |
| * @param labelMap adds files to set of known files of label. Used for resolving $(location) |
| * variables. |
| */ |
| private CommandHelper( |
| RuleContext ruleContext, |
| ImmutableList<Iterable<? extends TransitiveInfoCollection>> toolsList, |
| ImmutableMap<Label, ? extends Iterable<Artifact>> labelMap) { |
| |
| this.ruleContext = ruleContext; |
| |
| NestedSetBuilder<Artifact> resolvedToolsBuilder = NestedSetBuilder.stableOrder(); |
| ImmutableList.Builder<RunfilesSupplier> toolsRunfilesBuilder = ImmutableList.builder(); |
| Map<Label, Collection<Artifact>> tempLabelMap = new HashMap<>(); |
| |
| for (Map.Entry<Label, ? extends Iterable<Artifact>> entry : labelMap.entrySet()) { |
| Iterables.addAll(mapGet(tempLabelMap, entry.getKey()), entry.getValue()); |
| } |
| |
| for (Iterable<? extends TransitiveInfoCollection> tools : toolsList) { |
| for (TransitiveInfoCollection dep : tools) { // (Note: host configuration) |
| Label label = AliasProvider.getDependencyLabel(dep); |
| MiddlemanProvider toolMiddleman = dep.getProvider(MiddlemanProvider.class); |
| if (toolMiddleman != null) { |
| resolvedToolsBuilder.addTransitive(toolMiddleman.getMiddlemanArtifact()); |
| // It is not obviously correct to skip potentially adding getFilesToRun of the |
| // FilesToRunProvider. However, for all tools that we know of that provide a middleman, |
| // the middleman is equivalent to the list of files coming out of getFilesToRun(). |
| // Just adding all the files creates a substantial performance bottleneck. E.g. a C++ |
| // toolchain might consist of thousands of files and tracking them one by one for each |
| // action that uses them is inefficient. |
| continue; |
| } |
| |
| FilesToRunProvider tool = dep.getProvider(FilesToRunProvider.class); |
| if (tool == null) { |
| continue; |
| } |
| |
| NestedSet<Artifact> files = tool.getFilesToRun(); |
| resolvedToolsBuilder.addTransitive(files); |
| Artifact executableArtifact = tool.getExecutable(); |
| // If the label has an executable artifact add that to the multimaps. |
| if (executableArtifact != null) { |
| mapGet(tempLabelMap, label).add(executableArtifact); |
| // Also send the runfiles when running remotely. |
| toolsRunfilesBuilder.add(tool.getRunfilesSupplier()); |
| } else { |
| // Map all depArtifacts to the respective label using the multimaps. |
| Iterables.addAll(mapGet(tempLabelMap, label), files); |
| } |
| } |
| } |
| |
| this.resolvedTools = resolvedToolsBuilder.build(); |
| this.toolsRunfilesSuppliers = SkylarkList.createImmutable(toolsRunfilesBuilder.build()); |
| ImmutableMap.Builder<Label, ImmutableCollection<Artifact>> labelMapBuilder = |
| ImmutableMap.builder(); |
| for (Map.Entry<Label, Collection<Artifact>> entry : tempLabelMap.entrySet()) { |
| labelMapBuilder.put(entry.getKey(), ImmutableList.copyOf(entry.getValue())); |
| } |
| this.labelMap = labelMapBuilder.build(); |
| } |
| |
| public NestedSet<Artifact> getResolvedTools() { |
| return resolvedTools; |
| } |
| |
| public SkylarkList<RunfilesSupplier> getToolsRunfilesSuppliers() { |
| return toolsRunfilesSuppliers; |
| } |
| |
| public ImmutableMap<Label, ImmutableCollection<Artifact>> getLabelMap() { |
| return labelMap; |
| } |
| |
| // Returns the value in the specified corresponding to 'key', creating and |
| // inserting an empty container if absent. We use Map not Multimap because |
| // we need to distinguish the cases of "empty value" and "absent key". |
| private static Collection<Artifact> mapGet(Map<Label, Collection<Artifact>> map, Label key) { |
| Collection<Artifact> values = map.get(key); |
| if (values == null) { |
| // We use sets not lists, because it's conceivable that the same artifact |
| // could appear twice, e.g. in "srcs" and "deps". |
| values = Sets.newHashSet(); |
| map.put(key, values); |
| } |
| return values; |
| } |
| |
| /** |
| * Resolves a command, and expands known locations for $(location) |
| * variables. |
| */ |
| @Deprecated // Only exists to support a legacy Skylark API. |
| public String resolveCommandAndExpandLabels( |
| String command, @Nullable String attribute, boolean allowDataInLabel) { |
| LocationExpander expander; |
| if (allowDataInLabel) { |
| expander = LocationExpander.withExecPathsAndData(ruleContext, labelMap); |
| } else { |
| expander = LocationExpander.withExecPaths(ruleContext, labelMap); |
| } |
| if (attribute != null) { |
| command = expander.expandAttribute(attribute, command); |
| } else { |
| command = expander.expand(command); |
| } |
| return command; |
| } |
| |
| /** |
| * Expands labels occurring in the string "expr" in the rule 'cmd'. |
| * Each label must be valid, be a declared prerequisite, and expand to a |
| * unique path. |
| * |
| * <p>If the expansion fails, an attribute error is reported and the original |
| * expression is returned. |
| */ |
| public String expandLabelsHeuristically(String expr) { |
| try { |
| return LabelExpander.expand(expr, labelMap, ruleContext.getLabel()); |
| } catch (LabelExpander.NotUniqueExpansionException nuee) { |
| ruleContext.attributeError("cmd", nuee.getMessage()); |
| return expr; |
| } |
| } |
| |
| private static Pair<List<String>, Artifact> buildCommandLineMaybeWithScriptFile( |
| RuleContext ruleContext, String command, String scriptPostFix, PathFragment shellPath) { |
| List<String> argv; |
| Artifact scriptFileArtifact = null; |
| if (command.length() <= maxCommandLength) { |
| argv = buildCommandLineSimpleArgv(command, shellPath); |
| } else { |
| // Use script file. |
| scriptFileArtifact = buildCommandLineArtifact(ruleContext, command, scriptPostFix); |
| argv = buildCommandLineArgvWithArtifact(scriptFileArtifact, shellPath); |
| } |
| return Pair.of(argv, scriptFileArtifact); |
| } |
| |
| private static ImmutableList<String> buildCommandLineArgvWithArtifact(Artifact scriptFileArtifact, |
| PathFragment shellPath) { |
| return ImmutableList.of(shellPath.getPathString(), scriptFileArtifact.getExecPathString()); |
| } |
| |
| private static Artifact buildCommandLineArtifact(RuleContext ruleContext, String command, |
| String scriptPostFix) { |
| String scriptFileName = ruleContext.getTarget().getName() + scriptPostFix; |
| String scriptFileContents = "#!/bin/bash\n" + command; |
| Artifact scriptFileArtifact = FileWriteAction.createFile( |
| ruleContext, scriptFileName, scriptFileContents, /*executable=*/true); |
| return scriptFileArtifact; |
| } |
| |
| private static ImmutableList<String> buildCommandLineSimpleArgv(String command, |
| PathFragment shellPath) { |
| return ImmutableList.of(shellPath.getPathString(), "-c", command); |
| } |
| |
| /** |
| * If {@code command} is too long, creates a helper shell script that runs that command. |
| * |
| * <p>Returns the {@link Artifact} corresponding to that script. |
| * |
| * <p>Otherwise, when {@code command} is shorter than the platform's shell's command length limit, |
| * this method does nothing and returns null. |
| */ |
| @Nullable |
| public static Artifact shellCommandHelperScriptMaybe( |
| RuleContext ruleCtx, |
| String command, |
| String scriptPostFix, |
| Map<String, String> executionInfo) { |
| if (command.length() <= maxCommandLength) { |
| return null; |
| } else { |
| return buildCommandLineArtifact(ruleCtx, command, scriptPostFix); |
| } |
| } |
| |
| /** |
| * Builds the set of command-line arguments. Creates a bash script if the command line is longer |
| * than the allowed maximum {@link #maxCommandLength}. Fixes up the input artifact list with the |
| * created bash script when required. |
| */ |
| public List<String> buildCommandLine( |
| PathFragment shExecutable, |
| String command, |
| NestedSetBuilder<Artifact> inputs, |
| String scriptPostFix) { |
| return buildCommandLine( |
| shExecutable, command, inputs, scriptPostFix, ImmutableMap.<String, String>of()); |
| } |
| |
| /** |
| * Builds the set of command-line arguments using the specified shell path. Creates a bash script |
| * if the command line is longer than the allowed maximum {@link #maxCommandLength}. Fixes up the |
| * input artifact list with the created bash script when required. |
| * |
| * @param executionInfo an execution info map of the action associated with the command line to be |
| * built. |
| */ |
| public List<String> buildCommandLine( |
| PathFragment shExecutable, |
| String command, |
| NestedSetBuilder<Artifact> inputs, |
| String scriptPostFix, |
| Map<String, String> executionInfo) { |
| Pair<List<String>, Artifact> argvAndScriptFile = |
| buildCommandLineMaybeWithScriptFile( |
| ruleContext, command, scriptPostFix, shellPath(executionInfo, shExecutable)); |
| if (argvAndScriptFile.second != null) { |
| inputs.add(argvAndScriptFile.second); |
| } |
| return argvAndScriptFile.first; |
| } |
| |
| /** |
| * Builds the set of command-line arguments. Creates a bash script if the command line is longer |
| * than the allowed maximum {@link #maxCommandLength}. Fixes up the input artifact list with the |
| * created bash script when required. |
| */ |
| public List<String> buildCommandLine( |
| PathFragment shExecutable, |
| String command, |
| List<Artifact> inputs, |
| String scriptPostFix, |
| Map<String, String> executionInfo) { |
| Pair<List<String>, Artifact> argvAndScriptFile = |
| buildCommandLineMaybeWithScriptFile( |
| ruleContext, command, scriptPostFix, shellPath(executionInfo, shExecutable)); |
| if (argvAndScriptFile.second != null) { |
| inputs.add(argvAndScriptFile.second); |
| } |
| return argvAndScriptFile.first; |
| } |
| |
| /** Returns the path to the shell for an action with the given execution requirements. */ |
| private PathFragment shellPath(Map<String, String> executionInfo, PathFragment shExecutable) { |
| // Use vanilla /bin/bash for actions running on mac machines. |
| return executionInfo.containsKey(ExecutionRequirements.REQUIRES_DARWIN) |
| ? PathFragment.create("/bin/bash") |
| : shExecutable; |
| } |
| } |