| // Copyright 2014 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package com.google.devtools.build.lib.actions; |
| |
| import com.google.common.base.Preconditions; |
| import com.google.common.collect.ImmutableList; |
| import com.google.common.collect.ImmutableSet; |
| import com.google.common.collect.Iterables; |
| import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; |
| import com.google.devtools.build.lib.analysis.platform.PlatformInfo; |
| import com.google.devtools.build.lib.cmdline.Label; |
| import com.google.devtools.build.lib.collect.CollectionUtils; |
| import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; |
| import com.google.devtools.build.lib.collect.nestedset.Order; |
| import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; |
| import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; |
| import com.google.devtools.build.lib.events.Event; |
| import com.google.devtools.build.lib.events.EventHandler; |
| import com.google.devtools.build.lib.packages.AspectDescriptor; |
| import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec.VisibleForSerialization; |
| import com.google.devtools.build.lib.skylarkbuildapi.ActionApi; |
| import com.google.devtools.build.lib.skylarkinterface.SkylarkPrinter; |
| import com.google.devtools.build.lib.syntax.EvalException; |
| import com.google.devtools.build.lib.syntax.SkylarkDict; |
| import com.google.devtools.build.lib.syntax.SkylarkList; |
| import com.google.devtools.build.lib.syntax.SkylarkNestedSet; |
| import com.google.devtools.build.lib.util.Fingerprint; |
| import com.google.devtools.build.lib.vfs.FileSystem; |
| import com.google.devtools.build.lib.vfs.FileSystemUtils; |
| import com.google.devtools.build.lib.vfs.Path; |
| import com.google.devtools.build.lib.vfs.Root; |
| import com.google.devtools.build.lib.vfs.Symlinks; |
| import java.io.IOException; |
| import java.util.Collection; |
| import java.util.Map; |
| import javax.annotation.Nullable; |
| import javax.annotation.concurrent.GuardedBy; |
| |
| /** |
| * Abstract implementation of Action which implements basic functionality: the inputs, outputs, and |
| * toString method. Both input and output sets are immutable. Subclasses must be generally immutable |
| * - see the documentation on {@link Action}. |
| */ |
| @Immutable |
| @ThreadSafe |
| public abstract class AbstractAction implements Action, ActionApi { |
| /** |
| * An arbitrary default resource set. We assume that a typical subprocess is single-threaded |
| * (i.e., uses one CPU core) and CPU-bound, and uses a small-ish amount of memory. In the past, |
| * we've seen that assuming less than one core can lead to local overload. Unless you have data |
| * indicating otherwise (for example, we've observed in the past that C++ linking can use large |
| * amounts of memory), we suggest to use this default set. |
| */ |
| // TODO(ulfjack): Collect actual data to confirm that this is an acceptable approximation. |
| public static final ResourceSet DEFAULT_RESOURCE_SET = |
| ResourceSet.createWithRamCpu(250, 1); |
| |
| /** |
| * The owner/inputs/outputs attributes below should never be directly accessed even within |
| * AbstractAction itself. The appropriate getter methods should be used instead. This has to be |
| * done due to the fact that the getter methods can be overridden in subclasses. |
| */ |
| @VisibleForSerialization protected final ActionOwner owner; |
| |
| /** |
| * Tools are a subset of inputs and used by the WorkerSpawnStrategy to determine whether a |
| * compiler has changed since the last time it was used. This should include all artifacts that |
| * the tool does not dynamically reload / check on each unit of work - e.g. its own binary, the |
| * JDK for Java binaries, shared libraries, ... but not a configuration file, if it reloads that |
| * when it has changed. |
| * |
| * <p>If the "tools" set does not contain exactly the right set of artifacts, the following can |
| * happen: If an artifact that should be included is missing, the tool might not be restarted when |
| * it should, and builds can become incorrect (example: The compiler binary is not part of this |
| * set, then the compiler gets upgraded, but the worker strategy still reuses the old version). |
| * If an artifact that should *not* be included is accidentally part of this set, the worker |
| * process will be restarted more often that is necessary - e.g. if a file that is unique to each |
| * unit of work, e.g. the source code that a compiler should compile for a compile action, is |
| * part of this set, then the worker will never be reused and will be restarted for each unit of |
| * work. |
| */ |
| private final Iterable<Artifact> tools; |
| |
| @GuardedBy("this") |
| private boolean inputsDiscovered = false; // Only used when discoversInputs() returns true |
| |
| // The variable inputs is non-final only so that actions that discover their inputs can modify it. |
| @GuardedBy("this") |
| @VisibleForSerialization |
| protected Iterable<Artifact> inputs; |
| |
| protected final ActionEnvironment env; |
| private final RunfilesSupplier runfilesSupplier; |
| @VisibleForSerialization protected final ImmutableSet<Artifact> outputs; |
| |
| private String cachedKey; |
| |
| /** |
| * Construct an abstract action with the specified inputs and outputs; |
| */ |
| protected AbstractAction( |
| ActionOwner owner, |
| Iterable<Artifact> inputs, |
| Iterable<Artifact> outputs) { |
| this( |
| owner, |
| /*tools = */ImmutableList.of(), |
| inputs, |
| EmptyRunfilesSupplier.INSTANCE, |
| outputs, |
| ActionEnvironment.EMPTY); |
| } |
| |
| protected AbstractAction( |
| ActionOwner owner, |
| Iterable<Artifact> inputs, |
| Iterable<Artifact> outputs, |
| ActionEnvironment env) { |
| this( |
| owner, |
| /*tools = */ImmutableList.of(), |
| inputs, |
| EmptyRunfilesSupplier.INSTANCE, |
| outputs, |
| env); |
| } |
| |
| protected AbstractAction( |
| ActionOwner owner, |
| Iterable<Artifact> tools, |
| Iterable<Artifact> inputs, |
| RunfilesSupplier runfilesSupplier, |
| Iterable<Artifact> outputs, |
| ActionEnvironment env) { |
| Preconditions.checkNotNull(owner); |
| this.owner = owner; |
| this.tools = CollectionUtils.makeImmutable(tools); |
| this.inputs = CollectionUtils.makeImmutable(inputs); |
| this.env = Preconditions.checkNotNull(env); |
| this.outputs = ImmutableSet.copyOf(outputs); |
| this.runfilesSupplier = Preconditions.checkNotNull(runfilesSupplier); |
| Preconditions.checkArgument(!this.outputs.isEmpty(), "action outputs may not be empty"); |
| } |
| |
| @Override |
| public final ActionOwner getOwner() { |
| return owner; |
| } |
| |
| @Override |
| public final synchronized boolean inputsDiscovered() { |
| return discoversInputs() ? inputsDiscovered : true; |
| } |
| |
| /** |
| * Should be overridden by actions that do input discovery. |
| * |
| * <p>The value returned by each instance should be constant over the lifetime of that instance. |
| * |
| * <p>If this returns true, {@link #discoverInputs(ActionExecutionContext)} must also be |
| * implemented. |
| */ |
| @Override |
| public boolean discoversInputs() { |
| return false; |
| } |
| |
| /** |
| * Run input discovery on the action. |
| * |
| * <p>Called by Blaze if {@link #discoversInputs()} returns true. It must return the set of |
| * input artifacts that were not known at analysis time. May also call |
| * {@link #updateInputs(Iterable<Artifact>)}; if it doesn't, the action itself must arrange for |
| * the newly discovered artifacts to be available during action execution, probably by keeping |
| * state in the action instance and using a custom action execution context and for |
| * {@code #updateInputs()} to be called during the execution of the action. |
| * |
| * <p>Since keeping state within an action bad, don't do that unless there is a very good reason |
| * to do so. |
| */ |
| @Override |
| public Iterable<Artifact> discoverInputs(ActionExecutionContext actionExecutionContext) |
| throws ActionExecutionException, InterruptedException { |
| throw new IllegalStateException("discoverInputs cannot be called for " + this.prettyPrint() |
| + " since it does not discover inputs"); |
| } |
| |
| @Override |
| public Iterable<Artifact> getAllowedDerivedInputs() { |
| throw new IllegalStateException( |
| "Method must be overridden for actions that may have unknown inputs."); |
| } |
| |
| /** |
| * Should be called when the inputs of the action become known, that is, either during {@link |
| * #discoverInputs(ActionExecutionContext)} or during {@link #execute(ActionExecutionContext)}. |
| * |
| * <p>When an action discovers inputs, it must have been called by the time {@code #execute()} |
| * returns. It can be called both during {@code discoverInputs} and during {@code execute()}. |
| * |
| * <p>In addition to being called from action implementations, it will also be called by Bazel |
| * itself when an action is loaded from the on-disk action cache. |
| */ |
| @Override |
| public synchronized void updateInputs(Iterable<Artifact> inputs) { |
| Preconditions.checkState( |
| discoversInputs(), "Can't update inputs unless discovering: %s %s", this, inputs); |
| this.inputs = CollectionUtils.makeImmutable(inputs); |
| inputsDiscovered = true; |
| } |
| |
| @Override |
| public Iterable<Artifact> getTools() { |
| return tools; |
| } |
| |
| /** |
| * Should not be overridden (it's non-final only for tests) |
| */ |
| @Override |
| public synchronized Iterable<Artifact> getInputs() { |
| return inputs; |
| } |
| |
| public final ActionEnvironment getEnvironment() { |
| return env; |
| } |
| |
| @Override |
| public Iterable<String> getClientEnvironmentVariables() { |
| return env.getInheritedEnv(); |
| } |
| |
| @Override |
| public RunfilesSupplier getRunfilesSupplier() { |
| return runfilesSupplier; |
| } |
| |
| @Override |
| public ImmutableSet<Artifact> getOutputs() { |
| return outputs; |
| } |
| |
| @Override |
| public Artifact getPrimaryInput() { |
| // The default behavior is to return the first input artifact. |
| // Call through the method, not the field, because it may be overridden. |
| return Iterables.getFirst(getInputs(), null); |
| } |
| |
| @Override |
| public Artifact getPrimaryOutput() { |
| // Default behavior is to return the first output artifact. |
| // Use the method rather than field in case of overriding in subclasses. |
| return Iterables.getFirst(getOutputs(), null); |
| } |
| |
| @Override |
| public Iterable<Artifact> getMandatoryInputs() { |
| return getInputs(); |
| } |
| |
| @Override |
| public String toString() { |
| return prettyPrint() + " (" + getMnemonic() + "[" + ImmutableList.copyOf(getInputs()) |
| + (inputsDiscovered() ? " -> " : ", unknown inputs -> ") |
| + getOutputs() + "]" + ")"; |
| } |
| |
| @Override |
| public abstract String getMnemonic(); |
| |
| /** |
| * See the javadoc for {@link com.google.devtools.build.lib.actions.Action} and {@link |
| * ActionExecutionMetadata#getKey(ActionKeyContext)} for the contract for {@link |
| * #computeKey(ActionKeyContext, Fingerprint)}. |
| */ |
| protected abstract void computeKey(ActionKeyContext actionKeyContext, Fingerprint fp) |
| throws CommandLineExpansionException; |
| |
| @Override |
| public final synchronized String getKey(ActionKeyContext actionKeyContext) { |
| if (cachedKey == null) { |
| try { |
| Fingerprint fp = new Fingerprint(); |
| computeKey(actionKeyContext, fp); |
| |
| // Add a bool indicating whether the execution platform was set. |
| fp.addBoolean(getExecutionPlatform() != null); |
| if (getExecutionPlatform() != null) { |
| // Add the execution platform information. |
| getExecutionPlatform().addTo(fp); |
| } |
| |
| // Compute the actual key and store it. |
| cachedKey = fp.hexDigestAndReset(); |
| } catch (CommandLineExpansionException e) { |
| cachedKey = KEY_ERROR; |
| } |
| } |
| return cachedKey; |
| } |
| |
| @Override |
| public String describeKey() { |
| return null; |
| } |
| |
| @Override |
| public boolean executeUnconditionally() { |
| return false; |
| } |
| |
| @Override |
| public boolean isVolatile() { |
| return false; |
| } |
| |
| @Override |
| public boolean isShareable() { |
| return true; |
| } |
| |
| @Override |
| public boolean showsOutputUnconditionally() { |
| return false; |
| } |
| |
| @Override |
| public final String getProgressMessage() { |
| String message = getRawProgressMessage(); |
| if (message == null) { |
| return null; |
| } |
| String additionalInfo = getOwner().getAdditionalProgressInfo(); |
| return additionalInfo == null ? message : message + " [" + additionalInfo + "]"; |
| } |
| |
| /** |
| * Returns a progress message string that is specific for this action. This is |
| * then annotated with additional information, currently the string '[for host]' |
| * for actions in the host configurations. |
| * |
| * <p>A return value of null indicates no message should be reported. |
| */ |
| protected String getRawProgressMessage() { |
| // A cheesy default implementation. Subclasses are invited to do something |
| // more meaningful. |
| return defaultProgressMessage(); |
| } |
| |
| private String defaultProgressMessage() { |
| return getMnemonic() + " " + getPrimaryOutput().prettyPrint(); |
| } |
| |
| @Override |
| public String prettyPrint() { |
| return "action '" + describe() + "'"; |
| } |
| |
| @Override |
| public void repr(SkylarkPrinter printer) { |
| printer.append(prettyPrint()); // TODO(bazel-team): implement a readable representation |
| } |
| |
| /** |
| * Deletes all of the action's output files, if they exist. If any of the Artifacts refers to a |
| * directory recursively removes the contents of the directory. |
| * |
| * @param execRoot the exec root in which this action is executed |
| */ |
| protected void deleteOutputs(FileSystem fileSystem, Path execRoot) throws IOException { |
| for (Artifact output : getOutputs()) { |
| deleteOutput(fileSystem, output); |
| } |
| } |
| |
| /** |
| * Helper method to remove an Artifact. If the Artifact refers to a directory recursively removes |
| * the contents of the directory. |
| */ |
| protected void deleteOutput(FileSystem fileSystem, Artifact output) throws IOException { |
| Path path = output.getPath(); |
| try { |
| // Optimize for the common case: output artifacts are files. |
| path.delete(); |
| } catch (IOException e) { |
| // Handle a couple of scenarios where the output can still be deleted, but make sure we're not |
| // deleting random files on the filesystem. |
| if (output.getRoot() == null) { |
| throw e; |
| } |
| Root outputRoot = output.getRoot().getRoot(); |
| if (!outputRoot.contains(path)) { |
| throw e; |
| } |
| |
| Path parentDir = path.getParentDirectory(); |
| if (!parentDir.isWritable() && outputRoot.contains(parentDir)) { |
| // Retry deleting after making the parent writable. |
| parentDir.setWritable(true); |
| deleteOutput(fileSystem, output); |
| } else if (path.isDirectory(Symlinks.NOFOLLOW)) { |
| FileSystemUtils.deleteTree(path); |
| } else { |
| throw e; |
| } |
| } |
| } |
| |
| /** |
| * If the action might read directories as inputs in a way that is unsound wrt dependency |
| * checking, this method must be called. |
| */ |
| protected void checkInputsForDirectories( |
| EventHandler eventHandler, MetadataProvider metadataProvider) throws IOException { |
| // Report "directory dependency checking" warning only for non-generated directories (generated |
| // ones will be reported earlier). |
| for (Artifact input : getMandatoryInputs()) { |
| // Assume that if the file did not exist, we would not have gotten here. |
| if (input.isSourceArtifact() && metadataProvider.getMetadata(input).getType().isDirectory()) { |
| // TODO(ulfjack): What about dependency checking of special files? |
| eventHandler.handle( |
| Event.warn( |
| getOwner().getLocation(), |
| String.format( |
| "input '%s' to %s is a directory; " |
| + "dependency checking of directories is unsound", |
| input.prettyPrint(), getOwner().getLabel()))); |
| } |
| } |
| } |
| |
| @Override |
| public MiddlemanType getActionType() { |
| return MiddlemanType.NORMAL; |
| } |
| |
| /** If the action might create directories as outputs this method must be called. */ |
| protected void checkOutputsForDirectories(ActionExecutionContext actionExecutionContext) { |
| for (Artifact output : getOutputs()) { |
| Path path = actionExecutionContext.getInputPath(output); |
| String ownerString = Label.print(getOwner().getLabel()); |
| if (path.isDirectory()) { |
| actionExecutionContext |
| .getEventHandler() |
| .handle( |
| Event.warn( |
| getOwner().getLocation(), |
| "output '" |
| + output.prettyPrint() |
| + "' of " |
| + ownerString |
| + " is a directory; dependency checking of directories is unsound") |
| .withTag(ownerString)); |
| } |
| } |
| } |
| |
| @Override |
| public void prepare(FileSystem fileSystem, Path execRoot) throws IOException { |
| deleteOutputs(fileSystem, execRoot); |
| } |
| |
| @Override |
| public String describe() { |
| String progressMessage = getProgressMessage(); |
| return progressMessage != null ? progressMessage : defaultProgressMessage(); |
| } |
| |
| @Override |
| public boolean shouldReportPathPrefixConflict(ActionAnalysisMetadata action) { |
| return this != action; |
| } |
| |
| @Override |
| public ExtraActionInfo.Builder getExtraActionInfo(ActionKeyContext actionKeyContext) |
| throws CommandLineExpansionException { |
| ActionOwner owner = getOwner(); |
| ExtraActionInfo.Builder result = |
| ExtraActionInfo.newBuilder() |
| .setOwner(owner.getLabel().toString()) |
| .setId(getKey(actionKeyContext)) |
| .setMnemonic(getMnemonic()); |
| Iterable<AspectDescriptor> aspectDescriptors = owner.getAspectDescriptors(); |
| AspectDescriptor lastAspect = null; |
| |
| for (AspectDescriptor aspectDescriptor : aspectDescriptors) { |
| ExtraActionInfo.AspectDescriptor.Builder builder = |
| ExtraActionInfo.AspectDescriptor.newBuilder() |
| .setAspectName(aspectDescriptor.getAspectClass().getName()); |
| for (Map.Entry<String, Collection<String>> entry : |
| aspectDescriptor.getParameters().getAttributes().asMap().entrySet()) { |
| builder.putAspectParameters( |
| entry.getKey(), |
| ExtraActionInfo.AspectDescriptor.StringList.newBuilder() |
| .addAllValue(entry.getValue()) |
| .build() |
| ); |
| } |
| lastAspect = aspectDescriptor; |
| } |
| if (lastAspect != null) { |
| result.setAspectName(lastAspect.getAspectClass().getName()); |
| |
| for (Map.Entry<String, Collection<String>> entry : |
| lastAspect.getParameters().getAttributes().asMap().entrySet()) { |
| result.putAspectParameters( |
| entry.getKey(), |
| ExtraActionInfo.StringList.newBuilder().addAllValue(entry.getValue()).build()); |
| } |
| } |
| return result; |
| } |
| |
| @Override |
| public ImmutableSet<Artifact> getMandatoryOutputs() { |
| return ImmutableSet.of(); |
| } |
| |
| /** |
| * Returns input files that need to be present to allow extra_action rules to shadow this action |
| * correctly when run remotely. This is at least the normal inputs of the action, but may include |
| * other files as well. For example C(++) compilation may perform include file header scanning. |
| * This needs to be mirrored by the extra_action rule. Called by |
| * {@link com.google.devtools.build.lib.analysis.extra.ExtraAction} at execution time for actions |
| * that return true for {link #discoversInputs()}. |
| * |
| * @param actionExecutionContext Services in the scope of the action, like the Out/Err streams. |
| * @throws ActionExecutionException only when code called from this method |
| * throws that exception. |
| * @throws InterruptedException if interrupted |
| */ |
| @Override |
| public Iterable<Artifact> getInputFilesForExtraAction( |
| ActionExecutionContext actionExecutionContext) |
| throws ActionExecutionException, InterruptedException { |
| return ImmutableList.of(); |
| } |
| |
| @Override |
| public SkylarkNestedSet getSkylarkInputs() { |
| return SkylarkNestedSet.of(Artifact.class, NestedSetBuilder.wrap( |
| Order.STABLE_ORDER, getInputs())); |
| } |
| |
| @Override |
| public SkylarkNestedSet getSkylarkOutputs() { |
| return SkylarkNestedSet.of(Artifact.class, NestedSetBuilder.wrap( |
| Order.STABLE_ORDER, getOutputs())); |
| } |
| |
| @Override |
| public SkylarkList<String> getSkylarkArgv() throws EvalException { |
| return null; |
| } |
| |
| @Override |
| public String getSkylarkContent() throws IOException { |
| return null; |
| } |
| |
| @Override |
| public SkylarkDict<String, String> getSkylarkSubstitutions() { |
| return null; |
| } |
| |
| @Override |
| public SkylarkDict<String, String> getEnv() { |
| return SkylarkDict.copyOf(null, env.getFixedEnv().toMap()); |
| } |
| |
| @Override |
| @Nullable |
| public PlatformInfo getExecutionPlatform() { |
| return getOwner().getExecutionPlatform(); |
| } |
| } |