| // Copyright 2014 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| package com.google.devtools.build.lib.skyframe; |
| |
| import com.google.common.base.Function; |
| import com.google.common.base.MoreObjects; |
| import com.google.common.base.Preconditions; |
| import com.google.common.base.Predicate; |
| import com.google.common.collect.ImmutableList; |
| import com.google.common.collect.ImmutableMap; |
| import com.google.common.collect.Iterables; |
| import com.google.devtools.build.lib.actions.Action; |
| import com.google.devtools.build.lib.actions.ActionAnalysisMetadata; |
| import com.google.devtools.build.lib.actions.ActionAnalysisMetadata.MiddlemanType; |
| import com.google.devtools.build.lib.actions.ActionLookupData; |
| import com.google.devtools.build.lib.actions.ActionLookupValue; |
| import com.google.devtools.build.lib.actions.ActionLookupValue.ActionLookupKey; |
| import com.google.devtools.build.lib.actions.Artifact; |
| import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact; |
| import com.google.devtools.build.lib.actions.ArtifactFileMetadata; |
| import com.google.devtools.build.lib.actions.ArtifactOwner; |
| import com.google.devtools.build.lib.actions.ArtifactSkyKey; |
| import com.google.devtools.build.lib.actions.FileArtifactValue; |
| import com.google.devtools.build.lib.actions.FileValue; |
| import com.google.devtools.build.lib.actions.FilesetTraversalParams.DirectTraversalRoot; |
| import com.google.devtools.build.lib.actions.FilesetTraversalParams.PackageBoundaryMode; |
| import com.google.devtools.build.lib.actions.MissingInputFileException; |
| import com.google.devtools.build.lib.cmdline.Label; |
| import com.google.devtools.build.lib.events.Event; |
| import com.google.devtools.build.lib.events.EventHandler; |
| import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalFunction.RecursiveFilesystemTraversalException; |
| import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFile; |
| import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.TraversalRequest; |
| import com.google.devtools.build.lib.util.Fingerprint; |
| import com.google.devtools.build.lib.util.Pair; |
| import com.google.devtools.build.lib.vfs.RootedPath; |
| import com.google.devtools.build.skyframe.SkyFunction; |
| import com.google.devtools.build.skyframe.SkyFunctionException; |
| import com.google.devtools.build.skyframe.SkyFunctionException.Transience; |
| import com.google.devtools.build.skyframe.SkyKey; |
| import com.google.devtools.build.skyframe.SkyValue; |
| import java.io.IOException; |
| import java.util.Comparator; |
| import java.util.Map; |
| import java.util.function.Supplier; |
| import javax.annotation.Nullable; |
| |
| /** A builder of values for {@link ArtifactSkyKey} keys. */ |
| class ArtifactFunction implements SkyFunction { |
| |
| private final Supplier<Boolean> mkdirForTreeArtifacts; |
| |
| public ArtifactFunction(Supplier<Boolean> mkdirForTreeArtifacts) { |
| this.mkdirForTreeArtifacts = mkdirForTreeArtifacts; |
| } |
| |
| @Override |
| public SkyValue compute(SkyKey skyKey, Environment env) |
| throws ArtifactFunctionException, InterruptedException { |
| Artifact artifact = ArtifactSkyKey.artifact(skyKey); |
| if (artifact.isSourceArtifact()) { |
| try { |
| return createSourceValue(artifact, ArtifactSkyKey.isMandatory(skyKey), env); |
| } catch (MissingInputFileException e) { |
| // The error is not necessarily truly transient, but we mark it as such because we have |
| // the above side effect of posting an event to the EventBus. Importantly, that event |
| // is potentially used to report root causes. |
| throw new ArtifactFunctionException(e, Transience.TRANSIENT); |
| } catch (IOException e) { |
| throw new ArtifactFunctionException(e, Transience.TRANSIENT); |
| } |
| } |
| |
| ArtifactDependencies artifactDependencies = |
| ArtifactDependencies.discoverDependencies((Artifact.DerivedArtifact) artifact, env); |
| if (artifactDependencies == null) { |
| return null; |
| } |
| |
| // If the action is an ActionTemplate, we need to expand the ActionTemplate into concrete |
| // actions, execute those actions in parallel and then aggregate the action execution results. |
| if (artifactDependencies.isTemplateActionForTreeArtifact()) { |
| if (mkdirForTreeArtifacts.get()) { |
| mkdirForTreeArtifact(artifact, env); |
| } |
| return createTreeArtifactValueFromActionKey(artifactDependencies, env); |
| } |
| |
| ActionExecutionValue actionValue = |
| (ActionExecutionValue) |
| env.getValue(artifactDependencies.getNontemplateActionExecutionKey()); |
| if (actionValue == null) { |
| return null; |
| } |
| |
| if (artifact.isTreeArtifact()) { |
| // We get a request for the whole tree artifact. We can just return the associated |
| // TreeArtifactValue. |
| return Preconditions.checkNotNull(actionValue.getTreeArtifactValue(artifact), artifact); |
| } |
| |
| if (artifact.isMiddlemanArtifact()) { |
| Action action = |
| Preconditions.checkNotNull( |
| artifactDependencies.getAction(), "Null middleman action? %s", artifactDependencies); |
| if (isAggregatingValue(action)) { |
| return createAggregatingValue( |
| artifact, action, actionValue.getArtifactValue(artifact), env); |
| } |
| } |
| |
| return createSimpleFileArtifactValue(artifact, actionValue); |
| } |
| |
| private static void mkdirForTreeArtifact(Artifact artifact, Environment env) |
| throws ArtifactFunctionException { |
| try { |
| artifact.getPath().createDirectoryAndParents(); |
| } catch (IOException e) { |
| env.getListener() |
| .handle( |
| Event.error( |
| String.format( |
| "Failed to create output directory for TreeArtifact %s: %s", |
| artifact, e.getMessage()))); |
| throw new ArtifactFunctionException(e, Transience.TRANSIENT); |
| } |
| } |
| |
| private static TreeArtifactValue createTreeArtifactValueFromActionKey( |
| ArtifactDependencies artifactDependencies, Environment env) throws InterruptedException { |
| ActionLookupKey actionLookupKey = artifactDependencies.getActionLookupKey(); |
| int actionIndex = artifactDependencies.getActionIndex(); |
| Artifact treeArtifact = artifactDependencies.getArtifact(); |
| |
| // Request the list of expanded actions from the ActionTemplate. |
| ActionTemplateExpansion actionTemplateExpansion = |
| artifactDependencies.getActionTemplateExpansion(env); |
| if (actionTemplateExpansion == null) { |
| // The expanded actions are not yet available. |
| return null; |
| } |
| ActionTemplateExpansionValue expansionValue = actionTemplateExpansion.getValue(); |
| ImmutableList<ActionLookupData> expandedActionExecutionKeys = |
| actionTemplateExpansion.getExpandedActionExecutionKeys(); |
| |
| Map<SkyKey, SkyValue> expandedActionValueMap = env.getValues(expandedActionExecutionKeys); |
| if (env.valuesMissing()) { |
| // The execution values of the expanded actions are not yet all available. |
| return null; |
| } |
| |
| // Aggregate the ArtifactValues for individual TreeFileArtifacts into a TreeArtifactValue for |
| // the parent TreeArtifact. |
| ImmutableMap.Builder<TreeFileArtifact, FileArtifactValue> map = ImmutableMap.builder(); |
| for (int i = 0; i < expandedActionExecutionKeys.size(); i++) { |
| final ActionExecutionValue actionExecutionValue = |
| (ActionExecutionValue) |
| Preconditions.checkNotNull( |
| expandedActionValueMap.get(expandedActionExecutionKeys.get(i)), |
| "Missing tree value: %s %s %s %s %s", |
| treeArtifact, |
| actionLookupKey, |
| actionIndex, |
| expansionValue, |
| expandedActionValueMap); |
| Iterable<TreeFileArtifact> treeFileArtifacts = |
| Iterables.transform( |
| Iterables.filter( |
| actionExecutionValue.getAllFileValues().keySet(), |
| new Predicate<Artifact>() { |
| @Override |
| public boolean apply(Artifact artifact) { |
| Preconditions.checkState( |
| artifact.hasParent(), |
| "No parent: %s %s %s %s %s", |
| artifact, |
| treeArtifact, |
| actionExecutionValue, |
| actionLookupKey, |
| actionIndex); |
| return artifact.getParent().equals(treeArtifact); |
| } |
| }), |
| new Function<Artifact, TreeFileArtifact>() { |
| @Override |
| public TreeFileArtifact apply(Artifact artifact) { |
| return (TreeFileArtifact) artifact; |
| } |
| }); |
| |
| Preconditions.checkState( |
| !Iterables.isEmpty(treeFileArtifacts), |
| "Action denoted by %s does not output TreeFileArtifact under %s", |
| expandedActionExecutionKeys.get(i), |
| treeArtifact); |
| |
| for (TreeFileArtifact treeFileArtifact : treeFileArtifacts) { |
| FileArtifactValue value = |
| createSimpleFileArtifactValue(treeFileArtifact, actionExecutionValue); |
| map.put(treeFileArtifact, value); |
| } |
| } |
| |
| // Return the aggregated TreeArtifactValue. |
| return TreeArtifactValue.create(map.build()); |
| } |
| |
| private FileArtifactValue createSourceValue(Artifact artifact, boolean mandatory, Environment env) |
| throws MissingInputFileException, IOException, InterruptedException { |
| RootedPath path = RootedPath.toRootedPath(artifact.getRoot().getRoot(), artifact.getPath()); |
| SkyKey fileSkyKey = FileValue.key(path); |
| FileValue fileValue; |
| try { |
| fileValue = (FileValue) env.getValueOrThrow(fileSkyKey, IOException.class); |
| } catch (IOException e) { |
| throw makeMissingInputFileException(artifact, mandatory, e, env.getListener()); |
| } |
| if (fileValue == null) { |
| return null; |
| } |
| if (!fileValue.exists()) { |
| if (!mandatory) { |
| return FileArtifactValue.MISSING_FILE_MARKER; |
| } else { |
| throw makeMissingInputFileException(artifact, mandatory, null, env.getListener()); |
| } |
| } |
| // For directory artifacts that are not Filesets, we initiate a directory traversal here, and |
| // compute a hash from the directory structure. |
| if (fileValue.isDirectory() && TrackSourceDirectoriesFlag.trackSourceDirectories()) { |
| // We rely on the guarantees of RecursiveFilesystemTraversalFunction for correctness. |
| // |
| // This approach may have unexpected interactions with --package_path. In particular, the exec |
| // root is setup from the loading / analysis phase, and it is now too late to change it; |
| // therefore, this may traverse a different set of files depending on which targets are built |
| // at the same time and what the package-path layout is (this may be moot if there is only one |
| // entry). Or this may return a set of files that's inconsistent with those actually available |
| // to the action (for local execution). |
| // |
| // In the future, we need to make this result the source of truth for the files available to |
| // the action so that we at least have consistency. |
| TraversalRequest request = TraversalRequest.create( |
| DirectTraversalRoot.forRootedPath(path), |
| /*isRootGenerated=*/ false, |
| PackageBoundaryMode.CROSS, |
| /*strictOutputFiles=*/ true, |
| /*skipTestingForSubpackage=*/ true, |
| /*errorInfo=*/ null); |
| RecursiveFilesystemTraversalValue value; |
| try { |
| value = |
| (RecursiveFilesystemTraversalValue) env.getValueOrThrow( |
| request, RecursiveFilesystemTraversalException.class); |
| } catch (RecursiveFilesystemTraversalException e) { |
| throw new IOException(e); |
| } |
| if (value == null) { |
| return null; |
| } |
| Fingerprint fp = new Fingerprint(); |
| for (ResolvedFile file : value.getTransitiveFiles()) { |
| fp.addString(file.getNameInSymlinkTree().getPathString()); |
| fp.addInt(file.getMetadata().hashCode()); |
| } |
| return FileArtifactValue.createDirectoryWithHash(fp.digestAndReset()); |
| } |
| try { |
| return FileArtifactValue.create(artifact, fileValue); |
| } catch (IOException e) { |
| throw makeMissingInputFileException(artifact, mandatory, e, env.getListener()); |
| } |
| } |
| |
| private static MissingInputFileException makeMissingInputFileException( |
| Artifact artifact, boolean mandatory, Exception failure, EventHandler reporter) { |
| String extraMsg = (failure == null) ? "" : (":" + failure.getMessage()); |
| MissingInputFileException ex = |
| new MissingInputFileException(constructErrorMessage(artifact) + extraMsg, null); |
| if (mandatory) { |
| reporter.handle(Event.error(ex.getLocation(), ex.getMessage())); |
| } |
| return ex; |
| } |
| |
| // Non-aggregating artifact -- should contain at most one piece of artifact data. |
| // data may be null if and only if artifact is a middleman artifact. |
| private static FileArtifactValue createSimpleFileArtifactValue( |
| Artifact artifact, ActionExecutionValue actionValue) { |
| FileArtifactValue value = actionValue.getArtifactValue(artifact); |
| if (value != null) { |
| return value; |
| } |
| // Middleman artifacts have no corresponding files, so their ArtifactValues should have already |
| // been constructed during execution of the action. |
| Preconditions.checkState(!artifact.isMiddlemanArtifact(), artifact); |
| ArtifactFileMetadata data = |
| Preconditions.checkNotNull(actionValue.getData(artifact), "%s %s", artifact, actionValue); |
| Preconditions.checkNotNull( |
| data.getDigest(), "Digest should already have been calculated for %s (%s)", artifact, data); |
| // Directories are special-cased because their mtimes are used, so should have been constructed |
| // during execution of the action (in ActionMetadataHandler#maybeStoreAdditionalData). |
| Preconditions.checkState(data.isFile(), "Unexpected not file %s (%s)", artifact, data); |
| return FileArtifactValue.createNormalFile(data, !artifact.isConstantMetadata()); |
| } |
| |
| @Nullable |
| private static AggregatingArtifactValue createAggregatingValue( |
| Artifact artifact, |
| ActionAnalysisMetadata action, |
| FileArtifactValue value, |
| SkyFunction.Environment env) |
| throws InterruptedException { |
| ImmutableList.Builder<Pair<Artifact, FileArtifactValue>> fileInputsBuilder = |
| ImmutableList.builder(); |
| ImmutableList.Builder<Pair<Artifact, TreeArtifactValue>> directoryInputsBuilder = |
| ImmutableList.builder(); |
| for (Map.Entry<SkyKey, SkyValue> entry : env.getValues(action.getInputs()).entrySet()) { |
| Artifact input = ArtifactSkyKey.artifact(entry.getKey()); |
| SkyValue inputValue = entry.getValue(); |
| if (inputValue == null) { |
| return null; |
| } |
| if (inputValue instanceof FileArtifactValue) { |
| fileInputsBuilder.add(Pair.of(input, (FileArtifactValue) inputValue)); |
| } else if (inputValue instanceof TreeArtifactValue) { |
| directoryInputsBuilder.add(Pair.of(input, (TreeArtifactValue) inputValue)); |
| } else { |
| // We do not recurse in aggregating middleman artifacts. |
| Preconditions.checkState( |
| !(inputValue instanceof AggregatingArtifactValue), |
| "%s %s %s", |
| artifact, |
| action, |
| inputValue); |
| } |
| } |
| |
| ImmutableList<Pair<Artifact, FileArtifactValue>> fileInputs = |
| ImmutableList.sortedCopyOf( |
| Comparator.comparing(pair -> pair.getFirst().getExecPathString()), |
| fileInputsBuilder.build()); |
| ImmutableList<Pair<Artifact, TreeArtifactValue>> directoryInputs = |
| ImmutableList.sortedCopyOf( |
| Comparator.comparing(pair -> pair.getFirst().getExecPathString()), |
| directoryInputsBuilder.build()); |
| |
| return (action.getActionType() == MiddlemanType.AGGREGATING_MIDDLEMAN) |
| ? new AggregatingArtifactValue(fileInputs, directoryInputs, value) |
| : new RunfilesArtifactValue(fileInputs, directoryInputs, value); |
| } |
| |
| /** |
| * Returns whether this value needs to contain the data of all its inputs. Currently only tests to |
| * see if the action is an aggregating or runfiles middleman action. However, may include Fileset |
| * artifacts in the future. |
| */ |
| private static boolean isAggregatingValue(ActionAnalysisMetadata action) { |
| switch (action.getActionType()) { |
| case AGGREGATING_MIDDLEMAN: |
| case RUNFILES_MIDDLEMAN: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| @Override |
| public String extractTag(SkyKey skyKey) { |
| return Label.print(ArtifactSkyKey.artifact(skyKey).getOwner()); |
| } |
| |
| static ActionLookupKey getActionLookupKey(Artifact artifact) { |
| ArtifactOwner artifactOwner = artifact.getArtifactOwner(); |
| Preconditions.checkState( |
| artifactOwner instanceof ActionLookupKey, "%s %s", artifact, artifactOwner); |
| return (ActionLookupKey) artifactOwner; |
| } |
| |
| @Nullable |
| static ActionLookupValue getActionLookupValue( |
| SkyKey actionLookupKey, SkyFunction.Environment env, Artifact artifact) |
| throws InterruptedException { |
| ActionLookupValue value = (ActionLookupValue) env.getValue(actionLookupKey); |
| if (value == null) { |
| ArtifactOwner artifactOwner = artifact.getArtifactOwner(); |
| Preconditions.checkState( |
| artifactOwner == CoverageReportValue.COVERAGE_REPORT_KEY, |
| "Not-yet-present artifact owner: %s (%s %s)", |
| artifactOwner, |
| artifact, |
| actionLookupKey); |
| return null; |
| } |
| return value; |
| } |
| |
| static final class ArtifactFunctionException extends SkyFunctionException { |
| ArtifactFunctionException(MissingInputFileException e, Transience transience) { |
| super(e, transience); |
| } |
| |
| ArtifactFunctionException(IOException e, Transience transience) { |
| super(e, transience); |
| } |
| } |
| |
| private static String constructErrorMessage(Artifact artifact) { |
| if (artifact.getOwner() == null) { |
| return String.format("missing input file '%s'", artifact.getPath().getPathString()); |
| } else { |
| return String.format("missing input file '%s'", artifact.getOwner()); |
| } |
| } |
| |
| /** Describes dependencies of derived artifacts. */ |
| // TODO(b/19539699): extend this to comprehensively support all special artifact types (e.g. |
| // middleman, etc). |
| static class ArtifactDependencies { |
| |
| private final Artifact artifact; |
| private final ActionLookupKey actionLookupKey; |
| private final ActionLookupValue actionLookupValue; |
| private final int actionIndex; |
| |
| private ArtifactDependencies( |
| Artifact artifact, |
| ActionLookupKey actionLookupKey, |
| ActionLookupValue actionLookupValue, |
| int actionIndex) { |
| this.artifact = artifact; |
| this.actionLookupKey = actionLookupKey; |
| this.actionLookupValue = actionLookupValue; |
| this.actionIndex = actionIndex; |
| } |
| |
| /** |
| * Constructs an {@link ArtifactDependencies} for the provided {@code derivedArtifact}. Returns |
| * {@code null} if any dependencies are not yet ready. |
| */ |
| @Nullable |
| static ArtifactDependencies discoverDependencies( |
| Artifact.DerivedArtifact derivedArtifact, SkyFunction.Environment env) |
| throws InterruptedException { |
| |
| ActionLookupKey actionLookupKey = ArtifactFunction.getActionLookupKey(derivedArtifact); |
| ActionLookupValue actionLookupValue = |
| ArtifactFunction.getActionLookupValue(actionLookupKey, env, derivedArtifact); |
| if (actionLookupValue == null) { |
| return null; |
| } |
| Integer actionIndex = actionLookupValue.getGeneratingActionIndex(derivedArtifact); |
| if (derivedArtifact.hasParent() && actionIndex == null) { |
| // If a TreeFileArtifact is created by a templated action, then it should have the proper |
| // reference to its owner. However, if it was created as part of a directory, by the first |
| // TreeArtifact-generating action in a chain, then its parent's generating action also |
| // generated it. This catches that case. |
| actionIndex = actionLookupValue.getGeneratingActionIndex(derivedArtifact.getParent()); |
| } |
| Preconditions.checkNotNull( |
| actionIndex, "%s %s %s", derivedArtifact, actionLookupKey, actionLookupValue); |
| |
| return new ArtifactDependencies( |
| derivedArtifact, actionLookupKey, actionLookupValue, actionIndex); |
| } |
| |
| Artifact getArtifact() { |
| return artifact; |
| } |
| |
| ActionLookupKey getActionLookupKey() { |
| return actionLookupKey; |
| } |
| |
| int getActionIndex() { |
| return actionIndex; |
| } |
| |
| boolean isTemplateActionForTreeArtifact() { |
| return artifact.isTreeArtifact() && actionLookupValue.isActionTemplate(actionIndex); |
| } |
| |
| ActionLookupData getNontemplateActionExecutionKey() { |
| Preconditions.checkState( |
| !isTemplateActionForTreeArtifact(), "Action is unexpectedly template: %s", this); |
| return ActionExecutionValue.key(actionLookupKey, actionIndex); |
| } |
| |
| /** |
| * Returns action template expansion information or {@code null} if that information is |
| * unavailable. |
| * |
| * <p>Must not be called if {@code !isTemplateActionForTreeArtifact()}. |
| */ |
| @Nullable |
| ActionTemplateExpansion getActionTemplateExpansion(SkyFunction.Environment env) |
| throws InterruptedException { |
| Preconditions.checkState( |
| isTemplateActionForTreeArtifact(), "Action is unexpectedly non-template: %s", this); |
| ActionTemplateExpansionValue.ActionTemplateExpansionKey key = |
| ActionTemplateExpansionValue.key(actionLookupKey, actionIndex); |
| ActionTemplateExpansionValue value = (ActionTemplateExpansionValue) env.getValue(key); |
| if (value == null) { |
| return null; |
| } |
| return new ActionTemplateExpansion(key, value); |
| } |
| |
| Action getAction() { |
| return actionLookupValue.getAction(actionIndex); |
| } |
| |
| @Override |
| public String toString() { |
| return MoreObjects.toStringHelper(this) |
| .add("artifact", artifact) |
| .add("actionLookupKey", actionLookupKey) |
| .add("actionLookupValue", actionLookupValue) |
| .add("actionIndex", actionIndex) |
| .toString(); |
| } |
| } |
| |
| static class ActionTemplateExpansion { |
| private final ActionTemplateExpansionValue.ActionTemplateExpansionKey key; |
| private final ActionTemplateExpansionValue value; |
| |
| private ActionTemplateExpansion( |
| ActionTemplateExpansionValue.ActionTemplateExpansionKey key, |
| ActionTemplateExpansionValue value) { |
| this.key = key; |
| this.value = value; |
| } |
| |
| ActionTemplateExpansionValue.ActionTemplateExpansionKey getKey() { |
| return key; |
| } |
| |
| ActionTemplateExpansionValue getValue() { |
| return value; |
| } |
| |
| ImmutableList<ActionLookupData> getExpandedActionExecutionKeys() { |
| int numActions = value.getNumActions(); |
| ImmutableList.Builder<ActionLookupData> expandedActionExecutionKeys = |
| ImmutableList.builderWithExpectedSize(numActions); |
| for (int i = 0; i < numActions; i++) { |
| expandedActionExecutionKeys.add(ActionExecutionValue.key(key, i)); |
| } |
| return expandedActionExecutionKeys.build(); |
| } |
| } |
| } |
