blob: 4ba628b8ad5ad85f85af0133f19f373be0dae644 [file] [log] [blame]
// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.devtools.build.lib.actions.Action;
import com.google.devtools.build.lib.actions.ActionCacheChecker.Token;
import com.google.devtools.build.lib.actions.ActionCompletionEvent;
import com.google.devtools.build.lib.actions.ActionExecutedEvent;
import com.google.devtools.build.lib.actions.ActionExecutedEvent.ErrorTiming;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionExecutionException;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionInputDepOwnerMap;
import com.google.devtools.build.lib.actions.ActionInputDepOwners;
import com.google.devtools.build.lib.actions.ActionInputMap;
import com.google.devtools.build.lib.actions.ActionInputMapSink;
import com.google.devtools.build.lib.actions.ActionLookupData;
import com.google.devtools.build.lib.actions.ActionLookupValue;
import com.google.devtools.build.lib.actions.ActionRewoundEvent;
import com.google.devtools.build.lib.actions.AlreadyReportedActionExecutionException;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.ArtifactPathResolver;
import com.google.devtools.build.lib.actions.ArtifactSkyKey;
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.actions.FilesetOutputSymlink;
import com.google.devtools.build.lib.actions.LostInputsExecException.LostInputsActionExecutionException;
import com.google.devtools.build.lib.actions.MissingDepException;
import com.google.devtools.build.lib.actions.MissingInputFileException;
import com.google.devtools.build.lib.actions.NotifyOnActionCacheHit;
import com.google.devtools.build.lib.actions.PackageRootResolver;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.causes.Cause;
import com.google.devtools.build.lib.causes.LabelCause;
import com.google.devtools.build.lib.clock.BlazeClock;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.rules.cpp.IncludeScannable;
import com.google.devtools.build.lib.skyframe.ActionRewindStrategy.RewindPlan;
import com.google.devtools.build.lib.util.io.FileOutErr;
import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import com.google.devtools.build.skyframe.ValueOrException2;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.IntFunction;
import javax.annotation.Nullable;
/**
* A {@link SkyFunction} that creates {@link ActionExecutionValue}s. There are four points where
* this function can abort due to missing values in the graph:
*
* <ol>
* <li>For actions that discover inputs, if missing metadata needed to resolve an artifact from a
* string input in the action cache.
* <li>If missing metadata for artifacts in inputs (including the artifacts above).
* <li>For actions that discover inputs, if missing metadata for inputs discovered prior to
* execution.
* <li>For actions that discover inputs, but do so during execution, if missing metadata for
* inputs discovered during execution.
* </ol>
*/
public class ActionExecutionFunction implements SkyFunction, CompletionReceiver {
private final ActionRewindStrategy actionRewindStrategy = new ActionRewindStrategy();
private final SkyframeActionExecutor skyframeActionExecutor;
private final BlazeDirectories directories;
private final AtomicReference<TimestampGranularityMonitor> tsgm;
private ConcurrentMap<Action, ContinuationState> stateMap;
public ActionExecutionFunction(
SkyframeActionExecutor skyframeActionExecutor,
BlazeDirectories directories,
AtomicReference<TimestampGranularityMonitor> tsgm) {
this.skyframeActionExecutor = skyframeActionExecutor;
this.directories = directories;
this.tsgm = tsgm;
stateMap = Maps.newConcurrentMap();
}
@Override
public SkyValue compute(SkyKey skyKey, Environment env)
throws ActionExecutionFunctionException, InterruptedException {
ActionLookupData actionLookupData = (ActionLookupData) skyKey.argument();
Action action = getActionForLookupData(env, actionLookupData);
skyframeActionExecutor.noteActionEvaluationStarted(actionLookupData, action);
if (actionDependsOnBuildId(action)) {
PrecomputedValue.BUILD_ID.get(env);
}
// Look up the parts of the environment that influence the action.
Map<SkyKey, SkyValue> clientEnvLookup =
env.getValues(
Iterables.transform(
action.getClientEnvironmentVariables(), ClientEnvironmentFunction::key));
if (env.valuesMissing()) {
return null;
}
Map<String, String> clientEnv = new HashMap<>();
for (Map.Entry<SkyKey, SkyValue> entry : clientEnvLookup.entrySet()) {
ClientEnvironmentValue envValue = (ClientEnvironmentValue) entry.getValue();
if (envValue.getValue() != null) {
clientEnv.put((String) entry.getKey().argument(), envValue.getValue());
}
}
// For restarts of this ActionExecutionFunction we use a ContinuationState variable, below, to
// avoid redoing work.
//
// However, if two actions are shared and the first one executes, when the
// second one goes to execute, we should detect that and short-circuit, even without taking
// ContinuationState into account.
//
// Additionally, if an action restarted (in the Skyframe sense) after it executed because it
// discovered new inputs during execution, we should detect that and short-circuit.
ActionExecutionState previousExecution = skyframeActionExecutor.probeActionExecution(action);
// If this action was previously completed this build, then this evaluation must be happening
// because of rewinding. Prevent any ProgressLike events from being published a second time for
// this action; downstream consumers of action events reasonably don't expect them.
env = getProgressEventSuppressingEnvironmentIfPreviouslyCompleted(action, env);
if (action.discoversInputs()) {
// If this action previously failed due to a lost input found during input discovery, ensure
// that the input is regenerated before attempting discovery again.
if (declareDepsOnLostDiscoveredInputsIfAny(env, action)) {
return null;
}
}
ContinuationState state;
if (action.discoversInputs()) {
state = getState(action);
} else {
// Because this is a new state, all conditionals below about whether state has already done
// something will return false, and so we will execute all necessary steps.
state = new ContinuationState();
}
if (!state.hasCollectedInputs()) {
state.allInputs = collectInputs(action, env);
if (state.allInputs == null) {
// Missing deps.
return null;
}
} else if (state.allInputs.keysRequested != null) {
// Preserve the invariant that we ask for the same deps each build.
env.getValues(state.allInputs.keysRequested);
Preconditions.checkState(!env.valuesMissing(), "%s %s", action, state);
}
CheckInputResults checkedInputs = null;
Iterable<SkyKey> inputDepKeys =
toKeys(
state.allInputs.getAllInputs(),
action.discoversInputs() ? action.getMandatoryInputs() : null);
// Declare deps on known inputs to action. We do this unconditionally to maintain our
// invariant of asking for the same deps each build.
Map<SkyKey, ValueOrException2<MissingInputFileException, ActionExecutionException>> inputDeps =
env.getValuesOrThrow(
inputDepKeys, MissingInputFileException.class, ActionExecutionException.class);
try {
if (previousExecution == null && !state.hasArtifactData()) {
// Do we actually need to find our metadata?
checkedInputs = checkInputs(env, action, inputDeps);
}
} catch (ActionExecutionException e) {
// Remove action from state map in case it's there (won't be unless it discovers inputs).
stateMap.remove(action);
throw new ActionExecutionFunctionException(e);
}
if (env.valuesMissing()) {
// There was missing artifact metadata in the graph. Wait for it to be present.
// We must check this and return here before attempting to establish any Skyframe dependencies
// of the action; see establishSkyframeDependencies why.
return null;
}
Object skyframeDepsResult;
try {
skyframeDepsResult = establishSkyframeDependencies(env, action);
} catch (ActionExecutionException e) {
// Remove action from state map in case it's there (won't be unless it discovers inputs).
stateMap.remove(action);
throw new ActionExecutionFunctionException(
skyframeActionExecutor.processAndGetExceptionToThrow(
env.getListener(), null, action, e, new FileOutErr(), ErrorTiming.BEFORE_EXECUTION));
}
if (env.valuesMissing()) {
return null;
}
if (checkedInputs != null) {
Preconditions.checkState(!state.hasArtifactData(), "%s %s", state, action);
state.inputArtifactData = checkedInputs.actionInputMap;
state.expandedArtifacts = checkedInputs.expandedArtifacts;
state.expandedFilesets = checkedInputs.expandedFilesets;
if (skyframeActionExecutor.usesActionFileSystem()) {
state.actionFileSystem =
skyframeActionExecutor.createActionFileSystem(
directories.getRelativeOutputPath(),
checkedInputs.actionInputMap,
action.getOutputs());
}
}
long actionStartTime = BlazeClock.nanoTime();
ActionExecutionValue result;
try {
result =
checkCacheAndExecuteIfNeeded(
action,
state,
env,
clientEnv,
actionLookupData,
previousExecution,
skyframeDepsResult,
actionStartTime);
} catch (LostInputsActionExecutionException e) {
return handleLostInputs(e, actionLookupData, action, actionStartTime, env, inputDeps, state);
} catch (ActionExecutionException e) {
// Remove action from state map in case it's there (won't be unless it discovers inputs).
stateMap.remove(action);
// In this case we do not report the error to the action reporter because we have already
// done it in SkyframeActionExecutor.reportErrorIfNotAbortingMode() method. That method
// prints the error in the top-level reporter and also dumps the recorded StdErr for the
// action. Label can be null in the case of, e.g., the SystemActionOwner (for build-info.txt).
throw new ActionExecutionFunctionException(new AlreadyReportedActionExecutionException(e));
}
if (env.valuesMissing()) {
// Only input-discovering actions are present in the stateMap. Other actions may have
// valuesMissing() here in rare circumstances related to Fileset inputs being unavailable.
// See comments in ActionInputMapHelper#getFilesets().
Preconditions.checkState(!action.discoversInputs() || stateMap.containsKey(action), action);
return null;
}
// Remove action from state map in case it's there (won't be unless it discovers inputs).
stateMap.remove(action);
return result;
}
private Environment getProgressEventSuppressingEnvironmentIfPreviouslyCompleted(
Action action, Environment env) {
if (skyframeActionExecutor.probeCompletedAndReset(action)) {
return new ProgressEventSuppressingEnvironment(env);
}
return env;
}
private boolean declareDepsOnLostDiscoveredInputsIfAny(Environment env, Action action)
throws InterruptedException, ActionExecutionFunctionException {
ImmutableList<Artifact> previouslyLostDiscoveredInputs =
skyframeActionExecutor.getLostDiscoveredInputs(action);
if (previouslyLostDiscoveredInputs != null) {
Map<SkyKey, ValueOrException2<MissingInputFileException, ActionExecutionException>>
lostInputValues =
env.getValuesOrThrow(
previouslyLostDiscoveredInputs,
MissingInputFileException.class,
ActionExecutionException.class);
if (env.valuesMissing()) {
return true;
}
for (Map.Entry<SkyKey, ValueOrException2<MissingInputFileException, ActionExecutionException>>
lostInput : lostInputValues.entrySet()) {
try {
lostInput.getValue().get();
} catch (MissingInputFileException e) {
// MissingInputFileException comes from problems with source artifact construction.
// Rewinding never invalidates source artifacts.
throw new IllegalStateException(
"MissingInputFileException unexpected from rewound generated discovered input. key="
+ lostInput.getKey(),
e);
} catch (ActionExecutionException e) {
throw new ActionExecutionFunctionException(e);
}
}
}
return false;
}
/**
* Clean up state associated with the current action execution attempt and return a {@link
* Restart} value which rewinds the actions that generate the lost inputs.
*/
private SkyFunction.Restart handleLostInputs(
LostInputsActionExecutionException e,
ActionLookupData actionLookupData,
Action action,
long actionStartTime,
Environment env,
Map<SkyKey, ValueOrException2<MissingInputFileException, ActionExecutionException>> inputDeps,
ContinuationState state)
throws InterruptedException, ActionExecutionFunctionException {
// Remove action from state map in case it's there (won't be unless it discovers inputs).
stateMap.remove(action);
RewindPlan rewindPlan = null;
try {
// Reconstruct the relationship between lost inputs and this action's direct deps if any of
// the lost inputs came from runfiles:
ActionInputDepOwners runfilesDepOwners;
Set<ActionInput> lostRunfiles = e.getInputOwners().getRunfilesInputsAndOwners();
if (!lostRunfiles.isEmpty()) {
try {
runfilesDepOwners = getInputDepOwners(env, action, inputDeps, lostRunfiles);
} catch (ActionExecutionException unexpected) {
// getInputDepOwners should not be able to throw, because it does the same work as
// checkInputs, so if getInputDepOwners throws then checkInputs should have thrown, and if
// checkInputs threw then we shouldn't have reached this point in action execution.
throw new IllegalStateException(unexpected);
}
} else {
runfilesDepOwners = ActionInputDepOwners.EMPTY_INSTANCE;
}
// Collect the set of direct deps of this action which may be responsible for the lost inputs,
// some of which may be discovered.
ImmutableList<Artifact> lostDiscoveredInputs = ImmutableList.of();
Iterable<? extends SkyKey> failedActionDeps;
if (e.isFromInputDiscovery()) {
// Lost inputs found during input discovery are necessarily artifacts. These may not be
// direct deps yet, but the next time this Skyframe node is evaluated they will be. See
// SkyframeActionExecutor's lostDiscoveredInputsMap.
lostDiscoveredInputs =
e.getLostInputs().values().stream()
.map(i -> (Artifact) i)
.collect(ImmutableList.toImmutableList());
failedActionDeps = lostDiscoveredInputs;
} else if (state.discoveredInputs != null) {
failedActionDeps = Iterables.concat(inputDeps.keySet(), state.discoveredInputs);
} else {
failedActionDeps = inputDeps.keySet();
}
try {
rewindPlan =
actionRewindStrategy.getRewindPlan(
action, actionLookupData, failedActionDeps, e, runfilesDepOwners, env);
} catch (ActionExecutionException rewindingFailedException) {
// This call to processAndGetExceptionToThrow will emit an ActionExecutedEvent and report
// the error. The previous call to processAndGetExceptionToThrow didn't.
throw new ActionExecutionFunctionException(
new AlreadyReportedActionExecutionException(
skyframeActionExecutor.processAndGetExceptionToThrow(
env.getListener(),
e.getPrimaryOutputPath(),
action,
rewindingFailedException,
e.getFileOutErr(),
ActionExecutedEvent.ErrorTiming.AFTER_EXECUTION)));
}
if (e.isActionStartedEventAlreadyEmitted()) {
env.getListener().post(new ActionRewoundEvent(actionStartTime, action));
}
skyframeActionExecutor.resetFailedActionExecution(action, lostDiscoveredInputs);
for (Action actionToRestart : rewindPlan.getAdditionalActionsToRestart()) {
skyframeActionExecutor.resetPreviouslyCompletedActionExecution(actionToRestart);
}
return rewindPlan.getNodesToRestart();
} finally {
if (rewindPlan == null && e.isActionStartedEventAlreadyEmitted()) {
// Rewinding was unsuccessful. SkyframeActionExecutor's ActionRunner didn't emit an
// ActionCompletionEvent because it hoped rewinding would fix things. Because it won't, this
// must emit one to compensate.
env.getListener()
.post(new ActionCompletionEvent(actionStartTime, action, actionLookupData));
}
}
}
static Action getActionForLookupData(Environment env, ActionLookupData actionLookupData)
throws InterruptedException {
// Because of the phase boundary separating analysis and execution, all needed
// ActionLookupValues must have already been evaluated.
ActionLookupValue actionLookupValue =
Preconditions.checkNotNull(
(ActionLookupValue) env.getValue(actionLookupData.getActionLookupKey()),
"ActionLookupValue missing: %s",
actionLookupData);
return actionLookupValue.getAction(actionLookupData.getActionIndex());
}
/**
* An action's inputs needed for execution. May not just be the result of Action#getInputs(). If
* the action cache's view of this action contains additional inputs, it will request metadata for
* them, so we consider those inputs as dependencies of this action as well. Returns null if some
* dependencies were missing and this ActionExecutionFunction needs to restart.
*/
@Nullable
private AllInputs collectInputs(Action action, Environment env) throws InterruptedException {
Iterable<Artifact> allKnownInputs = action.getInputs();
if (action.inputsDiscovered()) {
return new AllInputs(allKnownInputs);
}
Preconditions.checkState(action.discoversInputs(), action);
PackageRootResolverWithEnvironment resolver = new PackageRootResolverWithEnvironment(env);
Iterable<Artifact> actionCacheInputs =
skyframeActionExecutor.getActionCachedInputs(action, resolver);
if (actionCacheInputs == null) {
Preconditions.checkState(env.valuesMissing(), action);
return null;
}
return new AllInputs(allKnownInputs, actionCacheInputs, resolver.keysRequested);
}
private static class AllInputs {
final Iterable<Artifact> defaultInputs;
@Nullable final Iterable<Artifact> actionCacheInputs;
@Nullable final List<SkyKey> keysRequested;
AllInputs(Iterable<Artifact> defaultInputs) {
this.defaultInputs = Preconditions.checkNotNull(defaultInputs);
this.actionCacheInputs = null;
this.keysRequested = null;
}
AllInputs(
Iterable<Artifact> defaultInputs,
Iterable<Artifact> actionCacheInputs,
List<SkyKey> keysRequested) {
this.defaultInputs = Preconditions.checkNotNull(defaultInputs);
this.actionCacheInputs = Preconditions.checkNotNull(actionCacheInputs);
this.keysRequested = keysRequested;
}
Iterable<Artifact> getAllInputs() {
return actionCacheInputs == null
? defaultInputs
: Iterables.concat(defaultInputs, actionCacheInputs);
}
}
/**
* Skyframe implementation of {@link PackageRootResolver}. Should be used only from SkyFunctions,
* because it uses SkyFunction.Environment for evaluation of ContainingPackageLookupValue.
*/
private static class PackageRootResolverWithEnvironment implements PackageRootResolver {
final List<SkyKey> keysRequested = new ArrayList<>();
private final Environment env;
private PackageRootResolverWithEnvironment(Environment env) {
this.env = env;
}
@Override
public Map<PathFragment, Root> findPackageRootsForFiles(Iterable<PathFragment> execPaths)
throws InterruptedException {
Preconditions.checkState(
keysRequested.isEmpty(),
"resolver should only be called once: %s %s",
keysRequested,
execPaths);
// Create SkyKeys list based on execPaths.
Map<PathFragment, SkyKey> depKeys = new HashMap<>();
for (PathFragment path : execPaths) {
PathFragment parent =
Preconditions.checkNotNull(
path.getParentDirectory(), "Must pass in files, not root directory");
Preconditions.checkArgument(!parent.isAbsolute(), path);
try {
SkyKey depKey =
ContainingPackageLookupValue.key(PackageIdentifier.discoverFromExecPath(path, true));
depKeys.put(path, depKey);
keysRequested.add(depKey);
} catch (LabelSyntaxException e) {
// This code is only used to do action cache checks. If one of the file names we got from
// the action cache is corrupted, or if the action cache is from a different Bazel
// binary, then the path may not be valid for this Bazel binary, and trigger this
// exception. In that case, it's acceptable for us to ignore the exception - we'll get an
// action cache miss and re-execute the action, which is what we should do.
continue;
}
}
Map<SkyKey, SkyValue> values = env.getValues(depKeys.values());
if (env.valuesMissing()) {
return null;
}
Map<PathFragment, Root> result = new HashMap<>();
for (PathFragment path : execPaths) {
if (!depKeys.containsKey(path)) {
continue;
}
ContainingPackageLookupValue value =
(ContainingPackageLookupValue) values.get(depKeys.get(path));
if (value.hasContainingPackage()) {
// We have found corresponding root for current execPath.
result.put(
path,
SkyframeExecutor.maybeTransformRootForRepository(
value.getContainingPackageRoot(),
value.getContainingPackageName().getRepository()));
} else {
// We haven't found corresponding root for current execPath.
result.put(path, null);
}
}
return result;
}
}
private ActionExecutionValue checkCacheAndExecuteIfNeeded(
Action action,
ContinuationState state,
Environment env,
Map<String, String> clientEnv,
ActionLookupData actionLookupData,
@Nullable ActionExecutionState previousAction,
Object skyframeDepsResult,
long actionStartTime)
throws ActionExecutionException, InterruptedException {
if (previousAction != null) {
// There are two cases where we can already have an executing action for a specific output:
// 1. Another instance of a shared action won the race and got executed first.
// 2. The action was already started earlier, and this SkyFunction got restarted since
// there's progress to be made.
// In either case, we must use this continuation to continue. Note that in the first case,
// we don't have any input metadata available, so we couldn't re-execute the action even if we
// wanted to.
return previousAction.getResultOrDependOnFuture(
env, actionLookupData, action, skyframeActionExecutor.getActionCompletedReceiver());
}
// The metadataHandler may be recreated if we discover inputs.
ArtifactPathResolver pathResolver = ArtifactPathResolver.createPathResolver(
state.actionFileSystem, skyframeActionExecutor.getExecRoot());
ActionMetadataHandler metadataHandler =
new ActionMetadataHandler(
state.inputArtifactData,
/* missingArtifactsAllowed= */ action.discoversInputs(),
action.getOutputs(),
tsgm.get(),
pathResolver,
state.actionFileSystem == null ? new OutputStore() : new MinimalOutputStore());
// We only need to check the action cache if we haven't done it on a previous run.
if (!state.hasCheckedActionCache()) {
state.token =
skyframeActionExecutor.checkActionCache(
env.getListener(),
action,
metadataHandler,
actionStartTime,
state.allInputs.actionCacheInputs,
clientEnv);
}
if (state.token == null) {
// We got a hit from the action cache -- no need to execute.
Preconditions.checkState(
!(action instanceof SkyframeAwareAction),
"Error, we're not re-executing a "
+ "SkyframeAwareAction which should be re-executed unconditionally. Action: %s",
action);
return ActionExecutionValue.createFromOutputStore(
metadataHandler.getOutputStore(),
/*outputSymlinks=*/ null,
(action instanceof IncludeScannable)
? ((IncludeScannable) action).getDiscoveredModules()
: null,
actionDependsOnBuildId(action));
}
// Delete the metadataHandler's cache of the action's outputs, since they are being deleted.
metadataHandler.discardOutputMetadata();
// This may be recreated if we discover inputs.
// TODO(shahan): this isn't used when using ActionFileSystem so we can avoid creating some
// unused objects.
if (action.discoversInputs()) {
if (state.discoveredInputs == null) {
try {
try {
state.updateFileSystemContext(
skyframeActionExecutor, env, metadataHandler, ImmutableMap.of());
} catch (IOException e) {
throw new ActionExecutionException(
"Failed to update filesystem context: ", e, action, /*catastrophe=*/ false);
}
state.discoveredInputs =
skyframeActionExecutor.discoverInputs(
action,
metadataHandler,
metadataHandler,
skyframeActionExecutor.probeCompletedAndReset(action)
? SkyframeActionExecutor.ProgressEventBehavior.SUPPRESS
: SkyframeActionExecutor.ProgressEventBehavior.EMIT,
env,
state.actionFileSystem);
Preconditions.checkState(
env.valuesMissing() == (state.discoveredInputs == null),
"discoverInputs() must return null iff requesting more dependencies.");
if (state.discoveredInputs == null) {
return null;
}
} catch (MissingDepException e) {
Preconditions.checkState(env.valuesMissing(), action);
return null;
}
}
addDiscoveredInputs(
state.inputArtifactData, state.expandedArtifacts, state.discoveredInputs, env);
if (env.valuesMissing()) {
return null;
}
metadataHandler =
new ActionMetadataHandler(
state.inputArtifactData,
/* missingArtifactsAllowed= */ false,
action.getOutputs(),
tsgm.get(),
pathResolver,
state.actionFileSystem == null ? new OutputStore() : new MinimalOutputStore());
// Set the MetadataHandler to accept output information.
metadataHandler.discardOutputMetadata();
}
// Make sure this is a regular HashMap rather than ImmutableMapBuilder so that we are safe
// in case of collisions.
Map<Artifact, ImmutableList<FilesetOutputSymlink>> filesetMappings = new HashMap<>();
for (Artifact actionInput : action.getInputs()) {
if (!actionInput.isFileset()) {
continue;
}
ImmutableList<FilesetOutputSymlink> mapping =
ActionInputMapHelper.getFilesets(env, actionInput);
if (mapping == null) {
return null;
}
filesetMappings.put(actionInput, mapping);
}
ImmutableMap<Artifact, ImmutableList<FilesetOutputSymlink>> topLevelFilesets =
ImmutableMap.copyOf(filesetMappings);
// Aggregate top-level Filesets with Filesets nested in Runfiles. Both should be used to update
// the FileSystem context.
state.expandedFilesets.forEach(filesetMappings::put);
ImmutableMap<Artifact, ImmutableList<FilesetOutputSymlink>> expandedFilesets =
ImmutableMap.copyOf(filesetMappings);
try {
state.updateFileSystemContext(skyframeActionExecutor, env, metadataHandler, expandedFilesets);
} catch (IOException e) {
throw new ActionExecutionException(
"Failed to update filesystem context: ", e, action, /*catastrophe=*/ false);
}
try (ActionExecutionContext actionExecutionContext =
skyframeActionExecutor.getContext(
metadataHandler,
metadataHandler,
skyframeActionExecutor.probeCompletedAndReset(action)
? SkyframeActionExecutor.ProgressEventBehavior.SUPPRESS
: SkyframeActionExecutor.ProgressEventBehavior.EMIT,
Collections.unmodifiableMap(state.expandedArtifacts),
expandedFilesets,
topLevelFilesets,
state.actionFileSystem,
skyframeDepsResult)) {
if (!state.hasExecutedAction()) {
state.value =
skyframeActionExecutor.executeAction(
env,
action,
metadataHandler,
actionStartTime,
actionExecutionContext,
actionLookupData);
// If an action is executed asynchronously, we may not have a result. In that case we return
// null here and expect the function to be restarted.
if (env.valuesMissing()) {
return null;
}
}
} catch (IOException e) {
throw new ActionExecutionException(
"Failed to close action output", e, action, /*catastrophe=*/ false);
}
if (action.discoversInputs()) {
Iterable<Artifact> newInputs = filterKnownInputs(action.getInputs(), state.inputArtifactData);
Map<SkyKey, SkyValue> metadataFoundDuringActionExecution =
env.getValues(toKeys(newInputs, action.getMandatoryInputs()));
state.discoveredInputs = newInputs;
if (env.valuesMissing()) {
return null;
}
if (!metadataFoundDuringActionExecution.isEmpty()) {
// We are in the interesting case of an action that discovered its inputs during
// execution, and found some new ones, but the new ones were already present in the graph.
// We must therefore cache the metadata for those new ones.
for (Map.Entry<SkyKey, SkyValue> entry : metadataFoundDuringActionExecution.entrySet()) {
state.inputArtifactData.putWithNoDepOwner(
ArtifactSkyKey.artifact(entry.getKey()), (FileArtifactValue) entry.getValue());
}
// TODO(ulfjack): This causes information loss about omitted and injected outputs. Also see
// the documentation on MetadataHandler.artifactOmitted. This works by accident because
// markOmitted is only called for remote execution, and this code only gets executed for
// local execution.
metadataHandler =
new ActionMetadataHandler(
state.inputArtifactData,
/*missingArtifactsAllowed=*/ false,
action.getOutputs(),
tsgm.get(),
pathResolver,
state.actionFileSystem == null ? new OutputStore() : new MinimalOutputStore());
}
}
Preconditions.checkState(!env.valuesMissing(), action);
skyframeActionExecutor.updateActionCacheIfReallyExecuted(
action, metadataHandler, state.token, clientEnv, actionLookupData);
return state.value;
}
private static final Function<Artifact, SkyKey> TO_NONMANDATORY_SKYKEY =
new Function<Artifact, SkyKey>() {
@Nullable
@Override
public SkyKey apply(@Nullable Artifact artifact) {
return ArtifactSkyKey.key(artifact, /*isMandatory=*/ false);
}
};
private static Iterable<SkyKey> newlyDiscoveredInputsToSkyKeys(
Iterable<Artifact> discoveredInputs, ActionInputMap inputArtifactData) {
return Iterables.transform(
filterKnownInputs(discoveredInputs, inputArtifactData), TO_NONMANDATORY_SKYKEY);
}
private static void addDiscoveredInputs(
ActionInputMap inputData,
Map<Artifact, Collection<Artifact>> expandedArtifacts,
Iterable<Artifact> discoveredInputs,
Environment env)
throws InterruptedException {
// We do not do a getValuesOrThrow() call for the following reasons:
// 1. No exceptions can be thrown for non-mandatory inputs;
// 2. Any derived inputs must be in the transitive closure of this action's inputs. Therefore,
// if there was an error building one of them, then that exception would have percolated up to
// this action already, through one of its declared inputs, and we would not have reached input
// discovery.
// Therefore there is no need to catch and rethrow exceptions as there is with #checkInputs.
Map<SkyKey, SkyValue> nonMandatoryDiscovered =
env.getValues(newlyDiscoveredInputsToSkyKeys(discoveredInputs, inputData));
if (!env.valuesMissing()) {
for (Map.Entry<SkyKey, SkyValue> entry : nonMandatoryDiscovered.entrySet()) {
Artifact input = ArtifactSkyKey.artifact(entry.getKey());
if (entry.getValue() instanceof TreeArtifactValue) {
TreeArtifactValue treeValue = (TreeArtifactValue) entry.getValue();
expandedArtifacts.put(input, ImmutableSet.<Artifact>copyOf(treeValue.getChildren()));
for (Map.Entry<Artifact.TreeFileArtifact, FileArtifactValue> child :
treeValue.getChildValues().entrySet()) {
inputData.putWithNoDepOwner(child.getKey(), child.getValue());
}
inputData.putWithNoDepOwner(input, treeValue.getSelfData());
} else {
inputData.putWithNoDepOwner(input, (FileArtifactValue) entry.getValue());
}
}
}
}
private static Object establishSkyframeDependencies(Environment env, Action action)
throws ActionExecutionException, InterruptedException {
// Before we may safely establish Skyframe dependencies, we must build all action inputs by
// requesting their ArtifactValues.
// This is very important to do, because the establishSkyframeDependencies method may request
// FileValues for input files of this action (directly requesting them, or requesting some other
// SkyValue whose builder requests FileValues), which may not yet exist if their generating
// actions have not yet run.
// See SkyframeAwareActionTest.testRaceConditionBetweenInputAcquisitionAndSkyframeDeps
Preconditions.checkState(!env.valuesMissing(), action);
if (action instanceof SkyframeAwareAction) {
// Skyframe-aware actions should be executed unconditionally, i.e. bypass action cache
// checking. See documentation of SkyframeAwareAction.
Preconditions.checkState(action.executeUnconditionally(), action);
try {
return ((SkyframeAwareAction) action).establishSkyframeDependencies(env);
} catch (SkyframeAwareAction.ExceptionBase e) {
throw new ActionExecutionException(e, action, false);
}
}
return null;
}
private static Iterable<SkyKey> toKeys(
Iterable<Artifact> inputs, Iterable<Artifact> mandatoryInputs) {
if (mandatoryInputs == null) {
// This is a non inputs-discovering action, so no need to distinguish mandatory from regular
// inputs.
return Iterables.transform(
inputs,
new Function<Artifact, SkyKey>() {
@Override
public SkyKey apply(Artifact artifact) {
return ArtifactSkyKey.key(artifact, true);
}
});
} else {
Collection<SkyKey> discoveredArtifacts = new HashSet<>();
Set<Artifact> mandatory = Sets.newHashSet(mandatoryInputs);
for (Artifact artifact : inputs) {
discoveredArtifacts.add(ArtifactSkyKey.key(artifact, mandatory.contains(artifact)));
}
return discoveredArtifacts;
}
}
private static class CheckInputResults {
/** Metadata about Artifacts consumed by this Action. */
private final ActionInputMap actionInputMap;
/** Artifact expansion mapping for Runfiles tree and tree artifacts. */
private final Map<Artifact, Collection<Artifact>> expandedArtifacts;
/** Artifact expansion mapping for Filesets embedded in Runfiles. */
private final Map<Artifact, ImmutableList<FilesetOutputSymlink>> expandedFilesets;
public CheckInputResults(ActionInputMap actionInputMap,
Map<Artifact, Collection<Artifact>> expandedArtifacts,
Map<Artifact, ImmutableList<FilesetOutputSymlink>> expandedFilesets) {
this.actionInputMap = actionInputMap;
this.expandedArtifacts = expandedArtifacts;
this.expandedFilesets = expandedFilesets;
}
}
private interface AccumulateInputResultsFactory<S extends ActionInputMapSink, R> {
R create(
S actionInputMapSink,
Map<Artifact, Collection<Artifact>> expandedArtifacts,
Map<Artifact, ImmutableList<FilesetOutputSymlink>> expandedFilesets);
}
/**
* Declare dependency on all known inputs of action. Throws exception if any are known to be
* missing. Some inputs may not yet be in the graph, in which case the builder should abort.
*/
private CheckInputResults checkInputs(
Environment env,
Action action,
Map<SkyKey, ValueOrException2<MissingInputFileException, ActionExecutionException>> inputDeps)
throws ActionExecutionException, InterruptedException {
return accumulateInputs(env, action, inputDeps, ActionInputMap::new, CheckInputResults::new);
}
/**
* Reconstructs the relationships between lost inputs and the direct deps responsible for them.
*/
private ActionInputDepOwners getInputDepOwners(
Environment env,
Action action,
Map<SkyKey, ValueOrException2<MissingInputFileException, ActionExecutionException>> inputDeps,
Collection<ActionInput> lostInputs)
throws ActionExecutionException, InterruptedException {
return accumulateInputs(
env,
action,
inputDeps,
ignoredInputDepsSize -> new ActionInputDepOwnerMap(lostInputs),
(actionInputMapSink, expandedArtifacts, expandedFilesets) -> actionInputMapSink);
}
private <S extends ActionInputMapSink, R> R accumulateInputs(
Environment env,
Action action,
Map<SkyKey, ValueOrException2<MissingInputFileException, ActionExecutionException>> inputDeps,
IntFunction<S> actionInputMapSinkFactory,
AccumulateInputResultsFactory<S, R> accumulateInputResultsFactory)
throws ActionExecutionException, InterruptedException {
int missingCount = 0;
int actionFailures = 0;
// Only populate input data if we have the input values, otherwise they'll just go unused.
// We still want to loop through the inputs to collect missing deps errors. During the
// evaluator "error bubbling", we may get one last chance at reporting errors even though
// some deps are still missing.
boolean populateInputData = !env.valuesMissing();
NestedSetBuilder<Cause> rootCauses = NestedSetBuilder.stableOrder();
S inputArtifactData = actionInputMapSinkFactory.apply(populateInputData ? inputDeps.size() : 0);
Map<Artifact, Collection<Artifact>> expandedArtifacts =
new HashMap<>(populateInputData ? 128 : 0);
Map<Artifact, ImmutableList<FilesetOutputSymlink>> expandedFilesets = new HashMap<>();
ActionExecutionException firstActionExecutionException = null;
for (Map.Entry<SkyKey, ValueOrException2<MissingInputFileException, ActionExecutionException>>
depsEntry : inputDeps.entrySet()) {
Artifact input = ArtifactSkyKey.artifact(depsEntry.getKey());
try {
SkyValue value = depsEntry.getValue().get();
if (populateInputData) {
ActionInputMapHelper.addToMap(
inputArtifactData,
expandedArtifacts,
expandedFilesets,
input,
value,
env);
}
} catch (MissingInputFileException e) {
missingCount++;
if (input.getOwner() != null) {
rootCauses.add(new LabelCause(input.getOwner(), e.getMessage()));
}
} catch (ActionExecutionException e) {
actionFailures++;
// Prefer a catastrophic exception as the one we propagate.
if (firstActionExecutionException == null
|| (!firstActionExecutionException.isCatastrophe() && e.isCatastrophe())) {
firstActionExecutionException = e;
}
rootCauses.addTransitive(e.getRootCauses());
}
}
// We need to rethrow first exception because it can contain useful error message
if (firstActionExecutionException != null) {
if (missingCount == 0 && actionFailures == 1) {
// In the case a single action failed, just propagate the exception upward. This avoids
// having to copy the root causes to the upwards transitive closure.
throw firstActionExecutionException;
}
throw new ActionExecutionException(
firstActionExecutionException.getMessage(),
firstActionExecutionException.getCause(),
action,
rootCauses.build(),
firstActionExecutionException.isCatastrophe(),
firstActionExecutionException.getExitCode());
}
if (missingCount > 0) {
for (Cause missingInput : rootCauses.build()) {
env.getListener()
.handle(
Event.error(
action.getOwner().getLocation(),
String.format(
"%s: missing input file '%s'",
action.getOwner().getLabel(), missingInput.getLabel())));
}
throw new ActionExecutionException(
missingCount + " input file(s) do not exist",
action,
rootCauses.build(),
/*catastrophe=*/ false);
}
return accumulateInputResultsFactory.create(
inputArtifactData, expandedArtifacts, expandedFilesets);
}
private static Iterable<Artifact> filterKnownInputs(
Iterable<Artifact> newInputs, ActionInputMap inputArtifactData) {
return Iterables.filter(newInputs, input -> inputArtifactData.getMetadata(input) == null);
}
static boolean actionDependsOnBuildId(Action action) {
// Volatile build actions may need to execute even if none of their known inputs have changed.
// Depending on the build id ensures that these actions have a chance to execute.
// SkyframeAwareActions do not need to depend on the build id because their volatility is due to
// their dependence on Skyframe nodes that are not captured in the action cache. Any changes to
// those nodes will cause this action to be rerun, so a build id dependency is unnecessary.
return (action.isVolatile() && !(action instanceof SkyframeAwareAction))
|| action instanceof NotifyOnActionCacheHit;
}
/** All info/warning messages associated with actions should be always displayed. */
@Override
public String extractTag(SkyKey skyKey) {
return null;
}
/**
* Should be called once execution is over, and the intra-build cache of in-progress computations
* should be discarded. If the cache is non-empty (due to an interrupted/failed build), failure to
* call complete() can both cause a memory leak and incorrect results on the subsequent build.
*/
@Override
public void complete() {
// Discard all remaining state (there should be none after a successful execution).
stateMap = Maps.newConcurrentMap();
actionRewindStrategy.reset();
}
private ContinuationState getState(Action action) {
ContinuationState state = stateMap.get(action);
if (state == null) {
state = new ContinuationState();
Preconditions.checkState(stateMap.put(action, state) == null, action);
}
return state;
}
/**
* State to save work across restarts of ActionExecutionFunction due to missing values in the
* graph for actions that discover inputs. There are three places where we save work, all for
* actions that discover inputs:
*
* <ol>
* <li>If not all known input metadata (coming from Action#getInputs) is available yet, then the
* calculated set of inputs (including the inputs resolved from the action cache) is saved.
* <li>If not all discovered inputs' metadata is available yet, then the known input metadata
* together with the set of discovered inputs is saved, as well as the Token used to
* identify this action to the action cache.
* <li>If, after execution, new inputs are discovered whose metadata is not yet available, then
* the same data as in the previous case is saved, along with the actual result of
* execution.
* </ol>
*/
private static class ContinuationState {
AllInputs allInputs;
/** Mutable map containing metadata for known artifacts. */
ActionInputMap inputArtifactData = null;
Map<Artifact, Collection<Artifact>> expandedArtifacts = null;
Map<Artifact, ImmutableList<FilesetOutputSymlink>> expandedFilesets = null;
Token token = null;
Iterable<Artifact> discoveredInputs = null;
ActionExecutionValue value = null;
FileSystem actionFileSystem = null;
boolean hasCollectedInputs() {
return allInputs != null;
}
boolean hasArtifactData() {
boolean result = inputArtifactData != null;
Preconditions.checkState(result == (expandedArtifacts != null), this);
return result;
}
boolean hasCheckedActionCache() {
// If token is null because there was an action cache hit, this method is never called again
// because we return immediately.
return token != null;
}
boolean hasExecutedAction() {
return value != null;
}
/** Must be called to assign values to the given variables as they change. */
void updateFileSystemContext(
SkyframeActionExecutor executor,
Environment env,
ActionMetadataHandler metadataHandler,
ImmutableMap<Artifact, ImmutableList<FilesetOutputSymlink>> filesets)
throws IOException {
if (actionFileSystem != null) {
executor.updateActionFileSystemContext(
actionFileSystem, env, metadataHandler.getOutputStore()::injectOutputData, filesets);
}
}
@Override
public String toString() {
return token
+ ", "
+ value
+ ", "
+ allInputs
+ ", "
+ inputArtifactData
+ ", "
+ discoveredInputs;
}
}
/**
* Used to declare all the exception types that can be wrapped in the exception thrown by {@link
* ActionExecutionFunction#compute}.
*/
static final class ActionExecutionFunctionException extends SkyFunctionException {
private final ActionExecutionException actionException;
ActionExecutionFunctionException(ActionExecutionException e) {
// We conservatively assume that the error is transient. We don't have enough information to
// distinguish non-transient errors (e.g. compilation error from a deterministic compiler)
// from transient ones (e.g. IO error).
// TODO(bazel-team): Have ActionExecutionExceptions declare their transience.
super(e, Transience.TRANSIENT);
this.actionException = e;
}
@Override
public boolean isCatastrophic() {
return actionException.isCatastrophe();
}
}
}