Update from Google.
--
MOE_MIGRATED_REVID=85702957
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupFunction.java
new file mode 100644
index 0000000..6dcc224
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupFunction.java
@@ -0,0 +1,177 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.packages.CachingPackageLocator;
+import com.google.devtools.build.lib.packages.RuleClassProvider;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.syntax.BuildFileAST;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * A SkyFunction for {@link ASTFileLookupValue}s. Tries to locate a file and load it as a
+ * syntax tree and cache the resulting {@link BuildFileAST}. If the file doesn't exist
+ * the function doesn't fail but returns a specific NO_FILE ASTLookupValue.
+ */
+public class ASTFileLookupFunction implements SkyFunction {
+
+ private abstract static class FileLookupResult {
+ /** Returns whether the file lookup was successful. */
+ public abstract boolean lookupSuccessful();
+
+ /** If {@code lookupSuccessful()}, returns the {@link RootedPath} to the file. */
+ public abstract RootedPath rootedPath();
+
+ static FileLookupResult noFile() {
+ return UnsuccessfulFileResult.INSTANCE;
+ }
+
+ static FileLookupResult file(RootedPath rootedPath) {
+ return new SuccessfulFileResult(rootedPath);
+ }
+
+ private static class SuccessfulFileResult extends FileLookupResult {
+ private final RootedPath rootedPath;
+
+ private SuccessfulFileResult(RootedPath rootedPath) {
+ this.rootedPath = rootedPath;
+ }
+
+ @Override
+ public boolean lookupSuccessful() {
+ return true;
+ }
+
+ @Override
+ public RootedPath rootedPath() {
+ return rootedPath;
+ }
+ }
+
+ private static class UnsuccessfulFileResult extends FileLookupResult {
+ private static final UnsuccessfulFileResult INSTANCE = new UnsuccessfulFileResult();
+ private UnsuccessfulFileResult() {
+ }
+
+ @Override
+ public boolean lookupSuccessful() {
+ return false;
+ }
+
+ @Override
+ public RootedPath rootedPath() {
+ throw new IllegalStateException("unsucessful lookup");
+ }
+ }
+ }
+
+ private final AtomicReference<PathPackageLocator> pkgLocator;
+ private final RuleClassProvider ruleClassProvider;
+ private final CachingPackageLocator packageManager;
+
+ public ASTFileLookupFunction(AtomicReference<PathPackageLocator> pkgLocator,
+ CachingPackageLocator packageManager,
+ RuleClassProvider ruleClassProvider) {
+ this.pkgLocator = pkgLocator;
+ this.packageManager = packageManager;
+ this.ruleClassProvider = ruleClassProvider;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException,
+ InterruptedException {
+ PathFragment astFilePathFragment = (PathFragment) skyKey.argument();
+ FileLookupResult lookupResult = getASTFile(env, astFilePathFragment);
+ if (lookupResult == null) {
+ return null;
+ }
+
+ BuildFileAST ast = null;
+ if (!lookupResult.lookupSuccessful()) {
+ // Return the specific NO_FILE ASTLookupValue instance if no file was found.
+ return ASTFileLookupValue.NO_FILE;
+ } else {
+ Path path = lookupResult.rootedPath().asPath();
+ // Skylark files end with bzl.
+ boolean parseAsSkylark = astFilePathFragment.getPathString().endsWith(".bzl");
+ try {
+ ast = parseAsSkylark
+ ? BuildFileAST.parseSkylarkFile(path, env.getListener(),
+ packageManager, ruleClassProvider.getSkylarkValidationEnvironment().clone())
+ : BuildFileAST.parseBuildFile(path, env.getListener(),
+ packageManager, false);
+ } catch (IOException e) {
+ throw new ASTLookupFunctionException(new ErrorReadingSkylarkExtensionException(
+ e.getMessage()), Transience.TRANSIENT);
+ }
+ }
+
+ return new ASTFileLookupValue(ast);
+ }
+
+ private FileLookupResult getASTFile(Environment env, PathFragment astFilePathFragment)
+ throws ASTLookupFunctionException {
+ for (Path packagePathEntry : pkgLocator.get().getPathEntries()) {
+ RootedPath rootedPath = RootedPath.toRootedPath(packagePathEntry, astFilePathFragment);
+ SkyKey fileSkyKey = FileValue.key(rootedPath);
+ FileValue fileValue = null;
+ try {
+ fileValue = (FileValue) env.getValueOrThrow(fileSkyKey, IOException.class,
+ FileSymlinkCycleException.class, InconsistentFilesystemException.class);
+ } catch (IOException | FileSymlinkCycleException e) {
+ throw new ASTLookupFunctionException(new ErrorReadingSkylarkExtensionException(
+ e.getMessage()), Transience.PERSISTENT);
+ } catch (InconsistentFilesystemException e) {
+ throw new ASTLookupFunctionException(e, Transience.PERSISTENT);
+ }
+ if (fileValue == null) {
+ return null;
+ }
+ if (fileValue.isFile()) {
+ return FileLookupResult.file(rootedPath);
+ }
+ }
+ return FileLookupResult.noFile();
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ private static final class ASTLookupFunctionException extends SkyFunctionException {
+ private ASTLookupFunctionException(ErrorReadingSkylarkExtensionException e,
+ Transience transience) {
+ super(e, transience);
+ }
+
+ private ASTLookupFunctionException(InconsistentFilesystemException e, Transience transience) {
+ super(e, transience);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupValue.java
new file mode 100644
index 0000000..1061c86
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupValue.java
@@ -0,0 +1,61 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.syntax.BuildFileAST;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import javax.annotation.Nullable;
+
+/**
+ * A value that represents an AST file lookup result.
+ */
+public class ASTFileLookupValue implements SkyValue {
+
+ static final ASTFileLookupValue NO_FILE = new ASTFileLookupValue(null);
+
+ @Nullable private final BuildFileAST ast;
+
+ public ASTFileLookupValue(@Nullable BuildFileAST ast) {
+ this.ast = ast;
+ }
+
+ /**
+ * Returns the original AST file.
+ */
+ @Nullable public BuildFileAST getAST() {
+ return ast;
+ }
+
+ static void checkInputArgument(PathFragment astFilePathFragment) throws ASTLookupInputException {
+ if (astFilePathFragment.isAbsolute()) {
+ throw new ASTLookupInputException(String.format(
+ "Input file '%s' cannot be an absolute path.", astFilePathFragment));
+ }
+ }
+
+ static SkyKey key(PathFragment astFilePathFragment) throws ASTLookupInputException {
+ checkInputArgument(astFilePathFragment);
+ return new SkyKey(SkyFunctions.AST_FILE_LOOKUP, astFilePathFragment);
+ }
+
+ static final class ASTLookupInputException extends Exception {
+ private ASTLookupInputException(String msg) {
+ super(msg);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/AbstractLabelCycleReporter.java b/src/main/java/com/google/devtools/build/lib/skyframe/AbstractLabelCycleReporter.java
new file mode 100644
index 0000000..797f158
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/AbstractLabelCycleReporter.java
@@ -0,0 +1,130 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.LoadedPackageProvider;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.CycleInfo;
+import com.google.devtools.build.skyframe.CyclesReporter;
+import com.google.devtools.build.skyframe.SkyKey;
+
+/** Reports cycles between skyframe values whose keys contains {@link Label}s. */
+abstract class AbstractLabelCycleReporter implements CyclesReporter.SingleCycleReporter {
+
+ private final LoadedPackageProvider loadedPackageProvider;
+
+ AbstractLabelCycleReporter(LoadedPackageProvider loadedPackageProvider) {
+ this.loadedPackageProvider = loadedPackageProvider;
+ }
+
+ /** Returns the String representation of the {@code SkyKey}. */
+ protected abstract String prettyPrint(SkyKey key);
+
+ /** Returns the associated Label of the SkyKey. */
+ protected abstract Label getLabel(SkyKey key);
+
+ protected abstract boolean canReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo);
+
+ protected String getAdditionalMessageAboutCycle(SkyKey topLevelKey, CycleInfo cycleInfo) {
+ return "";
+ }
+
+ @Override
+ public boolean maybeReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo,
+ boolean alreadyReported, EventHandler eventHandler) {
+ Preconditions.checkNotNull(eventHandler);
+ if (!canReportCycle(topLevelKey, cycleInfo)) {
+ return false;
+ }
+
+ if (alreadyReported) {
+ Label label = getLabel(topLevelKey);
+ Target target = getTargetForLabel(label);
+ eventHandler.handle(Event.error(target.getLocation(),
+ "in " + target.getTargetKind() + " " + label +
+ ": cycle in dependency graph: target depends on an already-reported cycle"));
+ } else {
+ StringBuilder cycleMessage = new StringBuilder("cycle in dependency graph:");
+ ImmutableList<SkyKey> pathToCycle = cycleInfo.getPathToCycle();
+ ImmutableList<SkyKey> cycle = cycleInfo.getCycle();
+ for (SkyKey value : pathToCycle) {
+ cycleMessage.append("\n ");
+ cycleMessage.append(prettyPrint(value));
+ }
+
+ SkyKey cycleValue = printCycle(cycle, cycleMessage, new Function<SkyKey, String>() {
+ @Override
+ public String apply(SkyKey input) {
+ return prettyPrint(input);
+ }
+ });
+
+ cycleMessage.append(getAdditionalMessageAboutCycle(topLevelKey, cycleInfo));
+
+ Label label = getLabel(cycleValue);
+ Target target = getTargetForLabel(label);
+ eventHandler.handle(
+ Event.error(target.getLocation(), "in " + target.getTargetKind() + " " + label
+ + ": " + cycleMessage.toString()));
+ }
+
+ return true;
+ }
+
+ /**
+ * Prints the SkyKey-s in cycle into cycleMessage using the print function.
+ */
+ static SkyKey printCycle(ImmutableList<SkyKey> cycle, StringBuilder cycleMessage,
+ Function<SkyKey, String> printFunction) {
+ Iterable<SkyKey> valuesToPrint = cycle.size() > 1
+ ? Iterables.concat(cycle, ImmutableList.of(cycle.get(0))) : cycle;
+ SkyKey cycleValue = null;
+ for (SkyKey value : valuesToPrint) {
+ if (cycleValue == null) {
+ cycleValue = value;
+ }
+ if (value == cycleValue) {
+ cycleMessage.append("\n * ");
+ } else {
+ cycleMessage.append("\n ");
+ }
+ cycleMessage.append(printFunction.apply(value));
+ }
+
+ if (cycle.size() == 1) {
+ cycleMessage.append(" [self-edge]");
+ }
+
+ return cycleValue;
+ }
+
+ protected final Target getTargetForLabel(Label label) {
+ try {
+ return loadedPackageProvider.getLoadedTarget(label);
+ } catch (NoSuchThingException e) {
+ // This method is used for getting the target from a label in a circular dependency.
+ // If we have a cycle that means that we need to have accessed the target (to get its
+ // dependencies). So all the labels in a dependency cycle need to exist.
+ throw new IllegalStateException(e);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionArtifactCycleReporter.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionArtifactCycleReporter.java
new file mode 100644
index 0000000..3105539
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionArtifactCycleReporter.java
@@ -0,0 +1,77 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
+import com.google.devtools.build.lib.pkgcache.LoadedPackageProvider;
+import com.google.devtools.build.lib.skyframe.ArtifactValue.OwnedArtifact;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.CycleInfo;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+
+/**
+ * Reports cycles between Actions and Artifacts. These indicates cycles within a rule.
+ */
+public class ActionArtifactCycleReporter extends AbstractLabelCycleReporter {
+
+ private static final Predicate<SkyKey> IS_ARTIFACT_OR_ACTION_SKY_KEY = Predicates.or(
+ SkyFunctions.isSkyFunction(SkyFunctions.ARTIFACT),
+ SkyFunctions.isSkyFunction(SkyFunctions.ACTION_EXECUTION),
+ SkyFunctions.isSkyFunction(SkyFunctions.TARGET_COMPLETION));
+
+ ActionArtifactCycleReporter(LoadedPackageProvider loadedPackageProvider) {
+ super(loadedPackageProvider);
+ }
+
+ @Override
+ protected String prettyPrint(SkyKey key) {
+ return prettyPrint(key.functionName(), key.argument());
+ }
+
+ private String prettyPrint(SkyFunctionName skyFunctionName, Object arg) {
+ if (arg instanceof OwnedArtifact) {
+ return "file: " + ((OwnedArtifact) arg).getArtifact().getRootRelativePathString();
+ } else if (arg instanceof Action) {
+ return "action: " + ((Action) arg).getMnemonic();
+ } else if (arg instanceof LabelAndConfiguration
+ && skyFunctionName == SkyFunctions.TARGET_COMPLETION) {
+ return "configured target: " + ((LabelAndConfiguration) arg).getLabel().toString();
+ }
+ throw new IllegalStateException(
+ "Argument is not Action, TargetCompletion, or OwnedArtifact: " + arg);
+ }
+
+ @Override
+ protected Label getLabel(SkyKey key) {
+ Object arg = key.argument();
+ if (arg instanceof OwnedArtifact) {
+ return ((OwnedArtifact) arg).getArtifact().getOwner();
+ } else if (arg instanceof Action) {
+ return ((Action) arg).getOwner().getLabel();
+ }
+ throw new IllegalStateException("Argument is not Action or OwnedArtifact: " + arg);
+ }
+
+ @Override
+ protected boolean canReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo) {
+ return IS_ARTIFACT_OR_ACTION_SKY_KEY.apply(topLevelKey)
+ && Iterables.all(cycleInfo.getCycle(), IS_ARTIFACT_OR_ACTION_SKY_KEY);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java
new file mode 100644
index 0000000..1420860
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java
@@ -0,0 +1,338 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionCacheChecker.Token;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.AlreadyReportedActionExecutionException;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.MissingInputFileException;
+import com.google.devtools.build.lib.actions.NotifyOnActionCacheHit;
+import com.google.devtools.build.lib.actions.cache.MetadataHandler;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import com.google.devtools.build.skyframe.ValueOrException2;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A builder for {@link ActionExecutionValue}s.
+ */
+public class ActionExecutionFunction implements SkyFunction {
+
+ private static final Predicate<Artifact> IS_SOURCE_ARTIFACT = new Predicate<Artifact>() {
+ @Override
+ public boolean apply(Artifact input) {
+ return input.isSourceArtifact();
+ }
+ };
+
+ private final SkyframeActionExecutor skyframeActionExecutor;
+ private final TimestampGranularityMonitor tsgm;
+
+ public ActionExecutionFunction(SkyframeActionExecutor skyframeActionExecutor,
+ TimestampGranularityMonitor tsgm) {
+ this.skyframeActionExecutor = skyframeActionExecutor;
+ this.tsgm = tsgm;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws ActionExecutionFunctionException,
+ InterruptedException {
+ Action action = (Action) skyKey.argument();
+ Map<Artifact, FileArtifactValue> inputArtifactData = null;
+ Map<Artifact, Collection<Artifact>> expandedMiddlemen = null;
+ boolean alreadyRan = skyframeActionExecutor.probeActionExecution(action);
+ try {
+ Pair<Map<Artifact, FileArtifactValue>, Map<Artifact, Collection<Artifact>>> checkedInputs =
+ checkInputs(env, action, alreadyRan); // Declare deps on known inputs to action.
+
+ if (checkedInputs != null) {
+ inputArtifactData = checkedInputs.first;
+ expandedMiddlemen = checkedInputs.second;
+ }
+ } catch (ActionExecutionException e) {
+ throw new ActionExecutionFunctionException(e);
+ }
+ // TODO(bazel-team): Non-volatile NotifyOnActionCacheHit actions perform worse in Skyframe than
+ // legacy when they are not at the top of the action graph. In legacy, they are stored
+ // separately, so notifying non-dirty actions is cheap. In Skyframe, they depend on the
+ // BUILD_ID, forcing invalidation of upward transitive closure on each build.
+ if (action.isVolatile() || action instanceof NotifyOnActionCacheHit) {
+ // Volatile build actions may need to execute even if none of their known inputs have changed.
+ // Depending on the buildID ensure that these actions have a chance to execute.
+ PrecomputedValue.BUILD_ID.get(env);
+ }
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ ActionExecutionValue result;
+ try {
+ result = checkCacheAndExecuteIfNeeded(action, inputArtifactData, expandedMiddlemen, env);
+ } catch (ActionExecutionException e) {
+ // In this case we do not report the error to the action reporter because we have already
+ // done it in SkyframeExecutor.reportErrorIfNotAbortingMode() method. That method
+ // prints the error in the top-level reporter and also dumps the recorded StdErr for the
+ // action. Label can be null in the case of, e.g., the SystemActionOwner (for build-info.txt).
+ throw new ActionExecutionFunctionException(new AlreadyReportedActionExecutionException(e));
+ } finally {
+ declareAdditionalDependencies(env, action);
+ }
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ return result;
+ }
+
+ private ActionExecutionValue checkCacheAndExecuteIfNeeded(
+ Action action,
+ Map<Artifact, FileArtifactValue> inputArtifactData,
+ Map<Artifact, Collection<Artifact>> expandedMiddlemen,
+ Environment env) throws ActionExecutionException, InterruptedException {
+ // Don't initialize the cache if the result has already been computed and this is just a
+ // rerun.
+ FileAndMetadataCache fileAndMetadataCache = null;
+ MetadataHandler metadataHandler = null;
+ Token token = null;
+ long actionStartTime = System.nanoTime();
+ // inputArtifactData is null exactly when we know that the execution result was already
+ // computed on a prior run of this SkyFunction. If it is null we don't need to initialize
+ // anything -- we will get the result directly from SkyframeActionExecutor's cache.
+ if (inputArtifactData != null) {
+ // Check action cache to see if we need to execute anything. Checking the action cache only
+ // needs to happen on the first run, since a cache hit means we'll return immediately, and
+ // there'll be no second run.
+ fileAndMetadataCache = new FileAndMetadataCache(
+ inputArtifactData,
+ expandedMiddlemen,
+ skyframeActionExecutor.getExecRoot(),
+ action.getOutputs(),
+ // Only give the metadata cache the ability to look up Skyframe values if the action
+ // might have undeclared inputs. If those undeclared inputs are generated, they are
+ // present in Skyframe, so we can save a stat by looking them up directly.
+ action.discoversInputs() ? env : null,
+ tsgm);
+ metadataHandler =
+ skyframeActionExecutor.constructMetadataHandler(fileAndMetadataCache);
+ token = skyframeActionExecutor.checkActionCache(action, metadataHandler, actionStartTime);
+ }
+ if (token == null && inputArtifactData != null) {
+ // We got a hit from the action cache -- no need to execute.
+ return new ActionExecutionValue(
+ fileAndMetadataCache.getOutputData(),
+ fileAndMetadataCache.getAdditionalOutputData());
+ } else {
+ ActionExecutionContext actionExecutionContext = null;
+ if (inputArtifactData != null) {
+ actionExecutionContext = skyframeActionExecutor.constructActionExecutionContext(
+ fileAndMetadataCache,
+ metadataHandler);
+ if (action.discoversInputs()) {
+ skyframeActionExecutor.discoverInputs(action, actionExecutionContext);
+ }
+ }
+ // If this is the second time we are here (because the action discovers inputs, and we had
+ // to restart the value builder after declaring our dependence on newly discovered inputs),
+ // the result returned here is the already-computed result from the first run.
+ // Similarly, if this is a shared action and the other action is the one that executed, we
+ // must use that other action's value, provided here, since it is populated with metadata
+ // for the outputs.
+ // If this action was not shared and this is the first run of the action, this returned
+ // result was computed during the call.
+ return skyframeActionExecutor.executeAction(action, fileAndMetadataCache, token,
+ actionStartTime, actionExecutionContext);
+ }
+ }
+
+ private static Iterable<SkyKey> toKeys(Iterable<Artifact> inputs,
+ Iterable<Artifact> mandatoryInputs) {
+ if (mandatoryInputs == null) {
+ // This is a non inputs-discovering action, so no need to distinguish mandatory from regular
+ // inputs.
+ return Iterables.transform(inputs, new Function<Artifact, SkyKey>() {
+ @Override
+ public SkyKey apply(Artifact artifact) {
+ return ArtifactValue.key(artifact, true);
+ }
+ });
+ } else {
+ Collection<SkyKey> discoveredArtifacts = new HashSet<>();
+ Set<Artifact> mandatory = Sets.newHashSet(mandatoryInputs);
+ for (Artifact artifact : inputs) {
+ discoveredArtifacts.add(ArtifactValue.key(artifact, mandatory.contains(artifact)));
+ }
+
+ // In case the action violates the invariant that getInputs() is a superset of
+ // getMandatoryInputs(), explicitly add the mandatory inputs. See bug about an
+ // "action not in canonical form" error message. Also note that we may add Skyframe edges on
+ // these potentially stale deps due to the way loading inputs from the action cache functions.
+ // In practice, this is safe since C++ actions (the only ones which discover inputs) only add
+ // possibly stale inputs on source artifacts, which we treat as non-mandatory.
+ for (Artifact artifact : mandatory) {
+ discoveredArtifacts.add(ArtifactValue.key(artifact, true));
+ }
+ return discoveredArtifacts;
+ }
+ }
+
+ /**
+ * Declare dependency on all known inputs of action. Throws exception if any are known to be
+ * missing. Some inputs may not yet be in the graph, in which case the builder should abort.
+ */
+ private Pair<Map<Artifact, FileArtifactValue>, Map<Artifact, Collection<Artifact>>> checkInputs(
+ Environment env, Action action, boolean alreadyRan) throws ActionExecutionException {
+ Map<SkyKey, ValueOrException2<MissingInputFileException, ActionExecutionException>> inputDeps =
+ env.getValuesOrThrow(toKeys(action.getInputs(), action.discoversInputs()
+ ? action.getMandatoryInputs() : null), MissingInputFileException.class,
+ ActionExecutionException.class);
+
+ // If the action was already run, then break out early. This avoids the cost of constructing the
+ // input map and expanded middlemen if they're not going to be used.
+ if (alreadyRan) {
+ return null;
+ }
+
+ int missingCount = 0;
+ int actionFailures = 0;
+ boolean catastrophe = false;
+ // Only populate input data if we have the input values, otherwise they'll just go unused.
+ // We still want to loop through the inputs to collect missing deps errors. During the
+ // evaluator "error bubbling", we may get one last chance at reporting errors even though
+ // some deps are stilling missing.
+ boolean populateInputData = !env.valuesMissing();
+ NestedSetBuilder<Label> rootCauses = NestedSetBuilder.stableOrder();
+ Map<Artifact, FileArtifactValue> inputArtifactData =
+ new HashMap<>(populateInputData ? inputDeps.size() : 0);
+ Map<Artifact, Collection<Artifact>> expandedMiddlemen =
+ new HashMap<>(populateInputData ? 128 : 0);
+
+ ActionExecutionException firstActionExecutionException = null;
+ for (Map.Entry<SkyKey, ValueOrException2<MissingInputFileException,
+ ActionExecutionException>> depsEntry : inputDeps.entrySet()) {
+ Artifact input = ArtifactValue.artifact(depsEntry.getKey());
+ try {
+ ArtifactValue value = (ArtifactValue) depsEntry.getValue().get();
+ if (populateInputData && value instanceof AggregatingArtifactValue) {
+ AggregatingArtifactValue aggregatingValue = (AggregatingArtifactValue) value;
+ for (Pair<Artifact, FileArtifactValue> entry : aggregatingValue.getInputs()) {
+ inputArtifactData.put(entry.first, entry.second);
+ }
+ // We have to cache the "digest" of the aggregating value itself, because the action cache
+ // checker may want it.
+ inputArtifactData.put(input, aggregatingValue.getSelfData());
+ expandedMiddlemen.put(input,
+ Collections2.transform(aggregatingValue.getInputs(),
+ Pair.<Artifact, FileArtifactValue>firstFunction()));
+ } else if (populateInputData && value instanceof FileArtifactValue) {
+ // TODO(bazel-team): Make sure middleman "virtual" artifact data is properly processed.
+ inputArtifactData.put(input, (FileArtifactValue) value);
+ }
+ } catch (MissingInputFileException e) {
+ missingCount++;
+ if (input.getOwner() != null) {
+ rootCauses.add(input.getOwner());
+ }
+ } catch (ActionExecutionException e) {
+ actionFailures++;
+ if (firstActionExecutionException == null) {
+ firstActionExecutionException = e;
+ }
+ catastrophe = catastrophe || e.isCatastrophe();
+ rootCauses.addTransitive(e.getRootCauses());
+ }
+ }
+ // We need to rethrow first exception because it can contain useful error message
+ if (firstActionExecutionException != null) {
+ if (missingCount == 0 && actionFailures == 1) {
+ // In the case a single action failed, just propagate the exception upward. This avoids
+ // having to copy the root causes to the upwards transitive closure.
+ throw firstActionExecutionException;
+ }
+ throw new ActionExecutionException(firstActionExecutionException.getMessage(),
+ firstActionExecutionException.getCause(), action, rootCauses.build(), catastrophe);
+ }
+
+ if (missingCount > 0) {
+ for (Label missingInput : rootCauses.build()) {
+ env.getListener().handle(Event.error(action.getOwner().getLocation(), String.format(
+ "%s: missing input file '%s'", action.getOwner().getLabel(), missingInput)));
+ }
+ throw new ActionExecutionException(missingCount + " input file(s) do not exist", action,
+ rootCauses.build(), /*catastrophe=*/false);
+ }
+ return Pair.of(
+ Collections.unmodifiableMap(inputArtifactData),
+ Collections.unmodifiableMap(expandedMiddlemen));
+ }
+
+ private static void declareAdditionalDependencies(Environment env, Action action) {
+ if (action.discoversInputs()) {
+ // TODO(bazel-team): Should this be all inputs, or just source files?
+ env.getValues(toKeys(Iterables.filter(action.getInputs(), IS_SOURCE_ARTIFACT),
+ action.getMandatoryInputs()));
+ }
+ }
+
+ /**
+ * All info/warning messages associated with actions should be always displayed.
+ */
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link ActionExecutionFunction#compute}.
+ */
+ private static final class ActionExecutionFunctionException extends SkyFunctionException {
+
+ private final ActionExecutionException actionException;
+
+ public ActionExecutionFunctionException(ActionExecutionException e) {
+ // We conservatively assume that the error is transient. We don't have enough information to
+ // distinguish non-transient errors (e.g. compilation error from a deterministic compiler)
+ // from transient ones (e.g. IO error).
+ // TODO(bazel-team): Have ActionExecutionExceptions declare their transience.
+ super(e, Transience.TRANSIENT);
+ this.actionException = e;
+ }
+
+ @Override
+ public boolean isCatastrophic() {
+ return actionException.isCatastrophe();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionInactivityWatchdog.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionInactivityWatchdog.java
new file mode 100644
index 0000000..87e3e0d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionInactivityWatchdog.java
@@ -0,0 +1,180 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * An object that can monitor whether actions are getting completed in a timely manner.
+ *
+ * <p>If there's nothing happening for a while, a background thread will print (and update) the
+ * "Still waiting for N actions to complete..." message.
+ */
+public final class ActionExecutionInactivityWatchdog {
+
+ /** An object used in monitoring action execution inactivity. */
+ public interface InactivityMonitor {
+
+ /** Returns whether action execution has started. */
+ boolean hasStarted();
+
+ /** Returns the number of enqueued but not yet completed actions. */
+ int getPending();
+
+ /**
+ * Waits for any action to complete, or the timeout to elapse.
+ *
+ * <p>The thread must wait at least for the specified timeout, unless some action completes in
+ * the meantime. It's not allowed to return 0 too early.
+ *
+ * <p>Note that it's acceptable to return (any value) later than specified by the timeout.
+ *
+ * @return the number of actions completed during the wait
+ */
+ int waitForNextCompletion(int timeoutMilliseconds) throws InterruptedException;
+ }
+
+ /** An object that the watchdog can report inactivity to. */
+ public interface InactivityReporter {
+
+ /**
+ * Report that actions are not getting completed in a timely manner.
+ *
+ * <p>Inactivity is typically not reported if tests with streaming output are being run.
+ */
+ void maybeReportInactivity();
+ }
+
+ @VisibleForTesting
+ interface Sleep {
+ void sleep(int durationMilliseconds) throws InterruptedException;
+ }
+
+ private static final class WaitTime {
+ private final int progressIntervalFlagValue;
+ private int prev;
+
+ public WaitTime(int progressIntervalFlagValue) {
+ this.progressIntervalFlagValue = progressIntervalFlagValue;
+ }
+
+ public void reset() {
+ prev = 0;
+ }
+
+ public int next() {
+ prev = ActionExecutionStatusReporter.getWaitTime(progressIntervalFlagValue, prev);
+ return prev;
+ }
+ }
+
+ private final AtomicBoolean isRunning = new AtomicBoolean(false);
+ private final InactivityMonitor monitor;
+ private final InactivityReporter reporter;
+ private final Sleep sleeper;
+ private final Thread thread;
+ private final WaitTime waitTime;
+
+ public ActionExecutionInactivityWatchdog(InactivityMonitor monitor, InactivityReporter reporter,
+ int progressIntervalFlagValue) {
+ this(monitor, reporter, progressIntervalFlagValue, new Sleep() {
+ @Override
+ public void sleep(int durationMilliseconds) throws InterruptedException {
+ Thread.sleep(durationMilliseconds);
+ }
+ });
+ }
+
+ @VisibleForTesting
+ public ActionExecutionInactivityWatchdog(InactivityMonitor monitor, InactivityReporter reporter,
+ int progressIntervalFlagValue, Sleep sleeper) {
+ this.monitor = Preconditions.checkNotNull(monitor);
+ this.reporter = Preconditions.checkNotNull(reporter);
+ this.sleeper = Preconditions.checkNotNull(sleeper);
+ this.waitTime = new WaitTime(progressIntervalFlagValue);
+ this.thread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ enterWatchdogLoop();
+ }
+ });
+ this.thread.setDaemon(true);
+ this.thread.setName("action-execution-watchdog");
+ }
+
+ /** Starts the watchdog thread. This method should only be called once. */
+ public void start() {
+ Preconditions.checkState(!isRunning.getAndSet(true));
+ thread.start();
+ }
+
+ /**
+ * Stops the watchdog thread. This method should only be called once.
+ *
+ * <p>The method waits for the thread to terminate. If the caller thread is interrupted
+ * in the meantime, the interrupted status will be set.
+ */
+ public void stop() {
+ Preconditions.checkState(isRunning.getAndSet(false));
+ thread.interrupt();
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ // When Thread.join throws, the interrupted status is cleared. We need to set it again.
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ private void enterWatchdogLoop() {
+ while (isRunning.get()) {
+ try {
+ // Wait a while for any SkyFunction to finish. The returned number indicates how many
+ // actions completed during the wait. It's possible that this is more than 1, since
+ // this thread may not immediately regain control.
+ int completedActions = monitor.waitForNextCompletion(waitTime.next() * 1000);
+ if (!isRunning.get()) {
+ break;
+ }
+
+ int pending = monitor.getPending();
+ if (!monitor.hasStarted() || completedActions > 0 || pending == 0) {
+ // If no keys have been enqueued yet (execution hasn't started), or some actions
+ // were completed since this thread was notified (we are making visible progress),
+ // or there are currently no enqueued actions waiting to be processed (perhaps all
+ // have completed and we are about to stop monitoring), then there's no need to
+ // display any messages.
+ waitTime.reset();
+
+ // Sleep a while before checking again. Actions might be executing at a nice rate, no
+ // need to worry about inactivity. This extra sleep isn't required but it's nice to
+ // have: without it we would, at times of high action completion rate, unnecessarily
+ // put the monitor into a fast sleep-wake cycle --- not a big problem but wasteful.
+ sleeper.sleep(1000);
+ } else {
+ // If actions are executing but we haven't made any progress in a while (no new
+ // action completion), then reassure the user that we're still running. Next time
+ // wait a little longer.
+ reporter.maybeReportInactivity();
+ }
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ return;
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionValue.java
new file mode 100644
index 0000000..de63c3b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionValue.java
@@ -0,0 +1,117 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Action.MiddlemanType;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * A value representing an executed action.
+ */
+@Immutable
+@ThreadSafe
+public class ActionExecutionValue implements SkyValue {
+ private final ImmutableMap<Artifact, FileValue> artifactData;
+ private final ImmutableMap<Artifact, FileArtifactValue> additionalOutputData;
+
+ /**
+ * @param artifactData Map from Artifacts to corresponding FileValues.
+ * @param additionalOutputData Map from Artifacts to values if the FileArtifactValue for this
+ * artifact cannot be derived from the corresponding FileValue (see {@link
+ * FileAndMetadataCache#getAdditionalOutputData} for when this is necessary).
+ */
+ ActionExecutionValue(Map<Artifact, FileValue> artifactData,
+ Map<Artifact, FileArtifactValue> additionalOutputData) {
+ this.artifactData = ImmutableMap.copyOf(artifactData);
+ this.additionalOutputData = ImmutableMap.copyOf(additionalOutputData);
+ }
+
+ /**
+ * Returns metadata for a given artifact, if that metadata cannot be inferred from the
+ * corresponding {@link #getData} call for that Artifact. See {@link
+ * FileAndMetadataCache#getAdditionalOutputData} for when that can happen.
+ */
+ @Nullable
+ FileArtifactValue getArtifactValue(Artifact artifact) {
+ return additionalOutputData.get(artifact);
+ }
+
+ /**
+ * @return The data for each non-middleman output of this action, in the form of the {@link
+ * FileValue} that would be created for the file if it were to be read from disk.
+ */
+ FileValue getData(Artifact artifact) {
+ Preconditions.checkState(!additionalOutputData.containsKey(artifact),
+ "Should not be requesting data for already-constructed FileArtifactValue: %s", artifact);
+ return artifactData.get(artifact);
+ }
+
+ /**
+ * @return The map from {@link Artifact} to the corresponding {@link FileValue} that would be
+ * returned by {@link #getData}. Should only be needed by {@link FilesystemValueChecker}.
+ */
+ ImmutableMap<Artifact, FileValue> getAllOutputArtifactData() {
+ return artifactData;
+ }
+
+ @ThreadSafe
+ @VisibleForTesting
+ public static SkyKey key(Action action) {
+ return new SkyKey(SkyFunctions.ACTION_EXECUTION, action);
+ }
+
+ /**
+ * Returns whether the key corresponds to a ActionExecutionValue worth reporting status about.
+ *
+ * <p>If an action can do real work, it's probably worth counting and reporting status about.
+ * Actions that don't really do any work (typically middleman actions) should not be counted
+ * towards enqueued and completed actions.
+ */
+ public static boolean isReportWorthyAction(SkyKey key) {
+ return key.functionName() == SkyFunctions.ACTION_EXECUTION
+ && isReportWorthyAction((Action) key.argument());
+ }
+
+ /**
+ * Returns whether the action is worth reporting status about.
+ *
+ * <p>If an action can do real work, it's probably worth counting and reporting status about.
+ * Actions that don't really do any work (typically middleman actions) should not be counted
+ * towards enqueued and completed actions.
+ */
+ public static boolean isReportWorthyAction(Action action) {
+ return action.getActionType() == MiddlemanType.NORMAL;
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("artifactData", artifactData)
+ .add("additionalOutputData", additionalOutputData)
+ .toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionLookupValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionLookupValue.java
new file mode 100644
index 0000000..1dfa722
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionLookupValue.java
@@ -0,0 +1,106 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Base class for all values which can provide the generating action of an artifact. The primary
+ * instance of such lookup values is {@link ConfiguredTargetValue}. Values that hold the generating
+ * actions of target completion values and build info artifacts also fall into this category.
+ */
+public class ActionLookupValue implements SkyValue {
+ protected final ImmutableMap<Artifact, Action> generatingActionMap;
+
+ ActionLookupValue(Iterable<Action> actions) {
+ // Duplicate/shared actions get passed in all the time. Blaze is weird. We can't double-register
+ // the generated artifacts in an immutable map builder, so we double-register them in a more
+ // forgiving map, and then use that map to create the immutable one.
+ Map<Artifact, Action> generatingActions = new HashMap<>();
+ for (Action action : actions) {
+ for (Artifact artifact : action.getOutputs()) {
+ generatingActions.put(artifact, action);
+ }
+ }
+ generatingActionMap = ImmutableMap.copyOf(generatingActions);
+ }
+
+ ActionLookupValue(Action action) {
+ this(ImmutableList.of(action));
+ }
+
+ Action getGeneratingAction(Artifact artifact) {
+ return generatingActionMap.get(artifact);
+ }
+
+ /** To be used only when checking consistency of the action graph -- not by other values. */
+ ImmutableMap<Artifact, Action> getMapForConsistencyCheck() {
+ return generatingActionMap;
+ }
+
+ /**
+ * To be used only when setting the owners of deserialized artifacts whose owners were unknown at
+ * creation time -- not by other callers or values.
+ */
+ Iterable<Action> getActionsForFindingArtifactOwners() {
+ return generatingActionMap.values();
+ }
+
+ @VisibleForTesting
+ public static SkyKey key(ActionLookupKey ownerKey) {
+ return ownerKey.getSkyKey();
+ }
+
+ /**
+ * ArtifactOwner is not a SkyKey, but we wish to convert any ArtifactOwner into a SkyKey as
+ * simply as possible. To that end, all subclasses of ActionLookupValue "own" artifacts with
+ * ArtifactOwners that are subclasses of ActionLookupKey. This allows callers to easily find the
+ * value key, while remaining agnostic to what ActionLookupValues actually exist.
+ *
+ * <p>The methods of this class should only be called by {@link ActionLookupValue#key}.
+ */
+ protected abstract static class ActionLookupKey implements ArtifactOwner {
+ @Override
+ public Label getLabel() {
+ return null;
+ }
+
+ /**
+ * Subclasses must override this to specify their specific value type, unless they override
+ * {@link #getSkyKey}, in which case they are free not to implement this method.
+ */
+ abstract SkyFunctionName getType();
+
+ /**
+ * Prefer {@link ActionLookupValue#key} to calling this method directly.
+ *
+ * <p>Subclasses may override if the value key contents should not be the key itself.
+ */
+ SkyKey getSkyKey() {
+ return new SkyKey(getType(), this);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/AggregatingArtifactValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/AggregatingArtifactValue.java
new file mode 100644
index 0000000..8374efe
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/AggregatingArtifactValue.java
@@ -0,0 +1,42 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.util.Pair;
+
+import java.util.Collection;
+
+/** Value for aggregating artifacts, which must be expanded to a set of other artifacts. */
+class AggregatingArtifactValue extends ArtifactValue {
+ private final FileArtifactValue selfData;
+ private final ImmutableList<Pair<Artifact, FileArtifactValue>> inputs;
+
+ AggregatingArtifactValue(ImmutableList<Pair<Artifact, FileArtifactValue>> inputs,
+ FileArtifactValue selfData) {
+ this.inputs = inputs;
+ this.selfData = selfData;
+ }
+
+ /** Returns the artifacts that this artifact expands to, together with their data. */
+ Collection<Pair<Artifact, FileArtifactValue>> getInputs() {
+ return inputs;
+ }
+
+ /** Returns the data of the artifact for this value, as computed by the action cache checker. */
+ FileArtifactValue getSelfData() {
+ return selfData;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java
new file mode 100644
index 0000000..e277476
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java
@@ -0,0 +1,230 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Action.MiddlemanType;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.actions.MissingInputFileException;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.skyframe.ActionLookupValue.ActionLookupKey;
+import com.google.devtools.build.lib.skyframe.ArtifactValue.OwnedArtifact;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * A builder for {@link ArtifactValue}s.
+ */
+class ArtifactFunction implements SkyFunction {
+
+ private final Predicate<PathFragment> allowedMissingInputs;
+
+ ArtifactFunction(Predicate<PathFragment> allowedMissingInputs) {
+ this.allowedMissingInputs = allowedMissingInputs;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws ArtifactFunctionException {
+ OwnedArtifact ownedArtifact = (OwnedArtifact) skyKey.argument();
+ Artifact artifact = ownedArtifact.getArtifact();
+ if (artifact.isSourceArtifact()) {
+ try {
+ return createSourceValue(artifact, ownedArtifact.isMandatory(), env);
+ } catch (MissingInputFileException e) {
+ // The error is not necessarily truly transient, but we mark it as such because we have
+ // the above side effect of posting an event to the EventBus. Importantly, that event
+ // is potentially used to report root causes.
+ throw new ArtifactFunctionException(e, Transience.TRANSIENT);
+ }
+ }
+
+ Action action = extractActionFromArtifact(artifact, env);
+ if (action == null) {
+ return null;
+ }
+
+ ActionExecutionValue actionValue =
+ (ActionExecutionValue) env.getValue(ActionExecutionValue.key(action));
+ if (actionValue == null) {
+ return null;
+ }
+
+ if (!isAggregatingValue(action)) {
+ try {
+ return createSimpleValue(artifact, actionValue);
+ } catch (IOException e) {
+ ActionExecutionException ex = new ActionExecutionException(e, action,
+ /*catastrophe=*/false);
+ env.getListener().handle(Event.error(ex.getLocation(), ex.getMessage()));
+ // This is a transient error since we did the work that led to the IOException.
+ throw new ArtifactFunctionException(ex, Transience.TRANSIENT);
+ }
+ } else {
+ return createAggregatingValue(artifact, action, actionValue.getArtifactValue(artifact), env);
+ }
+ }
+
+ private ArtifactValue createSourceValue(Artifact artifact, boolean mandatory, Environment env)
+ throws MissingInputFileException {
+ SkyKey fileSkyKey = FileValue.key(RootedPath.toRootedPath(artifact.getRoot().getPath(),
+ artifact.getPath()));
+ FileValue fileValue;
+ try {
+ fileValue = (FileValue) env.getValueOrThrow(fileSkyKey, IOException.class,
+ InconsistentFilesystemException.class, FileSymlinkCycleException.class);
+ } catch (IOException | InconsistentFilesystemException | FileSymlinkCycleException e) {
+ throw makeMissingInputFileExn(artifact, mandatory, e, env.getListener());
+ }
+ if (fileValue == null) {
+ return null;
+ }
+ if (!fileValue.exists()) {
+ if (allowedMissingInputs.apply(((RootedPath) fileSkyKey.argument()).getRelativePath())) {
+ return FileArtifactValue.MISSING_FILE_MARKER;
+ } else {
+ return missingInputFile(artifact, mandatory, null, env.getListener());
+ }
+ }
+ try {
+ return FileArtifactValue.create(artifact, fileValue);
+ } catch (IOException e) {
+ throw makeMissingInputFileExn(artifact, mandatory, e, env.getListener());
+ }
+ }
+
+ private static ArtifactValue missingInputFile(Artifact artifact, boolean mandatory,
+ Exception failure, EventHandler reporter) throws MissingInputFileException {
+ if (!mandatory) {
+ return FileArtifactValue.MISSING_FILE_MARKER;
+ }
+ throw makeMissingInputFileExn(artifact, mandatory, failure, reporter);
+ }
+
+ private static MissingInputFileException makeMissingInputFileExn(Artifact artifact,
+ boolean mandatory, Exception failure, EventHandler reporter) {
+ String extraMsg = (failure == null) ? "" : (":" + failure.getMessage());
+ MissingInputFileException ex = new MissingInputFileException(
+ constructErrorMessage(artifact) + extraMsg, null);
+ if (mandatory) {
+ reporter.handle(Event.error(ex.getLocation(), ex.getMessage()));
+ }
+ return ex;
+ }
+
+ // Non-aggregating artifact -- should contain at most one piece of artifact data.
+ // data may be null if and only if artifact is a middleman artifact.
+ private ArtifactValue createSimpleValue(Artifact artifact, ActionExecutionValue actionValue)
+ throws IOException {
+ ArtifactValue value = actionValue.getArtifactValue(artifact);
+ if (value != null) {
+ return value;
+ }
+ // Middleman artifacts have no corresponding files, so their ArtifactValues should have already
+ // been constructed during execution of the action.
+ Preconditions.checkState(!artifact.isMiddlemanArtifact(), artifact);
+ FileValue data = Preconditions.checkNotNull(actionValue.getData(artifact),
+ "%s %s", artifact, actionValue);
+ Preconditions.checkNotNull(data.getDigest(),
+ "Digest should already have been calculated for %s (%s)", artifact, data);
+ return FileArtifactValue.create(artifact, data);
+ }
+
+ private AggregatingArtifactValue createAggregatingValue(Artifact artifact, Action action,
+ FileArtifactValue value, SkyFunction.Environment env) {
+ // This artifact aggregates other artifacts. Keep track of them so callers can find them.
+ ImmutableList.Builder<Pair<Artifact, FileArtifactValue>> inputs = ImmutableList.builder();
+ for (Map.Entry<SkyKey, SkyValue> entry :
+ env.getValues(ArtifactValue.mandatoryKeys(action.getInputs())).entrySet()) {
+ Artifact input = ArtifactValue.artifact(entry.getKey());
+ ArtifactValue inputValue = (ArtifactValue) entry.getValue();
+ Preconditions.checkNotNull(inputValue, "%s has null dep %s", artifact, input);
+ if (!(inputValue instanceof FileArtifactValue)) {
+ // We do not recurse in aggregating middleman artifacts.
+ Preconditions.checkState(!(inputValue instanceof AggregatingArtifactValue),
+ "%s %s %s", artifact, action, inputValue);
+ continue;
+ }
+ inputs.add(Pair.of(input, (FileArtifactValue) inputValue));
+ }
+ return new AggregatingArtifactValue(inputs.build(), value);
+ }
+
+ /**
+ * Returns whether this value needs to contain the data of all its inputs. Currently only tests to
+ * see if the action is an aggregating middleman action. However, may include runfiles middleman
+ * actions and Fileset artifacts in the future.
+ */
+ private static boolean isAggregatingValue(Action action) {
+ return action.getActionType() == MiddlemanType.AGGREGATING_MIDDLEMAN;
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return Label.print(((OwnedArtifact) skyKey.argument()).getArtifact().getOwner());
+ }
+
+ private Action extractActionFromArtifact(Artifact artifact, SkyFunction.Environment env) {
+ ArtifactOwner artifactOwner = artifact.getArtifactOwner();
+
+ Preconditions.checkState(artifactOwner instanceof ActionLookupKey, "", artifact, artifactOwner);
+ SkyKey actionLookupKey = ActionLookupValue.key((ActionLookupKey) artifactOwner);
+ ActionLookupValue value = (ActionLookupValue) env.getValue(actionLookupKey);
+ if (value == null) {
+ Preconditions.checkState(artifactOwner == CoverageReportValue.ARTIFACT_OWNER,
+ "Not-yet-present artifact owner: %s", artifactOwner);
+ return null;
+ }
+ // The value should already exist (except for the coverage report action output artifacts):
+ // ConfiguredTargetValues were created during the analysis phase, and BuildInfo*Values
+ // were created during the first analysis of a configured target.
+ Preconditions.checkNotNull(value,
+ "Owner %s of %s not in graph %s", artifactOwner, artifact, actionLookupKey);
+ return Preconditions.checkNotNull(value.getGeneratingAction(artifact),
+ "Value %s does not contain generating action of %s", value, artifact);
+ }
+
+ private static final class ArtifactFunctionException extends SkyFunctionException {
+ ArtifactFunctionException(MissingInputFileException e, Transience transience) {
+ super(e, transience);
+ }
+
+ ArtifactFunctionException(ActionExecutionException e, Transience transience) {
+ super(e, transience);
+ }
+ }
+
+ private static String constructErrorMessage(Artifact artifact) {
+ if (artifact.getOwner() == null) {
+ return String.format("missing input file '%s'", artifact.getPath().getPathString());
+ } else {
+ return String.format("missing input file '%s'", artifact.getOwner());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactValue.java
new file mode 100644
index 0000000..6139d2e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactValue.java
@@ -0,0 +1,160 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Collection;
+
+/**
+ * A value representing an artifact. Source artifacts are checked for existence, while output
+ * artifacts imply creation of the output file.
+ *
+ * <p>There are effectively two kinds of output artifact values. The first corresponds to an
+ * ordinary artifact {@link FileArtifactValue}. It stores the relevant data for the artifact --
+ * digest/mtime and size. The second corresponds to an "aggregating" artifact -- the output of an
+ * aggregating middleman action. It stores the relevant data of all its inputs.
+ */
+@Immutable
+@ThreadSafe
+public abstract class ArtifactValue implements SkyValue {
+
+ @ThreadSafe
+ static SkyKey key(Artifact artifact, boolean isMandatory) {
+ return new SkyKey(SkyFunctions.ARTIFACT, artifact.isSourceArtifact()
+ ? new OwnedArtifact(artifact, isMandatory)
+ : new OwnedArtifact(artifact));
+ }
+
+ private static final Function<Artifact, SkyKey> TO_MANDATORY_KEY =
+ new Function<Artifact, SkyKey>() {
+ @Override
+ public SkyKey apply(Artifact artifact) {
+ return key(artifact, true);
+ }
+ };
+
+ @ThreadSafe
+ public static Iterable<SkyKey> mandatoryKeys(Iterable<Artifact> artifacts) {
+ return Iterables.transform(artifacts, TO_MANDATORY_KEY);
+ }
+
+ private static final Function<OwnedArtifact, Artifact> TO_ARTIFACT =
+ new Function<OwnedArtifact, Artifact>() {
+ @Override
+ public Artifact apply(OwnedArtifact key) {
+ return key.getArtifact();
+ }
+ };
+
+ public static Collection<Artifact> artifacts(Collection<? extends OwnedArtifact> keys) {
+ return Collections2.transform(keys, TO_ARTIFACT);
+ }
+
+ public static Artifact artifact(SkyKey key) {
+ return TO_ARTIFACT.apply((OwnedArtifact) key.argument());
+ }
+
+ /**
+ * Artifacts are compared using just their paths, but in Skyframe, the configured target that owns
+ * an artifact must also be part of the comparison. For example, suppose we build //foo:foo in
+ * configurationA, yielding artifact foo.out. If we change the configuration to configurationB in
+ * such a way that the path to the artifact does not change, requesting foo.out from the graph
+ * will result in the value entry for foo.out under configurationA being returned. This would
+ * prevent caching the graph in different configurations, and also causes big problems with change
+ * pruning, which assumes the invariant that a value's first dependency will always be the same.
+ * In this case, the value entry's old dependency on //foo:foo in configurationA would cause it to
+ * request (//foo:foo, configurationA) from the graph, causing an undesired re-analysis of
+ * (//foo:foo, configurationA).
+ *
+ * <p>In order to prevent that, instead of using Artifacts as keys in the graph, we use
+ * OwnedArtifacts, which compare for equality using both the Artifact, and the owner. The effect
+ * is functionally that of making Artifact.equals() check the owner, but only within Skyframe,
+ * since outside of Skyframe it is quite crucial that Artifacts with different owners be able to
+ * compare equal.
+ */
+ public static class OwnedArtifact {
+ private final Artifact artifact;
+ // Always true for derived artifacts.
+ private final boolean isMandatory;
+
+ /** Constructs an OwnedArtifact wrapper for a source artifact. */
+ private OwnedArtifact(Artifact sourceArtifact, boolean mandatory) {
+ Preconditions.checkArgument(sourceArtifact.isSourceArtifact());
+ this.artifact = Preconditions.checkNotNull(sourceArtifact);
+ this.isMandatory = mandatory;
+ }
+
+ /**
+ * Constructs an OwnedArtifact wrapper for a derived artifact. The mandatory attribute is
+ * not needed because a derived artifact must be a mandatory input for some action in order to
+ * ensure that it is built in the first place. If it fails to build, then that fact is cached
+ * in the node, so any action that has it as a non-mandatory input can retrieve that
+ * information from the node.
+ */
+ private OwnedArtifact(Artifact derivedArtifact) {
+ this.artifact = Preconditions.checkNotNull(derivedArtifact);
+ Preconditions.checkArgument(!derivedArtifact.isSourceArtifact(), derivedArtifact);
+ this.isMandatory = true; // Unused.
+ }
+
+ @Override
+ public int hashCode() {
+ int initialHash = artifact.hashCode() + artifact.getArtifactOwner().hashCode();
+ return isMandatory ? initialHash : 47 * initialHash + 1;
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (this == that) {
+ return true;
+ }
+ if (!(that instanceof OwnedArtifact)) {
+ return false;
+ }
+ OwnedArtifact thatOwnedArtifact = ((OwnedArtifact) that);
+ Artifact thatArtifact = thatOwnedArtifact.artifact;
+ return artifact.equals(thatArtifact)
+ && artifact.getArtifactOwner().equals(thatArtifact.getArtifactOwner())
+ && isMandatory == thatOwnedArtifact.isMandatory;
+ }
+
+ Artifact getArtifact() {
+ return artifact;
+ }
+
+ /**
+ * Returns whether the artifact is a mandatory input of its requesting action. May only be
+ * called for source artifacts, since a derived artifact must be a mandatory input of some
+ * action in order to have been built in the first place.
+ */
+ public boolean isMandatory() {
+ Preconditions.checkState(artifact.isSourceArtifact(), artifact);
+ return isMandatory;
+ }
+
+ @Override
+ public String toString() {
+ return artifact.prettyPrint() + " " + artifact.getArtifactOwner();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/AspectFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/AspectFunction.java
new file mode 100644
index 0000000..f1aa2f6e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/AspectFunction.java
@@ -0,0 +1,187 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ListMultimap;
+import com.google.devtools.build.lib.analysis.Aspect;
+import com.google.devtools.build.lib.analysis.CachingAnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget;
+import com.google.devtools.build.lib.analysis.TargetAndConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.packages.AspectFactory;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.skyframe.AspectValue.AspectKey;
+import com.google.devtools.build.lib.skyframe.ConfiguredTargetFunction.DependencyEvaluationException;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor.BuildViewProvider;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * The Skyframe function that generates aspects.
+ */
+public final class AspectFunction implements SkyFunction {
+ private final BuildViewProvider buildViewProvider;
+
+ public AspectFunction(BuildViewProvider buildViewProvider) {
+ this.buildViewProvider = buildViewProvider;
+ }
+
+ @Nullable
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env)
+ throws AspectFunctionException {
+ SkyframeBuildView view = buildViewProvider.getSkyframeBuildView();
+ AspectKey key = (AspectKey) skyKey.argument();
+ ConfiguredAspectFactory aspectFactory =
+ (ConfiguredAspectFactory) AspectFactory.Util.create(key.getAspect());
+
+ PackageValue packageValue =
+ (PackageValue) env.getValue(PackageValue.key(key.getLabel().getPackageIdentifier()));
+ if (packageValue == null) {
+ return null;
+ }
+
+ Target target;
+ try {
+ target = packageValue.getPackage().getTarget(key.getLabel().getName());
+ } catch (NoSuchTargetException e) {
+ throw new AspectFunctionException(skyKey, e);
+ }
+
+ if (!(target instanceof Rule)) {
+ throw new AspectFunctionException(new AspectCreationException(
+ "aspects must be attached to rules"));
+ }
+
+ RuleConfiguredTarget associatedTarget = (RuleConfiguredTarget)
+ ((ConfiguredTargetValue) env.getValue(ConfiguredTargetValue.key(
+ key.getLabel(), key.getConfiguration()))).getConfiguredTarget();
+
+ if (associatedTarget == null) {
+ return null;
+ }
+
+ SkyframeDependencyResolver resolver = view.createDependencyResolver(env);
+ if (resolver == null) {
+ return null;
+ }
+
+ TargetAndConfiguration ctgValue =
+ new TargetAndConfiguration(target, key.getConfiguration());
+
+ try {
+ // Get the configuration targets that trigger this rule's configurable attributes.
+ Set<ConfigMatchingProvider> configConditions =
+ ConfiguredTargetFunction.getConfigConditions(target, env, resolver, ctgValue);
+ if (configConditions == null) {
+ // Those targets haven't yet been resolved.
+ return null;
+ }
+
+ ListMultimap<Attribute, ConfiguredTarget> depValueMap =
+ ConfiguredTargetFunction.computeDependencies(env, resolver, ctgValue,
+ aspectFactory.getDefinition(), configConditions);
+
+ return createAspect(env, key, associatedTarget, configConditions, depValueMap);
+ } catch (DependencyEvaluationException e) {
+ throw new AspectFunctionException(e.getRootCauseSkyKey(), e.getCause());
+ }
+ }
+
+ @Nullable
+ private AspectValue createAspect(Environment env, AspectKey key,
+ RuleConfiguredTarget associatedTarget, Set<ConfigMatchingProvider> configConditions,
+ ListMultimap<Attribute, ConfiguredTarget> directDeps)
+ throws AspectFunctionException {
+ SkyframeBuildView view = buildViewProvider.getSkyframeBuildView();
+ BuildConfiguration configuration = associatedTarget.getConfiguration();
+ boolean extendedSanityChecks = configuration != null && configuration.extendedSanityChecks();
+
+ StoredEventHandler events = new StoredEventHandler();
+ CachingAnalysisEnvironment analysisEnvironment = view.createAnalysisEnvironment(
+ key, false, extendedSanityChecks, events, env, true);
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ ConfiguredAspectFactory aspectFactory =
+ (ConfiguredAspectFactory) AspectFactory.Util.create(key.getAspect());
+ Aspect aspect = view.createAspect(
+ analysisEnvironment, associatedTarget, aspectFactory, directDeps, configConditions);
+
+ events.replayOn(env.getListener());
+ if (events.hasErrors()) {
+ analysisEnvironment.disable(associatedTarget.getTarget());
+ throw new AspectFunctionException(new AspectCreationException(
+ "Analysis of target '" + associatedTarget.getLabel() + "' failed; build aborted"));
+ }
+ Preconditions.checkState(!analysisEnvironment.hasErrors(),
+ "Analysis environment hasError() but no errors reported");
+
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ analysisEnvironment.disable(associatedTarget.getTarget());
+ Preconditions.checkNotNull(aspect);
+
+ return new AspectValue(
+ aspect, ImmutableList.copyOf(analysisEnvironment.getRegisteredActions()));
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * An exception indicating that there was a problem creating an aspect.
+ */
+ public static final class AspectCreationException extends Exception {
+ public AspectCreationException(String message) {
+ super(message);
+ }
+ }
+
+ /**
+ * Used to indicate errors during the computation of an {@link AspectValue}.
+ */
+ private static final class AspectFunctionException extends SkyFunctionException {
+ public AspectFunctionException(Exception e) {
+ super(e, Transience.PERSISTENT);
+ }
+
+ /** Used to rethrow a child error that we cannot handle. */
+ public AspectFunctionException(SkyKey childKey, Exception transitiveError) {
+ super(transitiveError, childKey);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/AspectValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/AspectValue.java
new file mode 100644
index 0000000..9b863bd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/AspectValue.java
@@ -0,0 +1,109 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Objects;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.analysis.Aspect;
+import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+
+/**
+ * An aspect in the context of the Skyframe graph.
+ */
+public final class AspectValue extends ActionLookupValue {
+ /**
+ * The key of an action that is generated by an aspect.
+ */
+ public static final class AspectKey extends ActionLookupKey {
+ private final Label label;
+ private final BuildConfiguration configuration;
+ // TODO(bazel-team): class objects are not really hashable or comparable for equality other than
+ // by reference. We should identify the aspect here in a way that does not rely on comparison
+ // by reference so that keys can be serialized and deserialized properly.
+ private final Class<? extends ConfiguredAspectFactory> aspectFactory;
+
+ private AspectKey(Label label, BuildConfiguration configuration,
+ Class<? extends ConfiguredAspectFactory> aspectFactory) {
+ this.label = label;
+ this.configuration = configuration;
+ this.aspectFactory = aspectFactory;
+ }
+
+ @Override
+ public Label getLabel() {
+ return label;
+ }
+
+ public BuildConfiguration getConfiguration() {
+ return configuration;
+ }
+
+ public Class<? extends ConfiguredAspectFactory> getAspect() {
+ return aspectFactory;
+ }
+
+ @Override
+ SkyFunctionName getType() {
+ return SkyFunctions.ASPECT;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(label, configuration, aspectFactory);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+
+ if (!(other instanceof AspectKey)) {
+ return false;
+ }
+
+ AspectKey that = (AspectKey) other;
+ return Objects.equal(label, that.label)
+ && Objects.equal(configuration, that.configuration)
+ && Objects.equal(aspectFactory, that.aspectFactory);
+ }
+
+ @Override
+ public String toString() {
+ return label + "#" + aspectFactory.getSimpleName() + " "
+ + (configuration == null ? "null" : configuration.shortCacheKey());
+ }
+ }
+
+ private final Aspect aspect;
+
+ public AspectValue(Aspect aspect, Iterable<Action> actions) {
+ super(actions);
+ this.aspect = aspect;
+ }
+
+ public Aspect get() {
+ return aspect;
+ }
+
+ public static SkyKey key(Label label, BuildConfiguration configuration,
+ Class<? extends ConfiguredAspectFactory> aspectFactory) {
+ return new SkyKey(SkyFunctions.ASPECT, new AspectKey(label, configuration, aspectFactory));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BrokenDiffAwarenessException.java b/src/main/java/com/google/devtools/build/lib/skyframe/BrokenDiffAwarenessException.java
new file mode 100644
index 0000000..a5b0272
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/BrokenDiffAwarenessException.java
@@ -0,0 +1,27 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Thrown on {@link DiffAwareness#getDiff} to indicate that something is wrong with the
+ * {@link DiffAwareness} instance and it should not be used again.
+ */
+public class BrokenDiffAwarenessException extends Exception {
+
+ public BrokenDiffAwarenessException(String msg) {
+ super(Preconditions.checkNotNull(msg));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BuildInfoCollectionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/BuildInfoCollectionFunction.java
new file mode 100644
index 0000000..e717e51
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/BuildInfoCollectionFunction.java
@@ -0,0 +1,86 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Supplier;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactFactory;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoContext;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoType;
+import com.google.devtools.build.lib.skyframe.BuildInfoCollectionValue.BuildInfoKeyAndConfig;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Map;
+
+/**
+ * Creates a {@link BuildInfoCollectionValue}. Only depends on the unique
+ * {@link WorkspaceStatusValue} and the constant {@link PrecomputedValue#BUILD_INFO_FACTORIES}
+ * injected value.
+ */
+public class BuildInfoCollectionFunction implements SkyFunction {
+ // Supplier only because the artifact factory has not yet been created at constructor time.
+ private final Supplier<ArtifactFactory> artifactFactory;
+ private final Root buildDataDirectory;
+
+ BuildInfoCollectionFunction(Supplier<ArtifactFactory> artifactFactory,
+ Root buildDataDirectory) {
+ this.artifactFactory = artifactFactory;
+ this.buildDataDirectory = buildDataDirectory;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ final BuildInfoKeyAndConfig keyAndConfig = (BuildInfoKeyAndConfig) skyKey.argument();
+ WorkspaceStatusValue infoArtifactValue =
+ (WorkspaceStatusValue) env.getValue(WorkspaceStatusValue.SKY_KEY);
+ if (infoArtifactValue == null) {
+ return null;
+ }
+ Map<BuildInfoKey, BuildInfoFactory> buildInfoFactories =
+ PrecomputedValue.BUILD_INFO_FACTORIES.get(env);
+ if (buildInfoFactories == null) {
+ return null;
+ }
+ final ArtifactFactory factory = artifactFactory.get();
+ BuildInfoContext context = new BuildInfoContext() {
+ @Override
+ public Artifact getBuildInfoArtifact(PathFragment rootRelativePath, Root root,
+ BuildInfoType type) {
+ return type == BuildInfoType.NO_REBUILD
+ ? factory.getConstantMetadataArtifact(rootRelativePath, root, keyAndConfig)
+ : factory.getDerivedArtifact(rootRelativePath, root, keyAndConfig);
+ }
+
+ @Override
+ public Root getBuildDataDirectory() {
+ return buildDataDirectory;
+ }
+ };
+
+ return new BuildInfoCollectionValue(buildInfoFactories.get(
+ keyAndConfig.getInfoKey()).create(context, keyAndConfig.getConfig(),
+ infoArtifactValue.getStableArtifact(), infoArtifactValue.getVolatileArtifact()));
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BuildInfoCollectionValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/BuildInfoCollectionValue.java
new file mode 100644
index 0000000..8958e1b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/BuildInfoCollectionValue.java
@@ -0,0 +1,97 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoCollection;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+
+import java.util.Objects;
+
+/**
+ * Value that stores {@link BuildInfoCollection}s generated by {@link BuildInfoFactory} instances.
+ * These collections are used during analysis (see {@code CachingAnalysisEnvironment}).
+ */
+public class BuildInfoCollectionValue extends ActionLookupValue {
+ private final BuildInfoCollection collection;
+
+ BuildInfoCollectionValue(BuildInfoCollection collection) {
+ super(collection.getActions());
+ this.collection = collection;
+ }
+
+ public BuildInfoCollection getCollection() {
+ return collection;
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public String toString() {
+ return com.google.common.base.Objects.toStringHelper(getClass())
+ .add("collection", collection)
+ .add("generatingActionMap", generatingActionMap).toString();
+ }
+
+ /** Key for BuildInfoCollectionValues. */
+ public static class BuildInfoKeyAndConfig extends ActionLookupKey {
+ private final BuildInfoFactory.BuildInfoKey infoKey;
+ private final BuildConfiguration config;
+
+ public BuildInfoKeyAndConfig(BuildInfoFactory.BuildInfoKey key, BuildConfiguration config) {
+ this.infoKey = Preconditions.checkNotNull(key, config);
+ this.config = Preconditions.checkNotNull(config, key);
+ }
+
+ @Override
+ SkyFunctionName getType() {
+ return SkyFunctions.BUILD_INFO_COLLECTION;
+ }
+
+ BuildInfoFactory.BuildInfoKey getInfoKey() {
+ return infoKey;
+ }
+
+ BuildConfiguration getConfig() {
+ return config;
+ }
+
+ @Override
+ public Label getLabel() {
+ return null;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(infoKey, config);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null) {
+ return false;
+ }
+ if (this.getClass() != other.getClass()) {
+ return false;
+ }
+ BuildInfoKeyAndConfig that = (BuildInfoKeyAndConfig) other;
+ return Objects.equals(this.infoKey, that.infoKey) && Objects.equals(this.config, that.config);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java b/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java
new file mode 100644
index 0000000..7fdb55c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java
@@ -0,0 +1,75 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.BuildFailedException;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.TestExecException;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.util.AbruptExitException;
+
+import java.util.Collection;
+import java.util.Set;
+
+/**
+ * A Builder consumes top-level artifacts, targets, and tests,, and executes them in some
+ * topological order, possibly concurrently, using some dependency-checking policy.
+ *
+ * <p> The methods of the Builder interface are typically long-running, but honor the
+ * {@link java.lang.Thread#interrupt} contract: if an interrupt is delivered to the thread in which
+ * a call to buildTargets or buildArtifacts is active, the Builder attempts to terminate the call
+ * prematurely, throwing InterruptedException. No guarantee is made about the timeliness of such
+ * termination, as it depends on the ability of the Actions being executed to be interrupted, but
+ * typically any running subprocesses will be quickly killed.
+ */
+public interface Builder {
+
+ /**
+ * Transitively build all given artifacts, targets, and tests, and all necessary prerequisites
+ * thereof. For sequential implementations of this interface, the top-level requests will be
+ * built in the iteration order of the Set provided; for concurrent implementations, the order
+ * is undefined.
+ *
+ * <p>This method should not be invoked more than once concurrently on the same Builder instance.
+ *
+ * @param artifacts the set of Artifacts to build
+ * @param parallelTests tests to execute in parallel with the other top-level targetsToBuild and
+ * artifacts.
+ * @param exclusiveTests are executed one at a time, only after all other tasks have completed
+ * @param targetsToBuild Set of targets which will be built
+ * @param executor an opaque application-specific value that will be
+ * passed down to the execute() method of any Action executed during
+ * this call
+ * @param builtTargets (out) set of successfully built subset of targetsToBuild. This set is
+ * populated immediately upon confirmation that artifact is built so it will be
+ * valid even if a future action throws ActionExecutionException
+ * @throws BuildFailedException if there were problems establishing the action execution
+ * environment, if the the metadata of any file during the build could not be obtained,
+ * if any input files are missing, or if an action fails during execution
+ * @throws InterruptedException if there was an asynchronous stop request
+ * @throws TestExecException if any test fails
+ */
+ @ThreadCompatible
+ void buildArtifacts(Set<Artifact> artifacts,
+ Set<ConfiguredTarget> parallelTests,
+ Set<ConfiguredTarget> exclusiveTests,
+ Collection<ConfiguredTarget> targetsToBuild,
+ Executor executor,
+ Set<ConfiguredTarget> builtTargets,
+ boolean explain)
+ throws BuildFailedException, AbruptExitException, InterruptedException, TestExecException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationCollectionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationCollectionFunction.java
new file mode 100644
index 0000000..89828c3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationCollectionFunction.java
@@ -0,0 +1,164 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Supplier;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationKey;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigurationFactory;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.analysis.config.PackageProviderForConfigurations;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.skyframe.ConfigurationCollectionValue.ConfigurationCollectionKey;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A builder for {@link ConfigurationCollectionValue} instances.
+ */
+public class ConfigurationCollectionFunction implements SkyFunction {
+
+ private final Supplier<ConfigurationFactory> configurationFactory;
+ private final Supplier<Map<String, String>> clientEnv;
+ private final Supplier<Set<Package>> configurationPackages;
+
+ public ConfigurationCollectionFunction(
+ Supplier<ConfigurationFactory> configurationFactory,
+ Supplier<Map<String, String>> clientEnv,
+ Supplier<Set<Package>> configurationPackages) {
+ this.configurationFactory = configurationFactory;
+ this.clientEnv = clientEnv;
+ this.configurationPackages = configurationPackages;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException,
+ ConfigurationCollectionFunctionException {
+ ConfigurationCollectionKey collectionKey = (ConfigurationCollectionKey) skyKey.argument();
+ try {
+ // We are not using this value, because test_environment can be created from clientEnv. But
+ // we want ConfigurationCollection to be recomputed each time when test_environment changes.
+ PrecomputedValue.TEST_ENVIRONMENT_VARIABLES.get(env);
+ BlazeDirectories directories = PrecomputedValue.BLAZE_DIRECTORIES.get(env);
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ BuildConfigurationCollection result =
+ getConfigurations(env.getListener(),
+ new SkyframePackageLoaderWithValueEnvironment(env, configurationPackages.get()),
+ new BuildConfigurationKey(collectionKey.getBuildOptions(), directories, clientEnv.get(),
+ collectionKey.getMultiCpu()));
+
+ // BuildConfigurationCollection can be created, but dependencies to some files might be
+ // missing. In that case we need to build configurationCollection again.
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ for (BuildConfiguration config : result.getTargetConfigurations()) {
+ config.declareSkyframeDependencies(env);
+ }
+ if (env.valuesMissing()) {
+ return null;
+ }
+ return new ConfigurationCollectionValue(result, configurationPackages.get());
+ } catch (InvalidConfigurationException e) {
+ throw new ConfigurationCollectionFunctionException(e);
+ }
+ }
+
+ /** Create the build configurations with the given options. */
+ private BuildConfigurationCollection getConfigurations(EventHandler eventHandler,
+ PackageProviderForConfigurations loadedPackageProvider, BuildConfigurationKey key)
+ throws InvalidConfigurationException {
+ List<BuildConfiguration> targetConfigurations = new ArrayList<>();
+ if (!key.getMultiCpu().isEmpty()) {
+ for (String cpu : key.getMultiCpu()) {
+ BuildConfiguration targetConfiguration = createConfiguration(
+ eventHandler, loadedPackageProvider, key, cpu);
+ if (targetConfiguration == null || targetConfigurations.contains(targetConfiguration)) {
+ continue;
+ }
+ targetConfigurations.add(targetConfiguration);
+ }
+ if (loadedPackageProvider.valuesMissing()) {
+ return null;
+ }
+ } else {
+ BuildConfiguration targetConfiguration = createConfiguration(
+ eventHandler, loadedPackageProvider, key, null);
+ if (targetConfiguration == null) {
+ return null;
+ }
+ targetConfigurations.add(targetConfiguration);
+ }
+ return new BuildConfigurationCollection(targetConfigurations);
+ }
+
+ @Nullable
+ public BuildConfiguration createConfiguration(
+ EventHandler originalEventListener,
+ PackageProviderForConfigurations loadedPackageProvider,
+ BuildConfigurationKey key, String cpuOverride) throws InvalidConfigurationException {
+ StoredEventHandler errorEventListener = new StoredEventHandler();
+ BuildOptions buildOptions = key.getBuildOptions();
+ if (cpuOverride != null) {
+ // TODO(bazel-team): Options classes should be immutable. This is a bit of a hack.
+ buildOptions = buildOptions.clone();
+ buildOptions.get(BuildConfiguration.Options.class).cpu = cpuOverride;
+ }
+
+ BuildConfiguration targetConfig = configurationFactory.get().createConfiguration(
+ loadedPackageProvider, buildOptions, key, errorEventListener);
+ if (targetConfig == null) {
+ return null;
+ }
+ errorEventListener.replayOn(originalEventListener);
+ if (errorEventListener.hasErrors()) {
+ throw new InvalidConfigurationException("Build options are invalid");
+ }
+ return targetConfig;
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link ConfigurationCollectionFunction#compute}.
+ */
+ private static final class ConfigurationCollectionFunctionException extends
+ SkyFunctionException {
+ public ConfigurationCollectionFunctionException(InvalidConfigurationException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationCollectionValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationCollectionValue.java
new file mode 100644
index 0000000..30e4fd7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationCollectionValue.java
@@ -0,0 +1,100 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.Serializable;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * A Skyframe value representing a build configuration collection.
+ */
+@Immutable
+@ThreadSafe
+public class ConfigurationCollectionValue implements SkyValue {
+
+ private final BuildConfigurationCollection configurationCollection;
+ private final ImmutableSet<Package> configurationPackages;
+
+ ConfigurationCollectionValue(BuildConfigurationCollection configurationCollection,
+ Set<Package> configurationPackages) {
+ this.configurationCollection = Preconditions.checkNotNull(configurationCollection);
+ this.configurationPackages = ImmutableSet.copyOf(configurationPackages);
+ }
+
+ public BuildConfigurationCollection getConfigurationCollection() {
+ return configurationCollection;
+ }
+
+ /**
+ * Returns set of packages required for configuration.
+ */
+ public Set<Package> getConfigurationPackages() {
+ return configurationPackages;
+ }
+
+ @ThreadSafe
+ public static SkyKey key(BuildOptions buildOptions, ImmutableSet<String> multiCpu) {
+ return new SkyKey(SkyFunctions.CONFIGURATION_COLLECTION,
+ new ConfigurationCollectionKey(buildOptions, multiCpu));
+ }
+
+ static final class ConfigurationCollectionKey implements Serializable {
+ private final BuildOptions buildOptions;
+ private final ImmutableSet<String> multiCpu;
+ private final int hashCode;
+
+ public ConfigurationCollectionKey(BuildOptions buildOptions, ImmutableSet<String> multiCpu) {
+ this.buildOptions = Preconditions.checkNotNull(buildOptions);
+ this.multiCpu = Preconditions.checkNotNull(multiCpu);
+ this.hashCode = Objects.hash(buildOptions, multiCpu);
+ }
+
+ public BuildOptions getBuildOptions() {
+ return buildOptions;
+ }
+
+ public ImmutableSet<String> getMultiCpu() {
+ return multiCpu;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof ConfigurationCollectionKey)) {
+ return false;
+ }
+ ConfigurationCollectionKey confObject = (ConfigurationCollectionKey) o;
+ return Objects.equals(multiCpu, confObject.multiCpu)
+ && Objects.equals(buildOptions, confObject.buildOptions);
+ }
+
+ @Override
+ public int hashCode() {
+ return hashCode;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationFragmentFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationFragmentFunction.java
new file mode 100644
index 0000000..0393b16
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationFragmentFunction.java
@@ -0,0 +1,146 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment;
+import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.analysis.config.PackageProviderForConfigurations;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.skyframe.ConfigurationFragmentValue.ConfigurationFragmentKey;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.util.Set;
+
+/**
+ * A builder for {@link ConfigurationFragmentValue}s.
+ */
+public class ConfigurationFragmentFunction implements SkyFunction {
+
+ private final Supplier<ImmutableList<ConfigurationFragmentFactory>> configurationFragments;
+ private final Supplier<Set<Package>> configurationPackages;
+
+ public ConfigurationFragmentFunction(
+ Supplier<ImmutableList<ConfigurationFragmentFactory>> configurationFragments,
+ Supplier<Set<Package>> configurationPackages) {
+ this.configurationFragments = configurationFragments;
+ this.configurationPackages = configurationPackages;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException,
+ ConfigurationFragmentFunctionException {
+ ConfigurationFragmentKey configurationFragmentKey =
+ (ConfigurationFragmentKey) skyKey.argument();
+ BuildOptions buildOptions = configurationFragmentKey.getBuildOptions();
+ ConfigurationFragmentFactory factory = getFactory(configurationFragmentKey.getFragmentType());
+ try {
+ PackageProviderForConfigurations loadedPackageProvider =
+ new SkyframePackageLoaderWithValueEnvironment(env, configurationPackages.get());
+ ConfigurationEnvironment confEnv = new ConfigurationBuilderEnvironment(loadedPackageProvider);
+ Fragment fragment = factory.create(confEnv, buildOptions);
+
+ if (env.valuesMissing()) {
+ return null;
+ }
+ return new ConfigurationFragmentValue(fragment);
+ } catch (InvalidConfigurationException e) {
+ // TODO(bazel-team): Rework the control-flow here so that we're not actually throwing this
+ // exception with missing Skyframe dependencies.
+ if (env.valuesMissing()) {
+ return null;
+ }
+ throw new ConfigurationFragmentFunctionException(e);
+ }
+ }
+
+ private ConfigurationFragmentFactory getFactory(Class<? extends Fragment> fragmentType) {
+ for (ConfigurationFragmentFactory factory : configurationFragments.get()) {
+ if (factory.creates().equals(fragmentType)) {
+ return factory;
+ }
+ }
+ throw new IllegalStateException(
+ "There is no factory for fragment: " + fragmentType.getSimpleName());
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * A {@link ConfigurationEnvironment} implementation that can create dependencies on files.
+ */
+ private final class ConfigurationBuilderEnvironment implements ConfigurationEnvironment {
+ private final PackageProviderForConfigurations loadedPackageProvider;
+
+ ConfigurationBuilderEnvironment(
+ PackageProviderForConfigurations loadedPackageProvider) {
+ this.loadedPackageProvider = loadedPackageProvider;
+ }
+
+ @Override
+ public Target getTarget(Label label) throws NoSuchPackageException, NoSuchTargetException {
+ return loadedPackageProvider.getLoadedTarget(label);
+ }
+
+ @Override
+ public Path getPath(Package pkg, String fileName) {
+ Path result = pkg.getPackageDirectory().getRelative(fileName);
+ try {
+ loadedPackageProvider.addDependency(pkg, fileName);
+ } catch (IOException | SyntaxException e) {
+ return null;
+ }
+ return result;
+ }
+
+ @Override
+ public <T extends Fragment> T getFragment(BuildOptions buildOptions, Class<T> fragmentType)
+ throws InvalidConfigurationException {
+ return loadedPackageProvider.getFragment(buildOptions, fragmentType);
+ }
+
+ @Override
+ public BlazeDirectories getBlazeDirectories() {
+ return loadedPackageProvider.getDirectories();
+ }
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link ConfigurationFragmentFunction#compute}.
+ */
+ private static final class ConfigurationFragmentFunctionException extends SkyFunctionException {
+ public ConfigurationFragmentFunctionException(InvalidConfigurationException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationFragmentValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationFragmentValue.java
new file mode 100644
index 0000000..cc07216
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationFragmentValue.java
@@ -0,0 +1,90 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * A Skyframe node representing a build configuration fragment.
+ */
+@Immutable
+@ThreadSafe
+public class ConfigurationFragmentValue implements SkyValue {
+
+ @Nullable
+ private final BuildConfiguration.Fragment fragment;
+
+ ConfigurationFragmentValue(BuildConfiguration.Fragment fragment) {
+ this.fragment = fragment;
+ }
+
+ public BuildConfiguration.Fragment getFragment() {
+ return fragment;
+ }
+
+ @ThreadSafe
+ public static SkyKey key(BuildOptions buildOptions, Class<? extends Fragment> fragmentType) {
+ return new SkyKey(SkyFunctions.CONFIGURATION_FRAGMENT,
+ new ConfigurationFragmentKey(buildOptions, fragmentType));
+ }
+
+ static final class ConfigurationFragmentKey implements Serializable {
+ private final BuildOptions buildOptions;
+ private final Class<? extends Fragment> fragmentType;
+
+ public ConfigurationFragmentKey(BuildOptions buildOptions,
+ Class<? extends Fragment> fragmentType) {
+ this.buildOptions = Preconditions.checkNotNull(buildOptions);
+ this.fragmentType = Preconditions.checkNotNull(fragmentType);
+ }
+
+ public BuildOptions getBuildOptions() {
+ return buildOptions;
+ }
+
+ public Class<? extends Fragment> getFragmentType() {
+ return fragmentType;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof ConfigurationFragmentKey)) {
+ return false;
+ }
+ ConfigurationFragmentKey confObject = (ConfigurationFragmentKey) o;
+ return Objects.equals(fragmentType, confObject.fragmentType)
+ && Objects.equals(buildOptions, confObject.buildOptions);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(buildOptions, fragmentType);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java
new file mode 100644
index 0000000..9fc3df4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java
@@ -0,0 +1,578 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.ListMultimap;
+import com.google.devtools.build.lib.analysis.Aspect;
+import com.google.devtools.build.lib.analysis.CachingAnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.DependencyResolver.Dependency;
+import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget;
+import com.google.devtools.build.lib.analysis.TargetAndConfiguration;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.packages.AspectDefinition;
+import com.google.devtools.build.lib.packages.AspectFactory;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.InputFile;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.PackageGroup;
+import com.google.devtools.build.lib.packages.RawAttributeMapper;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.skyframe.AspectFunction.AspectCreationException;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor.BuildViewProvider;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import com.google.devtools.build.skyframe.ValueOrException2;
+import com.google.devtools.build.skyframe.ValueOrException3;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * SkyFunction for {@link ConfiguredTargetValue}s.
+ */
+final class ConfiguredTargetFunction implements SkyFunction {
+
+ /**
+ * Exception class that signals an error during the evaluation of a dependency.
+ */
+ public static class DependencyEvaluationException extends Exception {
+ private final SkyKey rootCauseSkyKey;
+
+ public DependencyEvaluationException(Exception cause) {
+ super(cause);
+ this.rootCauseSkyKey = null;
+ }
+
+ public DependencyEvaluationException(SkyKey rootCauseSkyKey, Exception cause) {
+ super(cause);
+ this.rootCauseSkyKey = rootCauseSkyKey;
+ }
+
+ /**
+ * Returns the key of the root cause or null if the problem was with this target.
+ */
+ public SkyKey getRootCauseSkyKey() {
+ return rootCauseSkyKey;
+ }
+
+ @Override
+ public Exception getCause() {
+ return (Exception) super.getCause();
+ }
+ }
+
+ private static final Function<Dependency, SkyKey> TO_KEYS =
+ new Function<Dependency, SkyKey>() {
+ @Override
+ public SkyKey apply(Dependency input) {
+ return ConfiguredTargetValue.key(input.getLabel(), input.getConfiguration());
+ }
+ };
+
+ private final BuildViewProvider buildViewProvider;
+
+ ConfiguredTargetFunction(BuildViewProvider buildViewProvider) {
+ this.buildViewProvider = buildViewProvider;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey key, Environment env) throws ConfiguredTargetFunctionException,
+ InterruptedException {
+ SkyframeBuildView view = buildViewProvider.getSkyframeBuildView();
+
+ ConfiguredTargetKey configuredTargetKey = (ConfiguredTargetKey) key.argument();
+ LabelAndConfiguration lc = LabelAndConfiguration.of(
+ configuredTargetKey.getLabel(), configuredTargetKey.getConfiguration());
+
+ BuildConfiguration configuration = lc.getConfiguration();
+
+ PackageValue packageValue =
+ (PackageValue) env.getValue(PackageValue.key(lc.getLabel().getPackageIdentifier()));
+ if (packageValue == null) {
+ return null;
+ }
+
+ Target target;
+ try {
+ target = packageValue.getPackage().getTarget(lc.getLabel().getName());
+ } catch (NoSuchTargetException e1) {
+ throw new ConfiguredTargetFunctionException(new NoSuchTargetException(lc.getLabel(),
+ "No such target"));
+ }
+ // TODO(bazel-team): This is problematic - we create the right key, but then end up with a value
+ // that doesn't match; we can even have the same value multiple times. However, I think it's
+ // only triggered in tests (i.e., in normal operation, the configuration passed in is already
+ // null).
+ if (target instanceof InputFile) {
+ // InputFileConfiguredTarget expects its configuration to be null since it's not used.
+ configuration = null;
+ } else if (target instanceof PackageGroup) {
+ // Same for PackageGroupConfiguredTarget.
+ configuration = null;
+ }
+ TargetAndConfiguration ctgValue =
+ new TargetAndConfiguration(target, configuration);
+
+ SkyframeDependencyResolver resolver = view.createDependencyResolver(env);
+ if (resolver == null) {
+ return null;
+ }
+
+ try {
+ // Get the configuration targets that trigger this rule's configurable attributes.
+ Set<ConfigMatchingProvider> configConditions =
+ getConfigConditions(ctgValue.getTarget(), env, resolver, ctgValue);
+ if (configConditions == null) {
+ // Those targets haven't yet been resolved.
+ return null;
+ }
+
+ ListMultimap<Attribute, ConfiguredTarget> depValueMap =
+ computeDependencies(env, resolver, ctgValue, null, configConditions);
+ return createConfiguredTarget(
+ view, env, target, configuration, depValueMap, configConditions);
+ } catch (DependencyEvaluationException e) {
+ throw new ConfiguredTargetFunctionException(e.getRootCauseSkyKey(), e.getCause());
+ }
+ }
+
+ /**
+ * Computes the direct dependencies of a node in the configured target graph (a configured
+ * target or an aspect).
+ *
+ * <p>Returns null if Skyframe hasn't evaluated the required dependencies yet. In this case, the
+ * caller should also return null to Skyframe.
+ *
+ * @param env the Skyframe environment
+ * @param resolver The dependency resolver
+ * @param ctgValue The label and the configuration of the node
+ * @param aspectDefinition the aspect of the node (if null, the node is a configured target,
+ * otherwise it's an asect)
+ * @param configConditions the configuration conditions for evaluating the attributes of the node
+ * @return an attribute -> direct dependency multimap
+ * @throws ConfiguredTargetFunctionException
+ */
+ @Nullable
+ static ListMultimap<Attribute, ConfiguredTarget> computeDependencies(
+ Environment env, SkyframeDependencyResolver resolver, TargetAndConfiguration ctgValue,
+ AspectDefinition aspectDefinition, Set<ConfigMatchingProvider> configConditions)
+ throws DependencyEvaluationException {
+
+ // 1. Create the map from attributes to list of (target, configuration) pairs.
+ ListMultimap<Attribute, Dependency> depValueNames;
+ try {
+ depValueNames = resolver.dependentNodeMap(ctgValue, aspectDefinition, configConditions);
+ } catch (EvalException e) {
+ env.getListener().handle(Event.error(e.getLocation(), e.getMessage()));
+ throw new DependencyEvaluationException(new ConfiguredValueCreationException(e.print()));
+ }
+
+ // 2. Resolve configured target dependencies and handle errors.
+ Map<SkyKey, ConfiguredTarget> depValues =
+ resolveConfiguredTargetDependencies(env, depValueNames.values(), ctgValue.getTarget());
+ if (depValues == null) {
+ return null;
+ }
+
+ // 3. Resolve required aspects.
+ ListMultimap<SkyKey, Aspect> depAspects = resolveAspectDependencies(
+ env, depValues, depValueNames.values());
+ if (depAspects == null) {
+ return null;
+ }
+
+ // 3. Merge the dependent configured targets and aspects into a single map.
+ return mergeAspects(depValueNames, depValues, depAspects);
+ }
+
+ /**
+ * Merges the each direct dependency configured target with the aspects associated with it.
+ *
+ * <p>Note that the combination of a configured target and its associated aspects are not
+ * represented by a Skyframe node. This is because there can possibly be many different
+ * combinations of aspects for a particular configured target, so it would result in a
+ * combinatiorial explosion of Skyframe nodes.
+ */
+ private static ListMultimap<Attribute, ConfiguredTarget> mergeAspects(
+ ListMultimap<Attribute, Dependency> depValueNames,
+ Map<SkyKey, ConfiguredTarget> depConfiguredTargetMap,
+ ListMultimap<SkyKey, Aspect> depAspectMap) {
+ ListMultimap<Attribute, ConfiguredTarget> result = ArrayListMultimap.create();
+
+ for (Map.Entry<Attribute, Dependency> entry : depValueNames.entries()) {
+ Dependency dep = entry.getValue();
+ SkyKey depKey = TO_KEYS.apply(dep);
+ ConfiguredTarget depConfiguredTarget = depConfiguredTargetMap.get(depKey);
+ result.put(entry.getKey(),
+ RuleConfiguredTarget.mergeAspects(depConfiguredTarget, depAspectMap.get(depKey)));
+ }
+
+ return result;
+ }
+
+ /**
+ * Given a list of {@link Dependency} objects, returns a multimap from the {@link SkyKey} of the
+ * dependency to the {@link Aspect} instances that should be merged into it.
+ *
+ * <p>Returns null if the required aspects are not computed yet.
+ */
+ @Nullable
+ private static ListMultimap<SkyKey, Aspect> resolveAspectDependencies(Environment env,
+ Map<SkyKey, ConfiguredTarget> configuredTargetMap, Iterable<Dependency> deps)
+ throws DependencyEvaluationException {
+ ListMultimap<SkyKey, Aspect> result = ArrayListMultimap.create();
+ Set<SkyKey> aspectKeys = new HashSet<>();
+ for (Dependency dep : deps) {
+ for (Class<? extends ConfiguredAspectFactory> depAspect : dep.getAspects()) {
+ aspectKeys.add(AspectValue.key(dep.getLabel(), dep.getConfiguration(), depAspect));
+ }
+ }
+
+ Map<SkyKey, ValueOrException3<
+ AspectCreationException, NoSuchThingException, ConfiguredValueCreationException>>
+ depAspects = env.getValuesOrThrow(aspectKeys, AspectCreationException.class,
+ NoSuchThingException.class, ConfiguredValueCreationException.class);
+
+ for (Dependency dep : deps) {
+ SkyKey depKey = TO_KEYS.apply(dep);
+ ConfiguredTarget depConfiguredTarget = configuredTargetMap.get(depKey);
+ List<AspectValue> aspects = new ArrayList<>();
+ for (Class<? extends ConfiguredAspectFactory> depAspect : dep.getAspects()) {
+ if (!aspectMatchesConfiguredTarget(depConfiguredTarget, depAspect)) {
+ continue;
+ }
+
+ SkyKey aspectKey = AspectValue.key(dep.getLabel(), dep.getConfiguration(), depAspect);
+ AspectValue aspectValue = null;
+ try {
+ aspectValue = (AspectValue) depAspects.get(aspectKey).get();
+ } catch (ConfiguredValueCreationException e) {
+ // The configured target should have been created in resolveConfiguredTargetDependencies()
+ throw new IllegalStateException(e);
+ } catch (NoSuchThingException | AspectCreationException e) {
+ AspectFactory depAspectFactory = AspectFactory.Util.create(depAspect);
+ throw new DependencyEvaluationException(new ConfiguredValueCreationException(
+ String.format("Evaluation of aspect %s on %s failed: %s",
+ depAspectFactory.getDefinition().getName(), dep.getLabel(), e.toString())));
+ }
+
+ if (aspectValue == null) {
+ // Dependent aspect has either not been computed yet or is in error.
+ return null;
+ }
+ result.put(depKey, aspectValue.get());
+ }
+ }
+
+ return result;
+ }
+
+ private static boolean aspectMatchesConfiguredTarget(ConfiguredTarget dep,
+ Class<? extends ConfiguredAspectFactory> aspectFactory) {
+ AspectDefinition aspectDefinition = AspectFactory.Util.create(aspectFactory).getDefinition();
+ for (Class<?> provider : aspectDefinition.getRequiredProviders()) {
+ if (dep.getProvider((Class<? extends TransitiveInfoProvider>) provider) == null) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * Returns which aspects are computable based on the precise set of providers direct dependencies
+ * publish (and not the upper estimate in their rule definition).
+ *
+ * <p>An aspect is computable for a particular configured target if the configured target supplies
+ * all the providers the aspect requires.
+ *
+ * @param upperEstimate a multimap from attribute to the upper estimates computed by
+ * {@link com.google.devtools.build.lib.analysis.DependencyResolver}.
+ * @param configuredTargetDeps a multimap from attribute to the directly dependent configured
+ * targets
+ * @return a multimap from attribute to the more precise {@link Dependency} objects
+ */
+ private static ListMultimap<Attribute, Dependency> getComputableAspects(
+ ListMultimap<Attribute, Dependency> upperEstimate,
+ Map<SkyKey, ConfiguredTarget> configuredTargetDeps) {
+ ListMultimap<Attribute, Dependency> result = ArrayListMultimap.create();
+ for (Map.Entry<Attribute, Dependency> entry : upperEstimate.entries()) {
+ ConfiguredTarget dep =
+ configuredTargetDeps.get(TO_KEYS.apply(entry.getValue()));
+ List<Class<? extends ConfiguredAspectFactory>> depAspects = new ArrayList<>();
+ for (Class<? extends ConfiguredAspectFactory> candidate : entry.getValue().getAspects()) {
+ boolean ok = true;
+ for (Class<?> requiredProvider :
+ AspectFactory.Util.create(candidate).getDefinition().getRequiredProviders()) {
+ if (dep.getProvider((Class<? extends TransitiveInfoProvider>) requiredProvider) == null) {
+ ok = false;
+ break;
+ }
+ }
+
+ if (ok) {
+ depAspects.add(candidate);
+ }
+ }
+
+ result.put(entry.getKey(), new Dependency(
+ entry.getValue().getLabel(), entry.getValue().getConfiguration(),
+ ImmutableSet.copyOf(depAspects)));
+ }
+
+ return result;
+ }
+
+ /**
+ * Returns the set of {@link ConfigMatchingProvider}s that key the configurable attributes
+ * used by this rule.
+ *
+ * <p>>If the configured targets supplying those providers aren't yet resolved by the
+ * dependency resolver, returns null.
+ */
+ @Nullable
+ static Set<ConfigMatchingProvider> getConfigConditions(Target target, Environment env,
+ SkyframeDependencyResolver resolver, TargetAndConfiguration ctgValue)
+ throws DependencyEvaluationException {
+ if (!(target instanceof Rule)) {
+ return ImmutableSet.of();
+ }
+
+ ImmutableSet.Builder<ConfigMatchingProvider> configConditions = ImmutableSet.builder();
+
+ // Collect the labels of the configured targets we need to resolve.
+ ListMultimap<Attribute, LabelAndConfiguration> configLabelMap = ArrayListMultimap.create();
+ RawAttributeMapper attributeMap = RawAttributeMapper.of(((Rule) target));
+ for (Attribute a : ((Rule) target).getAttributes()) {
+ for (Label configLabel : attributeMap.getConfigurabilityKeys(a.getName(), a.getType())) {
+ if (!Type.Selector.isReservedLabel(configLabel)) {
+ configLabelMap.put(a, LabelAndConfiguration.of(
+ configLabel, ctgValue.getConfiguration()));
+ }
+ }
+ }
+ if (configLabelMap.isEmpty()) {
+ return ImmutableSet.of();
+ }
+
+ // Collect the corresponding Skyframe configured target values. Abort early if they haven't
+ // been computed yet.
+ Collection<Dependency> configValueNames =
+ resolver.resolveRuleLabels(ctgValue, null, configLabelMap);
+ Map<SkyKey, ConfiguredTarget> configValues =
+ resolveConfiguredTargetDependencies(env, configValueNames, target);
+ if (configValues == null) {
+ return null;
+ }
+
+ // Get the configured targets as ConfigMatchingProvider interfaces.
+ for (Dependency entry : configValueNames) {
+ ConfiguredTarget value = configValues.get(TO_KEYS.apply(entry));
+ // The code above guarantees that value is non-null here.
+ ConfigMatchingProvider provider = value.getProvider(ConfigMatchingProvider.class);
+ if (provider != null) {
+ configConditions.add(provider);
+ } else {
+ // Not a valid provider for configuration conditions.
+ String message =
+ entry.getLabel() + " is not a valid configuration key for " + target.getLabel();
+ env.getListener().handle(Event.error(TargetUtils.getLocationMaybe(target), message));
+ throw new DependencyEvaluationException(new ConfiguredValueCreationException(message));
+ }
+ }
+
+ return configConditions.build();
+ }
+
+ /***
+ * Resolves the targets referenced in depValueNames and returns their ConfiguredTarget
+ * instances.
+ *
+ * <p>Returns null if not all instances are available yet.
+ *
+ */
+ @Nullable
+ private static Map<SkyKey, ConfiguredTarget> resolveConfiguredTargetDependencies(
+ Environment env, Collection<Dependency> deps, Target target)
+ throws DependencyEvaluationException {
+ boolean ok = !env.valuesMissing();
+ String message = null;
+ Iterable<SkyKey> depKeys = Iterables.transform(deps, TO_KEYS);
+ // TODO(bazel-team): maybe having a two-exception argument is better than typing a generic
+ // Exception here.
+ Map<SkyKey, ValueOrException2<NoSuchTargetException,
+ NoSuchPackageException>> depValuesOrExceptions = env.getValuesOrThrow(depKeys,
+ NoSuchTargetException.class, NoSuchPackageException.class);
+ Map<SkyKey, ConfiguredTarget> depValues = new HashMap<>(depValuesOrExceptions.size());
+ SkyKey childKey = null;
+ NoSuchThingException transitiveChildException = null;
+ for (Map.Entry<SkyKey, ValueOrException2<NoSuchTargetException, NoSuchPackageException>> entry
+ : depValuesOrExceptions.entrySet()) {
+ ConfiguredTargetKey depKey = (ConfiguredTargetKey) entry.getKey().argument();
+ LabelAndConfiguration depLabelAndConfiguration = LabelAndConfiguration.of(
+ depKey.getLabel(), depKey.getConfiguration());
+ Label depLabel = depLabelAndConfiguration.getLabel();
+ ConfiguredTargetValue depValue = null;
+ NoSuchThingException directChildException = null;
+ try {
+ depValue = (ConfiguredTargetValue) entry.getValue().get();
+ } catch (NoSuchTargetException e) {
+ if (depLabel.equals(e.getLabel())) {
+ directChildException = e;
+ } else {
+ childKey = entry.getKey();
+ transitiveChildException = e;
+ }
+ } catch (NoSuchPackageException e) {
+ if (depLabel.getPackageName().equals(e.getPackageName())) {
+ directChildException = e;
+ } else {
+ childKey = entry.getKey();
+ transitiveChildException = e;
+ }
+ }
+ // If an exception wasn't caused by a direct child target value, we'll treat it the same
+ // as any other missing dep by setting ok = false below, and returning null at the end.
+ if (directChildException != null) {
+ // Only update messages for missing targets we depend on directly.
+ message = TargetUtils.formatMissingEdge(target, depLabel, directChildException);
+ env.getListener().handle(Event.error(TargetUtils.getLocationMaybe(target), message));
+ }
+
+ if (depValue == null) {
+ ok = false;
+ } else {
+ depValues.put(entry.getKey(), depValue.getConfiguredTarget());
+ }
+ }
+ if (message != null) {
+ throw new DependencyEvaluationException(new NoSuchTargetException(message));
+ }
+ if (childKey != null) {
+ throw new DependencyEvaluationException(childKey, transitiveChildException);
+ }
+ if (!ok) {
+ return null;
+ } else {
+ return depValues;
+ }
+ }
+
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return Label.print(((ConfiguredTargetKey) skyKey.argument()).getLabel());
+ }
+
+ @Nullable
+ private ConfiguredTargetValue createConfiguredTarget(SkyframeBuildView view,
+ Environment env, Target target, BuildConfiguration configuration,
+ ListMultimap<Attribute, ConfiguredTarget> depValueMap,
+ Set<ConfigMatchingProvider> configConditions)
+ throws ConfiguredTargetFunctionException,
+ InterruptedException {
+ boolean extendedSanityChecks = configuration != null && configuration.extendedSanityChecks();
+
+ StoredEventHandler events = new StoredEventHandler();
+ BuildConfiguration ownerConfig = (configuration == null)
+ ? null : configuration.getArtifactOwnerConfiguration();
+ boolean allowRegisteringActions = configuration == null || configuration.isActionsEnabled();
+ CachingAnalysisEnvironment analysisEnvironment = view.createAnalysisEnvironment(
+ new ConfiguredTargetKey(target.getLabel(), ownerConfig), false,
+ extendedSanityChecks, events, env, allowRegisteringActions);
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ ConfiguredTarget configuredTarget = view.createConfiguredTarget(target, configuration,
+ analysisEnvironment, depValueMap, configConditions);
+
+ events.replayOn(env.getListener());
+ if (events.hasErrors()) {
+ analysisEnvironment.disable(target);
+ throw new ConfiguredTargetFunctionException(new ConfiguredValueCreationException(
+ "Analysis of target '" + target.getLabel() + "' failed; build aborted"));
+ }
+ Preconditions.checkState(!analysisEnvironment.hasErrors(),
+ "Analysis environment hasError() but no errors reported");
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ analysisEnvironment.disable(target);
+ Preconditions.checkNotNull(configuredTarget, target);
+
+ return new ConfiguredTargetValue(configuredTarget,
+ ImmutableList.copyOf(analysisEnvironment.getRegisteredActions()));
+ }
+
+ /**
+ * An exception indicating that there was a problem during the construction of
+ * a ConfiguredTargetValue.
+ */
+ public static final class ConfiguredValueCreationException extends Exception {
+
+ public ConfiguredValueCreationException(String message) {
+ super(message);
+ }
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link ConfiguredTargetFunction#compute}.
+ */
+ public static final class ConfiguredTargetFunctionException extends SkyFunctionException {
+ public ConfiguredTargetFunctionException(NoSuchTargetException e) {
+ super(e, Transience.PERSISTENT);
+ }
+
+ private ConfiguredTargetFunctionException(ConfiguredValueCreationException error) {
+ super(error, Transience.PERSISTENT);
+ };
+
+ private ConfiguredTargetFunctionException(
+ @Nullable SkyKey childKey, Exception transitiveError) {
+ super(transitiveError, childKey);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKey.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKey.java
new file mode 100644
index 0000000..ea744c1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKey.java
@@ -0,0 +1,96 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * A (Label, Configuration) pair. Note that this pair may be used to look up the generating action
+ * of an artifact. Callers may want to ensure that they have the correct configuration for this
+ * purpose by passing in {@link BuildConfiguration#getArtifactOwnerConfiguration} in preference to
+ * the raw configuration.
+ */
+public class ConfiguredTargetKey extends ActionLookupValue.ActionLookupKey {
+ private final Label label;
+ @Nullable
+ private final BuildConfiguration configuration;
+
+ public ConfiguredTargetKey(Label label, @Nullable BuildConfiguration configuration) {
+ this.label = Preconditions.checkNotNull(label);
+ this.configuration = configuration;
+ }
+
+ public ConfiguredTargetKey(ConfiguredTarget rule) {
+ this(rule.getTarget().getLabel(), rule.getConfiguration());
+ }
+
+ @Override
+ public Label getLabel() {
+ return label;
+ }
+
+ @Override
+ SkyFunctionName getType() {
+ return SkyFunctions.CONFIGURED_TARGET;
+ }
+
+ @Nullable
+ public BuildConfiguration getConfiguration() {
+ return configuration;
+ }
+
+ @Override
+ public int hashCode() {
+ int configVal = configuration == null ? 79 : configuration.hashCode();
+ return 31 * label.hashCode() + configVal;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof ConfiguredTargetKey)) {
+ return false;
+ }
+ ConfiguredTargetKey other = (ConfiguredTargetKey) obj;
+ return Objects.equals(label, other.label) && Objects.equals(configuration, other.configuration);
+ }
+
+ public String prettyPrint() {
+ if (label == null) {
+ return "null";
+ }
+ return (configuration != null && configuration.isHostConfiguration())
+ ? (label.toString() + " (host)") : label.toString();
+ }
+
+ @Override
+ public String toString() {
+ return label + " " + (configuration == null ? "null" : configuration.shortCacheKey());
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetValue.java
new file mode 100644
index 0000000..200e05f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetValue.java
@@ -0,0 +1,105 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyKey;
+
+import javax.annotation.Nullable;
+
+/**
+ * A configured target in the context of a Skyframe graph.
+ */
+@Immutable
+@ThreadSafe
+@VisibleForTesting
+public final class ConfiguredTargetValue extends ActionLookupValue {
+
+ // These variables are only non-final because they may be clear()ed to save memory. They are null
+ // only after they are cleared.
+ @Nullable private ConfiguredTarget configuredTarget;
+
+ // We overload this variable to check whether the value has been clear()ed. We don't use a
+ // separate variable in order to save memory.
+ @Nullable private volatile Iterable<Action> actions;
+
+ ConfiguredTargetValue(ConfiguredTarget configuredTarget, Iterable<Action> actions) {
+ super(actions);
+ this.configuredTarget = configuredTarget;
+ this.actions = actions;
+ }
+
+ @VisibleForTesting
+ public ConfiguredTarget getConfiguredTarget() {
+ Preconditions.checkNotNull(actions, configuredTarget);
+ return configuredTarget;
+ }
+
+ @VisibleForTesting
+ public Iterable<Action> getActions() {
+ return Preconditions.checkNotNull(actions, configuredTarget);
+ }
+
+ /**
+ * Clears configured target data from this value, leaving only the artifact->generating action
+ * map.
+ *
+ * <p>Should only be used when user specifies --discard_analysis_cache. Must be called at most
+ * once per value, after which {@link #getConfiguredTarget} and {@link #getActions} cannot be
+ * called.
+ */
+ public void clear() {
+ Preconditions.checkNotNull(actions, configuredTarget);
+ configuredTarget = null;
+ actions = null;
+ }
+
+ @VisibleForTesting
+ public static SkyKey key(Label label, BuildConfiguration configuration) {
+ return key(new ConfiguredTargetKey(label, configuration));
+ }
+
+ static ImmutableList<SkyKey> keys(Iterable<ConfiguredTargetKey> lacs) {
+ ImmutableList.Builder<SkyKey> keys = ImmutableList.builder();
+ for (ConfiguredTargetKey lac : lacs) {
+ keys.add(key(lac));
+ }
+ return keys.build();
+ }
+
+ /**
+ * Returns a label of ConfiguredTargetValue.
+ */
+ @ThreadSafe
+ static Label extractLabel(SkyKey value) {
+ Object valueName = value.argument();
+ Preconditions.checkState(valueName instanceof ConfiguredTargetKey, valueName);
+ return ((ConfiguredTargetKey) valueName).getLabel();
+ }
+
+ @Override
+ public String toString() {
+ return "ConfiguredTargetValue: "
+ + configuredTarget + ", actions: " + (actions == null ? null : Iterables.toString(actions));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupFunction.java
new file mode 100644
index 0000000..58cb67d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupFunction.java
@@ -0,0 +1,55 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import javax.annotation.Nullable;
+
+/**
+ * SkyFunction for {@link ContainingPackageLookupValue}s.
+ */
+public class ContainingPackageLookupFunction implements SkyFunction {
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ PackageIdentifier dir = (PackageIdentifier) skyKey.argument();
+ PackageLookupValue pkgLookupValue = null;
+ pkgLookupValue = (PackageLookupValue) env.getValue(PackageLookupValue.key(dir));
+ if (pkgLookupValue == null) {
+ return null;
+ }
+
+ if (pkgLookupValue.packageExists()) {
+ return ContainingPackageLookupValue.withContainingPackage(dir, pkgLookupValue.getRoot());
+ }
+
+ PathFragment parentDir = dir.getPackageFragment().getParentDirectory();
+ if (parentDir == null) {
+ return ContainingPackageLookupValue.noContainingPackage();
+ }
+ PackageIdentifier parentId = new PackageIdentifier(dir.getRepository(), parentDir);
+ return env.getValue(ContainingPackageLookupValue.key(parentId));
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupValue.java
new file mode 100644
index 0000000..16516b5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupValue.java
@@ -0,0 +1,111 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A value that represents the result of looking for the existence of a package that owns a
+ * specific directory path. Compare with {@link PackageLookupValue}, which deals with existence of
+ * a specific package.
+ */
+public abstract class ContainingPackageLookupValue implements SkyValue {
+ /** Returns whether there is a containing package. */
+ public abstract boolean hasContainingPackage();
+
+ /** If there is a containing package, returns its name. */
+ abstract PackageIdentifier getContainingPackageName();
+
+ /** If there is a containing package, returns its package root */
+ public abstract Path getContainingPackageRoot();
+
+ public static SkyKey key(PackageIdentifier id) {
+ Preconditions.checkArgument(!id.getPackageFragment().isAbsolute(), id);
+ return new SkyKey(SkyFunctions.CONTAINING_PACKAGE_LOOKUP, id);
+ }
+
+ static ContainingPackageLookupValue noContainingPackage() {
+ return NoContainingPackage.INSTANCE;
+ }
+
+ static ContainingPackageLookupValue withContainingPackage(PackageIdentifier pkgId, Path root) {
+ return new ContainingPackage(pkgId, root);
+ }
+
+ private static class NoContainingPackage extends ContainingPackageLookupValue {
+ private static final NoContainingPackage INSTANCE = new NoContainingPackage();
+
+ @Override
+ public boolean hasContainingPackage() {
+ return false;
+ }
+
+ @Override
+ public PackageIdentifier getContainingPackageName() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public Path getContainingPackageRoot() {
+ throw new IllegalStateException();
+ }
+ }
+
+ private static class ContainingPackage extends ContainingPackageLookupValue {
+ private final PackageIdentifier containingPackage;
+ private final Path containingPackageRoot;
+
+ private ContainingPackage(PackageIdentifier pkgId, Path containingPackageRoot) {
+ this.containingPackage = pkgId;
+ this.containingPackageRoot = containingPackageRoot;
+ }
+
+ @Override
+ public boolean hasContainingPackage() {
+ return true;
+ }
+
+ @Override
+ public PackageIdentifier getContainingPackageName() {
+ return containingPackage;
+ }
+
+ @Override
+ public Path getContainingPackageRoot() {
+ return containingPackageRoot;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof ContainingPackage)) {
+ return false;
+ }
+ ContainingPackage other = (ContainingPackage) obj;
+ return containingPackage.equals(other.containingPackage)
+ && containingPackageRoot.equals(other.containingPackageRoot);
+ }
+
+ @Override
+ public int hashCode() {
+ return containingPackage.hashCode();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/CoverageReportFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/CoverageReportFunction.java
new file mode 100644
index 0000000..8d6fe3d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/CoverageReportFunction.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A Skyframe function to calculate the coverage report Action and Artifacts.
+ */
+public class CoverageReportFunction implements SkyFunction {
+ CoverageReportFunction() {}
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ Preconditions.checkState(
+ CoverageReportValue.SKY_KEY.equals(skyKey), String.format(
+ "Expected %s for SkyKey but got %s instead", CoverageReportValue.SKY_KEY, skyKey));
+
+ Action action = PrecomputedValue.COVERAGE_REPORT_KEY.get(env);
+ if (action == null) {
+ return null;
+ }
+
+ return new CoverageReportValue(
+ action.getOutputs(),
+ action);
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/CoverageReportValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/CoverageReportValue.java
new file mode 100644
index 0000000..862e381
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/CoverageReportValue.java
@@ -0,0 +1,55 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+
+/**
+ * A SkyValue to store the coverage report Action and Artifacts.
+ */
+public class CoverageReportValue extends ActionLookupValue {
+ private final ImmutableSet<Artifact> coverageReportArtifacts;
+
+ // There should only ever be one CoverageReportValue value in the graph.
+ public static final SkyKey SKY_KEY = new SkyKey(SkyFunctions.COVERAGE_REPORT, "COVERAGE_REPORT");
+ public static final ArtifactOwner ARTIFACT_OWNER = new CoverageReportKey();
+
+ public CoverageReportValue(ImmutableSet<Artifact> coverageReportArtifacts,
+ Action coverageReportAction) {
+ super(coverageReportAction);
+ this.coverageReportArtifacts = coverageReportArtifacts;
+ }
+
+ public ImmutableSet<Artifact> getCoverageReportArtifacts() {
+ return coverageReportArtifacts;
+ }
+
+ private static class CoverageReportKey extends ActionLookupKey {
+ @Override
+ SkyFunctionName getType() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ SkyKey getSkyKey() {
+ return SKY_KEY;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DiffAwareness.java b/src/main/java/com/google/devtools/build/lib/skyframe/DiffAwareness.java
new file mode 100644
index 0000000..d0f4c99
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/DiffAwareness.java
@@ -0,0 +1,82 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.vfs.ModifiedFileSet;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.Closeable;
+
+import javax.annotation.Nullable;
+
+/**
+ * Interface for computing modifications of files under a package path entry.
+ *
+ * <p> Skyframe has a {@link DiffAwareness} instance per package-path entry, and each instance is
+ * responsible for all files under its path entry. At the beginning of each incremental build,
+ * skyframe queries for changes using {@link #getDiff}. Ideally, {@link #getDiff} should be
+ * constant-time; if it were linear in the number of files of interest, we might as well just
+ * detect modifications manually.
+ */
+public interface DiffAwareness extends Closeable {
+
+ /** Factory for creating {@link DiffAwareness} instances. */
+ public interface Factory {
+ /**
+ * Returns a {@link DiffAwareness} instance suitable for managing changes to files under the
+ * given package path entry, or {@code null} if this factory cannot create such an instance.
+ *
+ * <p> Skyframe has a collection of factories, and will create a {@link DiffAwareness} instance
+ * per package path entry using one of the factories that returns a non-null value.
+ */
+ @Nullable
+ DiffAwareness maybeCreate(Path pathEntry);
+ }
+
+ /** Opaque view of the filesystem under a package path entry at a specific point in time. */
+ interface View {
+ }
+
+ /**
+ * Returns the live view of the filesystem under the package path entry.
+ *
+ * @throws BrokenDiffAwarenessException if something is wrong and the caller should discard this
+ * {@link DiffAwareness} instance. The {@link DiffAwareness} is expected to close itself in
+ * this case.
+ */
+ View getCurrentView() throws BrokenDiffAwarenessException;
+
+ /**
+ * Returns the set of files of interest that have been modified between the given two views.
+ *
+ * <p>The given views must have come from previous calls to {@link #getCurrentView} on the
+ * {@link DiffAwareness} instance (i.e. using a {@link View} from another instance is not
+ * supported).
+ *
+ * @throws IncompatibleViewException if the given views are not compatible with this
+ * {@link DiffAwareness} instance. This probably indicates a bug.
+ * @throws BrokenDiffAwarenessException if something is wrong and the caller should discard this
+ * {@link DiffAwareness} instance. The {@link DiffAwareness} is expected to close itself in
+ * this case.
+ */
+ ModifiedFileSet getDiff(View oldView, View newView)
+ throws IncompatibleViewException, BrokenDiffAwarenessException;
+
+ /**
+ * Must be called whenever the {@link DiffAwareness} object is to be discarded. Using a
+ * {@link DiffAwareness} instance after calling {@link #close} on it is unspecified behavior.
+ */
+ @Override
+ void close();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DiffAwarenessManager.java b/src/main/java/com/google/devtools/build/lib/skyframe/DiffAwarenessManager.java
new file mode 100644
index 0000000..d1bb81e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/DiffAwarenessManager.java
@@ -0,0 +1,188 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.skyframe.DiffAwareness.View;
+import com.google.devtools.build.lib.vfs.ModifiedFileSet;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * Helper class to make it easier to correctly use the {@link DiffAwareness} interface in a
+ * sequential manner.
+ */
+public final class DiffAwarenessManager {
+
+ private final ImmutableSet<? extends DiffAwareness.Factory> diffAwarenessFactories;
+ private Map<Path, DiffAwarenessState> currentDiffAwarenessStates = Maps.newHashMap();
+ private final Reporter reporter;
+
+ public DiffAwarenessManager(Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories,
+ Reporter reporter) {
+ this.diffAwarenessFactories = ImmutableSet.copyOf(diffAwarenessFactories);
+ this.reporter = reporter;
+ }
+
+ private static class DiffAwarenessState {
+ private final DiffAwareness diffAwareness;
+ /**
+ * The {@link View} that should be the baseline for the next {@link #getDiff} call, or
+ * {@code null} if the next {@link #getDiff} will be the first incremental one.
+ */
+ @Nullable
+ private View baselineView;
+
+ private DiffAwarenessState(DiffAwareness diffAwareness, @Nullable View baselineView) {
+ this.diffAwareness = diffAwareness;
+ this.baselineView = baselineView;
+ }
+ }
+
+ /** Reset internal {@link DiffAwareness} state. */
+ public void reset() {
+ for (DiffAwarenessState diffAwarenessState : currentDiffAwarenessStates.values()) {
+ diffAwarenessState.diffAwareness.close();
+ }
+ currentDiffAwarenessStates.clear();
+ }
+
+ /** A set of modified files that should be marked as processed. */
+ public interface ProcessableModifiedFileSet {
+ ModifiedFileSet getModifiedFileSet();
+
+ /**
+ * This should be called when the changes have been noted. Otherwise, the result from the next
+ * call to {@link #getDiff} will be from the baseline of the old, unprocessed, diff.
+ */
+ void markProcessed();
+ }
+
+ /**
+ * Gets the set of changed files since the last call with this path entry, or
+ * {@code ModifiedFileSet.EVERYTHING_MODIFIED} if this is the first such call.
+ */
+ public ProcessableModifiedFileSet getDiff(Path pathEntry) {
+ DiffAwarenessState diffAwarenessState = maybeGetDiffAwarenessState(pathEntry);
+ if (diffAwarenessState == null) {
+ return BrokenProcessableModifiedFileSet.INSTANCE;
+ }
+ DiffAwareness diffAwareness = diffAwarenessState.diffAwareness;
+ View newView;
+ try {
+ newView = diffAwareness.getCurrentView();
+ } catch (BrokenDiffAwarenessException e) {
+ handleBrokenDiffAwareness(pathEntry, e);
+ return BrokenProcessableModifiedFileSet.INSTANCE;
+ }
+
+ View baselineView = diffAwarenessState.baselineView;
+ if (baselineView == null) {
+ diffAwarenessState.baselineView = newView;
+ return BrokenProcessableModifiedFileSet.INSTANCE;
+ }
+
+ ModifiedFileSet diff;
+ try {
+ diff = diffAwareness.getDiff(baselineView, newView);
+ } catch (BrokenDiffAwarenessException e) {
+ handleBrokenDiffAwareness(pathEntry, e);
+ return BrokenProcessableModifiedFileSet.INSTANCE;
+ } catch (IncompatibleViewException e) {
+ throw new IllegalStateException(pathEntry + " " + baselineView + " " + newView, e);
+ }
+ ProcessableModifiedFileSet result = new ProcessableModifiedFileSetImpl(diff, pathEntry,
+ newView);
+ return result;
+ }
+
+ private void handleBrokenDiffAwareness(Path pathEntry, BrokenDiffAwarenessException e) {
+ currentDiffAwarenessStates.remove(pathEntry);
+ reporter.handle(Event.warn(e.getMessage() + "... temporarily falling back to manually "
+ + "checking files for changes"));
+ }
+
+ /**
+ * Returns the current diff awareness for the given path entry, or a fresh one if there is no
+ * current one, or otherwise {@code null} if no factory could make a fresh one.
+ */
+ @Nullable
+ private DiffAwarenessState maybeGetDiffAwarenessState(Path pathEntry) {
+ DiffAwarenessState diffAwarenessState = currentDiffAwarenessStates.get(pathEntry);
+ if (diffAwarenessState != null) {
+ return diffAwarenessState;
+ }
+ for (DiffAwareness.Factory factory : diffAwarenessFactories) {
+ DiffAwareness newDiffAwareness = factory.maybeCreate(pathEntry);
+ if (newDiffAwareness != null) {
+ diffAwarenessState = new DiffAwarenessState(newDiffAwareness, /*previousView=*/null);
+ currentDiffAwarenessStates.put(pathEntry, diffAwarenessState);
+ return diffAwarenessState;
+ }
+ }
+ return null;
+ }
+
+ private class ProcessableModifiedFileSetImpl implements ProcessableModifiedFileSet {
+
+ private final ModifiedFileSet modifiedFileSet;
+ private final Path pathEntry;
+ /**
+ * The {@link View} that should be the baseline on the next {@link #getDiff} call after
+ * {@link #markProcessed} is called.
+ */
+ private final View nextView;
+
+ private ProcessableModifiedFileSetImpl(ModifiedFileSet modifiedFileSet, Path pathEntry,
+ View nextView) {
+ this.modifiedFileSet = modifiedFileSet;
+ this.pathEntry = pathEntry;
+ this.nextView = nextView;
+ }
+
+ @Override
+ public ModifiedFileSet getModifiedFileSet() {
+ return modifiedFileSet;
+ }
+
+ @Override
+ public void markProcessed() {
+ DiffAwarenessState diffAwarenessState = currentDiffAwarenessStates.get(pathEntry);
+ if (diffAwarenessState != null) {
+ diffAwarenessState.baselineView = nextView;
+ }
+ }
+ }
+
+ private static class BrokenProcessableModifiedFileSet implements ProcessableModifiedFileSet {
+
+ private static final BrokenProcessableModifiedFileSet INSTANCE =
+ new BrokenProcessableModifiedFileSet();
+
+ @Override
+ public ModifiedFileSet getModifiedFileSet() {
+ return ModifiedFileSet.EVERYTHING_MODIFIED;
+ }
+
+ @Override
+ public void markProcessed() {
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingFunction.java
new file mode 100644
index 0000000..93c3d75
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingFunction.java
@@ -0,0 +1,72 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import javax.annotation.Nullable;
+
+/**
+ * A {@link SkyFunction} for {@link DirectoryListingValue}s.
+ */
+final class DirectoryListingFunction implements SkyFunction {
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env)
+ throws DirectoryListingFunctionException {
+ RootedPath dirRootedPath = (RootedPath) skyKey.argument();
+
+ FileValue dirFileValue = (FileValue) env.getValue(FileValue.key(dirRootedPath));
+ if (dirFileValue == null) {
+ return null;
+ }
+
+ RootedPath realDirRootedPath = dirFileValue.realRootedPath();
+ if (!dirFileValue.isDirectory()) {
+ // Recall that the directory is assumed to exist (see DirectoryListingValue#key).
+ throw new DirectoryListingFunctionException(new InconsistentFilesystemException(
+ dirRootedPath.asPath() + " is no longer an existing directory. Did you delete it during "
+ + "the build?"));
+ }
+
+ DirectoryListingStateValue directoryListingStateValue =
+ (DirectoryListingStateValue) env.getValue(DirectoryListingStateValue.key(
+ realDirRootedPath));
+ if (directoryListingStateValue == null) {
+ return null;
+ }
+
+ return DirectoryListingValue.value(dirRootedPath, dirFileValue, directoryListingStateValue);
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link DirectoryListingFunction#compute}.
+ */
+ private static final class DirectoryListingFunctionException extends SkyFunctionException {
+ public DirectoryListingFunctionException(InconsistentFilesystemException e) {
+ super(e, Transience.TRANSIENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateFunction.java
new file mode 100644
index 0000000..6e47a2d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateFunction.java
@@ -0,0 +1,68 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+
+/**
+ * A {@link SkyFunction} for {@link DirectoryListingStateValue}s.
+ *
+ * <p>Merely calls DirectoryListingStateValue#create, but also has special handling for
+ * directories outside the package roots (see {@link ExternalFilesHelper}).
+ */
+public class DirectoryListingStateFunction implements SkyFunction {
+
+ private final ExternalFilesHelper externalFilesHelper;
+
+ public DirectoryListingStateFunction(ExternalFilesHelper externalFilesHelper) {
+ this.externalFilesHelper = externalFilesHelper;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env)
+ throws DirectoryListingStateFunctionException {
+ RootedPath dirRootedPath = (RootedPath) skyKey.argument();
+ externalFilesHelper.maybeAddDepOnBuildId(dirRootedPath, env);
+ if (env.valuesMissing()) {
+ return null;
+ }
+ try {
+ return DirectoryListingStateValue.create(dirRootedPath);
+ } catch (IOException e) {
+ throw new DirectoryListingStateFunctionException(e);
+ }
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link DirectoryListingStateFunction#compute}.
+ */
+ private static final class DirectoryListingStateFunctionException
+ extends SkyFunctionException {
+ public DirectoryListingStateFunctionException(IOException e) {
+ super(e, Transience.TRANSIENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateValue.java
new file mode 100644
index 0000000..87b9748
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateValue.java
@@ -0,0 +1,214 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.vfs.Dirent;
+import com.google.devtools.build.lib.vfs.Dirent.Type;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.lib.vfs.Symlinks;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.Objects;
+
+/**
+ * Encapsulates the filesystem operations needed to get the directory entries of a directory.
+ *
+ * <p>This class is an implementation detail of {@link DirectoryListingValue}.
+ */
+final class DirectoryListingStateValue implements SkyValue {
+
+ private final CompactSortedDirents compactSortedDirents;
+
+ private DirectoryListingStateValue(Collection<Dirent> dirents) {
+ this.compactSortedDirents = CompactSortedDirents.create(dirents);
+ }
+
+ @VisibleForTesting
+ public static DirectoryListingStateValue createForTesting(Collection<Dirent> dirents) {
+ return new DirectoryListingStateValue(dirents);
+ }
+
+ public static DirectoryListingStateValue create(RootedPath dirRootedPath) throws IOException {
+ Collection<Dirent> dirents = dirRootedPath.asPath().readdir(Symlinks.NOFOLLOW);
+ return new DirectoryListingStateValue(dirents);
+ }
+
+ @ThreadSafe
+ public static SkyKey key(RootedPath rootedPath) {
+ return new SkyKey(SkyFunctions.DIRECTORY_LISTING_STATE, rootedPath);
+ }
+
+ /**
+ * Returns the directory entries for this directory, in a stable order.
+ *
+ * <p>Symlinks are not expanded.
+ */
+ public Iterable<Dirent> getDirents() {
+ return compactSortedDirents;
+ }
+
+ @Override
+ public int hashCode() {
+ return compactSortedDirents.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof DirectoryListingStateValue)) {
+ return false;
+ }
+ DirectoryListingStateValue other = (DirectoryListingStateValue) obj;
+ return compactSortedDirents.equals(other.compactSortedDirents);
+ }
+
+ /** A space-efficient, sorted, immutable dirent structure. */
+ private static class CompactSortedDirents implements Iterable<Dirent>, Serializable {
+
+ private final String[] names;
+ private final BitSet packedTypes;
+
+ private CompactSortedDirents(String[] names, BitSet packedTypes) {
+ this.names = names;
+ this.packedTypes = packedTypes;
+ }
+
+ public static CompactSortedDirents create(Collection<Dirent> dirents) {
+ final Dirent[] direntArray = dirents.toArray(new Dirent[dirents.size()]);
+ Integer[] indices = new Integer[dirents.size()];
+ for (int i = 0; i < dirents.size(); i++) {
+ indices[i] = i;
+ }
+ Arrays.sort(indices,
+ new Comparator<Integer>() {
+ @Override
+ public int compare(Integer o1, Integer o2) {
+ return direntArray[o1].getName().compareTo(direntArray[o2].getName());
+ }
+ });
+ String[] names = new String[dirents.size()];
+ BitSet packedTypes = new BitSet(dirents.size() * 2);
+ for (int i = 0; i < dirents.size(); i++) {
+ Dirent dirent = direntArray[indices[i]];
+ names[i] = dirent.getName();
+ packType(packedTypes, dirent.getType(), i);
+ }
+ return new CompactSortedDirents(names, packedTypes);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof CompactSortedDirents)) {
+ return false;
+ }
+ if (this == obj) {
+ return true;
+ }
+ CompactSortedDirents other = (CompactSortedDirents) obj;
+ return Arrays.equals(names, other.names) && packedTypes.equals(other.packedTypes);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(Arrays.hashCode(names), packedTypes);
+ }
+
+ @Override
+ public Iterator<Dirent> iterator() {
+ return new Iterator<Dirent>() {
+
+ private int i = 0;
+
+ @Override
+ public boolean hasNext() {
+ return i < size();
+ }
+
+ @Override
+ public Dirent next() {
+ return direntAt(i++);
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ private int size() {
+ return names.length;
+ }
+
+ /** Returns the type of the ith dirent. */
+ private Dirent.Type unpackType(int i) {
+ int start = i * 2;
+ boolean upper = packedTypes.get(start);
+ boolean lower = packedTypes.get(start + 1);
+ if (!upper && !lower) {
+ return Type.FILE;
+ } else if (!upper && lower){
+ return Type.DIRECTORY;
+ } else if (upper && !lower) {
+ return Type.SYMLINK;
+ } else {
+ return Type.UNKNOWN;
+ }
+ }
+
+ /** Sets the type of the ith dirent. */
+ private static void packType(BitSet bitSet, Dirent.Type type, int i) {
+ int start = i * 2;
+ switch (type) {
+ case FILE:
+ pack(bitSet, start, false, false);
+ break;
+ case DIRECTORY:
+ pack(bitSet, start, false, true);
+ break;
+ case SYMLINK:
+ pack(bitSet, start, true, false);
+ break;
+ case UNKNOWN:
+ pack(bitSet, start, true, true);
+ break;
+ default:
+ throw new IllegalStateException("Unknown dirent type: " + type);
+ }
+ }
+
+ private static void pack(BitSet bitSet, int start, boolean upper, boolean lower) {
+ bitSet.set(start, upper);
+ bitSet.set(start + 1, lower);
+ }
+
+ private Dirent direntAt(int i) {
+ Preconditions.checkState(i >= 0 && i < size(), "i: %s, size: %s", i, size());
+ return new Dirent(names[i], unpackType(i));
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingValue.java
new file mode 100644
index 0000000..3fe6dba
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingValue.java
@@ -0,0 +1,134 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.vfs.Dirent;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Objects;
+
+/**
+ * A value that represents the list of files in a given directory under a given package path root.
+ * Anything in Skyframe that cares about the contents of a directory should have a dependency
+ * on the corresponding {@link DirectoryListingValue}.
+ *
+ * <p>This value only depends on the FileValue corresponding to the directory. In particular, note
+ * that it does not depend on any of its child entries.
+ *
+ * <p>Note that symlinks in dirents are <b>not</b> expanded. Dependents of the value are responsible
+ * for expanding the symlink entries by referring to FileValues that correspond to the symlinks.
+ * This is a little onerous, but correct: we do not need to reread the directory when a symlink
+ * inside it changes, therefore this value should not be invalidated in that case.
+ */
+@Immutable
+@ThreadSafe
+abstract class DirectoryListingValue implements SkyValue {
+
+ /**
+ * Returns the directory entries for this directory, in a stable order.
+ *
+ * <p>Symlinks are not expanded.
+ */
+ public abstract Iterable<Dirent> getDirents();
+
+ /**
+ * Returns a {@link SkyKey} for getting the directory entries of the given directory. The
+ * given path is assumed to be an existing directory (e.g. via {@link FileValue#isDirectory} or
+ * from a directory listing on its parent directory).
+ */
+ @ThreadSafe
+ static SkyKey key(RootedPath directoryUnderRoot) {
+ return new SkyKey(SkyFunctions.DIRECTORY_LISTING, directoryUnderRoot);
+ }
+
+ static DirectoryListingValue value(RootedPath dirRootedPath, FileValue dirFileValue,
+ DirectoryListingStateValue realDirectoryListingStateValue) {
+ return dirFileValue.realRootedPath().equals(dirRootedPath)
+ ? new RegularDirectoryListingValue(realDirectoryListingStateValue)
+ : new DifferentRealPathDirectoryListingValue(dirFileValue.realRootedPath(),
+ realDirectoryListingStateValue);
+ }
+
+ @ThreadSafe
+ private static final class RegularDirectoryListingValue extends DirectoryListingValue {
+
+ private final DirectoryListingStateValue directoryListingStateValue;
+
+ private RegularDirectoryListingValue(DirectoryListingStateValue directoryListingStateValue) {
+ this.directoryListingStateValue = directoryListingStateValue;
+ }
+
+ @Override
+ public Iterable<Dirent> getDirents() {
+ return directoryListingStateValue.getDirents();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof RegularDirectoryListingValue)) {
+ return false;
+ }
+ RegularDirectoryListingValue other = (RegularDirectoryListingValue) obj;
+ return directoryListingStateValue.equals(other.directoryListingStateValue);
+ }
+
+ @Override
+ public int hashCode() {
+ return directoryListingStateValue.hashCode();
+ }
+ }
+
+ @ThreadSafe
+ private static final class DifferentRealPathDirectoryListingValue extends DirectoryListingValue {
+
+ private final RootedPath realDirRootedPath;
+ private final DirectoryListingStateValue directoryListingStateValue;
+
+ private DifferentRealPathDirectoryListingValue(RootedPath realDirRootedPath,
+ DirectoryListingStateValue directoryListingStateValue) {
+ this.realDirRootedPath = realDirRootedPath;
+ this.directoryListingStateValue = directoryListingStateValue;
+ }
+
+ @Override
+ public Iterable<Dirent> getDirents() {
+ return directoryListingStateValue.getDirents();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof DifferentRealPathDirectoryListingValue)) {
+ return false;
+ }
+ DifferentRealPathDirectoryListingValue other = (DifferentRealPathDirectoryListingValue) obj;
+ return realDirRootedPath.equals(other.realDirRootedPath)
+ && directoryListingStateValue.equals(other.directoryListingStateValue);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(realDirRootedPath, directoryListingStateValue);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ErrorReadingSkylarkExtensionException.java b/src/main/java/com/google/devtools/build/lib/skyframe/ErrorReadingSkylarkExtensionException.java
new file mode 100644
index 0000000..8593afb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ErrorReadingSkylarkExtensionException.java
@@ -0,0 +1,21 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+/** Indicates some sort of IO error while dealing with a Skylark extension. */
+public class ErrorReadingSkylarkExtensionException extends Exception {
+ public ErrorReadingSkylarkExtensionException(String message) {
+ super(message);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ExternalFilesHelper.java b/src/main/java/com/google/devtools/build/lib/skyframe/ExternalFilesHelper.java
new file mode 100644
index 0000000..ce858de
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ExternalFilesHelper.java
@@ -0,0 +1,97 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
+/** Common utilities for dealing with files outside the package roots. */
+class ExternalFilesHelper {
+
+ private final AtomicReference<PathPackageLocator> pkgLocator;
+ private final Set<Path> immutableDirs;
+
+ ExternalFilesHelper(AtomicReference<PathPackageLocator> pkgLocator) {
+ this(pkgLocator, ImmutableSet.<Path>of());
+ }
+
+ ExternalFilesHelper(AtomicReference<PathPackageLocator> pkgLocator, Set<Path> immutableDirs) {
+ this.pkgLocator = pkgLocator;
+ this.immutableDirs = immutableDirs;
+ }
+
+ private enum FileType {
+ // A file inside the package roots.
+ INTERNAL_FILE,
+
+ // A file outside the package roots that we may pretends is immutable.
+ EXTERNAL_IMMUTABLE_FILE,
+
+ // A file outside the package roots about which we may make no other assumptions.
+ EXTERNAL_MUTABLE_FILE,
+ }
+
+ private FileType getFileType(RootedPath rootedPath) {
+ // TODO(bazel-team): This is inefficient when there are a lot of package roots or there are a
+ // lot of immutable directories. Consider either explicitly preventing this case or using a more
+ // efficient approach here (e.g. use a trie for determing if a file is under an immutable
+ // directory).
+ if (!pkgLocator.get().getPathEntries().contains(rootedPath.getRoot())) {
+ Path path = rootedPath.asPath();
+ for (Path immutableDir : immutableDirs) {
+ if (path.startsWith(immutableDir)) {
+ return FileType.EXTERNAL_IMMUTABLE_FILE;
+ }
+ }
+ return FileType.EXTERNAL_MUTABLE_FILE;
+ }
+ return FileType.INTERNAL_FILE;
+ }
+
+ public boolean shouldAssumeImmutable(RootedPath rootedPath) {
+ return getFileType(rootedPath) == FileType.EXTERNAL_IMMUTABLE_FILE;
+ }
+
+ public void maybeAddDepOnBuildId(RootedPath rootedPath, SkyFunction.Environment env) {
+ if (getFileType(rootedPath) == FileType.EXTERNAL_MUTABLE_FILE) {
+ // For files outside the package roots that are not assumed to be immutable, add a dependency
+ // on the build_id. This is sufficient for correctness; all other files will be handled by
+ // diff awareness of their respective package path, but these files need to be addressed
+ // separately.
+ //
+ // Using the build_id here seems to introduce a performance concern because the upward
+ // transitive closure of these external files will get eagerly invalidated on each
+ // incremental build (e.g. if every file had a transitive dependency on the filesystem root,
+ // then we'd have a big performance problem). But this a non-issue by design:
+ // - We don't add a dependency on the parent directory at the package root boundary, so the
+ // only transitive dependencies from files inside the package roots to external files are
+ // through symlinks. So the upwards transitive closure of external files is small.
+ // - The only way external source files get into the skyframe graph in the first place is
+ // through symlinks outside the package roots, which we neither want to encourage nor
+ // optimize for since it is not common. So the set of external files is small.
+ //
+ // The above reasoning doesn't hold for bazel, because external repositories
+ // (e.g. new_local_repository) cause lots of external symlinks to be present in the build.
+ // So bazel pretends that these external repositories are immutable to avoid the performance
+ // penalty described above.
+ PrecomputedValue.dependOnBuildId(env);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileAndMetadataCache.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileAndMetadataCache.java
new file mode 100644
index 0000000..2a0de78
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileAndMetadataCache.java
@@ -0,0 +1,466 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Interner;
+import com.google.common.collect.Interners;
+import com.google.common.collect.Sets;
+import com.google.common.io.BaseEncoding;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.cache.Digest;
+import com.google.devtools.build.lib.actions.cache.DigestUtils;
+import com.google.devtools.build.lib.actions.cache.Metadata;
+import com.google.devtools.build.lib.actions.cache.MetadataHandler;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.FileStatusWithDigest;
+import com.google.devtools.build.lib.vfs.FileStatusWithDigestAdapter;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.lib.vfs.Symlinks;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.protobuf.ByteString;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import javax.annotation.Nullable;
+
+/**
+ * Cache provided by an {@link ActionExecutionFunction}, allowing Blaze to obtain data from the
+ * graph and to inject data (e.g. file digests) back into the graph.
+ *
+ * <p>Data for the action's inputs is injected into this cache on construction, using the graph as
+ * the source of truth.
+ *
+ * <p>As well, this cache collects data about the action's output files, which is used in three
+ * ways. First, it is served as requested during action execution, primarily by the {@code
+ * ActionCacheChecker} when determining if the action must be rerun, and then after the action is
+ * run, to gather information about the outputs. Second, it is accessed by {@link
+ * ArtifactFunction}s in order to construct {@link ArtifactValue}s. Third, the {@link
+ * FilesystemValueChecker} uses it to determine the set of output files to check for inter-build
+ * modifications. Because all these use cases are slightly different, we must occasionally store two
+ * versions of the data for a value (see {@link #getAdditionalOutputData} for more.
+ */
+@VisibleForTesting
+public class FileAndMetadataCache implements ActionInputFileCache, MetadataHandler {
+ /** This should never be read directly. Use {@link #getInputFileArtifactValue} instead. */
+ private final Map<Artifact, FileArtifactValue> inputArtifactData;
+ private final Map<Artifact, Collection<Artifact>> expandedInputMiddlemen;
+ private final File execRoot;
+ private final Map<ByteString, Artifact> reverseMap = new ConcurrentHashMap<>();
+ private final ConcurrentMap<Artifact, FileValue> outputArtifactData =
+ new ConcurrentHashMap<>();
+ // See #getAdditionalOutputData for documentation of this field.
+ private final ConcurrentMap<Artifact, FileArtifactValue> additionalOutputData =
+ new ConcurrentHashMap<>();
+ private final Set<Artifact> injectedArtifacts = Sets.newConcurrentHashSet();
+ private final ImmutableSet<Artifact> outputs;
+ @Nullable private final SkyFunction.Environment env;
+ private final TimestampGranularityMonitor tsgm;
+
+ private static final Interner<ByteString> BYTE_INTERNER = Interners.newWeakInterner();
+
+ public FileAndMetadataCache(Map<Artifact, FileArtifactValue> inputArtifactData,
+ Map<Artifact, Collection<Artifact>> expandedInputMiddlemen, File execRoot,
+ Iterable<Artifact> outputs, @Nullable SkyFunction.Environment env,
+ TimestampGranularityMonitor tsgm) {
+ this.inputArtifactData = Preconditions.checkNotNull(inputArtifactData);
+ this.expandedInputMiddlemen = Preconditions.checkNotNull(expandedInputMiddlemen);
+ this.execRoot = Preconditions.checkNotNull(execRoot);
+ this.outputs = ImmutableSet.copyOf(outputs);
+ this.env = env;
+ this.tsgm = tsgm;
+ }
+
+ @Override
+ public Metadata getMetadataMaybe(Artifact artifact) {
+ try {
+ return getMetadata(artifact);
+ } catch (IOException e) {
+ return null;
+ }
+ }
+
+ private static Metadata metadataFromValue(FileArtifactValue value) throws FileNotFoundException {
+ if (value == FileArtifactValue.MISSING_FILE_MARKER) {
+ throw new FileNotFoundException();
+ }
+ // If the file is empty or a directory, we need to return the mtime because the action cache
+ // uses mtime to determine if this artifact has changed. We do not optimize for this code
+ // path (by storing the mtime somewhere) because we eventually may be switching to use digests
+ // for empty files. We want this code path to go away somehow too for directories (maybe by
+ // implementing FileSet in Skyframe).
+ return value.getSize() > 0
+ ? new Metadata(value.getDigest())
+ : new Metadata(value.getModifiedTime());
+ }
+
+ @Override
+ public Metadata getMetadata(Artifact artifact) throws IOException {
+ Metadata metadata = getRealMetadata(artifact);
+ return artifact.isConstantMetadata() ? Metadata.CONSTANT_METADATA : metadata;
+ }
+
+ @Nullable
+ private FileArtifactValue getInputFileArtifactValue(ActionInput input) {
+ FileArtifactValue value = inputArtifactData.get(input);
+ if (value != null) {
+ return value;
+ }
+ if (outputs.contains(input)) {
+ // When this method is called to calculate the metadata of an artifact, the artifact may be an
+ // output artifact. Don't try to do anything then.
+ return null;
+ }
+ if (!(input instanceof Artifact)) {
+ // Maybe we're being asked for some strange constructed ActionInput coming from runfiles or
+ // similar. We have no information about such things.
+ return null;
+ }
+ // TODO(bazel-team): Remove this codepath once Skyframe has native input discovery, so all
+ // inputs will already have metadata known.
+ // ActionExecutionFunction may have passed in null environment if this action does not
+ // discover inputs. In which case we should not have gotten here.
+ Preconditions.checkNotNull(env, input);
+ Artifact artifact = (Artifact) input;
+ if (artifact.isSourceArtifact()) {
+ // We might have no artifact data for discovered source inputs, and it's not worth storing
+ // it in this cache, because it won't be reused across actions -- while we could request an
+ // artifact from the graph, we would have to be tolerant to it not yet being present in the
+ // graph yet, which adds complexity. Instead, we let the undeclared inputs handler stat it, so
+ // it can be reused.
+ return null;
+ } else {
+ // This getValue call is not expected to return null, because if the artifact is a
+ // transitive dependency of this action (as it should be), it will already have been built,
+ // so this call will return a value.
+ // This getValue call is theoretically less efficient for a subsequent incremental build
+ // than it would be to do a bulk getValues call after action execution, as is done for
+ // source inputs. However, since almost all nodes requested here are transitive deps of an
+ // already-declared dependency, change pruning will request these values serially, but they
+ // will already have been built. So the only penalty is restarting ParallelEvaluator#run, as
+ // opposed to traversing the entire downward transitive closure on a single thread.
+ value = (FileArtifactValue) env.getValue(
+ FileArtifactValue.key(artifact, /*argument ignored for derived artifacts*/false));
+ return value;
+ }
+ }
+
+ /**
+ * We cache data for constant-metadata artifacts, even though it is technically unnecessary,
+ * because the data stored in this cache is consumed by various parts of Blaze via the {@link
+ * ActionExecutionValue} (for now, {@link FilesystemValueChecker} and {@link ArtifactFunction}).
+ * It is simpler for those parts if every output of the action is present in the cache. However,
+ * we must not return the actual metadata for a constant-metadata artifact.
+ */
+ private Metadata getRealMetadata(Artifact artifact) throws IOException {
+ FileArtifactValue value = getInputFileArtifactValue(artifact);
+ if (value != null) {
+ return metadataFromValue(value);
+ }
+ if (artifact.isSourceArtifact()) {
+ // A discovered input we didn't have data for.
+ // TODO(bazel-team): Change this to an assertion once Skyframe has native input discovery, so
+ // all inputs will already have metadata known.
+ return null;
+ } else if (artifact.isMiddlemanArtifact()) {
+ // A middleman artifact's data was either already injected from the action cache checker using
+ // #setDigestForVirtualArtifact, or it has the default middleman value.
+ value = additionalOutputData.get(artifact);
+ if (value != null) {
+ return metadataFromValue(value);
+ }
+ value = FileArtifactValue.DEFAULT_MIDDLEMAN;
+ FileArtifactValue oldValue = additionalOutputData.putIfAbsent(artifact, value);
+ checkInconsistentData(artifact, oldValue, value);
+ return metadataFromValue(value);
+ }
+ FileValue fileValue = outputArtifactData.get(artifact);
+ if (fileValue != null) {
+ // Non-middleman artifacts should only have additionalOutputData if they have
+ // outputArtifactData. We don't assert this because of concurrency possibilities, but at least
+ // we don't check additionalOutputData unless we expect that we might see the artifact there.
+ value = additionalOutputData.get(artifact);
+ // If additional output data is present for this artifact, we use it in preference to the
+ // usual calculation.
+ if (value != null) {
+ return metadataFromValue(value);
+ }
+ if (!fileValue.exists()) {
+ throw new FileNotFoundException(artifact.prettyPrint() + " does not exist");
+ }
+ return new Metadata(Preconditions.checkNotNull(fileValue.getDigest(), artifact));
+ }
+ // We do not cache exceptions besides nonexistence here, because it is unlikely that the file
+ // will be requested from this cache too many times.
+ fileValue = fileValueFromArtifact(artifact, null, tsgm);
+ FileValue oldFileValue = outputArtifactData.putIfAbsent(artifact, fileValue);
+ checkInconsistentData(artifact, oldFileValue, value);
+ return maybeStoreAdditionalData(artifact, fileValue, null);
+ }
+
+ /** Expands one of the input middlemen artifacts of the corresponding action. */
+ public Collection<Artifact> expandInputMiddleman(Artifact middlemanArtifact) {
+ Preconditions.checkState(middlemanArtifact.isMiddlemanArtifact(), middlemanArtifact);
+ Collection<Artifact> result = expandedInputMiddlemen.get(middlemanArtifact);
+ // Note that result may be null for non-aggregating middlemen.
+ return result == null ? ImmutableSet.<Artifact>of() : result;
+ }
+
+ /**
+ * Check that the new {@code data} we just calculated for an {@code artifact} agrees with the
+ * {@code oldData} (presumably calculated concurrently), if it was present.
+ */
+ // Not private only because used by SkyframeActionExecutor's metadata handler.
+ static void checkInconsistentData(Artifact artifact,
+ @Nullable Object oldData, Object data) throws IOException {
+ if (oldData != null && !oldData.equals(data)) {
+ // Another thread checked this file since we looked at the map, and got a different answer
+ // than we did. Presumably the user modified the file between reads.
+ throw new IOException("Data for " + artifact.prettyPrint() + " changed to " + data
+ + " after it was calculated as " + oldData);
+ }
+ }
+
+ /**
+ * See {@link #getAdditionalOutputData} for why we sometimes need to store additional data, even
+ * for normal (non-middleman) artifacts.
+ */
+ @Nullable
+ private Metadata maybeStoreAdditionalData(Artifact artifact, FileValue data,
+ @Nullable byte[] injectedDigest) throws IOException {
+ if (!data.exists()) {
+ // Nonexistent files should only occur before executing an action.
+ throw new FileNotFoundException(artifact.prettyPrint() + " does not exist");
+ }
+ boolean isFile = data.isFile();
+ boolean useDigest = DigestUtils.useFileDigest(artifact, isFile, isFile ? data.getSize() : 0);
+ if (useDigest && data.getDigest() != null) {
+ // We do not need to store the FileArtifactValue separately -- the digest is in the file value
+ // and that is all that is needed for this file's metadata.
+ return new Metadata(data.getDigest());
+ }
+ // Unfortunately, the FileValue does not contain enough information for us to calculate the
+ // corresponding FileArtifactValue -- either the metadata must use the modified time, which we
+ // do not expose in the FileValue, or the FileValue didn't store the digest So we store the
+ // metadata separately.
+ // Use the FileValue's digest if no digest was injected, or if the file can't be digested.
+ injectedDigest = injectedDigest != null || !isFile ? injectedDigest : data.getDigest();
+ FileArtifactValue value =
+ FileArtifactValue.create(artifact, isFile, isFile ? data.getSize() : 0, injectedDigest);
+ FileArtifactValue oldValue = additionalOutputData.putIfAbsent(artifact, value);
+ checkInconsistentData(artifact, oldValue, value);
+ return metadataFromValue(value);
+ }
+
+ @Override
+ public void setDigestForVirtualArtifact(Artifact artifact, Digest digest) {
+ Preconditions.checkState(artifact.isMiddlemanArtifact(), artifact);
+ Preconditions.checkNotNull(digest, artifact);
+ additionalOutputData.put(artifact,
+ FileArtifactValue.createMiddleman(digest.asMetadata().digest));
+ }
+
+ @Override
+ public void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest) {
+ if (output instanceof Artifact) {
+ Artifact artifact = (Artifact) output;
+ Preconditions.checkState(injectedArtifacts.add(artifact), artifact);
+ FileValue fileValue;
+ try {
+ // This call may do an unnecessary call to Path#getFastDigest to see if the digest is
+ // readily available. We cannot pass the digest in, though, because if it is not available
+ // from the filesystem, this FileValue will not compare equal to another one created for the
+ // same file, because the other one will be missing its digest.
+ fileValue = fileValueFromArtifact(artifact, FileStatusWithDigestAdapter.adapt(statNoFollow),
+ tsgm);
+ byte[] fileDigest = fileValue.getDigest();
+ Preconditions.checkState(fileDigest == null || Arrays.equals(digest, fileDigest),
+ "%s %s %s", artifact, digest, fileDigest);
+ outputArtifactData.put(artifact, fileValue);
+ } catch (IOException e) {
+ // Do nothing - we just failed to inject metadata. Real error handling will be done later,
+ // when somebody will try to access that file.
+ return;
+ }
+ // If needed, insert additional data. Note that this can only be true if the file is empty or
+ // the filesystem does not support fast digests. Since we usually only inject digests when
+ // running with a filesystem that supports fast digests, this is fairly unlikely.
+ try {
+ maybeStoreAdditionalData(artifact, fileValue, digest);
+ } catch (IOException e) {
+ if (fileValue.getSize() != 0) {
+ // Empty files currently have their mtimes examined, and so could throw. No other files
+ // should throw, since all filesystem access has already been done.
+ throw new IllegalStateException(
+ "Filesystem should not have been accessed while injecting data for "
+ + artifact.prettyPrint(), e);
+ }
+ // Ignore exceptions for empty files, as above.
+ }
+ }
+ }
+
+ @Override
+ public void discardMetadata(Collection<Artifact> artifactList) {
+ Preconditions.checkState(injectedArtifacts.isEmpty(),
+ "Artifacts cannot be injected before action execution: %s", injectedArtifacts);
+ outputArtifactData.keySet().removeAll(artifactList);
+ additionalOutputData.keySet().removeAll(artifactList);
+ }
+
+ @Override
+ public boolean artifactExists(Artifact artifact) {
+ return getMetadataMaybe(artifact) != null;
+ }
+
+ @Override
+ public boolean isRegularFile(Artifact artifact) {
+ // Currently this method is used only for genrule input directory checks. If we need to call
+ // this on output artifacts too, this could be more efficient.
+ FileArtifactValue value = getInputFileArtifactValue(artifact);
+ if (value != null && value.getDigest() != null) {
+ return true;
+ }
+ return artifact.getPath().isFile();
+ }
+
+ @Override
+ public boolean isInjected(Artifact artifact) {
+ return injectedArtifacts.contains(artifact);
+ }
+
+ /**
+ * @return data for output files that was computed during execution. Should include data for all
+ * non-middleman artifacts.
+ */
+ Map<Artifact, FileValue> getOutputData() {
+ return outputArtifactData;
+ }
+
+ /**
+ * Returns data for any output files whose metadata was not computable from the corresponding
+ * entry in {@link #getOutputData}.
+ *
+ * <p>There are three reasons why we might not be able to compute metadata for an artifact from
+ * the FileValue. First, middleman artifacts have no corresponding FileValues. Second, if
+ * computing a file's digest is not fast, the FileValue does not do so, so a file on a filesystem
+ * without fast digests has to have its metadata stored separately. Third, some files' metadata
+ * (directories, empty files) contain their mtimes, which the FileValue does not expose, so that
+ * has to be stored separately.
+ *
+ * <p>Note that for files that need digests, we can't easily inject the digest in the FileValue
+ * because it would complicate equality-checking on subsequent builds -- if our filesystem doesn't
+ * do fast digests, the comparison value would not have a digest.
+ */
+ Map<Artifact, FileArtifactValue> getAdditionalOutputData() {
+ return additionalOutputData;
+ }
+
+ @Override
+ public long getSizeInBytes(ActionInput input) throws IOException {
+ FileArtifactValue metadata = getInputFileArtifactValue(input);
+ if (metadata != null) {
+ return metadata.getSize();
+ }
+ return -1;
+ }
+
+ @Nullable
+ @Override
+ public File getFileFromDigest(ByteString digest) throws IOException {
+ Artifact artifact = reverseMap.get(digest);
+ if (artifact != null) {
+ String relPath = artifact.getExecPathString();
+ return relPath.startsWith("/") ? new File(relPath) : new File(execRoot, relPath);
+ }
+ return null;
+ }
+
+ @Nullable
+ @Override
+ public ByteString getDigest(ActionInput input) throws IOException {
+ FileArtifactValue value = getInputFileArtifactValue(input);
+ if (value != null) {
+ byte[] bytes = value.getDigest();
+ if (bytes != null) {
+ ByteString digest = ByteString.copyFrom(BaseEncoding.base16().lowerCase().encode(bytes)
+ .getBytes(StandardCharsets.US_ASCII));
+ reverseMap.put(BYTE_INTERNER.intern(digest), (Artifact) input);
+ return digest;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public boolean contentsAvailableLocally(ByteString digest) {
+ return reverseMap.containsKey(digest);
+ }
+
+ static FileValue fileValueFromArtifact(Artifact artifact,
+ @Nullable FileStatusWithDigest statNoFollow, TimestampGranularityMonitor tsgm)
+ throws IOException {
+ Path path = artifact.getPath();
+ RootedPath rootedPath =
+ RootedPath.toRootedPath(artifact.getRoot().getPath(), artifact.getRootRelativePath());
+ if (statNoFollow == null) {
+ statNoFollow = FileStatusWithDigestAdapter.adapt(path.statIfFound(Symlinks.NOFOLLOW));
+ if (statNoFollow == null) {
+ return FileValue.value(rootedPath, FileStateValue.NONEXISTENT_FILE_STATE_NODE,
+ rootedPath, FileStateValue.NONEXISTENT_FILE_STATE_NODE);
+ }
+ }
+ Path realPath = path;
+ // We use FileStatus#isSymbolicLink over Path#isSymbolicLink to avoid the unnecessary stat
+ // done by the latter.
+ if (statNoFollow.isSymbolicLink()) {
+ realPath = path.resolveSymbolicLinks();
+ // We need to protect against symlink cycles since FileValue#value assumes it's dealing with a
+ // file that's not in a symlink cycle.
+ if (realPath.equals(path)) {
+ throw new IOException("symlink cycle");
+ }
+ }
+ RootedPath realRootedPath = RootedPath.toRootedPathMaybeUnderRoot(realPath,
+ ImmutableList.of(artifact.getRoot().getPath()));
+ FileStateValue fileStateValue;
+ FileStateValue realFileStateValue;
+ try {
+ fileStateValue = FileStateValue.createWithStatNoFollow(rootedPath, statNoFollow, tsgm);
+ // TODO(bazel-team): consider avoiding a 'stat' here when the symlink target hasn't changed
+ // and is a source file (since changes to those are checked separately).
+ realFileStateValue = realPath.equals(path) ? fileStateValue
+ : FileStateValue.create(realRootedPath, tsgm);
+ } catch (InconsistentFilesystemException e) {
+ throw new IOException(e);
+ }
+ return FileValue.value(rootedPath, fileStateValue, realRootedPath, realFileStateValue);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java
new file mode 100644
index 0000000..7acc38c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java
@@ -0,0 +1,148 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.cache.DigestUtils;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import javax.annotation.Nullable;
+
+/**
+ * Stores the data of an artifact corresponding to a file. This file may be an ordinary file, in
+ * which case we would expect to see a digest and size; a directory, in which case we would expect
+ * to see an mtime; or an empty file, where we would expect to see a size (=0), mtime, and digest
+ */
+public class FileArtifactValue extends ArtifactValue {
+ /** Data for Middleman artifacts that did not have data specified. */
+ static final FileArtifactValue DEFAULT_MIDDLEMAN = new FileArtifactValue(null, 0, 0);
+ /** Data that marks that a file is not present on the filesystem. */
+ static final FileArtifactValue MISSING_FILE_MARKER = new FileArtifactValue(null, 1, 0);
+
+ @Nullable private final byte[] digest;
+ private final long mtime;
+ private final long size;
+
+ private FileArtifactValue(byte[] digest, long size) {
+ Preconditions.checkState(size >= 0, "size must be non-negative: %s %s", digest, size);
+ this.digest = Preconditions.checkNotNull(digest, size);
+ this.size = size;
+ this.mtime = -1;
+ }
+
+ // Only used by empty files (non-null digest) and directories (null digest).
+ private FileArtifactValue(byte[] digest, long mtime, long size) {
+ Preconditions.checkState(mtime >= 0, "mtime must be non-negative: %s %s", mtime, size);
+ Preconditions.checkState(size == 0, "size must be zero: %s %s", mtime, size);
+ this.digest = digest;
+ this.size = size;
+ this.mtime = mtime;
+ }
+
+ static FileArtifactValue create(Artifact artifact) throws IOException {
+ Path path = artifact.getPath();
+ FileStatus stat = path.stat();
+ boolean isFile = stat.isFile();
+ return create(artifact, isFile, isFile ? stat.getSize() : 0, null);
+ }
+
+ static FileArtifactValue create(Artifact artifact, FileValue fileValue) throws IOException {
+ boolean isFile = fileValue.isFile();
+ return create(artifact, isFile, isFile ? fileValue.getSize() : 0,
+ isFile ? fileValue.getDigest() : null);
+ }
+
+ static FileArtifactValue create(Artifact artifact, boolean isFile, long size,
+ @Nullable byte[] digest) throws IOException {
+ if (isFile && digest == null) {
+ digest = DigestUtils.getDigestOrFail(artifact.getPath(), size);
+ }
+ if (!DigestUtils.useFileDigest(artifact, isFile, size)) {
+ // In this case, we need to store the mtime because the action cache uses mtime to determine
+ // if this artifact has changed. This is currently true for empty files and directories. We
+ // do not optimize for this code path (by storing the mtime in a FileValue) because we do not
+ // like it and may remove this special-casing for empty files in the future. We want this code
+ // path to go away somehow too for directories (maybe by implementing FileSet
+ // in Skyframe)
+ return new FileArtifactValue(digest, artifact.getPath().getLastModifiedTime(), size);
+ }
+ Preconditions.checkState(digest != null, artifact);
+ return new FileArtifactValue(digest, size);
+ }
+
+ static FileArtifactValue createMiddleman(byte[] digest) {
+ Preconditions.checkNotNull(digest);
+ // The Middleman artifact values have size 1 because we want their digests to be used. This hack
+ // can be removed once empty files are digested.
+ return new FileArtifactValue(digest, /*size=*/1);
+ }
+
+ @Nullable
+ byte[] getDigest() {
+ return digest;
+ }
+
+ /** Gets the size of the file. Directories have size 0. */
+ long getSize() {
+ return size;
+ }
+
+ /**
+ * Gets last modified time of file. Should only be called if {@link DigestUtils#useFileDigest} was
+ * false for this artifact -- namely, either it is a directory or an empty file. Note that since
+ * we store directory sizes as 0, all files for which this method can be called have size 0.
+ */
+ long getModifiedTime() {
+ Preconditions.checkState(size == 0, "%s %s %s", digest, mtime, size);
+ return mtime;
+ }
+
+ @Override
+ public int hashCode() {
+ // Hash digest by content, not reference. Note that digest is the only array in this array.
+ return Arrays.deepHashCode(new Object[] {size, mtime, digest});
+ }
+
+ /**
+ * Two FileArtifactValues will only compare equal if they have the same content. This differs
+ * from the {@code Metadata#equivalence} method, which allows for comparison using mtime if
+ * one object does not have a digest available.
+ */
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (!(other instanceof FileArtifactValue)) {
+ return false;
+ }
+ FileArtifactValue that = (FileArtifactValue) other;
+ return this.mtime == that.mtime && this.size == that.size
+ && Arrays.equals(this.digest, that.digest);
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(FileArtifactValue.class)
+ .add("digest", digest)
+ .add("mtime", mtime)
+ .add("size", size).toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileContentsProxy.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileContentsProxy.java
new file mode 100644
index 0000000..344c364
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileContentsProxy.java
@@ -0,0 +1,66 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * In case we can't get a fast digest from the filesystem, we store this metadata as a proxy to
+ * the file contents. Currently it is a pair of the mtime and "value id" (which is right now just
+ * the ivalue number). We may wish to augment this object with the following data:
+ * a. the device number
+ * b. the ctime, which cannot be tampered with in userspace
+ *
+ * <p>For an example of why mtime alone is insufficient, note that 'mv' preserves timestamps. So if
+ * files 'a' and 'b' initially have the same timestamp, then we would think 'b' is unchanged after
+ * the user executes `mv a b` between two builds.
+ */
+public final class FileContentsProxy implements Serializable {
+ private final long mtime;
+ private final long valueId;
+
+ private FileContentsProxy(long mtime, long valueId) {
+ this.mtime = mtime;
+ this.valueId = valueId;
+ }
+
+ public static FileContentsProxy create(long mtime, long valueId) {
+ return new FileContentsProxy(mtime, valueId);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+
+ if (!(other instanceof FileContentsProxy)) {
+ return false;
+ }
+
+ FileContentsProxy that = (FileContentsProxy) other;
+ return mtime == that.mtime && valueId == that.valueId;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(mtime, valueId);
+ }
+
+ @Override
+ public String toString() {
+ return "mtime: " + mtime + " valueId: " + valueId;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileFunction.java
new file mode 100644
index 0000000..ad3cb74
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileFunction.java
@@ -0,0 +1,217 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.LinkedHashSet;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * A {@link SkyFunction} for {@link FileValue}s.
+ *
+ * <p>Most of the complexity in the implementation is associated to handling symlinks. Namely,
+ * this class makes sure that {@code FileValue}s corresponding to symlinks are correctly invalidated
+ * if the destination of the symlink is invalidated. Directory symlinks are also covered.
+ */
+public class FileFunction implements SkyFunction {
+
+ private final AtomicReference<PathPackageLocator> pkgLocator;
+ private final ExternalFilesHelper externalFilesHelper;
+
+ public FileFunction(AtomicReference<PathPackageLocator> pkgLocator,
+ ExternalFilesHelper externalFilesHelper) {
+ this.pkgLocator = pkgLocator;
+ this.externalFilesHelper = externalFilesHelper;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws FileFunctionException {
+ RootedPath rootedPath = (RootedPath) skyKey.argument();
+ RootedPath realRootedPath = rootedPath;
+ FileStateValue realFileStateValue = null;
+ PathFragment relativePath = rootedPath.getRelativePath();
+
+ // Resolve ancestor symlinks, but only if the current file is not the filesystem root (has no
+ // parent) or a package path root (treated opaquely and handled by skyframe's DiffAwareness
+ // interface) or otherwise assumed to be immutable (handling ancestors would add dependencies
+ // too aggressively). Note that this is the first thing we do - if an ancestor is part of a
+ // symlink cycle, we want to detect that quickly as it gives a more informative error message
+ // than we'd get doing bogus filesystem operations.
+ if (!relativePath.equals(PathFragment.EMPTY_FRAGMENT)
+ && !externalFilesHelper.shouldAssumeImmutable(rootedPath)) {
+ Pair<RootedPath, FileStateValue> resolvedState =
+ resolveFromAncestors(rootedPath, env);
+ if (resolvedState == null) {
+ return null;
+ }
+ realRootedPath = resolvedState.getFirst();
+ realFileStateValue = resolvedState.getSecond();
+ }
+
+ FileStateValue fileStateValue = (FileStateValue) env.getValue(FileStateValue.key(rootedPath));
+ if (fileStateValue == null) {
+ return null;
+ }
+ if (realFileStateValue == null) {
+ realFileStateValue = fileStateValue;
+ }
+
+ LinkedHashSet<RootedPath> seenPaths = Sets.newLinkedHashSet();
+ while (realFileStateValue.getType().equals(FileStateValue.Type.SYMLINK)) {
+ if (!seenPaths.add(realRootedPath)) {
+ FileSymlinkCycleException fileSymlinkCycleException =
+ makeFileSymlinkCycleException(realRootedPath, seenPaths);
+ if (env.getValue(FileSymlinkCycleUniquenessValue.key(fileSymlinkCycleException.getCycle()))
+ == null) {
+ // Note that this dependency is merely to ensure that each unique cycle gets reported
+ // exactly once.
+ return null;
+ }
+ throw new FileFunctionException(fileSymlinkCycleException);
+ }
+ Pair<RootedPath, FileStateValue> resolvedState = getSymlinkTargetRootedPath(realRootedPath,
+ realFileStateValue.getSymlinkTarget(), env);
+ if (resolvedState == null) {
+ return null;
+ }
+ realRootedPath = resolvedState.getFirst();
+ realFileStateValue = resolvedState.getSecond();
+ }
+ return FileValue.value(rootedPath, fileStateValue, realRootedPath, realFileStateValue);
+ }
+
+ /**
+ * Returns the path and file state of {@code rootedPath}, accounting for ancestor symlinks, or
+ * {@code null} if there was a missing dep.
+ */
+ @Nullable
+ private Pair<RootedPath, FileStateValue> resolveFromAncestors(RootedPath rootedPath,
+ Environment env) throws FileFunctionException {
+ PathFragment relativePath = rootedPath.getRelativePath();
+ RootedPath realRootedPath = rootedPath;
+ FileValue parentFileValue = null;
+ if (!relativePath.equals(PathFragment.EMPTY_FRAGMENT)) {
+ RootedPath parentRootedPath = RootedPath.toRootedPath(rootedPath.getRoot(),
+ relativePath.getParentDirectory());
+ parentFileValue = (FileValue) env.getValue(FileValue.key(parentRootedPath));
+ if (parentFileValue == null) {
+ return null;
+ }
+ PathFragment baseName = new PathFragment(relativePath.getBaseName());
+ RootedPath parentRealRootedPath = parentFileValue.realRootedPath();
+ realRootedPath = RootedPath.toRootedPath(parentRealRootedPath.getRoot(),
+ parentRealRootedPath.getRelativePath().getRelative(baseName));
+ }
+ FileStateValue realFileStateValue =
+ (FileStateValue) env.getValue(FileStateValue.key(realRootedPath));
+ if (realFileStateValue == null) {
+ return null;
+ }
+ if (realFileStateValue.getType() != FileStateValue.Type.NONEXISTENT
+ && parentFileValue != null && !parentFileValue.isDirectory()) {
+ String type = realFileStateValue.getType().toString().toLowerCase();
+ String message = type + " " + rootedPath.asPath() + " exists but its parent "
+ + "directory " + parentFileValue.realRootedPath().asPath() + " doesn't exist.";
+ throw new FileFunctionException(new InconsistentFilesystemException(message),
+ Transience.TRANSIENT);
+ }
+ return Pair.of(realRootedPath, realFileStateValue);
+ }
+
+ /**
+ * Returns the symlink target and file state of {@code rootedPath}'s symlink to
+ * {@code symlinkTarget}, accounting for ancestor symlinks, or {@code null} if there was a
+ * missing dep.
+ */
+ @Nullable
+ private Pair<RootedPath, FileStateValue> getSymlinkTargetRootedPath(RootedPath rootedPath,
+ PathFragment symlinkTarget, Environment env) throws FileFunctionException {
+ if (symlinkTarget.isAbsolute()) {
+ Path path = rootedPath.asPath().getFileSystem().getRootDirectory().getRelative(
+ symlinkTarget);
+ return resolveFromAncestors(
+ RootedPath.toRootedPathMaybeUnderRoot(path, pkgLocator.get().getPathEntries()), env);
+ }
+ Path path = rootedPath.asPath();
+ Path symlinkTargetPath;
+ if (path.getParentDirectory() != null) {
+ RootedPath parentRootedPath = RootedPath.toRootedPathMaybeUnderRoot(
+ path.getParentDirectory(), pkgLocator.get().getPathEntries());
+ FileValue parentFileValue = (FileValue) env.getValue(FileValue.key(parentRootedPath));
+ if (parentFileValue == null) {
+ return null;
+ }
+ symlinkTargetPath = parentFileValue.realRootedPath().asPath().getRelative(symlinkTarget);
+ } else {
+ // This means '/' is a symlink to 'symlinkTarget'.
+ symlinkTargetPath = path.getRelative(symlinkTarget);
+ }
+ RootedPath symlinkTargetRootedPath = RootedPath.toRootedPathMaybeUnderRoot(symlinkTargetPath,
+ pkgLocator.get().getPathEntries());
+ return resolveFromAncestors(symlinkTargetRootedPath, env);
+ }
+
+ private FileSymlinkCycleException makeFileSymlinkCycleException(RootedPath startOfCycle,
+ Iterable<RootedPath> symlinkPaths) {
+ boolean inPathToCycle = true;
+ ImmutableList.Builder<RootedPath> pathToCycleBuilder = ImmutableList.builder();
+ ImmutableList.Builder<RootedPath> cycleBuilder = ImmutableList.builder();
+ for (RootedPath path : symlinkPaths) {
+ if (path.equals(startOfCycle)) {
+ inPathToCycle = false;
+ }
+ if (inPathToCycle) {
+ pathToCycleBuilder.add(path);
+ } else {
+ cycleBuilder.add(path);
+ }
+ }
+ return new FileSymlinkCycleException(pathToCycleBuilder.build(), cycleBuilder.build());
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link FileFunction#compute}.
+ */
+ private static final class FileFunctionException extends SkyFunctionException {
+
+ public FileFunctionException(InconsistentFilesystemException e, Transience transience) {
+ super(e, transience);
+ }
+
+ public FileFunctionException(FileSymlinkCycleException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileStateFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateFunction.java
new file mode 100644
index 0000000..ec2e871
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateFunction.java
@@ -0,0 +1,76 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+
+/**
+ * A {@link SkyFunction} for {@link FileStateValue}s.
+ *
+ * <p>Merely calls FileStateValue#create, but also has special handling for files outside the
+ * package roots (see {@link ExternalFilesHelper}).
+ */
+public class FileStateFunction implements SkyFunction {
+
+ private final TimestampGranularityMonitor tsgm;
+ private final ExternalFilesHelper externalFilesHelper;
+
+ public FileStateFunction(TimestampGranularityMonitor tsgm,
+ ExternalFilesHelper externalFilesHelper) {
+ this.tsgm = tsgm;
+ this.externalFilesHelper = externalFilesHelper;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws FileStateFunctionException {
+ RootedPath rootedPath = (RootedPath) skyKey.argument();
+ externalFilesHelper.maybeAddDepOnBuildId(rootedPath, env);
+ if (env.valuesMissing()) {
+ return null;
+ }
+ try {
+ return FileStateValue.create(rootedPath, tsgm);
+ } catch (IOException e) {
+ throw new FileStateFunctionException(e);
+ } catch (InconsistentFilesystemException e) {
+ throw new FileStateFunctionException(e);
+ }
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link FileStateFunction#compute}.
+ */
+ private static final class FileStateFunctionException extends SkyFunctionException {
+ public FileStateFunctionException(IOException e) {
+ super(e, Transience.TRANSIENT);
+ }
+
+ public FileStateFunctionException(InconsistentFilesystemException e) {
+ super(e, Transience.TRANSIENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileStateValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateValue.java
new file mode 100644
index 0000000..8631ff3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateValue.java
@@ -0,0 +1,317 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.FileStatusWithDigest;
+import com.google.devtools.build.lib.vfs.FileStatusWithDigestAdapter;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.lib.vfs.Symlinks;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * Encapsulates the filesystem operations needed to get state for a path. This is at least a
+ * 'lstat' to determine what type of file the path is.
+ * <ul>
+ * <li> For a non-existent file, the non existence is noted.
+ * <li> For a symlink, the symlink target is noted.
+ * <li> For a directory, the existence is noted.
+ * <li> For a file, the existence is noted, along with metadata about the file (e.g.
+ * file digest). See {@link FileFileStateValue}.
+ * <ul>
+ *
+ * <p>This class is an implementation detail of {@link FileValue} and should not be used outside of
+ * {@link FileFunction}. Instead, {@link FileValue} should be used by consumers that care about
+ * files.
+ *
+ * <p>All subclasses must implement {@link #equals} and {@link #hashCode} properly.
+ */
+abstract class FileStateValue implements SkyValue {
+
+ public static final FileStateValue DIRECTORY_FILE_STATE_NODE = DirectoryFileStateValue.INSTANCE;
+ public static final FileStateValue NONEXISTENT_FILE_STATE_NODE =
+ NonexistentFileStateValue.INSTANCE;
+
+ public enum Type {
+ FILE,
+ DIRECTORY,
+ SYMLINK,
+ NONEXISTENT,
+ }
+
+ protected FileStateValue() {
+ }
+
+ public static FileStateValue create(RootedPath rootedPath,
+ @Nullable TimestampGranularityMonitor tsgm) throws InconsistentFilesystemException,
+ IOException {
+ Path path = rootedPath.asPath();
+ // Stat, but don't throw an exception for the common case of a nonexistent file. This still
+ // throws an IOException in case any other IO error is encountered.
+ FileStatus stat = path.statIfFound(Symlinks.NOFOLLOW);
+ if (stat == null) {
+ return NONEXISTENT_FILE_STATE_NODE;
+ }
+ return createWithStatNoFollow(rootedPath, FileStatusWithDigestAdapter.adapt(stat), tsgm);
+ }
+
+ public static FileStateValue createWithStatNoFollow(RootedPath rootedPath,
+ FileStatusWithDigest statNoFollow, @Nullable TimestampGranularityMonitor tsgm)
+ throws InconsistentFilesystemException, IOException {
+ Path path = rootedPath.asPath();
+ if (statNoFollow.isFile()) {
+ return FileFileStateValue.fromPath(path, statNoFollow, tsgm);
+ } else if (statNoFollow.isDirectory()) {
+ return DIRECTORY_FILE_STATE_NODE;
+ } else if (statNoFollow.isSymbolicLink()) {
+ return new SymlinkFileStateValue(path.readSymbolicLink());
+ }
+ throw new InconsistentFilesystemException("according to stat, existing path " + path + " is "
+ + "neither a file nor directory nor symlink.");
+ }
+
+ @ThreadSafe
+ static SkyKey key(RootedPath rootedPath) {
+ return new SkyKey(SkyFunctions.FILE_STATE, rootedPath);
+ }
+
+ abstract Type getType();
+
+ PathFragment getSymlinkTarget() {
+ throw new IllegalStateException();
+ }
+
+ long getSize() {
+ throw new IllegalStateException();
+ }
+
+ @Nullable
+ byte[] getDigest() {
+ throw new IllegalStateException();
+ }
+
+ /**
+ * Implementation of {@link FileStateValue} for files that exist.
+ *
+ * <p>A union of (digest, mtime). We use digests only if a fast digest lookup is available from
+ * the filesystem. If not, we fall back to mtime-based digests. This avoids the case where Blaze
+ * must read all files involved in the build in order to check for modifications in the case
+ * where fast digest lookups are not available.
+ */
+ @ThreadSafe
+ private static final class FileFileStateValue extends FileStateValue {
+ private final long size;
+ // Only needed for empty-file equality-checking. Otherwise is always -1.
+ // TODO(bazel-team): Consider getting rid of this special case for empty files.
+ private final long mtime;
+ @Nullable private final byte[] digest;
+ @Nullable private final FileContentsProxy contentsProxy;
+
+ private FileFileStateValue(long size, long mtime, byte[] digest,
+ FileContentsProxy contentsProxy) {
+ Preconditions.checkState((digest == null) != (contentsProxy == null));
+ this.size = size;
+ // mtime is forced to be -1 so that we do not accidentally depend on it for non-empty files,
+ // which should only be compared using digests.
+ this.mtime = size == 0 ? mtime : -1;
+ this.digest = digest;
+ this.contentsProxy = contentsProxy;
+ }
+
+ /**
+ * Create a FileFileStateValue instance corresponding to the given existing file.
+ * @param stat must be of type "File". (Not a symlink).
+ */
+ public static FileFileStateValue fromPath(Path path, FileStatusWithDigest stat,
+ @Nullable TimestampGranularityMonitor tsgm)
+ throws InconsistentFilesystemException {
+ Preconditions.checkState(stat.isFile(), path);
+ try {
+ byte[] digest = stat.getDigest();
+ if (digest == null) {
+ digest = path.getFastDigest();
+ }
+ if (digest == null) {
+ long mtime = stat.getLastModifiedTime();
+ // Note that TimestampGranularityMonitor#notifyDependenceOnFileTime is a thread-safe
+ // method.
+ if (tsgm != null) {
+ tsgm.notifyDependenceOnFileTime(mtime);
+ }
+ return new FileFileStateValue(stat.getSize(), stat.getLastModifiedTime(), null,
+ FileContentsProxy.create(mtime, stat.getNodeId()));
+ } else {
+ // We are careful here to avoid putting the value ID into FileMetadata if we already have
+ // a digest. Arbitrary filesystems may do weird things with the value ID; a digest is more
+ // robust.
+ return new FileFileStateValue(stat.getSize(), stat.getLastModifiedTime(), digest, null);
+ }
+ } catch (IOException e) {
+ String errorMessage = e.getMessage() != null
+ ? "error '" + e.getMessage() + "'" : "an error";
+ throw new InconsistentFilesystemException("'stat' said " + path + " is a file but then we "
+ + "later encountered " + errorMessage + " which indicates that " + path + " no longer "
+ + "exists. Did you delete it during the build?");
+ }
+ }
+
+ @Override
+ public Type getType() {
+ return Type.FILE;
+ }
+
+ @Override
+ public long getSize() {
+ return size;
+ }
+
+ @Override
+ @Nullable
+ public byte[] getDigest() {
+ return digest;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof FileFileStateValue) {
+ FileFileStateValue other = (FileFileStateValue) obj;
+ return size == other.size && mtime == other.mtime && Arrays.equals(digest, other.digest)
+ && Objects.equals(contentsProxy, other.contentsProxy);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(size, mtime, Arrays.hashCode(digest), contentsProxy);
+ }
+
+ @Override
+ public String toString() {
+ return "[size: " + size + " " + (mtime != -1 ? "mtime: " + mtime : "")
+ + (digest != null ? "digest: " + Arrays.toString(digest) : contentsProxy) + "]";
+ }
+ }
+
+ /** Implementation of {@link FileStateValue} for directories that exist. */
+ private static final class DirectoryFileStateValue extends FileStateValue {
+
+ public static final DirectoryFileStateValue INSTANCE = new DirectoryFileStateValue();
+
+ private DirectoryFileStateValue() {
+ }
+
+ @Override
+ public Type getType() {
+ return Type.DIRECTORY;
+ }
+
+ @Override
+ public String toString() {
+ return "directory";
+ }
+
+ // This object is normally a singleton, but deserialization produces copies.
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof DirectoryFileStateValue;
+ }
+
+ @Override
+ public int hashCode() {
+ return 7654321;
+ }
+ }
+
+ /** Implementation of {@link FileStateValue} for symlinks. */
+ private static final class SymlinkFileStateValue extends FileStateValue {
+
+ private final PathFragment symlinkTarget;
+
+ private SymlinkFileStateValue(PathFragment symlinkTarget) {
+ this.symlinkTarget = symlinkTarget;
+ }
+
+ @Override
+ public Type getType() {
+ return Type.SYMLINK;
+ }
+
+ @Override
+ public PathFragment getSymlinkTarget() {
+ return symlinkTarget;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof SymlinkFileStateValue)) {
+ return false;
+ }
+ SymlinkFileStateValue other = (SymlinkFileStateValue) obj;
+ return symlinkTarget.equals(other.symlinkTarget);
+ }
+
+ @Override
+ public int hashCode() {
+ return symlinkTarget.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "symlink to " + symlinkTarget;
+ }
+ }
+
+ /** Implementation of {@link FileStateValue} for nonexistent files. */
+ private static final class NonexistentFileStateValue extends FileStateValue {
+
+ public static final NonexistentFileStateValue INSTANCE = new NonexistentFileStateValue();
+
+ private NonexistentFileStateValue() {
+ }
+
+ @Override
+ public Type getType() {
+ return Type.NONEXISTENT;
+ }
+
+ @Override
+ public String toString() {
+ return "nonexistent";
+ }
+
+ // This object is normally a singleton, but deserialization produces copies.
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof NonexistentFileStateValue;
+ }
+
+ @Override
+ public int hashCode() {
+ return 8765432;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleException.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleException.java
new file mode 100644
index 0000000..d57fc42
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleException.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.vfs.RootedPath;
+
+/** Exception indicating that a cycle was found in the filesystem. */
+public class FileSymlinkCycleException extends Exception {
+
+ private final ImmutableList<RootedPath> pathToCycle;
+ private final ImmutableList<RootedPath> cycle;
+
+ FileSymlinkCycleException(ImmutableList<RootedPath> pathToCycle,
+ ImmutableList<RootedPath> cycle) {
+ // The cycle itself has already been reported by FileSymlinkCycleUniquenessValue, but we still
+ // want to have a readable #getMessage.
+ super("Symlink cycle");
+ this.pathToCycle = pathToCycle;
+ this.cycle = cycle;
+ }
+
+ /**
+ * The symlink path to the symlink cycle. For example, suppose 'a' -> 'b' -> 'c' -> 'd' -> 'c'.
+ * The path to the cycle is 'a', 'b'.
+ */
+ ImmutableList<RootedPath> getPathToCycle() {
+ return pathToCycle;
+ }
+
+ /**
+ * The symlink cycle. For example, suppose 'a' -> 'b' -> 'c' -> 'd' -> 'c'.
+ * The cycle is 'c', 'd'.
+ */
+ ImmutableList<RootedPath> getCycle() {
+ return cycle;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessFunction.java
new file mode 100644
index 0000000..a0604b5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessFunction.java
@@ -0,0 +1,45 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/** A value builder that has the side effect of reporting a file symlink cycle. */
+public class FileSymlinkCycleUniquenessFunction implements SkyFunction {
+
+ @SuppressWarnings("unchecked") // Cast from Object to ImmutableList<RootedPath>.
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ StringBuilder cycleMessage = new StringBuilder("circular symlinks detected\n");
+ cycleMessage.append("[start of symlink cycle]\n");
+ for (RootedPath rootedPath : (ImmutableList<RootedPath>) skyKey.argument()) {
+ cycleMessage.append(rootedPath.asPath() + "\n");
+ }
+ cycleMessage.append("[end of symlink cycle]");
+ // The purpose of this value builder is the side effect of emitting an error message exactly
+ // once per build per unique cycle.
+ env.getListener().handle(Event.error(cycleMessage.toString()));
+ return FileSymlinkCycleUniquenessValue.INSTANCE;
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessValue.java
new file mode 100644
index 0000000..627276d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessValue.java
@@ -0,0 +1,57 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A value for ensuring that a file symlink cycle is reported exactly once. This is achieved by
+ * forcing the same value key for two logically equivalent cycles (e.g. ['a' -> 'b' -> 'c' -> 'a']
+ * and ['b' -> 'c' -> 'a' -> 'b'], and letting Skyframe do its magic.
+ */
+class FileSymlinkCycleUniquenessValue implements SkyValue {
+
+ public static final FileSymlinkCycleUniquenessValue INSTANCE =
+ new FileSymlinkCycleUniquenessValue();
+
+ private FileSymlinkCycleUniquenessValue() {
+ }
+
+ static SkyKey key(ImmutableList<RootedPath> cycle) {
+ Preconditions.checkState(!cycle.isEmpty());
+ return new SkyKey(SkyFunctions.FILE_SYMLINK_CYCLE_UNIQUENESS, canonicalize(cycle));
+ }
+
+ private static ImmutableList<RootedPath> canonicalize(ImmutableList<RootedPath> cycle) {
+ int minPos = 0;
+ String minString = cycle.get(0).toString();
+ for (int i = 1; i < cycle.size(); i++) {
+ String candidateString = cycle.get(i).toString();
+ if (candidateString.compareTo(minString) < 0) {
+ minPos = i;
+ minString = candidateString;
+ }
+ }
+ ImmutableList.Builder<RootedPath> builder = ImmutableList.builder();
+ for (int i = 0; i < cycle.size(); i++) {
+ int pos = (minPos + i) % cycle.size();
+ builder.add(cycle.get(pos));
+ }
+ return builder.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileValue.java
new file mode 100644
index 0000000..1850fd9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileValue.java
@@ -0,0 +1,279 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.skyframe.FileStateValue.Type;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * A value that corresponds to a file (or directory or symlink or non-existent file), fully
+ * accounting for symlinks (e.g. proper dependencies on ancestor symlinks so as to be incrementally
+ * correct). Anything in Skyframe that cares about the fully resolved path of a file (e.g. anything
+ * that cares about the contents of a file) should have a dependency on the corresponding
+ * {@link FileValue}.
+ *
+ * <p>
+ * Note that the existence of a file value does not imply that the file exists on the filesystem.
+ * File values for missing files will be created on purpose in order to facilitate incremental
+ * builds in the case those files have reappeared.
+ *
+ * <p>
+ * This class contains the relevant metadata for a file, although not the contents. Note that
+ * since a FileValue doesn't store its corresponding SkyKey, it's possible for the FileValues for
+ * two different paths to be the same.
+ *
+ * <p>
+ * This should not be used for build outputs; use {@link ArtifactValue} for those.
+ */
+@Immutable
+@ThreadSafe
+public abstract class FileValue implements SkyValue {
+
+ boolean exists() {
+ return realFileStateValue().getType() != Type.NONEXISTENT;
+ }
+
+ public boolean isSymlink() {
+ return false;
+ }
+
+ /**
+ * Returns true if this value corresponds to a file or symlink to an existing file. If so, its
+ * parent directory is guaranteed to exist.
+ */
+ public boolean isFile() {
+ return realFileStateValue().getType() == Type.FILE;
+ }
+
+ /**
+ * Returns true if the file is a directory or a symlink to an existing directory. If so, its
+ * parent directory is guaranteed to exist.
+ */
+ public boolean isDirectory() {
+ return realFileStateValue().getType() == Type.DIRECTORY;
+ }
+
+ /**
+ * Returns the real rooted path of the file, taking ancestor symlinks into account. For example,
+ * the rooted path ['root']/['a/b'] is really ['root']/['c/b'] if 'a' is a symlink to 'b'. Note
+ * that ancestor symlinks outside the root boundary are not taken into consideration.
+ */
+ public abstract RootedPath realRootedPath();
+
+ abstract FileStateValue realFileStateValue();
+
+ /**
+ * Returns the unresolved link target if {@link #isSymlink()}.
+ *
+ * <p>This is useful if the caller wants to, for example, duplicate a relative symlink. An actual
+ * example could be a build rule that copies a set of input files to the output directory, but
+ * upon encountering symbolic links it can decide between copying or following them.
+ */
+ PathFragment getUnresolvedLinkTarget() {
+ throw new IllegalStateException(this.toString());
+ }
+
+ long getSize() {
+ Preconditions.checkState(isFile(), this);
+ return realFileStateValue().getSize();
+ }
+
+ @Nullable
+ byte[] getDigest() {
+ Preconditions.checkState(isFile(), this);
+ return realFileStateValue().getDigest();
+ }
+
+ /**
+ * Returns a key for building a file value for the given root-relative path.
+ */
+ @ThreadSafe
+ public static SkyKey key(RootedPath rootedPath) {
+ return new SkyKey(SkyFunctions.FILE, rootedPath);
+ }
+
+ /**
+ * Only intended to be used by {@link FileFunction}. Should not be used for symlink cycles.
+ */
+ static FileValue value(RootedPath rootedPath, FileStateValue fileStateValue,
+ RootedPath realRootedPath, FileStateValue realFileStateValue) {
+ if (rootedPath.equals(realRootedPath)) {
+ Preconditions.checkState(fileStateValue.getType() != FileStateValue.Type.SYMLINK,
+ "rootedPath: %s, fileStateValue: %s, realRootedPath: %s, realFileStateValue: %s",
+ rootedPath, fileStateValue, realRootedPath, realFileStateValue);
+ return new RegularFileValue(rootedPath, fileStateValue);
+ } else {
+ if (fileStateValue.getType() == FileStateValue.Type.SYMLINK) {
+ return new SymlinkFileValue(realRootedPath, realFileStateValue,
+ fileStateValue.getSymlinkTarget());
+ } else {
+ return new DifferentRealPathFileValue(realRootedPath, realFileStateValue);
+ }
+ }
+ }
+
+ /**
+ * Implementation of {@link FileValue} for files whose fully resolved path is the same as the
+ * requested path. For example, this is the case for the path "foo/bar/baz" if neither 'foo' nor
+ * 'foo/bar' nor 'foo/bar/baz' are symlinks.
+ */
+ private static final class RegularFileValue extends FileValue {
+
+ private final RootedPath rootedPath;
+ private final FileStateValue fileStateValue;
+
+ private RegularFileValue(RootedPath rootedPath, FileStateValue fileState) {
+ this.rootedPath = Preconditions.checkNotNull(rootedPath);
+ this.fileStateValue = Preconditions.checkNotNull(fileState);
+ }
+
+ @Override
+ public RootedPath realRootedPath() {
+ return rootedPath;
+ }
+
+ @Override
+ FileStateValue realFileStateValue() {
+ return fileStateValue;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (obj.getClass() != RegularFileValue.class) {
+ return false;
+ }
+ RegularFileValue other = (RegularFileValue) obj;
+ return rootedPath.equals(other.rootedPath) && fileStateValue.equals(other.fileStateValue);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(rootedPath, fileStateValue);
+ }
+
+ @Override
+ public String toString() {
+ return rootedPath + ", " + fileStateValue;
+ }
+ }
+
+ /**
+ * Base class for {@link FileValue}s for files whose fully resolved path is different than the
+ * requested path. For example, this is the case for the path "foo/bar/baz" if at least one of
+ * 'foo', 'foo/bar', or 'foo/bar/baz' is a symlink.
+ */
+ private static class DifferentRealPathFileValue extends FileValue {
+
+ protected final RootedPath realRootedPath;
+ protected final FileStateValue realFileStateValue;
+
+ private DifferentRealPathFileValue(RootedPath realRootedPath,
+ FileStateValue realFileStateValue) {
+ this.realRootedPath = Preconditions.checkNotNull(realRootedPath);
+ this.realFileStateValue = Preconditions.checkNotNull(realFileStateValue);
+ }
+
+ @Override
+ public RootedPath realRootedPath() {
+ return realRootedPath;
+ }
+
+ @Override
+ FileStateValue realFileStateValue() {
+ return realFileStateValue;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (obj.getClass() != DifferentRealPathFileValue.class) {
+ return false;
+ }
+ DifferentRealPathFileValue other = (DifferentRealPathFileValue) obj;
+ return realRootedPath.equals(other.realRootedPath)
+ && realFileStateValue.equals(other.realFileStateValue);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(realRootedPath, realFileStateValue);
+ }
+
+ @Override
+ public String toString() {
+ return realRootedPath + ", " + realFileStateValue + " (symlink ancestor)";
+ }
+ }
+
+ /** Implementation of {@link FileValue} for files that are symlinks. */
+ private static final class SymlinkFileValue extends DifferentRealPathFileValue {
+ private final PathFragment linkValue;
+
+ private SymlinkFileValue(RootedPath realRootedPath, FileStateValue realFileState,
+ PathFragment linkTarget) {
+ super(realRootedPath, realFileState);
+ this.linkValue = linkTarget;
+ }
+
+ @Override
+ public boolean isSymlink() {
+ return true;
+ }
+
+ @Override
+ public PathFragment getUnresolvedLinkTarget() {
+ return linkValue;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (obj.getClass() != SymlinkFileValue.class) {
+ return false;
+ }
+ SymlinkFileValue other = (SymlinkFileValue) obj;
+ return realRootedPath.equals(other.realRootedPath)
+ && realFileStateValue.equals(other.realFileStateValue)
+ && linkValue.equals(other.linkValue);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(realRootedPath, realFileStateValue, linkValue, Boolean.TRUE);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("symlink (real_path=%s, real_state=%s, link_value=%s)",
+ realRootedPath, realFileStateValue, linkValue);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunction.java
new file mode 100644
index 0000000..a559206
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunction.java
@@ -0,0 +1,320 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.actions.FilesetOutputSymlink;
+import com.google.devtools.build.lib.actions.FilesetTraversalParams;
+import com.google.devtools.build.lib.actions.FilesetTraversalParams.DirectTraversal;
+import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalFunction.DanglingSymlinkException;
+import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalFunction.RecursiveFilesystemTraversalException;
+import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFile;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+
+/** SkyFunction for {@link FilesetEntryValue}. */
+public final class FilesetEntryFunction implements SkyFunction {
+
+ private static final class MissingDepException extends Exception {}
+
+ private static final class FilesetEntryFunctionException extends SkyFunctionException {
+ FilesetEntryFunctionException(RecursiveFilesystemTraversalException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+
+ @Override
+ public SkyValue compute(SkyKey key, Environment env) throws FilesetEntryFunctionException {
+ FilesetTraversalParams t = (FilesetTraversalParams) key.argument();
+ Preconditions.checkState(
+ t.getNestedTraversal().isPresent() != t.getDirectTraversal().isPresent(),
+ "Exactly one of the nested and direct traversals must be specified: %s", t);
+
+ // Create the set of excluded files. Only top-level files can be excluded, i.e. ones that are
+ // directly under the root if the root is a directory.
+ Set<String> exclusions = createExclusionSet(t.getExcludedFiles());
+
+ // The map of output symlinks. Each key is the path of a output symlink that the Fileset must
+ // create, relative to the Fileset.out directory, and each value specifies extra information
+ // about the link (its target, associated metadata and again its name).
+ Map<PathFragment, FilesetOutputSymlink> outputSymlinks = new LinkedHashMap<>();
+
+ if (t.getNestedTraversal().isPresent()) {
+ // The "nested" traversal parameters are present if and only if FilesetEntry.srcdir specifies
+ // another Fileset (a "nested" one).
+ FilesetEntryValue nested = (FilesetEntryValue) env.getValue(
+ FilesetEntryValue.key(t.getNestedTraversal().get()));
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ for (FilesetOutputSymlink s : nested.getSymlinks()) {
+ maybeStoreSymlink(s, t.getDestPath(), exclusions, outputSymlinks);
+ }
+ } else {
+ // The "nested" traversal params are absent if and only if the "direct" traversal params are
+ // present, which is the case when the FilesetEntry specifies a package's BUILD file, a
+ // directory or a list of files.
+
+ // The root of the direct traversal is defined as follows.
+ //
+ // If FilesetEntry.files is specified, then a TraversalRequest is created for each entry, the
+ // root being the respective entry itself. These are all traversed for they may be
+ // directories or symlinks to directories, and we need to establish Skyframe dependencies on
+ // their contents for incremental correctness. If an entry is indeed a directory (but not when
+ // it's a symlink to one) then we have to create symlinks to each of their childen.
+ // (NB: there seems to be no good reason for this, it's just how legacy Fileset works. We may
+ // want to consider creating a symlink just for the directory and not for its child elements.)
+ //
+ // If FilesetEntry.files is not specified, then srcdir refers to either a BUILD file or a
+ // directory. For the former, the root will be the parent of the BUILD file. For the latter,
+ // the root will be srcdir itself.
+ DirectTraversal direct = t.getDirectTraversal().get();
+
+ RecursiveFilesystemTraversalValue rftv;
+ try {
+ // Traverse the filesystem to establish skyframe dependencies.
+ rftv = traverse(env, createErrorInfo(t), direct);
+ } catch (MissingDepException e) {
+ return null;
+ }
+
+ // The root can only be absent for the EMPTY rftv instance.
+ if (!rftv.getResolvedRoot().isPresent()) {
+ return FilesetEntryValue.EMPTY;
+ }
+
+ ResolvedFile resolvedRoot = rftv.getResolvedRoot().get();
+
+ // Handle dangling symlinks gracefully be returning empty results.
+ if (!resolvedRoot.type.exists()) {
+ return FilesetEntryValue.EMPTY;
+ }
+
+ // The prefix to remove is the entire path of the root. This is OK:
+ // - when the root is a file, this removes the entire path, but the traversal's destination
+ // path is actually the name of the output symlink, so this works out correctly
+ // - when the root is a directory or a symlink to one then we need to strip off the
+ // directory's path from every result (this is how the output symlinks must be created)
+ // before making them relative to the destination path
+ PathFragment prefixToRemove = direct.getRoot().getRelativePart();
+
+ Iterable<ResolvedFile> results = null;
+
+ if (direct.isRecursive()
+ || (resolvedRoot.type.isDirectory() && !resolvedRoot.type.isSymlink())) {
+ // The traversal is recursive (requested for an entire FilesetEntry.srcdir) or it was
+ // requested for a FilesetEntry.files entry which turned out to be a directory. We need to
+ // create an output symlink for every file in it and all of its subdirectories. Only
+ // exception is when the subdirectory is really a symlink to a directory -- no output
+ // shall be created for the contents of those.
+ // Now we create Dir objects to model the filesystem tree. The object employs a trick to
+ // find directory symlinks: directory symlinks have corresponding ResolvedFile entries and
+ // are added as files too, while their children, also added as files, contain the path of
+ // the parent. Finding and discarding the children is easy if we traverse the tree from
+ // root to leaf.
+ DirectoryTree root = new DirectoryTree();
+ for (ResolvedFile f : rftv.getTransitiveFiles().toCollection()) {
+ PathFragment path = f.getNameInSymlinkTree().relativeTo(prefixToRemove);
+ if (path.segmentCount() > 0) {
+ path = t.getDestPath().getRelative(path);
+ DirectoryTree dir = root;
+ for (int i = 0; i < path.segmentCount() - 1; ++i) {
+ dir = dir.addOrGetSubdir(path.getSegment(i));
+ }
+ dir.maybeAddFile(f);
+ }
+ }
+ // Here's where the magic happens. The returned iterable will yield all files in the
+ // directory that are not under symlinked directories, as well as all directory symlinks.
+ results = root.iterateFiles();
+ } else {
+ // If we're on this branch then the traversal was done for just one entry in
+ // FilesetEntry.files (which was not a directory, so it was either a file, a symlink to one
+ // or a symlink to a directory), meaning we'll have only one output symlink.
+ results = ImmutableList.of(resolvedRoot);
+ }
+
+ // Create one output symlink for each entry in the results.
+ for (ResolvedFile f : results) {
+ PathFragment targetName;
+ try {
+ targetName = f.getTargetInSymlinkTree(direct.isFollowingSymlinks());
+ } catch (DanglingSymlinkException e) {
+ throw new FilesetEntryFunctionException(e);
+ }
+
+ // Metadata field must be present. It can only be absent when stripped by tests.
+ String metadata = Integer.toHexString(f.metadata.get().hashCode());
+
+ // The linkName has to be under the traversal's root, which is also the prefix to remove.
+ PathFragment linkName = f.getNameInSymlinkTree().relativeTo(prefixToRemove);
+ maybeStoreSymlink(linkName, targetName, metadata, t.getDestPath(), exclusions,
+ outputSymlinks);
+ }
+ }
+
+ return FilesetEntryValue.of(ImmutableSet.copyOf(outputSymlinks.values()));
+ }
+
+ /** Stores an output symlink unless it's excluded or would overwrite an existing one. */
+ private static void maybeStoreSymlink(FilesetOutputSymlink nestedLink, PathFragment destPath,
+ Set<String> exclusions, Map<PathFragment, FilesetOutputSymlink> result) {
+ maybeStoreSymlink(nestedLink.name, nestedLink.target, nestedLink.metadata, destPath,
+ exclusions, result);
+ }
+
+ /** Stores an output symlink unless it's excluded or would overwrite an existing one. */
+ private static void maybeStoreSymlink(PathFragment linkName, PathFragment linkTarget,
+ String metadata, PathFragment destPath, Set<String> exclusions,
+ Map<PathFragment, FilesetOutputSymlink> result) {
+ if (!exclusions.contains(linkName.getPathString())) {
+ linkName = destPath.getRelative(linkName);
+ if (!result.containsKey(linkName)) {
+ result.put(linkName, new FilesetOutputSymlink(linkName, linkTarget, metadata));
+ }
+ }
+ }
+
+ private static Set<String> createExclusionSet(Set<String> input) {
+ return Sets.filter(input, new Predicate<String>() {
+ @Override
+ public boolean apply(String e) {
+ // Keep the top-level exclusions only. Do not look for "/" but count the path segments
+ // instead, in anticipation of future Windows support.
+ return new PathFragment(e).segmentCount() == 1;
+ }
+ });
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ private RecursiveFilesystemTraversalValue traverse(Environment env, String errorInfo,
+ DirectTraversal traversal) throws MissingDepException {
+ SkyKey depKey = RecursiveFilesystemTraversalValue.key(
+ new RecursiveFilesystemTraversalValue.TraversalRequest(traversal.getRoot().asRootedPath(),
+ traversal.isGenerated(), traversal.getCrossPackageBoundary(), traversal.isPackage(),
+ errorInfo));
+ RecursiveFilesystemTraversalValue v = (RecursiveFilesystemTraversalValue) env.getValue(depKey);
+ if (env.valuesMissing()) {
+ throw new MissingDepException();
+ }
+ return v;
+ }
+
+ private static String createErrorInfo(FilesetTraversalParams t) {
+ if (t.getDirectTraversal().isPresent()) {
+ DirectTraversal direct = t.getDirectTraversal().get();
+ return String.format("Fileset '%s' traversing %s %s", t.getOwnerLabel(),
+ direct.isPackage() ? "package" : "file (or directory)",
+ direct.getRoot().getRelativePart().getPathString());
+ } else {
+ return String.format("Fileset '%s' traversing another Fileset", t.getOwnerLabel());
+ }
+ }
+
+ /**
+ * Models a FilesetEntryFunction's portion of the symlink output tree created by a Fileset rule.
+ *
+ * <p>A Fileset rule's output is computed by zero or more {@link FilesetEntryFunction}s, resulting
+ * in one {@link FilesetEntryValue} for each. Each of those represents a portion of the grand
+ * output tree of the Fileset. These portions are later merged and written to the fileset manifest
+ * file, which is then consumed by a tool that ultimately creates the symlinks in the filesystem.
+ *
+ * <p>Because the Fileset doesn't process the lists in the FilesetEntryValues any further than
+ * merging them, they have to adhere to the conventions of the manifest file. One of these is that
+ * files are alphabetically ordered (enables the consumer of the manifest to work faster than
+ * otherwise) and another is that the contents of regular directories are listed, but contents
+ * of directory symlinks are not, only the symlinks are. (Other details of the manifest file are
+ * not relevant here.)
+ *
+ * <p>See {@link DirectoryTree#iterateFiles()} for more details.
+ */
+ private static final class DirectoryTree {
+ // Use TreeMaps for the benefit of alphabetically ordered iteration.
+ public final Map<String, ResolvedFile> files = new TreeMap<>();
+ public final Map<String, DirectoryTree> dirs = new TreeMap<>();
+
+ DirectoryTree addOrGetSubdir(String name) {
+ DirectoryTree result = dirs.get(name);
+ if (result == null) {
+ result = new DirectoryTree();
+ dirs.put(name, result);
+ }
+ return result;
+ }
+
+ void maybeAddFile(ResolvedFile r) {
+ String name = r.getNameInSymlinkTree().getBaseName();
+ if (!files.containsKey(name)) {
+ files.put(name, r);
+ }
+ }
+
+ /**
+ * Lazily yields all files in this directory and all of its subdirectories.
+ *
+ * <p>The function first yields all the files directly under this directory, in alphabetical
+ * order. Then come the contents of subdirectories, processed recursively in the same fashion
+ * as this directory, and also in alphabetical order.
+ *
+ * <p>If a directory symlink is encountered its contents are not listed, only the symlink is.
+ */
+ Iterable<ResolvedFile> iterateFiles() {
+ // 1. Filter directory symlinks. If the symlink target contains files, those were added
+ // as normal files so their parent directory (the symlink) would show up in the dirs map
+ // (as a directory) as well as in the files map (as a symlink to a directory).
+ final Set<String> fileNames = files.keySet();
+ Iterable<Map.Entry<String, DirectoryTree>> noDirSymlinkes = Iterables.filter(dirs.entrySet(),
+ new Predicate<Map.Entry<String, DirectoryTree>>() {
+ @Override
+ public boolean apply(Map.Entry<String, DirectoryTree> input) {
+ return !fileNames.contains(input.getKey());
+ }
+ });
+
+ // 2. Extract the iterables of the true subdirectories.
+ Iterable<Iterable<ResolvedFile>> subdirIters = Iterables.transform(noDirSymlinkes,
+ new Function<Map.Entry<String, DirectoryTree>, Iterable<ResolvedFile>>() {
+ @Override
+ public Iterable<ResolvedFile> apply(Entry<String, DirectoryTree> input) {
+ return input.getValue().iterateFiles();
+ }
+ });
+
+ // 3. Just concat all subdirectory iterations for one, seamless iteration.
+ Iterable<ResolvedFile> dirsIter = Iterables.concat(subdirIters);
+
+ return Iterables.concat(files.values(), dirsIter);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryValue.java
new file mode 100644
index 0000000..e7b6580
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryValue.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.actions.FilesetOutputSymlink;
+import com.google.devtools.build.lib.actions.FilesetTraversalParams;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/** Output symlinks produced by a whole FilesetEntry or by a single file in FilesetEntry.files. */
+public final class FilesetEntryValue implements SkyValue {
+ static final FilesetEntryValue EMPTY =
+ new FilesetEntryValue(ImmutableSet.<FilesetOutputSymlink>of());
+
+ private final ImmutableSet<FilesetOutputSymlink> symlinks;
+
+ private FilesetEntryValue(ImmutableSet<FilesetOutputSymlink> symlinks) {
+ this.symlinks = symlinks;
+ }
+
+ static FilesetEntryValue of(ImmutableSet<FilesetOutputSymlink> symlinks) {
+ if (symlinks.isEmpty()) {
+ return EMPTY;
+ } else {
+ return new FilesetEntryValue(symlinks);
+ }
+ }
+
+ /** Returns the list of output symlinks. */
+ public ImmutableSet<FilesetOutputSymlink> getSymlinks() {
+ return symlinks;
+ }
+
+ public static SkyKey key(FilesetTraversalParams params) {
+ return new SkyKey(SkyFunctions.FILESET_ENTRY, params);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java b/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java
new file mode 100644
index 0000000..be4f4e8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java
@@ -0,0 +1,398 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.concurrent.ExecutorShutdownUtil;
+import com.google.devtools.build.lib.concurrent.Sharder;
+import com.google.devtools.build.lib.concurrent.ThrowableRecordingRunnableWrapper;
+import com.google.devtools.build.lib.util.LoggingUtil;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.BatchStat;
+import com.google.devtools.build.lib.vfs.FileStatusWithDigest;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.Differencer;
+import com.google.devtools.build.skyframe.MemoizingEvaluator;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+
+/**
+ * A helper class to find dirty values by accessing the filesystem directly (contrast with
+ * {@link DiffAwareness}).
+ */
+class FilesystemValueChecker {
+
+ private static final int DIRTINESS_CHECK_THREADS = 50;
+ private static final Logger LOG = Logger.getLogger(FilesystemValueChecker.class.getName());
+
+ private static final Predicate<SkyKey> FILE_STATE_AND_DIRECTORY_LISTING_STATE_FILTER =
+ SkyFunctionName.functionIsIn(ImmutableSet.of(SkyFunctions.FILE_STATE,
+ SkyFunctions.DIRECTORY_LISTING_STATE));
+ private static final Predicate<SkyKey> ACTION_FILTER =
+ SkyFunctionName.functionIs(SkyFunctions.ACTION_EXECUTION);
+
+ private final TimestampGranularityMonitor tsgm;
+ private final Supplier<Map<SkyKey, SkyValue>> valuesSupplier;
+ private AtomicInteger modifiedOutputFilesCounter = new AtomicInteger(0);
+
+ FilesystemValueChecker(final MemoizingEvaluator evaluator, TimestampGranularityMonitor tsgm) {
+ this.tsgm = tsgm;
+
+ // Construct the full map view of the entire graph at most once ("memoized"), lazily. If
+ // getDirtyFilesystemValues(Iterable<SkyKey>) is called on an empty Iterable, we avoid having
+ // to create the Map of value keys to values. This is useful in the case where the graph
+ // getValues() method could be slow.
+ this.valuesSupplier = Suppliers.memoize(new Supplier<Map<SkyKey, SkyValue>>() {
+ @Override
+ public Map<SkyKey, SkyValue> get() {
+ return evaluator.getValues();
+ }
+ });
+ }
+
+ Iterable<SkyKey> getFilesystemSkyKeys() {
+ return Iterables.filter(valuesSupplier.get().keySet(),
+ FILE_STATE_AND_DIRECTORY_LISTING_STATE_FILTER);
+ }
+
+ Differencer.Diff getDirtyFilesystemSkyKeys() throws InterruptedException {
+ return getDirtyFilesystemValues(getFilesystemSkyKeys());
+ }
+
+ /**
+ * Check the given file and directory values for modifications. {@code values} is assumed to only
+ * have {@link FileValue}s and {@link DirectoryListingStateValue}s.
+ */
+ Differencer.Diff getDirtyFilesystemValues(Iterable<SkyKey> values)
+ throws InterruptedException {
+ return getDirtyValues(values, FILE_STATE_AND_DIRECTORY_LISTING_STATE_FILTER,
+ new DirtyChecker() {
+ @Override
+ public DirtyResult check(SkyKey key, SkyValue oldValue, TimestampGranularityMonitor tsgm) {
+ if (key.functionName() == SkyFunctions.FILE_STATE) {
+ return checkFileStateValue((RootedPath) key.argument(), (FileStateValue) oldValue,
+ tsgm);
+ } else if (key.functionName() == SkyFunctions.DIRECTORY_LISTING_STATE) {
+ return checkDirectoryListingStateValue((RootedPath) key.argument(),
+ (DirectoryListingStateValue) oldValue);
+ } else {
+ throw new IllegalStateException("Unexpected key type " + key);
+ }
+ }
+ });
+ }
+
+ /**
+ * Return a collection of action values which have output files that are not in-sync with
+ * the on-disk file value (were modified externally).
+ */
+ public Collection<SkyKey> getDirtyActionValues(@Nullable final BatchStat batchStatter)
+ throws InterruptedException {
+ // CPU-bound (usually) stat() calls, plus a fudge factor.
+ LOG.info("Accumulating dirty actions");
+ final int numOutputJobs = Runtime.getRuntime().availableProcessors() * 4;
+ final Set<SkyKey> actionSkyKeys =
+ Sets.filter(valuesSupplier.get().keySet(), ACTION_FILTER);
+ final Sharder<Pair<SkyKey, ActionExecutionValue>> outputShards =
+ new Sharder<>(numOutputJobs, actionSkyKeys.size());
+
+ for (SkyKey key : actionSkyKeys) {
+ outputShards.add(Pair.of(key, (ActionExecutionValue) valuesSupplier.get().get(key)));
+ }
+ LOG.info("Sharded action values for batching");
+
+ ExecutorService executor = Executors.newFixedThreadPool(
+ numOutputJobs,
+ new ThreadFactoryBuilder().setNameFormat("FileSystem Output File Invalidator %d").build());
+
+ Collection<SkyKey> dirtyKeys = Sets.newConcurrentHashSet();
+ ThrowableRecordingRunnableWrapper wrapper =
+ new ThrowableRecordingRunnableWrapper("FileSystemValueChecker#getDirtyActionValues");
+
+ modifiedOutputFilesCounter.set(0);
+ for (List<Pair<SkyKey, ActionExecutionValue>> shard : outputShards) {
+ Runnable job = (batchStatter == null)
+ ? outputStatJob(dirtyKeys, shard)
+ : batchStatJob(dirtyKeys, shard, batchStatter);
+ executor.submit(wrapper.wrap(job));
+ }
+
+ boolean interrupted = ExecutorShutdownUtil.interruptibleShutdown(executor);
+ Throwables.propagateIfPossible(wrapper.getFirstThrownError());
+ LOG.info("Completed output file stat checks");
+ if (interrupted) {
+ throw new InterruptedException();
+ }
+ return dirtyKeys;
+ }
+
+ private Runnable batchStatJob(final Collection<SkyKey> dirtyKeys,
+ final List<Pair<SkyKey, ActionExecutionValue>> shard,
+ final BatchStat batchStatter) {
+ return new Runnable() {
+ @Override
+ public void run() {
+ Map<Artifact, Pair<SkyKey, ActionExecutionValue>> artifactToKeyAndValue = new HashMap<>();
+ for (Pair<SkyKey, ActionExecutionValue> keyAndValue : shard) {
+ ActionExecutionValue actionValue = keyAndValue.getSecond();
+ if (actionValue == null) {
+ dirtyKeys.add(keyAndValue.getFirst());
+ } else {
+ for (Artifact artifact : actionValue.getAllOutputArtifactData().keySet()) {
+ artifactToKeyAndValue.put(artifact, keyAndValue);
+ }
+ }
+ }
+
+ List<Artifact> artifacts = ImmutableList.copyOf(artifactToKeyAndValue.keySet());
+ List<FileStatusWithDigest> stats;
+ try {
+ stats = batchStatter.batchStat(/*includeDigest=*/true, /*includeLinks=*/true,
+ Artifact.asPathFragments(artifacts));
+ } catch (IOException e) {
+ // Batch stat did not work. Log an exception and fall back on system calls.
+ LoggingUtil.logToRemote(Level.WARNING, "Unable to process batch stat", e);
+ outputStatJob(dirtyKeys, shard).run();
+ return;
+ } catch (InterruptedException e) {
+ // We handle interrupt in the main thread.
+ return;
+ }
+
+ Preconditions.checkState(artifacts.size() == stats.size(),
+ "artifacts.size() == %s stats.size() == %s", artifacts.size(), stats.size());
+ for (int i = 0; i < artifacts.size(); i++) {
+ Artifact artifact = artifacts.get(i);
+ FileStatusWithDigest stat = stats.get(i);
+ Pair<SkyKey, ActionExecutionValue> keyAndValue = artifactToKeyAndValue.get(artifact);
+ ActionExecutionValue actionValue = keyAndValue.getSecond();
+ SkyKey key = keyAndValue.getFirst();
+ FileValue lastKnownData = actionValue.getAllOutputArtifactData().get(artifact);
+ try {
+ FileValue newData = FileAndMetadataCache.fileValueFromArtifact(artifact, stat, tsgm);
+ if (!newData.equals(lastKnownData)) {
+ modifiedOutputFilesCounter.getAndIncrement();
+ dirtyKeys.add(key);
+ }
+ } catch (IOException e) {
+ // This is an unexpected failure getting a digest or symlink target.
+ modifiedOutputFilesCounter.getAndIncrement();
+ dirtyKeys.add(key);
+ }
+ }
+ }
+ };
+ }
+
+ private Runnable outputStatJob(final Collection<SkyKey> dirtyKeys,
+ final List<Pair<SkyKey, ActionExecutionValue>> shard) {
+ return new Runnable() {
+ @Override
+ public void run() {
+ for (Pair<SkyKey, ActionExecutionValue> keyAndValue : shard) {
+ ActionExecutionValue value = keyAndValue.getSecond();
+ if (value == null || actionValueIsDirtyWithDirectSystemCalls(value)) {
+ dirtyKeys.add(keyAndValue.getFirst());
+ }
+ }
+ }
+ };
+ }
+
+ /**
+ * Returns number of modified output files inside of dirty actions.
+ */
+ int getNumberOfModifiedOutputFiles() {
+ return modifiedOutputFilesCounter.get();
+ }
+
+ private boolean actionValueIsDirtyWithDirectSystemCalls(ActionExecutionValue actionValue) {
+ boolean isDirty = false;
+ for (Map.Entry<Artifact, FileValue> entry :
+ actionValue.getAllOutputArtifactData().entrySet()) {
+ Artifact artifact = entry.getKey();
+ FileValue lastKnownData = entry.getValue();
+ try {
+ if (!FileAndMetadataCache.fileValueFromArtifact(artifact, null, tsgm).equals(
+ lastKnownData)) {
+ modifiedOutputFilesCounter.getAndIncrement();
+ isDirty = true;
+ }
+ } catch (IOException e) {
+ // This is an unexpected failure getting a digest or symlink target.
+ modifiedOutputFilesCounter.getAndIncrement();
+ isDirty = true;
+ }
+ }
+ return isDirty;
+ }
+
+ private BatchDirtyResult getDirtyValues(Iterable<SkyKey> values,
+ Predicate<SkyKey> keyFilter,
+ final DirtyChecker checker) throws InterruptedException {
+ ExecutorService executor = Executors.newFixedThreadPool(DIRTINESS_CHECK_THREADS,
+ new ThreadFactoryBuilder().setNameFormat("FileSystem Value Invalidator %d").build());
+
+ final BatchDirtyResult batchResult = new BatchDirtyResult();
+ ThrowableRecordingRunnableWrapper wrapper =
+ new ThrowableRecordingRunnableWrapper("FilesystemValueChecker#getDirtyValues");
+ for (final SkyKey key : values) {
+ Preconditions.checkState(keyFilter.apply(key), key);
+ final SkyValue value = valuesSupplier.get().get(key);
+ executor.execute(wrapper.wrap(new Runnable() {
+ @Override
+ public void run() {
+ if (value == null) {
+ // value will be null if the value is in error or part of a cycle.
+ // TODO(bazel-team): This is overly conservative.
+ batchResult.add(key, /*newValue=*/null);
+ return;
+ }
+ DirtyResult result = checker.check(key, value, tsgm);
+ if (result.isDirty()) {
+ batchResult.add(key, result.getNewValue());
+ }
+ }
+ }));
+ }
+
+ boolean interrupted = ExecutorShutdownUtil.interruptibleShutdown(executor);
+ Throwables.propagateIfPossible(wrapper.getFirstThrownError());
+ if (interrupted) {
+ throw new InterruptedException();
+ }
+ return batchResult;
+ }
+
+ private static DirtyResult checkFileStateValue(RootedPath rootedPath,
+ FileStateValue fileStateValue, TimestampGranularityMonitor tsgm) {
+ try {
+ FileStateValue newValue = FileStateValue.create(rootedPath, tsgm);
+ return newValue.equals(fileStateValue)
+ ? DirtyResult.NOT_DIRTY : DirtyResult.dirtyWithNewValue(newValue);
+ } catch (InconsistentFilesystemException | IOException e) {
+ // TODO(bazel-team): An IOException indicates a failure to get a file digest or a symlink
+ // target, not a missing file. Such a failure really shouldn't happen, so failing early
+ // may be better here.
+ return DirtyResult.DIRTY;
+ }
+ }
+
+ private static DirtyResult checkDirectoryListingStateValue(RootedPath dirRootedPath,
+ DirectoryListingStateValue directoryListingStateValue) {
+ try {
+ DirectoryListingStateValue newValue = DirectoryListingStateValue.create(dirRootedPath);
+ return newValue.equals(directoryListingStateValue)
+ ? DirtyResult.NOT_DIRTY : DirtyResult.dirtyWithNewValue(newValue);
+ } catch (IOException e) {
+ return DirtyResult.DIRTY;
+ }
+ }
+
+ /**
+ * Result of a batch call to {@link DirtyChecker#check}. Partitions the dirty values based on
+ * whether we have a new value available for them or not.
+ */
+ private static class BatchDirtyResult implements Differencer.Diff {
+
+ private final Set<SkyKey> concurrentDirtyKeysWithoutNewValues =
+ Collections.newSetFromMap(new ConcurrentHashMap<SkyKey, Boolean>());
+ private final ConcurrentHashMap<SkyKey, SkyValue> concurrentDirtyKeysWithNewValues =
+ new ConcurrentHashMap<>();
+
+ private void add(SkyKey key, @Nullable SkyValue newValue) {
+ if (newValue == null) {
+ concurrentDirtyKeysWithoutNewValues.add(key);
+ } else {
+ concurrentDirtyKeysWithNewValues.put(key, newValue);
+ }
+ }
+
+ @Override
+ public Iterable<SkyKey> changedKeysWithoutNewValues() {
+ return concurrentDirtyKeysWithoutNewValues;
+ }
+
+ @Override
+ public Map<SkyKey, ? extends SkyValue> changedKeysWithNewValues() {
+ return concurrentDirtyKeysWithNewValues;
+ }
+ }
+
+ private static class DirtyResult {
+
+ static final DirtyResult NOT_DIRTY = new DirtyResult(false, null);
+ static final DirtyResult DIRTY = new DirtyResult(true, null);
+
+ private final boolean isDirty;
+ @Nullable private final SkyValue newValue;
+
+ private DirtyResult(boolean isDirty, @Nullable SkyValue newValue) {
+ this.isDirty = isDirty;
+ this.newValue = newValue;
+ }
+
+ boolean isDirty() {
+ return isDirty;
+ }
+
+ /**
+ * If {@code isDirty()}, then either returns the new value for the value or {@code null} if
+ * the new value wasn't computed. In the case where the value is dirty and a new value is
+ * available, then the new value can be injected into the skyframe graph. Otherwise, the value
+ * should simply be invalidated.
+ */
+ @Nullable
+ SkyValue getNewValue() {
+ Preconditions.checkState(isDirty());
+ return newValue;
+ }
+
+ static DirtyResult dirtyWithNewValue(SkyValue newValue) {
+ return new DirtyResult(true, newValue);
+ }
+ }
+
+ private static interface DirtyChecker {
+ DirtyResult check(SkyKey key, SkyValue oldValue, TimestampGranularityMonitor tsgm);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/GlobDescriptor.java b/src/main/java/com/google/devtools/build/lib/skyframe/GlobDescriptor.java
new file mode 100644
index 0000000..5baeae8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/GlobDescriptor.java
@@ -0,0 +1,113 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.util.StringCanonicalizer;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * A descriptor for a glob request.
+ *
+ * <p>{@code subdir} must be empty or point to an existing directory.</p>
+ *
+ * <p>{@code pattern} must be valid, as indicated by {@code UnixGlob#checkPatternForError}.
+ */
+@ThreadSafe
+public final class GlobDescriptor implements Serializable {
+ final PackageIdentifier packageId;
+ final PathFragment subdir;
+ final String pattern;
+ final boolean excludeDirs;
+
+ /**
+ * Constructs a GlobDescriptor.
+ *
+ * @param packageId the name of the owner package (must be an existing package)
+ * @param subdir the subdirectory being looked at (must exist and must be a directory. It's
+ * assumed that there are no other packages between {@code packageName} and
+ * {@code subdir}.
+ * @param pattern a valid glob pattern
+ * @param excludeDirs true if directories should be excluded from results
+ */
+ GlobDescriptor(PackageIdentifier packageId, PathFragment subdir, String pattern,
+ boolean excludeDirs) {
+ this.packageId = Preconditions.checkNotNull(packageId);
+ this.subdir = Preconditions.checkNotNull(subdir);
+ this.pattern = Preconditions.checkNotNull(StringCanonicalizer.intern(pattern));
+ this.excludeDirs = excludeDirs;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("<GlobDescriptor packageName=%s subdir=%s pattern=%s excludeDirs=%s>",
+ packageId, subdir, pattern, excludeDirs);
+ }
+
+ /**
+ * Returns the package that "owns" this glob.
+ *
+ * <p>The glob evaluation code ensures that the boundaries of this package are not crossed.
+ */
+ public PackageIdentifier getPackageId() {
+ return packageId;
+ }
+
+ /**
+ * Returns the subdirectory of the package under consideration.
+ */
+ PathFragment getSubdir() {
+ return subdir;
+ }
+
+ /**
+ * Returns the glob pattern under consideration. May contain wildcards.
+ *
+ * <p>As the glob evaluator traverses deeper into the file tree, components are added at the
+ * beginning of {@code subdir} and removed from the beginning of {@code pattern}.
+ */
+ String getPattern() {
+ return pattern;
+ }
+
+ /**
+ * Returns true if directories should be excluded from results.
+ */
+ boolean excludeDirs() {
+ return excludeDirs;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(packageId, subdir, pattern, excludeDirs);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof GlobDescriptor)) {
+ return false;
+ }
+ GlobDescriptor other = (GlobDescriptor) obj;
+ return packageId.equals(other.packageId) && subdir.equals(other.subdir)
+ && pattern.equals(other.pattern) && excludeDirs == other.excludeDirs;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/GlobFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/GlobFunction.java
new file mode 100644
index 0000000..a1fdcb2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/GlobFunction.java
@@ -0,0 +1,251 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.vfs.Dirent;
+import com.google.devtools.build.lib.vfs.Dirent.Type;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.regex.Pattern;
+
+import javax.annotation.Nullable;
+
+/**
+ * A {@link SkyFunction} for {@link GlobValue}s.
+ *
+ * <p>This code drives the glob matching process.
+ */
+final class GlobFunction implements SkyFunction {
+
+ private final Cache<String, Pattern> regexPatternCache =
+ CacheBuilder.newBuilder().concurrencyLevel(4).build();
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws GlobFunctionException {
+ GlobDescriptor glob = (GlobDescriptor) skyKey.argument();
+
+ PackageLookupValue globPkgLookupValue = (PackageLookupValue)
+ env.getValue(PackageLookupValue.key(glob.getPackageId()));
+ if (globPkgLookupValue == null) {
+ return null;
+ }
+ Preconditions.checkState(globPkgLookupValue.packageExists(), "%s isn't an existing package",
+ glob.getPackageId());
+ // Note that this implies that the package's BUILD file exists which implies that the
+ // package's directory exists.
+
+ PathFragment globSubdir = glob.getSubdir();
+ if (!globSubdir.equals(PathFragment.EMPTY_FRAGMENT)) {
+ PackageLookupValue globSubdirPkgLookupValue = (PackageLookupValue) env.getValue(
+ PackageLookupValue.key(glob.getPackageId().getPackageFragment()
+ .getRelative(globSubdir)));
+ if (globSubdirPkgLookupValue == null) {
+ return null;
+ }
+ if (globSubdirPkgLookupValue.packageExists()) {
+ // We crossed the package boundary, that is, pkg/subdir contains a BUILD file and thus
+ // defines another package, so glob expansion should not descend into that subdir.
+ return GlobValue.EMPTY;
+ }
+ }
+
+ String pattern = glob.getPattern();
+ // Split off the first path component of the pattern.
+ int slashPos = pattern.indexOf("/");
+ String patternHead;
+ String patternTail;
+ if (slashPos == -1) {
+ patternHead = pattern;
+ patternTail = null;
+ } else {
+ // Substrings will share the backing array of the original glob string. That should be fine.
+ patternHead = pattern.substring(0, slashPos);
+ patternTail = pattern.substring(slashPos + 1);
+ }
+
+ NestedSetBuilder<PathFragment> matches = NestedSetBuilder.stableOrder();
+
+ // "**" also matches an empty segment, so try the case where it is not present.
+ if ("**".equals(patternHead)) {
+ if (patternTail == null) {
+ if (!glob.excludeDirs()) {
+ matches.add(globSubdir);
+ }
+ } else {
+ SkyKey globKey = GlobValue.internalKey(
+ glob.getPackageId(), globSubdir, patternTail, glob.excludeDirs());
+ GlobValue globValue = (GlobValue) env.getValue(globKey);
+ if (globValue == null) {
+ return null;
+ }
+ matches.addTransitive(globValue.getMatches());
+ }
+ }
+
+ PathFragment dirPathFragment = glob.getPackageId().getPackageFragment().getRelative(globSubdir);
+ RootedPath dirRootedPath = RootedPath.toRootedPath(globPkgLookupValue.getRoot(),
+ dirPathFragment);
+ if (containsGlobs(patternHead)) {
+ // Pattern contains globs, so a directory listing is required.
+ //
+ // Note that we have good reason to believe the directory exists: if this is the
+ // top-level directory of the package, the package's existence implies the directory's
+ // existence; if this is a lower-level directory in the package, then we got here from
+ // previous directory listings. Filesystem operations concurrent with build could mean the
+ // directory no longer exists, but DirectoryListingFunction handles that gracefully.
+ DirectoryListingValue listingValue = (DirectoryListingValue)
+ env.getValue(DirectoryListingValue.key(dirRootedPath));
+ if (listingValue == null) {
+ return null;
+ }
+
+ for (Dirent dirent : listingValue.getDirents()) {
+ Type direntType = dirent.getType();
+ String fileName = dirent.getName();
+
+ boolean isDirectory = (direntType == Dirent.Type.DIRECTORY);
+
+ if (!UnixGlob.matches(patternHead, fileName, regexPatternCache)) {
+ continue;
+ }
+
+ if (direntType == Dirent.Type.SYMLINK) {
+ // TODO(bazel-team): Consider extracting the symlink resolution logic.
+ // For symlinks, look up the corresponding FileValue. This ensures that if the symlink
+ // changes and "switches types" (say, from a file to a directory), this value will be
+ // invalidated.
+ RootedPath symlinkRootedPath = RootedPath.toRootedPath(globPkgLookupValue.getRoot(),
+ dirPathFragment.getRelative(fileName));
+ FileValue symlinkFileValue = (FileValue) env.getValue(FileValue.key(symlinkRootedPath));
+ if (symlinkFileValue == null) {
+ continue;
+ }
+ if (!symlinkFileValue.isSymlink()) {
+ throw new GlobFunctionException(new InconsistentFilesystemException(
+ "readdir and stat disagree about whether " + symlinkRootedPath.asPath()
+ + " is a symlink."), Transience.TRANSIENT);
+ }
+ isDirectory = symlinkFileValue.isDirectory();
+ }
+
+ String subdirPattern = "**".equals(patternHead) ? glob.getPattern() : patternTail;
+ addFile(fileName, glob, subdirPattern, patternTail == null, isDirectory,
+ matches, env);
+ }
+ } else {
+ // Pattern does not contain globs, so a direct stat is enough.
+ String fileName = patternHead;
+ RootedPath fileRootedPath = RootedPath.toRootedPath(globPkgLookupValue.getRoot(),
+ dirPathFragment.getRelative(fileName));
+ FileValue fileValue = (FileValue) env.getValue(FileValue.key(fileRootedPath));
+ if (fileValue == null) {
+ return null;
+ }
+ if (fileValue.exists()) {
+ addFile(fileName, glob, patternTail, patternTail == null,
+ fileValue.isDirectory(), matches, env);
+ }
+ }
+
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ NestedSet<PathFragment> matchesBuilt = matches.build();
+ // Use the same value to represent that we did not match anything.
+ if (matchesBuilt.isEmpty()) {
+ return GlobValue.EMPTY;
+ }
+ return new GlobValue(matchesBuilt);
+ }
+
+ /**
+ * Returns true if the given pattern contains globs.
+ */
+ private boolean containsGlobs(String pattern) {
+ return pattern.contains("*") || pattern.contains("?");
+ }
+
+ /**
+ * Includes the given file/directory in the glob.
+ *
+ * <p>{@code fileName} must exist.
+ *
+ * <p>{@code isDirectory} must be true iff the file is a directory.
+ *
+ * <p>{@code directResult} must be set if the file should be included in the result set
+ * directly rather than recursed into if it is a directory.
+ */
+ private void addFile(String fileName, GlobDescriptor glob, String subdirPattern,
+ boolean directResult, boolean isDirectory, NestedSetBuilder<PathFragment> matches,
+ Environment env) {
+ if (isDirectory && subdirPattern != null) {
+ // This is a directory, and the pattern covers files under that directory.
+ SkyKey subdirGlobKey = GlobValue.internalKey(glob.getPackageId(),
+ glob.getSubdir().getRelative(fileName), subdirPattern, glob.excludeDirs());
+ GlobValue subdirGlob = (GlobValue) env.getValue(subdirGlobKey);
+ if (subdirGlob == null) {
+ return;
+ }
+ matches.addTransitive(subdirGlob.getMatches());
+ }
+
+ if (directResult && !(isDirectory && glob.excludeDirs())) {
+ if (isDirectory) {
+ // TODO(bazel-team): Refactor. This is basically inlined code from the next recursion level.
+ // Ensure that subdirectories that contain other packages are not picked up.
+ PathFragment directory = glob.getPackageId().getPackageFragment()
+ .getRelative(glob.getSubdir()).getRelative(fileName);
+ PackageLookupValue pkgLookupValue = (PackageLookupValue)
+ env.getValue(PackageLookupValue.key(directory));
+ if (pkgLookupValue == null) {
+ return;
+ }
+ if (pkgLookupValue.packageExists()) {
+ // The file is a directory and contains another package.
+ return;
+ }
+ }
+ matches.add(glob.getSubdir().getRelative(fileName));
+ }
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link GlobFunction#compute}.
+ */
+ private static final class GlobFunctionException extends SkyFunctionException {
+ public GlobFunctionException(InconsistentFilesystemException e, Transience transience) {
+ super(e, transience);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/GlobValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/GlobValue.java
new file mode 100644
index 0000000..6de0fbd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/GlobValue.java
@@ -0,0 +1,132 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A value corresponding to a glob.
+ */
+@Immutable
+@ThreadSafe
+final class GlobValue implements SkyValue {
+
+ static final GlobValue EMPTY = new GlobValue(
+ NestedSetBuilder.<PathFragment>emptySet(Order.STABLE_ORDER));
+
+ private final NestedSet<PathFragment> matches;
+
+ GlobValue(NestedSet<PathFragment> matches) {
+ this.matches = Preconditions.checkNotNull(matches);
+ }
+
+ /**
+ * Returns glob matches.
+ */
+ NestedSet<PathFragment> getMatches() {
+ return matches;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+ if (!(other instanceof GlobValue)) {
+ return false;
+ }
+ // shallowEquals() may fail to detect that two equivalent (according to toString())
+ // NestedSets are equal, but will always detect when two NestedSets are different.
+ // This makes this implementation of equals() overly strict, but we only call this
+ // method when doing change pruning, which can accept false negatives.
+ return getMatches().shallowEquals(((GlobValue) other).getMatches());
+ }
+
+ @Override
+ public int hashCode() {
+ return matches.shallowHashCode();
+ }
+
+ /**
+ * Constructs a {@link SkyKey} for a glob lookup. {@code packageName} is assumed to be an
+ * existing package. Trying to glob into a non-package is undefined behavior.
+ *
+ * @throws InvalidGlobPatternException if the pattern is not valid.
+ */
+ @ThreadSafe
+ static SkyKey key(PackageIdentifier packageId, String pattern, boolean excludeDirs)
+ throws InvalidGlobPatternException {
+ if (pattern.indexOf('?') != -1) {
+ throw new InvalidGlobPatternException(pattern, "wildcard ? forbidden");
+ }
+
+ String error = UnixGlob.checkPatternForError(pattern);
+ if (error != null) {
+ throw new InvalidGlobPatternException(pattern, error);
+ }
+
+ return internalKey(packageId, PathFragment.EMPTY_FRAGMENT, pattern, excludeDirs);
+ }
+
+ /**
+ * Constructs a {@link SkyKey} for a glob lookup.
+ *
+ * <p>Do not use outside {@code GlobFunction}.
+ */
+ @ThreadSafe
+ static SkyKey internalKey(PackageIdentifier packageId, PathFragment subdir, String pattern,
+ boolean excludeDirs) {
+ return new SkyKey(SkyFunctions.GLOB,
+ new GlobDescriptor(packageId, subdir, pattern, excludeDirs));
+ }
+
+ /**
+ * Constructs a {@link SkyKey} for a glob lookup.
+ *
+ * <p>Do not use outside {@code GlobFunction}.
+ */
+ @ThreadSafe
+ static SkyKey internalKey(GlobDescriptor glob, String subdirName) {
+ return internalKey(glob.packageId, glob.subdir.getRelative(subdirName),
+ glob.pattern, glob.excludeDirs);
+ }
+
+ /**
+ * An exception that indicates that a glob pattern is syntactically invalid.
+ */
+ @ThreadSafe
+ static final class InvalidGlobPatternException extends Exception {
+ private final String pattern;
+
+ InvalidGlobPatternException(String pattern, String error) {
+ super(error);
+ this.pattern = pattern;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("invalid glob pattern '%s': %s", pattern, getMessage());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/IncompatibleViewException.java b/src/main/java/com/google/devtools/build/lib/skyframe/IncompatibleViewException.java
new file mode 100644
index 0000000..9d6f550
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/IncompatibleViewException.java
@@ -0,0 +1,26 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Thrown on {@link DiffAwareness#getDiff} to indicate that the given {@link DiffAwareness.View}s
+ * are incompatible with the {@link DiffAwareness} instance.
+ */
+public class IncompatibleViewException extends Exception {
+ public IncompatibleViewException(String msg) {
+ super(Preconditions.checkNotNull(msg));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/InconsistentFilesystemException.java b/src/main/java/com/google/devtools/build/lib/skyframe/InconsistentFilesystemException.java
new file mode 100644
index 0000000..26cb02f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/InconsistentFilesystemException.java
@@ -0,0 +1,27 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+/**
+ * Used to indicate a filesystem inconsistency, e.g. file 'a/b' exists but directory 'a' doesn't
+ * exist. This generally means the result of the build is undefined but we shouldn't crash hard.
+ */
+public class InconsistentFilesystemException extends Exception {
+ public InconsistentFilesystemException(String inconsistencyMessage) {
+ super("Inconsistent filesystem operations. " + inconsistencyMessage + " The results of the "
+ + "build are not guaranteed to be correct. You should probably run 'blaze clean' and "
+ + "investigate the filesystem inconsistency (likely due to filesytem updates concurrent "
+ + "with the build)");
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/LocalDiffAwareness.java b/src/main/java/com/google/devtools/build/lib/skyframe/LocalDiffAwareness.java
new file mode 100644
index 0000000..861f89ac
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/LocalDiffAwareness.java
@@ -0,0 +1,329 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.HashBiMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.Constants;
+import com.google.devtools.build.lib.vfs.ModifiedFileSet;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.nio.file.ClosedWatchServiceException;
+import java.nio.file.FileSystems;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.StandardWatchEventKinds;
+import java.nio.file.WatchEvent;
+import java.nio.file.WatchEvent.Kind;
+import java.nio.file.WatchKey;
+import java.nio.file.WatchService;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * File system watcher for local filesystems. It's able to provide a list of changed
+ * files between two consecutive calls. Uses the standard Java WatchService, which uses
+ * 'inotify' on Linux.
+ */
+public class LocalDiffAwareness implements DiffAwareness {
+
+ /** Factory for creating {@link LocalDiffAwareness} instances. */
+ public static class Factory implements DiffAwareness.Factory {
+ @Override
+ public DiffAwareness maybeCreate(com.google.devtools.build.lib.vfs.Path pathEntry) {
+ com.google.devtools.build.lib.vfs.Path resolvedPathEntry;
+ try {
+ resolvedPathEntry = pathEntry.resolveSymbolicLinks();
+ } catch (IOException e) {
+ return null;
+ }
+ PathFragment resolvedPathEntryFragment = resolvedPathEntry.asFragment();
+ // There's no good way to automatically detect network file systems. We rely on a blacklist
+ // for now (and maybe add a command-line option in the future?).
+ for (String prefix : Constants.WATCHFS_BLACKLIST) {
+ if (resolvedPathEntryFragment.startsWith(new PathFragment(prefix))) {
+ return null;
+ }
+ }
+
+ WatchService watchService;
+ try {
+ watchService = FileSystems.getDefault().newWatchService();
+ } catch (IOException e) {
+ return null;
+ }
+ return new LocalDiffAwareness(resolvedPathEntryFragment.toString(),
+ watchService);
+ }
+ }
+
+ private int numGetCurrentViewCalls = 0;
+
+ /**
+ * Bijection from WatchKey to the (absolute) Path being watched. WatchKeys don't have this
+ * functionality built-in so we do it ourselves.
+ */
+ private final HashBiMap<WatchKey, Path> watchKeyToDirBiMap = HashBiMap.create();
+
+ /** Root directory to watch. This is an absolute path. */
+ private final Path watchRootPath;
+
+ /** Every directory is registered under this watch service. */
+ private WatchService watchService;
+
+ private LocalDiffAwareness(String watchRoot, WatchService watchService) {
+ this.watchRootPath = FileSystems.getDefault().getPath(watchRoot);
+ this.watchService = watchService;
+ }
+
+ /**
+ * The WatchService is inherently sequential and side-effectful, so we enforce this by only
+ * supporting {@link #getDiff} calls that happen to be sequential.
+ */
+ private static class SequentialView implements DiffAwareness.View {
+ private final LocalDiffAwareness owner;
+ private final int position;
+ private final Set<Path> modifiedAbsolutePaths;
+
+ public SequentialView(LocalDiffAwareness owner, int position, Set<Path> modifiedAbsolutePaths) {
+ this.owner = owner;
+ this.position = position;
+ this.modifiedAbsolutePaths = modifiedAbsolutePaths;
+ }
+
+ public static boolean areInSequence(SequentialView oldView, SequentialView newView) {
+ return oldView.owner == newView.owner && (oldView.position + 1) == newView.position;
+ }
+ }
+
+ @Override
+ public SequentialView getCurrentView() throws BrokenDiffAwarenessException {
+ Set<Path> modifiedAbsolutePaths;
+ if (numGetCurrentViewCalls++ == 0) {
+ try {
+ registerSubDirectoriesAndReturnContents(watchRootPath);
+ } catch (IOException e) {
+ close();
+ throw new BrokenDiffAwarenessException(
+ "Error encountered with local file system watcher " + e);
+ }
+ modifiedAbsolutePaths = ImmutableSet.of();
+ } else {
+ try {
+ modifiedAbsolutePaths = collectChanges();
+ } catch (BrokenDiffAwarenessException e) {
+ close();
+ throw e;
+ } catch (IOException e) {
+ close();
+ throw new BrokenDiffAwarenessException(
+ "Error encountered with local file system watcher " + e);
+ } catch (ClosedWatchServiceException e) {
+ throw new BrokenDiffAwarenessException(
+ "Internal error with the local file system watcher " + e);
+ }
+ }
+ return new SequentialView(this, numGetCurrentViewCalls, modifiedAbsolutePaths);
+ }
+
+ @Override
+ public ModifiedFileSet getDiff(View oldView, View newView)
+ throws IncompatibleViewException, BrokenDiffAwarenessException {
+ SequentialView oldSequentialView;
+ SequentialView newSequentialView;
+ try {
+ oldSequentialView = (SequentialView) oldView;
+ newSequentialView = (SequentialView) newView;
+ } catch (ClassCastException e) {
+ throw new IncompatibleViewException("Given views are not from LocalDiffAwareness");
+ }
+ if (!SequentialView.areInSequence(oldSequentialView, newSequentialView)) {
+ return ModifiedFileSet.EVERYTHING_MODIFIED;
+ }
+ return ModifiedFileSet.builder()
+ .modifyAll(Iterables.transform(newSequentialView.modifiedAbsolutePaths,
+ nioAbsolutePathToPathFragment))
+ .build();
+ }
+
+ @Override
+ public void close() {
+ try {
+ watchService.close();
+ } catch (IOException ignored) {
+ // Nothing we can do here.
+ }
+ }
+
+ /** Converts java.nio.file.Path objects to vfs.PathFragment. */
+ private final Function<Path, PathFragment> nioAbsolutePathToPathFragment =
+ new Function<Path, PathFragment>() {
+ @Override
+ public PathFragment apply(Path input) {
+ Preconditions.checkArgument(input.startsWith(watchRootPath), "%s %s", input,
+ watchRootPath);
+ return new PathFragment(watchRootPath.relativize(input).toString());
+ }
+ };
+
+ /** Returns the changed files caught by the watch service. */
+ private Set<Path> collectChanges() throws BrokenDiffAwarenessException, IOException {
+ Set<Path> createdFilesAndDirectories = new HashSet<Path>();
+ Set<Path> deletedOrModifiedFilesAndDirectories = new HashSet<Path>();
+ Set<Path> deletedTrackedDirectories = new HashSet<Path>();
+
+ WatchKey watchKey;
+ while ((watchKey = watchService.poll()) != null) {
+ Path dir = watchKeyToDirBiMap.get(watchKey);
+ Preconditions.checkArgument(dir != null);
+
+ // We replay all the events for this watched directory in chronological order and
+ // construct the diff of this directory since the last #collectChanges call.
+ for (WatchEvent<?> event : watchKey.pollEvents()) {
+ Kind<?> kind = event.kind();
+ if (kind == StandardWatchEventKinds.OVERFLOW) {
+ // TODO(bazel-team): find out when an overflow might happen, and maybe handle it more
+ // gently.
+ throw new BrokenDiffAwarenessException("Overflow when watching local filesystem for "
+ + "changes");
+ }
+ if (event.context() == null) {
+ // The WatchService documentation mentions that WatchEvent#context may return null, but
+ // doesn't explain how/why it would do so. Looking at the implementation, it only
+ // happens on an overflow event. But we make no assumptions about that implementation
+ // detail here.
+ throw new BrokenDiffAwarenessException("Insufficient information from local file system "
+ + "watcher");
+ }
+ // For the events we've registered, the context given is a relative path.
+ Path relativePath = (Path) event.context();
+ Path path = dir.resolve(relativePath);
+ Preconditions.checkState(path.isAbsolute(), path);
+ if (kind == StandardWatchEventKinds.ENTRY_CREATE) {
+ createdFilesAndDirectories.add(path);
+ deletedOrModifiedFilesAndDirectories.remove(path);
+ } else if (kind == StandardWatchEventKinds.ENTRY_DELETE) {
+ createdFilesAndDirectories.remove(path);
+ deletedOrModifiedFilesAndDirectories.add(path);
+ if (watchKeyToDirBiMap.containsValue(path)) {
+ // If the deleted directory has children, then there will also be events for the
+ // WatchKey of the directory itself. WatchService#poll doesn't specify the order in
+ // which WatchKeys are returned, so the key for the directory itself may be processed
+ // *after* the current key (the parent of the deleted directory), and so we don't want
+ // to remove the deleted directory from our bimap just yet.
+ //
+ // For example, suppose we have the file '/root/a/foo.txt' and are watching the
+ // directories '/root' and '/root/a'. If the directory '/root/a' gets deleted then the
+ // following is a valid sequence of events by key.
+ //
+ // WatchKey '/root/'
+ // WatchEvent EVENT_MODIFY 'a'
+ // WatchEvent EVENT_DELETE 'a'
+ // WatchKey '/root/a'
+ // WatchEvent EVENT_DELETE 'foo.txt'
+ deletedTrackedDirectories.add(path);
+ }
+ } else if (kind == StandardWatchEventKinds.ENTRY_MODIFY) {
+ // If a file was created and then modified, then the net diff is that it was
+ // created.
+ if (!createdFilesAndDirectories.contains(path)) {
+ deletedOrModifiedFilesAndDirectories.add(path);
+ }
+ }
+ }
+
+ if (!watchKey.reset()) {
+ // Watcher got deleted, directory no longer valid.
+ watchKeyToDirBiMap.remove(watchKey);
+ }
+ }
+
+ for (Path path : deletedTrackedDirectories) {
+ WatchKey staleKey = watchKeyToDirBiMap.inverse().get(path);
+ watchKeyToDirBiMap.remove(staleKey);
+ }
+ if (watchKeyToDirBiMap.isEmpty()) {
+ // No more directories to watch, something happened the root directory being watched.
+ throw new IOException("Root directory " + watchRootPath + " became inaccessible.");
+ }
+
+ Set<Path> changedPaths = new HashSet<Path>();
+ for (Path path : createdFilesAndDirectories) {
+ if (Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) {
+ // This is a new directory, so changes to it since its creation have not been watched.
+ // We manually traverse the directory tree to register all the new subdirectories and find
+ // all the new subdirectories and files.
+ changedPaths.addAll(registerSubDirectoriesAndReturnContents(path));
+ } else {
+ changedPaths.add(path);
+ }
+ }
+ changedPaths.addAll(deletedOrModifiedFilesAndDirectories);
+ return changedPaths;
+ }
+
+ /**
+ * Traverses directory tree to register subdirectories. Returns all paths traversed (as absolute
+ * paths).
+ */
+ private Set<Path> registerSubDirectoriesAndReturnContents(Path rootDir) throws IOException {
+ Set<Path> visitedAbsolutePaths = new HashSet<Path>();
+ // Note that this does not follow symlinks.
+ Files.walkFileTree(rootDir, new WatcherFileVisitor(visitedAbsolutePaths));
+ return visitedAbsolutePaths;
+ }
+
+ /** File visitor used by Files.walkFileTree() upon traversing subdirectories. */
+ private class WatcherFileVisitor extends SimpleFileVisitor<Path> {
+
+ private final Set<Path> visitedAbsolutePaths;
+
+ private WatcherFileVisitor(Set<Path> visitedPaths) {
+ this.visitedAbsolutePaths = visitedPaths;
+ }
+
+ @Override
+ public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) {
+ Preconditions.checkState(path.isAbsolute(), path);
+ visitedAbsolutePaths.add(path);
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs)
+ throws IOException {
+ // It's important that we register the directory before we visit its children. This way we
+ // are guaranteed to see new files/directories either on this #getDiff or the next one.
+ // Otherwise, e.g., an intra-build creation of a child directory will be forever missed if it
+ // happens before the directory is listed as part of the visitation.
+ WatchKey key = path.register(watchService,
+ StandardWatchEventKinds.ENTRY_CREATE,
+ StandardWatchEventKinds.ENTRY_MODIFY,
+ StandardWatchEventKinds.ENTRY_DELETE);
+ Preconditions.checkState(path.isAbsolute(), path);
+ visitedAbsolutePaths.add(path);
+ watchKeyToDirBiMap.put(key, path);
+ return FileVisitResult.CONTINUE;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/MutableSupplier.java b/src/main/java/com/google/devtools/build/lib/skyframe/MutableSupplier.java
new file mode 100644
index 0000000..86de11d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/MutableSupplier.java
@@ -0,0 +1,46 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Supplier;
+
+/**
+ * Supplier whose value can be changed by clients who have a reference to it as a MutableSupplier.
+ * Unlike an {@code AtomicReference}, clients who are passed a MutableSupplier as a Supplier cannot
+ * change its value without a reckless cast.
+ */
+public class MutableSupplier<T> implements Supplier<T> {
+ private T val;
+
+ @Override
+ public T get() {
+ return val;
+ }
+
+ /**
+ * Sets the value of the object supplied. Do not cast a Supplier to a MutableSupplier in order to
+ * call this method!
+ */
+ public void set(T newVal) {
+ val = newVal;
+ }
+
+ @SuppressWarnings("deprecation") // MoreObjects.toStringHelper() is not in Guava
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(getClass())
+ .add("val", val).toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PackageFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/PackageFunction.java
new file mode 100644
index 0000000..2404b99
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PackageFunction.java
@@ -0,0 +1,809 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.Constants;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
+import com.google.devtools.build.lib.packages.BuildFileNotFoundException;
+import com.google.devtools.build.lib.packages.CachingPackageLocator;
+import com.google.devtools.build.lib.packages.InvalidPackageNameException;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.PackageFactory.Globber;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.packages.PackageLoadedEvent;
+import com.google.devtools.build.lib.packages.Preprocessor;
+import com.google.devtools.build.lib.packages.RuleVisibility;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.skyframe.ASTFileLookupValue.ASTLookupInputException;
+import com.google.devtools.build.lib.skyframe.GlobValue.InvalidGlobPatternException;
+import com.google.devtools.build.lib.skyframe.SkylarkImportLookupFunction.SkylarkImportFailedException;
+import com.google.devtools.build.lib.syntax.BuildFileAST;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.ParserInputSource;
+import com.google.devtools.build.lib.syntax.SkylarkEnvironment;
+import com.google.devtools.build.lib.syntax.Statement;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.util.JavaClock;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import com.google.devtools.build.skyframe.ValueOrException3;
+import com.google.devtools.build.skyframe.ValueOrException4;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * A SkyFunction for {@link PackageValue}s.
+ */
+public class PackageFunction implements SkyFunction {
+
+ private final EventHandler reporter;
+ private final PackageFactory packageFactory;
+ private final CachingPackageLocator packageLocator;
+ private final ConcurrentMap<PackageIdentifier, Package.LegacyBuilder> packageFunctionCache;
+ private final AtomicBoolean showLoadingProgress;
+ private final AtomicReference<EventBus> eventBus;
+ private final AtomicInteger numPackagesLoaded;
+ private final Profiler profiler = Profiler.instance();
+
+ private static final PathFragment PRELUDE_FILE_FRAGMENT =
+ new PathFragment(Constants.PRELUDE_FILE_DEPOT_RELATIVE_PATH);
+
+ static final String DEFAULTS_PACKAGE_NAME = "tools/defaults";
+ public static final String EXTERNAL_PACKAGE_NAME = "external";
+
+ static {
+ Preconditions.checkArgument(!PRELUDE_FILE_FRAGMENT.isAbsolute());
+ }
+
+ public PackageFunction(Reporter reporter, PackageFactory packageFactory,
+ CachingPackageLocator pkgLocator, AtomicBoolean showLoadingProgress,
+ ConcurrentMap<PackageIdentifier, Package.LegacyBuilder> packageFunctionCache,
+ AtomicReference<EventBus> eventBus, AtomicInteger numPackagesLoaded) {
+ this.reporter = reporter;
+
+ this.packageFactory = packageFactory;
+ this.packageLocator = pkgLocator;
+ this.showLoadingProgress = showLoadingProgress;
+ this.packageFunctionCache = packageFunctionCache;
+ this.eventBus = eventBus;
+ this.numPackagesLoaded = numPackagesLoaded;
+ }
+
+ private static void maybeThrowFilesystemInconsistency(String packageName,
+ Exception skyframeException, boolean packageWasInError)
+ throws InternalInconsistentFilesystemException {
+ if (!packageWasInError) {
+ throw new InternalInconsistentFilesystemException(packageName, "Encountered error '"
+ + skyframeException.getMessage() + "' but didn't encounter it when doing the same thing "
+ + "earlier in the build");
+ }
+ }
+
+ /**
+ * Marks the given dependencies, and returns those already present. Ignores any exception
+ * thrown while building the dependency, except for filesystem inconsistencies.
+ *
+ * <p>We need to mark dependencies implicitly used by the legacy package loading code, but we
+ * don't care about any skyframe errors since the package knows whether it's in error or not.
+ */
+ private static Pair<? extends Map<PathFragment, PackageLookupValue>, Boolean>
+ getPackageLookupDepsAndPropagateInconsistentFilesystemExceptions(String packageName,
+ Iterable<SkyKey> depKeys, Environment env, boolean packageWasInError)
+ throws InternalInconsistentFilesystemException {
+ Preconditions.checkState(
+ Iterables.all(depKeys, SkyFunctions.isSkyFunction(SkyFunctions.PACKAGE_LOOKUP)), depKeys);
+ boolean packageShouldBeInError = packageWasInError;
+ ImmutableMap.Builder<PathFragment, PackageLookupValue> builder = ImmutableMap.builder();
+ for (Map.Entry<SkyKey, ValueOrException3<BuildFileNotFoundException,
+ InconsistentFilesystemException, FileSymlinkCycleException>> entry :
+ env.getValuesOrThrow(depKeys, BuildFileNotFoundException.class,
+ InconsistentFilesystemException.class,
+ FileSymlinkCycleException.class).entrySet()) {
+ PathFragment pkgName = ((PackageIdentifier) entry.getKey().argument()).getPackageFragment();
+ try {
+ PackageLookupValue value = (PackageLookupValue) entry.getValue().get();
+ if (value != null) {
+ builder.put(pkgName, value);
+ }
+ } catch (BuildFileNotFoundException e) {
+ maybeThrowFilesystemInconsistency(packageName, e, packageWasInError);
+ } catch (InconsistentFilesystemException e) {
+ throw new InternalInconsistentFilesystemException(packageName, e);
+ } catch (FileSymlinkCycleException e) {
+ // Legacy doesn't detect symlink cycles.
+ packageShouldBeInError = true;
+ }
+ }
+ return Pair.of(builder.build(), packageShouldBeInError);
+ }
+
+ private static boolean markFileDepsAndPropagateInconsistentFilesystemExceptions(
+ String packageName, Iterable<SkyKey> depKeys, Environment env, boolean packageWasInError)
+ throws InternalInconsistentFilesystemException {
+ Preconditions.checkState(
+ Iterables.all(depKeys, SkyFunctions.isSkyFunction(SkyFunctions.FILE)), depKeys);
+ boolean packageShouldBeInError = packageWasInError;
+ for (Map.Entry<SkyKey, ValueOrException3<IOException, FileSymlinkCycleException,
+ InconsistentFilesystemException>> entry : env.getValuesOrThrow(depKeys, IOException.class,
+ FileSymlinkCycleException.class, InconsistentFilesystemException.class).entrySet()) {
+ try {
+ entry.getValue().get();
+ } catch (IOException e) {
+ maybeThrowFilesystemInconsistency(packageName, e, packageWasInError);
+ } catch (FileSymlinkCycleException e) {
+ // Legacy doesn't detect symlink cycles.
+ packageShouldBeInError = true;
+ } catch (InconsistentFilesystemException e) {
+ throw new InternalInconsistentFilesystemException(packageName, e);
+ }
+ }
+ return packageShouldBeInError;
+ }
+
+ private static boolean markGlobDepsAndPropagateInconsistentFilesystemExceptions(
+ String packageName, Iterable<SkyKey> depKeys, Environment env, boolean packageWasInError)
+ throws InternalInconsistentFilesystemException {
+ Preconditions.checkState(
+ Iterables.all(depKeys, SkyFunctions.isSkyFunction(SkyFunctions.GLOB)), depKeys);
+ boolean packageShouldBeInError = packageWasInError;
+ for (Map.Entry<SkyKey, ValueOrException4<IOException, BuildFileNotFoundException,
+ FileSymlinkCycleException, InconsistentFilesystemException>> entry :
+ env.getValuesOrThrow(depKeys, IOException.class, BuildFileNotFoundException.class,
+ FileSymlinkCycleException.class, InconsistentFilesystemException.class).entrySet()) {
+ try {
+ entry.getValue().get();
+ } catch (IOException | BuildFileNotFoundException e) {
+ maybeThrowFilesystemInconsistency(packageName, e, packageWasInError);
+ } catch (FileSymlinkCycleException e) {
+ // Legacy doesn't detect symlink cycles.
+ packageShouldBeInError = true;
+ } catch (InconsistentFilesystemException e) {
+ throw new InternalInconsistentFilesystemException(packageName, e);
+ }
+ }
+ return packageShouldBeInError;
+ }
+
+ /**
+ * Marks dependencies implicitly used by legacy package loading code, after the fact. Note that
+ * the given package might already be in error.
+ *
+ * <p>Any skyframe exceptions encountered here are ignored, as similar errors should have
+ * already been encountered by legacy package loading (if not, then the filesystem is
+ * inconsistent).
+ */
+ private static boolean markDependenciesAndPropagateInconsistentFilesystemExceptions(
+ Package pkg, Environment env, Collection<Pair<String, Boolean>> globPatterns,
+ Map<Label, Path> subincludes) throws InternalInconsistentFilesystemException {
+ boolean packageShouldBeInError = pkg.containsErrors();
+
+ // TODO(bazel-team): This means that many packages will have to be preprocessed twice. Ouch!
+ // We need a better continuation mechanism to avoid repeating work. [skyframe-loading]
+
+ // TODO(bazel-team): It would be preferable to perform I/O from the package preprocessor via
+ // Skyframe rather than add (potentially incomplete) dependencies after the fact.
+ // [skyframe-loading]
+
+ Set<SkyKey> subincludePackageLookupDepKeys = Sets.newHashSet();
+ for (Label label : pkg.getSubincludeLabels()) {
+ // Declare a dependency on the package lookup for the package giving access to the label.
+ subincludePackageLookupDepKeys.add(PackageLookupValue.key(label.getPackageFragment()));
+ }
+ Pair<? extends Map<PathFragment, PackageLookupValue>, Boolean> subincludePackageLookupResult =
+ getPackageLookupDepsAndPropagateInconsistentFilesystemExceptions(pkg.getName(),
+ subincludePackageLookupDepKeys, env, pkg.containsErrors());
+ Map<PathFragment, PackageLookupValue> subincludePackageLookupDeps =
+ subincludePackageLookupResult.getFirst();
+ packageShouldBeInError = subincludePackageLookupResult.getSecond();
+ List<SkyKey> subincludeFileDepKeys = Lists.newArrayList();
+ for (Entry<Label, Path> subincludeEntry : subincludes.entrySet()) {
+ // Ideally, we would have a direct dependency on the target with the given label, but then
+ // subincluding a file from the same package will cause a dependency cycle, since targets
+ // depend on their containing packages.
+ Label label = subincludeEntry.getKey();
+ PackageLookupValue subincludePackageLookupValue =
+ subincludePackageLookupDeps.get(label.getPackageFragment());
+ if (subincludePackageLookupValue != null) {
+ // Declare a dependency on the actual file that was subincluded.
+ Path subincludeFilePath = subincludeEntry.getValue();
+ if (subincludeFilePath != null) {
+ if (!subincludePackageLookupValue.packageExists()) {
+ // Legacy blaze puts a non-null path when only when the package does indeed exist.
+ throw new InternalInconsistentFilesystemException(pkg.getName(), String.format(
+ "Unexpected package in %s. Was it modified during the build?", subincludeFilePath));
+ }
+ // Sanity check for consistency of Skyframe and legacy blaze.
+ Path subincludeFilePathSkyframe =
+ subincludePackageLookupValue.getRoot().getRelative(label.toPathFragment());
+ if (!subincludeFilePathSkyframe.equals(subincludeFilePath)) {
+ throw new InternalInconsistentFilesystemException(pkg.getName(), String.format(
+ "Inconsistent package location for %s: '%s' vs '%s'. "
+ + "Was the source tree modified during the build?",
+ label.getPackageFragment(), subincludeFilePathSkyframe, subincludeFilePath));
+ }
+ // The actual file may be under a different package root than the package being
+ // constructed.
+ SkyKey subincludeSkyKey =
+ FileValue.key(RootedPath.toRootedPath(subincludePackageLookupValue.getRoot(),
+ subincludeFilePath));
+ subincludeFileDepKeys.add(subincludeSkyKey);
+ }
+ }
+ }
+ packageShouldBeInError = markFileDepsAndPropagateInconsistentFilesystemExceptions(
+ pkg.getName(), subincludeFileDepKeys, env, pkg.containsErrors());
+ // Another concern is a subpackage cutting off the subinclude label, but this is already
+ // handled by the legacy package loading code which calls into our SkyframePackageLocator.
+
+ // TODO(bazel-team): In the long term, we want to actually resolve the glob patterns within
+ // Skyframe. For now, just logging the glob requests provides correct incrementality and
+ // adequate performance.
+ PackageIdentifier packageId = pkg.getPackageIdentifier();
+ List<SkyKey> globDepKeys = Lists.newArrayList();
+ for (Pair<String, Boolean> globPattern : globPatterns) {
+ String pattern = globPattern.getFirst();
+ boolean excludeDirs = globPattern.getSecond();
+ SkyKey globSkyKey;
+ try {
+ globSkyKey = GlobValue.key(packageId, pattern, excludeDirs);
+ } catch (InvalidGlobPatternException e) {
+ // Globs that make it to pkg.getGlobPatterns() should already be filtered for errors.
+ throw new IllegalStateException(e);
+ }
+ globDepKeys.add(globSkyKey);
+ }
+ packageShouldBeInError = markGlobDepsAndPropagateInconsistentFilesystemExceptions(
+ pkg.getName(), globDepKeys, env, pkg.containsErrors());
+ return packageShouldBeInError;
+ }
+
+ /**
+ * Adds a dependency on the WORKSPACE file, representing it as a special type of package.
+ * @throws PackageFunctionException if there is an error computing the workspace file or adding
+ * its rules to the //external package.
+ */
+ private SkyValue getExternalPackage(Environment env, Path packageLookupPath)
+ throws PackageFunctionException {
+ RootedPath workspacePath = RootedPath.toRootedPath(
+ packageLookupPath, new PathFragment("WORKSPACE"));
+ SkyKey workspaceKey = WorkspaceFileValue.key(workspacePath);
+ WorkspaceFileValue workspace = null;
+ try {
+ workspace = (WorkspaceFileValue) env.getValueOrThrow(workspaceKey, IOException.class,
+ FileSymlinkCycleException.class, InconsistentFilesystemException.class,
+ EvalException.class);
+ } catch (IOException | FileSymlinkCycleException | InconsistentFilesystemException
+ | EvalException e) {
+ throw new PackageFunctionException(new BadWorkspaceFileException(e.getMessage()),
+ Transience.PERSISTENT);
+ }
+ if (workspace == null) {
+ return null;
+ }
+
+ Package pkg = workspace.getPackage();
+ Event.replayEventsOn(env.getListener(), pkg.getEvents());
+ if (pkg.containsErrors()) {
+ throw new PackageFunctionException(new BuildFileContainsErrorsException("external",
+ "Package 'external' contains errors"),
+ pkg.containsTemporaryErrors() ? Transience.TRANSIENT : Transience.PERSISTENT);
+ }
+
+ return new PackageValue(pkg);
+ }
+
+ @Override
+ public SkyValue compute(SkyKey key, Environment env) throws PackageFunctionException,
+ InterruptedException {
+ PackageIdentifier packageId = (PackageIdentifier) key.argument();
+ PathFragment packageNameFragment = packageId.getPackageFragment();
+ String packageName = packageNameFragment.getPathString();
+
+ SkyKey packageLookupKey = PackageLookupValue.key(packageId);
+ PackageLookupValue packageLookupValue;
+ try {
+ packageLookupValue = (PackageLookupValue)
+ env.getValueOrThrow(packageLookupKey, BuildFileNotFoundException.class,
+ InconsistentFilesystemException.class);
+ } catch (BuildFileNotFoundException e) {
+ throw new PackageFunctionException(e, Transience.PERSISTENT);
+ } catch (InconsistentFilesystemException e) {
+ // This error is not transient from the perspective of the PackageFunction.
+ throw new PackageFunctionException(
+ new InternalInconsistentFilesystemException(packageName, e), Transience.PERSISTENT);
+ }
+ if (packageLookupValue == null) {
+ return null;
+ }
+
+ if (!packageLookupValue.packageExists()) {
+ switch (packageLookupValue.getErrorReason()) {
+ case NO_BUILD_FILE:
+ case DELETED_PACKAGE:
+ case NO_EXTERNAL_PACKAGE:
+ throw new PackageFunctionException(new BuildFileNotFoundException(packageName,
+ packageLookupValue.getErrorMsg()), Transience.PERSISTENT);
+ case INVALID_PACKAGE_NAME:
+ throw new PackageFunctionException(new InvalidPackageNameException(packageName,
+ packageLookupValue.getErrorMsg()), Transience.PERSISTENT);
+ default:
+ // We should never get here.
+ Preconditions.checkState(false);
+ }
+ }
+
+ if (packageName.equals(EXTERNAL_PACKAGE_NAME)) {
+ return getExternalPackage(env, packageLookupValue.getRoot());
+ }
+
+ RootedPath buildFileRootedPath = RootedPath.toRootedPath(packageLookupValue.getRoot(),
+ packageNameFragment.getChild("BUILD"));
+ FileValue buildFileValue;
+ try {
+ buildFileValue = (FileValue) env.getValueOrThrow(FileValue.key(buildFileRootedPath),
+ IOException.class, FileSymlinkCycleException.class,
+ InconsistentFilesystemException.class);
+ } catch (IOException | FileSymlinkCycleException | InconsistentFilesystemException e) {
+ throw new IllegalStateException("Package lookup succeeded but encountered error when "
+ + "getting FileValue for BUILD file directly.", e);
+ }
+ if (buildFileValue == null) {
+ return null;
+ }
+ Preconditions.checkState(buildFileValue.exists(),
+ "Package lookup succeeded but BUILD file doesn't exist");
+ Path buildFilePath = buildFileRootedPath.asPath();
+
+ String replacementContents = null;
+ if (packageName.equals(DEFAULTS_PACKAGE_NAME)) {
+ replacementContents = PrecomputedValue.DEFAULTS_PACKAGE_CONTENTS.get(env);
+ if (replacementContents == null) {
+ return null;
+ }
+ }
+
+ RuleVisibility defaultVisibility = PrecomputedValue.DEFAULT_VISIBILITY.get(env);
+ if (defaultVisibility == null) {
+ return null;
+ }
+
+ ASTFileLookupValue astLookupValue = null;
+ SkyKey astLookupKey = null;
+ try {
+ astLookupKey = ASTFileLookupValue.key(PRELUDE_FILE_FRAGMENT);
+ } catch (ASTLookupInputException e) {
+ // There's a static check ensuring that PRELUDE_FILE_FRAGMENT is relative.
+ throw new IllegalStateException(e);
+ }
+ try {
+ astLookupValue = (ASTFileLookupValue) env.getValueOrThrow(astLookupKey,
+ ErrorReadingSkylarkExtensionException.class, InconsistentFilesystemException.class);
+ } catch (ErrorReadingSkylarkExtensionException | InconsistentFilesystemException e) {
+ throw new PackageFunctionException(new BadPreludeFileException(packageName, e.getMessage()),
+ Transience.PERSISTENT);
+ }
+ if (astLookupValue == null) {
+ return null;
+ }
+ List<Statement> preludeStatements = astLookupValue == ASTFileLookupValue.NO_FILE
+ ? ImmutableList.<Statement>of() : astLookupValue.getAST().getStatements();
+
+ // Load the BUILD file AST and handle Skylark dependencies. This way BUILD files are
+ // only loaded twice if there are unavailable Skylark or package dependencies or an
+ // IOException occurs. Note that the BUILD files are still parsed two times.
+ ParserInputSource inputSource;
+ try {
+ if (showLoadingProgress.get() && !packageFunctionCache.containsKey(packageId)) {
+ // TODO(bazel-team): don't duplicate the loading message if there are unavailable
+ // Skylark dependencies.
+ reporter.handle(Event.progress("Loading package: " + packageName));
+ }
+ inputSource = ParserInputSource.create(buildFilePath);
+ } catch (IOException e) {
+ env.getListener().handle(Event.error(Location.fromFile(buildFilePath), e.getMessage()));
+ // Note that we did this work, so we should conservatively report this error as transient.
+ throw new PackageFunctionException(new BuildFileContainsErrorsException(
+ packageName, e.getMessage()), Transience.TRANSIENT);
+ }
+ SkylarkImportResult importResult = fetchImportsFromBuildFile(
+ buildFilePath, packageId.getRepository(), preludeStatements, inputSource, packageName, env);
+ if (importResult == null) {
+ return null;
+ }
+
+ Package.LegacyBuilder legacyPkgBuilder = loadPackage(inputSource, replacementContents,
+ packageId, buildFilePath, defaultVisibility, preludeStatements, importResult);
+ legacyPkgBuilder.buildPartial();
+ try {
+ handleLabelsCrossingSubpackagesAndPropagateInconsistentFilesystemExceptions(
+ packageLookupValue.getRoot(), packageId, legacyPkgBuilder, env);
+ } catch (InternalInconsistentFilesystemException e) {
+ packageFunctionCache.remove(packageId);
+ throw new PackageFunctionException(e,
+ e.isTransient() ? Transience.TRANSIENT : Transience.PERSISTENT);
+ }
+ if (env.valuesMissing()) {
+ // The package we just loaded will be in the {@code packageFunctionCache} next when this
+ // SkyFunction is called again.
+ return null;
+ }
+ Collection<Pair<String, Boolean>> globPatterns = legacyPkgBuilder.getGlobPatterns();
+ Map<Label, Path> subincludes = legacyPkgBuilder.getSubincludes();
+ Package pkg = legacyPkgBuilder.finishBuild();
+ Event.replayEventsOn(env.getListener(), pkg.getEvents());
+ boolean packageShouldBeConsideredInError = pkg.containsErrors();
+ try {
+ packageShouldBeConsideredInError =
+ markDependenciesAndPropagateInconsistentFilesystemExceptions(pkg, env,
+ globPatterns, subincludes);
+ } catch (InternalInconsistentFilesystemException e) {
+ packageFunctionCache.remove(packageId);
+ throw new PackageFunctionException(e,
+ e.isTransient() ? Transience.TRANSIENT : Transience.PERSISTENT);
+ }
+
+ if (env.valuesMissing()) {
+ return null;
+ }
+ // We know this SkyFunction will not be called again, so we can remove the cache entry.
+ packageFunctionCache.remove(packageId);
+
+ if (packageShouldBeConsideredInError) {
+ throw new PackageFunctionException(new BuildFileContainsErrorsException(pkg,
+ "Package '" + packageName + "' contains errors"),
+ pkg.containsTemporaryErrors() ? Transience.TRANSIENT : Transience.PERSISTENT);
+ }
+ return new PackageValue(pkg);
+ }
+
+ private SkylarkImportResult fetchImportsFromBuildFile(Path buildFilePath, RepositoryName repo,
+ List<Statement> preludeStatements, ParserInputSource inputSource,
+ String packageName, Environment env) throws PackageFunctionException {
+ StoredEventHandler eventHandler = new StoredEventHandler();
+ BuildFileAST buildFileAST = BuildFileAST.parseBuildFile(
+ inputSource, preludeStatements, eventHandler, null, true);
+
+ if (eventHandler.hasErrors()) {
+ // In case of Python preprocessing, errors have already been reported (see checkSyntax).
+ // In other cases, errors will be reported later.
+ // TODO(bazel-team): maybe we could get rid of checkSyntax and always report errors here?
+ return new SkylarkImportResult(
+ ImmutableMap.<PathFragment, SkylarkEnvironment>of(),
+ ImmutableList.<Label>of());
+ }
+
+ ImmutableCollection<PathFragment> imports = buildFileAST.getImports();
+ Map<PathFragment, SkylarkEnvironment> importMap = new HashMap<>();
+ ImmutableList.Builder<SkylarkFileDependency> fileDependencies = ImmutableList.builder();
+ try {
+ for (PathFragment importFile : imports) {
+ SkyKey importsLookupKey = SkylarkImportLookupValue.key(repo, importFile);
+ SkylarkImportLookupValue importLookupValue = (SkylarkImportLookupValue)
+ env.getValueOrThrow(importsLookupKey, SkylarkImportFailedException.class,
+ InconsistentFilesystemException.class, ASTLookupInputException.class,
+ BuildFileNotFoundException.class);
+ if (importLookupValue != null) {
+ importMap.put(importFile, importLookupValue.getImportedEnvironment());
+ fileDependencies.add(importLookupValue.getDependency());
+ }
+ }
+ } catch (SkylarkImportFailedException e) {
+ env.getListener().handle(Event.error(Location.fromFile(buildFilePath), e.getMessage()));
+ throw new PackageFunctionException(new BuildFileContainsErrorsException(packageName,
+ e.getMessage()), Transience.PERSISTENT);
+ } catch (InconsistentFilesystemException e) {
+ throw new PackageFunctionException(new InternalInconsistentFilesystemException(packageName,
+ e), Transience.PERSISTENT);
+ } catch (ASTLookupInputException e) {
+ // The load syntax is bad in the BUILD file so BuildFileContainsErrorsException is OK.
+ throw new PackageFunctionException(new BuildFileContainsErrorsException(packageName,
+ e.getMessage()), Transience.PERSISTENT);
+ } catch (BuildFileNotFoundException e) {
+ throw new PackageFunctionException(e, Transience.PERSISTENT);
+ }
+ if (env.valuesMissing()) {
+ // There are unavailable Skylark dependencies.
+ return null;
+ }
+ return new SkylarkImportResult(importMap, transitiveClosureOfLabels(fileDependencies.build()));
+ }
+
+ private ImmutableList<Label> transitiveClosureOfLabels(
+ ImmutableList<SkylarkFileDependency> immediateDeps) {
+ Set<Label> transitiveClosure = Sets.newHashSet();
+ transitiveClosureOfLabels(immediateDeps, transitiveClosure);
+ return ImmutableList.copyOf(transitiveClosure);
+ }
+
+ private void transitiveClosureOfLabels(
+ ImmutableList<SkylarkFileDependency> immediateDeps, Set<Label> transitiveClosure) {
+ for (SkylarkFileDependency dep : immediateDeps) {
+ if (!transitiveClosure.contains(dep.getLabel())) {
+ transitiveClosure.add(dep.getLabel());
+ transitiveClosureOfLabels(dep.getDependencies(), transitiveClosure);
+ }
+ }
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ private static void handleLabelsCrossingSubpackagesAndPropagateInconsistentFilesystemExceptions(
+ Path pkgRoot, PackageIdentifier pkgId, Package.LegacyBuilder pkgBuilder, Environment env)
+ throws InternalInconsistentFilesystemException {
+ Set<SkyKey> containingPkgLookupKeys = Sets.newHashSet();
+ Map<Target, SkyKey> targetToKey = new HashMap<>();
+ for (Target target : pkgBuilder.getTargets()) {
+ PathFragment dir = target.getLabel().toPathFragment().getParentDirectory();
+ PackageIdentifier dirId = new PackageIdentifier(pkgId.getRepository(), dir);
+ if (dir.equals(pkgId.getPackageFragment())) {
+ continue;
+ }
+ SkyKey key = ContainingPackageLookupValue.key(dirId);
+ targetToKey.put(target, key);
+ containingPkgLookupKeys.add(key);
+ }
+ Map<Label, SkyKey> subincludeToKey = new HashMap<>();
+ for (Label subincludeLabel : pkgBuilder.getSubincludeLabels()) {
+ PathFragment dir = subincludeLabel.toPathFragment().getParentDirectory();
+ PackageIdentifier dirId = new PackageIdentifier(pkgId.getRepository(), dir);
+ if (dir.equals(pkgId.getPackageFragment())) {
+ continue;
+ }
+ SkyKey key = ContainingPackageLookupValue.key(dirId);
+ subincludeToKey.put(subincludeLabel, key);
+ containingPkgLookupKeys.add(ContainingPackageLookupValue.key(dirId));
+ }
+ Map<SkyKey, ValueOrException3<BuildFileNotFoundException, InconsistentFilesystemException,
+ FileSymlinkCycleException>> containingPkgLookupValues = env.getValuesOrThrow(
+ containingPkgLookupKeys, BuildFileNotFoundException.class,
+ InconsistentFilesystemException.class, FileSymlinkCycleException.class);
+ if (env.valuesMissing()) {
+ return;
+ }
+ for (Target target : ImmutableSet.copyOf(pkgBuilder.getTargets())) {
+ SkyKey key = targetToKey.get(target);
+ if (!containingPkgLookupValues.containsKey(key)) {
+ continue;
+ }
+ ContainingPackageLookupValue containingPackageLookupValue =
+ getContainingPkgLookupValueAndPropagateInconsistentFilesystemExceptions(
+ pkgId.getPackageFragment().getPathString(), containingPkgLookupValues.get(key), env);
+ if (maybeAddEventAboutLabelCrossingSubpackage(pkgBuilder, pkgRoot, target.getLabel(),
+ target.getLocation(), containingPackageLookupValue)) {
+ pkgBuilder.removeTarget(target);
+ pkgBuilder.setContainsErrors();
+ }
+ }
+ for (Label subincludeLabel : pkgBuilder.getSubincludeLabels()) {
+ SkyKey key = subincludeToKey.get(subincludeLabel);
+ if (!containingPkgLookupValues.containsKey(key)) {
+ continue;
+ }
+ ContainingPackageLookupValue containingPackageLookupValue =
+ getContainingPkgLookupValueAndPropagateInconsistentFilesystemExceptions(
+ pkgId.getPackageFragment().getPathString(), containingPkgLookupValues.get(key), env);
+ if (maybeAddEventAboutLabelCrossingSubpackage(pkgBuilder, pkgRoot, subincludeLabel,
+ /*location=*/null, containingPackageLookupValue)) {
+ pkgBuilder.setContainsErrors();
+ }
+ }
+ }
+
+ @Nullable
+ private static ContainingPackageLookupValue
+ getContainingPkgLookupValueAndPropagateInconsistentFilesystemExceptions(String packageName,
+ ValueOrException3<BuildFileNotFoundException, InconsistentFilesystemException,
+ FileSymlinkCycleException> containingPkgLookupValueOrException, Environment env)
+ throws InternalInconsistentFilesystemException {
+ try {
+ return (ContainingPackageLookupValue) containingPkgLookupValueOrException.get();
+ } catch (BuildFileNotFoundException | FileSymlinkCycleException e) {
+ env.getListener().handle(Event.error(null, e.getMessage()));
+ return null;
+ } catch (InconsistentFilesystemException e) {
+ throw new InternalInconsistentFilesystemException(packageName, e);
+ }
+ }
+
+ private static boolean maybeAddEventAboutLabelCrossingSubpackage(
+ Package.LegacyBuilder pkgBuilder, Path pkgRoot, Label label, @Nullable Location location,
+ @Nullable ContainingPackageLookupValue containingPkgLookupValue) {
+ if (containingPkgLookupValue == null) {
+ return true;
+ }
+ if (!containingPkgLookupValue.hasContainingPackage()) {
+ // The missing package here is a problem, but it's not an error from the perspective of
+ // PackageFunction.
+ return false;
+ }
+ PackageIdentifier containingPkg = containingPkgLookupValue.getContainingPackageName();
+ if (containingPkg.equals(label.getPackageIdentifier())) {
+ // The label does not cross a subpackage boundary.
+ return false;
+ }
+ if (!containingPkg.getPackageFragment().startsWith(label.getPackageFragment())) {
+ // This label is referencing an imaginary package, because the containing package should
+ // extend the label's package: if the label is //a/b:c/d, the containing package could be
+ // //a/b/c or //a/b, but should never be //a. Usually such errors will be caught earlier, but
+ // in some exceptional cases (such as a Python-aware BUILD file catching its own io
+ // exceptions), it reaches here, and we tolerate it.
+ return false;
+ }
+ PathFragment labelNameFragment = new PathFragment(label.getName());
+ String message = String.format("Label '%s' crosses boundary of subpackage '%s'",
+ label, containingPkg);
+ Path containingRoot = containingPkgLookupValue.getContainingPackageRoot();
+ if (pkgRoot.equals(containingRoot)) {
+ PathFragment labelNameInContainingPackage = labelNameFragment.subFragment(
+ containingPkg.getPackageFragment().segmentCount()
+ - label.getPackageFragment().segmentCount(),
+ labelNameFragment.segmentCount());
+ message += " (perhaps you meant to put the colon here: "
+ + "'//" + containingPkg + ":" + labelNameInContainingPackage + "'?)";
+ } else {
+ message += " (have you deleted " + containingPkg + "/BUILD? "
+ + "If so, use the --deleted_packages=" + containingPkg + " option)";
+ }
+ pkgBuilder.addEvent(Event.error(location, message));
+ return true;
+ }
+
+ /**
+ * Constructs a {@link Package} object for the given package using legacy package loading.
+ * Note that the returned package may be in error.
+ */
+ private Package.LegacyBuilder loadPackage(ParserInputSource inputSource,
+ @Nullable String replacementContents,
+ PackageIdentifier packageId, Path buildFilePath, RuleVisibility defaultVisibility,
+ List<Statement> preludeStatements, SkylarkImportResult importResult)
+ throws InterruptedException {
+ ParserInputSource replacementSource = replacementContents == null ? null
+ : ParserInputSource.create(replacementContents, buildFilePath);
+ Package.LegacyBuilder pkgBuilder = packageFunctionCache.get(packageId);
+ if (pkgBuilder == null) {
+ Clock clock = new JavaClock();
+ long startTime = clock.nanoTime();
+ profiler.startTask(ProfilerTask.CREATE_PACKAGE, packageId.toString());
+ try {
+ Globber globber = packageFactory.createLegacyGlobber(buildFilePath.getParentDirectory(),
+ packageId, packageLocator);
+ StoredEventHandler localReporter = new StoredEventHandler();
+ Preprocessor.Result preprocessingResult = replacementSource == null
+ ? packageFactory.preprocess(packageId, buildFilePath, inputSource, globber,
+ localReporter)
+ : Preprocessor.Result.noPreprocessing(replacementSource);
+ pkgBuilder = packageFactory.createPackageFromPreprocessingResult(packageId, buildFilePath,
+ preprocessingResult, localReporter.getEvents(), preludeStatements,
+ importResult.importMap, importResult.fileDependencies, packageLocator,
+ defaultVisibility, globber);
+ if (eventBus.get() != null) {
+ eventBus.get().post(new PackageLoadedEvent(packageId.toString(),
+ (clock.nanoTime() - startTime) / (1000 * 1000),
+ // It's impossible to tell if the package was loaded before, so we always pass false.
+ /*reloading=*/false,
+ // This isn't completely correct since we may encounter errors later (e.g. filesystem
+ // inconsistencies)
+ !pkgBuilder.containsErrors()));
+ }
+ numPackagesLoaded.incrementAndGet();
+ packageFunctionCache.put(packageId, pkgBuilder);
+ } finally {
+ profiler.completeTask(ProfilerTask.CREATE_PACKAGE);
+ }
+ }
+ return pkgBuilder;
+ }
+
+ private static class InternalInconsistentFilesystemException extends NoSuchPackageException {
+ private boolean isTransient;
+
+ /**
+ * Used to represent a filesystem inconsistency discovered outside the
+ * {@link PackageFunction}.
+ */
+ public InternalInconsistentFilesystemException(String packageName,
+ InconsistentFilesystemException e) {
+ super(packageName, e.getMessage(), e);
+ // This is not a transient error from the perspective of the PackageFunction.
+ this.isTransient = false;
+ }
+
+ /** Used to represent a filesystem inconsistency discovered by the {@link PackageFunction}. */
+ public InternalInconsistentFilesystemException(String packageName,
+ String inconsistencyMessage) {
+ this(packageName, new InconsistentFilesystemException(inconsistencyMessage));
+ this.isTransient = true;
+ }
+
+ public boolean isTransient() {
+ return isTransient;
+ }
+ }
+
+ private static class BadWorkspaceFileException extends NoSuchPackageException {
+ private BadWorkspaceFileException(String message) {
+ super("external", "Error encountered while dealing with the WORKSPACE file: " + message);
+ }
+ }
+
+ private static class BadPreludeFileException extends NoSuchPackageException {
+ private BadPreludeFileException(String packageName, String message) {
+ super(packageName, "Error encountered while reading the prelude file: " + message);
+ }
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link PackageFunction#compute}.
+ */
+ private static class PackageFunctionException extends SkyFunctionException {
+ public PackageFunctionException(NoSuchPackageException e, Transience transience) {
+ super(e, transience);
+ }
+ }
+
+ /** A simple value class to store the result of the Skylark imports.*/
+ private static final class SkylarkImportResult {
+ private final Map<PathFragment, SkylarkEnvironment> importMap;
+ private final ImmutableList<Label> fileDependencies;
+ private SkylarkImportResult(Map<PathFragment, SkylarkEnvironment> importMap,
+ ImmutableList<Label> fileDependencies) {
+ this.importMap = importMap;
+ this.fileDependencies = fileDependencies;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PackageLookupFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/PackageLookupFunction.java
new file mode 100644
index 0000000..ae4ee55
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PackageLookupFunction.java
@@ -0,0 +1,180 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.cmdline.LabelValidator;
+import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
+import com.google.devtools.build.lib.packages.BuildFileNotFoundException;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * SkyFunction for {@link PackageLookupValue}s.
+ */
+class PackageLookupFunction implements SkyFunction {
+
+ private final AtomicReference<ImmutableSet<String>> deletedPackages;
+
+ PackageLookupFunction(AtomicReference<ImmutableSet<String>> deletedPackages) {
+ this.deletedPackages = deletedPackages;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws PackageLookupFunctionException {
+ PathPackageLocator pkgLocator = PrecomputedValue.PATH_PACKAGE_LOCATOR.get(env);
+ PackageIdentifier packageKey = (PackageIdentifier) skyKey.argument();
+ if (!packageKey.getRepository().isDefault()) {
+ return computeExternalPackageLookupValue(skyKey, env);
+ }
+ PathFragment pkg = packageKey.getPackageFragment();
+
+ // This represents a package lookup at the package root.
+ if (pkg.equals(PathFragment.EMPTY_FRAGMENT)) {
+ return PackageLookupValue.invalidPackageName("The empty package name is invalid");
+ }
+
+ String pkgName = pkg.getPathString();
+ String packageNameErrorMsg = LabelValidator.validatePackageName(pkgName);
+ if (packageNameErrorMsg != null) {
+ return PackageLookupValue.invalidPackageName("Invalid package name '" + pkgName + "': "
+ + packageNameErrorMsg);
+ }
+
+ if (deletedPackages.get().contains(pkg.getPathString())) {
+ return PackageLookupValue.deletedPackage();
+ }
+
+ // TODO(bazel-team): The following is O(n^2) on the number of elements on the package path due
+ // to having restart the SkyFunction after every new dependency. However, if we try to batch
+ // the missing value keys, more dependencies than necessary will be declared. This wart can be
+ // fixed once we have nicer continuation support [skyframe-loading]
+ for (Path packagePathEntry : pkgLocator.getPathEntries()) {
+ PackageLookupValue value = getPackageLookupValue(env, packagePathEntry, pkg);
+ if (value == null || value.packageExists()) {
+ return value;
+ }
+ }
+ return PackageLookupValue.noBuildFile();
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ private PackageLookupValue getPackageLookupValue(Environment env, Path packagePathEntry,
+ PathFragment pkgFragment) throws PackageLookupFunctionException {
+ PathFragment buildFileFragment;
+ if (pkgFragment.getPathString().equals(PackageFunction.EXTERNAL_PACKAGE_NAME)) {
+ buildFileFragment = new PathFragment("WORKSPACE");
+ } else {
+ buildFileFragment = pkgFragment.getChild("BUILD");
+ }
+ RootedPath buildFileRootedPath = RootedPath.toRootedPath(packagePathEntry,
+ buildFileFragment);
+ String basename = buildFileRootedPath.asPath().getBaseName();
+ SkyKey fileSkyKey = FileValue.key(buildFileRootedPath);
+ FileValue fileValue = null;
+ try {
+ fileValue = (FileValue) env.getValueOrThrow(fileSkyKey, IOException.class,
+ FileSymlinkCycleException.class, InconsistentFilesystemException.class);
+ } catch (IOException e) {
+ String pkgName = pkgFragment.getPathString();
+ // TODO(bazel-team): throw an IOException here and let PackageFunction wrap that into a
+ // BuildFileNotFoundException.
+ throw new PackageLookupFunctionException(new BuildFileNotFoundException(pkgName,
+ "IO errors while looking for " + basename + " file reading "
+ + buildFileRootedPath.asPath() + ": " + e.getMessage(), e),
+ Transience.PERSISTENT);
+ } catch (FileSymlinkCycleException e) {
+ String pkgName = buildFileRootedPath.asPath().getPathString();
+ throw new PackageLookupFunctionException(new BuildFileNotFoundException(pkgName,
+ "Symlink cycle detected while trying to find " + basename + " file "
+ + buildFileRootedPath.asPath()),
+ Transience.PERSISTENT);
+ } catch (InconsistentFilesystemException e) {
+ // This error is not transient from the perspective of the PackageLookupFunction.
+ throw new PackageLookupFunctionException(e, Transience.PERSISTENT);
+ }
+ if (fileValue == null) {
+ return null;
+ }
+ if (fileValue.isFile()) {
+ return PackageLookupValue.success(buildFileRootedPath.getRoot());
+ }
+ return PackageLookupValue.noBuildFile();
+ }
+
+ /**
+ * Gets a PackageLookupValue from a different Bazel repository.
+ *
+ * To do this, it looks up the "external" package and finds a path mapping for the repository
+ * name.
+ */
+ private PackageLookupValue computeExternalPackageLookupValue(
+ SkyKey skyKey, Environment env) throws PackageLookupFunctionException {
+ PackageIdentifier id = (PackageIdentifier) skyKey.argument();
+ SkyKey repositoryKey = RepositoryValue.key(id.getRepository());
+ RepositoryValue repositoryValue = null;
+ try {
+ repositoryValue = (RepositoryValue) env.getValueOrThrow(
+ repositoryKey, NoSuchPackageException.class, IOException.class, EvalException.class);
+ if (repositoryValue == null) {
+ return null;
+ }
+ } catch (NoSuchPackageException e) {
+ throw new PackageLookupFunctionException(e, Transience.PERSISTENT);
+ } catch (IOException e) {
+ throw new PackageLookupFunctionException(new BuildFileContainsErrorsException(
+ PackageFunction.EXTERNAL_PACKAGE_NAME, e.getMessage()), Transience.PERSISTENT);
+ } catch (EvalException e) {
+ throw new PackageLookupFunctionException(new BuildFileContainsErrorsException(
+ PackageFunction.EXTERNAL_PACKAGE_NAME, e.getMessage()), Transience.PERSISTENT);
+ }
+
+ return getPackageLookupValue(env, repositoryValue.getPath(), id.getPackageFragment());
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link PackageLookupFunction#compute}.
+ */
+ private static final class PackageLookupFunctionException extends SkyFunctionException {
+ public PackageLookupFunctionException(NoSuchPackageException e, Transience transience) {
+ super(e, transience);
+ }
+
+ public PackageLookupFunctionException(InconsistentFilesystemException e,
+ Transience transience) {
+ super(e, transience);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PackageLookupValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/PackageLookupValue.java
new file mode 100644
index 0000000..c877d38
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PackageLookupValue.java
@@ -0,0 +1,249 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A value that represents a package lookup result.
+ *
+ * <p>Package lookups will always produce a value. On success, the {@code #getRoot} returns the
+ * package path root under which the package resides and the package's BUILD file is guaranteed to
+ * exist; on failure, {@code #getErrorReason} and {@code #getErrorMsg} describe why the package
+ * doesn't exist.
+ *
+ * <p>Implementation detail: we use inheritance here to optimize for memory usage.
+ */
+abstract class PackageLookupValue implements SkyValue {
+
+ enum ErrorReason {
+ // There is no BUILD file.
+ NO_BUILD_FILE,
+
+ // The package name is invalid.
+ INVALID_PACKAGE_NAME,
+
+ // The package is considered deleted because of --deleted_packages.
+ DELETED_PACKAGE,
+
+ // The //external package could not be loaded, either because the WORKSPACE file could not be
+ // parsed or the packages it references cannot be loaded.
+ NO_EXTERNAL_PACKAGE
+ }
+
+ protected PackageLookupValue() {
+ }
+
+ public static PackageLookupValue success(Path root) {
+ return new SuccessfulPackageLookupValue(root);
+ }
+
+ public static PackageLookupValue noBuildFile() {
+ return NoBuildFilePackageLookupValue.INSTANCE;
+ }
+
+ public static PackageLookupValue noExternalPackage() {
+ return NoExternalPackageLookupValue.INSTANCE;
+ }
+
+ public static PackageLookupValue invalidPackageName(String errorMsg) {
+ return new InvalidNamePackageLookupValue(errorMsg);
+ }
+
+ public static PackageLookupValue deletedPackage() {
+ return DeletedPackageLookupValue.INSTANCE;
+ }
+
+ /**
+ * For a successful package lookup, returns the root (package path entry) that the package
+ * resides in.
+ */
+ public abstract Path getRoot();
+
+ /**
+ * Returns whether the package lookup was successful.
+ */
+ public abstract boolean packageExists();
+
+ /**
+ * For an unsuccessful package lookup, gets the reason why {@link #packageExists} returns
+ * {@code false}.
+ */
+ abstract ErrorReason getErrorReason();
+
+ /**
+ * For an unsuccessful package lookup, gets a detailed error message for {@link #getErrorReason}
+ * that is suitable for reporting to a user.
+ */
+ abstract String getErrorMsg();
+
+ static SkyKey key(PathFragment directory) {
+ Preconditions.checkArgument(!directory.isAbsolute(), directory);
+ return key(PackageIdentifier.createInDefaultRepo(directory));
+ }
+
+ static SkyKey key(PackageIdentifier pkgIdentifier) {
+ return new SkyKey(SkyFunctions.PACKAGE_LOOKUP, pkgIdentifier);
+ }
+
+ private static class SuccessfulPackageLookupValue extends PackageLookupValue {
+
+ private final Path root;
+
+ private SuccessfulPackageLookupValue(Path root) {
+ this.root = root;
+ }
+
+ @Override
+ public boolean packageExists() {
+ return true;
+ }
+
+ @Override
+ public Path getRoot() {
+ return root;
+ }
+
+ @Override
+ ErrorReason getErrorReason() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ String getErrorMsg() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof SuccessfulPackageLookupValue)) {
+ return false;
+ }
+ SuccessfulPackageLookupValue other = (SuccessfulPackageLookupValue) obj;
+ return root.equals(other.root);
+ }
+
+ @Override
+ public int hashCode() {
+ return root.hashCode();
+ }
+ }
+
+ private abstract static class UnsuccessfulPackageLookupValue extends PackageLookupValue {
+
+ @Override
+ public boolean packageExists() {
+ return false;
+ }
+
+ @Override
+ public Path getRoot() {
+ throw new IllegalStateException();
+ }
+ }
+
+ private static class NoBuildFilePackageLookupValue extends UnsuccessfulPackageLookupValue {
+
+ public static final NoBuildFilePackageLookupValue INSTANCE =
+ new NoBuildFilePackageLookupValue();
+
+ private NoBuildFilePackageLookupValue() {
+ }
+
+ @Override
+ ErrorReason getErrorReason() {
+ return ErrorReason.NO_BUILD_FILE;
+ }
+
+ @Override
+ String getErrorMsg() {
+ return "BUILD file not found on package path";
+ }
+ }
+
+ private static class NoExternalPackageLookupValue extends UnsuccessfulPackageLookupValue {
+
+ public static final NoExternalPackageLookupValue INSTANCE =
+ new NoExternalPackageLookupValue();
+
+ private NoExternalPackageLookupValue() {
+ }
+
+ @Override
+ ErrorReason getErrorReason() {
+ return ErrorReason.NO_EXTERNAL_PACKAGE;
+ }
+
+ @Override
+ String getErrorMsg() {
+ return "Error loading the //external package";
+ }
+ }
+
+ private static class InvalidNamePackageLookupValue extends UnsuccessfulPackageLookupValue {
+
+ private final String errorMsg;
+
+ private InvalidNamePackageLookupValue(String errorMsg) {
+ this.errorMsg = errorMsg;
+ }
+
+ @Override
+ ErrorReason getErrorReason() {
+ return ErrorReason.INVALID_PACKAGE_NAME;
+ }
+
+ @Override
+ String getErrorMsg() {
+ return errorMsg;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof InvalidNamePackageLookupValue)) {
+ return false;
+ }
+ InvalidNamePackageLookupValue other = (InvalidNamePackageLookupValue) obj;
+ return errorMsg.equals(other.errorMsg);
+ }
+
+ @Override
+ public int hashCode() {
+ return errorMsg.hashCode();
+ }
+ }
+
+ private static class DeletedPackageLookupValue extends UnsuccessfulPackageLookupValue {
+
+ public static final DeletedPackageLookupValue INSTANCE = new DeletedPackageLookupValue();
+
+ private DeletedPackageLookupValue() {
+ }
+
+ @Override
+ ErrorReason getErrorReason() {
+ return ErrorReason.DELETED_PACKAGE;
+ }
+
+ @Override
+ String getErrorMsg() {
+ return "Package is considered deleted due to --deleted_packages";
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PackageValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/PackageValue.java
new file mode 100644
index 0000000..65fd2af
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PackageValue.java
@@ -0,0 +1,55 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A Skyframe value representing a package.
+ */
+@Immutable
+@ThreadSafe
+public class PackageValue implements SkyValue {
+
+ private final Package pkg;
+
+ PackageValue(Package pkg) {
+ this.pkg = Preconditions.checkNotNull(pkg);
+ }
+
+ public Package getPackage() {
+ return pkg;
+ }
+
+ @Override
+ public String toString() {
+ return "<PackageValue name=" + pkg.getName() + ">";
+ }
+
+ @ThreadSafe
+ public static SkyKey key(PathFragment pkgName) {
+ return key(PackageIdentifier.createInDefaultRepo(pkgName));
+ }
+
+ public static SkyKey key(PackageIdentifier pkgIdentifier) {
+ return new SkyKey(SkyFunctions.PACKAGE, pkgIdentifier);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PerBuildSyscallCache.java b/src/main/java/com/google/devtools/build/lib/skyframe/PerBuildSyscallCache.java
new file mode 100644
index 0000000..5116a6f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PerBuildSyscallCache.java
@@ -0,0 +1,131 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.Dirent;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.Symlinks;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+
+import java.io.IOException;
+import java.util.Collection;
+
+/**
+ * A per-build cache of filesystem operations for Skyframe invocations of legacy package loading.
+ */
+class PerBuildSyscallCache implements UnixGlob.FilesystemCalls {
+
+ private final LoadingCache<Pair<Path, Symlinks>, FileStatus> statCache =
+ newStatMap();
+ private final LoadingCache<Pair<Path, Symlinks>, Pair<Collection<Dirent>, IOException>>
+ readdirCache = newReaddirMap();
+
+ private static final FileStatus NO_STATUS = new FakeFileStatus();
+
+ @Override
+ public Collection<Dirent> readdir(Path path, Symlinks symlinks) throws IOException {
+ Pair<Collection<Dirent>, IOException> result =
+ readdirCache.getUnchecked(Pair.of(path, symlinks));
+ Collection<Dirent> entries = result.getFirst();
+ if (entries != null) {
+ return entries;
+ }
+ throw result.getSecond();
+ }
+
+ @Override
+ public FileStatus statNullable(Path path, Symlinks symlinks) {
+ FileStatus status = statCache.getUnchecked(Pair.of(path, symlinks));
+ return (status == NO_STATUS) ? null : status;
+ }
+
+ // This is used because the cache implementations don't allow null.
+ private static final class FakeFileStatus implements FileStatus {
+ @Override
+ public long getLastChangeTime() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getNodeId() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getLastModifiedTime() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getSize() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isDirectory() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isFile() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isSymbolicLink() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ /**
+ * A cache of stat calls.
+ * Input: (path, following_symlinks)
+ * Output: FileStatus
+ */
+ private static LoadingCache<Pair<Path, Symlinks>, FileStatus> newStatMap() {
+ return CacheBuilder.newBuilder().build(
+ new CacheLoader<Pair<Path, Symlinks>, FileStatus>() {
+ @Override
+ public FileStatus load(Pair<Path, Symlinks> p) {
+ FileStatus f = p.first.statNullable(p.second);
+ return (f == null) ? NO_STATUS : f;
+ }
+ });
+ }
+
+ /**
+ * A cache of readdir calls.
+ * Input: (path, following_symlinks)
+ * Output: A union of (Dirents, IOException).
+ */
+ private static
+ LoadingCache<Pair<Path, Symlinks>, Pair<Collection<Dirent>, IOException>> newReaddirMap() {
+ return CacheBuilder.newBuilder().build(
+ new CacheLoader<Pair<Path, Symlinks>, Pair<Collection<Dirent>, IOException>>() {
+ @Override
+ public Pair<Collection<Dirent>, IOException> load(Pair<Path, Symlinks> p) {
+ try {
+ return Pair.of(p.first.readdir(p.second), null);
+ } catch (IOException e) {
+ return Pair.of(null, e);
+ }
+ }
+ });
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PostConfiguredTargetFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/PostConfiguredTargetFunction.java
new file mode 100644
index 0000000..03920b7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PostConfiguredTargetFunction.java
@@ -0,0 +1,145 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.DependencyResolver.Dependency;
+import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
+import com.google.devtools.build.lib.analysis.TargetAndConfiguration;
+import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.RawAttributeMapper;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ConflictException;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * Build a post-processed ConfiguredTarget, vetting it for action conflict issues.
+ */
+public class PostConfiguredTargetFunction implements SkyFunction {
+ private static final Function<Dependency, SkyKey> TO_KEYS =
+ new Function<Dependency, SkyKey>() {
+ @Override
+ public SkyKey apply(Dependency input) {
+ return PostConfiguredTargetValue.key(
+ new ConfiguredTargetKey(input.getLabel(), input.getConfiguration()));
+ }
+ };
+
+ private final SkyframeExecutor.BuildViewProvider buildViewProvider;
+
+ public PostConfiguredTargetFunction(
+ SkyframeExecutor.BuildViewProvider buildViewProvider) {
+ this.buildViewProvider = Preconditions.checkNotNull(buildViewProvider);
+ }
+
+ @Nullable
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ ImmutableMap<Action, ConflictException> badActions = PrecomputedValue.BAD_ACTIONS.get(env);
+ ConfiguredTargetValue ctValue = (ConfiguredTargetValue)
+ env.getValue(ConfiguredTargetValue.key((ConfiguredTargetKey) skyKey.argument()));
+ SkyframeDependencyResolver resolver =
+ buildViewProvider.getSkyframeBuildView().createDependencyResolver(env);
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ for (Action action : ctValue.getActions()) {
+ if (badActions.containsKey(action)) {
+ throw new ActionConflictFunctionException(badActions.get(action));
+ }
+ }
+
+ ConfiguredTarget ct = ctValue.getConfiguredTarget();
+ TargetAndConfiguration ctgValue =
+ new TargetAndConfiguration(ct.getTarget(), ct.getConfiguration());
+
+ Set<ConfigMatchingProvider> configConditions =
+ getConfigurableAttributeConditions(ctgValue, env);
+ if (configConditions == null) {
+ return null;
+ }
+
+ Collection<Dependency> deps = resolver.dependentNodes(ctgValue, configConditions);
+ env.getValues(Iterables.transform(deps, TO_KEYS));
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ return new PostConfiguredTargetValue(ct);
+ }
+
+ /**
+ * Returns the configurable attribute conditions necessary to evaluate the given configured
+ * target, or null if not all dependencies have yet been SkyFrame-evaluated.
+ */
+ @Nullable
+ private Set<ConfigMatchingProvider> getConfigurableAttributeConditions(
+ TargetAndConfiguration ctg, Environment env) {
+ if (!(ctg.getTarget() instanceof Rule)) {
+ return ImmutableSet.of();
+ }
+ Rule rule = (Rule) ctg.getTarget();
+ RawAttributeMapper mapper = RawAttributeMapper.of(rule);
+ Set<SkyKey> depKeys = new LinkedHashSet<>();
+ for (Attribute attribute : rule.getAttributes()) {
+ for (Label label : mapper.getConfigurabilityKeys(attribute.getName(), attribute.getType())) {
+ if (!Type.Selector.isReservedLabel(label)) {
+ depKeys.add(ConfiguredTargetValue.key(label, ctg.getConfiguration()));
+ }
+ }
+ }
+ Map<SkyKey, SkyValue> cts = env.getValues(depKeys);
+ if (env.valuesMissing()) {
+ return null;
+ }
+ ImmutableSet.Builder<ConfigMatchingProvider> conditions = ImmutableSet.builder();
+ for (SkyValue ctValue : cts.values()) {
+ ConfiguredTarget ct = ((ConfiguredTargetValue) ctValue).getConfiguredTarget();
+ conditions.add(Preconditions.checkNotNull(ct.getProvider(ConfigMatchingProvider.class)));
+ }
+ return conditions.build();
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return Label.print(((LabelAndConfiguration) skyKey.argument()).getLabel());
+ }
+
+ private static class ActionConflictFunctionException extends SkyFunctionException {
+ public ActionConflictFunctionException(ConflictException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PostConfiguredTargetValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/PostConfiguredTargetValue.java
new file mode 100644
index 0000000..42d2b38
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PostConfiguredTargetValue.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A post-processed ConfiguredTarget which is known to be transitively error-free from action
+ * conflict issues.
+ */
+class PostConfiguredTargetValue implements SkyValue {
+
+ private final ConfiguredTarget ct;
+
+ public PostConfiguredTargetValue(ConfiguredTarget ct) {
+ this.ct = Preconditions.checkNotNull(ct);
+ }
+
+ public static ImmutableList<SkyKey> keys(Iterable<ConfiguredTargetKey> lacs) {
+ ImmutableList.Builder<SkyKey> keys = ImmutableList.builder();
+ for (ConfiguredTargetKey lac : lacs) {
+ keys.add(key(lac));
+ }
+ return keys.build();
+ }
+
+ public static SkyKey key(ConfiguredTargetKey lac) {
+ return new SkyKey(SkyFunctions.POST_CONFIGURED_TARGET, lac);
+ }
+
+ public ConfiguredTarget getCt() {
+ return ct;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedFunction.java
new file mode 100644
index 0000000..8254987
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedFunction.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * Builder for {@link PrecomputedValue}s.
+ *
+ * <p>Always throws an error, because the values aren't computed inside the skyframe framework.
+ */
+public class PrecomputedFunction implements SkyFunction {
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException,
+ InterruptedException {
+ throw new IllegalStateException(skyKey + " not set");
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedValue.java
new file mode 100644
index 0000000..bb2656d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedValue.java
@@ -0,0 +1,182 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey;
+import com.google.devtools.build.lib.packages.RuleVisibility;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ConflictException;
+import com.google.devtools.build.skyframe.Injectable;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Map;
+import java.util.UUID;
+
+import javax.annotation.Nullable;
+
+/**
+ * A value that represents something computed outside of the skyframe framework. These values are
+ * "precomputed" from skyframe's perspective and so the graph needs to be prepopulated with them
+ * (e.g. via injection).
+ */
+public class PrecomputedValue implements SkyValue {
+ /**
+ * An externally-injected precomputed value. Exists so that modules can inject precomputed values
+ * into Skyframe's graph.
+ *
+ * <p>{@see com.google.devtools.build.lib.blaze.BlazeModule#getPrecomputedValues}.
+ */
+ public static final class Injected {
+ private final Precomputed<?> precomputed;
+ private final Supplier<? extends Object> supplier;
+
+ private Injected(Precomputed<?> precomputed, Supplier<? extends Object> supplier) {
+ this.precomputed = precomputed;
+ this.supplier = supplier;
+ }
+
+ void inject(Injectable injectable) {
+ injectable.inject(ImmutableMap.of(precomputed.key, new PrecomputedValue(supplier.get())));
+ }
+ }
+
+ public static <T> Injected injected(Precomputed<T> precomputed, Supplier<T> value) {
+ return new Injected(precomputed, value);
+ }
+
+ public static <T> Injected injected(Precomputed<T> precomputed, T value) {
+ return new Injected(precomputed, Suppliers.ofInstance(value));
+ }
+
+ static final Precomputed<String> DEFAULTS_PACKAGE_CONTENTS =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "default_pkg"));
+
+ static final Precomputed<RuleVisibility> DEFAULT_VISIBILITY =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "default_visibility"));
+
+ static final Precomputed<UUID> BUILD_ID =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "build_id"));
+
+ static final Precomputed<WorkspaceStatusAction> WORKSPACE_STATUS_KEY =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "workspace_status_action"));
+
+ static final Precomputed<Action> COVERAGE_REPORT_KEY =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "coverage_report_action"));
+
+ static final Precomputed<TopLevelArtifactContext> TOP_LEVEL_CONTEXT =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "top_level_context"));
+
+ static final Precomputed<Map<BuildInfoKey, BuildInfoFactory>> BUILD_INFO_FACTORIES =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "build_info_factories"));
+
+ static final Precomputed<Map<String, String>> TEST_ENVIRONMENT_VARIABLES =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "test_environment"));
+
+ static final Precomputed<BlazeDirectories> BLAZE_DIRECTORIES =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "blaze_directories"));
+
+ static final Precomputed<ImmutableMap<Action, ConflictException>> BAD_ACTIONS =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "bad_actions"));
+
+ public static final Precomputed<PathPackageLocator> PATH_PACKAGE_LOCATOR =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "path_package_locator"));
+
+ private final Object value;
+
+ public PrecomputedValue(Object value) {
+ this.value = Preconditions.checkNotNull(value);
+ }
+
+ /**
+ * Returns the value of the variable.
+ */
+ public Object get() {
+ return value;
+ }
+
+ @Override
+ public int hashCode() {
+ return value.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof PrecomputedValue)) {
+ return false;
+ }
+ PrecomputedValue other = (PrecomputedValue) obj;
+ return value.equals(other.value);
+ }
+
+ @Override
+ public String toString() {
+ return "<BuildVariable " + value + ">";
+ }
+
+ public static final void dependOnBuildId(SkyFunction.Environment env) {
+ BUILD_ID.get(env);
+ }
+
+ /**
+ * A helper object corresponding to a variable in Skyframe.
+ *
+ * <p>Instances do not have internal state.
+ */
+ public static final class Precomputed<T> {
+ private final SkyKey key;
+
+ public Precomputed(SkyKey key) {
+ this.key = key;
+ }
+
+ @VisibleForTesting
+ SkyKey getKeyForTesting() {
+ return key;
+ }
+
+ /**
+ * Retrieves the value of this variable from Skyframe.
+ *
+ * <p>If the value was not set, an exception will be raised.
+ */
+ @Nullable
+ @SuppressWarnings("unchecked")
+ public T get(SkyFunction.Environment env) {
+ PrecomputedValue value = (PrecomputedValue) env.getValue(key);
+ if (value == null) {
+ return null;
+ }
+ return (T) value.get();
+ }
+
+ /**
+ * Injects a new variable value.
+ */
+ void set(Injectable injectable, T value) {
+ injectable.inject(ImmutableMap.of(key, new PrecomputedValue(value)));
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java
new file mode 100644
index 0000000..d54e98f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java
@@ -0,0 +1,454 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Verify;
+import com.google.common.collect.Collections2;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFile;
+import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.TraversalRequest;
+import com.google.devtools.build.lib.vfs.Dirent;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/** A {@link SkyFunction} to build {@link RecursiveFilesystemTraversalValue}s. */
+public final class RecursiveFilesystemTraversalFunction implements SkyFunction {
+
+ private static final class MissingDepException extends Exception {}
+
+ /** Base class for exceptions that {@link RecursiveFilesystemTraversalFunctionException} wraps. */
+ public abstract static class RecursiveFilesystemTraversalException extends Exception {
+ protected RecursiveFilesystemTraversalException(String message) {
+ super(message);
+ }
+ }
+
+ /** Thrown when a generated directory's root-relative path conflicts with a package's path. */
+ public static final class GeneratedPathConflictException extends
+ RecursiveFilesystemTraversalException {
+ GeneratedPathConflictException(TraversalRequest traversal) {
+ super(String.format(
+ "Generated directory %s conflicts with package under the same path. Additional info: %s",
+ traversal.path.getRelativePath().getPathString(),
+ traversal.errorInfo != null ? traversal.errorInfo : traversal.toString()));
+ }
+ }
+
+ /**
+ * Thrown when the traversal encounters a subdirectory with a BUILD file but is not allowed to
+ * recurse into it.
+ */
+ public static final class CannotCrossPackageBoundaryException extends
+ RecursiveFilesystemTraversalException {
+ CannotCrossPackageBoundaryException(String message) {
+ super(message);
+ }
+ }
+
+ /**
+ * Thrown when a dangling symlink is attempted to be dereferenced.
+ *
+ * <p>Note: this class is not identical to the one in com.google.devtools.build.lib.view.fileset
+ * and it's not easy to merge the two because of the dependency structure. The other one will
+ * probably be removed along with the rest of the legacy Fileset code.
+ */
+ public static final class DanglingSymlinkException extends RecursiveFilesystemTraversalException {
+ public final String path;
+ public final String unresolvedLink;
+
+ public DanglingSymlinkException(String path, String unresolvedLink) {
+ super("Found dangling symlink: " + path + ", unresolved path: ");
+ Preconditions.checkArgument(path != null && !path.isEmpty());
+ Preconditions.checkArgument(unresolvedLink != null && !unresolvedLink.isEmpty());
+ this.path = path;
+ this.unresolvedLink = unresolvedLink;
+ }
+
+ public String getPath() {
+ return path;
+ }
+ }
+
+ /** Exception type thrown by {@link RecursiveFilesystemTraversalFunction#compute}. */
+ private static final class RecursiveFilesystemTraversalFunctionException extends
+ SkyFunctionException {
+ RecursiveFilesystemTraversalFunctionException(RecursiveFilesystemTraversalException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env)
+ throws RecursiveFilesystemTraversalFunctionException {
+ TraversalRequest traversal = (TraversalRequest) skyKey.argument();
+ try {
+ // Stat the traversal root.
+ FileInfo rootInfo = lookUpFileInfo(env, traversal);
+
+ if (!rootInfo.type.exists()) {
+ // May be a dangling symlink or a non-existent file. Handle gracefully.
+ if (rootInfo.type.isSymlink()) {
+ return resultForDanglingSymlink(traversal.path, rootInfo);
+ } else {
+ return RecursiveFilesystemTraversalValue.EMPTY;
+ }
+ }
+
+ if (rootInfo.type.isFile()) {
+ // The root is a file or a symlink to one.
+ return resultForFileRoot(traversal.path, rootInfo);
+ }
+
+ // Otherwise the root is a directory or a symlink to one.
+ PkgLookupResult pkgLookupResult = checkIfPackage(env, traversal, rootInfo);
+ traversal = pkgLookupResult.traversal;
+
+ if (pkgLookupResult.isConflicting()) {
+ // The traversal was requested for an output directory whose root-relative path conflicts
+ // with a source package. We can't handle that, bail out.
+ throw new RecursiveFilesystemTraversalFunctionException(
+ new GeneratedPathConflictException(traversal));
+ } else if (pkgLookupResult.isPackage() && !traversal.skipTestingForSubpackage) {
+ // The traversal was requested for a directory that defines a package.
+ if (traversal.crossPkgBoundaries) {
+ // We are free to traverse the subpackage but we need to display a warning.
+ String msg = traversal.errorInfo + " crosses package boundary into package rooted at "
+ + traversal.path.getRelativePath().getPathString();
+ env.getListener().handle(new Event(EventKind.WARNING, null, msg));
+ } else {
+ // We cannot traverse the subpackage and should skip it silently. Return empty results.
+ return RecursiveFilesystemTraversalValue.EMPTY;
+ }
+ }
+
+ // We are free to traverse this directory.
+ Collection<SkyKey> dependentKeys = createRecursiveTraversalKeys(env, traversal);
+ return resultForDirectory(traversal, rootInfo, traverseChildren(env, dependentKeys));
+ } catch (MissingDepException e) {
+ return null;
+ }
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ private static final class FileInfo {
+ final FileType type;
+ final FileStateValue metadata;
+ @Nullable final RootedPath realPath;
+ @Nullable final PathFragment unresolvedSymlinkTarget;
+
+ FileInfo(FileType type, FileStateValue metadata, @Nullable RootedPath realPath,
+ @Nullable PathFragment unresolvedSymlinkTarget) {
+ this.type = Preconditions.checkNotNull(type);
+ this.metadata = Preconditions.checkNotNull(metadata);
+ this.realPath = realPath;
+ this.unresolvedSymlinkTarget = unresolvedSymlinkTarget;
+ }
+
+ @Override
+ public String toString() {
+ if (type.isSymlink()) {
+ return String.format("(%s: link_value=%s, real_path=%s)", type,
+ unresolvedSymlinkTarget.getPathString(), realPath);
+ } else {
+ return String.format("(%s: real_path=%s)", type, realPath);
+ }
+ }
+ }
+
+ private static FileInfo lookUpFileInfo(Environment env, TraversalRequest traversal)
+ throws MissingDepException {
+ // Stat the file.
+ FileValue fileValue = (FileValue) getDependentSkyValue(env, FileValue.key(traversal.path));
+ if (fileValue.exists()) {
+ // If it exists, it may either be a symlink or a file/directory.
+ PathFragment unresolvedLinkTarget = null;
+ FileType type = null;
+ if (fileValue.isSymlink()) {
+ unresolvedLinkTarget = fileValue.getUnresolvedLinkTarget();
+ type = fileValue.isDirectory() ? FileType.SYMLINK_TO_DIRECTORY : FileType.SYMLINK_TO_FILE;
+ } else {
+ type = fileValue.isDirectory() ? FileType.DIRECTORY : FileType.FILE;
+ }
+ return new FileInfo(type, fileValue.realFileStateValue(),
+ fileValue.realRootedPath(), unresolvedLinkTarget);
+ } else {
+ // If it doesn't exist, or it's a dangling symlink, we still want to handle that gracefully.
+ return new FileInfo(
+ fileValue.isSymlink() ? FileType.DANGLING_SYMLINK : FileType.NONEXISTENT,
+ fileValue.realFileStateValue(), null,
+ fileValue.isSymlink() ? fileValue.getUnresolvedLinkTarget() : null);
+ }
+ }
+
+ private static final class PkgLookupResult {
+ private enum Type {
+ CONFLICT, DIRECTORY, PKG
+ }
+
+ private final Type type;
+ final TraversalRequest traversal;
+ final FileInfo rootInfo;
+
+ /** Result for a generated directory that conflicts with a source package. */
+ static PkgLookupResult conflict(TraversalRequest traversal, FileInfo rootInfo) {
+ return new PkgLookupResult(Type.CONFLICT, traversal, rootInfo);
+ }
+
+ /** Result for a source or generated directory (not a package). */
+ static PkgLookupResult directory(TraversalRequest traversal, FileInfo rootInfo) {
+ return new PkgLookupResult(Type.DIRECTORY, traversal, rootInfo);
+ }
+
+ /** Result for a package, i.e. a directory with a BUILD file. */
+ static PkgLookupResult pkg(TraversalRequest traversal, FileInfo rootInfo) {
+ return new PkgLookupResult(Type.PKG, traversal, rootInfo);
+ }
+
+ private PkgLookupResult(Type type, TraversalRequest traversal, FileInfo rootInfo) {
+ this.type = Preconditions.checkNotNull(type);
+ this.traversal = Preconditions.checkNotNull(traversal);
+ this.rootInfo = Preconditions.checkNotNull(rootInfo);
+ }
+
+ boolean isPackage() {
+ return type == Type.PKG;
+ }
+
+ boolean isConflicting() {
+ return type == Type.CONFLICT;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("(%s: info=%s, traversal=%s)", type, rootInfo, traversal);
+ }
+ }
+
+ /**
+ * Checks whether the {@code traversal}'s path refers to a package directory.
+ *
+ * @return the result of the lookup; it contains potentially new {@link TraversalRequest} and
+ * {@link FileInfo} so the caller should use these instead of the old ones (this happens when
+ * a package is found, but under a different root than expected)
+ */
+ private static PkgLookupResult checkIfPackage(Environment env, TraversalRequest traversal,
+ FileInfo rootInfo) throws MissingDepException {
+ Preconditions.checkArgument(rootInfo.type.exists() && !rootInfo.type.isFile(),
+ "{%s} {%s}", traversal, rootInfo);
+ PackageLookupValue pkgLookup = (PackageLookupValue) getDependentSkyValue(env,
+ PackageLookupValue.key(traversal.path.getRelativePath()));
+
+ if (pkgLookup.packageExists()) {
+ if (traversal.isGenerated) {
+ // The traversal's root was a generated directory, but its root-relative path conflicts with
+ // an existing package.
+ return PkgLookupResult.conflict(traversal, rootInfo);
+ } else {
+ // The traversal's root was a source directory and it defines a package.
+ Path pkgRoot = pkgLookup.getRoot();
+ if (!pkgRoot.equals(traversal.path.getRoot())) {
+ // However the root of this package is different from what we expected. stat() the real
+ // BUILD file of that package.
+ traversal = traversal.forChangedRootPath(pkgRoot);
+ rootInfo = lookUpFileInfo(env, traversal);
+ Verify.verify(rootInfo.type.exists(), "{%s} {%s}", traversal, rootInfo);
+ }
+ return PkgLookupResult.pkg(traversal, rootInfo);
+ }
+ } else {
+ // The traversal's root was a directory (source or generated one), no package exists under the
+ // same root-relative path.
+ return PkgLookupResult.directory(traversal, rootInfo);
+ }
+ }
+
+ /**
+ * List the directory and create {@code SkyKey}s to request contents of its children recursively.
+ *
+ * <p>The returned keys are of type {@link SkyFunctions#RECURSIVE_FILESYSTEM_TRAVERSAL}.
+ */
+ private static Collection<SkyKey> createRecursiveTraversalKeys(Environment env,
+ TraversalRequest traversal) throws MissingDepException {
+ // Use the traversal's path, even if it's a symlink. The contents of the directory, as listed
+ // in the result, must be relative to it.
+ DirectoryListingValue dirListing = (DirectoryListingValue) getDependentSkyValue(env,
+ DirectoryListingValue.key(traversal.path));
+
+ List<SkyKey> result = new ArrayList<>();
+ for (Dirent dirent : dirListing.getDirents()) {
+ RootedPath childPath = RootedPath.toRootedPath(traversal.path.getRoot(),
+ traversal.path.getRelativePath().getRelative(dirent.getName()));
+ TraversalRequest childTraversal = traversal.forChildEntry(childPath);
+ result.add(RecursiveFilesystemTraversalValue.key(childTraversal));
+ }
+ return result;
+ }
+
+ /**
+ * Creates result for a dangling symlink.
+ *
+ * @param linkName path to the symbolic link
+ * @param info the {@link FileInfo} associated with the link file
+ */
+ private static RecursiveFilesystemTraversalValue resultForDanglingSymlink(RootedPath linkName,
+ FileInfo info) {
+ Preconditions.checkState(info.type.isSymlink() && !info.type.exists(), "{%s} {%s}", linkName,
+ info.type);
+ return RecursiveFilesystemTraversalValue.of(
+ ResolvedFile.danglingSymlink(linkName, info.unresolvedSymlinkTarget, info.metadata));
+ }
+
+ /**
+ * Creates results for a file or for a symlink that points to one.
+ *
+ * <p>A symlink may be direct (points to a file) or transitive (points at a direct or transitive
+ * symlink).
+ */
+ private static RecursiveFilesystemTraversalValue resultForFileRoot(RootedPath path,
+ FileInfo info) {
+ Preconditions.checkState(info.type.isFile() && info.type.exists(), "{%s} {%s}", path,
+ info.type);
+ if (info.type.isSymlink()) {
+ return RecursiveFilesystemTraversalValue.of(ResolvedFile.symlinkToFile(info.realPath, path,
+ info.unresolvedSymlinkTarget, info.metadata));
+ } else {
+ return RecursiveFilesystemTraversalValue.of(ResolvedFile.regularFile(path, info.metadata));
+ }
+ }
+
+ private static RecursiveFilesystemTraversalValue resultForDirectory(TraversalRequest traversal,
+ FileInfo rootInfo, Collection<RecursiveFilesystemTraversalValue> subdirTraversals) {
+ // Collect transitive closure of files in subdirectories.
+ NestedSetBuilder<ResolvedFile> paths = NestedSetBuilder.stableOrder();
+ for (RecursiveFilesystemTraversalValue child : subdirTraversals) {
+ paths.addTransitive(child.getTransitiveFiles());
+ }
+ ResolvedFile root;
+ if (rootInfo.type.isSymlink()) {
+ root = ResolvedFile.symlinkToDirectory(rootInfo.realPath, traversal.path,
+ rootInfo.unresolvedSymlinkTarget, rootInfo.metadata);
+ paths.add(root);
+ } else {
+ root = ResolvedFile.directory(rootInfo.realPath);
+ }
+ return RecursiveFilesystemTraversalValue.of(root, paths.build());
+ }
+
+ private static SkyValue getDependentSkyValue(Environment env, SkyKey key)
+ throws MissingDepException {
+ SkyValue value = env.getValue(key);
+ if (env.valuesMissing()) {
+ throw new MissingDepException();
+ }
+ return value;
+ }
+
+ /**
+ * Requests Skyframe to compute the dependent values and returns them.
+ *
+ * <p>The keys must all be {@link SkyFunctions#RECURSIVE_FILESYSTEM_TRAVERSAL} keys.
+ */
+ private static Collection<RecursiveFilesystemTraversalValue> traverseChildren(
+ Environment env, Iterable<SkyKey> keys)
+ throws MissingDepException {
+ Map<SkyKey, SkyValue> values = env.getValues(keys);
+ if (env.valuesMissing()) {
+ throw new MissingDepException();
+ }
+ return Collections2.transform(values.values(),
+ new Function<SkyValue, RecursiveFilesystemTraversalValue>() {
+ @Override
+ public RecursiveFilesystemTraversalValue apply(SkyValue input) {
+ return (RecursiveFilesystemTraversalValue) input;
+ }
+ });
+ }
+
+ /** Type information about the filesystem entry residing at a path. */
+ enum FileType {
+ /** A regular file. */
+ FILE {
+ @Override boolean isFile() { return true; }
+ @Override boolean exists() { return true; }
+ @Override public String toString() { return "<f>"; }
+ },
+ /**
+ * A symlink to a regular file.
+ *
+ * <p>The symlink may be direct (points to a non-symlink (here a file)) or it may be transitive
+ * (points to a direct or transitive symlink).
+ */
+ SYMLINK_TO_FILE {
+ @Override boolean isFile() { return true; }
+ @Override boolean isSymlink() { return true; }
+ @Override boolean exists() { return true; }
+ @Override public String toString() { return "<lf>"; }
+ },
+ /** A directory. */
+ DIRECTORY {
+ @Override boolean isDirectory() { return true; }
+ @Override boolean exists() { return true; }
+ @Override public String toString() { return "<d>"; }
+ },
+ /**
+ * A symlink to a directory.
+ *
+ * <p>The symlink may be direct (points to a non-symlink (here a directory)) or it may be
+ * transitive (points to a direct or transitive symlink).
+ */
+ SYMLINK_TO_DIRECTORY {
+ @Override boolean isDirectory() { return true; }
+ @Override boolean isSymlink() { return true; }
+ @Override boolean exists() { return true; }
+ @Override public String toString() { return "<ld>"; }
+ },
+ /** A dangling symlink, i.e. one whose target is known not to exist. */
+ DANGLING_SYMLINK {
+ @Override boolean isFile() { throw new UnsupportedOperationException(); }
+ @Override boolean isDirectory() { throw new UnsupportedOperationException(); }
+ @Override boolean isSymlink() { return true; }
+ @Override public String toString() { return "<l?>"; }
+ },
+ /** A path that does not exist or should be ignored. */
+ NONEXISTENT {
+ @Override public String toString() { return "<?>"; }
+ };
+
+ boolean isFile() { return false; }
+ boolean isDirectory() { return false; }
+ boolean isSymlink() { return false; }
+ boolean exists() { return false; }
+ @Override public abstract String toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalValue.java
new file mode 100644
index 0000000..023b1cf
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalValue.java
@@ -0,0 +1,597 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalFunction.DanglingSymlinkException;
+import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalFunction.FileType;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import javax.annotation.Nullable;
+
+/**
+ * Collection of files found while recursively traversing a path.
+ *
+ * <p>The path may refer to files, symlinks or directories that may or may not exist.
+ *
+ * <p>Traversing a file or a symlink results in a single {@link ResolvedFile} corresponding to the
+ * file or symlink.
+ *
+ * <p>Traversing a directory results in a collection of {@link ResolvedFile}s for all files and
+ * symlinks under it, and in all of its subdirectories. The {@link TraversalRequest} can specify
+ * whether to traverse source subdirectories that are packages (have BUILD files in them).
+ *
+ * <p>Traversing a symlink that points to a directory is the same as traversing a normal directory.
+ * The paths in the result will not be resolved; the files will be listed under the symlink, as if
+ * it was the actual directory they reside in.
+ *
+ * <p>Editing a file that is part of this traversal, or adding or removing a file in a directory
+ * that is part of this traversal, will invalidate this {@link SkyValue}. This also applies to
+ * directories that are symlinked to.
+ */
+public final class RecursiveFilesystemTraversalValue implements SkyValue {
+ static final RecursiveFilesystemTraversalValue EMPTY = new RecursiveFilesystemTraversalValue(
+ Optional.<ResolvedFile>absent(),
+ NestedSetBuilder.<ResolvedFile>emptySet(Order.STABLE_ORDER));
+
+ /** The root of the traversal. May only be absent for the {@link #EMPTY} instance. */
+ private final Optional<ResolvedFile> resolvedRoot;
+
+ /** The transitive closure of {@link ResolvedFile}s. */
+ private final NestedSet<ResolvedFile> resolvedPaths;
+
+ private RecursiveFilesystemTraversalValue(Optional<ResolvedFile> resolvedRoot,
+ NestedSet<ResolvedFile> resolvedPaths) {
+ this.resolvedRoot = Preconditions.checkNotNull(resolvedRoot);
+ this.resolvedPaths = Preconditions.checkNotNull(resolvedPaths);
+ }
+
+ static RecursiveFilesystemTraversalValue of(ResolvedFile resolvedRoot,
+ NestedSet<ResolvedFile> resolvedPaths) {
+ if (resolvedPaths.isEmpty()) {
+ return EMPTY;
+ } else {
+ return new RecursiveFilesystemTraversalValue(Optional.of(resolvedRoot), resolvedPaths);
+ }
+ }
+
+ static RecursiveFilesystemTraversalValue of(ResolvedFile singleMember) {
+ return new RecursiveFilesystemTraversalValue(Optional.of(singleMember),
+ NestedSetBuilder.<ResolvedFile>create(Order.STABLE_ORDER, singleMember));
+ }
+
+ /** Returns the root of the traversal; absent only for the {@link #EMPTY} instance. */
+ public Optional<ResolvedFile> getResolvedRoot() {
+ return resolvedRoot;
+ }
+
+ /**
+ * Retrieves the set of {@link ResolvedFile}s that were found by this traversal.
+ *
+ * <p>The returned set may be empty if no files were found, or the ones found were to be
+ * considered non-existent. Unless it's empty, the returned set always includes the
+ * {@link #getResolvedRoot() resolved root}.
+ *
+ * <p>The returned set also includes symlinks. If a symlink points to a directory, its contents
+ * are also included in this set, and their path will start with the symlink's path, just like on
+ * a usual Unix file system.
+ */
+ public NestedSet<ResolvedFile> getTransitiveFiles() {
+ return resolvedPaths;
+ }
+
+ public static SkyKey key(TraversalRequest traversal) {
+ return new SkyKey(SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL, traversal);
+ }
+
+ /** The parameters of a file or directory traversal. */
+ public static final class TraversalRequest {
+
+ /** The path to start the traversal from; may be a file, a directory or a symlink. */
+ final RootedPath path;
+
+ /**
+ * Whether the path is in the output tree.
+ *
+ * <p>Such paths and all their subdirectories are assumed not to define packages, so package
+ * lookup for them is skipped.
+ */
+ final boolean isGenerated;
+
+ /** Whether traversal should descend into directories that are roots of subpackages. */
+ final boolean crossPkgBoundaries;
+
+ /**
+ * Whether to skip checking if the root (if it's a directory) contains a BUILD file.
+ *
+ * <p>Such directories are not considered to be packages when this flag is true. This needs to
+ * be true in order to traverse directories of packages, but should be false for <i>their</i>
+ * subdirectories.
+ */
+ final boolean skipTestingForSubpackage;
+
+ /** Information to be attached to any error messages that may be reported. */
+ @Nullable final String errorInfo;
+
+ public TraversalRequest(RootedPath path, boolean isRootGenerated,
+ boolean crossPkgBoundaries, boolean skipTestingForSubpackage,
+ @Nullable String errorInfo) {
+ this.path = path;
+ this.isGenerated = isRootGenerated;
+ this.crossPkgBoundaries = crossPkgBoundaries;
+ this.skipTestingForSubpackage = skipTestingForSubpackage;
+ this.errorInfo = errorInfo;
+ }
+
+ private TraversalRequest duplicate(RootedPath newRoot, boolean newSkipTestingForSubpackage) {
+ return new TraversalRequest(newRoot, isGenerated, crossPkgBoundaries,
+ newSkipTestingForSubpackage, errorInfo);
+ }
+
+ /** Creates a new request to traverse a child element in the current directory (the root). */
+ TraversalRequest forChildEntry(RootedPath newPath) {
+ return duplicate(newPath, false);
+ }
+
+ /**
+ * Creates a new request for a changed root.
+ *
+ * <p>This method can be used when a package is found out to be under a different root path than
+ * originally assumed.
+ */
+ TraversalRequest forChangedRootPath(Path newRoot) {
+ return duplicate(RootedPath.toRootedPath(newRoot, path.getRelativePath()),
+ skipTestingForSubpackage);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof TraversalRequest)) {
+ return false;
+ }
+ TraversalRequest o = (TraversalRequest) obj;
+ return path.equals(o.path) && isGenerated == o.isGenerated
+ && crossPkgBoundaries == o.crossPkgBoundaries
+ && skipTestingForSubpackage == o.skipTestingForSubpackage;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(path, isGenerated, crossPkgBoundaries, skipTestingForSubpackage);
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "TraversalParams(root=%s, is_generated=%d, skip_testing_for_subpkg=%d,"
+ + " pkg_boundaries=%d)", path, isGenerated ? 1 : 0, skipTestingForSubpackage ? 1 : 0,
+ crossPkgBoundaries ? 1 : 0);
+ }
+ }
+
+ /**
+ * Path and type information about a single file or symlink.
+ *
+ * <p>The object stores things such as the absolute path of the file or symlink, its exact type
+ * and, if it's a symlink, the resolved and unresolved link target paths.
+ */
+ public abstract static class ResolvedFile {
+ private static final class Symlink {
+ private final RootedPath linkName;
+ private final PathFragment unresolvedLinkTarget;
+ // The resolved link target is stored in ResolvedFile.path
+
+ private Symlink(RootedPath linkName, PathFragment unresolvedLinkTarget) {
+ this.linkName = Preconditions.checkNotNull(linkName);
+ this.unresolvedLinkTarget = Preconditions.checkNotNull(unresolvedLinkTarget);
+ }
+
+ PathFragment getNameInSymlinkTree() {
+ return linkName.getRelativePath();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof Symlink)) {
+ return false;
+ }
+ Symlink o = (Symlink) obj;
+ return linkName.equals(o.linkName) && unresolvedLinkTarget.equals(o.unresolvedLinkTarget);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(linkName, unresolvedLinkTarget);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Symlink(link_name=%s, unresolved_target=%s)",
+ linkName, unresolvedLinkTarget);
+ }
+ }
+
+ private static final class RegularFile extends ResolvedFile {
+ private RegularFile(RootedPath path) {
+ super(FileType.FILE, Optional.of(path), Optional.<FileStateValue>absent());
+ }
+
+ RegularFile(RootedPath path, FileStateValue metadata) {
+ super(FileType.FILE, Optional.of(path), Optional.of(metadata));
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof RegularFile)) {
+ return false;
+ }
+ return super.isEqualTo((RegularFile) obj);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("RegularFile(%s)", super.toString());
+ }
+
+ @Override
+ ResolvedFile stripMetadataForTesting() {
+ return new RegularFile(path.get());
+ }
+
+ @Override
+ public PathFragment getNameInSymlinkTree() {
+ return path.get().getRelativePath();
+ }
+
+ @Override
+ public PathFragment getTargetInSymlinkTree(boolean followSymlinks) {
+ return path.get().asPath().asFragment();
+ }
+ }
+
+ private static final class Directory extends ResolvedFile {
+ Directory(RootedPath path) {
+ super(FileType.DIRECTORY, Optional.of(path), Optional.of(
+ FileStateValue.DIRECTORY_FILE_STATE_NODE));
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof Directory)) {
+ return false;
+ }
+ return super.isEqualTo((Directory) obj);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Directory(%s)", super.toString());
+ }
+
+ @Override
+ ResolvedFile stripMetadataForTesting() {
+ return this;
+ }
+
+ @Override
+ public PathFragment getNameInSymlinkTree() {
+ return path.get().getRelativePath();
+ }
+
+ @Override
+ public PathFragment getTargetInSymlinkTree(boolean followSymlinks) {
+ return path.get().asPath().asFragment();
+ }
+ }
+
+ private static final class DanglingSymlink extends ResolvedFile {
+ private final Symlink symlink;
+
+ private DanglingSymlink(Symlink symlink) {
+ super(FileType.DANGLING_SYMLINK, Optional.<RootedPath>absent(),
+ Optional.<FileStateValue>absent());
+ this.symlink = symlink;
+ }
+
+ DanglingSymlink(RootedPath linkNamePath, PathFragment linkTargetPath,
+ FileStateValue metadata) {
+ super(FileType.DANGLING_SYMLINK, Optional.<RootedPath>absent(), Optional.of(metadata));
+ this.symlink = new Symlink(linkNamePath, linkTargetPath);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof DanglingSymlink)) {
+ return false;
+ }
+ DanglingSymlink o = (DanglingSymlink) obj;
+ return super.isEqualTo(o) && symlink.equals(o.symlink);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(super.hashCode(), symlink);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("DanglingSymlink(%s, %s)", super.toString(), symlink);
+ }
+
+ @Override
+ ResolvedFile stripMetadataForTesting() {
+ return new DanglingSymlink(symlink);
+ }
+
+ @Override
+ public PathFragment getNameInSymlinkTree() {
+ return symlink.getNameInSymlinkTree();
+ }
+
+ @Override
+ public PathFragment getTargetInSymlinkTree(boolean followSymlinks)
+ throws DanglingSymlinkException {
+ if (followSymlinks) {
+ throw new DanglingSymlinkException(symlink.linkName.asPath().getPathString(),
+ symlink.unresolvedLinkTarget.getPathString());
+ } else {
+ return symlink.unresolvedLinkTarget;
+ }
+ }
+ }
+
+ private static final class SymlinkToFile extends ResolvedFile {
+ private final Symlink symlink;
+
+ private SymlinkToFile(RootedPath targetPath, Symlink symlink) {
+ super(FileType.SYMLINK_TO_FILE, Optional.of(targetPath), Optional.<FileStateValue>absent());
+ this.symlink = symlink;
+ }
+
+ SymlinkToFile(RootedPath targetPath, RootedPath linkNamePath,
+ PathFragment linkTargetPath, FileStateValue metadata) {
+ super(FileType.SYMLINK_TO_FILE, Optional.of(targetPath), Optional.of(metadata));
+ this.symlink = new Symlink(linkNamePath, linkTargetPath);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof SymlinkToFile)) {
+ return false;
+ }
+ SymlinkToFile o = (SymlinkToFile) obj;
+ return super.isEqualTo(o) && symlink.equals(o.symlink);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(super.hashCode(), symlink);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("SymlinkToFile(%s, %s)", super.toString(), symlink);
+ }
+
+ @Override
+ ResolvedFile stripMetadataForTesting() {
+ return new SymlinkToFile(path.get(), symlink);
+ }
+
+ @Override
+ public PathFragment getNameInSymlinkTree() {
+ return symlink.getNameInSymlinkTree();
+ }
+
+ @Override
+ public PathFragment getTargetInSymlinkTree(boolean followSymlinks) {
+ return followSymlinks ? path.get().asPath().asFragment() : symlink.unresolvedLinkTarget;
+ }
+ }
+
+ private static final class SymlinkToDirectory extends ResolvedFile {
+ private final Symlink symlink;
+
+ private SymlinkToDirectory(RootedPath targetPath, Symlink symlink) {
+ super(FileType.SYMLINK_TO_DIRECTORY, Optional.of(targetPath),
+ Optional.<FileStateValue>absent());
+ this.symlink = symlink;
+ }
+
+ SymlinkToDirectory(RootedPath targetPath, RootedPath linkNamePath,
+ PathFragment linkValue, FileStateValue metadata) {
+ super(FileType.SYMLINK_TO_DIRECTORY, Optional.of(targetPath), Optional.of(metadata));
+ this.symlink = new Symlink(linkNamePath, linkValue);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof SymlinkToDirectory)) {
+ return false;
+ }
+ SymlinkToDirectory o = (SymlinkToDirectory) obj;
+ return super.isEqualTo(o) && symlink.equals(o.symlink);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(super.hashCode(), symlink);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("SymlinkToDirectory(%s, %s)", super.toString(), symlink);
+ }
+
+ @Override
+ ResolvedFile stripMetadataForTesting() {
+ return new SymlinkToDirectory(path.get(), symlink);
+ }
+
+ @Override
+ public PathFragment getNameInSymlinkTree() {
+ return symlink.getNameInSymlinkTree();
+ }
+
+ @Override
+ public PathFragment getTargetInSymlinkTree(boolean followSymlinks) {
+ return followSymlinks ? path.get().asPath().asFragment() : symlink.unresolvedLinkTarget;
+ }
+ }
+
+ /** Type of the entity under {@link #path}. */
+ final FileType type;
+
+ /**
+ * Path of the file, directory or resolved target of the symlink.
+ *
+ * <p>May only be absent for dangling symlinks.
+ */
+ protected final Optional<RootedPath> path;
+
+ /**
+ * Associated metadata.
+ *
+ * <p>This field must be stored so that this {@link ResolvedFile} is (also) the function of the
+ * stat() of the file, but otherwise it is likely not something the consumer of the
+ * {@link ResolvedFile} is directly interested in.
+ *
+ * <p>May only be absent if stripped for tests.
+ */
+ final Optional<FileStateValue> metadata;
+
+ private ResolvedFile(FileType type, Optional<RootedPath> path,
+ Optional<FileStateValue> metadata) {
+ this.type = Preconditions.checkNotNull(type);
+ this.path = Preconditions.checkNotNull(path);
+ this.metadata = Preconditions.checkNotNull(metadata);
+ }
+
+ static ResolvedFile regularFile(RootedPath path, FileStateValue metadata) {
+ return new RegularFile(path, metadata);
+ }
+
+ static ResolvedFile directory(RootedPath path) {
+ return new Directory(path);
+ }
+
+ static ResolvedFile symlinkToFile(RootedPath targetPath, RootedPath linkNamePath,
+ PathFragment linkTargetPath, FileStateValue metadata) {
+ return new SymlinkToFile(targetPath, linkNamePath, linkTargetPath, metadata);
+ }
+
+ static ResolvedFile symlinkToDirectory(RootedPath targetPath,
+ RootedPath linkNamePath, PathFragment linkValue, FileStateValue metadata) {
+ return new SymlinkToDirectory(targetPath, linkNamePath, linkValue, metadata);
+ }
+
+ static ResolvedFile danglingSymlink(RootedPath linkNamePath, PathFragment linkValue,
+ FileStateValue metadata) {
+ return new DanglingSymlink(linkNamePath, linkValue, metadata);
+ }
+
+ private boolean isEqualTo(ResolvedFile o) {
+ return type.equals(o.type) && path.equals(o.path) && metadata.equals(o.metadata);
+ }
+
+ @Override
+ public abstract boolean equals(Object obj);
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(type, path, metadata);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("type=%s, path=%s, metadata=%s", type, path,
+ metadata.isPresent() ? Integer.toHexString(metadata.get().hashCode()) : "(stripped)");
+ }
+
+ /**
+ * Returns the path of the Fileset-output symlink relative to the output directory.
+ *
+ * <p>The path should contain the FilesetEntry-specific destination directory (if any) and
+ * should have necessary prefixes stripped (if any).
+ */
+ public abstract PathFragment getNameInSymlinkTree();
+
+ /**
+ * Returns the path of the symlink target.
+ *
+ * @throws DanglingSymlinkException if the target cannot be resolved because the symlink is
+ * dangling
+ */
+ public abstract PathFragment getTargetInSymlinkTree(boolean followSymlinks)
+ throws DanglingSymlinkException;
+
+ /**
+ * Returns a copy of this object with the metadata stripped away.
+ *
+ * <p>This method should only be used by tests that wish to assert that this
+ * {@link ResolvedFile} refers to the expected absolute path and has the expected type, without
+ * asserting its actual contents (which the metadata is a function of).
+ */
+ @VisibleForTesting
+ abstract ResolvedFile stripMetadataForTesting();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof RecursiveFilesystemTraversalValue)) {
+ return false;
+ }
+ RecursiveFilesystemTraversalValue o = (RecursiveFilesystemTraversalValue) obj;
+ return resolvedRoot.equals(o.resolvedRoot) && resolvedPaths.equals(o.resolvedPaths);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(resolvedRoot, resolvedPaths);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePkgFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePkgFunction.java
new file mode 100644
index 0000000..11ed3be
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePkgFunction.java
@@ -0,0 +1,151 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.vfs.Dirent;
+import com.google.devtools.build.lib.vfs.Dirent.Type;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * RecursivePkgFunction builds up the set of packages underneath a given directory
+ * transitively.
+ *
+ * <p>Example: foo/BUILD, foo/sub/x, foo/subpkg/BUILD would yield transitive packages "foo" and
+ * "foo/subpkg".
+ */
+public class RecursivePkgFunction implements SkyFunction {
+
+ private static final Order ORDER = Order.STABLE_ORDER;
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ RootedPath rootedPath = (RootedPath) skyKey.argument();
+ Path root = rootedPath.getRoot();
+ PathFragment rootRelativePath = rootedPath.getRelativePath();
+
+ SkyKey fileKey = FileValue.key(rootedPath);
+ FileValue fileValue = (FileValue) env.getValue(fileKey);
+ if (fileValue == null) {
+ return null;
+ }
+
+ if (!fileValue.isDirectory()) {
+ return new RecursivePkgValue(NestedSetBuilder.<String>emptySet(ORDER));
+ }
+
+ if (fileValue.isSymlink()) {
+ // We do not follow directory symlinks when we look recursively for packages. It also
+ // prevents symlink loops.
+ return new RecursivePkgValue(NestedSetBuilder.<String>emptySet(ORDER));
+ }
+
+ PackageIdentifier packageId = PackageIdentifier.createInDefaultRepo(
+ rootRelativePath.getPathString());
+ PackageLookupValue pkgLookupValue =
+ (PackageLookupValue) env.getValue(PackageLookupValue.key(packageId));
+ if (pkgLookupValue == null) {
+ return null;
+ }
+
+ NestedSetBuilder<String> packages = new NestedSetBuilder<>(ORDER);
+
+ if (pkgLookupValue.packageExists()) {
+ if (pkgLookupValue.getRoot().equals(root)) {
+ try {
+ PackageValue pkgValue = (PackageValue)
+ env.getValueOrThrow(PackageValue.key(packageId),
+ NoSuchPackageException.class);
+ if (pkgValue == null) {
+ return null;
+ }
+ packages.add(pkgValue.getPackage().getName());
+ } catch (NoSuchPackageException e) {
+ // The package had errors, but don't fail-fast as there might subpackages below the
+ // current directory.
+ env.getListener().handle(Event.error(
+ "package contains errors: " + rootRelativePath.getPathString()));
+ if (e.getPackage() != null) {
+ packages.add(e.getPackage().getName());
+ }
+ }
+ }
+ // The package lookup succeeded, but was under a different root. We still, however, need to
+ // recursively consider subdirectories. For example:
+ //
+ // Pretend --package_path=rootA/workspace:rootB/workspace and these are the only files:
+ // rootA/workspace/foo/
+ // rootA/workspace/foo/bar/BUILD
+ // rootB/workspace/foo/BUILD
+ // If we're doing a recursive package lookup under 'rootA/workspace' starting at 'foo', note
+ // that even though the package 'foo' is under 'rootB/workspace', there is still a package
+ // 'foo/bar' under 'rootA/workspace'.
+ }
+
+ DirectoryListingValue dirValue = (DirectoryListingValue)
+ env.getValue(DirectoryListingValue.key(rootedPath));
+ if (dirValue == null) {
+ return null;
+ }
+
+ List<SkyKey> childDeps = Lists.newArrayList();
+ for (Dirent dirent : dirValue.getDirents()) {
+ if (dirent.getType() != Type.DIRECTORY) {
+ // Non-directories can never host packages, and we do not follow symlinks (see above).
+ continue;
+ }
+ String basename = dirent.getName();
+ if (rootRelativePath.equals(PathFragment.EMPTY_FRAGMENT)
+ && PathPackageLocator.DEFAULT_TOP_LEVEL_EXCLUDES.contains(basename)) {
+ continue;
+ }
+ SkyKey req = RecursivePkgValue.key(RootedPath.toRootedPath(root,
+ rootRelativePath.getRelative(basename)));
+ childDeps.add(req);
+ }
+ Map<SkyKey, SkyValue> childValueMap = env.getValues(childDeps);
+ if (env.valuesMissing()) {
+ return null;
+ }
+ // Aggregate the transitive subpackages.
+ for (SkyValue childValue : childValueMap.values()) {
+ if (childValue != null) {
+ packages.addTransitive(((RecursivePkgValue) childValue).getPackages());
+ }
+ }
+ return new RecursivePkgValue(packages.build());
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePkgValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePkgValue.java
new file mode 100644
index 0000000..4013b90
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePkgValue.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * This value represents the result of looking up all the packages under a given package path root,
+ * starting at a given directory.
+ */
+@Immutable
+@ThreadSafe
+public class RecursivePkgValue implements SkyValue {
+
+ private final NestedSet<String> packages;
+
+ public RecursivePkgValue(NestedSet<String> packages) {
+ this.packages = packages;
+ }
+
+ /**
+ * Create a transitive package lookup request.
+ */
+ @ThreadSafe
+ public static SkyKey key(RootedPath rootedPath) {
+ return new SkyKey(SkyFunctions.RECURSIVE_PKG, rootedPath);
+ }
+
+ public NestedSet<String> getPackages() {
+ return packages;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RepositoryValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/RepositoryValue.java
new file mode 100644
index 0000000..3183953
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/RepositoryValue.java
@@ -0,0 +1,75 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Objects;
+import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A local view of an external repository.
+ */
+public class RepositoryValue implements SkyValue {
+ private final Path path;
+
+ /**
+ * If path is a symlink, this will keep track of what the symlink actually points to (for
+ * checking equality).
+ */
+ private final FileValue details;
+
+ public RepositoryValue(Path path, FileValue repositoryDirectory) {
+ this.path = path;
+ this.details = repositoryDirectory;
+ }
+
+ /**
+ * Returns the path to the directory containing the repository's contents. This directory is
+ * guaranteed to exist. It may contain a full Bazel repository (with a WORKSPACE file,
+ * directories, and BUILD files) or simply contain a file (or set of files) for, say, a jar from
+ * Maven.
+ */
+ public Path getPath() {
+ return path;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+
+ if (other instanceof RepositoryValue) {
+ RepositoryValue otherValue = (RepositoryValue) other;
+ return path.equals(otherValue.path) && details.equals(otherValue.details);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(path, details);
+ }
+
+ /**
+ * Creates a key from the given repository name.
+ */
+ public static SkyKey key(RepositoryName repository) {
+ return new SkyKey(SkyFunctions.REPOSITORY, repository);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
new file mode 100644
index 0000000..e547166
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
@@ -0,0 +1,580 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.BuildView;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Preprocessor;
+import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.ResourceUsage;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.ModifiedFileSet;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.BuildDriver;
+import com.google.devtools.build.skyframe.Differencer;
+import com.google.devtools.build.skyframe.ImmutableDiff;
+import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator;
+import com.google.devtools.build.skyframe.Injectable;
+import com.google.devtools.build.skyframe.MemoizingEvaluator.EvaluatorSupplier;
+import com.google.devtools.build.skyframe.RecordingDifferencer;
+import com.google.devtools.build.skyframe.SequentialBuildDriver;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+
+/**
+ * A SkyframeExecutor that implicitly assumes that builds can be done incrementally from the most
+ * recent build. In other words, builds are "sequenced".
+ */
+public final class SequencedSkyframeExecutor extends SkyframeExecutor {
+ /** Lower limit for number of loaded packages to consider clearing CT values. */
+ private int valueCacheEvictionLimit = -1;
+
+ /** Union of labels of loaded packages since the last eviction of CT values. */
+ private Set<PackageIdentifier> allLoadedPackages = ImmutableSet.of();
+ private boolean lastAnalysisDiscarded = false;
+
+ // Can only be set once (to false) over the lifetime of this object. If false, the graph will not
+ // store edges, saving memory but making incremental builds impossible.
+ private boolean keepGraphEdges = true;
+
+ private RecordingDifferencer recordingDiffer;
+ private final DiffAwarenessManager diffAwarenessManager;
+
+ private SequencedSkyframeExecutor(Reporter reporter, EvaluatorSupplier evaluatorSupplier,
+ PackageFactory pkgFactory, TimestampGranularityMonitor tsgm,
+ BlazeDirectories directories, Factory workspaceStatusActionFactory,
+ ImmutableList<BuildInfoFactory> buildInfoFactories,
+ Set<Path> immutableDirectories,
+ Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories,
+ Predicate<PathFragment> allowedMissingInputs,
+ Preprocessor.Factory.Supplier preprocessorFactorySupplier,
+ ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions,
+ ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) {
+ super(reporter, evaluatorSupplier, pkgFactory, tsgm, directories,
+ workspaceStatusActionFactory, buildInfoFactories, immutableDirectories,
+ allowedMissingInputs, preprocessorFactorySupplier,
+ extraSkyFunctions, extraPrecomputedValues);
+ this.diffAwarenessManager = new DiffAwarenessManager(diffAwarenessFactories, reporter);
+ }
+
+ private SequencedSkyframeExecutor(Reporter reporter, PackageFactory pkgFactory,
+ TimestampGranularityMonitor tsgm, BlazeDirectories directories,
+ Factory workspaceStatusActionFactory,
+ ImmutableList<BuildInfoFactory> buildInfoFactories,
+ Set<Path> immutableDirectories,
+ Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories,
+ Predicate<PathFragment> allowedMissingInputs,
+ Preprocessor.Factory.Supplier preprocessorFactorySupplier,
+ ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions,
+ ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) {
+ this(reporter, InMemoryMemoizingEvaluator.SUPPLIER, pkgFactory, tsgm,
+ directories, workspaceStatusActionFactory, buildInfoFactories, immutableDirectories,
+ diffAwarenessFactories, allowedMissingInputs, preprocessorFactorySupplier,
+ extraSkyFunctions, extraPrecomputedValues);
+ }
+
+ private static SequencedSkyframeExecutor create(Reporter reporter,
+ EvaluatorSupplier evaluatorSupplier, PackageFactory pkgFactory,
+ TimestampGranularityMonitor tsgm, BlazeDirectories directories,
+ Factory workspaceStatusActionFactory, ImmutableList<BuildInfoFactory> buildInfoFactories,
+ Set<Path> immutableDirectories,
+ Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories,
+ Predicate<PathFragment> allowedMissingInputs,
+ Preprocessor.Factory.Supplier preprocessorFactorySupplier,
+ ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions,
+ ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) {
+ SequencedSkyframeExecutor skyframeExecutor = new SequencedSkyframeExecutor(reporter,
+ evaluatorSupplier, pkgFactory, tsgm, directories, workspaceStatusActionFactory,
+ buildInfoFactories, immutableDirectories, diffAwarenessFactories, allowedMissingInputs,
+ preprocessorFactorySupplier,
+ extraSkyFunctions, extraPrecomputedValues);
+ skyframeExecutor.init();
+ return skyframeExecutor;
+ }
+
+ public static SequencedSkyframeExecutor create(Reporter reporter, PackageFactory pkgFactory,
+ TimestampGranularityMonitor tsgm, BlazeDirectories directories,
+ Factory workspaceStatusActionFactory,
+ ImmutableList<BuildInfoFactory> buildInfoFactories,
+ Set<Path> immutableDirectories,
+ Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories,
+ Predicate<PathFragment> allowedMissingInputs,
+ Preprocessor.Factory.Supplier preprocessorFactorySupplier,
+ ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions,
+ ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) {
+ return create(reporter, InMemoryMemoizingEvaluator.SUPPLIER, pkgFactory, tsgm,
+ directories, workspaceStatusActionFactory, buildInfoFactories, immutableDirectories,
+ diffAwarenessFactories, allowedMissingInputs, preprocessorFactorySupplier,
+ extraSkyFunctions, extraPrecomputedValues);
+ }
+
+ @VisibleForTesting
+ public static SequencedSkyframeExecutor create(Reporter reporter, PackageFactory pkgFactory,
+ TimestampGranularityMonitor tsgm, BlazeDirectories directories,
+ WorkspaceStatusAction.Factory workspaceStatusActionFactory,
+ ImmutableList<BuildInfoFactory> buildInfoFactories,
+ Set<Path> immutableDirectories,
+ Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories) {
+ return create(reporter, pkgFactory, tsgm, directories, workspaceStatusActionFactory,
+ buildInfoFactories, immutableDirectories, diffAwarenessFactories,
+ Predicates.<PathFragment>alwaysFalse(),
+ Preprocessor.Factory.Supplier.NullSupplier.INSTANCE,
+ ImmutableMap.<SkyFunctionName, SkyFunction>of(),
+ ImmutableList.<PrecomputedValue.Injected>of());
+ }
+
+ @Override
+ protected BuildDriver newBuildDriver() {
+ return new SequentialBuildDriver(memoizingEvaluator);
+ }
+
+ @Override
+ protected void init() {
+ // Note that we need to set recordingDiffer first since SkyframeExecutor#init calls
+ // SkyframeExecutor#evaluatorDiffer.
+ recordingDiffer = new RecordingDifferencer();
+ super.init();
+ }
+
+ @Override
+ public void resetEvaluator() {
+ super.resetEvaluator();
+ diffAwarenessManager.reset();
+ }
+
+ @Override
+ protected Differencer evaluatorDiffer() {
+ return recordingDiffer;
+ }
+
+ @Override
+ protected Injectable injectable() {
+ return recordingDiffer;
+ }
+
+ @VisibleForTesting
+ public RecordingDifferencer getDifferencerForTesting() {
+ return recordingDiffer;
+ }
+
+ @Override
+ public void sync(PackageCacheOptions packageCacheOptions, Path workingDirectory,
+ String defaultsPackageContents, UUID commandId)
+ throws InterruptedException, AbruptExitException {
+ this.valueCacheEvictionLimit = packageCacheOptions.minLoadedPkgCountForCtNodeEviction;
+ super.sync(packageCacheOptions, workingDirectory, defaultsPackageContents, commandId);
+ handleDiffs();
+ }
+
+ /**
+ * The value types whose builders have direct access to the package locator, rather than accessing
+ * it via an explicit Skyframe dependency. They need to be invalidated if the package locator
+ * changes.
+ */
+ private static final Set<SkyFunctionName> PACKAGE_LOCATOR_DEPENDENT_VALUES = ImmutableSet.of(
+ SkyFunctions.FILE_STATE,
+ SkyFunctions.FILE,
+ SkyFunctions.DIRECTORY_LISTING_STATE,
+ SkyFunctions.TARGET_PATTERN,
+ SkyFunctions.WORKSPACE_FILE);
+
+ @Override
+ protected void onNewPackageLocator(PathPackageLocator oldLocator, PathPackageLocator pkgLocator) {
+ invalidate(SkyFunctionName.functionIsIn(PACKAGE_LOCATOR_DEPENDENT_VALUES));
+ }
+
+ @Override
+ protected void invalidate(Predicate<SkyKey> pred) {
+ recordingDiffer.invalidate(Iterables.filter(memoizingEvaluator.getValues().keySet(), pred));
+ }
+
+ private void invalidateDeletedPackages(Iterable<String> deletedPackages) {
+ ArrayList<SkyKey> packagesToInvalidate = Lists.newArrayList();
+ for (String deletedPackage : deletedPackages) {
+ PathFragment pathFragment = new PathFragment(deletedPackage);
+ packagesToInvalidate.add(PackageLookupValue.key(pathFragment));
+ }
+ recordingDiffer.invalidate(packagesToInvalidate);
+ }
+
+ /**
+ * Sets the packages that should be treated as deleted and ignored.
+ */
+ @Override
+ @VisibleForTesting // productionVisibility = Visibility.PRIVATE
+ public void setDeletedPackages(Iterable<String> pkgs) {
+ // Invalidate the old deletedPackages as they may exist now.
+ invalidateDeletedPackages(deletedPackages.get());
+ deletedPackages.set(ImmutableSet.copyOf(pkgs));
+ // Invalidate the new deletedPackages as we need to pretend that they don't exist now.
+ invalidateDeletedPackages(deletedPackages.get());
+ }
+
+ /**
+ * Uses diff awareness on all the package paths to invalidate changed files.
+ */
+ @VisibleForTesting
+ public void handleDiffs() throws InterruptedException {
+ if (lastAnalysisDiscarded) {
+ // Values were cleared last build, but they couldn't be deleted because they were needed for
+ // the execution phase. We can delete them now.
+ dropConfiguredTargetsNow();
+ lastAnalysisDiscarded = false;
+ }
+ modifiedFiles = 0;
+ Map<Path, DiffAwarenessManager.ProcessableModifiedFileSet> modifiedFilesByPathEntry =
+ Maps.newHashMap();
+ Set<Pair<Path, DiffAwarenessManager.ProcessableModifiedFileSet>>
+ pathEntriesWithoutDiffInformation = Sets.newHashSet();
+ for (Path pathEntry : pkgLocator.get().getPathEntries()) {
+ DiffAwarenessManager.ProcessableModifiedFileSet modifiedFileSet =
+ diffAwarenessManager.getDiff(pathEntry);
+ if (modifiedFileSet.getModifiedFileSet().treatEverythingAsModified()) {
+ pathEntriesWithoutDiffInformation.add(Pair.of(pathEntry, modifiedFileSet));
+ } else {
+ modifiedFilesByPathEntry.put(pathEntry, modifiedFileSet);
+ }
+ }
+ handleDiffsWithCompleteDiffInformation(modifiedFilesByPathEntry);
+ handleDiffsWithMissingDiffInformation(pathEntriesWithoutDiffInformation);
+ }
+
+ /**
+ * Invalidates files under path entries whose corresponding {@link DiffAwareness} gave an exact
+ * diff. Removes entries from the given map as they are processed. All of the files need to be
+ * invalidated, so the map should be empty upon completion of this function.
+ */
+ private void handleDiffsWithCompleteDiffInformation(
+ Map<Path, DiffAwarenessManager.ProcessableModifiedFileSet> modifiedFilesByPathEntry) {
+ // It's important that the below code be uninterruptible, since we already promised to
+ // invalidate these files.
+ for (Path pathEntry : ImmutableSet.copyOf(modifiedFilesByPathEntry.keySet())) {
+ DiffAwarenessManager.ProcessableModifiedFileSet processableModifiedFileSet =
+ modifiedFilesByPathEntry.get(pathEntry);
+ ModifiedFileSet modifiedFileSet = processableModifiedFileSet.getModifiedFileSet();
+ Preconditions.checkState(!modifiedFileSet.treatEverythingAsModified(), pathEntry);
+ Iterable<SkyKey> dirtyValues = getSkyKeysPotentiallyAffected(
+ modifiedFileSet.modifiedSourceFiles(), pathEntry);
+ handleChangedFiles(new ImmutableDiff(dirtyValues, ImmutableMap.<SkyKey, SkyValue>of()));
+ processableModifiedFileSet.markProcessed();
+ }
+ }
+
+ /**
+ * Finds and invalidates changed files under path entries whose corresponding
+ * {@link DiffAwareness} said all files may have been modified.
+ */
+ private void handleDiffsWithMissingDiffInformation(
+ Set<Pair<Path, DiffAwarenessManager.ProcessableModifiedFileSet>>
+ pathEntriesWithoutDiffInformation) throws InterruptedException {
+ if (pathEntriesWithoutDiffInformation.isEmpty()) {
+ return;
+ }
+ // Before running the FilesystemValueChecker, ensure that all values marked for invalidation
+ // have actually been invalidated (recall that invalidation happens at the beginning of the
+ // next evaluate() call), because checking those is a waste of time.
+ buildDriver.evaluate(ImmutableList.<SkyKey>of(), false,
+ DEFAULT_THREAD_COUNT, reporter);
+ FilesystemValueChecker fsnc = new FilesystemValueChecker(memoizingEvaluator, tsgm);
+ // We need to manually check for changes to known files. This entails finding all dirty file
+ // system values under package roots for which we don't have diff information. If at least
+ // one path entry doesn't have diff information, then we're going to have to iterate over
+ // the skyframe values at least once no matter what so we might as well do so now and avoid
+ // doing so more than once.
+ Iterable<SkyKey> filesystemSkyKeys = fsnc.getFilesystemSkyKeys();
+ // Partition by package path entry.
+ Multimap<Path, SkyKey> skyKeysByPathEntry = partitionSkyKeysByPackagePathEntry(
+ ImmutableSet.copyOf(pkgLocator.get().getPathEntries()), filesystemSkyKeys);
+ // Contains all file system values that we need to check for dirtiness.
+ List<Iterable<SkyKey>> valuesToCheckManually = Lists.newArrayList();
+ for (Pair<Path, DiffAwarenessManager.ProcessableModifiedFileSet> pair :
+ pathEntriesWithoutDiffInformation) {
+ Path pathEntry = pair.getFirst();
+ valuesToCheckManually.add(skyKeysByPathEntry.get(pathEntry));
+ }
+ Differencer.Diff diff = fsnc.getDirtyFilesystemValues(Iterables.concat(valuesToCheckManually));
+ handleChangedFiles(diff);
+ for (Pair<Path, DiffAwarenessManager.ProcessableModifiedFileSet> pair :
+ pathEntriesWithoutDiffInformation) {
+ DiffAwarenessManager.ProcessableModifiedFileSet processableModifiedFileSet = pair.getSecond();
+ processableModifiedFileSet.markProcessed();
+ }
+ }
+
+ /**
+ * Partitions the given filesystem values based on which package path root they are under.
+ * Returns a {@link Multimap} {@code m} such that {@code m.containsEntry(k, pe)} is true for
+ * each filesystem valuekey {@code k} under a package path root {@code pe}. Note that values not
+ * under a package path root are not present in the returned {@link Multimap}; these values are
+ * unconditionally checked for changes on each incremental build.
+ */
+ private static Multimap<Path, SkyKey> partitionSkyKeysByPackagePathEntry(
+ Set<Path> pkgRoots, Iterable<SkyKey> filesystemSkyKeys) {
+ ImmutableSetMultimap.Builder<Path, SkyKey> multimapBuilder =
+ ImmutableSetMultimap.builder();
+ for (SkyKey key : filesystemSkyKeys) {
+ Preconditions.checkState(key.functionName() == SkyFunctions.FILE_STATE
+ || key.functionName() == SkyFunctions.DIRECTORY_LISTING_STATE, key);
+ Path root = ((RootedPath) key.argument()).getRoot();
+ if (pkgRoots.contains(root)) {
+ multimapBuilder.put(root, key);
+ }
+ // We don't need to worry about FileStateValues for external files because they have a
+ // dependency on the build_id and so they get invalidated each build.
+ }
+ return multimapBuilder.build();
+ }
+
+ private void handleChangedFiles(Differencer.Diff diff) {
+ recordingDiffer.invalidate(diff.changedKeysWithoutNewValues());
+ recordingDiffer.inject(diff.changedKeysWithNewValues());
+ modifiedFiles += getNumberOfModifiedFiles(diff.changedKeysWithoutNewValues());
+ modifiedFiles += getNumberOfModifiedFiles(diff.changedKeysWithNewValues().keySet());
+ incrementalBuildMonitor.accrue(diff.changedKeysWithoutNewValues());
+ incrementalBuildMonitor.accrue(diff.changedKeysWithNewValues().keySet());
+ }
+
+ private static int getNumberOfModifiedFiles(Iterable<SkyKey> modifiedValues) {
+ // We are searching only for changed files, DirectoryListingValues don't depend on
+ // child values, that's why they are invalidated separately
+ return Iterables.size(Iterables.filter(modifiedValues,
+ SkyFunctionName.functionIs(SkyFunctions.FILE_STATE)));
+ }
+
+ @Override
+ public void decideKeepIncrementalState(boolean batch, BuildView.Options viewOptions) {
+ Preconditions.checkState(!active);
+ if (viewOptions == null) {
+ // Some blaze commands don't include the view options. Don't bother with them.
+ return;
+ }
+ if (batch && viewOptions.keepGoing && viewOptions.discardAnalysisCache) {
+ Preconditions.checkState(keepGraphEdges, "May only be called once if successful");
+ keepGraphEdges = false;
+ // Graph will be recreated on next sync.
+ }
+ }
+
+ @Override
+ public boolean hasIncrementalState() {
+ // TODO(bazel-team): Combine this method with clearSkyframeRelevantCaches() once legacy
+ // execution is removed [skyframe-execution].
+ return keepGraphEdges;
+ }
+
+ @Override
+ public void invalidateFilesUnderPathForTesting(ModifiedFileSet modifiedFileSet, Path pathEntry)
+ throws InterruptedException {
+ if (lastAnalysisDiscarded) {
+ // Values were cleared last build, but they couldn't be deleted because they were needed for
+ // the execution phase. We can delete them now.
+ dropConfiguredTargetsNow();
+ lastAnalysisDiscarded = false;
+ }
+ Iterable<SkyKey> keys;
+ if (modifiedFileSet.treatEverythingAsModified()) {
+ Differencer.Diff diff =
+ new FilesystemValueChecker(memoizingEvaluator, tsgm).getDirtyFilesystemSkyKeys();
+ keys = diff.changedKeysWithoutNewValues();
+ recordingDiffer.inject(diff.changedKeysWithNewValues());
+ } else {
+ keys = getSkyKeysPotentiallyAffected(modifiedFileSet.modifiedSourceFiles(), pathEntry);
+ }
+ syscalls.set(new PerBuildSyscallCache());
+ recordingDiffer.invalidate(keys);
+ // Blaze invalidates transient errors on every build.
+ invalidateTransientErrors();
+ }
+
+ @Override
+ public void invalidateTransientErrors() {
+ checkActive();
+ recordingDiffer.invalidateTransientErrors();
+ }
+
+ @Override
+ protected void invalidateDirtyActions(Iterable<SkyKey> dirtyActionValues) {
+ recordingDiffer.invalidate(dirtyActionValues);
+ }
+
+ /**
+ * Save memory by removing references to configured targets and actions in Skyframe.
+ *
+ * <p>These values must be recreated on subsequent builds. We do not clear the top-level target
+ * values, since their configured targets are needed for the target completion middleman values.
+ *
+ * <p>The values are not deleted during this method call, because they are needed for the
+ * execution phase. Instead, their data is cleared. The next build will delete the values (and
+ * recreate them if necessary).
+ */
+ private void discardAnalysisCache(Collection<ConfiguredTarget> topLevelTargets) {
+ lastAnalysisDiscarded = true;
+ for (Map.Entry<SkyKey, SkyValue> entry : memoizingEvaluator.getValues().entrySet()) {
+ if (!entry.getKey().functionName().equals(SkyFunctions.CONFIGURED_TARGET)) {
+ continue;
+ }
+ ConfiguredTargetValue ctValue = (ConfiguredTargetValue) entry.getValue();
+ // ctValue may be null if target was not successfully analyzed.
+ if (ctValue != null && !topLevelTargets.contains(ctValue.getConfiguredTarget())) {
+ ctValue.clear();
+ }
+ }
+ }
+
+ @Override
+ public void clearAnalysisCache(Collection<ConfiguredTarget> topLevelTargets) {
+ discardAnalysisCache(topLevelTargets);
+ }
+
+ @Override
+ public void dropConfiguredTargets() {
+ if (skyframeBuildView != null) {
+ skyframeBuildView.clearInvalidatedConfiguredTargets();
+ }
+ memoizingEvaluator.delete(
+ // We delete any value that can hold an action -- all subclasses of ActionLookupValue -- as
+ // well as ActionExecutionValues, since they do not depend on ActionLookupValues.
+ SkyFunctionName.functionIsIn(ImmutableSet.of(
+ SkyFunctions.CONFIGURED_TARGET,
+ SkyFunctions.ACTION_LOOKUP,
+ SkyFunctions.BUILD_INFO,
+ SkyFunctions.TARGET_COMPLETION,
+ SkyFunctions.BUILD_INFO_COLLECTION,
+ SkyFunctions.ACTION_EXECUTION))
+ );
+ }
+
+ /**
+ * Deletes all ConfiguredTarget values from the Skyframe cache.
+ *
+ * <p>After the execution of this method all invalidated and marked for deletion values
+ * (and the values depending on them) will be deleted from the cache.
+ *
+ * <p>WARNING: Note that a call to this method leaves legacy data inconsistent with Skyframe.
+ * The next build should clear the legacy caches.
+ */
+ private void dropConfiguredTargetsNow() {
+ dropConfiguredTargets();
+ // Run the invalidator to actually delete the values.
+ try {
+ progressReceiver.ignoreInvalidations = true;
+ callUninterruptibly(new Callable<Void>() {
+ @Override
+ public Void call() throws InterruptedException {
+ buildDriver.evaluate(ImmutableList.<SkyKey>of(), false,
+ ResourceUsage.getAvailableProcessors(), reporter);
+ return null;
+ }
+ });
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ } finally {
+ progressReceiver.ignoreInvalidations = false;
+ }
+ }
+
+ /**
+ * Returns true if the old set of Packages is a subset or superset of the new one.
+ *
+ * <p>Compares the names of packages instead of the Package objects themselves (Package doesn't
+ * yet override #equals). Since packages store their names as a String rather than a Label, it's
+ * easier to use strings here.
+ */
+ @VisibleForTesting
+ static boolean isBuildSubsetOrSupersetOfPreviousBuild(Set<PackageIdentifier> oldPackages,
+ Set<PackageIdentifier> newPackages) {
+ if (newPackages.size() <= oldPackages.size()) {
+ return Sets.difference(newPackages, oldPackages).isEmpty();
+ } else if (oldPackages.size() < newPackages.size()) {
+ // No need to check for <= here, since the first branch does that already.
+ // If size(A) = size(B), then then A\B = 0 iff B\A = 0
+ return Sets.difference(oldPackages, newPackages).isEmpty();
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void updateLoadedPackageSet(Set<PackageIdentifier> loadedPackages) {
+ Preconditions.checkState(valueCacheEvictionLimit >= 0,
+ "should have called setMinLoadedPkgCountForCtValueEviction earlier");
+
+ // Make a copy to avoid nesting SetView objects. It also computes size(), which we need below.
+ Set<PackageIdentifier> union = ImmutableSet.copyOf(
+ Sets.union(allLoadedPackages, loadedPackages));
+
+ if (union.size() < valueCacheEvictionLimit
+ || isBuildSubsetOrSupersetOfPreviousBuild(allLoadedPackages, loadedPackages)) {
+ allLoadedPackages = union;
+ } else {
+ dropConfiguredTargets();
+ allLoadedPackages = loadedPackages;
+ }
+ }
+
+ @Override
+ public void deleteOldNodes(long versionWindowForDirtyGc) {
+ // TODO(bazel-team): perhaps we should come up with a separate GC class dedicated to maintaining
+ // value garbage. If we ever do so, this logic should be moved there.
+ memoizingEvaluator.deleteDirty(versionWindowForDirtyGc);
+ }
+
+ @Override
+ public void dumpPackages(PrintStream out) {
+ Iterable<SkyKey> packageSkyKeys = Iterables.filter(memoizingEvaluator.getValues().keySet(),
+ SkyFunctions.isSkyFunction(SkyFunctions.PACKAGE));
+ out.println(Iterables.size(packageSkyKeys) + " packages");
+ for (SkyKey packageSkyKey : packageSkyKeys) {
+ Package pkg = ((PackageValue) memoizingEvaluator.getValues().get(packageSkyKey)).getPackage();
+ pkg.dump(out);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutorFactory.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutorFactory.java
new file mode 100644
index 0000000..7e277a7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutorFactory.java
@@ -0,0 +1,53 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.Preprocessor;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+
+import java.util.Set;
+
+/**
+ * A factory of SkyframeExecutors that returns SequencedSkyframeExecutor.
+ */
+public class SequencedSkyframeExecutorFactory implements SkyframeExecutorFactory {
+
+ @Override
+ public SkyframeExecutor create(Reporter reporter, PackageFactory pkgFactory,
+ TimestampGranularityMonitor tsgm, BlazeDirectories directories,
+ Factory workspaceStatusActionFactory, ImmutableList<BuildInfoFactory> buildInfoFactories,
+ Set<Path> immutableDirectories,
+ Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories,
+ Predicate<PathFragment> allowedMissingInputs,
+ Preprocessor.Factory.Supplier preprocessorFactorySupplier,
+ ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions,
+ ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) {
+ return SequencedSkyframeExecutor.create(reporter, pkgFactory, tsgm, directories,
+ workspaceStatusActionFactory, buildInfoFactories, immutableDirectories,
+ diffAwarenessFactories, allowedMissingInputs, preprocessorFactorySupplier,
+ extraSkyFunctions, extraPrecomputedValues);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java
new file mode 100644
index 0000000..316d27d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java
@@ -0,0 +1,81 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Predicate;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+
+/**
+ * Value types in Skyframe.
+ */
+public final class SkyFunctions {
+ public static final SkyFunctionName PRECOMPUTED = new SkyFunctionName("PRECOMPUTED", false);
+ public static final SkyFunctionName FILE_STATE = new SkyFunctionName("FILE_STATE", false);
+ public static final SkyFunctionName DIRECTORY_LISTING_STATE =
+ new SkyFunctionName("DIRECTORY_LISTING_STATE", false);
+ public static final SkyFunctionName FILE_SYMLINK_CYCLE_UNIQUENESS =
+ SkyFunctionName.computed("FILE_SYMLINK_CYCLE_UNIQUENESS_NODE");
+ public static final SkyFunctionName FILE = SkyFunctionName.computed("FILE");
+ public static final SkyFunctionName DIRECTORY_LISTING =
+ SkyFunctionName.computed("DIRECTORY_LISTING");
+ public static final SkyFunctionName PACKAGE_LOOKUP = SkyFunctionName.computed("PACKAGE_LOOKUP");
+ public static final SkyFunctionName CONTAINING_PACKAGE_LOOKUP =
+ SkyFunctionName.computed("CONTAINING_PACKAGE_LOOKUP");
+ public static final SkyFunctionName AST_FILE_LOOKUP = SkyFunctionName.computed("AST_FILE_LOOKUP");
+ public static final SkyFunctionName SKYLARK_IMPORTS_LOOKUP =
+ SkyFunctionName.computed("SKYLARK_IMPORTS_LOOKUP");
+ public static final SkyFunctionName GLOB = SkyFunctionName.computed("GLOB");
+ public static final SkyFunctionName PACKAGE = SkyFunctionName.computed("PACKAGE");
+ public static final SkyFunctionName TARGET_MARKER = SkyFunctionName.computed("TARGET_MARKER");
+ public static final SkyFunctionName TARGET_PATTERN = SkyFunctionName.computed("TARGET_PATTERN");
+ public static final SkyFunctionName RECURSIVE_PKG = SkyFunctionName.computed("RECURSIVE_PKG");
+ public static final SkyFunctionName TRANSITIVE_TARGET =
+ SkyFunctionName.computed("TRANSITIVE_TARGET");
+ public static final SkyFunctionName CONFIGURED_TARGET =
+ SkyFunctionName.computed("CONFIGURED_TARGET");
+ public static final SkyFunctionName ASPECT = SkyFunctionName.computed("ASPECT");
+ public static final SkyFunctionName POST_CONFIGURED_TARGET =
+ SkyFunctionName.computed("POST_CONFIGURED_TARGET");
+ public static final SkyFunctionName TARGET_COMPLETION =
+ SkyFunctionName.computed("TARGET_COMPLETION");
+ public static final SkyFunctionName TEST_COMPLETION =
+ SkyFunctionName.computed("TEST_COMPLETION");
+ public static final SkyFunctionName CONFIGURATION_FRAGMENT =
+ SkyFunctionName.computed("CONFIGURATION_FRAGMENT");
+ public static final SkyFunctionName CONFIGURATION_COLLECTION =
+ SkyFunctionName.computed("CONFIGURATION_COLLECTION");
+ public static final SkyFunctionName ARTIFACT = SkyFunctionName.computed("ARTIFACT");
+ public static final SkyFunctionName ACTION_EXECUTION =
+ SkyFunctionName.computed("ACTION_EXECUTION");
+ public static final SkyFunctionName ACTION_LOOKUP = SkyFunctionName.computed("ACTION_LOOKUP");
+ public static final SkyFunctionName RECURSIVE_FILESYSTEM_TRAVERSAL =
+ SkyFunctionName.computed("RECURSIVE_DIRECTORY_TRAVERSAL");
+ public static final SkyFunctionName FILESET_ENTRY = SkyFunctionName.computed("FILESET_ENTRY");
+ public static final SkyFunctionName BUILD_INFO_COLLECTION =
+ SkyFunctionName.computed("BUILD_INFO_COLLECTION");
+ public static final SkyFunctionName BUILD_INFO = SkyFunctionName.computed("BUILD_INFO");
+ public static final SkyFunctionName WORKSPACE_FILE = SkyFunctionName.computed("WORKSPACE_FILE");
+ public static final SkyFunctionName COVERAGE_REPORT = SkyFunctionName.computed("COVERAGE_REPORT");
+ public static final SkyFunctionName REPOSITORY = SkyFunctionName.computed("REPOSITORY");
+
+ public static Predicate<SkyKey> isSkyFunction(final SkyFunctionName functionName) {
+ return new Predicate<SkyKey>() {
+ @Override
+ public boolean apply(SkyKey key) {
+ return key.functionName() == functionName;
+ }
+ };
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java
new file mode 100644
index 0000000..bd591e1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java
@@ -0,0 +1,1152 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.devtools.build.lib.vfs.FileSystemUtils.createDirectoryAndParents;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import com.google.common.eventbus.EventBus;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionCacheChecker;
+import com.google.devtools.build.lib.actions.ActionCacheChecker.Token;
+import com.google.devtools.build.lib.actions.ActionCompletionEvent;
+import com.google.devtools.build.lib.actions.ActionExecutedEvent;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter;
+import com.google.devtools.build.lib.actions.ActionGraph;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.actions.ActionLogBufferPathGenerator;
+import com.google.devtools.build.lib.actions.ActionMiddlemanEvent;
+import com.google.devtools.build.lib.actions.ActionStartedEvent;
+import com.google.devtools.build.lib.actions.Actions;
+import com.google.devtools.build.lib.actions.AlreadyReportedActionExecutionException;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Artifact.MiddlemanExpander;
+import com.google.devtools.build.lib.actions.ArtifactPrefixConflictException;
+import com.google.devtools.build.lib.actions.CachedActionEvent;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.MapBasedActionGraph;
+import com.google.devtools.build.lib.actions.MutableActionGraph;
+import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException;
+import com.google.devtools.build.lib.actions.NotifyOnActionCacheHit;
+import com.google.devtools.build.lib.actions.ResourceManager;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.actions.TargetOutOfDateException;
+import com.google.devtools.build.lib.actions.cache.Digest;
+import com.google.devtools.build.lib.actions.cache.DigestUtils;
+import com.google.devtools.build.lib.actions.cache.Metadata;
+import com.google.devtools.build.lib.actions.cache.MetadataHandler;
+import com.google.devtools.build.lib.concurrent.ExecutorShutdownUtil;
+import com.google.devtools.build.lib.concurrent.Sharder;
+import com.google.devtools.build.lib.concurrent.ThrowableRecordingRunnableWrapper;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.io.FileOutErr;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.Symlinks;
+import com.google.protobuf.ByteString;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentNavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * Action executor: takes care of preparing an action for execution, executing it, validating that
+ * all output artifacts were created, error reporting, etc.
+ */
+public final class SkyframeActionExecutor {
+ private final Reporter reporter;
+ private final AtomicReference<EventBus> eventBus;
+ private final ResourceManager resourceManager;
+ private Executor executorEngine;
+ private ActionLogBufferPathGenerator actionLogBufferPathGenerator;
+ private ActionCacheChecker actionCacheChecker;
+ private ConcurrentMap<Artifact, Metadata> undeclaredInputsMetadata = new ConcurrentHashMap<>();
+ private final Profiler profiler = Profiler.instance();
+ private boolean explain;
+
+ // We keep track of actions already executed this build in order to avoid executing a shared
+ // action twice. Note that we may still unnecessarily re-execute the action on a subsequent
+ // build: say actions A and B are shared. If A is requested on the first build and then B is
+ // requested on the second build, we will execute B even though its output files are up to date.
+ // However, we will not re-execute A on a subsequent build.
+ // We do not allow the shared action to re-execute in the same build, even after the first
+ // action has finished execution, because a downstream action might be reading the output file
+ // at the same time as the shared action was writing to it.
+ // This map is also used for Actions that try to execute twice because they have discovered
+ // headers -- the SkyFunction tries to declare a dep on the missing headers and has to restart.
+ // We don't want to execute the action again on the second entry to the SkyFunction.
+ // In both cases, we store the already-computed ActionExecutionValue to avoid having to compute it
+ // again.
+ private ConcurrentMap<Artifact, Pair<Action, FutureTask<ActionExecutionValue>>> buildActionMap;
+
+ // Errors found when examining all actions in the graph are stored here, so that they can be
+ // thrown when execution of the action is requested. This field is set during each call to
+ // findAndStoreArtifactConflicts, and is preserved across builds otherwise.
+ private ImmutableMap<Action, ConflictException> badActionMap = ImmutableMap.of();
+ private boolean keepGoing;
+ private boolean hadExecutionError;
+ private ActionInputFileCache perBuildFileCache;
+ private ProgressSupplier progressSupplier;
+ private ActionCompletedReceiver completionReceiver;
+ private final AtomicReference<ActionExecutionStatusReporter> statusReporterRef;
+
+ SkyframeActionExecutor(Reporter reporter, ResourceManager resourceManager,
+ AtomicReference<EventBus> eventBus,
+ AtomicReference<ActionExecutionStatusReporter> statusReporterRef) {
+ this.reporter = reporter;
+ this.resourceManager = resourceManager;
+ this.eventBus = eventBus;
+ this.statusReporterRef = statusReporterRef;
+ }
+
+ /**
+ * A typed union of {@link ActionConflictException}, which indicates two actions that generate
+ * the same {@link Artifact}, and {@link ArtifactPrefixConflictException}, which indicates that
+ * the path of one {@link Artifact} is a prefix of another.
+ */
+ public static class ConflictException extends Exception {
+ @Nullable private final ActionConflictException ace;
+ @Nullable private final ArtifactPrefixConflictException apce;
+
+ public ConflictException(ActionConflictException e) {
+ super(e);
+ this.ace = e;
+ this.apce = null;
+ }
+
+ public ConflictException(ArtifactPrefixConflictException e) {
+ super(e);
+ this.ace = null;
+ this.apce = e;
+ }
+
+ void rethrowTyped() throws ActionConflictException, ArtifactPrefixConflictException {
+ if (ace == null) {
+ throw Preconditions.checkNotNull(apce);
+ }
+ if (apce == null) {
+ throw Preconditions.checkNotNull(ace);
+ }
+ throw new IllegalStateException();
+ }
+ }
+
+ /**
+ * Return the map of mostly recently executed bad actions to their corresponding exception.
+ * See {#findAndStoreArtifactConflicts()}.
+ */
+ public ImmutableMap<Action, ConflictException> badActions() {
+ // TODO(bazel-team): Move badActions() and findAndStoreArtifactConflicts() to SkyframeBuildView
+ // now that it's done in the analysis phase.
+ return badActionMap;
+ }
+
+ /**
+ * Basic implementation of {@link MetadataHandler} that delegates to Skyframe for metadata and
+ * caches missing source artifacts (which must be undeclared inputs: discovered headers) to avoid
+ * excessive filesystem access. The discovered-header cache is available across actions.
+ */
+ // TODO(bazel-team): remove when include scanning is skyframe-native.
+ private static class UndeclaredInputHandler implements MetadataHandler {
+ private final ConcurrentMap<Artifact, Metadata> undeclaredInputsMetadata;
+ private final MetadataHandler perActionHandler;
+
+ UndeclaredInputHandler(MetadataHandler perActionHandler,
+ ConcurrentMap<Artifact, Metadata> undeclaredInputsMetadata) {
+ // Shared across all UndeclaredInputHandlers in this build.
+ this.undeclaredInputsMetadata = undeclaredInputsMetadata;
+ this.perActionHandler = perActionHandler;
+ }
+
+ @Override
+ public Metadata getMetadataMaybe(Artifact artifact) {
+ try {
+ return getMetadata(artifact);
+ } catch (IOException e) {
+ return null;
+ }
+ }
+
+ @Override
+ public Metadata getMetadata(Artifact artifact) throws IOException {
+ Metadata metadata = perActionHandler.getMetadata(artifact);
+ if (metadata != null) {
+ return metadata;
+ }
+ // Skyframe stats all generated artifacts, because either they are outputs of the action being
+ // executed or they are generated files already present in the graph.
+ Preconditions.checkState(artifact.isSourceArtifact(), artifact);
+ metadata = undeclaredInputsMetadata.get(artifact);
+ if (metadata != null) {
+ return metadata;
+ }
+ FileStatus stat = artifact.getPath().stat();
+ if (DigestUtils.useFileDigest(artifact, stat.isFile(), stat.getSize())) {
+ metadata = new Metadata(Preconditions.checkNotNull(
+ DigestUtils.getDigestOrFail(artifact.getPath(), stat.getSize()), artifact));
+ } else {
+ metadata = new Metadata(stat.getLastModifiedTime());
+ }
+ // Cache for other actions that may also include without declaring.
+ Metadata oldMetadata = undeclaredInputsMetadata.put(artifact, metadata);
+ FileAndMetadataCache.checkInconsistentData(artifact, oldMetadata, metadata);
+ return metadata;
+ }
+
+ @Override
+ public void setDigestForVirtualArtifact(Artifact artifact, Digest digest) {
+ perActionHandler.setDigestForVirtualArtifact(artifact, digest);
+ }
+
+ @Override
+ public void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest) {
+ perActionHandler.injectDigest(output, statNoFollow, digest);
+ }
+
+ @Override
+ public boolean artifactExists(Artifact artifact) {
+ return perActionHandler.artifactExists(artifact);
+ }
+
+ @Override
+ public boolean isRegularFile(Artifact artifact) {
+ return perActionHandler.isRegularFile(artifact);
+ }
+
+ @Override
+ public boolean isInjected(Artifact artifact) throws IOException {
+ return perActionHandler.isInjected(artifact);
+ }
+
+ @Override
+ public void discardMetadata(Collection<Artifact> artifactList) {
+ // This input handler only caches undeclared inputs, which never need to be discarded
+ // intra-build.
+ perActionHandler.discardMetadata(artifactList);
+ }
+ }
+
+ /**
+ * Find conflicts between generated artifacts. There are two ways to have conflicts. First, if
+ * two (unshareable) actions generate the same output artifact, this will result in an {@link
+ * ActionConflictException}. Second, if one action generates an artifact whose path is a prefix of
+ * another artifact's path, those two artifacts cannot exist simultaneously in the output tree.
+ * This causes an {@link ArtifactPrefixConflictException}. The relevant exceptions are stored in
+ * the executor in {@code badActionMap}, and will be thrown immediately when that action is
+ * executed. Those exceptions persist, so that even if the action is not executed this build, the
+ * first time it is executed, the correct exception will be thrown.
+ *
+ * <p>This method must be called if a new action was added to the graph this build, so
+ * whenever a new configured target was analyzed this build. It is somewhat expensive (~1s
+ * range for a medium build as of 2014), so it should only be called when necessary.
+ *
+ * <p>Conflicts found may not be requested this build, and so we may overzealously throw an error.
+ * For instance, if actions A and B generate the same artifact foo, and the user first requests
+ * A' depending on A, and then in a subsequent build B' depending on B, we will fail the second
+ * build, even though it would have succeeded if it had been the only build. However, since
+ * Skyframe does not know the transitive dependencies of the request, we err on the conservative
+ * side.
+ *
+ * <p>If the user first runs one action on the first build, and on the second build adds a
+ * conflicting action, only the second action's error may be reported (because the first action
+ * will be cached), whereas if both actions were requested for the first time, both errors would
+ * be reported. However, the first time an action is added to the build, we are guaranteed to find
+ * any conflicts it has, since this method will compare it against all other actions. So there is
+ * no sequence of builds that can evade the error.
+ */
+ void findAndStoreArtifactConflicts(Iterable<ActionLookupValue> actionLookupValues)
+ throws InterruptedException {
+ ConcurrentMap<Action, ConflictException> temporaryBadActionMap = new ConcurrentHashMap<>();
+ Pair<ActionGraph, SortedMap<PathFragment, Artifact>> result;
+ result = constructActionGraphAndPathMap(actionLookupValues, temporaryBadActionMap);
+ ActionGraph actionGraph = result.first;
+ SortedMap<PathFragment, Artifact> artifactPathMap = result.second;
+
+ // Report an error for every derived artifact which is a prefix of another.
+ // If x << y << z (where x << y means "y starts with x"), then we only report (x,y), (x,z), but
+ // not (y,z).
+ Iterator<PathFragment> iter = artifactPathMap.keySet().iterator();
+ if (!iter.hasNext()) {
+ // No actions in graph -- currently happens only in tests. Special-cased because .next() call
+ // below is unconditional.
+ this.badActionMap = ImmutableMap.of();
+ return;
+ }
+ for (PathFragment pathJ = iter.next(); iter.hasNext(); ) {
+ // For each comparison, we have a prefix candidate (pathI) and a suffix candidate (pathJ).
+ // At the beginning of the loop, we set pathI to the last suffix candidate, since it has not
+ // yet been tested as a prefix candidate, and then set pathJ to the paths coming after pathI,
+ // until we come to one that does not contain pathI as a prefix. pathI is then verified not to
+ // be the prefix of any path, so we start the next run of the loop.
+ PathFragment pathI = pathJ;
+ // Compare pathI to the paths coming after it.
+ while (iter.hasNext()) {
+ pathJ = iter.next();
+ if (pathJ.startsWith(pathI)) { // prefix conflict.
+ Artifact artifactI = Preconditions.checkNotNull(artifactPathMap.get(pathI), pathI);
+ Artifact artifactJ = Preconditions.checkNotNull(artifactPathMap.get(pathJ), pathJ);
+ Action actionI =
+ Preconditions.checkNotNull(actionGraph.getGeneratingAction(artifactI), artifactI);
+ Action actionJ =
+ Preconditions.checkNotNull(actionGraph.getGeneratingAction(artifactJ), artifactJ);
+ if (actionI.shouldReportPathPrefixConflict(actionJ)) {
+ ArtifactPrefixConflictException exception = new ArtifactPrefixConflictException(pathI,
+ pathJ, actionI.getOwner().getLabel(), actionJ.getOwner().getLabel());
+ temporaryBadActionMap.put(actionI, new ConflictException(exception));
+ temporaryBadActionMap.put(actionJ, new ConflictException(exception));
+ }
+ } else { // pathJ didn't have prefix pathI, so no conflict possible for pathI.
+ break;
+ }
+ }
+ }
+ this.badActionMap = ImmutableMap.copyOf(temporaryBadActionMap);
+ }
+
+ /**
+ * Simultaneously construct an action graph for all the actions in Skyframe and a map from
+ * {@link PathFragment}s to their respective {@link Artifact}s. We do this in a threadpool to save
+ * around 1.5 seconds on a mid-sized build versus a single-threaded operation.
+ */
+ private static Pair<ActionGraph, SortedMap<PathFragment, Artifact>>
+ constructActionGraphAndPathMap(
+ Iterable<ActionLookupValue> values,
+ ConcurrentMap<Action, ConflictException> badActionMap) throws InterruptedException {
+ MutableActionGraph actionGraph = new MapBasedActionGraph();
+ ConcurrentNavigableMap<PathFragment, Artifact> artifactPathMap = new ConcurrentSkipListMap<>();
+ // Action graph construction is CPU-bound.
+ int numJobs = Runtime.getRuntime().availableProcessors();
+ // No great reason for expecting 5000 action lookup values, but not worth counting size of
+ // values.
+ Sharder<ActionLookupValue> actionShards = new Sharder<>(numJobs, 5000);
+ for (ActionLookupValue value : values) {
+ actionShards.add(value);
+ }
+
+ ThrowableRecordingRunnableWrapper wrapper = new ThrowableRecordingRunnableWrapper(
+ "SkyframeActionExecutor#constructActionGraphAndPathMap");
+
+ ExecutorService executor = Executors.newFixedThreadPool(
+ numJobs,
+ new ThreadFactoryBuilder().setNameFormat("ActionLookupValue Processor %d").build());
+ for (List<ActionLookupValue> shard : actionShards) {
+ executor.execute(
+ wrapper.wrap(actionRegistration(shard, actionGraph, artifactPathMap, badActionMap)));
+ }
+ boolean interrupted = ExecutorShutdownUtil.interruptibleShutdown(executor);
+ Throwables.propagateIfPossible(wrapper.getFirstThrownError());
+ if (interrupted) {
+ throw new InterruptedException();
+ }
+ return Pair.<ActionGraph, SortedMap<PathFragment, Artifact>>of(actionGraph, artifactPathMap);
+ }
+
+ private static Runnable actionRegistration(
+ final List<ActionLookupValue> values,
+ final MutableActionGraph actionGraph,
+ final ConcurrentMap<PathFragment, Artifact> artifactPathMap,
+ final ConcurrentMap<Action, ConflictException> badActionMap) {
+ return new Runnable() {
+ @Override
+ public void run() {
+ for (ActionLookupValue value : values) {
+ Set<Action> registeredActions = new HashSet<>();
+ for (Map.Entry<Artifact, Action> entry : value.getMapForConsistencyCheck().entrySet()) {
+ Action action = entry.getValue();
+ // We have an entry for each <action, artifact> pair. Only try to register each action
+ // once.
+ if (registeredActions.add(action)) {
+ try {
+ actionGraph.registerAction(action);
+ } catch (ActionConflictException e) {
+ Exception oldException = badActionMap.put(action, new ConflictException(e));
+ Preconditions.checkState(oldException == null,
+ "%s | %s | %s", action, e, oldException);
+ // We skip the rest of the loop, and do not add the path->artifact mapping for this
+ // artifact below -- we don't need to check it since this action is already in
+ // error.
+ continue;
+ }
+ }
+ artifactPathMap.put(entry.getKey().getExecPath(), entry.getKey());
+ }
+ }
+ }
+ };
+ }
+
+ void prepareForExecution(Executor executor, boolean keepGoing,
+ boolean explain, ActionCacheChecker actionCacheChecker) {
+ this.executorEngine = Preconditions.checkNotNull(executor);
+
+ // Start with a new map each build so there's no issue with internal resizing.
+ this.buildActionMap = Maps.newConcurrentMap();
+ this.keepGoing = keepGoing;
+ this.hadExecutionError = false;
+ this.actionCacheChecker = Preconditions.checkNotNull(actionCacheChecker);
+ // Don't cache possibly stale data from the last build.
+ undeclaredInputsMetadata = new ConcurrentHashMap<>();
+ this.explain = explain;
+ }
+
+ public void setActionLogBufferPathGenerator(
+ ActionLogBufferPathGenerator actionLogBufferPathGenerator) {
+ this.actionLogBufferPathGenerator = actionLogBufferPathGenerator;
+ }
+
+ void executionOver() {
+ // This transitively holds a bunch of heavy objects, so it's important to clear it at the
+ // end of a build.
+ this.executorEngine = null;
+ }
+
+ File getExecRoot() {
+ return executorEngine.getExecRoot().getPathFile();
+ }
+
+ boolean probeActionExecution(Action action) {
+ return buildActionMap.containsKey(action.getPrimaryOutput());
+ }
+
+ /**
+ * Executes the provided action on the current thread. Returns the ActionExecutionValue with the
+ * result, either computed here or already computed on another thread.
+ *
+ * <p>For use from {@link ArtifactFunction} only.
+ */
+ ActionExecutionValue executeAction(Action action, FileAndMetadataCache graphFileCache,
+ Token token, long actionStartTime,
+ ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ Exception exception = badActionMap.get(action);
+ if (exception != null) {
+ // If action had a conflict with some other action in the graph, report it now.
+ reportError(exception.getMessage(), exception, action, null);
+ }
+ Artifact primaryOutput = action.getPrimaryOutput();
+ FutureTask<ActionExecutionValue> actionTask =
+ new FutureTask<>(new ActionRunner(action, graphFileCache, token,
+ actionStartTime, actionExecutionContext));
+ // Check to see if another action is already executing/has executed this value.
+ Pair<Action, FutureTask<ActionExecutionValue>> oldAction =
+ buildActionMap.putIfAbsent(primaryOutput, Pair.of(action, actionTask));
+
+ if (oldAction == null) {
+ actionTask.run();
+ } else if (action == oldAction.first) {
+ // We only allow the same action to be executed twice if it discovers inputs. We allow that
+ // because we need to declare additional dependencies on those new inputs.
+ Preconditions.checkState(action.discoversInputs(),
+ "Same action shouldn't execute twice in build: %s", action);
+ actionTask = oldAction.second;
+ } else {
+ Preconditions.checkState(Actions.canBeShared(oldAction.first, action),
+ "Actions cannot be shared: %s %s", oldAction.first, action);
+ // Wait for other action to finish, so any actions that depend on its outputs can execute.
+ actionTask = oldAction.second;
+ }
+ try {
+ return actionTask.get();
+ } catch (ExecutionException e) {
+ Throwables.propagateIfPossible(e.getCause(),
+ ActionExecutionException.class, InterruptedException.class);
+ throw new IllegalStateException(e);
+ } finally {
+ String message = action.getProgressMessage();
+ if (message != null) {
+ // Tell the receiver that the action has completed *before* telling the reporter.
+ // This way the latter will correctly show the number of completed actions when task
+ // completion messages are enabled (--show_task_finish).
+ if (completionReceiver != null) {
+ completionReceiver.actionCompleted(action);
+ }
+ reporter.finishTask(null, prependExecPhaseStats(message));
+ }
+ }
+ }
+
+ /**
+ * Returns an ActionExecutionContext suitable for executing a particular action. The caller should
+ * pass the returned context to {@link #executeAction}, and any other method that needs to execute
+ * tasks related to that action.
+ */
+ ActionExecutionContext constructActionExecutionContext(final FileAndMetadataCache graphFileCache,
+ MetadataHandler metadataHandler) {
+ FileOutErr fileOutErr = actionLogBufferPathGenerator.generate();
+ return new ActionExecutionContext(
+ executorEngine,
+ new DelegatingPairFileCache(graphFileCache, perBuildFileCache),
+ metadataHandler,
+ fileOutErr,
+ new MiddlemanExpander() {
+ @Override
+ public void expand(Artifact middlemanArtifact,
+ Collection<? super Artifact> output) {
+ // Legacy code is more permissive regarding "mm" in that it expands any middleman,
+ // not just inputs of this action. Skyframe doesn't have access to a global action
+ // graph, therefore this implementation can't expand any middleman, only the
+ // inputs of this action.
+ // This is fine though: actions should only hold references to their input
+ // artifacts, otherwise hermeticity would be violated.
+ output.addAll(graphFileCache.expandInputMiddleman(middlemanArtifact));
+ }
+ });
+ }
+
+ /**
+ * Returns a MetadataHandler for use when executing a particular action. The caller can pass the
+ * returned handler in whenever a MetadataHandler is needed in the course of executing the action.
+ */
+ MetadataHandler constructMetadataHandler(MetadataHandler graphFileCache) {
+ return new UndeclaredInputHandler(graphFileCache, undeclaredInputsMetadata);
+ }
+
+ /**
+ * Checks the action cache to see if {@code action} needs to be executed, or is up to date.
+ * Returns a token with the semantics of {@link ActionCacheChecker#getTokenIfNeedToExecute}: null
+ * if the action is up to date, and non-null if it needs to be executed, in which case that token
+ * should be provided to the ActionCacheChecker after execution.
+ */
+ Token checkActionCache(Action action, MetadataHandler metadataHandler, long actionStartTime) {
+ profiler.startTask(ProfilerTask.ACTION_CHECK, action);
+ Token token = actionCacheChecker.getTokenIfNeedToExecute(
+ action, explain ? reporter : null, metadataHandler);
+ profiler.completeTask(ProfilerTask.ACTION_CHECK);
+ if (token == null) {
+ boolean eventPosted = false;
+ // Notify BlazeRuntimeStatistics about the action middleman 'execution'.
+ if (action.getActionType().isMiddleman()) {
+ postEvent(new ActionMiddlemanEvent(action, actionStartTime));
+ eventPosted = true;
+ }
+
+ if (action instanceof NotifyOnActionCacheHit) {
+ NotifyOnActionCacheHit notify = (NotifyOnActionCacheHit) action;
+ notify.actionCacheHit(executorEngine);
+ }
+
+ // We still need to check the outputs so that output file data is available to the value.
+ checkOutputs(action, metadataHandler);
+ if (!eventPosted) {
+ postEvent(new CachedActionEvent(action, actionStartTime));
+ }
+ }
+ return token;
+ }
+
+ /**
+ * Perform dependency discovery for action, which must discover its inputs.
+ *
+ * <p>This method is just a wrapper around {@link Action#discoverInputs} that properly processes
+ * any ActionExecutionException thrown before rethrowing it to the caller.
+ */
+ void discoverInputs(Action action, ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ try {
+ action.discoverInputs(actionExecutionContext);
+ } catch (ActionExecutionException e) {
+ processAndThrow(e, action, actionExecutionContext.getFileOutErr());
+ }
+ }
+
+ /**
+ * This method should be called if the builder encounters an error during
+ * execution. This allows the builder to record that it encountered at
+ * least one error, and may make it swallow its output to prevent
+ * spamming the user any further.
+ */
+ private void recordExecutionError() {
+ hadExecutionError = true;
+ }
+
+ /**
+ * Returns true if the Builder is winding down (i.e. cancelling outstanding
+ * actions and preparing to abort.)
+ * The builder is winding down iff:
+ * <ul>
+ * <li>we had an execution error
+ * <li>we are not running with --keep_going
+ * </ul>
+ */
+ private boolean isBuilderAborting() {
+ return hadExecutionError && !keepGoing;
+ }
+
+ void setFileCache(ActionInputFileCache fileCache) {
+ this.perBuildFileCache = fileCache;
+ }
+
+ private class ActionRunner implements Callable<ActionExecutionValue> {
+ private final Action action;
+ private final FileAndMetadataCache graphFileCache;
+ private Token token;
+ private long actionStartTime;
+ private ActionExecutionContext actionExecutionContext;
+
+ ActionRunner(Action action, FileAndMetadataCache graphFileCache, Token token,
+ long actionStartTime,
+ ActionExecutionContext actionExecutionContext) {
+ this.action = action;
+ this.graphFileCache = graphFileCache;
+ this.token = token;
+ this.actionStartTime = actionStartTime;
+ this.actionExecutionContext = actionExecutionContext;
+ }
+
+ @Override
+ public ActionExecutionValue call() throws ActionExecutionException, InterruptedException {
+ profiler.startTask(ProfilerTask.ACTION, action);
+ try {
+ if (actionCacheChecker.isActionExecutionProhibited(action)) {
+ // We can't execute an action (e.g. because --check_???_up_to_date option was used). Fail
+ // the build instead.
+ synchronized (reporter) {
+ TargetOutOfDateException e = new TargetOutOfDateException(action);
+ reporter.handle(Event.error(e.getMessage()));
+ recordExecutionError();
+ throw e;
+ }
+ }
+
+ String message = action.getProgressMessage();
+ if (message != null) {
+ reporter.startTask(null, prependExecPhaseStats(message));
+ }
+ statusReporterRef.get().setPreparing(action);
+
+ createOutputDirectories(action);
+
+ prepareScheduleExecuteAndCompleteAction(action, token,
+ actionExecutionContext, actionStartTime);
+ return new ActionExecutionValue(
+ graphFileCache.getOutputData(), graphFileCache.getAdditionalOutputData());
+ } finally {
+ profiler.completeTask(ProfilerTask.ACTION);
+ }
+ }
+ }
+
+ private void createOutputDirectories(Action action) throws ActionExecutionException {
+ try {
+ Set<Path> done = new HashSet<>(); // avoid redundant calls for the same directory.
+ for (Artifact outputFile : action.getOutputs()) {
+ Path outputDir = outputFile.getPath().getParentDirectory();
+ if (done.add(outputDir)) {
+ try {
+ createDirectoryAndParents(outputDir);
+ continue;
+ } catch (IOException e) {
+ /* Fall through to plan B. */
+ }
+
+ // Possibly some direct ancestors are not directories. In that case, we unlink all the
+ // ancestors until we reach a directory, then try again. This handles the case where a
+ // file becomes a directory, either from one build to another, or within a single build.
+ //
+ // Symlinks should not be followed so in order to clean up symlinks pointing to Fileset
+ // outputs from previous builds. See bug [incremental build of Fileset fails if
+ // Fileset.out was changed to be a subdirectory of the old value].
+ try {
+ for (Path p = outputDir; !p.isDirectory(Symlinks.NOFOLLOW);
+ p = p.getParentDirectory()) {
+ // p may be a file or dangling symlink, or a symlink to an old Fileset output
+ p.delete(); // throws IOException
+ }
+ createDirectoryAndParents(outputDir);
+ } catch (IOException e) {
+ throw new ActionExecutionException(
+ "failed to create output directory '" + outputDir + "'", e, action, false);
+ }
+ }
+ }
+ } catch (ActionExecutionException ex) {
+ printError(ex.getMessage(), action, null);
+ throw ex;
+ }
+ }
+
+ private String prependExecPhaseStats(String message) {
+ if (progressSupplier != null) {
+ // Prints a progress message like:
+ // [2608/6445] Compiling foo/bar.cc [host]
+ return progressSupplier.getProgressString() + " " + message;
+ } else {
+ // progressSupplier may be null in tests
+ return message;
+ }
+ }
+
+ /**
+ * Prepare, schedule, execute, and then complete the action.
+ * When this function is called, we know that this action needs to be executed.
+ * This function will prepare for the action's execution (i.e. delete the outputs);
+ * schedule its execution; execute the action;
+ * and then do some post-execution processing to complete the action:
+ * set the outputs readonly and executable, and insert the action results in the
+ * action cache.
+ *
+ * @param action The action to execute
+ * @param token The non-null token returned by dependencyChecker.getTokenIfNeedToExecute()
+ * @param context services in the scope of the action
+ * @param actionStartTime time when we started the first phase of the action execution.
+ * @throws ActionExecutionException if the execution of the specified action
+ * failed for any reason.
+ * @throws InterruptedException if the thread was interrupted.
+ */
+ private void prepareScheduleExecuteAndCompleteAction(Action action, Token token,
+ ActionExecutionContext context, long actionStartTime)
+ throws ActionExecutionException, InterruptedException {
+ Preconditions.checkNotNull(token, action);
+ // Delete the metadataHandler's cache of the action's outputs, since they are being deleted.
+ context.getMetadataHandler().discardMetadata(action.getOutputs());
+ // Delete the outputs before executing the action, just to ensure that
+ // the action really does produce the outputs.
+ try {
+ action.prepare(context.getExecutor().getExecRoot());
+ } catch (IOException e) {
+ reportError("failed to delete output files before executing action", e, action, null);
+ }
+
+ postEvent(new ActionStartedEvent(action, actionStartTime));
+ ResourceSet estimate = action.estimateResourceConsumption(executorEngine);
+ ActionExecutionStatusReporter statusReporter = statusReporterRef.get();
+ try {
+ if (estimate == null || estimate == ResourceSet.ZERO) {
+ statusReporter.setRunningFromBuildData(action);
+ } else {
+ // If estimated resource consumption is null, action will manually call
+ // resource manager when it knows what resources are needed.
+ resourceManager.acquireResources(action, estimate);
+ }
+ boolean outputDumped = executeActionTask(action, context);
+ completeAction(action, token, context.getMetadataHandler(),
+ context.getFileOutErr(), outputDumped);
+ } finally {
+ if (estimate != null) {
+ resourceManager.releaseResources(action, estimate);
+ }
+ statusReporter.remove(action);
+ postEvent(new ActionCompletionEvent(action));
+ }
+ }
+
+ private ActionExecutionException processAndThrow(
+ ActionExecutionException e, Action action, FileOutErr outErrBuffer)
+ throws ActionExecutionException {
+ reportActionExecution(action, e, outErrBuffer);
+ boolean reported = reportErrorIfNotAbortingMode(e, outErrBuffer);
+
+ ActionExecutionException toThrow = e;
+ if (reported){
+ // If we already printed the error for the exception we mark it as already reported
+ // so that we do not print it again in upper levels.
+ // Note that we need to report it here since we want immediate feedback of the errors
+ // and in some cases the upper-level printing mechanism only prints one of the errors.
+ toThrow = new AlreadyReportedActionExecutionException(e);
+ }
+
+ // Now, rethrow the exception.
+ // This can have two effects:
+ // If we're still building, the exception will get retrieved by the
+ // completor and rethrown.
+ // If we're aborting, the exception will never be retrieved from the
+ // completor, since the completor is waiting for all outstanding jobs
+ // to finish. After they have finished, it will only rethrow the
+ // exception that initially caused it to abort will and not check the
+ // exit status of any actions that had finished in the meantime.
+ throw toThrow;
+ }
+
+ /**
+ * Execute the specified action, in a profiler task.
+ * The caller is responsible for having already checked that we need to
+ * execute it and for acquiring/releasing any scheduling locks needed.
+ *
+ * <p>This is thread-safe so long as you don't try to execute the same action
+ * twice at the same time (or overlapping times).
+ * May execute in a worker thread.
+ *
+ * @throws ActionExecutionException if the execution of the specified action
+ * failed for any reason.
+ * @throws InterruptedException if the thread was interrupted.
+ * @return true if the action output was dumped, false otherwise.
+ */
+ private boolean executeActionTask(Action action, ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ profiler.startTask(ProfilerTask.ACTION_EXECUTE, action);
+ // ActionExecutionExceptions that occur as the thread is interrupted are
+ // assumed to be a result of that, so we throw InterruptedException
+ // instead.
+ FileOutErr outErrBuffer = actionExecutionContext.getFileOutErr();
+ try {
+ action.execute(actionExecutionContext);
+
+ // Action terminated fine, now report the output.
+ // The .showOutput() method is not necessarily a quick check: in its
+ // current implementation it uses regular expression matching.
+ if (outErrBuffer.hasRecordedOutput()
+ && (action.showsOutputUnconditionally()
+ || reporter.showOutput(Label.print(action.getOwner().getLabel())))) {
+ dumpRecordedOutErr(action, outErrBuffer);
+ return true;
+ }
+ // Defer reporting action success until outputs are checked
+ } catch (ActionExecutionException e) {
+ processAndThrow(e, action, outErrBuffer);
+ } finally {
+ profiler.completeTask(ProfilerTask.ACTION_EXECUTE);
+ }
+ return false;
+ }
+
+ private void completeAction(Action action, Token token, MetadataHandler metadataHandler,
+ FileOutErr fileOutErr, boolean outputAlreadyDumped) throws ActionExecutionException {
+ try {
+ Preconditions.checkState(action.inputsKnown(),
+ "Action %s successfully executed, but inputs still not known", action);
+
+ profiler.startTask(ProfilerTask.ACTION_COMPLETE, action);
+ try {
+ if (!checkOutputs(action, metadataHandler)) {
+ reportError("not all outputs were created", null, action,
+ outputAlreadyDumped ? null : fileOutErr);
+ }
+ // Prevent accidental stomping on files.
+ // This will also throw a FileNotFoundException
+ // if any of the output files doesn't exist.
+ try {
+ setOutputsReadOnlyAndExecutable(action, metadataHandler);
+ } catch (IOException e) {
+ reportError("failed to set outputs read-only", e, action, null);
+ }
+ try {
+ actionCacheChecker.afterExecution(action, token, metadataHandler);
+ } catch (IOException e) {
+ // Skyframe does all the filesystem access needed during the previous calls, and if those
+ // calls failed, we should already have thrown. So an IOException is impossible here.
+ throw new IllegalStateException(
+ "failed to update action cache for " + action.prettyPrint()
+ + ", but all outputs should already have been checked", e);
+ }
+ } finally {
+ profiler.completeTask(ProfilerTask.ACTION_COMPLETE);
+ }
+ reportActionExecution(action, null, fileOutErr);
+ } catch (ActionExecutionException actionException) {
+ // Success in execution but failure in completion.
+ reportActionExecution(action, actionException, fileOutErr);
+ throw actionException;
+ } catch (IllegalStateException exception) {
+ // More serious internal error, but failure still reported.
+ reportActionExecution(action,
+ new ActionExecutionException(exception, action, true), fileOutErr);
+ throw exception;
+ }
+ }
+
+ /**
+ * For each of the action's outputs that is a regular file (not a symbolic
+ * link or directory), make it read-only and executable.
+ *
+ * <p>Making the outputs read-only helps preventing accidental editing of
+ * them (e.g. in case of generated source code), while making them executable
+ * helps running generated files (such as generated shell scripts) on the
+ * command line.
+ *
+ * <p>May execute in a worker thread.
+ *
+ * <p>Note: setting these bits maintains transparency regarding the locality of the build;
+ * because the remote execution engine sets them, they should be set for local builds too.
+ *
+ * @throws IOException if an I/O error occurred.
+ */
+ private final void setOutputsReadOnlyAndExecutable(Action action, MetadataHandler metadataHandler)
+ throws IOException {
+ Preconditions.checkState(!action.getActionType().isMiddleman());
+
+ for (Artifact output : action.getOutputs()) {
+ Path path = output.getPath();
+ if (metadataHandler.isInjected(output)) {
+ // We trust the files created by the execution-engine to be non symlinks with expected
+ // chmod() settings already applied. The follow stanza implies a total of 6 system calls,
+ // since the UnixFileSystem implementation of setWritable() and setExecutable() both
+ // do a stat() internally.
+ continue;
+ }
+ if (path.isFile(Symlinks.NOFOLLOW)) { // i.e. regular files only.
+ path.setWritable(false);
+ path.setExecutable(true);
+ }
+ }
+ }
+
+ private void reportMissingOutputFile(Action action, Artifact output, Reporter reporter,
+ boolean isSymlink) {
+ boolean genrule = action.getMnemonic().equals("Genrule");
+ String prefix = (genrule ? "declared output '" : "output '") + output.prettyPrint() + "' ";
+ if (isSymlink) {
+ reporter.handle(Event.error(
+ action.getOwner().getLocation(), prefix + "is a dangling symbolic link"));
+ } else {
+ String suffix = genrule ? " by genrule. This is probably "
+ + "because the genrule actually didn't create this output, or because the output was a "
+ + "directory and the genrule was run remotely (note that only the contents of "
+ + "declared file outputs are copied from genrules run remotely)" : "";
+ reporter.handle(Event.error(
+ action.getOwner().getLocation(), prefix + "was not created" + suffix));
+ }
+ }
+
+ /**
+ * Validates that all action outputs were created.
+ *
+ * @return false if some outputs are missing, true - otherwise.
+ */
+ private boolean checkOutputs(Action action, MetadataHandler metadataHandler) {
+ boolean success = true;
+ for (Artifact output : action.getOutputs()) {
+ if (!metadataHandler.artifactExists(output)) {
+ reportMissingOutputFile(action, output, reporter, output.getPath().isSymbolicLink());
+ success = false;
+ }
+ }
+ return success;
+ }
+
+ private void postEvent(Object event) {
+ EventBus bus = eventBus.get();
+ if (bus != null) {
+ bus.post(event);
+ }
+ }
+
+ /**
+ * Convenience function for reporting that the action failed due to a
+ * the exception cause, if there is an additional explanatory message that
+ * clarifies the message of the exception. Combines the user-provided message
+ * and the exceptions' message and reports the combination as error.
+ * Then, throws an ActionExecutionException with the reported error as
+ * message and the provided exception as the cause.
+ *
+ * @param message A small text that explains why the action failed
+ * @param cause The exception that caused the action to fail
+ * @param action The action that failed
+ * @param actionOutput The output of the failed Action.
+ * May be null, if there is no output to display
+ */
+ private void reportError(String message, Throwable cause, Action action, FileOutErr actionOutput)
+ throws ActionExecutionException {
+ ActionExecutionException ex;
+ if (cause == null) {
+ ex = new ActionExecutionException(message, action, false);
+ } else {
+ ex = new ActionExecutionException(message, cause, action, false);
+ }
+ printError(ex.getMessage(), action, actionOutput);
+ throw ex;
+ }
+
+ /**
+ * For the action 'action' that failed due to 'ex' with the output
+ * 'actionOutput', notify the user about the error. To notify the user, the
+ * method first displays the output of the action and then reports an error
+ * via the reporter. The method ensures that the two messages appear next to
+ * each other by locking the outErr object where the output is displayed.
+ *
+ * @param message The reason why the action failed
+ * @param action The action that failed, must not be null.
+ * @param actionOutput The output of the failed Action.
+ * May be null, if there is no output to display
+ */
+ private void printError(String message, Action action, FileOutErr actionOutput) {
+ synchronized (reporter) {
+ if (actionOutput != null && actionOutput.hasRecordedOutput()) {
+ dumpRecordedOutErr(action, actionOutput);
+ }
+ if (keepGoing) {
+ message = "Couldn't " + describeAction(action) + ": " + message;
+ }
+ reporter.handle(Event.error(action.getOwner().getLocation(), message));
+ recordExecutionError();
+ }
+ }
+
+ /** Describe an action, for use in error messages. */
+ private static String describeAction(Action action) {
+ if (action.getOutputs().isEmpty()) {
+ return "run " + action.prettyPrint();
+ } else if (action.getActionType().isMiddleman()) {
+ return "build " + action.prettyPrint();
+ } else {
+ return "build file " + action.getPrimaryOutput().prettyPrint();
+ }
+ }
+
+ /**
+ * Dump the output from the action.
+ *
+ * @param action The action whose output is being dumped
+ * @param outErrBuffer The OutErr that recorded the actions output
+ */
+ private void dumpRecordedOutErr(Action action, FileOutErr outErrBuffer) {
+ StringBuilder message = new StringBuilder("");
+ message.append("From ");
+ message.append(action.describe());
+ message.append(":");
+
+ // Synchronize this on the reporter, so that the output from multiple
+ // actions will not be interleaved.
+ synchronized (reporter) {
+ // Only print the output if we're not winding down.
+ if (isBuilderAborting()) {
+ return;
+ }
+ reporter.handle(Event.info(message.toString()));
+
+ OutErr outErr = this.reporter.getOutErr();
+ outErrBuffer.dumpOutAsLatin1(outErr.getOutputStream());
+ outErrBuffer.dumpErrAsLatin1(outErr.getErrorStream());
+ }
+ }
+
+ private void reportActionExecution(Action action,
+ ActionExecutionException exception, FileOutErr outErr) {
+ String stdout = null;
+ String stderr = null;
+
+ if (outErr.hasRecordedStdout()) {
+ stdout = outErr.getOutputFile().toString();
+ }
+ if (outErr.hasRecordedStderr()) {
+ stderr = outErr.getErrorFile().toString();
+ }
+ postEvent(new ActionExecutedEvent(action, exception, stdout, stderr));
+ }
+
+ /**
+ * Returns true if the exception was reported. False otherwise. Currently this is a copy of what
+ * we did in pre-Skyframe execution. The main implication is that we are printing the error to the
+ * top level reporter instead of the action reporter. Because of that Skyframe values do not know
+ * about the errors happening in the execution phase. Even if we change in the future to log to
+ * the action reporter (that would be done in ActionExecutionFunction.compute() when we get an
+ * ActionExecutionException), we probably do not want to also store the StdErr output, so
+ * dumpRecordedOutErr() should still be called here.
+ */
+ private boolean reportErrorIfNotAbortingMode(ActionExecutionException ex,
+ FileOutErr outErrBuffer) {
+ // For some actions (e.g. many local actions) the pollInterruptedStatus()
+ // won't notice that we had an interrupted job. It will continue.
+ // For that reason we must take care to NOT report errors if we're
+ // in the 'aborting' mode: Any cancelled action would show up here.
+ // For some actions (e.g. many local actions) the pollInterruptedStatus()
+ // won't notice that we had an interrupted job. It will continue.
+ // For that reason we must take care to NOT report errors if we're
+ // in the 'aborting' mode: Any cancelled action would show up here.
+ synchronized (this.reporter) {
+ if (!isBuilderAborting()) {
+ // Oops. The action aborted. Report the problem.
+ printError(ex.getMessage(), ex.getAction(), outErrBuffer);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** An object supplying data for action execution progress reporting. */
+ public interface ProgressSupplier {
+ /** Returns the progress string to prefix action execution messages with. */
+ String getProgressString();
+ }
+
+ /** An object that can be notified about action completion. */
+ public interface ActionCompletedReceiver {
+ /** Receives a completed action. */
+ void actionCompleted(Action action);
+ }
+
+ public void setActionExecutionProgressReportingObjects(
+ @Nullable ProgressSupplier progressSupplier,
+ @Nullable ActionCompletedReceiver completionReceiver) {
+ this.progressSupplier = progressSupplier;
+ this.completionReceiver = completionReceiver;
+ }
+
+ private static class DelegatingPairFileCache implements ActionInputFileCache {
+ private final ActionInputFileCache perActionCache;
+ private final ActionInputFileCache perBuildFileCache;
+
+ private DelegatingPairFileCache(ActionInputFileCache mainCache,
+ ActionInputFileCache perBuildFileCache) {
+ this.perActionCache = mainCache;
+ this.perBuildFileCache = perBuildFileCache;
+ }
+
+ @Override
+ public ByteString getDigest(ActionInput actionInput) throws IOException {
+ ByteString digest = perActionCache.getDigest(actionInput);
+ return digest != null ? digest : perBuildFileCache.getDigest(actionInput);
+ }
+
+ @Override
+ public long getSizeInBytes(ActionInput actionInput) throws IOException {
+ long size = perActionCache.getSizeInBytes(actionInput);
+ return size > -1 ? size : perBuildFileCache.getSizeInBytes(actionInput);
+ }
+
+ @Override
+ public boolean contentsAvailableLocally(ByteString digest) {
+ return perActionCache.contentsAvailableLocally(digest)
+ || perBuildFileCache.contentsAvailableLocally(digest);
+ }
+
+ @Nullable
+ @Override
+ public File getFileFromDigest(ByteString digest) throws IOException {
+ File file = perActionCache.getFileFromDigest(digest);
+ return file != null ? file : perBuildFileCache.getFileFromDigest(digest);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java
new file mode 100644
index 0000000..857c231
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java
@@ -0,0 +1,510 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactFactory;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.actions.ArtifactPrefixConflictException;
+import com.google.devtools.build.lib.actions.MutableActionGraph;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.AnalysisFailureEvent;
+import com.google.devtools.build.lib.analysis.Aspect;
+import com.google.devtools.build.lib.analysis.CachingAnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.ConfiguredTargetFactory;
+import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget;
+import com.google.devtools.build.lib.analysis.ViewCreationFailedException;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey;
+import com.google.devtools.build.lib.analysis.config.BinTools;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
+import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.skyframe.ActionLookupValue.ActionLookupKey;
+import com.google.devtools.build.lib.skyframe.BuildInfoCollectionValue.BuildInfoKeyAndConfig;
+import com.google.devtools.build.lib.skyframe.ConfiguredTargetFunction.ConfiguredValueCreationException;
+import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ConflictException;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.CycleInfo;
+import com.google.devtools.build.skyframe.ErrorInfo;
+import com.google.devtools.build.skyframe.EvaluationProgressReceiver;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.SkyFunction.Environment;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * Skyframe-based driver of analysis.
+ *
+ * <p>Covers enough functionality to work as a substitute for {@code BuildView#configureTargets}.
+ */
+public final class SkyframeBuildView {
+
+ private final ConfiguredTargetFactory factory;
+ private final ArtifactFactory artifactFactory;
+ @Nullable private EventHandler warningListener;
+ private final SkyframeExecutor skyframeExecutor;
+ private final Runnable legacyDataCleaner;
+ private final BinTools binTools;
+ private boolean enableAnalysis = false;
+
+ // This hack allows us to see when a configured target has been invalidated, and thus when the set
+ // of artifact conflicts needs to be recomputed (whenever a configured target has been invalidated
+ // or newly evaluated).
+ private final EvaluationProgressReceiver invalidationReceiver =
+ new ConfiguredTargetValueInvalidationReceiver();
+ private final Set<SkyKey> evaluatedConfiguredTargets = Sets.newConcurrentHashSet();
+ // Used to see if checks of graph consistency need to be done after analysis.
+ private volatile boolean someConfiguredTargetEvaluated = false;
+
+ // We keep the set of invalidated configuration targets so that we can know if something
+ // has been invalidated after graph pruning has been executed.
+ private Set<ConfiguredTargetValue> dirtyConfiguredTargets = Sets.newConcurrentHashSet();
+ private volatile boolean anyConfiguredTargetDeleted = false;
+ private SkyKey configurationKey = null;
+
+ public SkyframeBuildView(ConfiguredTargetFactory factory,
+ ArtifactFactory artifactFactory,
+ SkyframeExecutor skyframeExecutor, Runnable legacyDataCleaner, BinTools binTools) {
+ this.factory = factory;
+ this.artifactFactory = artifactFactory;
+ this.skyframeExecutor = skyframeExecutor;
+ this.legacyDataCleaner = legacyDataCleaner;
+ this.binTools = binTools;
+ skyframeExecutor.setArtifactFactoryAndBinTools(artifactFactory, binTools);
+ }
+
+ public void setWarningListener(@Nullable EventHandler warningListener) {
+ this.warningListener = warningListener;
+ }
+
+ public void setConfigurationSkyKey(SkyKey skyKey) {
+ this.configurationKey = skyKey;
+ }
+
+ public void resetEvaluatedConfiguredTargetKeysSet() {
+ evaluatedConfiguredTargets.clear();
+ }
+
+ public Set<SkyKey> getEvaluatedTargetKeys() {
+ return ImmutableSet.copyOf(evaluatedConfiguredTargets);
+ }
+
+ private void setDeserializedArtifactOwners() throws ViewCreationFailedException {
+ Map<PathFragment, Artifact> deserializedArtifactMap =
+ artifactFactory.getDeserializedArtifacts();
+ Set<Artifact> deserializedArtifacts = new HashSet<>();
+ for (Artifact artifact : deserializedArtifactMap.values()) {
+ if (!artifact.getExecPath().getBaseName().endsWith(".gcda")) {
+ // gcda files are classified as generated artifacts, but are not actually generated. All
+ // others need owners.
+ deserializedArtifacts.add(artifact);
+ }
+ }
+ if (deserializedArtifacts.isEmpty()) {
+ // If there are no deserialized artifacts to process, don't pay the price of iterating over
+ // the graph.
+ return;
+ }
+ for (Map.Entry<SkyKey, ActionLookupValue> entry :
+ skyframeExecutor.getActionLookupValueMap().entrySet()) {
+ for (Action action : entry.getValue().getActionsForFindingArtifactOwners()) {
+ for (Artifact output : action.getOutputs()) {
+ Artifact deserializedArtifact = deserializedArtifactMap.get(output.getExecPath());
+ if (deserializedArtifact != null) {
+ deserializedArtifact.setArtifactOwner((ActionLookupKey) entry.getKey().argument());
+ deserializedArtifacts.remove(deserializedArtifact);
+ }
+ }
+ }
+ }
+ if (!deserializedArtifacts.isEmpty()) {
+ throw new ViewCreationFailedException("These artifacts were read in from the FDO profile but"
+ + " have no generating action that could be found. If you are confident that your profile was"
+ + " collected from the same source state at which you're building, please report this:\n"
+ + Artifact.asExecPaths(deserializedArtifacts));
+ }
+ artifactFactory.clearDeserializedArtifacts();
+ }
+
+ /**
+ * Analyzes the specified targets using Skyframe as the driving framework.
+ *
+ * @return the configured targets that should be built
+ */
+ public Collection<ConfiguredTarget> configureTargets(List<ConfiguredTargetKey> values,
+ EventBus eventBus, boolean keepGoing)
+ throws InterruptedException, ViewCreationFailedException {
+ enableAnalysis(true);
+ EvaluationResult<ConfiguredTargetValue> result;
+ try {
+ result = skyframeExecutor.configureTargets(values, keepGoing);
+ } finally {
+ enableAnalysis(false);
+ }
+ // For Skyframe m1, note that we already reported action conflicts during action registration
+ // in the legacy action graph.
+ ImmutableMap<Action, ConflictException> badActions = skyframeExecutor.findArtifactConflicts();
+
+ // Filter out all CTs that have a bad action and convert to a list of configured targets. This
+ // code ensures that the resulting list of configured targets has the same order as the incoming
+ // list of values, i.e., that the order is deterministic.
+ Collection<ConfiguredTarget> goodCts = Lists.newArrayListWithCapacity(values.size());
+ for (ConfiguredTargetKey value : values) {
+ ConfiguredTargetValue ctValue = result.get(ConfiguredTargetValue.key(value));
+ if (ctValue == null) {
+ continue;
+ }
+ goodCts.add(ctValue.getConfiguredTarget());
+ }
+
+ if (!result.hasError() && badActions.isEmpty()) {
+ setDeserializedArtifactOwners();
+ return goodCts;
+ }
+
+ // --nokeep_going so we fail with an exception for the first error.
+ // TODO(bazel-team): We might want to report the other errors through the event bus but
+ // for keeping this code in parity with legacy we just report the first error for now.
+ if (!keepGoing) {
+ for (Map.Entry<Action, ConflictException> bad : badActions.entrySet()) {
+ ConflictException ex = bad.getValue();
+ try {
+ ex.rethrowTyped();
+ } catch (MutableActionGraph.ActionConflictException ace) {
+ ace.reportTo(skyframeExecutor.getReporter());
+ String errorMsg = "Analysis of target '" + bad.getKey().getOwner().getLabel()
+ + "' failed; build aborted";
+ throw new ViewCreationFailedException(errorMsg);
+ } catch (ArtifactPrefixConflictException apce) {
+ skyframeExecutor.getReporter().handle(Event.error(apce.getMessage()));
+ }
+ throw new ViewCreationFailedException(ex.getMessage());
+ }
+
+ Map.Entry<SkyKey, ErrorInfo> error = result.errorMap().entrySet().iterator().next();
+ SkyKey topLevel = error.getKey();
+ ErrorInfo errorInfo = error.getValue();
+ assertSaneAnalysisError(errorInfo, topLevel);
+ skyframeExecutor.getCyclesReporter().reportCycles(errorInfo.getCycleInfo(), topLevel,
+ skyframeExecutor.getReporter());
+ Throwable cause = errorInfo.getException();
+ Preconditions.checkState(cause != null || !Iterables.isEmpty(errorInfo.getCycleInfo()),
+ errorInfo);
+ String errorMsg = "Analysis of target '" + ConfiguredTargetValue.extractLabel(topLevel)
+ + "' failed; build aborted";
+ throw new ViewCreationFailedException(errorMsg);
+ }
+
+ // --keep_going : We notify the error and return a ConfiguredTargetValue
+ for (Map.Entry<SkyKey, ErrorInfo> errorEntry : result.errorMap().entrySet()) {
+ if (values.contains(errorEntry.getKey().argument())) {
+ SkyKey errorKey = errorEntry.getKey();
+ ConfiguredTargetKey label = (ConfiguredTargetKey) errorKey.argument();
+ ErrorInfo errorInfo = errorEntry.getValue();
+ assertSaneAnalysisError(errorInfo, errorKey);
+
+ skyframeExecutor.getCyclesReporter().reportCycles(errorInfo.getCycleInfo(), errorKey,
+ skyframeExecutor.getReporter());
+ // We try to get the root cause key first from ErrorInfo rootCauses. If we don't have one
+ // we try to use the cycle culprit if the error is a cycle. Otherwise we use the top-level
+ // error key.
+ Label root;
+ if (!Iterables.isEmpty(errorEntry.getValue().getRootCauses())) {
+ SkyKey culprit = Preconditions.checkNotNull(Iterables.getFirst(
+ errorEntry.getValue().getRootCauses(), null));
+ root = ((ConfiguredTargetKey) culprit.argument()).getLabel();
+ } else {
+ root = maybeGetConfiguredTargetCycleCulprit(errorInfo.getCycleInfo());
+ }
+ if (warningListener != null) {
+ warningListener.handle(Event.warn("errors encountered while analyzing target '"
+ + label + "': it will not be built"));
+ }
+ eventBus.post(new AnalysisFailureEvent(
+ LabelAndConfiguration.of(label.getLabel(), label.getConfiguration()), root));
+ }
+ }
+
+ Collection<Exception> reportedExceptions = Sets.newHashSet();
+ for (Map.Entry<Action, ConflictException> bad : badActions.entrySet()) {
+ ConflictException ex = bad.getValue();
+ try {
+ ex.rethrowTyped();
+ } catch (MutableActionGraph.ActionConflictException ace) {
+ ace.reportTo(skyframeExecutor.getReporter());
+ if (warningListener != null) {
+ warningListener.handle(Event.warn("errors encountered while analyzing target '"
+ + bad.getKey().getOwner().getLabel() + "': it will not be built"));
+ }
+ } catch (ArtifactPrefixConflictException apce) {
+ if (reportedExceptions.add(apce)) {
+ skyframeExecutor.getReporter().handle(Event.error(apce.getMessage()));
+ }
+ }
+ }
+
+ if (!badActions.isEmpty()) {
+ // In order to determine the set of configured targets transitively error free from action
+ // conflict issues, we run a post-processing update() that uses the bad action map.
+ EvaluationResult<PostConfiguredTargetValue> actionConflictResult =
+ skyframeExecutor.postConfigureTargets(values, keepGoing, badActions);
+
+ goodCts = Lists.newArrayListWithCapacity(values.size());
+ for (ConfiguredTargetKey value : values) {
+ PostConfiguredTargetValue postCt =
+ actionConflictResult.get(PostConfiguredTargetValue.key(value));
+ if (postCt != null) {
+ goodCts.add(postCt.getCt());
+ }
+ }
+ }
+ setDeserializedArtifactOwners();
+ return goodCts;
+ }
+
+ @Nullable
+ Label maybeGetConfiguredTargetCycleCulprit(Iterable<CycleInfo> cycleInfos) {
+ for (CycleInfo cycleInfo : cycleInfos) {
+ SkyKey culprit = Iterables.getFirst(cycleInfo.getCycle(), null);
+ if (culprit == null) {
+ continue;
+ }
+ if (culprit.functionName().equals(SkyFunctions.CONFIGURED_TARGET)) {
+ return ((LabelAndConfiguration) culprit.argument()).getLabel();
+ }
+ }
+ return null;
+ }
+
+ private static void assertSaneAnalysisError(ErrorInfo errorInfo, SkyKey key) {
+ Throwable cause = errorInfo.getException();
+ if (cause != null) {
+ // We should only be trying to configure targets when the loading phase succeeds, meaning
+ // that the only errors should be analysis errors.
+ Preconditions.checkState(cause instanceof ConfiguredValueCreationException,
+ "%s -> %s", key, errorInfo);
+ }
+ }
+
+ ArtifactFactory getArtifactFactory() {
+ return artifactFactory;
+ }
+
+ @Nullable
+ EventHandler getWarningListener() {
+ return warningListener;
+ }
+
+ /**
+ * Because we don't know what build-info artifacts this configured target may request, we
+ * conservatively register a dep on all of them.
+ */
+ // TODO(bazel-team): Allow analysis to return null so the value builder can exit and wait for a
+ // restart deps are not present.
+ private boolean getWorkspaceStatusValues(Environment env) {
+ env.getValue(WorkspaceStatusValue.SKY_KEY);
+ Map<BuildInfoKey, BuildInfoFactory> buildInfoFactories =
+ PrecomputedValue.BUILD_INFO_FACTORIES.get(env);
+ if (buildInfoFactories == null) {
+ return false;
+ }
+ BuildConfigurationCollection configurations = getBuildConfigurationCollection(env);
+ if (configurations == null) {
+ return false;
+ }
+ // These factories may each create their own build info artifacts, all depending on the basic
+ // build-info.txt and build-changelist.txt.
+ List<SkyKey> depKeys = Lists.newArrayList();
+ for (BuildInfoKey key : buildInfoFactories.keySet()) {
+ for (BuildConfiguration config : configurations.getAllConfigurations()) {
+ if (buildInfoFactories.get(key).isEnabled(config)) {
+ depKeys.add(BuildInfoCollectionValue.key(new BuildInfoKeyAndConfig(key, config)));
+ }
+ }
+ }
+ env.getValues(depKeys);
+ return !env.valuesMissing();
+ }
+
+ /** Returns null if any build-info values are not ready. */
+ @Nullable
+ CachingAnalysisEnvironment createAnalysisEnvironment(ArtifactOwner owner,
+ boolean isSystemEnv, boolean extendedSanityChecks, EventHandler eventHandler,
+ Environment env, boolean allowRegisteringActions) {
+ if (!getWorkspaceStatusValues(env)) {
+ return null;
+ }
+ return new CachingAnalysisEnvironment(
+ artifactFactory, owner, isSystemEnv, extendedSanityChecks, eventHandler, env,
+ allowRegisteringActions, binTools);
+ }
+
+ /**
+ * Invokes the appropriate constructor to create a {@link ConfiguredTarget} instance.
+ *
+ * <p>For use in {@code ConfiguredTargetFunction}.
+ *
+ * <p>Returns null if Skyframe deps are missing or upon certain errors.
+ */
+ @Nullable
+ ConfiguredTarget createConfiguredTarget(Target target, BuildConfiguration configuration,
+ CachingAnalysisEnvironment analysisEnvironment,
+ ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap,
+ Set<ConfigMatchingProvider> configConditions)
+ throws InterruptedException {
+ Preconditions.checkState(enableAnalysis,
+ "Already in execution phase %s %s", target, configuration);
+ return factory.createConfiguredTarget(analysisEnvironment, artifactFactory, target,
+ configuration, prerequisiteMap, configConditions);
+ }
+
+ @Nullable
+ public Aspect createAspect(
+ AnalysisEnvironment env, RuleConfiguredTarget associatedTarget,
+ ConfiguredAspectFactory aspectFactory,
+ ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap,
+ Set<ConfigMatchingProvider> configConditions) {
+ return factory.createAspect(
+ env, associatedTarget, aspectFactory, prerequisiteMap, configConditions);
+ }
+
+ @Nullable
+ private BuildConfigurationCollection getBuildConfigurationCollection(Environment env) {
+ ConfigurationCollectionValue configurationsValue =
+ (ConfigurationCollectionValue) env.getValue(configurationKey);
+ return configurationsValue == null ? null : configurationsValue.getConfigurationCollection();
+ }
+
+ @Nullable
+ SkyframeDependencyResolver createDependencyResolver(Environment env) {
+ BuildConfigurationCollection configurations = getBuildConfigurationCollection(env);
+ return configurations == null ? null : new SkyframeDependencyResolver(env);
+ }
+
+ /**
+ * Workaround to clear all legacy data, like the action graph and the artifact factory. We need
+ * to clear them to avoid conflicts.
+ * TODO(bazel-team): Remove this workaround. [skyframe-execution]
+ */
+ void clearLegacyData() {
+ legacyDataCleaner.run();
+ }
+
+ /**
+ * Hack to invalidate actions in legacy action graph when their values are invalidated in
+ * skyframe.
+ */
+ EvaluationProgressReceiver getInvalidationReceiver() {
+ return invalidationReceiver;
+ }
+
+ /** Clear the invalidated configured targets detected during loading and analysis phases. */
+ public void clearInvalidatedConfiguredTargets() {
+ dirtyConfiguredTargets = Sets.newConcurrentHashSet();
+ anyConfiguredTargetDeleted = false;
+ }
+
+ public boolean isSomeConfiguredTargetInvalidated() {
+ return anyConfiguredTargetDeleted || !dirtyConfiguredTargets.isEmpty();
+ }
+
+ /**
+ * Called from SkyframeExecutor to see whether the graph needs to be checked for artifact
+ * conflicts. Returns true if some configured target has been evaluated since the last time the
+ * graph was checked for artifact conflicts (with that last time marked by a call to
+ * {@link #resetEvaluatedConfiguredTargetFlag()}).
+ */
+ boolean isSomeConfiguredTargetEvaluated() {
+ Preconditions.checkState(!enableAnalysis);
+ return someConfiguredTargetEvaluated;
+ }
+
+ /**
+ * Called from SkyframeExecutor after the graph is checked for artifact conflicts so that
+ * the next time {@link #isSomeConfiguredTargetEvaluated} is called, it will return true only if
+ * some configured target has been evaluated since the last check for artifact conflicts.
+ */
+ void resetEvaluatedConfiguredTargetFlag() {
+ someConfiguredTargetEvaluated = false;
+ }
+
+ /**
+ * {@link #createConfiguredTarget} will only create configured targets if this is set to true. It
+ * should be set to true before any Skyframe update call that might call into {@link
+ * #createConfiguredTarget}, and false immediately after the call. Use it to fail-fast in the case
+ * that a target is requested for analysis not during the analysis phase.
+ */
+ void enableAnalysis(boolean enable) {
+ this.enableAnalysis = enable;
+ }
+
+ private class ConfiguredTargetValueInvalidationReceiver implements EvaluationProgressReceiver {
+ @Override
+ public void invalidated(SkyValue value, InvalidationState state) {
+ if (value instanceof ConfiguredTargetValue) {
+ ConfiguredTargetValue ctValue = (ConfiguredTargetValue) value;
+ // If the value was just dirtied and not deleted, then it may not be truly invalid, since
+ // it may later get re-validated.
+ if (state == InvalidationState.DELETED) {
+ anyConfiguredTargetDeleted = true;
+ } else {
+ dirtyConfiguredTargets.add(ctValue);
+ }
+ }
+ }
+
+ @Override
+ public void enqueueing(SkyKey skyKey) {}
+
+ @Override
+ public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) {
+ if (skyKey.functionName() == SkyFunctions.CONFIGURED_TARGET) {
+ if (state == EvaluationState.BUILT) {
+ evaluatedConfiguredTargets.add(skyKey);
+ // During multithreaded operation, this is only set to true, so no concurrency issues.
+ someConfiguredTargetEvaluated = true;
+ }
+ Preconditions.checkNotNull(value, "%s %s", skyKey, state);
+ ConfiguredTargetValue ctValue = (ConfiguredTargetValue) value;
+ dirtyConfiguredTargets.remove(ctValue);
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeDependencyResolver.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeDependencyResolver.java
new file mode 100644
index 0000000..1c7cbfa
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeDependencyResolver.java
@@ -0,0 +1,68 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.analysis.DependencyResolver;
+import com.google.devtools.build.lib.analysis.TargetAndConfiguration;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunction.Environment;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import javax.annotation.Nullable;
+
+/**
+ * A dependency resolver for use within Skyframe. Loads packages lazily when possible.
+ */
+public final class SkyframeDependencyResolver extends DependencyResolver {
+
+ private final Environment env;
+
+ public SkyframeDependencyResolver(Environment env) {
+ this.env = env;
+ }
+
+ @Override
+ protected void invalidVisibilityReferenceHook(TargetAndConfiguration value, Label label) {
+ env.getListener().handle(
+ Event.error(TargetUtils.getLocationMaybe(value.getTarget()), String.format(
+ "Label '%s' in visibility attribute does not refer to a package group", label)));
+ }
+
+ @Override
+ protected void invalidPackageGroupReferenceHook(TargetAndConfiguration value, Label label) {
+ env.getListener().handle(
+ Event.error(TargetUtils.getLocationMaybe(value.getTarget()), String.format(
+ "label '%s' does not refer to a package group", label)));
+ }
+
+ @Nullable
+ @Override
+ protected Target getTarget(Label label) throws NoSuchThingException {
+ if (env.getValue(TargetMarkerValue.key(label)) == null) {
+ return null;
+ }
+ SkyKey key = PackageValue.key(label.getPackageIdentifier());
+ SkyValue value = env.getValue(key);
+ if (value == null) {
+ return null;
+ }
+ PackageValue packageValue = (PackageValue) value;
+ return packageValue.getPackage().getTarget(label.getName());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
new file mode 100644
index 0000000..29b4c23
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
@@ -0,0 +1,1476 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionCacheChecker;
+import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter;
+import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.actions.ActionLogBufferPathGenerator;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactFactory;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceManager;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.Aspect;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.BuildView.Options;
+import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.DependencyResolver.Dependency;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey;
+import com.google.devtools.build.lib.analysis.config.BinTools;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationKey;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigurationFactory;
+import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Preprocessor;
+import com.google.devtools.build.lib.packages.RuleVisibility;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
+import com.google.devtools.build.lib.pkgcache.PackageManager;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader;
+import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ActionCompletedReceiver;
+import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ProgressSupplier;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.ResourceUsage;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.BatchStat;
+import com.google.devtools.build.lib.vfs.ModifiedFileSet;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+import com.google.devtools.build.skyframe.BuildDriver;
+import com.google.devtools.build.skyframe.CycleInfo;
+import com.google.devtools.build.skyframe.CyclesReporter;
+import com.google.devtools.build.skyframe.Differencer;
+import com.google.devtools.build.skyframe.ErrorInfo;
+import com.google.devtools.build.skyframe.EvaluationProgressReceiver;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.Injectable;
+import com.google.devtools.build.skyframe.MemoizingEvaluator;
+import com.google.devtools.build.skyframe.MemoizingEvaluator.EvaluatorSupplier;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+
+/**
+ * A helper object to support Skyframe-driven execution.
+ *
+ * <p>This object is mostly used to inject external state, such as the executor engine or
+ * some additional artifacts (workspace status and build info artifacts) into SkyFunctions
+ * for use during the build.
+ */
+public abstract class SkyframeExecutor {
+ private final EvaluatorSupplier evaluatorSupplier;
+ protected MemoizingEvaluator memoizingEvaluator;
+ private final MemoizingEvaluator.EmittedEventState emittedEventState =
+ new MemoizingEvaluator.EmittedEventState();
+ protected final Reporter reporter;
+ private final PackageFactory pkgFactory;
+ private final WorkspaceStatusAction.Factory workspaceStatusActionFactory;
+ private final BlazeDirectories directories;
+ @Nullable
+ private BatchStat batchStatter;
+
+ // TODO(bazel-team): Figure out how to handle value builders that block internally. Blocking
+ // operations may need to be handled in another (bigger?) thread pool. Also, we should detect
+ // the number of cores and use that as the thread-pool size for CPU-bound operations.
+ // I just bumped this to 200 to get reasonable execution phase performance; that may cause
+ // significant overhead for CPU-bound processes (i.e. analysis). [skyframe-analysis]
+ @VisibleForTesting
+ public static final int DEFAULT_THREAD_COUNT = 200;
+
+ // Stores Packages between reruns of the PackageFunction (because of missing dependencies,
+ // within the same evaluate() run) to avoid loading the same package twice (first time loading
+ // to find subincludes and declare value dependencies).
+ // TODO(bazel-team): remove this cache once we have skyframe-native package loading
+ // [skyframe-loading]
+ private final ConcurrentMap<PackageIdentifier, Package.LegacyBuilder> packageFunctionCache =
+ Maps.newConcurrentMap();
+ private final AtomicInteger numPackagesLoaded = new AtomicInteger(0);
+
+ protected SkyframeBuildView skyframeBuildView;
+ private EventHandler errorEventListener;
+ private ActionLogBufferPathGenerator actionLogBufferPathGenerator;
+
+ protected BuildDriver buildDriver;
+
+ // AtomicReferences are used here as mutable boxes shared with value builders.
+ private final AtomicBoolean showLoadingProgress = new AtomicBoolean();
+ protected final AtomicReference<UnixGlob.FilesystemCalls> syscalls =
+ new AtomicReference<>(UnixGlob.DEFAULT_SYSCALLS);
+ protected final AtomicReference<PathPackageLocator> pkgLocator =
+ new AtomicReference<>();
+ protected final AtomicReference<ImmutableSet<String>> deletedPackages =
+ new AtomicReference<>(ImmutableSet.<String>of());
+ private final AtomicReference<EventBus> eventBus = new AtomicReference<>();
+
+ private final ImmutableList<BuildInfoFactory> buildInfoFactories;
+ // Under normal circumstances, the artifact factory persists for the life of a Blaze server, but
+ // since it is not yet created when we create the value builders, we have to use a supplier,
+ // initialized when the build view is created.
+ private final MutableSupplier<ArtifactFactory> artifactFactory = new MutableSupplier<>();
+ // Used to give to WriteBuildInfoAction via a supplier. Relying on BuildVariableValue.BUILD_ID
+ // would be preferable, but we have no way to have the Action depend on that value directly.
+ // Having the BuildInfoFunction own the supplier is currently not possible either, because then
+ // it would be invalidated on every build, since it would depend on the build id value.
+ private MutableSupplier<UUID> buildId = new MutableSupplier<>();
+
+ protected boolean active = true;
+ private final PackageManager packageManager;
+
+ private final Preprocessor.Factory.Supplier preprocessorFactorySupplier;
+ private Preprocessor.Factory preprocessorFactory;
+
+ protected final TimestampGranularityMonitor tsgm;
+
+ private final ResourceManager resourceManager;
+
+ /** Used to lock evaluator on legacy calls to get existing values. */
+ private final Object valueLookupLock = new Object();
+ private final AtomicReference<ActionExecutionStatusReporter> statusReporterRef =
+ new AtomicReference<>();
+ private final SkyframeActionExecutor skyframeActionExecutor;
+ protected SkyframeProgressReceiver progressReceiver;
+ private final AtomicReference<CyclesReporter> cyclesReporter = new AtomicReference<>();
+
+ private final Set<Path> immutableDirectories;
+
+ private BinTools binTools = null;
+ private boolean needToInjectEmbeddedArtifacts = true;
+ private boolean needToInjectPrecomputedValuesForAnalysis = true;
+ protected int modifiedFiles;
+ private final Predicate<PathFragment> allowedMissingInputs;
+
+ private final ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions;
+ private final ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues;
+
+ protected SkyframeIncrementalBuildMonitor incrementalBuildMonitor =
+ new SkyframeIncrementalBuildMonitor();
+
+ private MutableSupplier<ConfigurationFactory> configurationFactory = new MutableSupplier<>();
+ private MutableSupplier<Map<String, String>> clientEnv = new MutableSupplier<>();
+ private MutableSupplier<ImmutableList<ConfigurationFragmentFactory>> configurationFragments =
+ new MutableSupplier<>();
+ private MutableSupplier<Set<Package>> configurationPackages = new MutableSupplier<>();
+ private SkyKey configurationSkyKey = null;
+
+ private static final Logger LOG = Logger.getLogger(SkyframeExecutor.class.getName());
+
+ protected SkyframeExecutor(
+ Reporter reporter,
+ EvaluatorSupplier evaluatorSupplier,
+ PackageFactory pkgFactory,
+ TimestampGranularityMonitor tsgm,
+ BlazeDirectories directories,
+ Factory workspaceStatusActionFactory,
+ ImmutableList<BuildInfoFactory> buildInfoFactories,
+ Set<Path> immutableDirectories,
+ Predicate<PathFragment> allowedMissingInputs,
+ Preprocessor.Factory.Supplier preprocessorFactorySupplier,
+ ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions,
+ ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) {
+ // Strictly speaking, these arguments are not required for initialization, but all current
+ // callsites have them at hand, so we might as well set them during construction.
+ this.reporter = Preconditions.checkNotNull(reporter);
+ this.evaluatorSupplier = evaluatorSupplier;
+ this.pkgFactory = pkgFactory;
+ this.pkgFactory.setSyscalls(syscalls);
+ this.tsgm = tsgm;
+ this.workspaceStatusActionFactory = workspaceStatusActionFactory;
+ this.packageManager = new SkyframePackageManager(
+ new SkyframePackageLoader(), new SkyframeTransitivePackageLoader(),
+ new SkyframeTargetPatternEvaluator(this), syscalls, cyclesReporter, pkgLocator,
+ numPackagesLoaded, this);
+ this.errorEventListener = this.reporter;
+ this.resourceManager = ResourceManager.instance();
+ this.skyframeActionExecutor = new SkyframeActionExecutor(reporter, resourceManager, eventBus,
+ statusReporterRef);
+ this.directories = Preconditions.checkNotNull(directories);
+ this.buildInfoFactories = buildInfoFactories;
+ this.immutableDirectories = immutableDirectories;
+ this.allowedMissingInputs = allowedMissingInputs;
+ this.preprocessorFactorySupplier = preprocessorFactorySupplier;
+ this.extraSkyFunctions = extraSkyFunctions;
+ this.extraPrecomputedValues = extraPrecomputedValues;
+ }
+
+ private ImmutableMap<SkyFunctionName, SkyFunction> skyFunctions(
+ Root buildDataDirectory,
+ PackageFactory pkgFactory,
+ Predicate<PathFragment> allowedMissingInputs) {
+ ExternalFilesHelper externalFilesHelper = new ExternalFilesHelper(pkgLocator,
+ immutableDirectories);
+ // We use an immutable map builder for the nice side effect that it throws if a duplicate key
+ // is inserted.
+ ImmutableMap.Builder<SkyFunctionName, SkyFunction> map = ImmutableMap.builder();
+ map.put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction());
+ map.put(SkyFunctions.FILE_STATE, new FileStateFunction(tsgm, externalFilesHelper));
+ map.put(SkyFunctions.DIRECTORY_LISTING_STATE,
+ new DirectoryListingStateFunction(externalFilesHelper));
+ map.put(SkyFunctions.FILE_SYMLINK_CYCLE_UNIQUENESS,
+ new FileSymlinkCycleUniquenessFunction());
+ map.put(SkyFunctions.FILE, new FileFunction(pkgLocator, externalFilesHelper));
+ map.put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction());
+ map.put(SkyFunctions.PACKAGE_LOOKUP, new PackageLookupFunction(deletedPackages));
+ map.put(SkyFunctions.CONTAINING_PACKAGE_LOOKUP, new ContainingPackageLookupFunction());
+ map.put(SkyFunctions.AST_FILE_LOOKUP, new ASTFileLookupFunction(
+ pkgLocator, packageManager, pkgFactory.getRuleClassProvider()));
+ map.put(SkyFunctions.SKYLARK_IMPORTS_LOOKUP, new SkylarkImportLookupFunction(
+ pkgFactory.getRuleClassProvider(), pkgFactory));
+ map.put(SkyFunctions.GLOB, new GlobFunction());
+ map.put(SkyFunctions.TARGET_PATTERN, new TargetPatternFunction(pkgLocator));
+ map.put(SkyFunctions.RECURSIVE_PKG, new RecursivePkgFunction());
+ map.put(SkyFunctions.PACKAGE, new PackageFunction(
+ reporter, pkgFactory, packageManager, showLoadingProgress, packageFunctionCache,
+ eventBus, numPackagesLoaded));
+ map.put(SkyFunctions.TARGET_MARKER, new TargetMarkerFunction());
+ map.put(SkyFunctions.TRANSITIVE_TARGET, new TransitiveTargetFunction());
+ map.put(SkyFunctions.CONFIGURED_TARGET,
+ new ConfiguredTargetFunction(new BuildViewProvider()));
+ map.put(SkyFunctions.ASPECT, new AspectFunction(new BuildViewProvider()));
+ map.put(SkyFunctions.POST_CONFIGURED_TARGET,
+ new PostConfiguredTargetFunction(new BuildViewProvider()));
+ map.put(SkyFunctions.CONFIGURATION_COLLECTION, new ConfigurationCollectionFunction(
+ configurationFactory, clientEnv, configurationPackages));
+ map.put(SkyFunctions.CONFIGURATION_FRAGMENT, new ConfigurationFragmentFunction(
+ configurationFragments, configurationPackages));
+ map.put(SkyFunctions.WORKSPACE_FILE, new WorkspaceFileFunction(pkgFactory));
+ map.put(SkyFunctions.TARGET_COMPLETION, new TargetCompletionFunction(eventBus));
+ map.put(SkyFunctions.TEST_COMPLETION, new TestCompletionFunction());
+ map.put(SkyFunctions.ARTIFACT, new ArtifactFunction(allowedMissingInputs));
+ map.put(SkyFunctions.BUILD_INFO_COLLECTION, new BuildInfoCollectionFunction(artifactFactory,
+ buildDataDirectory));
+ map.put(SkyFunctions.BUILD_INFO, new WorkspaceStatusFunction());
+ map.put(SkyFunctions.COVERAGE_REPORT, new CoverageReportFunction());
+ map.put(SkyFunctions.ACTION_EXECUTION,
+ new ActionExecutionFunction(skyframeActionExecutor, tsgm));
+ map.put(SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL,
+ new RecursiveFilesystemTraversalFunction());
+ map.put(SkyFunctions.FILESET_ENTRY, new FilesetEntryFunction());
+ map.putAll(extraSkyFunctions);
+ return map.build();
+ }
+
+ @ThreadCompatible
+ public void setActive(boolean active) {
+ this.active = active;
+ }
+
+ protected void checkActive() {
+ Preconditions.checkState(active);
+ }
+
+ public void setFileCache(ActionInputFileCache fileCache) {
+ this.skyframeActionExecutor.setFileCache(fileCache);
+ }
+
+ public void dump(boolean summarize, PrintStream out) {
+ memoizingEvaluator.dump(summarize, out);
+ }
+
+ public abstract void dumpPackages(PrintStream out);
+
+ public void setBatchStatter(@Nullable BatchStat batchStatter) {
+ this.batchStatter = batchStatter;
+ }
+
+ /**
+ * Notify listeners about changed files, and release any associated memory afterwards.
+ */
+ public void drainChangedFiles() {
+ incrementalBuildMonitor.alertListeners(getEventBus());
+ incrementalBuildMonitor = null;
+ }
+
+ @VisibleForTesting
+ public BuildDriver getDriverForTesting() {
+ return buildDriver;
+ }
+
+ /**
+ * This method exists only to allow a module to make a top-level Skyframe call during the
+ * transition to making it fully Skyframe-compatible. Do not add additional callers!
+ */
+ public <E extends Exception> SkyValue evaluateSkyKeyForCodeMigration(final SkyKey key,
+ final Class<E> clazz) throws E {
+ try {
+ return callUninterruptibly(new Callable<SkyValue>() {
+ @Override
+ public SkyValue call() throws E, InterruptedException {
+ synchronized (valueLookupLock) {
+ // We evaluate in keepGoing mode because in the case that the graph does not store its
+ // edges, nokeepGoing builds are not allowed, whereas keepGoing builds are always
+ // permitted.
+ EvaluationResult<ActionLookupValue> result = buildDriver.evaluate(
+ ImmutableList.of(key), true, ResourceUsage.getAvailableProcessors(),
+ errorEventListener);
+ if (!result.hasError()) {
+ return Preconditions.checkNotNull(result.get(key), "%s %s", result, key);
+ }
+ ErrorInfo errorInfo = Preconditions.checkNotNull(result.getError(key),
+ "%s %s", key, result);
+ Throwables.propagateIfPossible(errorInfo.getException(), clazz);
+ if (errorInfo.getException() != null) {
+ throw new IllegalStateException(errorInfo.getException());
+ }
+ throw new IllegalStateException(errorInfo.toString());
+ }
+ }
+ });
+ } catch (Exception e) {
+ Throwables.propagateIfPossible(e, clazz);
+ throw new IllegalStateException(e);
+ }
+ }
+
+ class BuildViewProvider {
+ /**
+ * Returns the current {@link SkyframeBuildView} instance.
+ */
+ SkyframeBuildView getSkyframeBuildView() {
+ return skyframeBuildView;
+ }
+ }
+
+ /**
+ * Must be called before the {@link SkyframeExecutor} can be used (should only be called in
+ * factory methods and as an implementation detail of {@link #resetEvaluator}).
+ */
+ protected void init() {
+ progressReceiver = new SkyframeProgressReceiver();
+ Map<SkyFunctionName, SkyFunction> skyFunctions = skyFunctions(
+ directories.getBuildDataDirectory(), pkgFactory, allowedMissingInputs);
+ memoizingEvaluator = evaluatorSupplier.create(
+ skyFunctions, evaluatorDiffer(), progressReceiver, emittedEventState,
+ hasIncrementalState());
+ buildDriver = newBuildDriver();
+ }
+
+ /**
+ * Reinitializes the Skyframe evaluator, dropping all previously computed values.
+ *
+ * <p>Be careful with this method as it also deletes all injected values. You need to make sure
+ * that any necessary precomputed values are reinjected before the next build. Constants can be
+ * put in {@link #reinjectConstantValuesLazily}.
+ */
+ public void resetEvaluator() {
+ init();
+ emittedEventState.clear();
+ if (skyframeBuildView != null) {
+ skyframeBuildView.clearLegacyData();
+ }
+ reinjectConstantValuesLazily();
+ }
+
+ protected abstract Differencer evaluatorDiffer();
+
+ protected abstract BuildDriver newBuildDriver();
+
+ /**
+ * Values whose values are known at startup and guaranteed constant are still wiped from the
+ * evaluator when we create a new one, so they must be re-injected each time we create a new
+ * evaluator.
+ */
+ private void reinjectConstantValuesLazily() {
+ needToInjectEmbeddedArtifacts = true;
+ needToInjectPrecomputedValuesForAnalysis = true;
+ }
+
+ /**
+ * Deletes all ConfiguredTarget values from the Skyframe cache. This is done to save memory (e.g.
+ * on a configuration change); since the configuration is part of the key, these key/value pairs
+ * will be sitting around doing nothing until the configuration changes back to the previous
+ * value.
+ *
+ * <p>The next evaluation will delete all invalid values.
+ */
+ public abstract void dropConfiguredTargets();
+
+ /**
+ * Removes ConfigurationFragmentValuess and ConfigurationCollectionValues from the cache.
+ */
+ @VisibleForTesting
+ public void invalidateConfigurationCollection() {
+ invalidate(SkyFunctionName.functionIsIn(ImmutableSet.of(SkyFunctions.CONFIGURATION_FRAGMENT,
+ SkyFunctions.CONFIGURATION_COLLECTION)));
+ }
+
+ /**
+ * Decides if graph edges should be stored for this build. If not, re-creates the graph to not
+ * store graph edges. Necessary conditions to not store graph edges are:
+ * (1) batch (since incremental builds are not possible);
+ * (2) skyframe build (since otherwise the memory savings are too slight to bother);
+ * (3) keep-going (since otherwise bubbling errors up may require edges of done nodes);
+ * (4) discard_analysis_cache (since otherwise user isn't concerned about saving memory this way).
+ */
+ public void decideKeepIncrementalState(boolean batch, Options viewOptions) {
+ // Assume incrementality.
+ }
+
+ public boolean hasIncrementalState() {
+ return true;
+ }
+
+ @VisibleForTesting
+ protected abstract Injectable injectable();
+
+ /**
+ * Saves memory by clearing analysis objects from Skyframe. If using legacy execution, actually
+ * deletes the relevant values. If using Skyframe execution, clears their data without deleting
+ * them (they will be deleted on the next build).
+ */
+ public abstract void clearAnalysisCache(Collection<ConfiguredTarget> topLevelTargets);
+
+ /**
+ * Injects the contents of the computed tools/defaults package.
+ */
+ @VisibleForTesting
+ public void setupDefaultPackage(String defaultsPackageContents) {
+ PrecomputedValue.DEFAULTS_PACKAGE_CONTENTS.set(injectable(), defaultsPackageContents);
+ }
+
+ /**
+ * Injects the top-level artifact options.
+ */
+ public void injectTopLevelContext(TopLevelArtifactContext options) {
+ PrecomputedValue.TOP_LEVEL_CONTEXT.set(injectable(), options);
+ }
+
+ public void injectWorkspaceStatusData() {
+ PrecomputedValue.WORKSPACE_STATUS_KEY.set(injectable(),
+ workspaceStatusActionFactory.createWorkspaceStatusAction(
+ artifactFactory.get(), WorkspaceStatusValue.ARTIFACT_OWNER, buildId));
+ }
+
+ public void injectCoverageReportData(Action action) {
+ PrecomputedValue.COVERAGE_REPORT_KEY.set(injectable(), action);
+ }
+
+ /**
+ * Sets the default visibility.
+ */
+ private void setDefaultVisibility(RuleVisibility defaultVisibility) {
+ PrecomputedValue.DEFAULT_VISIBILITY.set(injectable(), defaultVisibility);
+ }
+
+ private void maybeInjectPrecomputedValuesForAnalysis() {
+ if (needToInjectPrecomputedValuesForAnalysis) {
+ injectBuildInfoFactories();
+ injectExtraPrecomputedValues();
+ needToInjectPrecomputedValuesForAnalysis = false;
+ }
+ }
+
+ private void injectExtraPrecomputedValues() {
+ for (PrecomputedValue.Injected injected : extraPrecomputedValues) {
+ injected.inject(injectable());
+ }
+ }
+
+ /**
+ * Injects the build info factory map that will be used when constructing build info
+ * actions/artifacts. Unchanged across the life of the Blaze server, although it must be injected
+ * each time the evaluator is created.
+ */
+ private void injectBuildInfoFactories() {
+ ImmutableMap.Builder<BuildInfoKey, BuildInfoFactory> factoryMapBuilder =
+ ImmutableMap.builder();
+ for (BuildInfoFactory factory : buildInfoFactories) {
+ factoryMapBuilder.put(factory.getKey(), factory);
+ }
+ PrecomputedValue.BUILD_INFO_FACTORIES.set(injectable(), factoryMapBuilder.build());
+ }
+
+ private void setShowLoadingProgress(boolean showLoadingProgressValue) {
+ showLoadingProgress.set(showLoadingProgressValue);
+ }
+
+ @VisibleForTesting
+ public void setCommandId(UUID commandId) {
+ PrecomputedValue.BUILD_ID.set(injectable(), commandId);
+ buildId.set(commandId);
+ }
+
+ /** Returns the build-info.txt and build-changelist.txt artifacts. */
+ public Collection<Artifact> getWorkspaceStatusArtifacts() throws InterruptedException {
+ // Should already be present, unless the user didn't request any targets for analysis.
+ EvaluationResult<WorkspaceStatusValue> result = buildDriver.evaluate(
+ ImmutableList.of(WorkspaceStatusValue.SKY_KEY), /*keepGoing=*/true, /*numThreads=*/1,
+ reporter);
+ WorkspaceStatusValue value =
+ Preconditions.checkNotNull(result.get(WorkspaceStatusValue.SKY_KEY));
+ return ImmutableList.of(value.getStableArtifact(), value.getVolatileArtifact());
+ }
+
+ // TODO(bazel-team): Make this take a PackageIdentifier.
+ public Map<PathFragment, Root> getArtifactRoots(Iterable<PathFragment> execPaths) {
+ final List<SkyKey> packageKeys = new ArrayList<>();
+ for (PathFragment execPath : execPaths) {
+ Preconditions.checkArgument(!execPath.isAbsolute(), execPath);
+ packageKeys.add(ContainingPackageLookupValue.key(
+ PackageIdentifier.createInDefaultRepo(execPath)));
+ }
+
+ EvaluationResult<ContainingPackageLookupValue> result;
+ try {
+ result = callUninterruptibly(new Callable<EvaluationResult<ContainingPackageLookupValue>>() {
+ @Override
+ public EvaluationResult<ContainingPackageLookupValue> call() throws InterruptedException {
+ return buildDriver.evaluate(packageKeys, /*keepGoing=*/true, /*numThreads=*/1, reporter);
+ }
+ });
+ } catch (Exception e) {
+ throw new IllegalStateException(e); // Should never happen.
+ }
+
+ Map<PathFragment, Root> roots = new HashMap<>();
+ for (PathFragment execPath : execPaths) {
+ ContainingPackageLookupValue value = result.get(ContainingPackageLookupValue.key(
+ PackageIdentifier.createInDefaultRepo(execPath)));
+ if (value.hasContainingPackage()) {
+ roots.put(execPath, Root.asSourceRoot(value.getContainingPackageRoot()));
+ } else {
+ roots.put(execPath, null);
+ }
+ }
+ return roots;
+ }
+
+ @VisibleForTesting
+ public WorkspaceStatusAction getLastWorkspaceStatusActionForTesting() {
+ PrecomputedValue value = (PrecomputedValue) buildDriver.getGraphForTesting()
+ .getExistingValueForTesting(PrecomputedValue.WORKSPACE_STATUS_KEY.getKeyForTesting());
+ return (WorkspaceStatusAction) value.get();
+ }
+
+ /**
+ * Informs user about number of modified files (source and output files).
+ */
+ // Note, that number of modified files in some cases can be bigger than actual number of
+ // modified files for targets in current request. Skyframe may check for modification all files
+ // from previous requests.
+ protected void informAboutNumberOfModifiedFiles() {
+ LOG.info(String.format("Found %d modified files from last build", modifiedFiles));
+ }
+
+ public Reporter getReporter() {
+ return reporter;
+ }
+
+ public EventBus getEventBus() {
+ return eventBus.get();
+ }
+
+ /**
+ * The map from package names to the package root where each package was found; this is used to
+ * set up the symlink tree.
+ */
+ public ImmutableMap<PackageIdentifier, Path> getPackageRoots() {
+ // Make a map of the package names to their root paths.
+ ImmutableMap.Builder<PackageIdentifier, Path> packageRoots = ImmutableMap.builder();
+ for (Package pkg : configurationPackages.get()) {
+ packageRoots.put(pkg.getPackageIdentifier(), pkg.getSourceRoot());
+ }
+ return packageRoots.build();
+ }
+
+ @VisibleForTesting
+ ImmutableList<Path> getPathEntries() {
+ return pkgLocator.get().getPathEntries();
+ }
+
+ protected abstract void invalidate(Predicate<SkyKey> pred);
+
+ protected static Iterable<SkyKey> getSkyKeysPotentiallyAffected(
+ Iterable<PathFragment> modifiedSourceFiles, final Path pathEntry) {
+ // TODO(bazel-team): change ModifiedFileSet to work with RootedPaths instead of PathFragments.
+ Iterable<SkyKey> fileStateSkyKeys = Iterables.transform(modifiedSourceFiles,
+ new Function<PathFragment, SkyKey>() {
+ @Override
+ public SkyKey apply(PathFragment pathFragment) {
+ Preconditions.checkState(!pathFragment.isAbsolute(),
+ "found absolute PathFragment: %s", pathFragment);
+ return FileStateValue.key(RootedPath.toRootedPath(pathEntry, pathFragment));
+ }
+ });
+ // TODO(bazel-team): Strictly speaking, we only need to invalidate directory values when a file
+ // has been created or deleted, not when it has been modified. Unfortunately we
+ // do not have that information here, although fancy filesystems could provide it with a
+ // hypothetically modified DiffAwareness interface.
+ // TODO(bazel-team): Even if we don't have that information, we could avoid invalidating
+ // directories when the state of a file does not change by statting them and comparing
+ // the new filetype (nonexistent/file/symlink/directory) with the old one.
+ Iterable<SkyKey> dirListingStateSkyKeys = Iterables.transform(
+ modifiedSourceFiles,
+ new Function<PathFragment, SkyKey>() {
+ @Override
+ public SkyKey apply(PathFragment pathFragment) {
+ Preconditions.checkState(!pathFragment.isAbsolute(),
+ "found absolute PathFragment: %s", pathFragment);
+ return DirectoryListingStateValue.key(RootedPath.toRootedPath(pathEntry,
+ pathFragment.getParentDirectory()));
+ }
+ });
+ return Iterables.concat(fileStateSkyKeys, dirListingStateSkyKeys);
+ }
+
+ protected static SkyKey createFileStateKey(RootedPath rootedPath) {
+ return FileStateValue.key(rootedPath);
+ }
+
+ protected static SkyKey createDirectoryListingStateKey(RootedPath rootedPath) {
+ return DirectoryListingStateValue.key(rootedPath);
+ }
+
+ /**
+ * Creates a FileValue pointing of type directory. No matter that the rootedPath points to a
+ * symlink.
+ *
+ * <p> Use it with caution as it would prevent invalidation when the destination file in the
+ * symlink changes.
+ */
+ protected static FileValue createFileDirValue(RootedPath rootedPath) {
+ return FileValue.value(rootedPath, FileStateValue.DIRECTORY_FILE_STATE_NODE,
+ rootedPath, FileStateValue.DIRECTORY_FILE_STATE_NODE);
+ }
+
+ /**
+ * Sets the packages that should be treated as deleted and ignored.
+ */
+ @VisibleForTesting // productionVisibility = Visibility.PRIVATE
+ public abstract void setDeletedPackages(Iterable<String> pkgs);
+
+ /**
+ * Prepares the evaluator for loading.
+ *
+ * <p>MUST be run before every incremental build.
+ */
+ @VisibleForTesting // productionVisibility = Visibility.PRIVATE
+ public void preparePackageLoading(PathPackageLocator pkgLocator, RuleVisibility defaultVisibility,
+ boolean showLoadingProgress,
+ String defaultsPackageContents, UUID commandId) {
+ Preconditions.checkNotNull(pkgLocator);
+ setActive(true);
+
+ maybeInjectPrecomputedValuesForAnalysis();
+ setCommandId(commandId);
+ setShowLoadingProgress(showLoadingProgress);
+ setDefaultVisibility(defaultVisibility);
+ setupDefaultPackage(defaultsPackageContents);
+ setPackageLocator(pkgLocator);
+
+ syscalls.set(new PerBuildSyscallCache());
+ checkPreprocessorFactory();
+ emittedEventState.clear();
+
+ // If the PackageFunction was interrupted, there may be stale entries here.
+ packageFunctionCache.clear();
+ numPackagesLoaded.set(0);
+
+ // Reset the stateful SkyframeCycleReporter, which contains cycles from last run.
+ cyclesReporter.set(createCyclesReporter());
+ }
+
+ @SuppressWarnings("unchecked")
+ private void setPackageLocator(PathPackageLocator pkgLocator) {
+ PathPackageLocator oldLocator = this.pkgLocator.getAndSet(pkgLocator);
+ PrecomputedValue.PATH_PACKAGE_LOCATOR.set(injectable(), pkgLocator);
+
+ if (!pkgLocator.equals(oldLocator)) {
+ // The package path is read not only by SkyFunctions but also by some other code paths.
+ // We need to take additional steps to keep the corresponding data structures in sync.
+ // (Some of the additional steps are carried out by ConfiguredTargetValueInvalidationListener,
+ // and some by BuildView#buildHasIncompatiblePackageRoots and #updateSkyframe.)
+ onNewPackageLocator(oldLocator, pkgLocator);
+ }
+ }
+
+ protected abstract void onNewPackageLocator(PathPackageLocator oldLocator,
+ PathPackageLocator pkgLocator);
+
+ private void checkPreprocessorFactory() {
+ if (preprocessorFactory == null) {
+ Preprocessor.Factory newPreprocessorFactory = preprocessorFactorySupplier.getFactory(
+ packageManager);
+ pkgFactory.setPreprocessorFactory(newPreprocessorFactory);
+ preprocessorFactory = newPreprocessorFactory;
+ } else if (!preprocessorFactory.isStillValid()) {
+ Preprocessor.Factory newPreprocessorFactory = preprocessorFactorySupplier.getFactory(
+ packageManager);
+ invalidate(SkyFunctionName.functionIs(SkyFunctions.PACKAGE));
+ pkgFactory.setPreprocessorFactory(newPreprocessorFactory);
+ preprocessorFactory = newPreprocessorFactory;
+ }
+ }
+
+ /**
+ * Specifies the current {@link SkyframeBuildView} instance. This should only be set once over the
+ * lifetime of the Blaze server, except in tests.
+ */
+ public void setSkyframeBuildView(SkyframeBuildView skyframeBuildView) {
+ this.skyframeBuildView = skyframeBuildView;
+ setConfigurationSkyKey(configurationSkyKey);
+ this.artifactFactory.set(skyframeBuildView.getArtifactFactory());
+ if (skyframeBuildView.getWarningListener() != null) {
+ setErrorEventListener(skyframeBuildView.getWarningListener());
+ }
+ }
+
+ /**
+ * Sets the eventBus to use for posting events.
+ */
+ public void setEventBus(EventBus eventBus) {
+ this.eventBus.set(eventBus);
+ }
+
+ /**
+ * Sets the eventHandler to use for reporting errors.
+ */
+ public void setErrorEventListener(EventHandler eventHandler) {
+ this.errorEventListener = eventHandler;
+ }
+
+ /**
+ * Sets the path for action log buffers.
+ */
+ public void setActionOutputRoot(Path actionOutputRoot) {
+ Preconditions.checkNotNull(actionOutputRoot);
+ this.actionLogBufferPathGenerator = new ActionLogBufferPathGenerator(actionOutputRoot);
+ this.skyframeActionExecutor.setActionLogBufferPathGenerator(actionLogBufferPathGenerator);
+ }
+
+ private void setConfigurationSkyKey(SkyKey skyKey) {
+ this.configurationSkyKey = skyKey;
+ if (skyframeBuildView != null) {
+ skyframeBuildView.setConfigurationSkyKey(skyKey);
+ }
+ }
+
+ @VisibleForTesting
+ public void setConfigurationDataForTesting(BuildOptions options,
+ BlazeDirectories directories, ConfigurationFactory configurationFactory) {
+ SkyKey skyKey = ConfigurationCollectionValue.key(options, ImmutableSet.<String>of());
+ setConfigurationSkyKey(skyKey);
+ PrecomputedValue.BLAZE_DIRECTORIES.set(injectable(), directories);
+ this.configurationFactory.set(configurationFactory);
+ this.configurationFragments.set(ImmutableList.copyOf(configurationFactory.getFactories()));
+ this.configurationPackages.set(Sets.<Package>newConcurrentHashSet());
+ }
+
+ @VisibleForTesting
+ public BuildConfigurationCollection createConfigurations(
+ ConfigurationFactory configurationFactory, BuildConfigurationKey configurationKey)
+ throws InvalidConfigurationException, InterruptedException {
+ return createConfigurations(false, configurationFactory, configurationKey);
+ }
+
+ /**
+ * Asks the Skyframe evaluator to build the value for BuildConfigurationCollection and
+ * returns result. Also invalidates {@link PrecomputedValue#TEST_ENVIRONMENT_VARIABLES} and
+ * {@link PrecomputedValue#BLAZE_DIRECTORIES} if they have changed.
+ */
+ public BuildConfigurationCollection createConfigurations(boolean keepGoing,
+ ConfigurationFactory configurationFactory, BuildConfigurationKey configurationKey)
+ throws InvalidConfigurationException, InterruptedException {
+
+ this.configurationPackages.set(Sets.<Package>newConcurrentHashSet());
+ this.clientEnv.set(configurationKey.getClientEnv());
+ this.configurationFactory.set(configurationFactory);
+ this.configurationFragments.set(ImmutableList.copyOf(configurationFactory.getFactories()));
+ BuildOptions buildOptions = configurationKey.getBuildOptions();
+ Map<String, String> testEnv = BuildConfiguration.getTestEnv(
+ buildOptions.get(BuildConfiguration.Options.class).testEnvironment,
+ configurationKey.getClientEnv());
+ // TODO(bazel-team): find a way to use only BuildConfigurationKey instead of
+ // TestEnvironmentVariables and BlazeDirectories. There is a problem only with
+ // TestEnvironmentVariables because BuildConfigurationKey stores client environment variables
+ // and we don't want to rebuild everything when any variable changes.
+ PrecomputedValue.TEST_ENVIRONMENT_VARIABLES.set(injectable(), testEnv);
+ PrecomputedValue.BLAZE_DIRECTORIES.set(injectable(), configurationKey.getDirectories());
+
+ SkyKey skyKey = ConfigurationCollectionValue.key(configurationKey.getBuildOptions(),
+ configurationKey.getMultiCpu());
+ setConfigurationSkyKey(skyKey);
+ EvaluationResult<ConfigurationCollectionValue> result = buildDriver.evaluate(
+ Arrays.asList(skyKey), keepGoing, DEFAULT_THREAD_COUNT, errorEventListener);
+ if (result.hasError()) {
+ Throwable e = result.getError(skyKey).getException();
+ // Wrap loading failed exceptions
+ if (e instanceof NoSuchThingException) {
+ e = new InvalidConfigurationException(e);
+ }
+ Throwables.propagateIfInstanceOf(e, InvalidConfigurationException.class);
+ throw new IllegalStateException(
+ "Unknown error during ConfigurationCollectionValue evaluation", e);
+ }
+ Preconditions.checkState(result.values().size() == 1,
+ "Result of evaluate() must contain exactly one value %s", result);
+ ConfigurationCollectionValue configurationValue =
+ Iterables.getOnlyElement(result.values());
+ this.configurationPackages.set(
+ Sets.newConcurrentHashSet(configurationValue.getConfigurationPackages()));
+ return configurationValue.getConfigurationCollection();
+ }
+
+ private Iterable<ActionLookupValue> getActionLookupValues() {
+ // This filter keeps subclasses of ActionLookupValue.
+ return Iterables.filter(memoizingEvaluator.getDoneValues().values(), ActionLookupValue.class);
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ Map<SkyKey, ActionLookupValue> getActionLookupValueMap() {
+ return (Map) Maps.filterValues(memoizingEvaluator.getDoneValues(),
+ Predicates.instanceOf(ActionLookupValue.class));
+ }
+
+ /**
+ * Checks the actions in Skyframe for conflicts between their output artifacts. Delegates to
+ * {@link SkyframeActionExecutor#findAndStoreArtifactConflicts} to do the work, since any
+ * conflicts found will only be reported during execution.
+ */
+ ImmutableMap<Action, SkyframeActionExecutor.ConflictException> findArtifactConflicts()
+ throws InterruptedException {
+ if (skyframeBuildView.isSomeConfiguredTargetEvaluated()
+ || skyframeBuildView.isSomeConfiguredTargetInvalidated()) {
+ // This operation is somewhat expensive, so we only do it if the graph might have changed in
+ // some way -- either we analyzed a new target or we invalidated an old one.
+ skyframeActionExecutor.findAndStoreArtifactConflicts(getActionLookupValues());
+ skyframeBuildView.resetEvaluatedConfiguredTargetFlag();
+ // The invalidated configured targets flag will be reset later in the evaluate() call.
+ }
+ return skyframeActionExecutor.badActions();
+ }
+
+ /**
+ * Asks the Skyframe evaluator to build the given artifacts and targets, and to test the
+ * given test targets.
+ */
+ public EvaluationResult<?> buildArtifacts(
+ Executor executor,
+ Set<Artifact> artifactsToBuild,
+ Collection<ConfiguredTarget> targetsToBuild,
+ Collection<ConfiguredTarget> targetsToTest,
+ boolean exclusiveTesting,
+ boolean keepGoing,
+ boolean explain,
+ int numJobs,
+ ActionCacheChecker actionCacheChecker,
+ @Nullable EvaluationProgressReceiver executionProgressReceiver) throws InterruptedException {
+ checkActive();
+ Preconditions.checkState(actionLogBufferPathGenerator != null);
+
+ skyframeActionExecutor.prepareForExecution(executor, keepGoing, explain, actionCacheChecker);
+
+ resourceManager.resetResourceUsage();
+ try {
+ progressReceiver.executionProgressReceiver = executionProgressReceiver;
+ Iterable<SkyKey> artifactKeys = ArtifactValue.mandatoryKeys(artifactsToBuild);
+ Iterable<SkyKey> targetKeys = TargetCompletionValue.keys(targetsToBuild);
+ Iterable<SkyKey> testKeys = TestCompletionValue.keys(targetsToTest, exclusiveTesting);
+ return buildDriver.evaluate(Iterables.concat(artifactKeys, targetKeys, testKeys), keepGoing,
+ numJobs, errorEventListener);
+ } finally {
+ progressReceiver.executionProgressReceiver = null;
+ // Also releases thread locks.
+ resourceManager.resetResourceUsage();
+ skyframeActionExecutor.executionOver();
+ }
+ }
+
+ @VisibleForTesting
+ public void prepareBuildingForTestingOnly(Executor executor, boolean keepGoing, boolean explain,
+ ActionCacheChecker checker) {
+ skyframeActionExecutor.prepareForExecution(executor, keepGoing, explain, checker);
+ }
+
+ EvaluationResult<TargetPatternValue> targetPatterns(Iterable<SkyKey> patternSkyKeys,
+ boolean keepGoing, EventHandler eventHandler) throws InterruptedException {
+ checkActive();
+ return buildDriver.evaluate(patternSkyKeys, keepGoing, DEFAULT_THREAD_COUNT,
+ eventHandler);
+ }
+
+ /**
+ * Returns the {@link ConfiguredTarget}s corresponding to the given keys.
+ *
+ * <p>For use for legacy support from {@code BuildView} only.
+ *
+ * <p>If a requested configured target is in error, the corresponding value is omitted from the
+ * returned list.
+ */
+ @ThreadSafety.ThreadSafe
+ public ImmutableList<ConfiguredTarget> getConfiguredTargets(Iterable<Dependency> keys) {
+ checkActive();
+ if (skyframeBuildView == null) {
+ // If build view has not yet been initialized, no configured targets can have been created.
+ // This is most likely to happen after a failed loading phase.
+ return ImmutableList.of();
+ }
+ final List<SkyKey> skyKeys = new ArrayList<>();
+ for (Dependency key : keys) {
+ skyKeys.add(ConfiguredTargetValue.key(key.getLabel(), key.getConfiguration()));
+ for (Class<? extends ConfiguredAspectFactory> aspect : key.getAspects()) {
+ skyKeys.add(AspectValue.key(key.getLabel(), key.getConfiguration(), aspect));
+ }
+ }
+
+ EvaluationResult<SkyValue> result;
+ try {
+ result = callUninterruptibly(new Callable<EvaluationResult<SkyValue>>() {
+ @Override
+ public EvaluationResult<SkyValue> call() throws Exception {
+ synchronized (valueLookupLock) {
+ try {
+ skyframeBuildView.enableAnalysis(true);
+ return buildDriver.evaluate(skyKeys, false, DEFAULT_THREAD_COUNT,
+ errorEventListener);
+ } finally {
+ skyframeBuildView.enableAnalysis(false);
+ }
+ }
+ }
+ });
+ } catch (Exception e) {
+ throw new IllegalStateException(e); // Should never happen.
+ }
+
+ ImmutableList.Builder<ConfiguredTarget> cts = ImmutableList.builder();
+
+ DependentNodeLoop:
+ for (Dependency key : keys) {
+ SkyKey configuredTargetKey = ConfiguredTargetValue.key(
+ key.getLabel(), key.getConfiguration());
+ if (result.get(configuredTargetKey) == null) {
+ continue;
+ }
+
+ ConfiguredTarget configuredTarget =
+ ((ConfiguredTargetValue) result.get(configuredTargetKey)).getConfiguredTarget();
+ List<Aspect> aspects = new ArrayList<>();
+
+ for (Class<? extends ConfiguredAspectFactory> aspect : key.getAspects()) {
+ SkyKey aspectKey = AspectValue.key(key.getLabel(), key.getConfiguration(), aspect);
+ if (result.get(aspectKey) == null) {
+ continue DependentNodeLoop;
+ }
+
+ aspects.add(((AspectValue) result.get(aspectKey)).get());
+ }
+
+ cts.add(RuleConfiguredTarget.mergeAspects(configuredTarget, aspects));
+ }
+
+ return cts.build();
+ }
+
+ /**
+ * Returns a particular configured target.
+ *
+ * <p>Used only for testing.
+ */
+ @VisibleForTesting
+ @Nullable
+ public ConfiguredTarget getConfiguredTargetForTesting(
+ Label label, BuildConfiguration configuration) {
+ if (memoizingEvaluator.getExistingValueForTesting(
+ PrecomputedValue.WORKSPACE_STATUS_KEY.getKeyForTesting()) == null) {
+ injectWorkspaceStatusData();
+ }
+ return Iterables.getFirst(getConfiguredTargets(ImmutableList.of(
+ new Dependency(label, configuration))), null);
+ }
+
+ /**
+ * Invalidates Skyframe values corresponding to the given set of modified files under the given
+ * path entry.
+ *
+ * <p>May throw an {@link InterruptedException}, which means that no values have been invalidated.
+ */
+ @VisibleForTesting
+ public abstract void invalidateFilesUnderPathForTesting(ModifiedFileSet modifiedFileSet,
+ Path pathEntry) throws InterruptedException;
+
+ /**
+ * Invalidates SkyFrame values that may have failed for transient reasons.
+ */
+ public abstract void invalidateTransientErrors();
+
+ @VisibleForTesting
+ public TimestampGranularityMonitor getTimestampGranularityMonitorForTesting() {
+ return tsgm;
+ }
+
+ /**
+ * Configures a given set of configured targets.
+ */
+ public EvaluationResult<ConfiguredTargetValue> configureTargets(
+ List<ConfiguredTargetKey> values, boolean keepGoing) throws InterruptedException {
+ checkActive();
+
+ // Make sure to not run too many analysis threads. This can cause memory thrashing.
+ return buildDriver.evaluate(ConfiguredTargetValue.keys(values), keepGoing,
+ ResourceUsage.getAvailableProcessors(), errorEventListener);
+ }
+
+ /**
+ * Post-process the targets. Values in the EvaluationResult are known to be transitively
+ * error-free from action conflicts.
+ */
+ public EvaluationResult<PostConfiguredTargetValue> postConfigureTargets(
+ List<ConfiguredTargetKey> values, boolean keepGoing,
+ ImmutableMap<Action, SkyframeActionExecutor.ConflictException> badActions)
+ throws InterruptedException {
+ checkActive();
+ PrecomputedValue.BAD_ACTIONS.set(injectable(), badActions);
+ // Make sure to not run too many analysis threads. This can cause memory thrashing.
+ EvaluationResult<PostConfiguredTargetValue> result =
+ buildDriver.evaluate(PostConfiguredTargetValue.keys(values), keepGoing,
+ ResourceUsage.getAvailableProcessors(), errorEventListener);
+
+ // Remove all post-configured target values immediately for memory efficiency. We are OK with
+ // this mini-phase being non-incremental as the failure mode of action conflict is rare.
+ memoizingEvaluator.delete(SkyFunctionName.functionIs(SkyFunctions.POST_CONFIGURED_TARGET));
+
+ return result;
+ }
+
+ /**
+ * Returns a Skyframe-based {@link SkyframeTransitivePackageLoader} implementation.
+ */
+ @VisibleForTesting
+ public TransitivePackageLoader pkgLoader() {
+ checkActive();
+ return new SkyframeLabelVisitor(new SkyframeTransitivePackageLoader(), cyclesReporter);
+ }
+
+ class SkyframeTransitivePackageLoader {
+ /**
+ * Loads the specified {@link TransitiveTargetValue}s.
+ */
+ EvaluationResult<TransitiveTargetValue> loadTransitiveTargets(
+ Iterable<Target> targetsToVisit, Iterable<Label> labelsToVisit, boolean keepGoing)
+ throws InterruptedException {
+ List<SkyKey> valueNames = new ArrayList<>();
+ for (Target target : targetsToVisit) {
+ valueNames.add(TransitiveTargetValue.key(target.getLabel()));
+ }
+ for (Label label : labelsToVisit) {
+ valueNames.add(TransitiveTargetValue.key(label));
+ }
+
+ return buildDriver.evaluate(valueNames, keepGoing, DEFAULT_THREAD_COUNT,
+ errorEventListener);
+ }
+
+ public Set<Package> retrievePackages(Set<PackageIdentifier> packageIds) {
+ final List<SkyKey> valueNames = new ArrayList<>();
+ for (PackageIdentifier pkgId : packageIds) {
+ valueNames.add(PackageValue.key(pkgId));
+ }
+
+ try {
+ return callUninterruptibly(new Callable<Set<Package>>() {
+ @Override
+ public Set<Package> call() throws Exception {
+ EvaluationResult<PackageValue> result = buildDriver.evaluate(
+ valueNames, false, ResourceUsage.getAvailableProcessors(), errorEventListener);
+ Preconditions.checkState(!result.hasError(),
+ "unexpected errors: %s", result.errorMap());
+ Set<Package> packages = Sets.newHashSet();
+ for (PackageValue value : result.values()) {
+ packages.add(value.getPackage());
+ }
+ return packages;
+ }
+ });
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+
+ }
+ }
+
+ /**
+ * Returns the generating {@link Action} of the given {@link Artifact}.
+ *
+ * <p>For use for legacy support from {@code BuildView} only.
+ */
+ @ThreadSafety.ThreadSafe
+ public Action getGeneratingAction(final Artifact artifact) {
+ if (artifact.isSourceArtifact()) {
+ return null;
+ }
+
+ try {
+ return callUninterruptibly(new Callable<Action>() {
+ @Override
+ public Action call() throws InterruptedException {
+ ArtifactOwner artifactOwner = artifact.getArtifactOwner();
+ Preconditions.checkState(artifactOwner instanceof ActionLookupValue.ActionLookupKey,
+ "%s %s", artifact, artifactOwner);
+ SkyKey actionLookupKey =
+ ActionLookupValue.key((ActionLookupValue.ActionLookupKey) artifactOwner);
+
+ synchronized (valueLookupLock) {
+ // Note that this will crash (attempting to run a configured target value builder after
+ // analysis) after a failed --nokeep_going analysis in which the configured target that
+ // failed was a (transitive) dependency of the configured target that should generate
+ // this action. We don't expect callers to query generating actions in such cases.
+ EvaluationResult<ActionLookupValue> result = buildDriver.evaluate(
+ ImmutableList.of(actionLookupKey), false, ResourceUsage.getAvailableProcessors(),
+ errorEventListener);
+ return result.hasError()
+ ? null
+ : result.get(actionLookupKey).getGeneratingAction(artifact);
+ }
+ }
+ });
+ } catch (Exception e) {
+ throw new IllegalStateException("Error getting generating action: " + artifact.prettyPrint(),
+ e);
+ }
+ }
+
+ public PackageManager getPackageManager() {
+ return packageManager;
+ }
+
+ class SkyframePackageLoader {
+ /**
+ * Looks up a particular package (mostly used after the loading phase, so packages should
+ * already be present, but occasionally used pre-loading phase). Use should be discouraged,
+ * since this cannot be used inside a Skyframe evaluation, and concurrent calls are
+ * synchronized.
+ *
+ * <p>Note that this method needs to be synchronized since InMemoryMemoizingEvaluator.evaluate()
+ * method does not support concurrent calls.
+ */
+ Package getPackage(EventHandler eventHandler, PackageIdentifier pkgName)
+ throws InterruptedException, NoSuchPackageException {
+ synchronized (valueLookupLock) {
+ SkyKey key = PackageValue.key(pkgName);
+ // Any call to this method post-loading phase should either be error-free or be in a
+ // keep_going build, since otherwise the build would have failed during loading. Thus
+ // we set keepGoing=true unconditionally.
+ EvaluationResult<PackageValue> result =
+ buildDriver.evaluate(ImmutableList.of(key), /*keepGoing=*/true,
+ DEFAULT_THREAD_COUNT, eventHandler);
+ if (result.hasError()) {
+ ErrorInfo error = result.getError();
+ if (!Iterables.isEmpty(error.getCycleInfo())) {
+ reportCycles(result.getError().getCycleInfo(), key);
+ // This can only happen if a package is freshly loaded outside of the target parsing
+ // or loading phase
+ throw new BuildFileContainsErrorsException(pkgName.toString(),
+ "Cycle encountered while loading package " + pkgName);
+ }
+ Throwable e = error.getException();
+ // PackageFunction should be catching, swallowing, and rethrowing all transitive
+ // errors as NoSuchPackageExceptions.
+ Throwables.propagateIfInstanceOf(e, NoSuchPackageException.class);
+ throw new IllegalStateException("Unexpected Exception type from PackageValue for '"
+ + pkgName + "'' with root causes: " + Iterables.toString(error.getRootCauses()), e);
+ }
+ return result.get(key).getPackage();
+ }
+ }
+
+ Package getLoadedPackage(final PackageIdentifier pkgName) throws NoSuchPackageException {
+ // Note that in Skyframe there is no way to tell if the package has been loaded before or not,
+ // so this will never throw for packages that are not loaded. However, no code currently
+ // relies on having the exception thrown.
+ try {
+ return callUninterruptibly(new Callable<Package>() {
+ @Override
+ public Package call() throws Exception {
+ return getPackage(errorEventListener, pkgName);
+ }
+ });
+ } catch (NoSuchPackageException e) {
+ if (e.getPackage() != null) {
+ return e.getPackage();
+ }
+ throw e;
+ } catch (Exception e) {
+ throw new IllegalStateException(e); // Should never happen.
+ }
+ }
+
+ /**
+ * Returns whether the given package should be consider deleted and thus should be ignored.
+ */
+ public boolean isPackageDeleted(String packageName) {
+ return deletedPackages.get().contains(packageName);
+ }
+
+ /** Same as {@link PackageManager#partiallyClear}. */
+ void partiallyClear() {
+ packageFunctionCache.clear();
+ }
+ }
+
+ /**
+ * Calls the given callable uninterruptibly.
+ *
+ * <p>If the callable throws {@link InterruptedException}, calls it again, until the callable
+ * returns a result. Sets the {@code currentThread().interrupted()} bit if the callable threw
+ * {@link InterruptedException} at least once.
+ *
+ * <p>This is almost identical to {@code Uninterruptibles#getUninterruptibly}.
+ */
+ protected static final <T> T callUninterruptibly(Callable<T> callable) throws Exception {
+ boolean interrupted = false;
+ try {
+ while (true) {
+ try {
+ return callable.call();
+ } catch (InterruptedException e) {
+ interrupted = true;
+ }
+ }
+ } finally {
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ @VisibleForTesting
+ public MemoizingEvaluator getEvaluatorForTesting() {
+ return memoizingEvaluator;
+ }
+
+ /**
+ * Stores the set of loaded packages and, if needed, evicts ConfiguredTarget values.
+ *
+ * <p>The set represents all packages from the transitive closure of the top-level targets from
+ * the latest build.
+ */
+ @ThreadCompatible
+ public abstract void updateLoadedPackageSet(Set<PackageIdentifier> loadedPackages);
+
+ public void sync(PackageCacheOptions packageCacheOptions, Path workingDirectory,
+ String defaultsPackageContents, UUID commandId) throws InterruptedException,
+ AbruptExitException{
+
+ preparePackageLoading(
+ createPackageLocator(packageCacheOptions, directories.getWorkspace(), workingDirectory),
+ packageCacheOptions.defaultVisibility, packageCacheOptions.showLoadingProgress,
+ defaultsPackageContents, commandId);
+ setDeletedPackages(ImmutableSet.copyOf(packageCacheOptions.deletedPackages));
+
+ incrementalBuildMonitor = new SkyframeIncrementalBuildMonitor();
+ invalidateTransientErrors();
+ }
+
+ protected PathPackageLocator createPackageLocator(PackageCacheOptions packageCacheOptions,
+ Path workspace, Path workingDirectory) throws AbruptExitException{
+ return PathPackageLocator.create(
+ packageCacheOptions.packagePath, getReporter(), workspace, workingDirectory);
+ }
+
+ private CyclesReporter createCyclesReporter() {
+ return new CyclesReporter(
+ new TransitiveTargetCycleReporter(packageManager),
+ new ActionArtifactCycleReporter(packageManager),
+ new SkylarkModuleCycleReporter());
+ }
+
+ CyclesReporter getCyclesReporter() {
+ return cyclesReporter.get();
+ }
+
+ /** Convenience method with same semantics as {@link CyclesReporter#reportCycles}. */
+ public void reportCycles(Iterable<CycleInfo> cycles, SkyKey topLevelKey) {
+ getCyclesReporter().reportCycles(cycles, topLevelKey, errorEventListener);
+ }
+
+ public void setActionExecutionProgressReportingObjects(@Nullable ProgressSupplier supplier,
+ @Nullable ActionCompletedReceiver completionReceiver,
+ @Nullable ActionExecutionStatusReporter statusReporter) {
+ skyframeActionExecutor.setActionExecutionProgressReportingObjects(supplier, completionReceiver);
+ this.statusReporterRef.set(statusReporter);
+ }
+
+ /**
+ * This should be called at most once in the lifetime of the SkyframeExecutor (except for
+ * tests), and it should be called before the execution phase.
+ */
+ void setArtifactFactoryAndBinTools(ArtifactFactory artifactFactory, BinTools binTools) {
+ this.artifactFactory.set(artifactFactory);
+ this.binTools = binTools;
+ }
+
+ public void prepareExecution(boolean checkOutputFiles) throws AbruptExitException,
+ InterruptedException {
+ maybeInjectEmbeddedArtifacts();
+
+ if (checkOutputFiles) {
+ // Detect external modifications in the output tree.
+ FilesystemValueChecker fsnc = new FilesystemValueChecker(memoizingEvaluator, tsgm);
+ invalidateDirtyActions(fsnc.getDirtyActionValues(batchStatter));
+ modifiedFiles += fsnc.getNumberOfModifiedOutputFiles();
+ }
+ informAboutNumberOfModifiedFiles();
+ }
+
+ protected abstract void invalidateDirtyActions(Iterable<SkyKey> dirtyActionValues);
+
+ @VisibleForTesting void maybeInjectEmbeddedArtifacts() throws AbruptExitException {
+ // The blaze client already ensures that the contents of the embedded binaries never change,
+ // so we just need to make sure that the appropriate artifacts are present in the skyframe
+ // graph.
+
+ if (!needToInjectEmbeddedArtifacts) {
+ return;
+ }
+
+ Preconditions.checkNotNull(artifactFactory.get());
+ Preconditions.checkNotNull(binTools);
+ Map<SkyKey, SkyValue> values = Maps.newHashMap();
+ // Blaze separately handles the symlinks that target these binaries. See BinTools#setupTool.
+ for (Artifact artifact : binTools.getAllEmbeddedArtifacts(artifactFactory.get())) {
+ FileArtifactValue fileArtifactValue;
+ try {
+ fileArtifactValue = FileArtifactValue.create(artifact);
+ } catch (IOException e) {
+ // See ExtractData in blaze.cc.
+ String message = "Error: corrupt installation: file " + artifact.getPath() + " missing. "
+ + "Please remove '" + directories.getInstallBase() + "' and try again.";
+ throw new AbruptExitException(message, ExitCode.LOCAL_ENVIRONMENTAL_ERROR, e);
+ }
+ values.put(ArtifactValue.key(artifact, /*isMandatory=*/true), fileArtifactValue);
+ }
+ injectable().inject(values);
+ needToInjectEmbeddedArtifacts = false;
+ }
+
+ /**
+ * Mark dirty values for deletion if they've been dirty for longer than N versions.
+ *
+ * <p>Specifying a value N means, if the current version is V and a value was dirtied (and
+ * has remained so) in version U, and U + N <= V, then the value will be marked for deletion
+ * and purged in version V+1.
+ */
+ public abstract void deleteOldNodes(long versionWindowForDirtyGc);
+
+ /**
+ * A progress received to track analysis invalidation and update progress messages.
+ */
+ protected class SkyframeProgressReceiver implements EvaluationProgressReceiver {
+ /**
+ * This flag is needed in order to avoid invalidating legacy data when we clear the
+ * analysis cache because of --discard_analysis_cache flag. For that case we want to keep
+ * the legacy data but get rid of the Skyframe data.
+ */
+ protected boolean ignoreInvalidations = false;
+ /** This receiver is only needed for execution, so it is null otherwise. */
+ @Nullable EvaluationProgressReceiver executionProgressReceiver = null;
+
+ @Override
+ public void invalidated(SkyValue value, InvalidationState state) {
+ if (ignoreInvalidations) {
+ return;
+ }
+ if (skyframeBuildView != null) {
+ skyframeBuildView.getInvalidationReceiver().invalidated(value, state);
+ }
+ }
+
+ @Override
+ public void enqueueing(SkyKey skyKey) {
+ if (ignoreInvalidations) {
+ return;
+ }
+ if (skyframeBuildView != null) {
+ skyframeBuildView.getInvalidationReceiver().enqueueing(skyKey);
+ }
+ if (executionProgressReceiver != null) {
+ executionProgressReceiver.enqueueing(skyKey);
+ }
+ }
+
+ @Override
+ public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) {
+ if (ignoreInvalidations) {
+ return;
+ }
+ if (skyframeBuildView != null) {
+ skyframeBuildView.getInvalidationReceiver().evaluated(skyKey, value, state);
+ }
+ if (executionProgressReceiver != null) {
+ executionProgressReceiver.evaluated(skyKey, value, state);
+ }
+ }
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutorFactory.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutorFactory.java
new file mode 100644
index 0000000..a1615cf
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutorFactory.java
@@ -0,0 +1,68 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.Preprocessor;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+
+import java.util.Set;
+
+/**
+* A factory that creates instances of SkyframeExecutor.
+*/
+public interface SkyframeExecutorFactory {
+
+ /**
+ * Creates an instance of SkyframeExecutor
+ *
+ * @param reporter the reporter to be used by the executor
+ * @param pkgFactory the package factory
+ * @param skyframeBuild use Skyframe for the build phase. Should be always true after we are in
+ * the skyframe full mode.
+ * @param tsgm timestamp granularity monitor
+ * @param directories Blaze directories
+ * @param workspaceStatusActionFactory a factory for creating WorkspaceStatusAction objects
+ * @param buildInfoFactories list of BuildInfoFactories
+ * @param diffAwarenessFactories
+ * @param allowedMissingInputs
+ * @param preprocessorFactorySupplier
+ * @param extraSkyFunctions
+ * @param extraPrecomputedValues
+ * @return an instance of the SkyframeExecutor
+ * @throws AbruptExitException if the executor cannot be created
+ */
+ SkyframeExecutor create(Reporter reporter, PackageFactory pkgFactory,
+ TimestampGranularityMonitor tsgm, BlazeDirectories directories,
+ Factory workspaceStatusActionFactory,
+ ImmutableList<BuildInfoFactory> buildInfoFactories,
+ Set<Path> immutableDirectories,
+ Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories,
+ Predicate<PathFragment> allowedMissingInputs,
+ Preprocessor.Factory.Supplier preprocessorFactorySupplier,
+ ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions,
+ ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) throws AbruptExitException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeIncrementalBuildMonitor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeIncrementalBuildMonitor.java
new file mode 100644
index 0000000..c0fea26
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeIncrementalBuildMonitor.java
@@ -0,0 +1,59 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.actions.ChangedFilesMessage;
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyKey;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A package-private class intended to track a small number of modified files during the build. This
+ * class should stop recording changed files if there are too many of them, instead of holding onto
+ * a large collection of files.
+ */
+@ThreadSafety.ThreadCompatible
+class SkyframeIncrementalBuildMonitor {
+ private Set<PathFragment> files = new HashSet<>();
+ private static final int MAX_FILES = 100;
+
+ public void accrue(Iterable<SkyKey> invalidatedValues) {
+ for (SkyKey skyKey : invalidatedValues) {
+ if (skyKey.functionName() == SkyFunctions.FILE_STATE) {
+ RootedPath file = (RootedPath) skyKey.argument();
+ maybeAddFile(file.getRelativePath());
+ }
+ }
+ }
+
+ private void maybeAddFile(PathFragment path) {
+ if (files != null) {
+ files.add(path);
+ if (files.size() >= MAX_FILES) {
+ files = null;
+ }
+ }
+ }
+
+ public void alertListeners(EventBus eventBus) {
+ if (files != null) {
+ eventBus.post(new ChangedFilesMessage(files));
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeLabelVisitor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeLabelVisitor.java
new file mode 100644
index 0000000..2844cc0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeLabelVisitor.java
@@ -0,0 +1,262 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor.SkyframeTransitivePackageLoader;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.CycleInfo;
+import com.google.devtools.build.skyframe.CyclesReporter;
+import com.google.devtools.build.skyframe.ErrorInfo;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.SkyKey;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * Skyframe-based transitive package loader.
+ */
+final class SkyframeLabelVisitor implements TransitivePackageLoader {
+
+ private final SkyframeTransitivePackageLoader transitivePackageLoader;
+ private final AtomicReference<CyclesReporter> skyframeCyclesReporter;
+
+ private Set<PackageIdentifier> allVisitedPackages;
+ private Set<PackageIdentifier> errorFreeVisitedPackages;
+ private Set<Label> visitedTargets;
+ private Set<TransitiveTargetValue> previousBuildTargetValueSet = null;
+ private boolean lastBuildKeepGoing = false;
+ private final Multimap<Label, Label> rootCauses = HashMultimap.create();
+
+ SkyframeLabelVisitor(SkyframeTransitivePackageLoader transitivePackageLoader,
+ AtomicReference<CyclesReporter> skyframeCyclesReporter) {
+ this.transitivePackageLoader = transitivePackageLoader;
+ this.skyframeCyclesReporter = skyframeCyclesReporter;
+ }
+
+ @Override
+ public boolean sync(EventHandler eventHandler, Set<Target> targetsToVisit,
+ Set<Label> labelsToVisit, boolean keepGoing, int parallelThreads, int maxDepth)
+ throws InterruptedException {
+ rootCauses.clear();
+ lastBuildKeepGoing = false;
+ EvaluationResult<TransitiveTargetValue> result =
+ transitivePackageLoader.loadTransitiveTargets(targetsToVisit, labelsToVisit, keepGoing);
+ updateVisitedValues(result.values());
+ lastBuildKeepGoing = keepGoing;
+
+ if (!hasErrors(result)) {
+ return true;
+ }
+
+ Set<Entry<SkyKey, ErrorInfo>> errors = result.errorMap().entrySet();
+ if (!keepGoing) {
+ // We may have multiple errors, but in non keep_going builds, we're obligated to print only
+ // one of them.
+ Preconditions.checkState(!errors.isEmpty(), result);
+ Entry<SkyKey, ErrorInfo> error = errors.iterator().next();
+ ErrorInfo errorInfo = error.getValue();
+ SkyKey topLevel = error.getKey();
+ Label topLevelLabel = (Label) topLevel.argument();
+ if (!Iterables.isEmpty(errorInfo.getCycleInfo())) {
+ skyframeCyclesReporter.get().reportCycles(errorInfo.getCycleInfo(), topLevel, eventHandler);
+ errorAboutLoadingFailure(topLevelLabel, null, eventHandler);
+ } else if (isDirectErrorFromTopLevelLabel(topLevelLabel, labelsToVisit, errorInfo)) {
+ // An error caused by a non-top-level label has already been reported during error
+ // bubbling but an error caused by the top-level non-target label itself hasn't been
+ // reported yet. Note that errors from top-level targets have already been reported
+ // during target parsing.
+ errorAboutLoadingFailure(topLevelLabel, errorInfo.getException(), eventHandler);
+ }
+ return false;
+ }
+
+ for (Entry<SkyKey, ErrorInfo> errorEntry : errors) {
+ SkyKey key = errorEntry.getKey();
+ ErrorInfo errorInfo = errorEntry.getValue();
+ Preconditions.checkState(key.functionName().equals(SkyFunctions.TRANSITIVE_TARGET), errorEntry);
+ Label topLevelLabel = (Label) key.argument();
+ if (!Iterables.isEmpty(errorInfo.getCycleInfo())) {
+ skyframeCyclesReporter.get().reportCycles(errorInfo.getCycleInfo(), key, eventHandler);
+ for (Label rootCause : getRootCausesOfCycles(topLevelLabel, errorInfo.getCycleInfo())) {
+ rootCauses.put(topLevelLabel, rootCause);
+ }
+ }
+ if (isDirectErrorFromTopLevelLabel(topLevelLabel, labelsToVisit, errorInfo)) {
+ // Unlike top-level targets, which have already gone through target parsing,
+ // errors directly coming from top-level labels have not been reported yet.
+ //
+ // See the note in the --nokeep_going case above.
+ eventHandler.handle(Event.error(errorInfo.getException().getMessage()));
+ }
+ warnAboutLoadingFailure(topLevelLabel, eventHandler);
+ for (SkyKey badKey : errorInfo.getRootCauses()) {
+ Preconditions.checkState(badKey.argument() instanceof Label,
+ "%s %s %s", key, errorInfo, badKey);
+ rootCauses.put(topLevelLabel, (Label) badKey.argument());
+ }
+ }
+ for (Label topLevelLabel : result.<Label>keyNames()) {
+ SkyKey topLevelTransitiveTargetKey = TransitiveTargetValue.key(topLevelLabel);
+ TransitiveTargetValue topLevelTransitiveTargetValue = result.get(topLevelTransitiveTargetKey);
+ if (topLevelTransitiveTargetValue.getTransitiveRootCauses() != null) {
+ for (Label rootCause : topLevelTransitiveTargetValue.getTransitiveRootCauses()) {
+ rootCauses.put(topLevelLabel, rootCause);
+ }
+ warnAboutLoadingFailure(topLevelLabel, eventHandler);
+ }
+ }
+ return false;
+ }
+
+ private static boolean hasErrors(EvaluationResult<TransitiveTargetValue> result) {
+ if (result.hasError()) {
+ return true;
+ }
+ for (TransitiveTargetValue transitiveTargetValue : result.values()) {
+ if (transitiveTargetValue.getTransitiveRootCauses() != null) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean isDirectErrorFromTopLevelLabel(Label label, Set<Label> topLevelLabels,
+ ErrorInfo errorInfo) {
+ return errorInfo.getException() != null && topLevelLabels.contains(label)
+ && Iterables.contains(errorInfo.getRootCauses(), TransitiveTargetValue.key(label));
+ }
+
+ private static void errorAboutLoadingFailure(Label topLevelLabel, @Nullable Throwable throwable,
+ EventHandler eventHandler) {
+ eventHandler.handle(Event.error(
+ "Loading of target '" + topLevelLabel + "' failed; build aborted" +
+ (throwable == null ? "" : ": " + throwable.getMessage())));
+ }
+
+ private static void warnAboutLoadingFailure(Label label, EventHandler eventHandler) {
+ eventHandler.handle(Event.warn(
+ // TODO(bazel-team): We use 'analyzing' here so that we print the same message as legacy
+ // Blaze. Once we get rid of legacy we should be able to change to 'loading' or
+ // similar.
+ "errors encountered while analyzing target '" + label + "': it will not be built"));
+ }
+
+ private static Set<Label> getRootCausesOfCycles(Label labelToLoad, Iterable<CycleInfo> cycles) {
+ ImmutableSet.Builder<Label> builder = ImmutableSet.builder();
+ for (CycleInfo cycleInfo : cycles) {
+ // The root cause of a cycle depends on the type of a cycle.
+
+ SkyKey culprit = Iterables.getFirst(cycleInfo.getCycle(), null);
+ if (culprit == null) {
+ continue;
+ }
+ if (culprit.functionName().equals(SkyFunctions.TRANSITIVE_TARGET)) {
+ // For a cycle between build targets, the root cause is the first element of the cycle.
+ builder.add((Label) culprit.argument());
+ } else {
+ // For other types of cycles (e.g. file symlink cycles), the root cause is the furthest
+ // target dependency that itself depended on the cycle.
+ Label furthestTarget = labelToLoad;
+ for (SkyKey skyKey : cycleInfo.getPathToCycle()) {
+ if (skyKey.functionName().equals(SkyFunctions.TRANSITIVE_TARGET)) {
+ furthestTarget = (Label) skyKey.argument();
+ } else {
+ break;
+ }
+ }
+ builder.add(furthestTarget);
+ }
+ }
+ return builder.build();
+ }
+
+ // Unfortunately we have to do an effective O(TC) visitation after the eval() call above to
+ // determine all of the packages in the closure.
+ private void updateVisitedValues(Collection<TransitiveTargetValue> targetValues) {
+ Set<TransitiveTargetValue> currentBuildTargetValueSet = new HashSet<>(targetValues);
+ if (Objects.equals(previousBuildTargetValueSet, currentBuildTargetValueSet)) {
+ // The next stanza is slow (and scales with the edge count of the target graph), so avoid
+ // the computation if the previous build already did it.
+ return;
+ }
+ NestedSetBuilder<PackageIdentifier> nestedAllPkgsBuilder = NestedSetBuilder.stableOrder();
+ NestedSetBuilder<PackageIdentifier> nestedErrorFreePkgsBuilder = NestedSetBuilder.stableOrder();
+ NestedSetBuilder<Label> nestedTargetBuilder = NestedSetBuilder.stableOrder();
+ for (TransitiveTargetValue value : targetValues) {
+ nestedAllPkgsBuilder.addTransitive(value.getTransitiveSuccessfulPackages());
+ nestedAllPkgsBuilder.addTransitive(value.getTransitiveUnsuccessfulPackages());
+ nestedErrorFreePkgsBuilder.addTransitive(value.getTransitiveSuccessfulPackages());
+ nestedTargetBuilder.addTransitive(value.getTransitiveTargets());
+ }
+ allVisitedPackages = nestedAllPkgsBuilder.build().toSet();
+ errorFreeVisitedPackages = nestedErrorFreePkgsBuilder.build().toSet();
+ visitedTargets = nestedTargetBuilder.build().toSet();
+ previousBuildTargetValueSet = currentBuildTargetValueSet;
+ }
+
+
+ @Override
+ public Set<PackageIdentifier> getVisitedPackageNames() {
+ return allVisitedPackages;
+ }
+
+ @Override
+ public Set<Package> getErrorFreeVisitedPackages() {
+ return transitivePackageLoader.retrievePackages(errorFreeVisitedPackages);
+ }
+
+ /**
+ * Doesn't necessarily include all top-level targets visited in error, because of issues with
+ * skyframe semantics (e.g. impossible to load a target if it transitively depends on a file
+ * symlink cycle). This is actually fine for the non-test usages of this method since such bad
+ * targets get filtered out.
+ */
+ @Override
+ public Set<Label> getVisitedTargets() {
+ return visitedTargets;
+ }
+
+ @Override
+ public Multimap<Label, Label> getRootCauses(final Collection<Label> targetsToLoad) {
+ Preconditions.checkState(lastBuildKeepGoing);
+ return Multimaps.filterKeys(rootCauses,
+ new Predicate<Label>() {
+ @Override
+ public boolean apply(Label label) {
+ return targetsToLoad.contains(label);
+ }
+ });
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageLoaderWithValueEnvironment.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageLoaderWithValueEnvironment.java
new file mode 100644
index 0000000..e467ae0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageLoaderWithValueEnvironment.java
@@ -0,0 +1,120 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.analysis.config.PackageProviderForConfigurations;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor.SkyframePackageLoader;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.MemoizingEvaluator;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+
+import java.io.IOException;
+import java.util.Set;
+
+/**
+ * Repeats functionality of {@link SkyframePackageLoader} but uses
+ * {@link SkyFunction.Environment#getValue} instead of {@link MemoizingEvaluator#evaluate}
+ * for node evaluation
+ */
+class SkyframePackageLoaderWithValueEnvironment implements
+ PackageProviderForConfigurations {
+ private final SkyFunction.Environment env;
+ private final Set<Package> packages;
+
+ public SkyframePackageLoaderWithValueEnvironment(SkyFunction.Environment env,
+ Set<Package> packages) {
+ this.env = env;
+ this.packages = packages;
+ }
+
+ private Package getPackage(PackageIdentifier pkgIdentifier) throws NoSuchPackageException{
+ SkyKey key = PackageValue.key(pkgIdentifier);
+ PackageValue value = (PackageValue) env.getValueOrThrow(key, NoSuchPackageException.class);
+ if (value != null) {
+ packages.add(value.getPackage());
+ return value.getPackage();
+ }
+ return null;
+ }
+
+ @Override
+ public Package getLoadedPackage(final PackageIdentifier pkgIdentifier)
+ throws NoSuchPackageException {
+ try {
+ return getPackage(pkgIdentifier);
+ } catch (NoSuchPackageException e) {
+ if (e.getPackage() != null) {
+ return e.getPackage();
+ }
+ throw e;
+ }
+ }
+
+ @Override
+ public Target getLoadedTarget(Label label) throws NoSuchPackageException,
+ NoSuchTargetException {
+ Package pkg = getLoadedPackage(label.getPackageIdentifier());
+ return pkg == null ? null : pkg.getTarget(label.getName());
+ }
+
+ @Override
+ public boolean isTargetCurrent(Target target) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void addDependency(Package pkg, String fileName) throws SyntaxException, IOException {
+ RootedPath fileRootedPath = RootedPath.toRootedPath(pkg.getSourceRoot(),
+ pkg.getNameFragment().getRelative(fileName));
+ FileValue result = (FileValue) env.getValue(FileValue.key(fileRootedPath));
+ if (result != null && !result.exists()) {
+ throw new IOException();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T extends Fragment> T getFragment(BuildOptions buildOptions, Class<T> fragmentType)
+ throws InvalidConfigurationException {
+ ConfigurationFragmentValue fragmentNode = (ConfigurationFragmentValue) env.getValueOrThrow(
+ ConfigurationFragmentValue.key(buildOptions, fragmentType),
+ InvalidConfigurationException.class);
+ if (fragmentNode == null) {
+ return null;
+ }
+ return (T) fragmentNode.getFragment();
+ }
+
+ @Override
+ public BlazeDirectories getDirectories() {
+ return PrecomputedValue.BLAZE_DIRECTORIES.get(env);
+ }
+
+ @Override
+ public boolean valuesMissing() {
+ return env.valuesMissing();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageManager.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageManager.java
new file mode 100644
index 0000000..cc32bf8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageManager.java
@@ -0,0 +1,177 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.cmdline.LabelValidator;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.PackageManager;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator;
+import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor.SkyframePackageLoader;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+import com.google.devtools.build.skyframe.CyclesReporter;
+
+import java.io.PrintStream;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Skyframe-based package manager.
+ *
+ * <p>This is essentially a compatibility shim between the native Skyframe and non-Skyframe
+ * parts of Blaze and should not be long-lived.
+ */
+class SkyframePackageManager implements PackageManager {
+
+ private final SkyframePackageLoader packageLoader;
+ private final SkyframeExecutor.SkyframeTransitivePackageLoader transitiveLoader;
+ private final TargetPatternEvaluator patternEvaluator;
+ private final AtomicReference<UnixGlob.FilesystemCalls> syscalls;
+ private final AtomicReference<CyclesReporter> skyframeCyclesReporter;
+ private final AtomicReference<PathPackageLocator> pkgLocator;
+ private final AtomicInteger numPackagesLoaded;
+ private final SkyframeExecutor skyframeExecutor;
+
+ public SkyframePackageManager(SkyframePackageLoader packageLoader,
+ SkyframeExecutor.SkyframeTransitivePackageLoader transitiveLoader,
+ TargetPatternEvaluator patternEvaluator,
+ AtomicReference<UnixGlob.FilesystemCalls> syscalls,
+ AtomicReference<CyclesReporter> skyframeCyclesReporter,
+ AtomicReference<PathPackageLocator> pkgLocator,
+ AtomicInteger numPackagesLoaded,
+ SkyframeExecutor skyframeExecutor) {
+ this.packageLoader = packageLoader;
+ this.transitiveLoader = transitiveLoader;
+ this.patternEvaluator = patternEvaluator;
+ this.skyframeCyclesReporter = skyframeCyclesReporter;
+ this.pkgLocator = pkgLocator;
+ this.syscalls = syscalls;
+ this.numPackagesLoaded = numPackagesLoaded;
+ this.skyframeExecutor = skyframeExecutor;
+ }
+
+ @Override
+ public Package getLoadedPackage(PackageIdentifier pkgIdentifier) throws NoSuchPackageException {
+ return packageLoader.getLoadedPackage(pkgIdentifier);
+ }
+
+ @ThreadSafe
+ @Override
+ public Package getPackage(EventHandler eventHandler, PackageIdentifier packageIdentifier)
+ throws NoSuchPackageException, InterruptedException {
+ try {
+ return packageLoader.getPackage(eventHandler, packageIdentifier);
+ } catch (NoSuchPackageException e) {
+ if (e.getPackage() != null) {
+ return e.getPackage();
+ }
+ throw e;
+ }
+ }
+
+ @Override
+ public Target getLoadedTarget(Label label) throws NoSuchPackageException, NoSuchTargetException {
+ return getLoadedPackage(label.getPackageIdentifier()).getTarget(label.getName());
+ }
+
+ @Override
+ public Target getTarget(EventHandler eventHandler, Label label)
+ throws NoSuchPackageException, NoSuchTargetException, InterruptedException {
+ return getPackage(eventHandler, label.getPackageIdentifier()).getTarget(label.getName());
+ }
+
+ @Override
+ public boolean isTargetCurrent(Target target) {
+ Package pkg = target.getPackage();
+ try {
+ return getLoadedPackage(target.getLabel().getPackageIdentifier()) == pkg;
+ } catch (NoSuchPackageException e) {
+ return false;
+ }
+ }
+
+ @Override
+ public void partiallyClear() {
+ packageLoader.partiallyClear();
+ }
+
+ @Override
+ public PackageManagerStatistics getStatistics() {
+ return new PackageManagerStatistics() {
+ @Override
+ public int getPackagesLoaded() {
+ return numPackagesLoaded.get();
+ }
+
+ @Override
+ public int getPackagesLookedUp() {
+ return -1;
+ }
+
+ @Override
+ public int getCacheSize() {
+ return -1;
+ }
+ };
+ }
+
+ @Override
+ public boolean isPackage(String packageName) {
+ return getBuildFileForPackage(packageName) != null;
+ }
+
+ @Override
+ public void dump(PrintStream printStream) {
+ skyframeExecutor.dumpPackages(printStream);
+ }
+
+ @ThreadSafe
+ @Override
+ public Path getBuildFileForPackage(String packageName) {
+ // Note that this method needs to be thread-safe, as it is currently used concurrently by
+ // legacy blaze code.
+ if (packageLoader.isPackageDeleted(packageName)
+ || LabelValidator.validatePackageName(packageName) != null) {
+ return null;
+ }
+ // TODO(bazel-team): Use a PackageLookupValue here [skyframe-loading]
+ // TODO(bazel-team): The implementation in PackageCache also checks for duplicate packages, see
+ // BuildFileCache#getBuildFile [skyframe-loading]
+ return pkgLocator.get().getPackageBuildFileNullable(packageName, syscalls);
+ }
+
+ @Override
+ public PathPackageLocator getPackagePath() {
+ return pkgLocator.get();
+ }
+
+ @Override
+ public TransitivePackageLoader newTransitiveLoader() {
+ return new SkyframeLabelVisitor(transitiveLoader, skyframeCyclesReporter);
+ }
+
+ @Override
+ public TargetPatternEvaluator getTargetPatternEvaluator() {
+ return patternEvaluator;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeTargetPatternEvaluator.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeTargetPatternEvaluator.java
new file mode 100644
index 0000000..9e619e3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeTargetPatternEvaluator.java
@@ -0,0 +1,146 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.cmdline.ResolvedTargets;
+import com.google.devtools.build.lib.cmdline.TargetParsingException;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.FilteringPolicies;
+import com.google.devtools.build.lib.pkgcache.FilteringPolicy;
+import com.google.devtools.build.lib.pkgcache.ParseFailureListener;
+import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.ErrorInfo;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.SkyKey;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Skyframe-based target pattern parsing.
+ */
+final class SkyframeTargetPatternEvaluator implements TargetPatternEvaluator {
+ private final SkyframeExecutor skyframeExecutor;
+ private String offset = "";
+
+ SkyframeTargetPatternEvaluator(SkyframeExecutor skyframeExecutor) {
+ this.skyframeExecutor = skyframeExecutor;
+ }
+
+ @Override
+ public ResolvedTargets<Target> parseTargetPatternList(EventHandler eventHandler,
+ List<String> targetPatterns, FilteringPolicy policy, boolean keepGoing)
+ throws TargetParsingException, InterruptedException {
+ return parseTargetPatternList(offset, eventHandler, targetPatterns, policy, keepGoing);
+ }
+
+ @Override
+ public ResolvedTargets<Target> parseTargetPattern(EventHandler eventHandler,
+ String pattern, boolean keepGoing) throws TargetParsingException, InterruptedException {
+ return parseTargetPatternList(eventHandler, ImmutableList.of(pattern),
+ FilteringPolicies.NO_FILTER, keepGoing);
+ }
+
+ @Override
+ public void updateOffset(PathFragment relativeWorkingDirectory) {
+ offset = relativeWorkingDirectory.getPathString();
+ }
+
+ @Override
+ public String getOffset() {
+ return offset;
+ }
+
+ @Override
+ public Map<String, ResolvedTargets<Target>> preloadTargetPatterns(EventHandler eventHandler,
+ Collection<String> patterns, boolean keepGoing)
+ throws TargetParsingException, InterruptedException {
+ // TODO(bazel-team): This is used only in "blaze query". There are plans to dramatically change
+ // how query works on Skyframe, in which case this method is likely to go away.
+ // We cannot use an ImmutableMap here because there may be null values.
+ Map<String, ResolvedTargets<Target>> result = Maps.newHashMapWithExpectedSize(patterns.size());
+ for (String pattern : patterns) {
+ // TODO(bazel-team): This could be parallelized to improve performance. [skyframe-loading]
+ result.put(pattern, parseTargetPattern(eventHandler, pattern, keepGoing));
+ }
+ return result;
+ }
+
+ /**
+ * Loads a list of target patterns (eg, "foo/...").
+ */
+ ResolvedTargets<Target> parseTargetPatternList(String offset, EventHandler eventHandler,
+ List<String> targetPatterns, FilteringPolicy policy, boolean keepGoing)
+ throws InterruptedException, TargetParsingException {
+ Iterable<SkyKey> patternSkyKeys = TargetPatternValue.keys(targetPatterns, policy, offset);
+ EvaluationResult<TargetPatternValue> result =
+ skyframeExecutor.targetPatterns(patternSkyKeys, keepGoing, eventHandler);
+
+ String errorMessage = null;
+ ResolvedTargets.Builder<Target> builder = ResolvedTargets.builder();
+ for (SkyKey key : patternSkyKeys) {
+ TargetPatternValue resultValue = result.get(key);
+ if (resultValue != null) {
+ ResolvedTargets<Target> results = resultValue.getTargets();
+ if (((TargetPatternValue.TargetPattern) key.argument()).isNegative()) {
+ builder.filter(Predicates.not(Predicates.in(results.getTargets())));
+ } else {
+ builder.merge(results);
+ }
+ } else {
+ TargetPatternValue.TargetPattern pattern =
+ (TargetPatternValue.TargetPattern) key.argument();
+ String rawPattern = pattern.getPattern();
+ ErrorInfo error = result.errorMap().get(key);
+ if (error == null) {
+ Preconditions.checkState(!keepGoing);
+ continue;
+ }
+ if (error.getException() != null) {
+ errorMessage = error.getException().getMessage();
+ } else if (!Iterables.isEmpty(error.getCycleInfo())) {
+ errorMessage = "cycles detected during target parsing";
+ skyframeExecutor.getCyclesReporter().reportCycles(
+ error.getCycleInfo(), key, eventHandler);
+ } else {
+ throw new IllegalStateException(error.toString());
+ }
+ if (keepGoing) {
+ eventHandler.handle(Event.error("Skipping '" + rawPattern + "': " + errorMessage));
+ }
+ builder.setError();
+
+ if (eventHandler instanceof ParseFailureListener) {
+ ParseFailureListener parseListener = (ParseFailureListener) eventHandler;
+ parseListener.parsingError(rawPattern, errorMessage);
+ }
+ }
+ }
+
+ if (!keepGoing && result.hasError()) {
+ Preconditions.checkState(errorMessage != null, "unexpected errors: %s", result.errorMap());
+ throw new TargetParsingException(errorMessage);
+ }
+ return builder.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkFileDependency.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkFileDependency.java
new file mode 100644
index 0000000..02d2e91
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkFileDependency.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * A simple value class to store the direct Skylark file dependencies of a Skylark
+ * extension file. It also contains a Label identifying the extension file.
+ */
+class SkylarkFileDependency {
+
+ private final Label label;
+ private final ImmutableList<SkylarkFileDependency> dependencies;
+
+ SkylarkFileDependency(Label label, ImmutableList<SkylarkFileDependency> dependencies) {
+ this.label = label;
+ this.dependencies = dependencies;
+ }
+
+ /**
+ * Returns the list of direct Skylark file dependencies of the Skylark extension file
+ * corresponding to this object.
+ */
+ ImmutableList<SkylarkFileDependency> getDependencies() {
+ return dependencies;
+ }
+
+ /**
+ * Returns the Label of the Skylark extension file corresponding to this object.
+ */
+ Label getLabel() {
+ return label;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupFunction.java
new file mode 100644
index 0000000..02d41e6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupFunction.java
@@ -0,0 +1,238 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.packages.BuildFileNotFoundException;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.packages.RuleClassProvider;
+import com.google.devtools.build.lib.skyframe.ASTFileLookupValue.ASTLookupInputException;
+import com.google.devtools.build.lib.syntax.BuildFileAST;
+import com.google.devtools.build.lib.syntax.Function;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.syntax.SkylarkEnvironment;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A Skyframe function to look up and import a single Skylark extension.
+ */
+public class SkylarkImportLookupFunction implements SkyFunction {
+
+ private final RuleClassProvider ruleClassProvider;
+ private final ImmutableList<Function> nativeRuleFunctions;
+
+ public SkylarkImportLookupFunction(
+ RuleClassProvider ruleClassProvider, PackageFactory packageFactory) {
+ this.ruleClassProvider = ruleClassProvider;
+ this.nativeRuleFunctions = packageFactory.collectNativeRuleFunctions();
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException,
+ InterruptedException {
+ PackageIdentifier arg = (PackageIdentifier) skyKey.argument();
+ PathFragment file = arg.getPackageFragment();
+ ASTFileLookupValue astLookupValue = null;
+ try {
+ SkyKey astLookupKey = ASTFileLookupValue.key(file);
+ astLookupValue = (ASTFileLookupValue) env.getValueOrThrow(astLookupKey,
+ ErrorReadingSkylarkExtensionException.class, InconsistentFilesystemException.class);
+ } catch (ErrorReadingSkylarkExtensionException e) {
+ throw new SkylarkImportLookupFunctionException(SkylarkImportFailedException.errorReadingFile(
+ file, e.getMessage()));
+ } catch (InconsistentFilesystemException e) {
+ throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT);
+ } catch (ASTLookupInputException e) {
+ throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT);
+ }
+ if (astLookupValue == null) {
+ return null;
+ }
+ if (astLookupValue == ASTFileLookupValue.NO_FILE) {
+ // Skylark import files have to exist.
+ throw new SkylarkImportLookupFunctionException(SkylarkImportFailedException.noFile(file));
+ }
+
+ Map<PathFragment, SkylarkEnvironment> importMap = new HashMap<>();
+ ImmutableList.Builder<SkylarkFileDependency> fileDependencies = ImmutableList.builder();
+ BuildFileAST ast = astLookupValue.getAST();
+ // TODO(bazel-team): Refactor this code and PackageFunction to reduce code duplications.
+ for (PathFragment importFile : ast.getImports()) {
+ try {
+ SkyKey importsLookupKey = SkylarkImportLookupValue.key(arg.getRepository(), importFile);
+ SkylarkImportLookupValue importsLookupValue;
+ importsLookupValue = (SkylarkImportLookupValue) env.getValueOrThrow(
+ importsLookupKey, ASTLookupInputException.class);
+ if (importsLookupValue != null) {
+ importMap.put(importFile, importsLookupValue.getImportedEnvironment());
+ fileDependencies.add(importsLookupValue.getDependency());
+ }
+ } catch (ASTLookupInputException e) {
+ throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT);
+ }
+ }
+ Label label = pathFragmentToLabel(arg.getRepository(), file, env);
+ if (env.valuesMissing()) {
+ // This means some imports are unavailable.
+ return null;
+ }
+
+ if (ast.containsErrors()) {
+ throw new SkylarkImportLookupFunctionException(SkylarkImportFailedException.skylarkErrors(
+ file));
+ }
+
+ SkylarkEnvironment extensionEnv = createEnv(ast, importMap, env);
+ // Skylark UserDefinedFunctions are sharing function definition Environments, so it's extremely
+ // important not to modify them from this point. Ideally they should be only used to import
+ // symbols and serve as global Environments of UserDefinedFunctions.
+ return new SkylarkImportLookupValue(
+ extensionEnv, new SkylarkFileDependency(label, fileDependencies.build()));
+ }
+
+ /**
+ * Converts the PathFragment of the Skylark file to a Label using the BUILD file closest to the
+ * Skylark file in its directory hierarchy - finds the package to which the Skylark file belongs.
+ * Throws an exception if no such BUILD file exists.
+ */
+ private Label pathFragmentToLabel(RepositoryName repo, PathFragment file, Environment env)
+ throws SkylarkImportLookupFunctionException {
+ ContainingPackageLookupValue containingPackageLookupValue = null;
+ try {
+ PackageIdentifier newPkgId = new PackageIdentifier(repo, file.getParentDirectory());
+ containingPackageLookupValue = (ContainingPackageLookupValue) env.getValueOrThrow(
+ ContainingPackageLookupValue.key(newPkgId),
+ BuildFileNotFoundException.class, InconsistentFilesystemException.class);
+ } catch (BuildFileNotFoundException e) {
+ // Thrown when there are IO errors looking for BUILD files.
+ throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT);
+ } catch (InconsistentFilesystemException e) {
+ throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT);
+ }
+
+ if (containingPackageLookupValue == null) {
+ return null;
+ }
+
+ if (!containingPackageLookupValue.hasContainingPackage()) {
+ throw new SkylarkImportLookupFunctionException(
+ SkylarkImportFailedException.noBuildFile(file));
+ }
+
+ PathFragment pkgName =
+ containingPackageLookupValue.getContainingPackageName().getPackageFragment();
+ PathFragment fileInPkg = file.relativeTo(pkgName);
+
+ try {
+ // This code relies on PackageIdentifier.RepositoryName.toString()
+ return Label.parseRepositoryLabel(repo + "//" + pkgName.getPathString() + ":" + fileInPkg);
+ } catch (SyntaxException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /**
+ * Creates the SkylarkEnvironment to be imported. After it's returned, the Environment
+ * must not be modified.
+ */
+ private SkylarkEnvironment createEnv(BuildFileAST ast,
+ Map<PathFragment, SkylarkEnvironment> importMap, Environment env)
+ throws InterruptedException {
+ StoredEventHandler eventHandler = new StoredEventHandler();
+ // TODO(bazel-team): this method overestimates the changes which can affect the
+ // Skylark RuleClass. For example changes to comments or unused functions can modify the hash.
+ // A more accurate - however much more complicated - way would be to calculate a hash based on
+ // the transitive closure of the accessible AST nodes.
+ SkylarkEnvironment extensionEnv = ruleClassProvider
+ .createSkylarkRuleClassEnvironment(eventHandler, ast.getContentHashCode());
+ // Adding native rules module for build extensions.
+ // TODO(bazel-team): this might not be the best place to do this.
+ extensionEnv.update("native", ruleClassProvider.getNativeModule());
+ for (Function function : nativeRuleFunctions) {
+ extensionEnv.registerFunction(
+ ruleClassProvider.getNativeModule().getClass(), function.getName(), function);
+ }
+ extensionEnv.setImportedExtensions(importMap);
+ ast.exec(extensionEnv, eventHandler);
+ // Don't fail just replay the events so the original package lookup can fail.
+ Event.replayEventsOn(env.getListener(), eventHandler.getEvents());
+ return extensionEnv;
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ static final class SkylarkImportFailedException extends Exception {
+ private SkylarkImportFailedException(String errorMessage) {
+ super(errorMessage);
+ }
+
+ static SkylarkImportFailedException errorReadingFile(PathFragment file, String error) {
+ return new SkylarkImportFailedException(
+ String.format("Encountered error while reading extension file '%s': %s", file, error));
+ }
+
+ static SkylarkImportFailedException noFile(PathFragment file) {
+ return new SkylarkImportFailedException(
+ String.format("Extension file not found: '%s'", file));
+ }
+
+ static SkylarkImportFailedException noBuildFile(PathFragment file) {
+ return new SkylarkImportFailedException(
+ String.format("Every .bzl file must have a corresponding package, but '%s' "
+ + "does not have one. Please create a BUILD file in the same or any parent directory."
+ + " Note that this BUILD file does not need to do anything except exist.", file));
+ }
+
+ static SkylarkImportFailedException skylarkErrors(PathFragment file) {
+ return new SkylarkImportFailedException(String.format("Extension '%s' has errors", file));
+ }
+ }
+
+ private static final class SkylarkImportLookupFunctionException extends SkyFunctionException {
+ private SkylarkImportLookupFunctionException(SkylarkImportFailedException cause) {
+ super(cause, Transience.PERSISTENT);
+ }
+
+ private SkylarkImportLookupFunctionException(InconsistentFilesystemException e,
+ Transience transience) {
+ super(e, transience);
+ }
+
+ private SkylarkImportLookupFunctionException(ASTLookupInputException e,
+ Transience transience) {
+ super(e, transience);
+ }
+
+ private SkylarkImportLookupFunctionException(BuildFileNotFoundException e,
+ Transience transience) {
+ super(e, transience);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupValue.java
new file mode 100644
index 0000000..3c87431
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupValue.java
@@ -0,0 +1,70 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.skyframe.ASTFileLookupValue.ASTLookupInputException;
+import com.google.devtools.build.lib.syntax.SkylarkEnvironment;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A value that represents a Skylark import lookup result. The lookup value corresponds to
+ * exactly one Skylark file, identified by the PathFragment SkyKey argument.
+ */
+public class SkylarkImportLookupValue implements SkyValue {
+
+ private final SkylarkEnvironment importedEnvironment;
+ /**
+ * The immediate Skylark file dependency descriptor class corresponding to this value.
+ * Using this reference it's possible to reach the transitive closure of Skylark files
+ * on which this Skylark file depends.
+ */
+ private final SkylarkFileDependency dependency;
+
+ public SkylarkImportLookupValue(
+ SkylarkEnvironment importedEnvironment, SkylarkFileDependency dependency) {
+ this.importedEnvironment = Preconditions.checkNotNull(importedEnvironment);
+ this.dependency = Preconditions.checkNotNull(dependency);
+ }
+
+ /**
+ * Returns the imported SkylarkEnvironment.
+ */
+ public SkylarkEnvironment getImportedEnvironment() {
+ return importedEnvironment;
+ }
+
+ /**
+ * Returns the immediate Skylark file dependency corresponding to this import lookup value.
+ */
+ public SkylarkFileDependency getDependency() {
+ return dependency;
+ }
+
+ static SkyKey key(PackageIdentifier pkgIdentifier) throws ASTLookupInputException {
+ return key(pkgIdentifier.getRepository(), pkgIdentifier.getPackageFragment());
+ }
+
+ static SkyKey key(RepositoryName repo, PathFragment fileToImport) throws ASTLookupInputException {
+ // Skylark import lookup keys need to be valid AST file lookup keys.
+ ASTFileLookupValue.checkInputArgument(fileToImport);
+ return new SkyKey(
+ SkyFunctions.SKYLARK_IMPORTS_LOOKUP,
+ new PackageIdentifier(repo, fileToImport));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkModuleCycleReporter.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkModuleCycleReporter.java
new file mode 100644
index 0000000..a0f37a9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkModuleCycleReporter.java
@@ -0,0 +1,71 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.skyframe.CycleInfo;
+import com.google.devtools.build.skyframe.CyclesReporter;
+import com.google.devtools.build.skyframe.SkyKey;
+
+/**
+ * Reports cycles of recursive import of Skylark files.
+ */
+public class SkylarkModuleCycleReporter implements CyclesReporter.SingleCycleReporter {
+
+ private static final Predicate<SkyKey> IS_SKYLARK_MODULE_SKY_KEY =
+ SkyFunctions.isSkyFunction(SkyFunctions.SKYLARK_IMPORTS_LOOKUP);
+
+ private static final Predicate<SkyKey> IS_PACKAGE_SKY_KEY =
+ SkyFunctions.isSkyFunction(SkyFunctions.PACKAGE);
+
+ @Override
+ public boolean maybeReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo, boolean alreadyReported,
+ EventHandler eventHandler) {
+ ImmutableList<SkyKey> pathToCycle = cycleInfo.getPathToCycle();
+ if (pathToCycle.size() == 0) {
+ return false;
+ }
+ SkyKey lastPathElement = cycleInfo.getPathToCycle().get(pathToCycle.size() - 1);
+ if (alreadyReported) {
+ return true;
+ } else if (Iterables.all(cycleInfo.getCycle(), IS_SKYLARK_MODULE_SKY_KEY)
+ // The last element of the path to the cycle has to be a PackageFunction.
+ && IS_PACKAGE_SKY_KEY.apply(lastPathElement)) {
+ StringBuilder cycleMessage = new StringBuilder()
+ .append(((PackageIdentifier) lastPathElement.argument()).toString() + "/BUILD: ")
+ .append("cycle in referenced extension files: ");
+
+ AbstractLabelCycleReporter.printCycle(cycleInfo.getCycle(), cycleMessage,
+ new Function<SkyKey, String>() {
+ @Override
+ public String apply(SkyKey input) {
+ return ((PackageIdentifier) input.argument()).toString();
+ }
+ });
+
+ // TODO(bazel-team): it would be nice to pass the Location of the load Statement in the
+ // BUILD file.
+ eventHandler.handle(Event.error(null, cycleMessage.toString()));
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetCompletionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetCompletionFunction.java
new file mode 100644
index 0000000..2e4aea9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetCompletionFunction.java
@@ -0,0 +1,138 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.MissingInputFileException;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
+import com.google.devtools.build.lib.analysis.TargetCompleteEvent;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactHelper;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import com.google.devtools.build.skyframe.ValueOrException2;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * TargetCompletionFunction builds the artifactsToBuild collection of a {@link ConfiguredTarget}.
+ */
+public final class TargetCompletionFunction implements SkyFunction {
+
+ private final AtomicReference<EventBus> eventBusRef;
+
+ public TargetCompletionFunction(AtomicReference<EventBus> eventBusRef) {
+ this.eventBusRef = eventBusRef;
+ }
+
+ @Nullable
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws TargetCompletionFunctionException {
+ LabelAndConfiguration lac = (LabelAndConfiguration) skyKey.argument();
+ ConfiguredTargetValue ctValue = (ConfiguredTargetValue)
+ env.getValue(ConfiguredTargetValue.key(lac.getLabel(), lac.getConfiguration()));
+ TopLevelArtifactContext topLevelContext = PrecomputedValue.TOP_LEVEL_CONTEXT.get(env);
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ Map<SkyKey, ValueOrException2<MissingInputFileException, ActionExecutionException>> inputDeps =
+ env.getValuesOrThrow(ArtifactValue.mandatoryKeys(
+ TopLevelArtifactHelper.getAllArtifactsToBuild(
+ ctValue.getConfiguredTarget(), topLevelContext)),
+ MissingInputFileException.class, ActionExecutionException.class);
+
+ int missingCount = 0;
+ ActionExecutionException firstActionExecutionException = null;
+ MissingInputFileException missingInputException = null;
+ NestedSetBuilder<Label> rootCausesBuilder = NestedSetBuilder.stableOrder();
+ for (Map.Entry<SkyKey, ValueOrException2<MissingInputFileException,
+ ActionExecutionException>> depsEntry : inputDeps.entrySet()) {
+ Artifact input = ArtifactValue.artifact(depsEntry.getKey());
+ try {
+ depsEntry.getValue().get();
+ } catch (MissingInputFileException e) {
+ missingCount++;
+ if (input.getOwner() != null) {
+ rootCausesBuilder.add(input.getOwner());
+ env.getListener().handle(Event.error(
+ ctValue.getConfiguredTarget().getTarget().getLocation(),
+ String.format("%s: missing input file '%s'",
+ lac.getLabel(), input.getOwner())));
+ }
+ } catch (ActionExecutionException e) {
+ rootCausesBuilder.addTransitive(e.getRootCauses());
+ if (firstActionExecutionException == null) {
+ firstActionExecutionException = e;
+ }
+ }
+ }
+
+ if (missingCount > 0) {
+ missingInputException = new MissingInputFileException(
+ ctValue.getConfiguredTarget().getTarget().getLocation() + " " + missingCount
+ + " input file(s) do not exist", ctValue.getConfiguredTarget().getTarget().getLocation());
+ }
+
+ NestedSet<Label> rootCauses = rootCausesBuilder.build();
+ if (!rootCauses.isEmpty()) {
+ eventBusRef.get().post(
+ TargetCompleteEvent.createFailed(ctValue.getConfiguredTarget(), rootCauses));
+ if (firstActionExecutionException != null) {
+ throw new TargetCompletionFunctionException(firstActionExecutionException);
+ } else {
+ throw new TargetCompletionFunctionException(missingInputException);
+ }
+ }
+
+ return env.valuesMissing() ? null : new TargetCompletionValue(ctValue.getConfiguredTarget());
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return Label.print(((LabelAndConfiguration) skyKey.argument()).getLabel());
+ }
+
+ private static final class TargetCompletionFunctionException extends SkyFunctionException {
+
+ private final ActionExecutionException actionException;
+
+ public TargetCompletionFunctionException(ActionExecutionException e) {
+ super(e, Transience.PERSISTENT);
+ this.actionException = e;
+ }
+
+ public TargetCompletionFunctionException(MissingInputFileException e) {
+ super(e, Transience.TRANSIENT);
+ this.actionException = null;
+ }
+
+ @Override
+ public boolean isCatastrophic() {
+ return actionException != null && actionException.isCatastrophe();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetCompletionValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetCompletionValue.java
new file mode 100644
index 0000000..5d7153d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetCompletionValue.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Collection;
+
+/**
+ * The value of a TargetCompletion. Currently this just stores a ConfiguredTarget.
+ */
+public class TargetCompletionValue implements SkyValue {
+ private final ConfiguredTarget ct;
+
+ TargetCompletionValue(ConfiguredTarget ct) {
+ this.ct = ct;
+ }
+
+ public ConfiguredTarget getConfiguredTarget() {
+ return ct;
+ }
+
+ public static SkyKey key(LabelAndConfiguration labelAndConfiguration) {
+ return new SkyKey(SkyFunctions.TARGET_COMPLETION, labelAndConfiguration);
+ }
+
+ public static Iterable<SkyKey> keys(Collection<ConfiguredTarget> targets) {
+ return Iterables.transform(targets, new Function<ConfiguredTarget, SkyKey>() {
+ @Override
+ public SkyKey apply(ConfiguredTarget ct) {
+ return new SkyKey(SkyFunctions.TARGET_COMPLETION, new LabelAndConfiguration(ct));
+ }
+ });
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetMarkerFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetMarkerFunction.java
new file mode 100644
index 0000000..3f9f22f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetMarkerFunction.java
@@ -0,0 +1,134 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.packages.BuildFileNotFoundException;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A SkyFunction for {@link TargetMarkerValue}s.
+ */
+public final class TargetMarkerFunction implements SkyFunction {
+
+ public TargetMarkerFunction() {
+ }
+
+ @Override
+ public SkyValue compute(SkyKey key, Environment env) throws TargetMarkerFunctionException {
+ Label label = (Label) key.argument();
+ PathFragment pkgForLabel = label.getPackageFragment();
+
+ if (label.getName().contains("/")) {
+ // This target is in a subdirectory, therefore it could potentially be invalidated by
+ // a new BUILD file appearing in the hierarchy.
+ PathFragment containingDirectory = label.toPathFragment().getParentDirectory();
+ ContainingPackageLookupValue containingPackageLookupValue = null;
+ try {
+ PackageIdentifier newPkgId = new PackageIdentifier(
+ label.getPackageIdentifier().getRepository(), containingDirectory);
+ containingPackageLookupValue = (ContainingPackageLookupValue) env.getValueOrThrow(
+ ContainingPackageLookupValue.key(newPkgId),
+ BuildFileNotFoundException.class, InconsistentFilesystemException.class);
+ } catch (BuildFileNotFoundException e) {
+ // Thrown when there are IO errors looking for BUILD files.
+ throw new TargetMarkerFunctionException(e);
+ } catch (InconsistentFilesystemException e) {
+ throw new TargetMarkerFunctionException(new NoSuchTargetException(label,
+ e.getMessage()));
+ }
+ if (containingPackageLookupValue == null) {
+ return null;
+ }
+ if (!containingPackageLookupValue.hasContainingPackage()) {
+ // This means the label's package doesn't exist. E.g. there is no package 'a' and we are
+ // trying to build the target for label 'a:b/foo'.
+ throw new TargetMarkerFunctionException(new BuildFileNotFoundException(
+ pkgForLabel.getPathString(), "BUILD file not found on package path for '"
+ + pkgForLabel.getPathString() + "'"));
+ }
+ if (!containingPackageLookupValue.getContainingPackageName().equals(
+ label.getPackageIdentifier())) {
+ throw new TargetMarkerFunctionException(new NoSuchTargetException(label,
+ String.format("Label '%s' crosses boundary of subpackage '%s'", label,
+ containingPackageLookupValue.getContainingPackageName())));
+ }
+ }
+
+ SkyKey pkgSkyKey = PackageValue.key(label.getPackageIdentifier());
+ NoSuchPackageException nspe = null;
+ Package pkg;
+ try {
+ PackageValue value = (PackageValue)
+ env.getValueOrThrow(pkgSkyKey, NoSuchPackageException.class);
+ if (value == null) {
+ return null;
+ }
+ pkg = value.getPackage();
+ } catch (NoSuchPackageException e) {
+ // For consistency with pre-Skyframe Blaze, we can return a valid Target from a Package
+ // containing errors.
+ pkg = e.getPackage();
+ if (pkg == null) {
+ // Re-throw this exception with our key because root causes should be targets, not packages.
+ throw new TargetMarkerFunctionException(e);
+ }
+ nspe = e;
+ }
+
+ Target target;
+ try {
+ target = pkg.getTarget(label.getName());
+ } catch (NoSuchTargetException e) {
+ throw new TargetMarkerFunctionException(e);
+ }
+
+ if (nspe != null) {
+ // There is a target, but its package is in error. We rethrow so that the root cause is the
+ // target, not the package. Note that targets are only in error when their package is
+ // "in error" (because a package is in error if there was an error evaluating the package, or
+ // if one of its targets was in error).
+ throw new TargetMarkerFunctionException(new NoSuchTargetException(target, nspe));
+ }
+ return TargetMarkerValue.TARGET_MARKER_INSTANCE;
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return Label.print((Label) skyKey.argument());
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link TargetMarkerFunction#compute}.
+ */
+ private static final class TargetMarkerFunctionException extends SkyFunctionException {
+ public TargetMarkerFunctionException(NoSuchTargetException e) {
+ super(e, Transience.PERSISTENT);
+ }
+
+ public TargetMarkerFunctionException(NoSuchPackageException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetMarkerValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetMarkerValue.java
new file mode 100644
index 0000000..eef7084
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetMarkerValue.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * Value represents visited target in the Skyframe graph after error checking.
+ */
+@Immutable
+@ThreadSafe
+public final class TargetMarkerValue implements SkyValue {
+
+ static final TargetMarkerValue TARGET_MARKER_INSTANCE = new TargetMarkerValue();
+
+ private TargetMarkerValue() {
+ }
+
+ @ThreadSafe
+ public static SkyKey key(Label label) {
+ return new SkyKey(SkyFunctions.TARGET_MARKER, label);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternFunction.java
new file mode 100644
index 0000000..6e631ed
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternFunction.java
@@ -0,0 +1,278 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.cmdline.LabelValidator;
+import com.google.devtools.build.lib.cmdline.ResolvedTargets;
+import com.google.devtools.build.lib.cmdline.TargetParsingException;
+import com.google.devtools.build.lib.cmdline.TargetPattern;
+import com.google.devtools.build.lib.cmdline.TargetPatternResolver;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.FilteringPolicies;
+import com.google.devtools.build.lib.pkgcache.FilteringPolicy;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.pkgcache.TargetPatternResolverUtil;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * TargetPatternFunction translates a target pattern (eg, "foo/...") into a set of resolved
+ * Targets.
+ */
+public class TargetPatternFunction implements SkyFunction {
+
+ private final AtomicReference<PathPackageLocator> pkgPath;
+
+ public TargetPatternFunction(AtomicReference<PathPackageLocator> pkgPath) {
+ this.pkgPath = pkgPath;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey key, Environment env) throws TargetPatternFunctionException,
+ InterruptedException {
+ TargetPatternValue.TargetPattern patternKey =
+ ((TargetPatternValue.TargetPattern) key.argument());
+
+ TargetPattern.Parser parser = new TargetPattern.Parser(patternKey.getOffset());
+ try {
+ Resolver resolver = new Resolver(env, patternKey.getPolicy(), pkgPath);
+ TargetPattern resolvedPattern = parser.parse(patternKey.getPattern());
+ return new TargetPatternValue(resolvedPattern.eval(resolver));
+ } catch (TargetParsingException e) {
+ throw new TargetPatternFunctionException(e);
+ } catch (TargetPatternResolver.MissingDepException e) {
+ return null;
+ }
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ private static class Resolver implements TargetPatternResolver<Target> {
+ private final Environment env;
+ private final FilteringPolicy policy;
+ private final AtomicReference<PathPackageLocator> pkgPath;
+
+ public Resolver(Environment env, FilteringPolicy policy,
+ AtomicReference<PathPackageLocator> pkgPath) {
+ this.policy = policy;
+ this.env = env;
+ this.pkgPath = pkgPath;
+ }
+
+ @Override
+ public void warn(String msg) {
+ env.getListener().handle(Event.warn(msg));
+ }
+
+ /**
+ * Gets a Package via the Skyframe env. May return a Package that has errors.
+ */
+ private Package getPackage(PackageIdentifier pkgIdentifier)
+ throws MissingDepException, NoSuchThingException {
+ SkyKey pkgKey = PackageValue.key(pkgIdentifier);
+ Package pkg;
+ try {
+ PackageValue pkgValue =
+ (PackageValue) env.getValueOrThrow(pkgKey, NoSuchThingException.class);
+ if (pkgValue == null) {
+ throw new MissingDepException();
+ }
+ pkg = pkgValue.getPackage();
+ } catch (NoSuchPackageException e) {
+ pkg = e.getPackage();
+ if (pkg == null) {
+ throw e;
+ }
+ }
+ return pkg;
+ }
+
+ @Override
+ public Target getTargetOrNull(String targetName) throws InterruptedException,
+ MissingDepException {
+ try {
+ Label label = Label.parseAbsolute(targetName);
+ if (!isPackage(label.getPackageName())) {
+ return null;
+ }
+ Package pkg = getPackage(label.getPackageIdentifier());
+ return pkg.getTarget(label.getName());
+ } catch (Label.SyntaxException | NoSuchThingException e) {
+ return null;
+ }
+ }
+
+ @Override
+ public ResolvedTargets<Target> getExplicitTarget(String targetName)
+ throws TargetParsingException, InterruptedException, MissingDepException {
+ Label label = TargetPatternResolverUtil.label(targetName);
+ try {
+ Package pkg = getPackage(label.getPackageIdentifier());
+ Target target = pkg.getTarget(label.getName());
+ return policy.shouldRetain(target, true)
+ ? ResolvedTargets.of(target)
+ : ResolvedTargets.<Target>empty();
+ } catch (NoSuchThingException e) {
+ throw new TargetParsingException(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public ResolvedTargets<Target> getTargetsInPackage(String originalPattern, String packageName,
+ boolean rulesOnly)
+ throws TargetParsingException, InterruptedException, MissingDepException {
+ FilteringPolicy actualPolicy = rulesOnly
+ ? FilteringPolicies.and(FilteringPolicies.RULES_ONLY, policy)
+ : policy;
+ return getTargetsInPackage(originalPattern, packageName, actualPolicy);
+ }
+
+ private ResolvedTargets<Target> getTargetsInPackage(String originalPattern, String packageName,
+ FilteringPolicy policy)
+ throws TargetParsingException, MissingDepException {
+ // Normalise, e.g "foo//bar" -> "foo/bar"; "foo/" -> "foo":
+ PathFragment packageNameFragment = new PathFragment(packageName);
+ packageName = packageNameFragment.toString();
+
+ // It's possible for this check to pass, but for
+ // Label.validatePackageNameFull to report an error because the
+ // package name is illegal. That's a little weird, but we can live with
+ // that for now--see test case: testBadPackageNameButGoodEnoughForALabel.
+ // (BTW I tried duplicating that validation logic in Label but it was
+ // extremely tricky.)
+ if (LabelValidator.validatePackageName(packageName) != null) {
+ throw new TargetParsingException("'" + packageName + "' is not a valid package name");
+ }
+ if (!isPackage(packageName)) {
+ throw new TargetParsingException(
+ TargetPatternResolverUtil.getParsingErrorMessage(
+ "no such package '" + packageName + "': BUILD file not found on package path",
+ originalPattern));
+ }
+
+ try {
+ Package pkg = getPackage(
+ PackageIdentifier.createInDefaultRepo(packageNameFragment.toString()));
+ return TargetPatternResolverUtil.resolvePackageTargets(pkg, policy);
+ } catch (NoSuchThingException e) {
+ String message = TargetPatternResolverUtil.getParsingErrorMessage(
+ "package contains errors", originalPattern);
+ throw new TargetParsingException(message, e);
+ }
+ }
+
+ @Override
+ public boolean isPackage(String packageName) throws MissingDepException {
+ SkyKey packageLookupKey;
+ packageLookupKey = PackageLookupValue.key(new PathFragment(packageName));
+ PackageLookupValue packageLookupValue = (PackageLookupValue) env.getValue(packageLookupKey);
+ if (packageLookupValue == null) {
+ throw new MissingDepException();
+ }
+ return packageLookupValue.packageExists();
+ }
+
+ @Override
+ public String getTargetKind(Target target) {
+ return target.getTargetKind();
+ }
+
+ @Override
+ public ResolvedTargets<Target> findTargetsBeneathDirectory(
+ String originalPattern, String pathPrefix, boolean rulesOnly)
+ throws TargetParsingException, MissingDepException {
+ FilteringPolicy actualPolicy = rulesOnly
+ ? FilteringPolicies.and(FilteringPolicies.RULES_ONLY, policy)
+ : policy;
+
+ PathFragment directory = new PathFragment(pathPrefix);
+ if (directory.containsUplevelReferences()) {
+ throw new TargetParsingException("up-level references are not permitted: '"
+ + directory.getPathString() + "'");
+ }
+ if (!pathPrefix.isEmpty() && (LabelValidator.validatePackageName(pathPrefix) != null)) {
+ throw new TargetParsingException("'" + pathPrefix + "' is not a valid package name");
+ }
+
+ ResolvedTargets.Builder<Target> builder = ResolvedTargets.builder();
+
+ List<RecursivePkgValue> lookupValues = new ArrayList<>();
+ for (Path root : pkgPath.get().getPathEntries()) {
+ SkyKey key = RecursivePkgValue.key(RootedPath.toRootedPath(root, directory));
+ RecursivePkgValue lookup = (RecursivePkgValue) env.getValue(key);
+ if (lookup != null) {
+ lookupValues.add(lookup);
+ }
+ }
+ if (env.valuesMissing()) {
+ throw new MissingDepException();
+ }
+
+ for (RecursivePkgValue value : lookupValues) {
+ for (String pkg : value.getPackages()) {
+ builder.merge(getTargetsInPackage(originalPattern, pkg, FilteringPolicies.NO_FILTER));
+ }
+ }
+
+ if (builder.isEmpty()) {
+ throw new TargetParsingException("no targets found beneath '" + directory + "'");
+ }
+
+ // Apply the transform after the check so we only return the
+ // error if the tree really contains no targets.
+ ResolvedTargets<Target> intermediateResult = builder.build();
+ ResolvedTargets.Builder<Target> filteredBuilder = ResolvedTargets.builder();
+ if (intermediateResult.hasError()) {
+ filteredBuilder.setError();
+ }
+ for (Target target : intermediateResult.getTargets()) {
+ if (actualPolicy.shouldRetain(target, false)) {
+ filteredBuilder.add(target);
+ }
+ }
+ return filteredBuilder.build();
+ }
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link TargetPatternFunction#compute}.
+ */
+ private static final class TargetPatternFunctionException extends SkyFunctionException {
+ public TargetPatternFunctionException(TargetParsingException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternValue.java
new file mode 100644
index 0000000..c97194f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternValue.java
@@ -0,0 +1,212 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.cmdline.ResolvedTargets;
+import com.google.devtools.build.lib.cmdline.ResolvedTargets.Builder;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.FilteringPolicies;
+import com.google.devtools.build.lib.pkgcache.FilteringPolicy;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * A value referring to a computed set of resolved targets. This is used for the results of target
+ * pattern parsing.
+ */
+@Immutable
+@ThreadSafe
+public final class TargetPatternValue implements SkyValue {
+
+ private ResolvedTargets<Target> targets;
+
+ TargetPatternValue(ResolvedTargets<Target> targets) {
+ this.targets = Preconditions.checkNotNull(targets);
+ }
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ Set<Package> packages = new LinkedHashSet<>();
+ List<String> ts = new ArrayList<>();
+ List<String> filteredTs = new ArrayList<>();
+ for (Target target : targets.getTargets()) {
+ packages.add(target.getPackage());
+ ts.add(target.getLabel().toString());
+ }
+ for (Target target : targets.getFilteredTargets()) {
+ packages.add(target.getPackage());
+ filteredTs.add(target.getLabel().toString());
+ }
+
+ out.writeObject(packages);
+ out.writeObject(ts);
+ out.writeObject(filteredTs);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ Set<Package> packages = (Set<Package>) in.readObject();
+ List<String> ts = (List<String>) in.readObject();
+ List<String> filteredTs = (List<String>) in.readObject();
+
+ Map<String, Package> packageMap = new HashMap<>();
+ for (Package p : packages) {
+ packageMap.put(p.getName(), p);
+ }
+
+ Builder<Target> builder = ResolvedTargets.<Target>builder();
+ for (String labelString : ts) {
+ builder.add(lookupTarget(packageMap, labelString));
+ }
+
+ for (String labelString : filteredTs) {
+ builder.remove(lookupTarget(packageMap, labelString));
+ }
+ this.targets = builder.build();
+ }
+
+ private static Target lookupTarget(Map<String, Package> packageMap, String labelString) {
+ Label label = Label.parseAbsoluteUnchecked(labelString);
+ Package p = packageMap.get(label.getPackageName());
+ try {
+ return p.getTarget(label.getName());
+ } catch (NoSuchTargetException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @SuppressWarnings("unused")
+ private void readObjectNoData() {
+ throw new IllegalStateException();
+ }
+
+ /**
+ * Create a target pattern value key.
+ *
+ * @param pattern The pattern, eg "-foo/biz...". If the first character is "-", the pattern
+ * is treated as a negative pattern.
+ * @param policy The filtering policy, eg "only return test targets"
+ * @param offset The offset to apply to relative target patterns.
+ */
+ @ThreadSafe
+ public static SkyKey key(String pattern,
+ FilteringPolicy policy,
+ String offset) {
+ return new SkyKey(SkyFunctions.TARGET_PATTERN,
+ pattern.startsWith("-")
+ // Don't apply filters to negative patterns.
+ ? new TargetPattern(pattern.substring(1), FilteringPolicies.NO_FILTER, true, offset)
+ : new TargetPattern(pattern, policy, false, offset));
+ }
+
+ /**
+ * Like above, but accepts a collection of target patterns for the same filtering policy.
+ *
+ * @param patterns The collection of patterns, eg "-foo/biz...". If the first character is "-",
+ * the pattern is treated as a negative pattern.
+ * @param policy The filtering policy, eg "only return test targets"
+ * @param offset The offset to apply to relative target patterns.
+ */
+ @ThreadSafe
+ public static Iterable<SkyKey> keys(Collection<String> patterns,
+ FilteringPolicy policy,
+ String offset) {
+ List<SkyKey> keys = Lists.newArrayListWithCapacity(patterns.size());
+ for (String pattern : patterns) {
+ keys.add(key(pattern, policy, offset));
+ }
+ return keys;
+ }
+
+ public ResolvedTargets<Target> getTargets() {
+ return targets;
+ }
+
+ /**
+ * A TargetPattern is a tuple of pattern (eg, "foo/..."), filtering policy, a relative pattern
+ * offset, and whether it is a positive or negative match.
+ */
+ @ThreadSafe
+ public static class TargetPattern implements Serializable {
+ private final String pattern;
+ private final FilteringPolicy policy;
+ private final boolean isNegative;
+
+ private final String offset;
+
+ public TargetPattern(String pattern, FilteringPolicy policy,
+ boolean isNegative, String offset) {
+ this.pattern = Preconditions.checkNotNull(pattern);
+ this.policy = Preconditions.checkNotNull(policy);
+ this.isNegative = isNegative;
+ this.offset = offset;
+ }
+
+ public String getPattern() {
+ return pattern;
+ }
+
+ public boolean isNegative() {
+ return isNegative;
+ }
+
+ public FilteringPolicy getPolicy() {
+ return policy;
+ }
+
+ public String getOffset() {
+ return offset;
+ }
+
+ @Override
+ public String toString() {
+ return (isNegative ? "-" : "") + pattern;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(pattern, isNegative, policy, offset);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof TargetPattern)) {
+ return false;
+ }
+ TargetPattern other = (TargetPattern) obj;
+
+ return other.isNegative == this.isNegative && other.pattern.equals(this.pattern) &&
+ other.offset.equals(this.offset) && other.policy.equals(this.policy);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TestCompletionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/TestCompletionFunction.java
new file mode 100644
index 0000000..b6f9606
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TestCompletionFunction.java
@@ -0,0 +1,71 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
+import com.google.devtools.build.lib.rules.test.TestProvider;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * TargetCompletionFunction builds all relevant test artifacts of a {@link
+ * com.google.devtools.build.lib.analysis.ConfiguredTarget}. This includes test shards and repeated
+ * runs.
+ */
+public final class TestCompletionFunction implements SkyFunction {
+
+ public TestCompletionFunction() {
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ TestCompletionValue.TestCompletionKey key =
+ (TestCompletionValue.TestCompletionKey) skyKey.argument();
+ LabelAndConfiguration lac = key.getLabelAndConfiguration();
+ if (env.getValue(TargetCompletionValue.key(lac)) == null) {
+ return null;
+ }
+
+ ConfiguredTargetValue ctValue = (ConfiguredTargetValue)
+ env.getValue(ConfiguredTargetValue.key(lac.getLabel(), lac.getConfiguration()));
+ if (ctValue == null) {
+ return null;
+ }
+
+ ConfiguredTarget ct = ctValue.getConfiguredTarget();
+ if (key.isExclusiveTesting()) {
+ // Request test artifacts iteratively if testing exclusively.
+ for (Artifact testArtifact : TestProvider.getTestStatusArtifacts(ct)) {
+ if (env.getValue(ArtifactValue.key(testArtifact, /*isMandatory=*/true)) == null) {
+ return null;
+ }
+ }
+ } else {
+ env.getValues(ArtifactValue.mandatoryKeys(TestProvider.getTestStatusArtifacts(ct)));
+ if (env.valuesMissing()) {
+ return null;
+ }
+ }
+ return TestCompletionValue.TEST_COMPLETION_MARKER;
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return Label.print(((LabelAndConfiguration) skyKey.argument()).getLabel());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TestCompletionValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TestCompletionValue.java
new file mode 100644
index 0000000..da944ed
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TestCompletionValue.java
@@ -0,0 +1,66 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Collection;
+
+/**
+ * A test completion value represents the completion of a test target. This includes the execution
+ * of all test shards and repeated runs, if applicable.
+ */
+public class TestCompletionValue implements SkyValue {
+ static final TestCompletionValue TEST_COMPLETION_MARKER = new TestCompletionValue();
+
+ private TestCompletionValue() { }
+
+ public static SkyKey key(LabelAndConfiguration lac, boolean exclusive) {
+ return new SkyKey(SkyFunctions.TEST_COMPLETION, new TestCompletionKey(lac, exclusive));
+ }
+
+ public static Iterable<SkyKey> keys(Collection<ConfiguredTarget> targets,
+ final boolean exclusive) {
+ return Iterables.transform(targets, new Function<ConfiguredTarget, SkyKey>() {
+ @Override
+ public SkyKey apply(ConfiguredTarget ct) {
+ return new SkyKey(SkyFunctions.TEST_COMPLETION,
+ new TestCompletionKey(new LabelAndConfiguration(ct), exclusive));
+ }
+ });
+ }
+
+ static class TestCompletionKey {
+ private final LabelAndConfiguration lac;
+ private final boolean exclusiveTesting;
+
+ TestCompletionKey(LabelAndConfiguration lac, boolean exclusive) {
+ this.lac = lac;
+ this.exclusiveTesting = exclusive;
+ }
+
+ public LabelAndConfiguration getLabelAndConfiguration() {
+ return lac;
+ }
+
+ public boolean isExclusiveTesting() {
+ return exclusiveTesting;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetCycleReporter.java b/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetCycleReporter.java
new file mode 100644
index 0000000..03bbd25
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetCycleReporter.java
@@ -0,0 +1,86 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.packages.PackageGroup;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.LoadedPackageProvider;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.CycleInfo;
+import com.google.devtools.build.skyframe.SkyKey;
+
+import java.util.List;
+
+/**
+ * Reports cycles between {@link TransitiveTargetValue}s. These indicates cycles between targets
+ * (e.g. '//a:foo' depends on '//b:bar' and '//b:bar' depends on '//a:foo').
+ */
+class TransitiveTargetCycleReporter extends AbstractLabelCycleReporter {
+
+ private static final Predicate<SkyKey> IS_TRANSITIVE_TARGET_SKY_KEY =
+ SkyFunctions.isSkyFunction(SkyFunctions.TRANSITIVE_TARGET);
+
+ TransitiveTargetCycleReporter(LoadedPackageProvider loadedPackageProvider) {
+ super(loadedPackageProvider);
+ }
+
+ @Override
+ protected boolean canReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo) {
+ return Iterables.all(Iterables.concat(ImmutableList.of(topLevelKey),
+ cycleInfo.getPathToCycle(), cycleInfo.getCycle()),
+ IS_TRANSITIVE_TARGET_SKY_KEY);
+ }
+
+ @Override
+ public String prettyPrint(SkyKey key) {
+ return getLabel(key).toString();
+ }
+
+ @Override
+ protected Label getLabel(SkyKey key) {
+ return (Label) key.argument();
+ }
+
+ @Override
+ protected String getAdditionalMessageAboutCycle(SkyKey topLevelKey, CycleInfo cycleInfo) {
+ Target currentTarget = getTargetForLabel(getLabel(topLevelKey));
+ List<SkyKey> keys = Lists.newArrayList();
+ if (!cycleInfo.getPathToCycle().isEmpty()) {
+ keys.add(topLevelKey);
+ keys.addAll(cycleInfo.getPathToCycle());
+ }
+ keys.addAll(cycleInfo.getCycle());
+ // Make sure we check the edge from the last element of the cycle to the first element of the
+ // cycle.
+ keys.add(cycleInfo.getCycle().get(0));
+ for (SkyKey nextKey : keys) {
+ Label nextLabel = getLabel(nextKey);
+ Target nextTarget = getTargetForLabel(nextLabel);
+ // This is inefficient but it's no big deal since we only do this when there's a cycle.
+ if (currentTarget.getVisibility().getDependencyLabels().contains(nextLabel)
+ && !nextTarget.getTargetKind().equals(PackageGroup.targetKind())) {
+ return "\nThe cycle is caused by a visibility edge from " + currentTarget.getLabel()
+ + " to the non-package-group target " + nextTarget.getLabel() + " . Note that "
+ + "visibility labels are supposed to be package group targets (which prevents cycles "
+ + "of this form)";
+ }
+ currentTarget = nextTarget;
+ }
+ return "";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetFunction.java
new file mode 100644
index 0000000..417cfca
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetFunction.java
@@ -0,0 +1,234 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.InputFile;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.OutputFile;
+import com.google.devtools.build.lib.packages.PackageGroup;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import com.google.devtools.build.skyframe.ValueOrException;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This class builds transitive Target values such that evaluating a Target value is similar to
+ * running it through the LabelVisitor.
+ */
+public class TransitiveTargetFunction implements SkyFunction {
+
+ @Override
+ public SkyValue compute(SkyKey key, Environment env) throws TransitiveTargetFunctionException {
+ Label label = (Label) key.argument();
+ SkyKey packageKey = PackageValue.key(label.getPackageIdentifier());
+ SkyKey targetKey = TargetMarkerValue.key(label);
+ Target target;
+ boolean packageLoadedSuccessfully;
+ boolean successfulTransitiveLoading = true;
+ NestedSetBuilder<Label> transitiveRootCauses = NestedSetBuilder.stableOrder();
+ NoSuchTargetException errorLoadingTarget = null;
+ try {
+ TargetMarkerValue targetValue = (TargetMarkerValue) env.getValueOrThrow(targetKey,
+ NoSuchThingException.class);
+ if (targetValue == null) {
+ return null;
+ }
+ PackageValue packageValue = (PackageValue) env.getValueOrThrow(packageKey,
+ NoSuchThingException.class);
+ if (packageValue == null) {
+ return null;
+ }
+
+ packageLoadedSuccessfully = true;
+ target = packageValue.getPackage().getTarget(label.getName());
+ } catch (NoSuchTargetException e) {
+ target = e.getTarget();
+ if (target == null) {
+ throw new TransitiveTargetFunctionException(e);
+ }
+ successfulTransitiveLoading = false;
+ transitiveRootCauses.add(label);
+ errorLoadingTarget = e;
+ packageLoadedSuccessfully = e.getPackageLoadedSuccessfully();
+ } catch (NoSuchPackageException e) {
+ throw new TransitiveTargetFunctionException(e);
+ } catch (NoSuchThingException e) {
+ throw new IllegalStateException(e
+ + " not NoSuchTargetException or NoSuchPackageException");
+ }
+
+ NestedSetBuilder<PackageIdentifier> transitiveSuccessfulPkgs = NestedSetBuilder.stableOrder();
+ NestedSetBuilder<PackageIdentifier> transitiveUnsuccessfulPkgs = NestedSetBuilder.stableOrder();
+ NestedSetBuilder<Label> transitiveTargets = NestedSetBuilder.stableOrder();
+
+ PackageIdentifier packageId = target.getPackage().getPackageIdentifier();
+ if (packageLoadedSuccessfully) {
+ transitiveSuccessfulPkgs.add(packageId);
+ } else {
+ transitiveUnsuccessfulPkgs.add(packageId);
+ }
+ transitiveTargets.add(target.getLabel());
+ for (Map.Entry<SkyKey, ValueOrException<NoSuchThingException>> entry :
+ env.getValuesOrThrow(getLabelDepKeys(target), NoSuchThingException.class).entrySet()) {
+ Label depLabel = (Label) entry.getKey().argument();
+ TransitiveTargetValue transitiveTargetValue;
+ try {
+ transitiveTargetValue = (TransitiveTargetValue) entry.getValue().get();
+ if (transitiveTargetValue == null) {
+ continue;
+ }
+ } catch (NoSuchPackageException | NoSuchTargetException e) {
+ successfulTransitiveLoading = false;
+ transitiveRootCauses.add(depLabel);
+ maybeReportErrorAboutMissingEdge(target, depLabel, e, env.getListener());
+ continue;
+ } catch (NoSuchThingException e) {
+ throw new IllegalStateException("Unexpected Exception type from TransitiveTargetValue.", e);
+ }
+ transitiveSuccessfulPkgs.addTransitive(
+ transitiveTargetValue.getTransitiveSuccessfulPackages());
+ transitiveUnsuccessfulPkgs.addTransitive(
+ transitiveTargetValue.getTransitiveUnsuccessfulPackages());
+ transitiveTargets.addTransitive(transitiveTargetValue.getTransitiveTargets());
+ NestedSet<Label> rootCauses = transitiveTargetValue.getTransitiveRootCauses();
+ if (rootCauses != null) {
+ successfulTransitiveLoading = false;
+ transitiveRootCauses.addTransitive(rootCauses);
+ if (transitiveTargetValue.getErrorLoadingTarget() != null) {
+ maybeReportErrorAboutMissingEdge(target, depLabel,
+ transitiveTargetValue.getErrorLoadingTarget(), env.getListener());
+ }
+ }
+ }
+
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ NestedSet<PackageIdentifier> successfullyLoadedPackages = transitiveSuccessfulPkgs.build();
+ NestedSet<PackageIdentifier> unsuccessfullyLoadedPackages = transitiveUnsuccessfulPkgs.build();
+ NestedSet<Label> loadedTargets = transitiveTargets.build();
+ if (successfulTransitiveLoading) {
+ return TransitiveTargetValue.successfulTransitiveLoading(successfullyLoadedPackages,
+ unsuccessfullyLoadedPackages, loadedTargets);
+ } else {
+ NestedSet<Label> rootCauses = transitiveRootCauses.build();
+ return TransitiveTargetValue.unsuccessfulTransitiveLoading(successfullyLoadedPackages,
+ unsuccessfullyLoadedPackages, loadedTargets, rootCauses, errorLoadingTarget);
+ }
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return Label.print(((Label) skyKey.argument()));
+ }
+
+ private static void maybeReportErrorAboutMissingEdge(Target target, Label depLabel,
+ NoSuchThingException e, EventHandler eventHandler) {
+ if (e instanceof NoSuchTargetException) {
+ NoSuchTargetException nste = (NoSuchTargetException) e;
+ if (depLabel.equals(nste.getLabel())) {
+ eventHandler.handle(Event.error(TargetUtils.getLocationMaybe(target),
+ TargetUtils.formatMissingEdge(target, depLabel, e)));
+ }
+ } else if (e instanceof NoSuchPackageException) {
+ NoSuchPackageException nspe = (NoSuchPackageException) e;
+ if (nspe.getPackageName().equals(depLabel.getPackageName())) {
+ eventHandler.handle(Event.error(TargetUtils.getLocationMaybe(target),
+ TargetUtils.formatMissingEdge(target, depLabel, e)));
+ }
+ }
+ }
+
+ private static Iterable<SkyKey> getLabelDepKeys(Target target) {
+ List<SkyKey> depKeys = Lists.newArrayList();
+ for (Label depLabel : getLabelDeps(target)) {
+ depKeys.add(TransitiveTargetValue.key(depLabel));
+ }
+ return depKeys;
+ }
+
+ // TODO(bazel-team): Unify this logic with that in LabelVisitor, and possibly DependencyResolver.
+ private static Iterable<Label> getLabelDeps(Target target) {
+ final Set<Label> labels = new HashSet<>();
+ if (target instanceof OutputFile) {
+ Rule rule = ((OutputFile) target).getGeneratingRule();
+ labels.add(rule.getLabel());
+ visitTargetVisibility(target, labels);
+ } else if (target instanceof InputFile) {
+ visitTargetVisibility(target, labels);
+ } else if (target instanceof Rule) {
+ visitTargetVisibility(target, labels);
+ labels.addAll(((Rule) target).getLabels(Rule.NO_NODEP_ATTRIBUTES));
+ } else if (target instanceof PackageGroup) {
+ visitPackageGroup((PackageGroup) target, labels);
+ }
+ return labels;
+ }
+
+ private static void visitTargetVisibility(Target target, Set<Label> labels) {
+ for (Label label : target.getVisibility().getDependencyLabels()) {
+ labels.add(label);
+ }
+ }
+
+ private static void visitPackageGroup(PackageGroup packageGroup, Set<Label> labels) {
+ for (final Label include : packageGroup.getIncludes()) {
+ labels.add(include);
+ }
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link TransitiveTargetFunction#compute}.
+ */
+ private static class TransitiveTargetFunctionException extends SkyFunctionException {
+ /**
+ * Used to propagate an error from a direct target dependency to the
+ * target that depended on it.
+ */
+ public TransitiveTargetFunctionException(NoSuchPackageException e) {
+ super(e, Transience.PERSISTENT);
+ }
+
+ /**
+ * In nokeep_going mode, used to propagate an error from a direct target dependency to the
+ * target that depended on it.
+ *
+ * In keep_going mode, used the same way, but only for targets that could not be loaded at all
+ * (we proceed with transitive loading on targets that contain errors).
+ */
+ public TransitiveTargetFunctionException(NoSuchTargetException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetValue.java
new file mode 100644
index 0000000..69b9638
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetValue.java
@@ -0,0 +1,142 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A <i>transitive</i> target reference that, when built in skyframe, loads the entire
+ * transitive closure of a target.
+ *
+ * This will probably be unnecessary once other refactorings occur throughout the codebase
+ * which make loading/analysis interleaving more feasible, or we will migrate "blaze query" to
+ * use this to evaluate its Target graph.
+ */
+@Immutable
+@ThreadSafe
+public class TransitiveTargetValue implements SkyValue {
+
+ // Non-final for serialization purposes.
+ private NestedSet<PackageIdentifier> transitiveSuccessfulPkgs;
+ private NestedSet<PackageIdentifier> transitiveUnsuccessfulPkgs;
+ private NestedSet<Label> transitiveTargets;
+ @Nullable private NestedSet<Label> transitiveRootCauses;
+ @Nullable private NoSuchTargetException errorLoadingTarget;
+
+ private TransitiveTargetValue(NestedSet<PackageIdentifier> transitiveSuccessfulPkgs,
+ NestedSet<PackageIdentifier> transitiveUnsuccessfulPkgs, NestedSet<Label> transitiveTargets,
+ @Nullable NestedSet<Label> transitiveRootCauses,
+ @Nullable NoSuchTargetException errorLoadingTarget) {
+ this.transitiveSuccessfulPkgs = transitiveSuccessfulPkgs;
+ this.transitiveUnsuccessfulPkgs = transitiveUnsuccessfulPkgs;
+ this.transitiveTargets = transitiveTargets;
+ this.transitiveRootCauses = transitiveRootCauses;
+ this.errorLoadingTarget = errorLoadingTarget;
+ }
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ // It helps to flatten the transitiveSuccessfulPkgs nested set as it has lots of duplicates.
+ Set<PackageIdentifier> successfulPkgs = transitiveSuccessfulPkgs.toSet();
+ out.writeInt(successfulPkgs.size());
+ for (PackageIdentifier pkg : successfulPkgs) {
+ out.writeObject(pkg);
+ }
+
+ out.writeObject(transitiveUnsuccessfulPkgs);
+ // Deliberately do not write out transitiveTargets. There is a lot of those and they drive
+ // serialization costs through the roof, both in terms of space and of time.
+ // TODO(bazel-team): Deal with this properly once we have efficient serialization of NestedSets.
+ out.writeObject(transitiveRootCauses);
+ out.writeObject(errorLoadingTarget);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ int successfulPkgCount = in.readInt();
+ NestedSetBuilder<PackageIdentifier> pkgs = NestedSetBuilder.stableOrder();
+ for (int i = 0; i < successfulPkgCount; i++) {
+ pkgs.add((PackageIdentifier) in.readObject());
+ }
+ transitiveSuccessfulPkgs = pkgs.build();
+ transitiveUnsuccessfulPkgs = (NestedSet<PackageIdentifier>) in.readObject();
+ // TODO(bazel-team): Deal with transitiveTargets properly.
+ transitiveTargets = NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ transitiveRootCauses = (NestedSet<Label>) in.readObject();
+ errorLoadingTarget = (NoSuchTargetException) in.readObject();
+ }
+
+ static TransitiveTargetValue unsuccessfulTransitiveLoading(
+ NestedSet<PackageIdentifier> transitiveSuccessfulPkgs,
+ NestedSet<PackageIdentifier> transitiveUnsuccessfulPkgs, NestedSet<Label> transitiveTargets,
+ NestedSet<Label> rootCauses, @Nullable NoSuchTargetException errorLoadingTarget) {
+ return new TransitiveTargetValue(transitiveSuccessfulPkgs, transitiveUnsuccessfulPkgs,
+ transitiveTargets, rootCauses, errorLoadingTarget);
+ }
+
+ static TransitiveTargetValue successfulTransitiveLoading(
+ NestedSet<PackageIdentifier> transitiveSuccessfulPkgs,
+ NestedSet<PackageIdentifier> transitiveUnsuccessfulPkgs,
+ NestedSet<Label> transitiveTargets) {
+ return new TransitiveTargetValue(transitiveSuccessfulPkgs, transitiveUnsuccessfulPkgs,
+ transitiveTargets, null, null);
+ }
+
+ /** Returns the error, if any, from loading the target. */
+ @Nullable
+ public NoSuchTargetException getErrorLoadingTarget() {
+ return errorLoadingTarget;
+ }
+
+ /** Returns the packages that were transitively successfully loaded. */
+ public NestedSet<PackageIdentifier> getTransitiveSuccessfulPackages() {
+ return transitiveSuccessfulPkgs;
+ }
+
+ /** Returns the packages that were transitively successfully loaded. */
+ public NestedSet<PackageIdentifier> getTransitiveUnsuccessfulPackages() {
+ return transitiveUnsuccessfulPkgs;
+ }
+
+ /** Returns the targets that were transitively loaded. */
+ public NestedSet<Label> getTransitiveTargets() {
+ return transitiveTargets;
+ }
+
+ /** Returns the root causes, if any, of why targets weren't loaded. */
+ @Nullable
+ public NestedSet<Label> getTransitiveRootCauses() {
+ return transitiveRootCauses;
+ }
+
+ @ThreadSafe
+ public static SkyKey key(Label label) {
+ return new SkyKey(SkyFunctions.TRANSITIVE_TARGET, label);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java
new file mode 100644
index 0000000..f518b8a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java
@@ -0,0 +1,224 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.devtools.build.lib.syntax.Environment.NONE;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.cmdline.LabelValidator;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.packages.ExternalPackage.Binding;
+import com.google.devtools.build.lib.packages.ExternalPackage.ExternalPackageBuilder;
+import com.google.devtools.build.lib.packages.ExternalPackage.ExternalPackageBuilder.NoSuchBindingException;
+import com.google.devtools.build.lib.packages.Package.NameConflictException;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleFactory;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.packages.Type.ConversionException;
+import com.google.devtools.build.lib.syntax.AbstractFunction;
+import com.google.devtools.build.lib.syntax.BuildFileAST;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.FuncallExpression;
+import com.google.devtools.build.lib.syntax.Function;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.syntax.MixedModeFunction;
+import com.google.devtools.build.lib.syntax.ParserInputSource;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A SkyFunction to parse WORKSPACE files.
+ */
+public class WorkspaceFileFunction implements SkyFunction {
+
+ private static final String BIND = "bind";
+
+ private final PackageFactory packageFactory;
+
+ WorkspaceFileFunction(PackageFactory packageFactory) {
+ this.packageFactory = packageFactory;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws WorkspaceFileFunctionException,
+ InterruptedException {
+ RootedPath workspaceRoot = (RootedPath) skyKey.argument();
+ // Explicitly make skyframe load this file.
+ if (env.getValue(FileValue.key(workspaceRoot)) == null) {
+ return null;
+ }
+ Path workspaceFilePath = workspaceRoot.getRoot().getRelative(workspaceRoot.getRelativePath());
+ WorkspaceNameHolder holder = new WorkspaceNameHolder();
+ ExternalPackageBuilder builder = new ExternalPackageBuilder(workspaceFilePath);
+ StoredEventHandler localReporter = new StoredEventHandler();
+ BuildFileAST buildFileAST;
+ ParserInputSource inputSource = null;
+
+ try {
+ inputSource = ParserInputSource.create(workspaceFilePath);
+ } catch (IOException e) {
+ throw new WorkspaceFileFunctionException(e, Transience.TRANSIENT);
+ }
+ buildFileAST = BuildFileAST.parseBuildFile(inputSource, localReporter, null, false);
+ if (buildFileAST.containsErrors()) {
+ localReporter.handle(Event.error("WORKSPACE file could not be parsed"));
+ } else {
+ try {
+ if (!evaluateWorkspaceFile(buildFileAST, holder, builder)) {
+ localReporter.handle(
+ Event.error("Error evaluating WORKSPACE file " + workspaceFilePath));
+ }
+ } catch (EvalException e) {
+ throw new WorkspaceFileFunctionException(e);
+ }
+ }
+
+ builder.addEvents(localReporter.getEvents());
+ if (localReporter.hasErrors()) {
+ builder.setContainsErrors();
+ }
+ return new WorkspaceFileValue(holder.workspaceName, builder.build());
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ private static Function newWorkspaceNameFunction(final WorkspaceNameHolder holder) {
+ List<String> params = ImmutableList.of("name");
+ return new MixedModeFunction("workspace", params, 1, true) {
+ @Override
+ public Object call(Object[] namedArgs, FuncallExpression ast) throws EvalException,
+ ConversionException, InterruptedException {
+ String name = Type.STRING.convert(namedArgs[0], "'name' argument");
+ String errorMessage = LabelValidator.validateTargetName(name);
+ if (errorMessage != null) {
+ throw new EvalException(ast.getLocation(), errorMessage);
+ }
+ holder.workspaceName = name;
+ return NONE;
+ }
+ };
+ }
+
+ private static Function newBindFunction(final ExternalPackageBuilder builder) {
+ List<String> params = ImmutableList.of("name", "actual");
+ return new MixedModeFunction(BIND, params, 2, true) {
+ @Override
+ public Object call(Object[] namedArgs, FuncallExpression ast)
+ throws EvalException, ConversionException {
+ String name = Type.STRING.convert(namedArgs[0], "'name' argument");
+ String actual = Type.STRING.convert(namedArgs[1], "'actual' argument");
+
+ Label nameLabel = null;
+ try {
+ nameLabel = Label.parseAbsolute("//external:" + name);
+ builder.addBinding(
+ nameLabel, new Binding(Label.parseRepositoryLabel(actual), ast.getLocation()));
+ } catch (SyntaxException e) {
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ }
+
+ return NONE;
+ }
+ };
+ }
+
+ /**
+ * Returns a function-value implementing the build rule "ruleClass" (e.g. cc_library) in the
+ * specified package context.
+ */
+ private static Function newRuleFunction(final RuleFactory ruleFactory,
+ final ExternalPackageBuilder builder, final String ruleClassName) {
+ return new AbstractFunction(ruleClassName) {
+ @Override
+ public Object call(List<Object> args, Map<String, Object> kwargs, FuncallExpression ast,
+ com.google.devtools.build.lib.syntax.Environment env)
+ throws EvalException {
+ if (!args.isEmpty()) {
+ throw new EvalException(ast.getLocation(),
+ "build rules do not accept positional parameters");
+ }
+
+ try {
+ RuleClass ruleClass = ruleFactory.getRuleClass(ruleClassName);
+ builder.createAndAddRepositoryRule(ruleClass, kwargs, ast);
+ } catch (RuleFactory.InvalidRuleException | NameConflictException | SyntaxException e) {
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ }
+ return NONE;
+ }
+ };
+ }
+
+ public boolean evaluateWorkspaceFile(BuildFileAST buildFileAST, WorkspaceNameHolder holder,
+ ExternalPackageBuilder builder)
+ throws InterruptedException, EvalException, WorkspaceFileFunctionException {
+ // Environment is defined in SkyFunction and the syntax package.
+ com.google.devtools.build.lib.syntax.Environment workspaceEnv =
+ new com.google.devtools.build.lib.syntax.Environment();
+
+ RuleFactory ruleFactory = new RuleFactory(packageFactory.getRuleClassProvider());
+ for (String ruleClass : ruleFactory.getRuleClassNames()) {
+ Function ruleFunction = newRuleFunction(ruleFactory, builder, ruleClass);
+ workspaceEnv.update(ruleClass, ruleFunction);
+ }
+
+ workspaceEnv.update(BIND, newBindFunction(builder));
+ workspaceEnv.update("workspace", newWorkspaceNameFunction(holder));
+
+ StoredEventHandler eventHandler = new StoredEventHandler();
+ if (!buildFileAST.exec(workspaceEnv, eventHandler)) {
+ return false;
+ }
+ try {
+ builder.resolveBindTargets(packageFactory.getRuleClass(BIND));
+ } catch (NoSuchBindingException e) {
+ throw new WorkspaceFileFunctionException(e);
+ }
+ return true;
+ }
+
+ private static final class WorkspaceNameHolder {
+ String workspaceName;
+ }
+
+ private static final class WorkspaceFileFunctionException extends SkyFunctionException {
+ public WorkspaceFileFunctionException(IOException e, Transience transience) {
+ super(e, transience);
+ }
+
+ public WorkspaceFileFunctionException(NoSuchBindingException e) {
+ super(e, Transience.PERSISTENT);
+ }
+
+ public WorkspaceFileFunctionException(EvalException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileValue.java
new file mode 100644
index 0000000..200f23f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileValue.java
@@ -0,0 +1,59 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.packages.ExternalPackage;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import javax.annotation.Nullable;
+
+/**
+ * Holds the contents of a WORKSPACE file as the //external package.
+ */
+public class WorkspaceFileValue implements SkyValue {
+
+ private final String workspace;
+ private final ExternalPackage pkg;
+
+ public WorkspaceFileValue(String workspace, ExternalPackage pkg) {
+ this.workspace = workspace;
+ this.pkg = pkg;
+ }
+
+ /**
+ * Returns the name of this workspace (or null for the default workspace).
+ */
+ @Nullable
+ public String getWorkspace() {
+ return workspace;
+ }
+
+ /**
+ * Returns the //external package.
+ */
+ public ExternalPackage getPackage() {
+ return pkg;
+ }
+
+ /**
+ * Generates a SkyKey based on the path to the WORKSPACE file.
+ */
+ public static SkyKey key(RootedPath workspacePath) {
+ return new SkyKey(SkyFunctions.WORKSPACE_FILE, workspacePath);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceStatusFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceStatusFunction.java
new file mode 100644
index 0000000..b1c5ef3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceStatusFunction.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/** Creates the workspace status artifacts and action. */
+public class WorkspaceStatusFunction implements SkyFunction {
+ WorkspaceStatusFunction() {
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ Preconditions.checkState(
+ WorkspaceStatusValue.SKY_KEY.equals(skyKey), WorkspaceStatusValue.SKY_KEY);
+
+ WorkspaceStatusAction action = PrecomputedValue.WORKSPACE_STATUS_KEY.get(env);
+ if (action == null) {
+ return null;
+ }
+
+ return new WorkspaceStatusValue(
+ action.getStableStatus(),
+ action.getVolatileStatus(),
+ action);
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceStatusValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceStatusValue.java
new file mode 100644
index 0000000..21b9215
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceStatusValue.java
@@ -0,0 +1,62 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+
+/**
+ * Value that stores the workspace status artifacts and their generating action. There should be
+ * only one of these values in the graph at any time.
+ */
+// TODO(bazel-team): This seems to be superfluous now, but it cannot be removed without making
+// PrecomputedValue public instead of package-private
+public class WorkspaceStatusValue extends ActionLookupValue {
+ private final Artifact stableArtifact;
+ private final Artifact volatileArtifact;
+
+ // There should only ever be one BuildInfo value in the graph.
+ public static final SkyKey SKY_KEY = new SkyKey(SkyFunctions.BUILD_INFO, "BUILD_INFO");
+ static final ArtifactOwner ARTIFACT_OWNER = new BuildInfoKey();
+
+ public WorkspaceStatusValue(Artifact stableArtifact, Artifact volatileArtifact,
+ WorkspaceStatusAction action) {
+ super(action);
+ this.stableArtifact = stableArtifact;
+ this.volatileArtifact = volatileArtifact;
+ }
+
+ public Artifact getStableArtifact() {
+ return stableArtifact;
+ }
+
+ public Artifact getVolatileArtifact() {
+ return volatileArtifact;
+ }
+
+ private static class BuildInfoKey extends ActionLookupKey {
+ @Override
+ SkyFunctionName getType() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ SkyKey getSkyKey() {
+ return SKY_KEY;
+ }
+ }
+}
