Introduce interleaved package and transitive target loading
Adds SkyFunctions and assorted values that implement interleaved
loading of packages and their targets' transitive dependencies.
They are not hooked up to any graph loading components, yet.
--
MOS_MIGRATED_REVID=97278368
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternFunction.java
new file mode 100644
index 0000000..bf32843
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternFunction.java
@@ -0,0 +1,225 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.cmdline.ResolvedTargets;
+import com.google.devtools.build.lib.cmdline.TargetParsingException;
+import com.google.devtools.build.lib.cmdline.TargetPattern;
+import com.google.devtools.build.lib.cmdline.TargetPatternResolver;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.FilteringPolicies;
+import com.google.devtools.build.lib.pkgcache.FilteringPolicy;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.pkgcache.TargetPatternResolverUtil;
+import com.google.devtools.build.lib.skyframe.EnvironmentBackedRecursivePackageProvider.MissingDepException;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * PrepareDepsOfPatternFunction ensures the graph loads targets matching the pattern and its
+ * transitive dependencies.
+ */
+public class PrepareDepsOfPatternFunction implements SkyFunction {
+
+ private final AtomicReference<PathPackageLocator> pkgPath;
+
+ public PrepareDepsOfPatternFunction(AtomicReference<PathPackageLocator> pkgPath) {
+ this.pkgPath = pkgPath;
+ }
+
+ @Nullable
+ @Override
+ public SkyValue compute(SkyKey key, Environment env)
+ throws SkyFunctionException, InterruptedException {
+ TargetPatternValue.TargetPatternKey patternKey =
+ ((TargetPatternValue.TargetPatternKey) key.argument());
+ try {
+ TargetPattern parsedPattern = patternKey.getParsedPattern();
+ DepsOfPatternPreparer preparer =
+ new DepsOfPatternPreparer(env, patternKey.getPolicy(), pkgPath.get());
+ ImmutableSet<String> excludedSubdirectories = patternKey.getExcludedSubdirectories();
+ parsedPattern.eval(preparer, excludedSubdirectories);
+ } catch (TargetParsingException e) {
+ throw new PrepareDepsOfPatternFunctionException(e);
+ } catch (MissingDepException e) {
+ // The DepsOfPatternPreparer constructed above might throw MissingDepException to signal
+ // when it has a dependency on a missing Environment value.
+ return null;
+ }
+ return PrepareDepsOfPatternValue.INSTANCE;
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by {@link
+ * PrepareDepsOfPatternFunction#compute}.
+ */
+ private static final class PrepareDepsOfPatternFunctionException extends SkyFunctionException {
+
+ public PrepareDepsOfPatternFunctionException(TargetParsingException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+
+ /**
+ * A {@link TargetPatternResolver} backed by an {@link Environment} whose methods do not actually
+ * return resolved targets, but that ensures the graph loads the matching targets <b>and</b> their
+ * transitive dependencies. Its methods may throw {@link MissingDepException} if the package
+ * values this depends on haven't been calculated and added to its environment.
+ */
+ static class DepsOfPatternPreparer implements TargetPatternResolver<Void> {
+
+ private final EnvironmentBackedRecursivePackageProvider packageProvider;
+ private final Environment env;
+ private final FilteringPolicy policy;
+ private final PathPackageLocator pkgPath;
+
+ public DepsOfPatternPreparer(Environment env, FilteringPolicy policy,
+ PathPackageLocator pkgPath) {
+ this.env = env;
+ this.packageProvider = new EnvironmentBackedRecursivePackageProvider(env);
+ this.policy = policy;
+ this.pkgPath = pkgPath;
+ }
+
+ @Override
+ public void warn(String msg) {
+ env.getListener().handle(Event.warn(msg));
+ }
+
+ @Override
+ public Void getTargetOrNull(String targetName) throws InterruptedException {
+ // Note:
+ // This method is used in just one place, TargetPattern.TargetsInPackage#getWildcardConflict.
+ // Returning null tells #getWildcardConflict that there is not a target with a name like
+ // "all" or "all-targets", which means that TargetPattern.TargetsInPackage will end up
+ // calling DepsOfTargetPreparer#getTargetsInPackage.
+ // TODO (bazel-team): Consider replacing this with an isTarget method on the interface.
+ return null;
+ }
+
+ @Override
+ public ResolvedTargets<Void> getExplicitTarget(String targetName)
+ throws TargetParsingException, InterruptedException {
+ Label label = TargetPatternResolverUtil.label(targetName);
+ try {
+ Target target = packageProvider.getTarget(env.getListener(), label);
+ SkyKey key = TransitiveTargetValue.key(target.getLabel());
+ SkyValue token =
+ env.getValueOrThrow(key, NoSuchPackageException.class, NoSuchTargetException.class);
+ if (token == null) {
+ throw new MissingDepException();
+ }
+ return ResolvedTargets.empty();
+ } catch (NoSuchThingException e) {
+ throw new TargetParsingException(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public ResolvedTargets<Void> getTargetsInPackage(String originalPattern, String packageName,
+ boolean rulesOnly) throws TargetParsingException, InterruptedException {
+ FilteringPolicy actualPolicy = rulesOnly
+ ? FilteringPolicies.and(FilteringPolicies.RULES_ONLY, policy)
+ : policy;
+ return getTargetsInPackage(originalPattern, new PathFragment(packageName), actualPolicy);
+ }
+
+ private ResolvedTargets<Void> getTargetsInPackage(String originalPattern,
+ PathFragment packageNameFragment, FilteringPolicy policy)
+ throws TargetParsingException, InterruptedException {
+ TargetPatternResolverUtil.validatePatternPackage(originalPattern, packageNameFragment, this);
+ try {
+ PackageIdentifier packageId = PackageIdentifier.createInDefaultRepo(packageNameFragment);
+ Package pkg = packageProvider.getPackage(env.getListener(), packageId);
+ ResolvedTargets<Target> packageTargets =
+ TargetPatternResolverUtil.resolvePackageTargets(pkg, policy);
+ ImmutableList.Builder<SkyKey> builder = ImmutableList.builder();
+ for (Target target : packageTargets.getTargets()) {
+ builder.add(TransitiveTargetValue.key(target.getLabel()));
+ }
+ ImmutableList<SkyKey> skyKeys = builder.build();
+ env.getValuesOrThrow(skyKeys, NoSuchPackageException.class, NoSuchTargetException.class);
+ if (env.valuesMissing()) {
+ throw new MissingDepException();
+ }
+ return ResolvedTargets.empty();
+ } catch (NoSuchThingException e) {
+ String message = TargetPatternResolverUtil.getParsingErrorMessage(
+ "package contains errors", originalPattern);
+ throw new TargetParsingException(message, e);
+ }
+ }
+
+ @Override
+ public boolean isPackage(String packageName) {
+ return packageProvider.isPackage(env.getListener(), packageName);
+ }
+
+ @Override
+ public String getTargetKind(Void target) {
+ // Note:
+ // This method is used in just one place, TargetPattern.TargetsInPackage#getWildcardConflict.
+ // Because DepsOfPatternPreparer#getTargetOrNull always returns null, this method is never
+ // called.
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ResolvedTargets<Void> findTargetsBeneathDirectory(String originalPattern,
+ String directory, boolean rulesOnly, ImmutableSet<String> excludedSubdirectories)
+ throws TargetParsingException, InterruptedException {
+ FilteringPolicy actualPolicy = rulesOnly
+ ? FilteringPolicies.and(FilteringPolicies.RULES_ONLY, policy)
+ : policy;
+ ImmutableSet<PathFragment> excludedPathFragments =
+ TargetPatternResolverUtil.getPathFragments(excludedSubdirectories);
+ PathFragment pathFragment = TargetPatternResolverUtil.getPathFragment(directory);
+ for (Path root : pkgPath.getPathEntries()) {
+ RootedPath rootedPath = RootedPath.toRootedPath(root, pathFragment);
+ SkyValue token = env.getValue(PrepareDepsOfTargetsUnderDirectoryValue.key(rootedPath,
+ excludedPathFragments, actualPolicy));
+ if (token == null) {
+ // A null token value means there is a missing dependency, because RecursivePkgFunction
+ // never throws.
+ throw new MissingDepException();
+ }
+ }
+ return ResolvedTargets.empty();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternValue.java
new file mode 100644
index 0000000..638b078
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternValue.java
@@ -0,0 +1,126 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.cmdline.TargetParsingException;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.pkgcache.FilteringPolicy;
+import com.google.devtools.build.lib.skyframe.TargetPatternValue.TargetPatternKey;
+import com.google.devtools.build.lib.skyframe.TargetPatternValue.TargetPatternSkyKeyOrException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.List;
+
+/**
+ * The value returned by {@link PrepareDepsOfPatternFunction}. Because that function is
+ * invoked only for its side effect (i.e. ensuring the graph contains targets matching the
+ * pattern and its transitive dependencies), this value carries no information.
+ *
+ * <p>Because the returned value is always the same object, this value and the
+ * {@link PrepareDepsOfPatternFunction} which computes it are incompatible with change pruning. It
+ * should only be requested by consumers who do not require reevaluation when
+ * {@link PrepareDepsOfPatternFunction} is reevaluated. Safe consumers include, e.g., top-level
+ * consumers, and other functions which invoke {@link PrepareDepsOfPatternFunction} solely for its
+ * side-effects.
+ */
+public class PrepareDepsOfPatternValue implements SkyValue {
+ public static final PrepareDepsOfPatternValue INSTANCE = new PrepareDepsOfPatternValue();
+
+ private PrepareDepsOfPatternValue() {
+ }
+
+ /**
+ * Returns an iterable of {@link PrepareDepsOfPatternSkyKeyOrException}, with
+ * {@link TargetPatternKey} arguments. If a provided pattern fails to parse, an element in the
+ * returned iterable will throw when its
+ * {@link PrepareDepsOfPatternSkyKeyOrException#getSkyKey} method is called and will return the
+ * failing pattern when its {@link PrepareDepsOfPatternSkyKeyOrException#getOriginalPattern}
+ * method is called.
+ *
+ * <p>There may be fewer returned elements than patterns provided as input. This function may
+ * combine patterns to return an iterable of SkyKeys that is equivalent but more efficient to
+ * evaluate, and will omit SkyKeys associated with negative patterns.
+ *
+ * @param patterns The list of patterns, e.g. "-foo/biz...". If a pattern's first character is
+ * "-", it is treated as a negative pattern.
+ * @param policy The filtering policy, e.g. "only return test targets"
+ * @param offset The offset to apply to relative target patterns.
+ */
+ @ThreadSafe
+ public static Iterable<PrepareDepsOfPatternSkyKeyOrException> keys(List<String> patterns,
+ FilteringPolicy policy, String offset) {
+ Iterable<TargetPatternSkyKeyOrException> keysMaybe =
+ TargetPatternValue.keys(patterns, policy, offset);
+ ImmutableList.Builder<PrepareDepsOfPatternSkyKeyOrException> builder = ImmutableList.builder();
+ for (TargetPatternSkyKeyOrException keyMaybe : keysMaybe) {
+ try {
+ SkyKey skyKey = keyMaybe.getSkyKey();
+ if (!((TargetPatternKey) skyKey.argument()).isNegative()) {
+ builder.add(new PrepareDepsOfPatternSkyKeyOrExceptionImpl(keyMaybe));
+ }
+ } catch (TargetParsingException e) {
+ // keyMaybe.getSkyKey() may throw TargetParsingException if its corresponding pattern
+ // failed to parse. If so, wrap the exception-holding TargetPatternSkyKeyOrException and
+ // return it, so that our caller can deal with it.
+ builder.add(new PrepareDepsOfPatternSkyKeyOrExceptionImpl(keyMaybe));
+ }
+ }
+ return builder.build();
+ }
+
+ /**
+ * Wrapper for a prepare deps of pattern {@link SkyKey} or the {@link TargetParsingException}
+ * thrown when trying to create it.
+ */
+ public interface PrepareDepsOfPatternSkyKeyOrException {
+
+ /**
+ * Returns the stored {@link SkyKey} or throws {@link TargetParsingException} if one was thrown
+ * when creating the key.
+ */
+ SkyKey getSkyKey() throws TargetParsingException;
+
+ /**
+ * Returns the pattern that resulted in the stored {@link SkyKey} or {@link
+ * TargetParsingException}.
+ */
+ String getOriginalPattern();
+ }
+
+ /**
+ * Converts from a {@link TargetPatternSkyKeyOrException} to a
+ * {@link PrepareDepsOfPatternSkyKeyOrException}.
+ */
+ private static class PrepareDepsOfPatternSkyKeyOrExceptionImpl implements
+ PrepareDepsOfPatternSkyKeyOrException {
+
+ private final TargetPatternSkyKeyOrException wrapped;
+
+ private PrepareDepsOfPatternSkyKeyOrExceptionImpl(TargetPatternSkyKeyOrException wrapped) {
+ this.wrapped = wrapped;
+ }
+
+ @Override
+ public SkyKey getSkyKey() throws TargetParsingException {
+ return new SkyKey(SkyFunctions.PREPARE_DEPS_OF_PATTERN, wrapped.getSkyKey().argument());
+ }
+
+ @Override
+ public String getOriginalPattern() {
+ return wrapped.getOriginalPattern();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfTargetsUnderDirectoryFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfTargetsUnderDirectoryFunction.java
new file mode 100644
index 0000000..1edf7a5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfTargetsUnderDirectoryFunction.java
@@ -0,0 +1,144 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.cmdline.ResolvedTargets;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.FilteringPolicy;
+import com.google.devtools.build.lib.pkgcache.TargetPatternResolverUtil;
+import com.google.devtools.build.lib.skyframe.PrepareDepsOfTargetsUnderDirectoryValue.PrepareDepsOfTargetsUnderDirectoryKey;
+import com.google.devtools.build.lib.skyframe.RecursivePkgValue.RecursivePkgKey;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * Ensures the graph contains the targets in the directory's package, if any, and in the
+ * non-excluded packages in its subdirectories, and all those targets' transitive dependencies,
+ * after a successful evaluation.
+ *
+ * <p>Computes {@link PrepareDepsOfTargetsUnderDirectoryValue} which describes whether the
+ * directory is a package and how many non-excluded packages exist below each of the directory's
+ * subdirectories.
+ */
+public class PrepareDepsOfTargetsUnderDirectoryFunction implements SkyFunction {
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ PrepareDepsOfTargetsUnderDirectoryKey argument =
+ (PrepareDepsOfTargetsUnderDirectoryKey) skyKey.argument();
+ FilteringPolicy filteringPolicy = argument.getFilteringPolicy();
+ RecursivePkgKey recursivePkgKey = argument.getRecursivePkgKey();
+ return new MyTraversalFunction(filteringPolicy).visitDirectory(recursivePkgKey, env);
+ }
+
+ private static class MyTraversalFunction
+ extends RecursiveDirectoryTraversalFunction<MyVisitor,
+ PrepareDepsOfTargetsUnderDirectoryValue> {
+
+ private final FilteringPolicy filteringPolicy;
+
+ private MyTraversalFunction(FilteringPolicy filteringPolicy) {
+ this.filteringPolicy = filteringPolicy;
+ }
+
+ @Override
+ protected PrepareDepsOfTargetsUnderDirectoryValue getEmptyReturn() {
+ return PrepareDepsOfTargetsUnderDirectoryValue.EMPTY;
+ }
+
+ @Override
+ protected MyVisitor getInitialVisitor() {
+ return new MyVisitor(filteringPolicy);
+ }
+
+ @Override
+ protected SkyKey getSkyKeyForSubdirectory(RootedPath subdirectory,
+ ImmutableSet<PathFragment> excludedSubdirectoriesBeneathSubdirectory) {
+ return PrepareDepsOfTargetsUnderDirectoryValue.key(subdirectory,
+ excludedSubdirectoriesBeneathSubdirectory, filteringPolicy);
+ }
+
+ @Override
+ protected PrepareDepsOfTargetsUnderDirectoryValue aggregateWithSubdirectorySkyValues(
+ MyVisitor visitor, Map<SkyKey, SkyValue> subdirectorySkyValues) {
+ // Aggregate the child subdirectory package counts.
+ ImmutableMap.Builder<RootedPath, Integer> builder = ImmutableMap.builder();
+ for (SkyKey key : subdirectorySkyValues.keySet()) {
+ PrepareDepsOfTargetsUnderDirectoryKey prepDepsKey =
+ (PrepareDepsOfTargetsUnderDirectoryKey) key.argument();
+ PrepareDepsOfTargetsUnderDirectoryValue prepDepsValue =
+ (PrepareDepsOfTargetsUnderDirectoryValue) subdirectorySkyValues.get(key);
+ int numPackagesInSubdirectory = (prepDepsValue.isDirectoryPackage() ? 1 : 0);
+ for (Integer numPkgsInSubSub : prepDepsValue.getSubdirectoryPackageCount().values()) {
+ numPackagesInSubdirectory += numPkgsInSubSub;
+ }
+ builder.put(prepDepsKey.getRecursivePkgKey().getRootedPath(), numPackagesInSubdirectory);
+ }
+ return new PrepareDepsOfTargetsUnderDirectoryValue(visitor.isDirectoryPackage(),
+ builder.build());
+ }
+ }
+
+ private static class MyVisitor implements RecursiveDirectoryTraversalFunction.Visitor {
+
+ private final FilteringPolicy filteringPolicy;
+ private boolean isDirectoryPackage;
+
+ private MyVisitor(FilteringPolicy filteringPolicy) {
+ this.filteringPolicy = Preconditions.checkNotNull(filteringPolicy);
+ }
+
+ @Override
+ public void visitPackageValue(Package pkg, Environment env) {
+ isDirectoryPackage = true;
+ loadTransitiveTargets(env, pkg, filteringPolicy);
+ }
+
+ public boolean isDirectoryPackage() {
+ return isDirectoryPackage;
+ }
+ }
+
+ private static void loadTransitiveTargets(Environment env, Package pkg,
+ FilteringPolicy filteringPolicy) {
+ ResolvedTargets<Target> packageTargets =
+ TargetPatternResolverUtil.resolvePackageTargets(pkg, filteringPolicy);
+ ImmutableList.Builder<SkyKey> builder = ImmutableList.builder();
+ for (Target target : packageTargets.getTargets()) {
+ builder.add(TransitiveTargetValue.key(target.getLabel()));
+ }
+ ImmutableList<SkyKey> skyKeys = builder.build();
+ env.getValuesOrThrow(skyKeys, NoSuchPackageException.class, NoSuchTargetException.class);
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfTargetsUnderDirectoryValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfTargetsUnderDirectoryValue.java
new file mode 100644
index 0000000..840ce7b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfTargetsUnderDirectoryValue.java
@@ -0,0 +1,152 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.pkgcache.FilteringPolicies;
+import com.google.devtools.build.lib.pkgcache.FilteringPolicy;
+import com.google.devtools.build.lib.skyframe.RecursivePkgValue.RecursivePkgKey;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * The value computed by {@link PrepareDepsOfTargetsUnderDirectoryFunction}. Contains a mapping for
+ * all its non-excluded directories to the count of packages beneath them.
+ *
+ * <p>This value is used by {@link GraphBackedRecursivePackageProvider#getPackagesUnderDirectory}
+ * to help it traverse the graph and find the set of packages under a directory, and recursively by
+ * {@link PrepareDepsOfTargetsUnderDirectoryFunction} which computes a value for a directory by
+ * aggregating results calculated from its subdirectories.
+ *
+ * <p>Note that even though the {@link PrepareDepsOfTargetsUnderDirectoryFunction} is evaluated in
+ * part because of its side-effects (i.e. loading transitive dependencies of targets), this value
+ * interacts safely with change pruning, despite the fact that this value is a lossy representation
+ * of the packages beneath a directory (i.e. it doesn't care <b>which</b> packages are under a
+ * directory, just the count of them). When the targets in a package change, the
+ * {@link PackageValue} that {@link PrepareDepsOfTargetsUnderDirectoryFunction} depends on will be
+ * invalidated, and the PrepareDeps function for that package's directory will be reevaluated,
+ * loading any new transitive dependencies. Change pruning may prevent the reevaluation of
+ * PrepareDeps for directories above that one, but they don't need to be re-run.
+ */
+public final class PrepareDepsOfTargetsUnderDirectoryValue implements SkyValue {
+ public static final PrepareDepsOfTargetsUnderDirectoryValue EMPTY =
+ new PrepareDepsOfTargetsUnderDirectoryValue(false, ImmutableMap.<RootedPath, Integer>of());
+
+ private final boolean isDirectoryPackage;
+ private final ImmutableMap<RootedPath, Integer> subdirectoryPackageCount;
+
+ public PrepareDepsOfTargetsUnderDirectoryValue(boolean isDirectoryPackage,
+ ImmutableMap<RootedPath, Integer> subdirectoryPackageCount) {
+ this.isDirectoryPackage = isDirectoryPackage;
+ this.subdirectoryPackageCount = Preconditions.checkNotNull(subdirectoryPackageCount);
+ }
+
+ /** Whether the directory is a package (i.e. contains a BUILD file). */
+ public boolean isDirectoryPackage() {
+ return isDirectoryPackage;
+ }
+
+ /**
+ * Returns a map from non-excluded immediate subdirectories of this directory to the number of
+ * non-excluded packages under them.
+ */
+ public ImmutableMap<RootedPath, Integer> getSubdirectoryPackageCount() {
+ return subdirectoryPackageCount;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof PrepareDepsOfTargetsUnderDirectoryValue)) {
+ return false;
+ }
+ PrepareDepsOfTargetsUnderDirectoryValue that = (PrepareDepsOfTargetsUnderDirectoryValue) o;
+ return isDirectoryPackage == that.isDirectoryPackage
+ && Objects.equals(subdirectoryPackageCount, that.subdirectoryPackageCount);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(isDirectoryPackage, subdirectoryPackageCount);
+ }
+
+ /** Create a prepare deps of targets under directory request. */
+ @ThreadSafe
+ public static SkyKey key(RootedPath rootedPath, ImmutableSet<PathFragment> excludedPaths) {
+ return key(rootedPath, excludedPaths, FilteringPolicies.NO_FILTER);
+ }
+
+ /**
+ * Create a prepare deps of targets under directory request, specifying a filtering policy for
+ * targets.
+ */
+ @ThreadSafe
+ public static SkyKey key(RootedPath rootedPath, ImmutableSet<PathFragment> excludedPaths,
+ FilteringPolicy filteringPolicy) {
+ return new SkyKey(SkyFunctions.PREPARE_DEPS_OF_TARGETS_UNDER_DIRECTORY,
+ new PrepareDepsOfTargetsUnderDirectoryKey(new RecursivePkgKey(rootedPath, excludedPaths),
+ filteringPolicy));
+ }
+
+ /**
+ * The argument value for {@link SkyKey}s of {@link PrepareDepsOfTargetsUnderDirectoryFunction}.
+ */
+ public static final class PrepareDepsOfTargetsUnderDirectoryKey implements Serializable {
+ private final RecursivePkgKey recursivePkgKey;
+ private final FilteringPolicy filteringPolicy;
+
+ private PrepareDepsOfTargetsUnderDirectoryKey(RecursivePkgKey recursivePkgKey,
+ FilteringPolicy filteringPolicy) {
+ this.recursivePkgKey = Preconditions.checkNotNull(recursivePkgKey);
+ this.filteringPolicy = Preconditions.checkNotNull(filteringPolicy);
+ }
+
+ public RecursivePkgKey getRecursivePkgKey() {
+ return recursivePkgKey;
+ }
+
+ public FilteringPolicy getFilteringPolicy() {
+ return filteringPolicy;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof PrepareDepsOfTargetsUnderDirectoryKey)) {
+ return false;
+ }
+
+ PrepareDepsOfTargetsUnderDirectoryKey that = (PrepareDepsOfTargetsUnderDirectoryKey) o;
+ return Objects.equals(recursivePkgKey, that.recursivePkgKey)
+ && Objects.equals(filteringPolicy, that.filteringPolicy);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(recursivePkgKey, filteringPolicy);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
index 452fb86..ef22ab9 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
@@ -230,6 +230,7 @@
SkyFunctions.FILE,
SkyFunctions.DIRECTORY_LISTING_STATE,
SkyFunctions.TARGET_PATTERN,
+ SkyFunctions.PREPARE_DEPS_OF_PATTERN,
SkyFunctions.WORKSPACE_FILE);
@Override
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java
index 047db51..b6944c5 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java
@@ -42,6 +42,10 @@
public static final SkyFunctionName TARGET_PATTERN = SkyFunctionName.create("TARGET_PATTERN");
public static final SkyFunctionName PREPARE_DEPS_OF_PATTERNS =
SkyFunctionName.create("PREPARE_DEPS_OF_PATTERNS");
+ public static final SkyFunctionName PREPARE_DEPS_OF_PATTERN =
+ SkyFunctionName.create("PREPARE_DEPS_OF_PATTERN");
+ public static final SkyFunctionName PREPARE_DEPS_OF_TARGETS_UNDER_DIRECTORY =
+ SkyFunctionName.create("PREPARE_DEPS_OF_TARGETS_UNDER_DIRECTORY");
public static final SkyFunctionName RECURSIVE_PKG = SkyFunctionName.create("RECURSIVE_PKG");
public static final SkyFunctionName TRANSITIVE_TARGET =
SkyFunctionName.create("TRANSITIVE_TARGET");
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
index 561c9db..b2ed3c6 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
@@ -308,6 +308,9 @@
map.put(SkyFunctions.GLOB, new GlobFunction());
map.put(SkyFunctions.TARGET_PATTERN, new TargetPatternFunction(pkgLocator));
map.put(SkyFunctions.PREPARE_DEPS_OF_PATTERNS, new PrepareDepsOfPatternsFunction());
+ map.put(SkyFunctions.PREPARE_DEPS_OF_PATTERN, new PrepareDepsOfPatternFunction(pkgLocator));
+ map.put(SkyFunctions.PREPARE_DEPS_OF_TARGETS_UNDER_DIRECTORY,
+ new PrepareDepsOfTargetsUnderDirectoryFunction());
map.put(SkyFunctions.RECURSIVE_PKG, new RecursivePkgFunction());
map.put(SkyFunctions.PACKAGE, new PackageFunction(
reporter, pkgFactory, packageManager, showLoadingProgress, packageFunctionCache,
