Make files and directories under managed directories be correctly processed by SequencedSkyframeExecutor.
TL;DR - two main changes: how to invalidate changed managed directories files, and how to force owning external repository to be evaluated before managed directories files.
- In ExternalFilesHelper, introduce one more file type - EXTERNAL_REPO_IN_USER_DIRECTORY, for the files under managed directories.
- For files under managed directories, require owning RepositoryDirectoryValue to be evaluated first.
- For correct dirtying of files under managed directories, both with watchfs flag and without, the following should be taken into account: not only that new values for external repositories and managed directories files can not be injected at the stage of dirtying, but also files that used to be under managed directories on the previous evaluator invocation.
The latter are still cached in evaluator, and the fact that they used to depend on their RepositoryDirectory values would prevent injection of the new values for them.
To meet those conditions, in SequencedSkyframeExecutor.handleChangedFiles() filtering of the going-to-be-injected files is added.
(The change can not be done inside ExternalDirtinessChecker only, as then it does not affect watchfs=true case).
- ManagedDirectoriesBlackBoxTest added to demonstrate and validate the functionality of managed directories.
PiperOrigin-RevId: 246496823
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
index 1d0fa39..961a95a 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
@@ -51,6 +51,7 @@
import com.google.devtools.build.lib.rules.repository.LocalRepositoryFunction;
import com.google.devtools.build.lib.rules.repository.LocalRepositoryRule;
import com.google.devtools.build.lib.rules.repository.ManagedDirectoriesKnowledgeImpl;
+import com.google.devtools.build.lib.rules.repository.ManagedDirectoriesKnowledgeImpl.ManagedDirectoriesListener;
import com.google.devtools.build.lib.rules.repository.NewLocalRepositoryFunction;
import com.google.devtools.build.lib.rules.repository.NewLocalRepositoryRule;
import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction;
@@ -70,6 +71,7 @@
import com.google.devtools.build.lib.skyframe.SkyFunctions;
import com.google.devtools.build.lib.skylarkbuildapi.repository.RepositoryBootstrap;
import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
@@ -87,7 +89,6 @@
/** Adds support for fetching external code. */
public class BazelRepositoryModule extends BlazeModule {
-
// Default location (relative to output user root) of the repository cache.
public static final String DEFAULT_CACHE_LOCATION = "cache/repos/v1";
@@ -107,12 +108,28 @@
private FileSystem filesystem;
// We hold the precomputed value of the managed directories here, so that the dependency
// on WorkspaceFileValue is not registered for each FileStateValue.
- private final ManagedDirectoriesKnowledgeImpl managedDirectoriesKnowledge =
- new ManagedDirectoriesKnowledgeImpl();
+ private final ManagedDirectoriesKnowledgeImpl managedDirectoriesKnowledge;
public BazelRepositoryModule() {
this.skylarkRepositoryFunction = new SkylarkRepositoryFunction(httpDownloader);
this.repositoryHandlers = repositoryRules(httpDownloader, mavenDownloader);
+ ManagedDirectoriesListener listener =
+ repositoryNamesWithManagedDirs -> {
+ Set<String> conflicting =
+ overrides.keySet().stream()
+ .filter(repositoryNamesWithManagedDirs::contains)
+ .map(RepositoryName::getName)
+ .collect(Collectors.toSet());
+ if (!conflicting.isEmpty()) {
+ String message =
+ "Overriding repositories is not allowed"
+ + " for the repositories with managed directories.\n"
+ + "The following overridden external repositories have managed directories: "
+ + String.join(", ", conflicting.toArray(new String[0]));
+ throw new AbruptExitException(message, ExitCode.COMMAND_LINE_ERROR);
+ }
+ };
+ managedDirectoriesKnowledge = new ManagedDirectoriesKnowledgeImpl(listener);
}
public static ImmutableMap<String, RepositoryFunction> repositoryRules(
@@ -153,10 +170,7 @@
@Override
public void workspaceInit(
BlazeRuntime runtime, BlazeDirectories directories, WorkspaceBuilder builder) {
- builder.setWorkspaceFileHeaderListener(
- value ->
- managedDirectoriesKnowledge.setManagedDirectories(
- value != null ? value.getManagedDirectories() : ImmutableMap.of()));
+ builder.setManagedDirectoriesKnowledge(managedDirectoriesKnowledge);
RepositoryDirectoryDirtinessChecker customDirtinessChecker =
new RepositoryDirectoryDirtinessChecker(managedDirectoriesKnowledge);
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/ManagedDirectoriesKnowledge.java b/src/main/java/com/google/devtools/build/lib/rules/repository/ManagedDirectoriesKnowledge.java
index 66ca156..2af4b8b 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/repository/ManagedDirectoriesKnowledge.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/ManagedDirectoriesKnowledge.java
@@ -16,8 +16,9 @@
import com.google.common.collect.ImmutableSet;
import com.google.devtools.build.lib.cmdline.RepositoryName;
+import com.google.devtools.build.lib.packages.WorkspaceFileValue;
+import com.google.devtools.build.lib.skyframe.SequencedSkyframeExecutor.WorkspaceFileHeaderListener;
import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.devtools.build.lib.vfs.RootedPath;
import javax.annotation.Nullable;
/**
@@ -29,12 +30,18 @@
* <p>Having managed directories as a separate component (and not SkyValue) allows to skip recording
* the dependency in Skyframe for each FileStateValue and DirectoryListingStateValue.
*/
-public interface ManagedDirectoriesKnowledge {
+public interface ManagedDirectoriesKnowledge extends WorkspaceFileHeaderListener {
ManagedDirectoriesKnowledge NO_MANAGED_DIRECTORIES =
new ManagedDirectoriesKnowledge() {
+ @Override
+ public boolean workspaceHeaderReloaded(
+ @Nullable WorkspaceFileValue oldValue, @Nullable WorkspaceFileValue newValue) {
+ return false;
+ }
+
@Nullable
@Override
- public RepositoryName getOwnerRepository(RootedPath rootedPath, boolean old) {
+ public RepositoryName getOwnerRepository(PathFragment relativePathFragment) {
return null;
}
@@ -44,8 +51,15 @@
}
};
+ /**
+ * Returns the owning repository for the incrementally updated path, or null.
+ *
+ * @param relativePathFragment path to check, relative to workspace root
+ * @return RepositoryName or null if there is no owning repository
+ */
@Nullable
- RepositoryName getOwnerRepository(RootedPath rootedPath, boolean old);
+ RepositoryName getOwnerRepository(PathFragment relativePathFragment);
+ /** Returns managed directories for the passed repository. */
ImmutableSet<PathFragment> getManagedDirectories(RepositoryName repositoryName);
}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/ManagedDirectoriesKnowledgeImpl.java b/src/main/java/com/google/devtools/build/lib/rules/repository/ManagedDirectoriesKnowledgeImpl.java
index 08b44fa..b2344dd 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/repository/ManagedDirectoriesKnowledgeImpl.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/ManagedDirectoriesKnowledgeImpl.java
@@ -14,61 +14,37 @@
package com.google.devtools.build.lib.rules.repository;
-import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.devtools.build.lib.cmdline.RepositoryName;
+import com.google.devtools.build.lib.packages.WorkspaceFileValue;
+import com.google.devtools.build.lib.util.AbruptExitException;
import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.devtools.build.lib.vfs.RootedPath;
import java.util.Comparator;
import java.util.Map;
-import java.util.NavigableMap;
+import java.util.Objects;
import java.util.Set;
-import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.Nullable;
/** Managed directories component. {@link ManagedDirectoriesKnowledge} */
public class ManagedDirectoriesKnowledgeImpl implements ManagedDirectoriesKnowledge {
- private final AtomicReference<ImmutableSortedMap<PathFragment, RepositoryName>>
- managedDirectoriesRef = new AtomicReference<>(ImmutableSortedMap.of());
- private final AtomicReference<Map<RepositoryName, ImmutableSet<PathFragment>>> repoToDirMapRef =
- new AtomicReference<>(ImmutableMap.of());
+ private final ManagedDirectoriesListener listener;
- /**
- * During build commands execution, Skyframe caches the states of files (FileStateValue) and
- * directory listings (DirectoryListingStateValue). In the case when the files/directories are
- * under a managed directory or inside an external repository, evaluation of file/directory
- * listing states requires that the RepositoryDirectoryValue of the owning external repository is
- * evaluated beforehand. (So that the repository rule generates the files.) So there is a
- * dependency on RepositoryDirectoryValue for files under managed directories and external
- * repositories. This dependency is recorded by Skyframe,
- *
- * <p>From the other side, by default Skyframe injects the new values of changed files already at
- * the stage of checking what files have changed. Only the values without any dependencies can be
- * injected into Skyframe. Skyframe can be specifically instructed to not inject new values and
- * only register them as changed.
- *
- * <p>When the values of managed directories change, some files appear to become files under
- * managed directories, or they are no longer files under managed directories. This implies that
- * corresponding file/directory listing states gain the dependency (RepositoryDirectoryValue) or
- * they lose this dependency. In both cases, we should prevent Skyframe from injecting those new
- * values of file/directory listing states at the stage of checking changed files.
- *
- * <p>That is why we need to keep track of the previous value of the managed directories.
- */
- private final AtomicReference<ImmutableSortedMap<PathFragment, RepositoryName>>
- oldManagedDirectoriesRef = new AtomicReference<>(ImmutableSortedMap.of());
+ private ImmutableSortedMap<PathFragment, RepositoryName> dirToRepoMap = ImmutableSortedMap.of();
+ private ImmutableSortedMap<RepositoryName, ImmutableSet<PathFragment>> repoToDirMap =
+ ImmutableSortedMap.of();
+
+ public ManagedDirectoriesKnowledgeImpl(ManagedDirectoriesListener listener) {
+ this.listener = listener;
+ }
@Override
@Nullable
- public RepositoryName getOwnerRepository(RootedPath rootedPath, boolean old) {
- PathFragment relativePath = rootedPath.getRootRelativePath();
- NavigableMap<PathFragment, RepositoryName> map =
- old ? oldManagedDirectoriesRef.get() : managedDirectoriesRef.get();
- Map.Entry<PathFragment, RepositoryName> entry = map.floorEntry(relativePath);
- if (entry != null && relativePath.startsWith(entry.getKey())) {
+ public RepositoryName getOwnerRepository(PathFragment relativePathFragment) {
+ Map.Entry<PathFragment, RepositoryName> entry = dirToRepoMap.floorEntry(relativePathFragment);
+ if (entry != null && relativePathFragment.startsWith(entry.getKey())) {
return entry.getValue();
}
return null;
@@ -76,26 +52,53 @@
@Override
public ImmutableSet<PathFragment> getManagedDirectories(RepositoryName repositoryName) {
- ImmutableSet<PathFragment> pathFragments = repoToDirMapRef.get().get(repositoryName);
+ ImmutableSet<PathFragment> pathFragments = repoToDirMap.get(repositoryName);
return pathFragments != null ? pathFragments : ImmutableSet.of();
}
- public void setManagedDirectories(ImmutableMap<PathFragment, RepositoryName> map) {
- oldManagedDirectoriesRef.set(managedDirectoriesRef.get());
- ImmutableSortedMap<PathFragment, RepositoryName> pathsMap = ImmutableSortedMap.copyOf(map);
- managedDirectoriesRef.set(pathsMap);
+ @Override
+ public boolean workspaceHeaderReloaded(
+ @Nullable WorkspaceFileValue oldValue, @Nullable WorkspaceFileValue newValue)
+ throws AbruptExitException {
+ if (Objects.equals(oldValue, newValue)) {
+ listener.onManagedDirectoriesRefreshed(repoToDirMap.keySet());
+ return false;
+ }
+ Map<PathFragment, RepositoryName> oldDirToRepoMap = dirToRepoMap;
+ refreshMappings(newValue);
+ if (!Objects.equals(oldDirToRepoMap, dirToRepoMap)) {
+ listener.onManagedDirectoriesRefreshed(repoToDirMap.keySet());
+ return true;
+ }
+ return false;
+ }
+
+ private void refreshMappings(@Nullable WorkspaceFileValue newValue) {
+ if (newValue == null) {
+ dirToRepoMap = ImmutableSortedMap.of();
+ repoToDirMap = ImmutableSortedMap.of();
+ return;
+ }
+
+ dirToRepoMap = ImmutableSortedMap.copyOf(newValue.getManagedDirectories());
Map<RepositoryName, Set<PathFragment>> reposMap = Maps.newHashMap();
- for (Map.Entry<PathFragment, RepositoryName> entry : pathsMap.entrySet()) {
+ for (Map.Entry<PathFragment, RepositoryName> entry : dirToRepoMap.entrySet()) {
RepositoryName repositoryName = entry.getValue();
reposMap.computeIfAbsent(repositoryName, name -> Sets.newTreeSet()).add(entry.getKey());
}
- ImmutableSortedMap.Builder<RepositoryName, ImmutableSet<PathFragment>> builder =
+ ImmutableSortedMap.Builder<RepositoryName, ImmutableSet<PathFragment>> reposMapBuilder =
new ImmutableSortedMap.Builder<>(Comparator.comparing(RepositoryName::getName));
for (Map.Entry<RepositoryName, Set<PathFragment>> entry : reposMap.entrySet()) {
- builder.put(entry.getKey(), ImmutableSet.copyOf(entry.getValue()));
+ reposMapBuilder.put(entry.getKey(), ImmutableSet.copyOf(entry.getValue()));
}
- repoToDirMapRef.set(builder.build());
+ repoToDirMap = reposMapBuilder.build();
+ }
+
+ /** Interface allows {@link BazelRepositoryModule} to react to managed directories refreshes. */
+ public interface ManagedDirectoriesListener {
+ void onManagedDirectoriesRefreshed(Set<RepositoryName> repositoryNames)
+ throws AbruptExitException;
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java
index 2b2332b..a996df9 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java
@@ -99,10 +99,10 @@
* {@link RepositoryDelegatorFunction} has to know how to catch.</p>
*/
public static class RepositoryFunctionException extends SkyFunctionException {
+
public RepositoryFunctionException(NoSuchPackageException cause, Transience transience) {
super(cause, transience);
}
-
/**
* Error reading or writing to the filesystem.
*/
@@ -117,11 +117,11 @@
super(cause, transience);
}
}
-
/**
* Exception thrown when something a repository rule cannot be found.
*/
public static final class RepositoryNotFoundException extends RepositoryFunctionException {
+
public RepositoryNotFoundException(String repositoryName) {
super(
new BuildFileContainsErrorsException(
@@ -130,7 +130,6 @@
Transience.PERSISTENT);
}
}
-
/**
* An exception thrown when a dependency is missing to notify the SkyFunction from an evaluation.
*/
@@ -140,7 +139,6 @@
super(Location.BUILTIN, "Internal exception");
}
}
-
/**
* repository functions can throw the result of this function to notify the RepositoryFunction
* that a dependency was missing and the evaluation of the function must be restarted.
@@ -528,7 +526,7 @@
*/
public static void addExternalFilesDependencies(
RootedPath rootedPath, boolean isDirectory, BlazeDirectories directories, Environment env)
- throws IOException, InterruptedException {
+ throws InterruptedException {
Path externalRepoDir = getExternalRepositoryDirectory(directories);
PathFragment repositoryPath = rootedPath.asPath().relativeTo(externalRepoDir);
if (repositoryPath.isEmpty()) {
@@ -579,7 +577,6 @@
// Alternatively, the repository might still be provided by an override. Therefore, in
// any case, register the dependency on the repository overrides.
RepositoryDelegatorFunction.REPOSITORY_OVERRIDES.get(env);
- return;
} catch (ExternalPackageException ex) {
// This should never happen.
throw new IllegalStateException(
@@ -588,6 +585,22 @@
}
/**
+ * For paths that are under managed directories, we require that the corresponding FileStateValue
+ * or DirectoryListingStateValue is evaluated only after RepositoryDirectoryValue is evaluated.
+ * This way we guarantee that the repository rule is given a chance to update the managed
+ * directory before the files under the managed directory are accessed.
+ *
+ * <p>We do not need to require anything else (comparing to dependencies required for external
+ * repositories files), as overriding external repositories with managed directories is currently
+ * forbidden; also, we do not have do perform special checks for local_repository targets, since
+ * such targets cannot have managed directories by definition.
+ */
+ public static void addManagedDirectoryDependencies(RepositoryName repositoryName, Environment env)
+ throws InterruptedException {
+ env.getValue(RepositoryDirectoryValue.key(repositoryName));
+ }
+
+ /**
* Sets up a mapping of environment variables to use.
*/
public void setClientEnvironment(Map<String, String> clientEnvironment) {
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/WorkspaceBuilder.java b/src/main/java/com/google/devtools/build/lib/runtime/WorkspaceBuilder.java
index bf10273..a86a406 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/WorkspaceBuilder.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/WorkspaceBuilder.java
@@ -23,8 +23,8 @@
import com.google.devtools.build.lib.exec.BinTools;
import com.google.devtools.build.lib.packages.PackageFactory;
import com.google.devtools.build.lib.profiler.memory.AllocationTracker;
+import com.google.devtools.build.lib.rules.repository.ManagedDirectoriesKnowledge;
import com.google.devtools.build.lib.skyframe.DiffAwareness;
-import com.google.devtools.build.lib.skyframe.SequencedSkyframeExecutor.WorkspaceFileHeaderListener;
import com.google.devtools.build.lib.skyframe.SequencedSkyframeExecutorFactory;
import com.google.devtools.build.lib.skyframe.SkyValueDirtinessChecker;
import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
@@ -53,7 +53,7 @@
private final ImmutableList.Builder<SkyValueDirtinessChecker> customDirtinessCheckers =
ImmutableList.builder();
private AllocationTracker allocationTracker;
- private WorkspaceFileHeaderListener workspaceFileHeaderListener;
+ private ManagedDirectoriesKnowledge managedDirectoriesKnowledge;
WorkspaceBuilder(BlazeDirectories directories, BinTools binTools) {
this.directories = directories;
@@ -82,7 +82,7 @@
diffAwarenessFactories.build(),
skyFunctions.build(),
customDirtinessCheckers.build(),
- workspaceFileHeaderListener);
+ managedDirectoriesKnowledge);
return new BlazeWorkspace(
runtime,
directories,
@@ -157,9 +157,9 @@
return this;
}
- public WorkspaceBuilder setWorkspaceFileHeaderListener(
- WorkspaceFileHeaderListener workspaceFileHeaderListener) {
- this.workspaceFileHeaderListener = workspaceFileHeaderListener;
+ public WorkspaceBuilder setManagedDirectoriesKnowledge(
+ ManagedDirectoriesKnowledge managedDirectoriesKnowledge) {
+ this.managedDirectoriesKnowledge = managedDirectoriesKnowledge;
return this;
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateFunction.java
index c3d3ce9..8e8b5bb 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateFunction.java
@@ -13,6 +13,7 @@
// limitations under the License.
package com.google.devtools.build.lib.skyframe;
+import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.FileType;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.lib.vfs.Symlinks;
import com.google.devtools.build.lib.vfs.UnixGlob.FilesystemCalls;
@@ -51,10 +52,17 @@
RootedPath dirRootedPath = (RootedPath) skyKey.argument();
try {
- externalFilesHelper.maybeHandleExternalFile(dirRootedPath, true, env);
+ FileType fileType = externalFilesHelper.maybeHandleExternalFile(dirRootedPath, true, env);
if (env.valuesMissing()) {
return null;
}
+ if (fileType == FileType.EXTERNAL_REPO
+ || fileType == FileType.EXTERNAL_IN_MANAGED_DIRECTORY) {
+ // Do not use syscallCache as files under repositories get generated during the build,
+ // while syscallCache is used independently from Skyframe and generally assumes
+ // the file system is frozen at the beginning of the build command.
+ return DirectoryListingStateValue.create(dirRootedPath);
+ }
return DirectoryListingStateValue.create(
syscallCache.get().readdir(dirRootedPath.asPath(), Symlinks.NOFOLLOW));
} catch (ExternalFilesHelper.NonexistentImmutableExternalFileException e) {
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirtinessCheckerUtils.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirtinessCheckerUtils.java
index ce4de19..860f437 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/DirtinessCheckerUtils.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirtinessCheckerUtils.java
@@ -106,7 +106,11 @@
}
}
- /** Checks files outside of the package roots for changes. */
+ /**
+ * Serves for tracking whether there are external and output files {@see ExternalFilesKnowledge}.
+ * Filtering of files, for which the new values should not be injected into evaluator, is done in
+ * SequencedSkyframeExecutor.handleChangedFiles().
+ */
static final class ExternalDirtinessChecker extends BasicFilesystemDirtinessChecker {
private final ExternalFilesHelper externalFilesHelper;
private final EnumSet<FileType> fileTypesToCheck;
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ExternalFilesHelper.java b/src/main/java/com/google/devtools/build/lib/skyframe/ExternalFilesHelper.java
index abc7577..1fcbc4e 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/ExternalFilesHelper.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ExternalFilesHelper.java
@@ -14,12 +14,17 @@
package com.google.devtools.build.lib.skyframe;
import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.FileStateValue;
+import com.google.devtools.build.lib.actions.FileValue;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.cmdline.LabelConstants;
+import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.rules.repository.ManagedDirectoriesKnowledge;
import com.google.devtools.build.lib.rules.repository.RepositoryFunction;
+import com.google.devtools.build.lib.util.Pair;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.skyframe.SkyFunction;
@@ -45,40 +50,59 @@
private boolean anyOutputFilesSeen = false;
private boolean anyNonOutputExternalFilesSeen = false;
+ private final ManagedDirectoriesKnowledge managedDirectoriesKnowledge;
+
private ExternalFilesHelper(
AtomicReference<PathPackageLocator> pkgLocator,
ExternalFileAction externalFileAction,
BlazeDirectories directories,
- int maxNumExternalFilesToLog) {
+ int maxNumExternalFilesToLog,
+ ManagedDirectoriesKnowledge managedDirectoriesKnowledge) {
this.pkgLocator = pkgLocator;
this.externalFileAction = externalFileAction;
this.directories = directories;
this.maxNumExternalFilesToLog = maxNumExternalFilesToLog;
+ this.managedDirectoriesKnowledge = managedDirectoriesKnowledge;
}
public static ExternalFilesHelper create(
AtomicReference<PathPackageLocator> pkgLocator,
ExternalFileAction externalFileAction,
- BlazeDirectories directories) {
+ BlazeDirectories directories,
+ ManagedDirectoriesKnowledge managedDirectoriesKnowledge) {
return IN_TEST
- ? createForTesting(pkgLocator, externalFileAction, directories)
+ ? createForTesting(pkgLocator, externalFileAction, directories, managedDirectoriesKnowledge)
: new ExternalFilesHelper(
pkgLocator,
externalFileAction,
directories,
- /*maxNumExternalFilesToLog=*/ 100);
+ /*maxNumExternalFilesToLog=*/ 100,
+ managedDirectoriesKnowledge);
}
public static ExternalFilesHelper createForTesting(
AtomicReference<PathPackageLocator> pkgLocator,
ExternalFileAction externalFileAction,
BlazeDirectories directories) {
+ return createForTesting(
+ pkgLocator,
+ externalFileAction,
+ directories,
+ ManagedDirectoriesKnowledge.NO_MANAGED_DIRECTORIES);
+ }
+
+ private static ExternalFilesHelper createForTesting(
+ AtomicReference<PathPackageLocator> pkgLocator,
+ ExternalFileAction externalFileAction,
+ BlazeDirectories directories,
+ ManagedDirectoriesKnowledge managedDirectoriesKnowledge) {
return new ExternalFilesHelper(
pkgLocator,
externalFileAction,
directories,
// These log lines are mostly spam during unit and integration tests.
- /*maxNumExternalFilesToLog=*/ 0);
+ /*maxNumExternalFilesToLog=*/ 0,
+ managedDirectoriesKnowledge);
}
@@ -105,12 +129,12 @@
/** Classification of a path encountered by Bazel. */
public enum FileType {
- /** A path inside the package roots or in an external repository. */
+ /** A path inside the package roots. */
INTERNAL,
/**
- * A non {@link #EXTERNAL_REPO} path outside the package roots about which we may make no other
- * assumptions.
+ * A non {@link #EXTERNAL_REPO} or {@link #EXTERNAL_IN_MANAGED_DIRECTORY} path outside the
+ * package roots about which we may make no other assumptions.
*/
EXTERNAL,
@@ -132,9 +156,22 @@
/**
* A path in the part of Bazel's output tree that contains (/ symlinks to) to external
- * repositories.
+ * repositories. Every such path under the external repository is generated by the execution of
+ * the corresponding repository rule, so these paths should not be cached by Skyframe before the
+ * RepositoryDirectoryValue is computed.
*/
EXTERNAL_REPO,
+
+ /**
+ * A path is under one of the managed directories. Managed directories are user-owned
+ * directories, which can be incrementally updated by repository rules, so that the updated
+ * files are visible for the actions in the same build.
+ *
+ * <p>Every such path under the managed directory is generated or updated by the execution of
+ * the corresponding repository rule, so these paths should not be cached by Skyframe before the
+ * RepositoryDirectoryValue is computed. {@link ManagedDirectoriesKnowledge}
+ */
+ EXTERNAL_IN_MANAGED_DIRECTORY,
}
/**
@@ -168,10 +205,35 @@
ExternalFilesHelper cloneWithFreshExternalFilesKnowledge() {
return new ExternalFilesHelper(
- pkgLocator, externalFileAction, directories, maxNumExternalFilesToLog);
+ pkgLocator,
+ externalFileAction,
+ directories,
+ maxNumExternalFilesToLog,
+ managedDirectoriesKnowledge);
}
- FileType getAndNoteFileType(RootedPath rootedPath) {
+ public FileType getAndNoteFileType(RootedPath rootedPath) {
+ return getFileTypeAndRepository(rootedPath).getFirst();
+ }
+
+ private Pair<FileType, RepositoryName> getFileTypeAndRepository(RootedPath rootedPath) {
+ RepositoryName repositoryName =
+ managedDirectoriesKnowledge.getOwnerRepository(rootedPath.getRootRelativePath());
+ if (repositoryName != null) {
+ anyNonOutputExternalFilesSeen = true;
+ return Pair.of(FileType.EXTERNAL_IN_MANAGED_DIRECTORY, repositoryName);
+ }
+ FileType fileType = detectFileType(rootedPath);
+ if (FileType.EXTERNAL == fileType || FileType.EXTERNAL_REPO == fileType) {
+ anyNonOutputExternalFilesSeen = true;
+ }
+ if (FileType.OUTPUT == fileType) {
+ anyOutputFilesSeen = true;
+ }
+ return Pair.of(fileType, null);
+ }
+
+ private FileType detectFileType(RootedPath rootedPath) {
PathPackageLocator packageLocator = pkgLocator.get();
if (packageLocator.getPathEntries().contains(rootedPath.getRoot())) {
return FileType.INTERNAL;
@@ -204,27 +266,39 @@
* a {@link NonexistentImmutableExternalFileException} instead.
*/
@ThreadSafe
- void maybeHandleExternalFile(
+ FileType maybeHandleExternalFile(
RootedPath rootedPath, boolean isDirectory, SkyFunction.Environment env)
throws NonexistentImmutableExternalFileException, IOException, InterruptedException {
- FileType fileType = getAndNoteFileType(rootedPath);
- if (fileType == FileType.INTERNAL) {
- return;
+ Pair<FileType, RepositoryName> pair = getFileTypeAndRepository(rootedPath);
+
+ FileType fileType = Preconditions.checkNotNull(pair.getFirst());
+ switch (fileType) {
+ case EXTERNAL_IN_MANAGED_DIRECTORY:
+ Preconditions.checkState(
+ externalFileAction == ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS,
+ externalFileAction);
+ RepositoryFunction.addManagedDirectoryDependencies(pair.getSecond(), env);
+ break;
+ case INTERNAL:
+ break;
+ case EXTERNAL:
+ if (numExternalFilesLogged.incrementAndGet() < maxNumExternalFilesToLog) {
+ logger.info("Encountered an external path " + rootedPath);
+ }
+ // fall through
+ case OUTPUT:
+ if (externalFileAction
+ == ExternalFileAction.ASSUME_NON_EXISTENT_AND_IMMUTABLE_FOR_EXTERNAL_PATHS) {
+ throw new NonexistentImmutableExternalFileException();
+ }
+ break;
+ case EXTERNAL_REPO:
+ Preconditions.checkState(
+ externalFileAction == ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS,
+ externalFileAction);
+ RepositoryFunction.addExternalFilesDependencies(rootedPath, isDirectory, directories, env);
+ break;
}
- if (fileType == FileType.OUTPUT || fileType == FileType.EXTERNAL) {
- if (externalFileAction
- == ExternalFileAction.ASSUME_NON_EXISTENT_AND_IMMUTABLE_FOR_EXTERNAL_PATHS) {
- throw new NonexistentImmutableExternalFileException();
- }
- if (fileType == FileType.EXTERNAL
- && numExternalFilesLogged.incrementAndGet() < maxNumExternalFilesToLog) {
- logger.info("Encountered an external path " + rootedPath);
- }
- return;
- }
- Preconditions.checkState(
- externalFileAction == ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS,
- externalFileAction);
- RepositoryFunction.addExternalFilesDependencies(rootedPath, isDirectory, directories, env);
+ return fileType;
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileStateFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateFunction.java
index 289c943..8ece6ee 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/FileStateFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateFunction.java
@@ -14,6 +14,7 @@
package com.google.devtools.build.lib.skyframe;
import com.google.devtools.build.lib.actions.FileStateValue;
+import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.FileType;
import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.lib.vfs.UnixGlob.FilesystemCalls;
@@ -50,10 +51,15 @@
RootedPath rootedPath = (RootedPath) skyKey.argument();
try {
- externalFilesHelper.maybeHandleExternalFile(rootedPath, false, env);
+ FileType fileType = externalFilesHelper.maybeHandleExternalFile(rootedPath, false, env);
if (env.valuesMissing()) {
return null;
}
+ if (fileType == FileType.EXTERNAL_REPO
+ || fileType == FileType.EXTERNAL_IN_MANAGED_DIRECTORY) {
+ // do not use syscallCache as files under repositories get generated during the build
+ return FileStateValue.create(rootedPath, tsgm.get());
+ }
return FileStateValue.create(rootedPath, syscallCache.get(), tsgm.get());
} catch (ExternalFilesHelper.NonexistentImmutableExternalFileException e) {
return FileStateValue.NONEXISTENT_FILE_STATE_NODE;
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
index 08f5fa1..8130c4d 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
@@ -62,7 +62,9 @@
import com.google.devtools.build.lib.profiler.ProfilerTask;
import com.google.devtools.build.lib.profiler.SilentCloseable;
import com.google.devtools.build.lib.query2.AqueryActionFilter;
+import com.google.devtools.build.lib.rules.repository.ManagedDirectoriesKnowledge;
import com.google.devtools.build.lib.skyframe.AspectValue.AspectKey;
+import com.google.devtools.build.lib.skyframe.DiffAwarenessManager.ProcessableModifiedFileSet;
import com.google.devtools.build.lib.skyframe.DirtinessCheckerUtils.BasicFilesystemDirtinessChecker;
import com.google.devtools.build.lib.skyframe.DirtinessCheckerUtils.ExternalDirtinessChecker;
import com.google.devtools.build.lib.skyframe.DirtinessCheckerUtils.MissingDiffDirtinessChecker;
@@ -86,6 +88,7 @@
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.skyframe.BuildDriver;
import com.google.devtools.build.skyframe.Differencer;
+import com.google.devtools.build.skyframe.Differencer.Diff;
import com.google.devtools.build.skyframe.EvaluationContext;
import com.google.devtools.build.skyframe.GraphInconsistencyReceiver;
import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator;
@@ -109,7 +112,6 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
@@ -149,7 +151,8 @@
private Duration sourceDiffCheckingDuration = Duration.ofSeconds(-1L);
private Duration outputTreeDiffCheckingDuration = Duration.ofSeconds(-1L);
- @Nullable private final WorkspaceFileHeaderListener workspaceFileHeaderListener;
+ // If this is null then workspace header pre-calculation won't happen.
+ @Nullable private final ManagedDirectoriesKnowledge managedDirectoriesKnowledge;
private SequencedSkyframeExecutor(
Consumer<SkyframeExecutor> skyframeExecutorConsumerOnInit,
@@ -170,7 +173,7 @@
ActionOnIOExceptionReadingBuildFile actionOnIOExceptionReadingBuildFile,
BuildOptions defaultBuildOptions,
MutableArtifactFactorySupplier mutableArtifactFactorySupplier,
- @Nullable WorkspaceFileHeaderListener workspaceFileHeaderListener) {
+ @Nullable ManagedDirectoriesKnowledge managedDirectoriesKnowledge) {
super(
skyframeExecutorConsumerOnInit,
evaluatorSupplier,
@@ -193,10 +196,11 @@
new PackageProgressReceiver(),
mutableArtifactFactorySupplier,
new ConfiguredTargetProgressReceiver(),
- /*nonexistentFileReceiver=*/ null);
+ /*nonexistentFileReceiver=*/ null,
+ managedDirectoriesKnowledge);
this.diffAwarenessManager = new DiffAwarenessManager(diffAwarenessFactories);
this.customDirtinessCheckers = customDirtinessCheckers;
- this.workspaceFileHeaderListener = workspaceFileHeaderListener;
+ this.managedDirectoriesKnowledge = managedDirectoriesKnowledge;
}
@Override
@@ -330,8 +334,10 @@
TimestampGranularityMonitor tsgm = this.tsgm.get();
modifiedFiles = 0;
- if (workspaceFileHeaderListener != null) {
- refreshWorkspaceHeader(eventHandler);
+ boolean managedDirectoriesChanged =
+ managedDirectoriesKnowledge != null && refreshWorkspaceHeader(eventHandler);
+ if (managedDirectoriesChanged) {
+ invalidateCachedWorkspacePathsStates();
}
Map<Root, DiffAwarenessManager.ProcessableModifiedFileSet> modifiedFilesByPathEntry =
@@ -347,12 +353,47 @@
modifiedFilesByPathEntry.put(pathEntry, modifiedFileSet);
}
}
- handleDiffsWithCompleteDiffInformation(tsgm, modifiedFilesByPathEntry);
- handleDiffsWithMissingDiffInformation(eventHandler, tsgm, pathEntriesWithoutDiffInformation,
- checkOutputFiles);
+ handleDiffsWithCompleteDiffInformation(
+ tsgm, modifiedFilesByPathEntry, managedDirectoriesChanged);
+ handleDiffsWithMissingDiffInformation(
+ eventHandler,
+ tsgm,
+ pathEntriesWithoutDiffInformation,
+ checkOutputFiles,
+ managedDirectoriesChanged);
handleClientEnvironmentChanges();
}
+ /**
+ * Skyframe caches the states of files (FileStateValue) and directory listings
+ * (DirectoryListingStateValue). In the case when the files/directories are under a managed
+ * directory or inside an external repository, evaluation of file/directory listing states
+ * requires that RepositoryDirectoryValue of the owning external repository is evaluated
+ * beforehand. (So that the repository rule generates the files.) So there is a dependency on
+ * RepositoryDirectoryValue for files under managed directories and external repositories. This
+ * dependency is recorded by Skyframe.
+ *
+ * <p>From the other side, by default Skyframe injects the new values of changed files already at
+ * the stage of checking what files have changed. Only values without any dependencies can be
+ * injected into Skyframe.
+ *
+ * <p>When the values of managed directories change, whether a file is under a managed directory
+ * can change. This implies that corresponding file/directory listing states gain the dependency
+ * (RepositoryDirectoryValue) or they lose this dependency. In both cases, we should prevent
+ * Skyframe from injecting those new values of file/directory listing states at the stage of
+ * checking changed files, because the files have not been generated yet.
+ *
+ * <p>The selected approach is to invalidate PACKAGE_LOCATOR_DEPENDENT_VALUES, which includes
+ * invalidating all cached file/directory listing state values. Additionally, no new
+ * FileStateValues and DirectoryStateValues should be injected.
+ */
+ private void invalidateCachedWorkspacePathsStates() {
+ logger.info(
+ "Invalidating cached packages and paths states"
+ + " because managed directories configuration has changed.");
+ invalidate(SkyFunctionName.functionIsIn(PACKAGE_LOCATOR_DEPENDENT_VALUES));
+ }
+
/** Invalidates entries in the client environment. */
private void handleClientEnvironmentChanges() {
// Remove deleted client environmental variables.
@@ -382,15 +423,18 @@
*/
private void handleDiffsWithCompleteDiffInformation(
TimestampGranularityMonitor tsgm,
- Map<Root, DiffAwarenessManager.ProcessableModifiedFileSet> modifiedFilesByPathEntry)
+ Map<Root, ProcessableModifiedFileSet> modifiedFilesByPathEntry,
+ boolean managedDirectoriesChanged)
throws InterruptedException {
for (Root pathEntry : ImmutableSet.copyOf(modifiedFilesByPathEntry.keySet())) {
DiffAwarenessManager.ProcessableModifiedFileSet processableModifiedFileSet =
modifiedFilesByPathEntry.get(pathEntry);
ModifiedFileSet modifiedFileSet = processableModifiedFileSet.getModifiedFileSet();
Preconditions.checkState(!modifiedFileSet.treatEverythingAsModified(), pathEntry);
- handleChangedFiles(ImmutableList.of(pathEntry),
- getDiff(tsgm, modifiedFileSet.modifiedSourceFiles(), pathEntry));
+ handleChangedFiles(
+ ImmutableList.of(pathEntry),
+ getDiff(tsgm, modifiedFileSet.modifiedSourceFiles(), pathEntry),
+ managedDirectoriesChanged);
processableModifiedFileSet.markProcessed();
}
}
@@ -402,9 +446,9 @@
private void handleDiffsWithMissingDiffInformation(
ExtendedEventHandler eventHandler,
TimestampGranularityMonitor tsgm,
- Set<Pair<Root, DiffAwarenessManager.ProcessableModifiedFileSet>>
- pathEntriesWithoutDiffInformation,
- boolean checkOutputFiles)
+ Set<Pair<Root, ProcessableModifiedFileSet>> pathEntriesWithoutDiffInformation,
+ boolean checkOutputFiles,
+ boolean managedDirectoriesChanged)
throws InterruptedException {
ExternalFilesKnowledge externalFilesKnowledge =
externalFilesHelper.getExternalFilesKnowledge();
@@ -464,7 +508,7 @@
new ExternalDirtinessChecker(tmpExternalFilesHelper, fileTypesToCheck),
new MissingDiffDirtinessChecker(diffPackageRootsUnderWhichToCheck)))));
}
- handleChangedFiles(diffPackageRootsUnderWhichToCheck, diff);
+ handleChangedFiles(diffPackageRootsUnderWhichToCheck, diff, managedDirectoriesChanged);
for (Pair<Root, DiffAwarenessManager.ProcessableModifiedFileSet> pair :
pathEntriesWithoutDiffInformation) {
@@ -478,18 +522,33 @@
}
private void handleChangedFiles(
- Collection<Root> diffPackageRootsUnderWhichToCheck, Differencer.Diff diff) {
- Collection<SkyKey> changedKeysWithoutNewValues = diff.changedKeysWithoutNewValues();
+ Collection<Root> diffPackageRootsUnderWhichToCheck,
+ Diff diff,
+ boolean managedDirectoriesChanged) {
+ int numWithoutNewValues = diff.changedKeysWithoutNewValues().size();
+ Iterable<SkyKey> keysToBeChangedLaterInThisBuild = diff.changedKeysWithoutNewValues();
Map<SkyKey, SkyValue> changedKeysWithNewValues = diff.changedKeysWithNewValues();
- logDiffInfo(diffPackageRootsUnderWhichToCheck, changedKeysWithoutNewValues,
+ // If managed directories settings changed, do not inject any new values, just invalidate
+ // keys of the changed values. {@link #invalidateCachedWorkspacePathsStates()}
+ if (managedDirectoriesChanged) {
+ numWithoutNewValues += changedKeysWithNewValues.size();
+ keysToBeChangedLaterInThisBuild =
+ Iterables.concat(keysToBeChangedLaterInThisBuild, changedKeysWithNewValues.keySet());
+ changedKeysWithNewValues = ImmutableMap.of();
+ }
+
+ logDiffInfo(
+ diffPackageRootsUnderWhichToCheck,
+ keysToBeChangedLaterInThisBuild,
+ numWithoutNewValues,
changedKeysWithNewValues);
- recordingDiffer.invalidate(changedKeysWithoutNewValues);
+ recordingDiffer.invalidate(keysToBeChangedLaterInThisBuild);
recordingDiffer.inject(changedKeysWithNewValues);
- modifiedFiles += getNumberOfModifiedFiles(changedKeysWithoutNewValues);
+ modifiedFiles += getNumberOfModifiedFiles(keysToBeChangedLaterInThisBuild);
modifiedFiles += getNumberOfModifiedFiles(changedKeysWithNewValues.keySet());
- incrementalBuildMonitor.accrue(changedKeysWithoutNewValues);
+ incrementalBuildMonitor.accrue(keysToBeChangedLaterInThisBuild);
incrementalBuildMonitor.accrue(changedKeysWithNewValues.keySet());
}
@@ -497,9 +556,10 @@
private static void logDiffInfo(
Iterable<Root> pathEntries,
- Collection<SkyKey> changedWithoutNewValue,
+ Iterable<SkyKey> changedWithoutNewValue,
+ int numWithoutNewValues,
Map<SkyKey, ? extends SkyValue> changedWithNewValue) {
- int numModified = changedWithNewValue.size() + changedWithoutNewValue.size();
+ int numModified = changedWithNewValue.size() + numWithoutNewValues;
StringBuilder result = new StringBuilder("DiffAwareness found ")
.append(numModified)
.append(" modified source files and directory listings");
@@ -826,10 +886,8 @@
* and call the listener, if the value has changed. Needed for incremental update of user-owned
* directories by repository rules.
*/
- private void refreshWorkspaceHeader(ExtendedEventHandler eventHandler)
+ private boolean refreshWorkspaceHeader(ExtendedEventHandler eventHandler)
throws InterruptedException, AbruptExitException {
- Preconditions.checkNotNull(workspaceFileHeaderListener);
-
Root workspaceRoot = Root.fromPath(directories.getWorkspace());
RootedPath workspacePath =
RootedPath.toRootedPath(workspaceRoot, LabelConstants.WORKSPACE_FILE_NAME);
@@ -840,9 +898,7 @@
maybeInvalidateWorkspaceFileStateValue(workspacePath);
WorkspaceFileValue newValue =
(WorkspaceFileValue) evaluateSingleValue(workspaceFileKey, eventHandler);
- if (!Objects.equals(newValue, oldValue)) {
- workspaceFileHeaderListener.workspaceHeaderChanged(newValue);
- }
+ return managedDirectoriesKnowledge.workspaceHeaderReloaded(oldValue, newValue);
}
// We only check the FileStateValue of the WORKSPACE file; we do not support the case
@@ -901,13 +957,12 @@
protected ActionKeyContext actionKeyContext;
protected ImmutableList<BuildInfoFactory> buildInfoFactories;
protected BuildOptions defaultBuildOptions;
-
private ImmutableSet<PathFragment> hardcodedBlacklistedPackagePrefixes;
private PathFragment additionalBlacklistedPackagePrefixesFile;
private CrossRepositoryLabelViolationStrategy crossRepositoryLabelViolationStrategy;
private List<BuildFileName> buildFilesByPriority;
private ActionOnIOExceptionReadingBuildFile actionOnIOExceptionReadingBuildFile;
- private WorkspaceFileHeaderListener workspaceFileHeaderListener;
+ @Nullable private ManagedDirectoriesKnowledge managedDirectoriesKnowledge;
// Fields with default values.
private ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions = ImmutableMap.of();
@@ -954,16 +1009,11 @@
actionOnIOExceptionReadingBuildFile,
defaultBuildOptions,
mutableArtifactFactorySupplier,
- workspaceFileHeaderListener);
+ managedDirectoriesKnowledge);
skyframeExecutor.init();
return skyframeExecutor;
}
- public Builder modify(Consumer<Builder> consumer) {
- consumer.accept(this);
- return this;
- }
-
public Builder setPkgFactory(PackageFactory pkgFactory) {
this.pkgFactory = pkgFactory;
return this;
@@ -1051,27 +1101,28 @@
this.mutableArtifactFactorySupplier = mutableArtifactFactorySupplier;
return this;
}
-
public Builder setSkyframeExecutorConsumerOnInit(
Consumer<SkyframeExecutor> skyframeExecutorConsumerOnInit) {
this.skyframeExecutorConsumerOnInit = skyframeExecutorConsumerOnInit;
return this;
}
- public Builder setWorkspaceFileHeaderListener(
- WorkspaceFileHeaderListener workspaceFileHeaderListener) {
- this.workspaceFileHeaderListener = workspaceFileHeaderListener;
+ public Builder setManagedDirectoriesKnowledge(
+ @Nullable ManagedDirectoriesKnowledge managedDirectoriesKnowledge) {
+ this.managedDirectoriesKnowledge = managedDirectoriesKnowledge;
return this;
}
}
/**
- * Listener class to subscribe for WORKSPACE file header changes.
+ * Listener class to subscribe for WORKSPACE file header refreshes.
*
* <p>Changes to WORKSPACE file header are computed before the files difference is computed in
* {@link #handleDiffs(ExtendedEventHandler, boolean, OptionsProvider)}
*/
public interface WorkspaceFileHeaderListener {
- void workspaceHeaderChanged(@Nullable WorkspaceFileValue newValue);
+ boolean workspaceHeaderReloaded(
+ @Nullable WorkspaceFileValue oldValue, @Nullable WorkspaceFileValue newValue)
+ throws AbruptExitException;
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutorFactory.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutorFactory.java
index ca668ea..33f5a61 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutorFactory.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutorFactory.java
@@ -21,7 +21,7 @@
import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
import com.google.devtools.build.lib.analysis.config.BuildOptions;
import com.google.devtools.build.lib.packages.PackageFactory;
-import com.google.devtools.build.lib.skyframe.SequencedSkyframeExecutor.WorkspaceFileHeaderListener;
+import com.google.devtools.build.lib.rules.repository.ManagedDirectoriesKnowledge;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionName;
@@ -49,7 +49,7 @@
Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories,
ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions,
Iterable<SkyValueDirtinessChecker> customDirtinessCheckers,
- @Nullable WorkspaceFileHeaderListener workspaceFileHeaderListener) {
+ @Nullable ManagedDirectoriesKnowledge managedDirectoriesKnowledge) {
return BazelSkyframeExecutorConstants.newBazelSkyframeExecutorBuilder()
.setPkgFactory(pkgFactory)
.setFileSystem(fileSystem)
@@ -61,7 +61,7 @@
.setDiffAwarenessFactories(diffAwarenessFactories)
.setExtraSkyFunctions(extraSkyFunctions)
.setCustomDirtinessCheckers(customDirtinessCheckers)
- .setWorkspaceFileHeaderListener(workspaceFileHeaderListener)
+ .setManagedDirectoriesKnowledge(managedDirectoriesKnowledge)
.build();
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
index e89873b..ced09be 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
@@ -70,6 +70,7 @@
import com.google.devtools.build.lib.analysis.PlatformOptions;
import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory;
import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
@@ -128,6 +129,7 @@
import com.google.devtools.build.lib.profiler.SilentCloseable;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
import com.google.devtools.build.lib.remote.options.RemoteOutputsMode;
+import com.google.devtools.build.lib.rules.repository.ManagedDirectoriesKnowledge;
import com.google.devtools.build.lib.rules.repository.ResolvedFileFunction;
import com.google.devtools.build.lib.rules.repository.ResolvedHashesFunction;
import com.google.devtools.build.lib.runtime.KeepGoingOption;
@@ -381,7 +383,7 @@
FileSystem fileSystem,
BlazeDirectories directories,
ActionKeyContext actionKeyContext,
- WorkspaceStatusAction.Factory workspaceStatusActionFactory,
+ Factory workspaceStatusActionFactory,
ImmutableList<BuildInfoFactory> buildInfoFactories,
ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions,
ExternalFileAction externalFileAction,
@@ -396,7 +398,8 @@
@Nullable PackageProgressReceiver packageProgress,
MutableArtifactFactorySupplier artifactResolverSupplier,
@Nullable ConfiguredTargetProgressReceiver configuredTargetProgress,
- @Nullable NonexistentFileReceiver nonexistentFileReceiver) {
+ @Nullable NonexistentFileReceiver nonexistentFileReceiver,
+ @Nullable ManagedDirectoriesKnowledge managedDirectoriesKnowledge) {
// Strictly speaking, these arguments are not required for initialization, but all current
// callsites have them at hand, so we might as well set them during construction.
this.skyframeExecutorConsumerOnInit = skyframeExecutorConsumerOnInit;
@@ -443,7 +446,13 @@
this.artifactFactory = artifactResolverSupplier;
this.artifactFactory.set(skyframeBuildView.getArtifactFactory());
this.externalFilesHelper =
- ExternalFilesHelper.create(pkgLocator, externalFileAction, directories);
+ ExternalFilesHelper.create(
+ pkgLocator,
+ externalFileAction,
+ directories,
+ managedDirectoriesKnowledge != null
+ ? managedDirectoriesKnowledge
+ : ManagedDirectoriesKnowledge.NO_MANAGED_DIRECTORIES);
this.crossRepositoryLabelViolationStrategy = crossRepositoryLabelViolationStrategy;
this.buildFilesByPriority = buildFilesByPriority;
this.actionOnIOExceptionReadingBuildFile = actionOnIOExceptionReadingBuildFile;
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutorFactory.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutorFactory.java
index 504e4a4..e7c7106 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutorFactory.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutorFactory.java
@@ -20,7 +20,7 @@
import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory;
import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
import com.google.devtools.build.lib.packages.PackageFactory;
-import com.google.devtools.build.lib.skyframe.SequencedSkyframeExecutor.WorkspaceFileHeaderListener;
+import com.google.devtools.build.lib.rules.repository.ManagedDirectoriesKnowledge;
import com.google.devtools.build.lib.util.AbruptExitException;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.skyframe.SkyFunction;
@@ -52,6 +52,6 @@
Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories,
ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions,
Iterable<SkyValueDirtinessChecker> customDirtinessCheckers,
- WorkspaceFileHeaderListener workspaceFileHeaderListener)
+ ManagedDirectoriesKnowledge managedDirectoriesKnowledge)
throws AbruptExitException;
}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java b/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java
index 240eeec..69072d4 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java
@@ -43,6 +43,7 @@
import com.google.devtools.build.lib.packages.PackageFactory.EnvironmentExtension;
import com.google.devtools.build.lib.packages.WorkspaceFileValue;
import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.rules.repository.ManagedDirectoriesKnowledge;
import com.google.devtools.build.lib.skyframe.ASTFileLookupFunction;
import com.google.devtools.build.lib.skyframe.BlacklistedPackagePrefixesFunction;
import com.google.devtools.build.lib.skyframe.ContainingPackageLookupFunction;
@@ -232,7 +233,11 @@
public final PackageLoader build() {
validate();
externalFilesHelper =
- ExternalFilesHelper.create(pkgLocatorRef, externalFileAction, directories);
+ ExternalFilesHelper.create(
+ pkgLocatorRef,
+ externalFileAction,
+ directories,
+ ManagedDirectoriesKnowledge.NO_MANAGED_DIRECTORIES);
return buildImpl();
}
