Clean up FilesystemCalls: rename to SyscallCache, move to a top-level class, and get rid of AtomicReference wherever possible. Use a SyscallCache for any command-scoped classes, and Supplier<SyscallCache> for others. This change should have no runtime-observable effects (all underlying caches are the same).
Also clean up AtomicReference<TimestampGranularityMonitor> in the same way, since the parameters are passed together, and set it to non-null wherever possible.
PiperOrigin-RevId: 424687480
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionContext.java b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionContext.java
index c155dc9..9230394 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionContext.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionContext.java
@@ -31,7 +31,7 @@
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.Root;
-import com.google.devtools.build.lib.vfs.UnixGlob.FilesystemCalls;
+import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.common.options.OptionsProvider;
import java.io.Closeable;
@@ -76,7 +76,7 @@
private final ArtifactPathResolver pathResolver;
private final DiscoveredModulesPruner discoveredModulesPruner;
- private final FilesystemCalls syscalls;
+ private final SyscallCache syscallCache;
private final ThreadStateReceiver threadStateReceiverForMetrics;
private ActionExecutionContext(
@@ -96,7 +96,7 @@
@Nullable FileSystem actionFileSystem,
@Nullable Object skyframeDepsResult,
DiscoveredModulesPruner discoveredModulesPruner,
- FilesystemCalls syscalls,
+ SyscallCache syscallCache,
ThreadStateReceiver threadStateReceiverForMetrics) {
this.actionInputFileCache = actionInputFileCache;
this.actionInputPrefetcher = actionInputPrefetcher;
@@ -118,7 +118,7 @@
// executor is only ever null in testing.
executor == null ? null : executor.getExecRoot());
this.discoveredModulesPruner = discoveredModulesPruner;
- this.syscalls = syscalls;
+ this.syscallCache = syscallCache;
}
public ActionExecutionContext(
@@ -137,7 +137,7 @@
@Nullable FileSystem actionFileSystem,
@Nullable Object skyframeDepsResult,
DiscoveredModulesPruner discoveredModulesPruner,
- FilesystemCalls syscalls,
+ SyscallCache syscallCache,
ThreadStateReceiver threadStateReceiverForMetrics) {
this(
executor,
@@ -156,7 +156,7 @@
actionFileSystem,
skyframeDepsResult,
discoveredModulesPruner,
- syscalls,
+ syscallCache,
threadStateReceiverForMetrics);
}
@@ -174,7 +174,7 @@
Environment env,
@Nullable FileSystem actionFileSystem,
DiscoveredModulesPruner discoveredModulesPruner,
- FilesystemCalls syscalls,
+ SyscallCache syscalls,
ThreadStateReceiver threadStateReceiverForMetrics) {
return new ActionExecutionContext(
executor,
@@ -382,12 +382,12 @@
}
/**
- * This only exists for loose header checking (and shouldn't be exist at all).
+ * This only exists for loose header checking (and shouldn't exist at all).
*
* <p>Do NOT use from any other place.
*/
- public FilesystemCalls getSyscalls() {
- return syscalls;
+ public SyscallCache getSyscallCache() {
+ return syscallCache;
}
public ThreadStateReceiver getThreadStateReceiverForMetrics() {
@@ -428,7 +428,7 @@
actionFileSystem,
skyframeDepsResult,
discoveredModulesPruner,
- syscalls,
+ syscallCache,
threadStateReceiverForMetrics);
}
@@ -451,7 +451,7 @@
actionFileSystem,
skyframeDepsResult,
discoveredModulesPruner,
- syscalls,
+ syscallCache,
threadStateReceiverForMetrics);
}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/FileStateValue.java b/src/main/java/com/google/devtools/build/lib/actions/FileStateValue.java
index 8ce0bbc..c3289d5 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/FileStateValue.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/FileStateValue.java
@@ -34,7 +34,7 @@
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.lib.vfs.Symlinks;
-import com.google.devtools.build.lib.vfs.UnixGlob.FilesystemCalls;
+import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.skyframe.AbstractSkyKey;
import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.build.skyframe.SkyValue;
@@ -76,9 +76,7 @@
private FileStateValue() {}
public static FileStateValue create(
- RootedPath rootedPath,
- FilesystemCalls syscallCache,
- @Nullable TimestampGranularityMonitor tsgm)
+ RootedPath rootedPath, SyscallCache syscallCache, @Nullable TimestampGranularityMonitor tsgm)
throws IOException {
Path path = rootedPath.asPath();
Dirent.Type type = syscallCache.getType(path, Symlinks.NOFOLLOW);
diff --git a/src/main/java/com/google/devtools/build/lib/includescanning/IncludeParser.java b/src/main/java/com/google/devtools/build/lib/includescanning/IncludeParser.java
index ab119a4..fdf1621 100644
--- a/src/main/java/com/google/devtools/build/lib/includescanning/IncludeParser.java
+++ b/src/main/java/com/google/devtools/build/lib/includescanning/IncludeParser.java
@@ -52,8 +52,8 @@
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
+import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.lib.vfs.UnixGlob;
-import com.google.devtools.build.lib.vfs.UnixGlob.FilesystemCalls;
import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
@@ -67,7 +67,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
@@ -77,6 +76,9 @@
* Scans a source file and extracts the literal inclusions it specifies. Does not store results --
* repeated requests to the same file will result in repeated scans. Clients should implement a
* caching layer in order to avoid unnecessary disk access when requesting an already scanned file.
+ *
+ * <p>Both this class and the static inner class {@link Hints} have lifetime of a single build (or a
+ * single include scanning operation in the case of the {@link SwigIncludeParser}).
*/
@VisibleForTesting
class IncludeParser {
@@ -188,7 +190,8 @@
private final ImmutableList<Rule> rules;
private final ArtifactFactory artifactFactory;
- private final AtomicReference<FilesystemCalls> syscallCache = new AtomicReference<>();
+ private final SyscallCache syscallCache =
+ PerBuildSyscallCache.newBuilder().setInitialCapacity(HINTS_CACHE_CONCURRENCY).build();
private final LoadingCache<Artifact, ImmutableList<Artifact>> fileLevelHintsCache =
Caffeine.newBuilder()
@@ -203,7 +206,6 @@
Hints(HintsRules hintsRules, ArtifactFactory artifactFactory) {
this.artifactFactory = artifactFactory;
this.rules = hintsRules.rules;
- clearCachedLegacyHints();
}
static HintsRules getRules(Path hintsFile) throws IOException {
@@ -242,16 +244,6 @@
return new HintsRules(rules.build());
}
- /**
- * Clears legacy inclusions cache to maintain inter-build correctness, since filesystem changes
- * are not tracked by cache.
- */
- void clearCachedLegacyHints() {
- fileLevelHintsCache.invalidateAll();
- syscallCache.set(
- PerBuildSyscallCache.newBuilder().setInitialCapacity(HINTS_CACHE_CONCURRENCY).build());
- }
-
/** Returns the "file" type hinted inclusions for a given path, caching results by path. */
ImmutableList<Artifact> getFileLevelHintedInclusionsLegacy(Artifact path) {
if (!path.getExecPathString().startsWith(ALLOWED_PREFIX)) {
diff --git a/src/main/java/com/google/devtools/build/lib/includescanning/IncludeScannerSupplier.java b/src/main/java/com/google/devtools/build/lib/includescanning/IncludeScannerSupplier.java
index ee4bc79..c521146 100644
--- a/src/main/java/com/google/devtools/build/lib/includescanning/IncludeScannerSupplier.java
+++ b/src/main/java/com/google/devtools/build/lib/includescanning/IncludeScannerSupplier.java
@@ -131,10 +131,5 @@
Preconditions.checkState(includeParseCache.isEmpty(), includeParseCache);
Preconditions.checkState(scanners.asMap().isEmpty(), scanners);
this.includeParser = Preconditions.checkNotNull(includeParser);
- if (this.includeParser.getHints() != null) {
- // The Hints object lives across the lifetime of the Blaze server, but its cached hints may
- // be stale.
- this.includeParser.getHints().clearCachedLegacyHints();
- }
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/includescanning/IncludeScanningModule.java b/src/main/java/com/google/devtools/build/lib/includescanning/IncludeScanningModule.java
index 9bd17e3..4352d0c 100644
--- a/src/main/java/com/google/devtools/build/lib/includescanning/IncludeScanningModule.java
+++ b/src/main/java/com/google/devtools/build/lib/includescanning/IncludeScanningModule.java
@@ -254,7 +254,7 @@
new SpawnIncludeScanner(
env.getExecRoot(),
options.experimentalRemoteExtractionThreshold,
- env.getSkyframeExecutor().getSyscalls()));
+ env.getSyscallCache()));
this.spawnScannerSupplier = spawnScannerSupplier;
env.getEventBus().register(this);
}
diff --git a/src/main/java/com/google/devtools/build/lib/includescanning/SpawnIncludeScanner.java b/src/main/java/com/google/devtools/build/lib/includescanning/SpawnIncludeScanner.java
index 3aa231b..edd1261 100644
--- a/src/main/java/com/google/devtools/build/lib/includescanning/SpawnIncludeScanner.java
+++ b/src/main/java/com/google/devtools/build/lib/includescanning/SpawnIncludeScanner.java
@@ -51,17 +51,16 @@
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Symlinks;
-import com.google.devtools.build.lib.vfs.UnixGlob.FilesystemCalls;
+import com.google.devtools.build.lib.vfs.SyscallCache;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
import java.util.List;
-import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.Nullable;
/**
* C include scanner. Scans C/C++ source files using spawns to determine the bounding set of
- * transitively referenced include files.
+ * transitively referenced include files. Has lifetime of a single build.
*/
public class SpawnIncludeScanner {
/** The grep-includes tool is very lightweight, so don't use the default from AbstractAction. */
@@ -72,11 +71,11 @@
private OutputService outputService;
private boolean inMemoryOutput;
private final int remoteExtractionThreshold;
- private final AtomicReference<FilesystemCalls> syscallCache;
+ private final SyscallCache syscallCache;
/** Constructs a new SpawnIncludeScanner. */
public SpawnIncludeScanner(
- Path execRoot, int remoteExtractionThreshold, AtomicReference<FilesystemCalls> syscallCache) {
+ Path execRoot, int remoteExtractionThreshold, SyscallCache syscallCache) {
this.execRoot = execRoot;
this.remoteExtractionThreshold = remoteExtractionThreshold;
this.syscallCache = syscallCache;
@@ -126,7 +125,7 @@
if (remoteExtractionThreshold == 0 || (outputService != null && !file.isSourceArtifact())) {
return true;
}
- FileStatus status = syscallCache.get().statIfFound(file.getPath(), Symlinks.FOLLOW);
+ FileStatus status = syscallCache.statIfFound(file.getPath(), Symlinks.FOLLOW);
return status == null || status.getSize() > remoteExtractionThreshold;
}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/GlobCache.java b/src/main/java/com/google/devtools/build/lib/packages/GlobCache.java
index 11caecc..818eef4 100644
--- a/src/main/java/com/google/devtools/build/lib/packages/GlobCache.java
+++ b/src/main/java/com/google/devtools/build/lib/packages/GlobCache.java
@@ -27,6 +27,7 @@
import com.google.devtools.build.lib.util.Pair;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.lib.vfs.UnixGlob;
import com.google.devtools.build.lib.vfs.UnixGlobPathDiscriminator;
import java.io.IOException;
@@ -42,9 +43,8 @@
import java.util.concurrent.Executor;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-/** Caches the results of glob expansion for a package. */
+/** Caches the results of glob expansion for a package. Has lifetime of a single build. */
@ThreadSafety.ThreadCompatible
public class GlobCache {
/**
@@ -65,7 +65,7 @@
private final PackageIdentifier packageId;
/** System call caching layer. */
- private final AtomicReference<? extends UnixGlob.FilesystemCalls> syscalls;
+ private final SyscallCache syscallCache;
private final int maxDirectoriesToEagerlyVisit;
@@ -95,7 +95,7 @@
final PackageIdentifier packageId,
final ImmutableSet<PathFragment> ignoredGlobPrefixes,
final CachingPackageLocator locator,
- AtomicReference<? extends UnixGlob.FilesystemCalls> syscalls,
+ SyscallCache syscallCache,
Executor globExecutor,
int maxDirectoriesToEagerlyVisit,
ThreadStateReceiver threadStateReceiverForMetrics) {
@@ -110,7 +110,7 @@
command.run();
}
});
- this.syscalls = syscalls == null ? new AtomicReference<>(UnixGlob.DEFAULT_SYSCALLS) : syscalls;
+ this.syscallCache = syscallCache;
this.maxDirectoriesToEagerlyVisit = maxDirectoriesToEagerlyVisit;
Preconditions.checkNotNull(locator);
@@ -223,7 +223,7 @@
.addPattern(pattern)
.setPathDiscriminator(new GlobUnixPathDiscriminator(globberOperation))
.setExecutor(globExecutor)
- .setFilesystemCalls(syscalls)
+ .setFilesystemCalls(syscallCache)
.globAsync();
} catch (UnixGlob.BadPattern ex) {
throw new BadGlobException(ex.getMessage());
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java b/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java
index 9cd3424..e07a3ef 100644
--- a/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java
+++ b/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java
@@ -46,15 +46,15 @@
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.RootedPath;
-import com.google.devtools.build.lib.vfs.UnixGlob;
+import com.google.devtools.build.lib.vfs.SyscallCache;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.OptionalLong;
import java.util.Set;
import java.util.concurrent.ForkJoinPool;
-import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
+import java.util.function.Supplier;
import net.starlark.java.eval.Dict;
import net.starlark.java.eval.EvalException;
import net.starlark.java.eval.Module;
@@ -114,7 +114,7 @@
private final RuleFactory ruleFactory;
private final RuleClassProvider ruleClassProvider;
- private AtomicReference<? extends UnixGlob.FilesystemCalls> syscalls;
+ private Supplier<? extends SyscallCache> syscalls;
private ForkJoinPool executor;
@@ -211,7 +211,7 @@
}
/** Sets the syscalls cache used in globbing. */
- public void setSyscalls(AtomicReference<? extends UnixGlob.FilesystemCalls> syscalls) {
+ public void setSyscallCache(Supplier<? extends SyscallCache> syscalls) {
this.syscalls = Preconditions.checkNotNull(syscalls);
}
@@ -481,7 +481,7 @@
packageId,
ignoredGlobPrefixes,
locator,
- syscalls,
+ syscalls.get(),
executor,
maxDirectoriesToEagerlyVisitInGlobbing,
threadStateReceiverForMetrics));
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/PathPackageLocator.java b/src/main/java/com/google/devtools/build/lib/pkgcache/PathPackageLocator.java
index c5c4be5..e7fda08 100644
--- a/src/main/java/com/google/devtools/build/lib/pkgcache/PathPackageLocator.java
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/PathPackageLocator.java
@@ -32,12 +32,11 @@
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.lib.vfs.Symlinks;
-import com.google.devtools.build.lib.vfs.UnixGlob;
+import com.google.devtools.build.lib.vfs.SyscallCache;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
-import java.util.concurrent.atomic.AtomicReference;
/**
* A mapping from the name of a package to the location of its BUILD file. The implementation
@@ -77,7 +76,7 @@
* first path that matches always wins.
*/
public Path getPackageBuildFile(PackageIdentifier packageName) throws NoSuchPackageException {
- Path buildFile = getPackageBuildFileNullable(packageName, UnixGlob.DEFAULT_SYSCALLS_REF);
+ Path buildFile = getPackageBuildFileNullable(packageName, SyscallCache.NO_CACHE);
if (buildFile == null) {
String message = "BUILD file not found on package path";
throw new BuildFileNotFoundException(
@@ -102,9 +101,7 @@
* @param cache a filesystem-level cache of stat() calls.
* @return the {@link Path} to the correct build file, or {@code null} if none was found
*/
- public Path getPackageBuildFileNullable(
- PackageIdentifier packageIdentifier,
- AtomicReference<? extends UnixGlob.FilesystemCalls> cache) {
+ public Path getPackageBuildFileNullable(PackageIdentifier packageIdentifier, SyscallCache cache) {
if (packageIdentifier.getRepository().isMain()) {
for (BuildFileName buildFileName : buildFilesByPriority) {
Path buildFilePath =
@@ -133,7 +130,7 @@
.getRelative(packageIdentifier.getSourceRoot())
.getRelative(buildFileName.getFilenameFragment());
try {
- FileStatus stat = cache.get().statIfFound(buildFile, Symlinks.FOLLOW);
+ FileStatus stat = cache.statIfFound(buildFile, Symlinks.FOLLOW);
if (stat != null && stat.isFile()) {
return buildFile;
}
@@ -257,22 +254,21 @@
* wins.
*/
public Path getWorkspaceFile() {
- AtomicReference<? extends UnixGlob.FilesystemCalls> cache = UnixGlob.DEFAULT_SYSCALLS_REF;
// TODO(bazel-team): correctness in the presence of changes to the location of the WORKSPACE
// file.
- Path workspaceFile = getFilePath(LabelConstants.WORKSPACE_DOT_BAZEL_FILE_NAME, cache);
+ Path workspaceFile =
+ getFilePath(LabelConstants.WORKSPACE_DOT_BAZEL_FILE_NAME, SyscallCache.NO_CACHE);
if (workspaceFile != null) {
return workspaceFile;
}
- return getFilePath(LabelConstants.WORKSPACE_FILE_NAME, cache);
+ return getFilePath(LabelConstants.WORKSPACE_FILE_NAME, SyscallCache.NO_CACHE);
}
- private Path getFilePath(PathFragment suffix,
- AtomicReference<? extends UnixGlob.FilesystemCalls> cache) {
+ private Path getFilePath(PathFragment suffix, SyscallCache cache) {
for (Root pathEntry : pathEntries) {
Path buildFile = pathEntry.getRelative(suffix);
try {
- Dirent.Type type = cache.get().getType(buildFile, Symlinks.FOLLOW);
+ Dirent.Type type = cache.getType(buildFile, Symlinks.FOLLOW);
if (type == Dirent.Type.FILE || type == Dirent.Type.UNKNOWN) {
return buildFile;
}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java
index fc377ee..51d68ef 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java
@@ -1117,7 +1117,7 @@
// TODO(b/187366935): Consider globbing everything eagerly instead.
fileStatus =
actionExecutionContext
- .getSyscalls()
+ .getSyscallCache()
.statIfFound(dirOrPackage.getRelative(BUILD_PATH_FRAGMENT), Symlinks.FOLLOW);
} catch (IOException e) {
// Previously, we used Path.exists() to check whether the BUILD file exists. This did
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java
index 80020ca..7d2fce9 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java
@@ -51,6 +51,7 @@
import com.google.devtools.build.lib.vfs.OutputService;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.common.options.OptionsParsingResult;
import com.google.devtools.common.options.OptionsProvider;
import com.google.protobuf.Any;
@@ -98,6 +99,7 @@
private final PathPackageLocator packageLocator;
private final Path workingDirectory;
private final PathFragment relativeWorkingDirectory;
+ private final SyscallCache syscallCache;
private final Duration waitTime;
private final long commandStartTime;
private final ImmutableList<Any> commandExtensions;
@@ -218,6 +220,7 @@
} else {
this.packageLocator = null;
}
+ this.syscallCache = workspace.getSkyframeExecutor().getCurrentSyscallCache();
workspace.getSkyframeExecutor().setEventBus(eventBus);
ClientOptions clientOptions =
@@ -794,6 +797,10 @@
}
}
+ public SyscallCache getSyscallCache() {
+ return syscallCache;
+ }
+
/**
* Returns the {@linkplain
* com.google.devtools.build.lib.server.CommandProtos.RunRequest#getCommandExtensions extensions}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java
index 22c0c1f..f646fb8 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java
@@ -112,7 +112,6 @@
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.function.IntFunction;
import java.util.function.Predicate;
@@ -143,13 +142,13 @@
private final ActionRewindStrategy actionRewindStrategy = new ActionRewindStrategy();
private final SkyframeActionExecutor skyframeActionExecutor;
private final BlazeDirectories directories;
- private final AtomicReference<TimestampGranularityMonitor> tsgm;
+ private final Supplier<TimestampGranularityMonitor> tsgm;
private final BugReporter bugReporter;
public ActionExecutionFunction(
SkyframeActionExecutor skyframeActionExecutor,
BlazeDirectories directories,
- AtomicReference<TimestampGranularityMonitor> tsgm,
+ Supplier<TimestampGranularityMonitor> tsgm,
BugReporter bugReporter) {
this.skyframeActionExecutor = skyframeActionExecutor;
this.directories = directories;
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateFunction.java
index 8295cec..5df2ca8 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateFunction.java
@@ -15,12 +15,12 @@
import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.FileType;
import com.google.devtools.build.lib.vfs.RootedPath;
-import com.google.devtools.build.lib.vfs.UnixGlob.FilesystemCalls;
+import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyKey;
import java.io.IOException;
-import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
/**
* A {@link SkyFunction} for {@link DirectoryListingStateValue}s.
@@ -37,10 +37,10 @@
* re-use the results of expensive readdir() operations, that are likely already executed for
* evaluating globs.
*/
- private final AtomicReference<FilesystemCalls> syscallCache;
+ private final Supplier<SyscallCache> syscallCache;
public DirectoryListingStateFunction(
- ExternalFilesHelper externalFilesHelper, AtomicReference<FilesystemCalls> syscallCache) {
+ ExternalFilesHelper externalFilesHelper, Supplier<SyscallCache> syscallCache) {
this.externalFilesHelper = externalFilesHelper;
this.syscallCache = syscallCache;
}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileStateFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateFunction.java
index de7f6d4..41399fc 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/FileStateFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateFunction.java
@@ -18,12 +18,12 @@
import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.FileType;
import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
import com.google.devtools.build.lib.vfs.RootedPath;
-import com.google.devtools.build.lib.vfs.UnixGlob.FilesystemCalls;
+import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyKey;
import java.io.IOException;
-import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
/**
* A {@link SkyFunction} for {@link FileStateValue}s.
@@ -33,13 +33,13 @@
*/
public class FileStateFunction implements SkyFunction {
- private final AtomicReference<TimestampGranularityMonitor> tsgm;
- private final AtomicReference<FilesystemCalls> syscallCache;
+ private final Supplier<TimestampGranularityMonitor> tsgm;
+ private final Supplier<SyscallCache> syscallCache;
private final ExternalFilesHelper externalFilesHelper;
public FileStateFunction(
- AtomicReference<TimestampGranularityMonitor> tsgm,
- AtomicReference<FilesystemCalls> syscallCache,
+ Supplier<TimestampGranularityMonitor> tsgm,
+ Supplier<SyscallCache> syscallCache,
ExternalFilesHelper externalFilesHelper) {
this.tsgm = tsgm;
this.syscallCache = syscallCache;
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PerBuildSyscallCache.java b/src/main/java/com/google/devtools/build/lib/skyframe/PerBuildSyscallCache.java
index 00ea4b1..b9e9af8 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/PerBuildSyscallCache.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PerBuildSyscallCache.java
@@ -22,7 +22,7 @@
import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.Symlinks;
-import com.google.devtools.build.lib.vfs.UnixGlob;
+import com.google.devtools.build.lib.vfs.SyscallCache;
import java.io.IOException;
import java.util.Collection;
@@ -31,7 +31,7 @@
*
* <p>Mostly used by non-Skyframe globbing and include parsing.
*/
-public final class PerBuildSyscallCache implements UnixGlob.FilesystemCalls {
+public final class PerBuildSyscallCache implements SyscallCache {
private final LoadingCache<Pair<Path, Symlinks>, Object> statCache;
@@ -139,14 +139,14 @@
if (result == NO_STATUS) {
return null;
}
- return UnixGlob.statusToDirentType((FileStatus) result);
+ return SyscallCache.statusToDirentType((FileStatus) result);
}
}
// If this is a root directory, we must stat, there is no parent.
Path parent = path.getParentDirectory();
if (parent == null) {
- return UnixGlob.statusToDirentType(statIfFound(path, symlinks));
+ return SyscallCache.statusToDirentType(statIfFound(path, symlinks));
}
// Answer based on a cached readdir() call if possible. The cache might already be populated
@@ -171,14 +171,14 @@
}
if (dirent.getType() == Dirent.Type.SYMLINK && symlinks == Symlinks.FOLLOW) {
// See above: We don't want to follow symlinks with readdir(). Do a stat() instead.
- return UnixGlob.statusToDirentType(statIfFound(path, Symlinks.FOLLOW));
+ return SyscallCache.statusToDirentType(statIfFound(path, Symlinks.FOLLOW));
}
return dirent.getType();
}
return null;
}
- return UnixGlob.statusToDirentType(statIfFound(path, symlinks));
+ return SyscallCache.statusToDirentType(statIfFound(path, symlinks));
}
public void clear() {
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java
index dd63561..2683512 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java
@@ -13,7 +13,6 @@
// limitations under the License.
package com.google.devtools.build.lib.skyframe;
-import static com.google.devtools.build.lib.vfs.UnixGlob.DEFAULT_SYSCALLS;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.common.annotations.VisibleForTesting;
@@ -54,6 +53,7 @@
import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.lib.vfs.Symlinks;
+import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyKey;
@@ -339,7 +339,7 @@
// remove some of the filesystem operations we're doing here.
Path path = traversal.root.asRootedPath().asPath();
FileStateValue fileState =
- FileStateValue.create(traversal.root.asRootedPath(), DEFAULT_SYSCALLS, null);
+ FileStateValue.create(traversal.root.asRootedPath(), SyscallCache.NO_CACHE, null);
if (fileState.getType() == FileStateType.NONEXISTENT) {
throw new IOException("Missing file: " + path);
}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
index 32a2bcb..787a263 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
@@ -771,6 +771,7 @@
throws InterruptedException, AbruptExitException {
TimestampGranularityMonitor tsgm = this.tsgm.get();
Differencer.Diff diff;
+ syscallCacheRef.set(getPerBuildSyscallCache(/*globbingThreads=*/ 42));
if (modifiedFileSet.treatEverythingAsModified()) {
diff =
new FilesystemValueChecker(tsgm, /*lastExecutionTimeRange=*/ null, /*numThreads=*/ 200)
@@ -778,7 +779,6 @@
} else {
diff = getDiff(tsgm, modifiedFileSet, pathEntry, /* fsvcThreads= */ 200);
}
- syscalls.set(getPerBuildSyscallCache(/*globbingThreads=*/ 42));
recordingDiffer.invalidate(diff.changedKeysWithoutNewValues());
recordingDiffer.inject(diff.changedKeysWithNewValues());
// Blaze invalidates transient errors on every build.
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java
index a1edb6e..111b866 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java
@@ -101,7 +101,7 @@
import com.google.devtools.build.lib.vfs.OutputService.ActionFileSystemType;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.Root;
-import com.google.devtools.build.lib.vfs.UnixGlob.FilesystemCalls;
+import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.common.options.OptionsProvider;
@@ -146,7 +146,7 @@
private final ActionKeyContext actionKeyContext;
private final MetadataConsumerForMetrics outputArtifactsSeen;
private final MetadataConsumerForMetrics outputArtifactsFromActionCache;
- private final AtomicReference<FilesystemCalls> syscalls;
+ private final Supplier<SyscallCache> syscallCache;
private final Function<SkyKey, ThreadStateReceiver> threadStateReceiverFactory;
private Reporter reporter;
private Map<String, String> clientEnv = ImmutableMap.of();
@@ -218,14 +218,14 @@
MetadataConsumerForMetrics outputArtifactsFromActionCache,
AtomicReference<ActionExecutionStatusReporter> statusReporterRef,
Supplier<ImmutableList<Root>> sourceRootSupplier,
- AtomicReference<FilesystemCalls> syscalls,
+ Supplier<SyscallCache> syscallCache,
Function<SkyKey, ThreadStateReceiver> threadStateReceiverFactory) {
this.actionKeyContext = actionKeyContext;
this.outputArtifactsSeen = outputArtifactsSeen;
this.outputArtifactsFromActionCache = outputArtifactsFromActionCache;
this.statusReporterRef = statusReporterRef;
this.sourceRootSupplier = sourceRootSupplier;
- this.syscalls = syscalls;
+ this.syscallCache = syscallCache;
this.threadStateReceiverFactory = threadStateReceiverFactory;
}
@@ -546,7 +546,7 @@
actionFileSystem,
skyframeDepsResult,
discoveredModulesPruner,
- syscalls.get(),
+ syscallCache.get(),
threadStateReceiverFactory.apply(actionLookupData));
}
@@ -767,7 +767,7 @@
env,
actionFileSystem,
discoveredModulesPruner,
- syscalls.get(),
+ syscallCache.get(),
threadStateReceiverFactory.apply(actionLookupData));
if (actionFileSystem != null) {
updateActionFileSystemContext(
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
index 705ae37..ec1a6f8 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
@@ -184,7 +184,7 @@
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.lib.vfs.RootedPath;
-import com.google.devtools.build.lib.vfs.UnixGlob;
+import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.skyframe.CycleInfo;
import com.google.devtools.build.skyframe.CyclesReporter;
import com.google.devtools.build.skyframe.Differencer;
@@ -294,8 +294,8 @@
// AtomicReferences are used here as mutable boxes shared with value builders.
private final AtomicBoolean showLoadingProgress = new AtomicBoolean();
- protected final AtomicReference<UnixGlob.FilesystemCalls> syscalls =
- new AtomicReference<>(UnixGlob.DEFAULT_SYSCALLS);
+ protected final AtomicReference<SyscallCache> syscallCacheRef =
+ new AtomicReference<>(SyscallCache.NO_CACHE);
protected final AtomicReference<PathPackageLocator> pkgLocator = new AtomicReference<>();
protected final AtomicReference<ImmutableSet<PackageIdentifier>> deletedPackages =
new AtomicReference<>(ImmutableSet.of());
@@ -422,15 +422,15 @@
this.shouldUnblockCpuWorkWhenFetchingDeps = shouldUnblockCpuWorkWhenFetchingDeps;
this.skyKeyStateReceiver = skyKeyStateReceiver;
this.bugReporter = bugReporter;
- this.pkgFactory.setSyscalls(syscalls);
+ this.pkgFactory.setSyscallCache(syscallCacheRef::get);
this.workspaceStatusActionFactory = workspaceStatusActionFactory;
this.packageManager =
new SkyframePackageManager(
new SkyframePackageLoader(),
new QueryTransitivePackagePreloader(
() -> memoizingEvaluator, this::newEvaluationContextBuilder),
- syscalls,
- pkgLocator,
+ syscallCacheRef::get,
+ pkgLocator::get,
numPackagesLoaded);
this.fileSystem = fileSystem;
this.directories = Preconditions.checkNotNull(directories);
@@ -446,7 +446,7 @@
outputArtifactsFromActionCache,
statusReporterRef,
this::getPathEntries,
- syscalls,
+ syscallCacheRef::get,
skyKeyStateReceiver::makeThreadStateReceiver);
this.artifactFactory =
new ArtifactFactory(
@@ -606,7 +606,7 @@
map.put(SkyFunctions.BUILD_INFO, new WorkspaceStatusFunction(this::makeWorkspaceStatusAction));
map.put(SkyFunctions.COVERAGE_REPORT, new CoverageReportFunction(actionKeyContext));
ActionExecutionFunction actionExecutionFunction =
- new ActionExecutionFunction(skyframeActionExecutor, directories, tsgm, bugReporter);
+ new ActionExecutionFunction(skyframeActionExecutor, directories, tsgm::get, bugReporter);
map.put(SkyFunctions.ACTION_EXECUTION, actionExecutionFunction);
this.actionExecutionFunction = actionExecutionFunction;
map.put(
@@ -651,11 +651,11 @@
}
protected SkyFunction newFileStateFunction() {
- return new FileStateFunction(tsgm, syscalls, externalFilesHelper);
+ return new FileStateFunction(tsgm::get, syscallCacheRef::get, externalFilesHelper);
}
protected SkyFunction newDirectoryListingStateFunction() {
- return new DirectoryListingStateFunction(externalFilesHelper, syscalls);
+ return new DirectoryListingStateFunction(externalFilesHelper, syscallCacheRef::get);
}
protected GlobFunction newGlobFunction() {
@@ -700,8 +700,8 @@
return perBuildSyscallCache;
}
- public AtomicReference<UnixGlob.FilesystemCalls> getSyscalls() {
- return syscalls;
+ public final SyscallCache getCurrentSyscallCache() {
+ return syscallCacheRef.get();
}
@ThreadCompatible
@@ -1364,7 +1364,7 @@
starlarkSemantics.getBool(BuildLanguageOptions.EXPERIMENTAL_SIBLING_REPOSITORY_LAYOUT));
setPackageLocator(pkgLocator);
- syscalls.set(getPerBuildSyscallCache(packageOptions.globbingThreads));
+ syscallCacheRef.set(getPerBuildSyscallCache(packageOptions.globbingThreads));
this.pkgFactory.setGlobbingThreads(packageOptions.globbingThreads);
this.pkgFactory.setMaxDirectoriesToEagerlyVisitInGlobbing(
packageOptions.maxDirectoriesToEagerlyVisitInGlobbing);
@@ -2245,7 +2245,7 @@
@VisibleForTesting
public final void turnOffSyscallCacheForTesting() {
- syscalls.set(UnixGlob.DEFAULT_SYSCALLS);
+ syscallCacheRef.set(SyscallCache.NO_CACHE);
}
protected abstract void invalidateFilesUnderPathForTestingImpl(
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageManager.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageManager.java
index ce43277..2b78b67 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageManager.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageManager.java
@@ -32,28 +32,28 @@
import com.google.devtools.build.lib.pkgcache.QueryTransitivePackagePreloader;
import com.google.devtools.build.lib.skyframe.SkyframeExecutor.SkyframePackageLoader;
import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.UnixGlob;
+import com.google.devtools.build.lib.vfs.SyscallCache;
import java.io.PrintStream;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
class SkyframePackageManager implements PackageManager, CachingPackageLocator {
private final SkyframePackageLoader packageLoader;
private final QueryTransitivePackagePreloader transitiveLoader;
- private final AtomicReference<UnixGlob.FilesystemCalls> syscalls;
- private final AtomicReference<PathPackageLocator> pkgLocator;
+ private final Supplier<SyscallCache> syscallCache;
+ private final Supplier<PathPackageLocator> pkgLocator;
private final AtomicInteger numPackagesLoaded;
public SkyframePackageManager(
SkyframePackageLoader packageLoader,
QueryTransitivePackagePreloader transitiveLoader,
- AtomicReference<UnixGlob.FilesystemCalls> syscalls,
- AtomicReference<PathPackageLocator> pkgLocator,
+ Supplier<SyscallCache> syscallCache,
+ Supplier<PathPackageLocator> pkgLocator,
AtomicInteger numPackagesLoaded) {
this.packageLoader = packageLoader;
this.transitiveLoader = transitiveLoader;
this.pkgLocator = pkgLocator;
- this.syscalls = syscalls;
+ this.syscallCache = syscallCache;
this.numPackagesLoaded = numPackagesLoaded;
}
@@ -99,7 +99,7 @@
// TODO(bazel-team): Use a PackageLookupValue here [skyframe-loading]
// TODO(bazel-team): The implementation in PackageCache also checks for duplicate packages, see
// BuildFileCache#getBuildFile [skyframe-loading]
- return pkgLocator.get().getPackageBuildFileNullable(packageName, syscalls);
+ return pkgLocator.get().getPackageBuildFileNullable(packageName, syscallCache.get());
}
@Override
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java b/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java
index 54f5c51..1f13bc1 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java
@@ -86,7 +86,6 @@
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
-import com.google.devtools.build.lib.vfs.UnixGlob.FilesystemCalls;
import com.google.devtools.build.skyframe.Differencer;
import com.google.devtools.build.skyframe.ErrorInfo;
import com.google.devtools.build.skyframe.EvaluationContext;
@@ -435,14 +434,10 @@
protected abstract ActionOnIOExceptionReadingBuildFile getActionOnIOExceptionReadingBuildFile();
private ImmutableMap<SkyFunctionName, SkyFunction> makeFreshSkyFunctions() {
- AtomicReference<TimestampGranularityMonitor> tsgm =
- new AtomicReference<>(new TimestampGranularityMonitor(BlazeClock.instance()));
- AtomicReference<FilesystemCalls> syscallCacheRef =
- new AtomicReference<>(
- PerBuildSyscallCache.newBuilder()
- .setInitialCapacity(nonSkyframeGlobbingThreads)
- .build());
- pkgFactory.setSyscalls(syscallCacheRef);
+ TimestampGranularityMonitor tsgm = new TimestampGranularityMonitor(BlazeClock.instance());
+ PerBuildSyscallCache syscallCache =
+ PerBuildSyscallCache.newBuilder().setInitialCapacity(nonSkyframeGlobbingThreads).build();
+ pkgFactory.setSyscallCache(() -> syscallCache);
pkgFactory.setMaxDirectoriesToEagerlyVisitInGlobbing(
MAX_DIRECTORIES_TO_EAGERLY_VISIT_IN_GLOBBING);
CachingPackageLocator cachingPackageLocator =
@@ -450,7 +445,7 @@
@Override
@Nullable
public Path getBuildFileForPackage(PackageIdentifier packageName) {
- return pkgLocatorRef.get().getPackageBuildFileNullable(packageName, syscallCacheRef);
+ return pkgLocatorRef.get().getPackageBuildFileNullable(packageName, syscallCache);
}
@Override
@@ -464,7 +459,7 @@
.put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction())
.put(
FileStateValue.FILE_STATE,
- new FileStateFunction(tsgm, syscallCacheRef, externalFilesHelper))
+ new FileStateFunction(() -> tsgm, () -> syscallCache, externalFilesHelper))
.put(FileSymlinkCycleUniquenessFunction.NAME, new FileSymlinkCycleUniquenessFunction())
.put(
FileSymlinkInfiniteExpansionUniquenessFunction.NAME,
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java b/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java
index d3e6b2a..bacc17b 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java
@@ -41,7 +41,7 @@
import com.google.devtools.build.lib.skyframe.SkyFunctions;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.Root;
-import com.google.devtools.build.lib.vfs.UnixGlob;
+import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionName;
import java.util.Optional;
@@ -117,7 +117,7 @@
.put(
SkyFunctions.DIRECTORY_LISTING_STATE,
new DirectoryListingStateFunction(
- externalFilesHelper, new AtomicReference<>(UnixGlob.DEFAULT_SYSCALLS)))
+ externalFilesHelper, () -> SyscallCache.NO_CACHE))
.put(SkyFunctions.ACTION_ENVIRONMENT_VARIABLE, new ActionEnvironmentFunction())
.put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction())
.put(
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/SyscallCache.java b/src/main/java/com/google/devtools/build/lib/vfs/SyscallCache.java
new file mode 100644
index 0000000..752197a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/SyscallCache.java
@@ -0,0 +1,68 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.vfs;
+
+import java.io.IOException;
+import java.util.Collection;
+
+/**
+ * Centralized point to perform filesystem calls, to promote caching. Ideally all filesystem
+ * operations would be cached in Skyframe, but even then, implementations of this interface may do
+ * batch operations and prefetching to improve performance.
+ */
+public interface SyscallCache {
+ SyscallCache NO_CACHE =
+ new SyscallCache() {
+ @Override
+ public Collection<Dirent> readdir(Path path) throws IOException {
+ return path.readdir(Symlinks.NOFOLLOW);
+ }
+
+ @Override
+ public FileStatus statIfFound(Path path, Symlinks symlinks) throws IOException {
+ return path.statIfFound(symlinks);
+ }
+
+ @Override
+ public Dirent.Type getType(Path path, Symlinks symlinks) throws IOException {
+ return statusToDirentType(statIfFound(path, symlinks));
+ }
+ };
+
+ /** Gets directory entries and their types. Does not follow symlinks. */
+ Collection<Dirent> readdir(Path path) throws IOException;
+
+ /** Returns the stat() for the given path, or null. */
+ FileStatus statIfFound(Path path, Symlinks symlinks) throws IOException;
+
+ /**
+ * Returns the type of a specific file. This may be answered using stat() or readdir(). Returns
+ * null if the path does not exist.
+ */
+ Dirent.Type getType(Path path, Symlinks symlinks) throws IOException;
+
+ static Dirent.Type statusToDirentType(FileStatus status) {
+ if (status == null) {
+ return null;
+ } else if (status.isFile()) {
+ return Dirent.Type.FILE;
+ } else if (status.isDirectory()) {
+ return Dirent.Type.DIRECTORY;
+ } else if (status.isSymbolicLink()) {
+ return Dirent.Type.SYMLINK;
+ }
+ return Dirent.Type.UNKNOWN;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/UnixGlob.java b/src/main/java/com/google/devtools/build/lib/vfs/UnixGlob.java
index 92f1c33..bbbf535 100644
--- a/src/main/java/com/google/devtools/build/lib/vfs/UnixGlob.java
+++ b/src/main/java/com/google/devtools/build/lib/vfs/UnixGlob.java
@@ -14,6 +14,8 @@
package com.google.devtools.build.lib.vfs;
+import static com.google.common.base.Preconditions.checkNotNull;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
@@ -74,7 +76,7 @@
Path base,
Collection<String> patterns,
UnixGlobPathDiscriminator pathDiscriminator,
- FilesystemCalls syscalls,
+ SyscallCache syscalls,
Executor executor)
throws IOException, InterruptedException, BadPattern {
GlobVisitor visitor = new GlobVisitor(executor);
@@ -85,7 +87,7 @@
Path base,
Collection<String> patterns,
UnixGlobPathDiscriminator pathDiscriminator,
- FilesystemCalls syscalls,
+ SyscallCache syscalls,
Executor executor)
throws IOException, BadPattern {
GlobVisitor visitor = new GlobVisitor(executor);
@@ -96,7 +98,7 @@
Path base,
Collection<String> patterns,
UnixGlobPathDiscriminator pathDiscriminator,
- FilesystemCalls syscalls,
+ SyscallCache syscalls,
Executor executor)
throws IOException, InterruptedException, BadPattern {
GlobVisitor visitor = new GlobVisitor(executor);
@@ -108,7 +110,7 @@
Path base,
Collection<String> patterns,
UnixGlobPathDiscriminator pathDiscriminator,
- FilesystemCalls syscalls,
+ SyscallCache syscalls,
Executor executor)
throws BadPattern {
Preconditions.checkNotNull(executor, "%s %s", base, patterns);
@@ -278,57 +280,6 @@
return Pattern.compile(regexp.toString());
}
- /**
- * Filesystem calls required for glob().
- */
- public interface FilesystemCalls {
- /** Get directory entries and their types. Does not follow symlinks. */
- Collection<Dirent> readdir(Path path) throws IOException;
-
- /** Return the stat() for the given path, or null. */
- FileStatus statIfFound(Path path, Symlinks symlinks) throws IOException;
-
- /**
- * Return the type of a specific file. This may be answered using stat() or readdir(). Returns
- * null if the path does not exist.
- */
- Dirent.Type getType(Path path, Symlinks symlinks) throws IOException;
- }
-
- public static final FilesystemCalls DEFAULT_SYSCALLS =
- new FilesystemCalls() {
- @Override
- public Collection<Dirent> readdir(Path path) throws IOException {
- return path.readdir(Symlinks.NOFOLLOW);
- }
-
- @Override
- public FileStatus statIfFound(Path path, Symlinks symlinks) throws IOException {
- return path.statIfFound(symlinks);
- }
-
- @Override
- public Dirent.Type getType(Path path, Symlinks symlinks) throws IOException {
- return statusToDirentType(statIfFound(path, symlinks));
- }
- };
-
- public static Dirent.Type statusToDirentType(FileStatus status) {
- if (status == null) {
- return null;
- } else if (status.isFile()) {
- return Dirent.Type.FILE;
- } else if (status.isDirectory()) {
- return Dirent.Type.DIRECTORY;
- } else if (status.isSymbolicLink()) {
- return Dirent.Type.SYMLINK;
- }
- return Dirent.Type.UNKNOWN;
- }
-
- public static final AtomicReference<FilesystemCalls> DEFAULT_SYSCALLS_REF =
- new AtomicReference<>(DEFAULT_SYSCALLS);
-
public static Builder forPath(Path path) {
return new Builder(path);
}
@@ -339,12 +290,11 @@
*
*/
public static class Builder {
- private Path base;
- private List<String> patterns;
+ private final Path base;
+ private final List<String> patterns;
private UnixGlobPathDiscriminator pathDiscriminator = DEFAULT_DISCRIMINATOR;
private Executor executor;
- private AtomicReference<? extends FilesystemCalls> syscalls =
- new AtomicReference<>(DEFAULT_SYSCALLS);
+ private SyscallCache syscallCache = SyscallCache.NO_CACHE;
/**
* Creates a glob builder with the given base path.
@@ -384,13 +334,9 @@
return this;
}
- /**
- * Sets the FilesystemCalls interface to use on this glob().
- */
- public Builder setFilesystemCalls(AtomicReference<? extends FilesystemCalls> syscalls) {
- this.syscalls = (syscalls == null)
- ? new AtomicReference<FilesystemCalls>(DEFAULT_SYSCALLS)
- : syscalls;
+ /** Sets the SyscallCache interface to use on this glob(). */
+ public Builder setFilesystemCalls(SyscallCache syscallCache) {
+ this.syscallCache = checkNotNull(syscallCache);
return this;
}
@@ -422,8 +368,7 @@
/** Executes the glob. */
public List<Path> glob() throws IOException, BadPattern {
- return globInternalUninterruptible(
- base, patterns, pathDiscriminator, syscalls.get(), executor);
+ return globInternalUninterruptible(base, patterns, pathDiscriminator, syscallCache, executor);
}
/**
@@ -432,14 +377,14 @@
* @throws InterruptedException if the thread is interrupted.
*/
public List<Path> globInterruptible() throws IOException, InterruptedException, BadPattern {
- return globInternal(base, patterns, pathDiscriminator, syscalls.get(), executor);
+ return globInternal(base, patterns, pathDiscriminator, syscallCache, executor);
}
@VisibleForTesting
public long globInterruptibleAndReturnNumGlobTasksForTesting()
throws IOException, InterruptedException, BadPattern {
return globInternalAndReturnNumGlobTasksForTesting(
- base, patterns, pathDiscriminator, syscalls.get(), executor);
+ base, patterns, pathDiscriminator, syscallCache, executor);
}
/**
@@ -447,7 +392,7 @@
* non-null argument.
*/
public Future<List<Path>> globAsync() throws BadPattern {
- return globAsyncInternal(base, patterns, pathDiscriminator, syscalls.get(), executor);
+ return globAsyncInternal(base, patterns, pathDiscriminator, syscallCache, executor);
}
}
@@ -531,7 +476,7 @@
Path base,
Collection<String> patterns,
UnixGlobPathDiscriminator pathDiscriminator,
- FilesystemCalls syscalls)
+ SyscallCache syscalls)
throws IOException, InterruptedException, BadPattern {
try {
return globAsync(base, patterns, pathDiscriminator, syscalls).get();
@@ -546,7 +491,7 @@
Path base,
Collection<String> patterns,
UnixGlobPathDiscriminator pathDiscriminator,
- FilesystemCalls syscalls)
+ SyscallCache syscalls)
throws IOException, BadPattern {
try {
return Uninterruptibles.getUninterruptibly(
@@ -571,7 +516,7 @@
Path base,
Collection<String> patterns,
UnixGlobPathDiscriminator pathDiscriminator,
- FilesystemCalls syscalls)
+ SyscallCache syscalls)
throws BadPattern {
FileStatus baseStat;
try {
@@ -710,12 +655,12 @@
private class GlobTaskContext {
private final String[] patternParts;
private final UnixGlobPathDiscriminator pathDiscriminator;
- private final FilesystemCalls syscalls;
+ private final SyscallCache syscalls;
GlobTaskContext(
String[] patternParts,
UnixGlobPathDiscriminator pathDiscriminator,
- FilesystemCalls syscalls) {
+ SyscallCache syscalls) {
this.patternParts = patternParts;
this.pathDiscriminator = pathDiscriminator;
this.syscalls = syscalls;
@@ -766,7 +711,7 @@
private RecursiveGlobTaskContext(
String[] patternParts,
UnixGlobPathDiscriminator pathDiscriminator,
- FilesystemCalls syscalls) {
+ SyscallCache syscalls) {
super(patternParts, pathDiscriminator, syscalls);
}
