Add drop state nodes flag and apply it to `SkyframeProgressReceiver#evaluated()` method.
Due to the fact that the vast majority of the `_STATE` nodes only have one rdeps, we decided to drop them after the evaluation of `FILE` or `DIRECTORY_LISTING` nodes. So in this change:
* A new `--experimental_drop_state_nodes` is added in `CommonCommandOptions` in order to control whether `_STATE` nodes should be dropped or not;
* Inject the flag to `SequencedSkyframeExecutor` and `SkybuildV2SkyframeExecutor`;
* `SkyframeProgressReceiver#evaluated(...)` method decides whether to drop `_STATE` nodes based on the flag value and the type of input `SkyKey`.
PiperOrigin-RevId: 495949249
Change-Id: I7b4b84f7d91e8c3a7ce44f5ccb1af62ed134ebeb
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java
index 432f787..60e9b66 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java
@@ -758,7 +758,9 @@
AnalysisOptions viewOptions = options.getOptions(AnalysisOptions.class);
skyframeExecutor.decideKeepIncrementalState(
runtime.getStartupOptionsProvider().getOptions(BlazeServerStartupOptions.class).batch,
- commonOptions.keepStateAfterBuild, commonOptions.trackIncrementalState,
+ commonOptions.keepStateAfterBuild,
+ commonOptions.trackIncrementalState,
+ commonOptions.experimentalHeuristicallyDropNodes,
viewOptions != null && viewOptions.discardAnalysisCache,
reporter);
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommonCommandOptions.java b/src/main/java/com/google/devtools/build/lib/runtime/CommonCommandOptions.java
index a4b79f6..ef71cc9 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/CommonCommandOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/CommonCommandOptions.java
@@ -535,6 +535,18 @@
+ " invalidating the action graph.")
public List<Map.Entry<String, String>> repositoryEnvironment;
+ @Option(
+ name = "experimental_heuristically_drop_nodes",
+ defaultValue = "false",
+ documentationCategory = OptionDocumentationCategory.BUILD_TIME_OPTIMIZATION,
+ effectTags = {OptionEffectTag.LOSES_INCREMENTAL_STATE},
+ help =
+ "If true, Blaze will remove FileState and DirectoryListingState nodes after related File"
+ + " and DirectoryListing node is done to save memory. We expect that it is less"
+ + " likely that these nodes will be needed again. If so, the program will re-evaluate"
+ + " them.")
+ public boolean experimentalHeuristicallyDropNodes;
+
/** The option converter to check that the user can only specify legal profiler tasks. */
public static class ProfilerTaskConverter extends EnumConverter<ProfilerTask> {
public ProfilerTaskConverter() {
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
index 53bc922..7e6bffb 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
@@ -667,6 +667,7 @@
boolean batch,
boolean keepStateAfterBuild,
boolean shouldTrackIncrementalState,
+ boolean heuristicallyDropNodes,
boolean discardAnalysisCache,
EventHandler eventHandler) {
Preconditions.checkState(!active);
@@ -696,6 +697,19 @@
}
}
+ if (trackIncrementalState) {
+ if (heuristicallyDropNodes) {
+ eventHandler.handle(
+ Event.warn(
+ "--experimental_heuristically_drop_nodes was specified with track incremental state"
+ + " also being true. The flag is ignored and no node is heuristically dropped"
+ + " in the track incremental mode."));
+ }
+ this.heuristicallyDropNodes = false;
+ } else {
+ this.heuristicallyDropNodes = heuristicallyDropNodes;
+ }
+
// Now check if it is necessary to wipe the previous state. We do this if either the previous
// or current incrementalStateRetentionStrategy requires the build to have been isolated.
if (oldValueOfTrackIncrementalState != trackIncrementalState) {
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
index a584b9e..5e38075 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
@@ -401,6 +401,11 @@
// Reset while preparing for execution in each build.
private Optional<IncrementalPackageRoots> incrementalPackageRoots = Optional.empty();
+ // This boolean controls whether FILE_STATE or DIRECTORY_LISTING_STATE nodes are dropped after the
+ // corresponding FILE or DIRECTORY_LISTING nodes are evaluated.
+ // See b/261019506.
+ protected boolean heuristicallyDropNodes = false;
+
class PathResolverFactoryImpl implements PathResolverFactory {
@Override
public boolean shouldCreatePathResolverForArtifactValues() {
@@ -948,6 +953,7 @@
boolean batch,
boolean keepStateAfterBuild,
boolean trackIncrementalState,
+ boolean heuristicallyDropNodes,
boolean discardAnalysisCache,
EventHandler eventHandler) {
// Assume incrementality.
@@ -3100,6 +3106,26 @@
@Nullable ErrorInfo newError,
Supplier<EvaluationSuccessState> evaluationSuccessState,
EvaluationState state) {
+ if (heuristicallyDropNodes) {
+ Object argument = skyKey.argument();
+ if (skyKey.functionName().equals(FileValue.FILE)) {
+ Preconditions.checkArgument(
+ argument instanceof RootedPath,
+ "FILE SkyKey (%s) does not have a RootedPath typed argument (%s)",
+ skyKey,
+ argument);
+ memoizingEvaluator.getAllValuesMutable().remove(argument);
+ } else if (skyKey.functionName().equals(SkyFunctions.DIRECTORY_LISTING)) {
+ Preconditions.checkArgument(
+ argument instanceof RootedPath,
+ "DIRECTORY_LISTING SkyKey (%s) does not have a RootedPath typed argument (%s)",
+ skyKey,
+ argument);
+ SkyKey directoryListingStateKey = DirectoryListingStateValue.key((RootedPath) argument);
+ memoizingEvaluator.getAllValuesMutable().remove(directoryListingStateKey);
+ }
+ }
+
if (EvaluationState.BUILT.equals(state)) {
skyKeyStateReceiver.evaluated(skyKey);
}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutorTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutorTest.java
index 5f4bedc..fa7e702 100644
--- a/src/test/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutorTest.java
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutorTest.java
@@ -1590,10 +1590,11 @@
initializeSkyframeExecutor();
skyframeExecutor.setActive(false);
skyframeExecutor.decideKeepIncrementalState(
- /*batch=*/ false,
- /*keepStateAfterBuild=*/ true,
- /*shouldTrackIncrementalState=*/ false,
- /*discardAnalysisCache=*/ false,
+ /* batch= */ false,
+ /* keepStateAfterBuild= */ true,
+ /* shouldTrackIncrementalState= */ false,
+ /* heuristicallyDropNodes= */ false,
+ /* discardAnalysisCache= */ false,
reporter);
skyframeExecutor.setActive(true);
syncSkyframeExecutor();
@@ -2140,10 +2141,11 @@
options.parse("--keep_going", "--jobs=1", "--discard_analysis_cache");
skyframeExecutor.setActive(false);
skyframeExecutor.decideKeepIncrementalState(
- /*batch=*/ true,
- /*keepStateAfterBuild=*/ true,
- /*shouldTrackIncrementalState=*/ true,
- /*discardAnalysisCache=*/ true,
+ /* batch= */ true,
+ /* keepStateAfterBuild= */ true,
+ /* shouldTrackIncrementalState= */ true,
+ /* heuristicallyDropNodes= */ false,
+ /* discardAnalysisCache= */ true,
reporter);
skyframeExecutor.setActive(true);
runCatastropheHaltsBuild();
@@ -2538,10 +2540,11 @@
skyframeExecutor.setActive(false);
skyframeExecutor.decideKeepIncrementalState(
- /*batch=*/ false,
- /*keepStateAfterBuild=*/ true,
+ /* batch= */ false,
+ /* keepStateAfterBuild= */ true,
trackIncrementalState,
- /*discardAnalysisCache=*/ false,
+ /* heuristicallyDropNodes= */ false,
+ /* discardAnalysisCache= */ false,
reporter);
skyframeExecutor.setActive(true);
syncSkyframeExecutor();
