Blaze actions: add support for in-memory unused inputs list (dependency pruning). RELNOTES: PiperOrigin-RevId: 268878533
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java index 8d932f4..921b136 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java
@@ -290,7 +290,9 @@ * if the subprocess execution returns normally, not in case of errors (non-zero exit, * setup/network failures, etc.). */ - protected void afterExecute(ActionExecutionContext actionExecutionContext) throws IOException {} + protected void afterExecute( + ActionExecutionContext actionExecutionContext, List<SpawnResult> spawnResults) + throws IOException {} @Override public final ActionContinuationOrResult beginExecution( @@ -720,6 +722,7 @@ commandLineLimits, isShellCommand, env, + configuration, configuration == null ? executionInfo : configuration.modifiedExecutionInfo(executionInfo, mnemonic), @@ -740,6 +743,7 @@ CommandLineLimits commandLineLimits, boolean isShellCommand, ActionEnvironment env, + @Nullable BuildConfiguration configuration, ImmutableMap<String, String> executionInfo, CharSequence progressMessage, RunfilesSupplier runfilesSupplier, @@ -1343,7 +1347,8 @@ if (resultConsumer != null) { resultConsumer.accept(Pair.of(actionExecutionContext, nextContinuation.get())); } - afterExecute(actionExecutionContext); + List<SpawnResult> spawnResults = nextContinuation.get(); + afterExecute(actionExecutionContext, spawnResults); return ActionContinuationOrResult.of(ActionResult.create(nextContinuation.get())); } return new SpawnActionContinuation(actionExecutionContext, nextContinuation);
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/StarlarkAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/StarlarkAction.java index 197b78a..a7801e1 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/actions/StarlarkAction.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/StarlarkAction.java
@@ -25,15 +25,21 @@ import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.CommandLines; import com.google.devtools.build.lib.actions.CommandLines.CommandLineLimits; +import com.google.devtools.build.lib.actions.ExecutionRequirements; import com.google.devtools.build.lib.actions.ResourceSet; import com.google.devtools.build.lib.actions.RunfilesSupplier; +import com.google.devtools.build.lib.actions.SpawnResult; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; +import javax.annotation.Nullable; /** A Starlark specific SpawnAction. */ public final class StarlarkAction extends SpawnAction { @@ -133,8 +139,25 @@ return allInputs; } + private InputStream getUnusedInputListInputStream( + ActionExecutionContext actionExecutionContext, List<SpawnResult> spawnResults) + throws IOException { + + // Check if the file is in-memory. + // Note: SpawnActionContext guarantees that the first list entry exists and corresponds to the + // executed spawn. + InputStream inputStream = spawnResults.get(0).getInMemoryOutput(unusedInputsList.get()); + if (inputStream != null) { + return inputStream; + } + // Fallback to reading from disk. + return actionExecutionContext.getPathResolver().toPath(unusedInputsList.get()).getInputStream(); + } + @Override - protected void afterExecute(ActionExecutionContext actionExecutionContext) throws IOException { + protected void afterExecute( + ActionExecutionContext actionExecutionContext, List<SpawnResult> spawnResults) + throws IOException { if (!unusedInputsList.isPresent()) { return; } @@ -145,11 +168,7 @@ try (BufferedReader br = new BufferedReader( new InputStreamReader( - actionExecutionContext - .getPathResolver() - .toPath(unusedInputsList.get()) - .getInputStream(), - UTF_8))) { + getUnusedInputListInputStream(actionExecutionContext, spawnResults), UTF_8))) { String line; while ((line = br.readLine()) != null) { line = line.trim(); @@ -172,6 +191,11 @@ return this; } + private static boolean getInMemoryUnusedInputsListFileFlag( + @Nullable BuildConfiguration configuration) { + return configuration == null ? false : configuration.inmemoryUnusedInputsList(); + } + /** Creates a SpawnAction. */ @Override protected SpawnAction createSpawnAction( @@ -185,10 +209,20 @@ CommandLineLimits commandLineLimits, boolean isShellCommand, ActionEnvironment env, + @Nullable BuildConfiguration configuration, ImmutableMap<String, String> executionInfo, CharSequence progressMessage, RunfilesSupplier runfilesSupplier, String mnemonic) { + if (unusedInputsList.isPresent() && getInMemoryUnusedInputsListFileFlag(configuration)) { + executionInfo = + ImmutableMap.<String, String>builderWithExpectedSize(executionInfo.size() + 1) + .putAll(executionInfo) + .put( + ExecutionRequirements.REMOTE_EXECUTION_INLINE_OUTPUTS, + unusedInputsList.get().getExecPathString()) + .build(); + } return new StarlarkAction( owner, tools,
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java index 767261d..44294e7 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
@@ -751,6 +751,10 @@ return options.actionListeners; } + public boolean inmemoryUnusedInputsList() { + return options.inmemoryUnusedInputsList; + } + /** * Returns whether FileWriteAction may transparently compress its contents in the analysis phase * to save memory. Semantics are not affected.
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java b/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java index 471e726..883eb9f 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java
@@ -753,6 +753,21 @@ help = "Whether to use graphless query and disable output ordering.") public boolean useGraphlessQuery; + @Option( + name = "experimental_inmemory_unused_inputs_list", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.BUILD_TIME_OPTIMIZATION, + effectTags = { + OptionEffectTag.LOADING_AND_ANALYSIS, + OptionEffectTag.EXECUTION, + OptionEffectTag.AFFECTS_OUTPUTS + }, + metadataTags = {OptionMetadataTag.EXPERIMENTAL}, + help = + "If enabled, the optional 'unused_inputs_list' file will be passed through in memory " + + "directly from the remote build nodes instead of being written to disk.") + public boolean inmemoryUnusedInputsList; + @Override public FragmentOptions getHost() { CoreOptions host = (CoreOptions) getDefault();
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/extra/ExtraAction.java b/src/main/java/com/google/devtools/build/lib/analysis/extra/ExtraAction.java index c4cfef3..87c338d 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/extra/ExtraAction.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/extra/ExtraAction.java
@@ -33,6 +33,7 @@ import com.google.devtools.build.lib.actions.CompositeRunfilesSupplier; import com.google.devtools.build.lib.actions.RunfilesSupplier; import com.google.devtools.build.lib.actions.SpawnActionContext; +import com.google.devtools.build.lib.actions.SpawnResult; import com.google.devtools.build.lib.analysis.actions.SpawnAction; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; @@ -40,6 +41,7 @@ import com.google.devtools.build.lib.vfs.FileSystemUtils; import java.io.IOException; import java.util.Collection; +import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -151,7 +153,9 @@ } @Override - protected void afterExecute(ActionExecutionContext actionExecutionContext) throws IOException { + protected void afterExecute( + ActionExecutionContext actionExecutionContext, List<SpawnResult> spawnResults) + throws IOException { // PHASE 3: create dummy output. // If the user didn't specify output, we need to create dummy output // to make blaze schedule this action.
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendAction.java index c4fc42d..9718be1 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendAction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendAction.java
@@ -30,6 +30,7 @@ import com.google.devtools.build.lib.actions.ResourceSet; import com.google.devtools.build.lib.actions.RunfilesSupplier; import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.util.Fingerprint; import com.google.devtools.build.lib.vfs.FileSystemUtils; @@ -232,6 +233,7 @@ CommandLineLimits commandLineLimits, boolean isShellCommand, ActionEnvironment env, + @Nullable BuildConfiguration configuration, ImmutableMap<String, String> executionInfo, CharSequence progressMessage, RunfilesSupplier runfilesSupplier,
diff --git a/src/main/java/com/google/devtools/build/lib/rules/genrule/GenRuleAction.java b/src/main/java/com/google/devtools/build/lib/rules/genrule/GenRuleAction.java index 6e69d32..6578947 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/genrule/GenRuleAction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/genrule/GenRuleAction.java
@@ -24,10 +24,12 @@ import com.google.devtools.build.lib.actions.CommandLines; import com.google.devtools.build.lib.actions.CommandLines.CommandLineLimits; import com.google.devtools.build.lib.actions.RunfilesSupplier; +import com.google.devtools.build.lib.actions.SpawnResult; import com.google.devtools.build.lib.analysis.actions.SpawnAction; import com.google.devtools.build.lib.skyframe.TrackSourceDirectoriesFlag; import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; import java.io.IOException; +import java.util.List; /** * A spawn action for genrules. Genrules are handled specially in that inputs and outputs are @@ -77,7 +79,8 @@ } @Override - protected void afterExecute(ActionExecutionContext actionExecutionContext) { + protected void afterExecute( + ActionExecutionContext actionExecutionContext, List<SpawnResult> spawnResults) { checkOutputsForDirectories(actionExecutionContext); } }
diff --git a/src/test/shell/integration/skylark_dependency_pruning_test.sh b/src/test/shell/integration/skylark_dependency_pruning_test.sh index 3251afe..1591704 100755 --- a/src/test/shell/integration/skylark_dependency_pruning_test.sh +++ b/src/test/shell/integration/skylark_dependency_pruning_test.sh
@@ -265,4 +265,23 @@ expect_log "Use --experimental_starlark_unused_inputs_list" } +# Test with orphaned artifacts features. +# This requires --experimental_inmemory_unused_inputs_list flag. +function test_orphaned_artifacts() { + options="--discard_orphaned_artifacts \ + --force_multigroup_accounting \ + --nokeep_state_after_build" + + # Use in-memory files. + bazel build ${options} --experimental_inmemory_unused_inputs_list \ + //pkg:output || fail "build failed" + + # This should fail. + bazel build ${options} --noexperimental_inmemory_unused_inputs_list \ + //pkg:output >& $TEST_log && fail "Expected failure" + exitcode=$? + assert_equals 1 "$exitcode" + expect_log "bin/pkg/output.unused (No such file or directory)" +} + run_suite "Tests Skylark dependency pruning"