Expand param files in the execution log with --expand_param_files (#31808)
### Description
`--expand_param_files` now also applies to the arguments recorded by `--execution_log_{compact,binary,json}_file`. Spawns are still executed with param files and the execution log still records the param files as inputs, so input digests are unaffected.
### Motivation
This makes it possible to analyze full command lines from the execution log, e.g. to estimate the effect of path mapping across configurations.
### Build API Changes
No
### Checklist
- [x] I have added tests for the new use cases (if any).
- [ ] I have updated the documentation (if applicable).
### Release Notes
RELNOTES: `--expand_param_files` now also expands the arguments persisted in the execution log.
Closes #31808
COPYBARA_INTEGRATE_REVIEW=https://github.com/bazelbuild/bazel/pull/31808 from fmeum:execution-log-expand-param-files d1bd91cae5ff7eb54c451f5a274b35befaaf3208
PiperOrigin-RevId: 1009134630
Change-Id: I3c577693fa2c554171f71ed22e63f9a3b4e16603
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java b/src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java
index 0abd857..25c73f5 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java
@@ -202,7 +202,8 @@
compressionService,
env.getCommandId(),
env.getReporter(),
- logSpawnPredicate);
+ logSpawnPredicate,
+ executionOptions.getExpandParamFiles());
} else {
boolean binaryElseJson = executionOptions.getExecutionLogBinaryFile() != null;
// Use a well-known temporary path to avoid accumulation of potentially large files in /tmp
@@ -222,7 +223,8 @@
env.getRuntime().getFileSystem().getDigestFunction(),
xattrProvider,
uriFuture != null,
- logSpawnPredicate);
+ logSpawnPredicate,
+ executionOptions.getExpandParamFiles());
}
} catch (InterruptedException e) {
env.getReporter()
diff --git a/src/main/java/com/google/devtools/build/lib/exec/CompactSpawnLogContext.java b/src/main/java/com/google/devtools/build/lib/exec/CompactSpawnLogContext.java
index c150983..7a625df 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/CompactSpawnLogContext.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/CompactSpawnLogContext.java
@@ -187,9 +187,10 @@
CompressionService compressionService,
UUID invocationId,
ExtendedEventHandler reporter,
- Predicate<Spawn> logSpawnPredicate)
+ Predicate<Spawn> logSpawnPredicate,
+ boolean expandParamFiles)
throws IOException, InterruptedException {
- super(logSpawnPredicate);
+ super(logSpawnPredicate, expandParamFiles);
this.execRoot = execRoot;
this.workspaceName = workspaceName;
this.remoteOptions = remoteOptions;
@@ -242,7 +243,8 @@
try (SilentCloseable c = Profiler.instance().profile(SPAWN_LOG, "logSpawn")) {
ExecLogEntry.Spawn.Builder builder = ExecLogEntry.Spawn.newBuilder();
- builder.addAllArgs(Lists.transform(spawn.getArguments(), StringEncoding::internalToUnicode));
+ builder.addAllArgs(
+ Lists.transform(getLoggedArguments(spawn), StringEncoding::internalToUnicode));
builder.addAllEnvVars(getEnvironmentVariables(spawn));
Platform platform = getPlatform(spawn, remoteOptions);
if (platform != null) {
diff --git a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java
index 64c5647..229cce5 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java
@@ -221,10 +221,12 @@
effectTags = {OptionEffectTag.TERMINAL_OUTPUT},
help =
"""
- When displaying subcommands (`--subcommands`) or printing the command line of a failed
- action (`--verbose_failures`), expand the contents of param files. When enabled,
- param file references like `@path/to/param_file` are replaced with the actual
- arguments they contain.
+ When displaying subcommands (`--subcommands`), printing the command line of a failed
+ action (`--verbose_failures`) or writing the execution log
+ (`--execution_log_{compact,binary,json}_file`), expand the contents of param files.
+ When enabled, param file references like `@path/to/param_file` are replaced with
+ the actual arguments they contain. Spawns are still executed with param files,
+ and the execution log still records them as inputs.
""")
public abstract boolean getExpandParamFiles();
diff --git a/src/main/java/com/google/devtools/build/lib/exec/ExpandedSpawnLogContext.java b/src/main/java/com/google/devtools/build/lib/exec/ExpandedSpawnLogContext.java
index 4a80514..5ada682 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/ExpandedSpawnLogContext.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/ExpandedSpawnLogContext.java
@@ -107,9 +107,10 @@
DigestHashFunction digestHashFunction,
XattrProvider xattrProvider,
boolean shouldPublish,
- Predicate<Spawn> logSpawnPredicate)
+ Predicate<Spawn> logSpawnPredicate,
+ boolean expandParamFiles)
throws IOException {
- super(logSpawnPredicate);
+ super(logSpawnPredicate, expandParamFiles);
this.encoding = encoding;
this.sorted = sorted;
this.tempPath = tempPath;
@@ -166,7 +167,7 @@
}
try (SilentCloseable c = Profiler.instance().profile("logSpawn")) {
SpawnExec.Builder builder = SpawnExec.newBuilder();
- builder.addAllCommandArgs(spawn.getArguments());
+ builder.addAllCommandArgs(getLoggedArguments(spawn));
builder.addAllEnvironmentVariables(getEnvironmentVariables(spawn));
ImmutableSet<? extends ActionInput> toolFiles = spawn.getToolFiles().toSet();
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java b/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java
index fd36575..1d114bf 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java
@@ -58,9 +58,16 @@
/** An {@link ActionContext} providing the ability to log executed spawns. */
public abstract class SpawnLogContext implements ActionContext {
protected final Predicate<Spawn> logSpawnPredicate;
+ private final boolean expandParamFiles;
- protected SpawnLogContext(Predicate<Spawn> logSpawnPredicate) {
+ protected SpawnLogContext(Predicate<Spawn> logSpawnPredicate, boolean expandParamFiles) {
this.logSpawnPredicate = logSpawnPredicate;
+ this.expandParamFiles = expandParamFiles;
+ }
+
+ /** Returns the arguments to record for the spawn, with param files expanded if requested. */
+ protected ImmutableList<String> getLoggedArguments(Spawn spawn) {
+ return expandParamFiles ? spawn.getArgumentsWithExpandedParamFiles() : spawn.getArguments();
}
/** Returns true if the spawn should be logged. */
diff --git a/src/test/java/com/google/devtools/build/lib/exec/CompactSpawnLogContextTest.java b/src/test/java/com/google/devtools/build/lib/exec/CompactSpawnLogContextTest.java
index 8541033..2d3166f 100644
--- a/src/test/java/com/google/devtools/build/lib/exec/CompactSpawnLogContextTest.java
+++ b/src/test/java/com/google/devtools/build/lib/exec/CompactSpawnLogContextTest.java
@@ -27,6 +27,8 @@
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.Artifact.TreeArtifact;
import com.google.devtools.build.lib.actions.BuildConfigurationEvent;
+import com.google.devtools.build.lib.actions.ParamFileActionInput;
+import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType;
import com.google.devtools.build.lib.actions.RunfilesTree;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
@@ -345,6 +347,45 @@
}
@Test
+ public void testExpandParamFiles(@TestParameter boolean expandParamFiles) throws Exception {
+ ParamFileActionInput paramFileInput =
+ new ParamFileActionInput(
+ PathFragment.create("foo.params"),
+ "@foo.params",
+ ImmutableList.of("a", "b", "c"),
+ ParameterFileType.UNQUOTED);
+
+ SpawnBuilder spawn =
+ new SpawnBuilder("cmd", "--opt", "@foo.params", "--last").withInputs(paramFileInput);
+
+ SpawnLogContext context =
+ createSpawnLogContext(
+ ImmutableMap.of(), /* logSpawnPredicate= */ s -> true, expandParamFiles);
+
+ context.logSpawn(
+ spawn.build(),
+ // ParamFileActionInputs appear in the input map but not in the metadata provider.
+ createInputMetadataProvider(),
+ createInputMap(paramFileInput),
+ fs,
+ defaultTimeout(),
+ defaultSpawnResult());
+
+ ImmutableList<String> expectedArgs =
+ expandParamFiles
+ ? ImmutableList.of("cmd", "--opt", "a", "b", "c", "--last")
+ : ImmutableList.of("cmd", "--opt", "@foo.params", "--last");
+ closeAndAssertLog(
+ context,
+ defaultSpawnExecBuilder()
+ .clearCommandArgs()
+ .addAllCommandArgs(expectedArgs)
+ // The param file is still logged as an input either way.
+ .addInputs(File.newBuilder().setPath("foo.params").setDigest(getDigest("a\nb\nc\n")))
+ .build());
+ }
+
+ @Test
public void testStreaming() throws Exception {
Artifact file = ActionsTestUtil.createArtifact(rootDir, "file");
writeFile(file, "abc");
@@ -366,7 +407,8 @@
new CompressionServiceImpl(),
UUID.fromString("00000000-0000-0000-0000-000000000000"),
storedEventHandler,
- /* logSpawnPredicate= */ s -> true);
+ /* logSpawnPredicate= */ s -> true,
+ /* expandParamFiles= */ false);
context.logSpawn(
spawn.build(),
@@ -408,6 +450,15 @@
SpawnLogContext createSpawnLogContext(
ImmutableMap<String, String> platformProperties, Predicate<Spawn> logSpawnPredicate)
throws IOException, InterruptedException {
+ return createSpawnLogContext(
+ platformProperties, logSpawnPredicate, /* expandParamFiles= */ false);
+ }
+
+ SpawnLogContext createSpawnLogContext(
+ ImmutableMap<String, String> platformProperties,
+ Predicate<Spawn> logSpawnPredicate,
+ boolean expandParamFiles)
+ throws IOException, InterruptedException {
RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
remoteOptions.setRemoteDefaultExecPropertiesField(platformProperties.entrySet().asList());
@@ -422,7 +473,8 @@
new CompressionServiceImpl(),
UUID.fromString("00000000-0000-0000-0000-000000000000"),
storedEventHandler,
- logSpawnPredicate);
+ logSpawnPredicate,
+ expandParamFiles);
}
@Override
diff --git a/src/test/java/com/google/devtools/build/lib/exec/ExpandedSpawnLogContextTest.java b/src/test/java/com/google/devtools/build/lib/exec/ExpandedSpawnLogContextTest.java
index 7cce17d..9c98ce0 100644
--- a/src/test/java/com/google/devtools/build/lib/exec/ExpandedSpawnLogContextTest.java
+++ b/src/test/java/com/google/devtools/build/lib/exec/ExpandedSpawnLogContextTest.java
@@ -88,7 +88,8 @@
DigestHashFunction.SHA256,
SyscallCache.NO_CACHE,
/* shouldPublish= */ false,
- /* logSpawnPredicate= */ s -> true);
+ /* logSpawnPredicate= */ s -> true,
+ /* expandParamFiles= */ false);
context.logSpawn(
spawn.build(),
@@ -140,7 +141,8 @@
DigestHashFunction.SHA256,
SyscallCache.NO_CACHE,
/* shouldPublish= */ false,
- logSpawnPredicate);
+ logSpawnPredicate,
+ /* expandParamFiles= */ false);
}
@Override