Delay creation of params files until we know where the writable execroot is.
When using workers or sandboxes, the output tree is not locked until the
spawn completes execution. However, some virtual inputs, like params files,
have to be materialized before running the spawn.
The previous code was materializing these files directly into the output
tree without previously grabbing the dynamic execution lock, causing trouble.
To fix this, postpone the creation of these files until we know where the
writable execroot for a spawn is (which, when using workers or sandboxes is
in a location separate from the actual execroot).
This fix is gated behind --experimental_delay_virtual_input_materialization
so that we can roll it out separately from a release given the limited testing
we can do on builds that do local execution. I'll remove the old code paths
once we have flipped the flag.
Problem diagnosed by felly@.
RELNOTES: None.
PiperOrigin-RevId: 301849551
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/AbstractContainerizingSandboxedSpawn.java b/src/main/java/com/google/devtools/build/lib/sandbox/AbstractContainerizingSandboxedSpawn.java
index a886383..27f6691 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/AbstractContainerizingSandboxedSpawn.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/AbstractContainerizingSandboxedSpawn.java
@@ -99,6 +99,7 @@
public void createFileSystem() throws IOException {
createDirectories();
createInputs(inputs);
+ inputs.materializeVirtualInputs(sandboxExecRoot);
}
/**
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxHelpers.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxHelpers.java
index f7dc09b..c37c7ce 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxHelpers.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxHelpers.java
@@ -31,8 +31,10 @@
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.TreeMap;
/**
@@ -41,13 +43,35 @@
* <p>All sandboxed strategies within a build should share the same instance of this object.
*/
public final class SandboxHelpers {
+
+ /**
+ * If true, materialize virtual inputs only inside the sandbox, not the output tree. This flag
+ * exists purely to support rolling this out as the defaut in a controlled manner.
+ */
+ private final boolean delayVirtualInputMaterialization;
+
+ /**
+ * Constructs a new collection of helpers.
+ *
+ * @param delayVirtualInputMaterialization whether to materialize virtual inputs only inside the
+ * sandbox
+ */
+ public SandboxHelpers(boolean delayVirtualInputMaterialization) {
+ this.delayVirtualInputMaterialization = delayVirtualInputMaterialization;
+ }
+
/** Wrapper class for the inputs of a sandbox. */
public static final class SandboxInputs {
private final Map<PathFragment, Path> files;
+ private final Set<VirtualActionInput> virtualInputs;
private final Map<PathFragment, PathFragment> symlinks;
- public SandboxInputs(Map<PathFragment, Path> files, Map<PathFragment, PathFragment> symlinks) {
+ public SandboxInputs(
+ Map<PathFragment, Path> files,
+ Set<VirtualActionInput> virtualInputs,
+ Map<PathFragment, PathFragment> symlinks) {
this.files = files;
+ this.virtualInputs = virtualInputs;
this.symlinks = symlinks;
}
@@ -58,15 +82,66 @@
public Map<PathFragment, PathFragment> getSymlinks() {
return symlinks;
}
+
+ /**
+ * Materializes a single virtual input inside the given execroot.
+ *
+ * @param input virtual input to materialize
+ * @param execroot path to the execroot under which to materialize the virtual input
+ * @param needsDelete whether to attempt to delete a previous instance of this virtual input.
+ * When materializing under a new sandbox execroot, we can expect the input to not exist,
+ * but we cannot make the same assumption for the non-sandboxed execroot.
+ * @throws IOException if the virtual input cannot be materialized
+ */
+ private static void materializeVirtualInput(
+ VirtualActionInput input, Path execroot, boolean needsDelete) throws IOException {
+ if (input instanceof ParamFileActionInput) {
+ ParamFileActionInput paramFileInput = (ParamFileActionInput) input;
+ Path outputPath = execroot.getRelative(paramFileInput.getExecPath());
+ if (needsDelete && outputPath.exists()) {
+ outputPath.delete();
+ }
+
+ outputPath.getParentDirectory().createDirectoryAndParents();
+ try (OutputStream outputStream = outputPath.getOutputStream()) {
+ paramFileInput.writeTo(outputStream);
+ }
+ } else {
+ // TODO(b/150963503): We can turn this into an unreachable code path when the old
+ // !delayVirtualInputMaterialization code path is deleted.
+ // TODO(ulfjack): Handle all virtual inputs, e.g., by writing them to a file.
+ Preconditions.checkState(input instanceof EmptyActionInput);
+ }
+ }
+
+ /**
+ * Materializes virtual files inside the sandboxed execroot once it is known.
+ *
+ * <p>These are files that do not have to exist in the execroot: we can materialize them only
+ * inside the sandbox, which means we can create them <i>before</i> we grab the output tree lock
+ * (but assuming we do so inside the sandbox only).
+ *
+ * @param sandboxExecRoot the path to the <i>sandboxed</i> execroot
+ * @throws IOException if any virtual input cannot be materialized
+ */
+ public void materializeVirtualInputs(Path sandboxExecRoot) throws IOException {
+ for (VirtualActionInput input : virtualInputs) {
+ materializeVirtualInput(input, sandboxExecRoot, /*needsDelete=*/ false);
+ }
+ }
}
/**
* Returns the inputs of a Spawn as a map of PathFragments relative to an execRoot to paths in the
* host filesystem where the input files can be found.
*
- * <p>Also writes any supported {@link VirtualActionInput}s found.
+ * <p>This does not (and must not) write any {@link VirtualActionInput}s found because we do not
+ * yet know where they should be written to. We have a path to an {@code execRoot}, but this path
+ * should be treated as read-only because we may not be holding its lock. The caller should use
+ * {@link SandboxInputs#materializeVirtualInputs(Path)} to later write these inputs when it knows
+ * where they should be written to.
*
- * @throws IOException If any files could not be written.
+ * @throws IOException if processing symlinks fails
*/
public SandboxInputs processInputFiles(
Map<PathFragment, ActionInput> inputMap,
@@ -92,43 +167,48 @@
}
Map<PathFragment, Path> inputFiles = new TreeMap<>();
+ Set<VirtualActionInput> virtualInputs = new HashSet<>();
Map<PathFragment, PathFragment> inputSymlinks = new TreeMap<>();
for (Map.Entry<PathFragment, ActionInput> e : inputMap.entrySet()) {
PathFragment pathFragment = e.getKey();
ActionInput actionInput = e.getValue();
- if (actionInput instanceof VirtualActionInput) {
- if (actionInput instanceof ParamFileActionInput) {
- ParamFileActionInput paramFileInput = (ParamFileActionInput) actionInput;
- Path outputPath = execRoot.getRelative(paramFileInput.getExecPath());
- if (outputPath.exists()) {
- outputPath.delete();
- }
- outputPath.getParentDirectory().createDirectoryAndParents();
- try (OutputStream outputStream = outputPath.getOutputStream()) {
- paramFileInput.writeTo(outputStream);
+ // TODO(b/150963503): Make delayVirtualInputMaterialization the default and remove the
+ // alternate code path.
+ if (delayVirtualInputMaterialization) {
+ if (actionInput instanceof VirtualActionInput) {
+ if (actionInput instanceof EmptyActionInput) {
+ inputFiles.put(pathFragment, null);
+ } else {
+ virtualInputs.add((VirtualActionInput) actionInput);
}
+ } else if (actionInput.isSymlink()) {
+ Path inputPath = execRoot.getRelative(actionInput.getExecPath());
+ inputSymlinks.put(pathFragment, inputPath.readSymbolicLink());
} else {
- // TODO(ulfjack): Handle all virtual inputs, e.g., by writing them to a file.
- Preconditions.checkState(actionInput instanceof EmptyActionInput);
+ Path inputPath = execRoot.getRelative(actionInput.getExecPath());
+ inputFiles.put(pathFragment, inputPath);
+ }
+ } else {
+ if (actionInput instanceof VirtualActionInput) {
+ SandboxInputs.materializeVirtualInput(
+ (VirtualActionInput) actionInput, execRoot, /*needsDelete=*/ true);
+ }
+
+ if (actionInput.isSymlink()) {
+ Path inputPath = execRoot.getRelative(actionInput.getExecPath());
+ inputSymlinks.put(pathFragment, inputPath.readSymbolicLink());
+ } else {
+ Path inputPath =
+ actionInput instanceof EmptyActionInput
+ ? null
+ : execRoot.getRelative(actionInput.getExecPath());
+ inputFiles.put(pathFragment, inputPath);
}
}
-
- if (actionInput.isSymlink()) {
- Path inputPath = execRoot.getRelative(actionInput.getExecPath());
- // TODO(lberki): This does I/O. This method already throws IOException, so I suppose that is
- // A-OK?
- inputSymlinks.put(pathFragment, inputPath.readSymbolicLink());
- } else {
- Path inputPath =
- actionInput instanceof EmptyActionInput
- ? null
- : execRoot.getRelative(actionInput.getExecPath());
- inputFiles.put(pathFragment, inputPath);
- }
}
- return new SandboxInputs(inputFiles, inputSymlinks);
+ return new SandboxInputs(inputFiles, virtualInputs, inputSymlinks);
}
/** The file and directory outputs of a sandboxed spawn. */
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java
index 21fe4ac..939747c 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java
@@ -180,7 +180,7 @@
SandboxOptions options = checkNotNull(env.getOptions().getOptions(SandboxOptions.class));
sandboxBase = computeSandboxBase(options, env);
- SandboxHelpers helpers = new SandboxHelpers();
+ SandboxHelpers helpers = new SandboxHelpers(options.delayVirtualInputMaterialization);
// Do not remove the sandbox base when --sandbox_debug was specified so that people can check
// out the contents of the generated sandbox directories.
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java
index c72118e..efe65ed 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java
@@ -355,6 +355,17 @@
+ " grows to the size specified by this flag when the server is idle.")
public int asyncTreeDeleteIdleThreads;
+ @Option(
+ name = "experimental_delay_virtual_input_materialization",
+ defaultValue = "false",
+ documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY,
+ effectTags = {OptionEffectTag.EXECUTION},
+ help =
+ "If set to true, creates virtual inputs (like params files) only inside the sandbox, "
+ + "not in the execroot, which fixes a race condition when using the dynamic "
+ + "scheduler. This flag exists purely to support rolling this bug fix out.")
+ public boolean delayVirtualInputMaterialization;
+
/** Converter for the number of threads used for asynchronous tree deletion. */
public static final class AsyncTreeDeletesConverter extends ResourceConverter {
public AsyncTreeDeletesConverter() {
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxfsSandboxedSpawn.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxfsSandboxedSpawn.java
index a69ed2f..f7a454e 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxfsSandboxedSpawn.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxfsSandboxedSpawn.java
@@ -181,6 +181,8 @@
public void createFileSystem() throws IOException {
sandboxScratchDir.createDirectory();
+ inputs.materializeVirtualInputs(sandboxScratchDir);
+
Set<PathFragment> dirsToCreate = new HashSet<>(writableDirs);
for (PathFragment output : outputs.files()) {
dirsToCreate.add(output.getParentDirectory());
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/WindowsSandboxedSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/sandbox/WindowsSandboxedSpawnRunner.java
index afbfa6e..022e686 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/WindowsSandboxedSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/WindowsSandboxedSpawnRunner.java
@@ -76,6 +76,8 @@
context.getArtifactExpander(),
execRoot);
+ readablePaths.materializeVirtualInputs(execRoot);
+
ImmutableSet.Builder<Path> writablePaths = ImmutableSet.builder();
writablePaths.addAll(getWritableDirs(execRoot, environment));
for (ActionInput output : spawn.getOutputFiles()) {
diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerExecRoot.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerExecRoot.java
index 3c39717..67ed3c9 100644
--- a/src/main/java/com/google/devtools/build/lib/worker/WorkerExecRoot.java
+++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerExecRoot.java
@@ -74,6 +74,8 @@
// Finally, create anything that is still missing.
createDirectories(dirsToCreate);
createInputs(inputsToCreate);
+
+ inputs.materializeVirtualInputs(workDir);
}
/** Populates the provided sets with the inputs and directories than need to be created. */
diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java
index adcf574..77441d8 100644
--- a/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java
+++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java
@@ -140,21 +140,20 @@
@Override
public void executorInit(CommandEnvironment env, BuildRequest request, ExecutorBuilder builder) {
Preconditions.checkNotNull(workerPool);
+ SandboxOptions sandboxOptions = env.getOptions().getOptions(SandboxOptions.class);
ImmutableMultimap<String, String> extraFlags =
ImmutableMultimap.copyOf(env.getOptions().getOptions(WorkerOptions.class).workerExtraFlags);
LocalEnvProvider localEnvProvider = LocalEnvProvider.forCurrentOs(env.getClientEnv());
WorkerSpawnRunner spawnRunner =
new WorkerSpawnRunner(
- new SandboxHelpers(),
+ new SandboxHelpers(sandboxOptions.delayVirtualInputMaterialization),
env.getExecRoot(),
workerPool,
extraFlags,
env.getReporter(),
createFallbackRunner(env, localEnvProvider),
localEnvProvider,
- env.getOptions()
- .getOptions(SandboxOptions.class)
- .symlinkedSandboxExpandsTreeArtifactsInRunfilesTree,
+ sandboxOptions.symlinkedSandboxExpandsTreeArtifactsInRunfilesTree,
env.getBlazeWorkspace().getBinTools(),
env.getLocalResourceManager(),
// TODO(buchgr): Replace singleton by a command-scoped RunfilesTreeUpdater
diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java
index e9b54c3..6eaebb8 100644
--- a/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java
@@ -322,6 +322,17 @@
ActionExecutionMetadata owner = spawn.getResourceOwner();
try {
try {
+ inputFiles.materializeVirtualInputs(execRoot);
+ } catch (IOException e) {
+ throw new UserExecException(
+ ErrorMessage.builder()
+ .message("IOException while materializing virtual inputs:")
+ .exception(e)
+ .build()
+ .toString());
+ }
+
+ try {
worker = workers.borrowObject(key);
request =
createWorkRequest(spawn, context, flagFiles, inputFileCache, worker.getWorkerId());
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/SandboxfsSandboxedSpawnTest.java b/src/test/java/com/google/devtools/build/lib/sandbox/SandboxfsSandboxedSpawnTest.java
index 5ad8852..ac4a601 100644
--- a/src/test/java/com/google/devtools/build/lib/sandbox/SandboxfsSandboxedSpawnTest.java
+++ b/src/test/java/com/google/devtools/build/lib/sandbox/SandboxfsSandboxedSpawnTest.java
@@ -73,6 +73,7 @@
ImmutableMap.of(),
new SandboxInputs(
ImmutableMap.of(PathFragment.create("such/input.txt"), helloTxt),
+ ImmutableSet.of(),
ImmutableMap.of()),
SandboxOutputs.create(
ImmutableSet.of(PathFragment.create("very/output.txt")), ImmutableSet.of()),
@@ -102,7 +103,7 @@
"some-workspace-name",
ImmutableList.of("/bin/true"),
ImmutableMap.of(),
- new SandboxInputs(ImmutableMap.of(), ImmutableMap.of()),
+ new SandboxInputs(ImmutableMap.of(), ImmutableSet.of(), ImmutableMap.of()),
SandboxOutputs.create(ImmutableSet.of(), ImmutableSet.of()),
ImmutableSet.of(),
/* mapSymlinkTargets= */ false,
@@ -128,6 +129,7 @@
ImmutableMap.of(),
new SandboxInputs(
ImmutableMap.of(PathFragment.create("such/input.txt"), helloTxt),
+ ImmutableSet.of(),
ImmutableMap.of()),
SandboxOutputs.create(
ImmutableSet.of(PathFragment.create("very/output.txt")), ImmutableSet.of()),
@@ -161,7 +163,7 @@
"workspace",
ImmutableList.of("/bin/true"),
ImmutableMap.of(),
- new SandboxInputs(ImmutableMap.of(), ImmutableMap.of()),
+ new SandboxInputs(ImmutableMap.of(), ImmutableSet.of(), ImmutableMap.of()),
SandboxOutputs.create(ImmutableSet.of(outputFile), ImmutableSet.of()),
ImmutableSet.of(),
/* mapSymlinkTargets= */ false,
@@ -225,6 +227,7 @@
PathFragment.create("such/link-1.txt"), linkToInput1,
PathFragment.create("such/link-to-link.txt"), linkToLink,
PathFragment.create("such/abs-link.txt"), linkToAbsolutePath),
+ ImmutableSet.of(),
ImmutableMap.of()),
SandboxOutputs.create(
ImmutableSet.of(PathFragment.create("very/output.txt")), ImmutableSet.of()),
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/SymlinkedSandboxedSpawnTest.java b/src/test/java/com/google/devtools/build/lib/sandbox/SymlinkedSandboxedSpawnTest.java
index e3efe7e..97321e5 100644
--- a/src/test/java/com/google/devtools/build/lib/sandbox/SymlinkedSandboxedSpawnTest.java
+++ b/src/test/java/com/google/devtools/build/lib/sandbox/SymlinkedSandboxedSpawnTest.java
@@ -70,6 +70,7 @@
ImmutableMap.of(),
new SandboxInputs(
ImmutableMap.of(PathFragment.create("such/input.txt"), helloTxt),
+ ImmutableSet.of(),
ImmutableMap.of()),
SandboxOutputs.create(
ImmutableSet.of(PathFragment.create("very/output.txt")), ImmutableSet.of()),
@@ -97,7 +98,7 @@
execRoot,
ImmutableList.of("/bin/true"),
ImmutableMap.of(),
- new SandboxInputs(ImmutableMap.of(), ImmutableMap.of()),
+ new SandboxInputs(ImmutableMap.of(), ImmutableSet.of(), ImmutableMap.of()),
SandboxOutputs.create(
ImmutableSet.of(outputFile.relativeTo(execRoot)), ImmutableSet.of()),
ImmutableSet.of(),
diff --git a/src/test/java/com/google/devtools/build/lib/worker/WorkerExecRootTest.java b/src/test/java/com/google/devtools/build/lib/worker/WorkerExecRootTest.java
index 27f73e2..6b602b2 100644
--- a/src/test/java/com/google/devtools/build/lib/worker/WorkerExecRootTest.java
+++ b/src/test/java/com/google/devtools/build/lib/worker/WorkerExecRootTest.java
@@ -65,7 +65,9 @@
new WorkerExecRoot(
execRoot,
new SandboxInputs(
- ImmutableMap.of(PathFragment.create("worker.sh"), workerSh), ImmutableMap.of()),
+ ImmutableMap.of(PathFragment.create("worker.sh"), workerSh),
+ ImmutableSet.of(),
+ ImmutableMap.of()),
SandboxOutputs.create(
ImmutableSet.of(PathFragment.create("very/output.txt")), ImmutableSet.of()),
ImmutableSet.of(PathFragment.create("worker.sh")));
@@ -104,6 +106,7 @@
execRoot,
new SandboxInputs(
ImmutableMap.of(),
+ ImmutableSet.of(),
ImmutableMap.of(
PathFragment.create("dir/input_symlink_1"), PathFragment.create("new_content"),
PathFragment.create("dir/input_symlink_2"), PathFragment.create("unchanged"))),
@@ -141,6 +144,7 @@
execRoot,
new SandboxInputs(
ImmutableMap.of(PathFragment.create("needed_file"), neededWorkspaceFile),
+ ImmutableSet.of(),
ImmutableMap.of()),
SandboxOutputs.create(ImmutableSet.of(), ImmutableSet.of()),
ImmutableSet.of());
@@ -167,7 +171,7 @@
WorkerExecRoot workerExecRoot =
new WorkerExecRoot(
execRoot,
- new SandboxInputs(inputs, ImmutableMap.of()),
+ new SandboxInputs(inputs, ImmutableSet.of(), ImmutableMap.of()),
SandboxOutputs.create(ImmutableSet.of(), ImmutableSet.of()),
ImmutableSet.of());
diff --git a/src/test/shell/integration/BUILD b/src/test/shell/integration/BUILD
index 71f4a3d..b7d72bc 100644
--- a/src/test/shell/integration/BUILD
+++ b/src/test/shell/integration/BUILD
@@ -607,6 +607,7 @@
data = [
":test-deps",
],
+ shard_count = 4,
tags = ["no_windows"],
)
diff --git a/src/test/shell/integration/sandboxing_test.sh b/src/test/shell/integration/sandboxing_test.sh
index 683fefe..433df86 100755
--- a/src/test/shell/integration/sandboxing_test.sh
+++ b/src/test/shell/integration/sandboxing_test.sh
@@ -167,4 +167,119 @@
done
}
+# Builds a target with the given strategy and ensures that the actions require
+# params files to be written in the output base.
+function build_with_params() {
+ local strategy="${1}"; shift
+
+ # Build a Java binary during this test because the Java rules work well with
+ # sandboxing and support workers.
+ mkdir pkg
+ cat >pkg/BUILD <<'EOF'
+java_binary(
+ name = "java",
+ srcs = ["Main.java"],
+ main_class = "pkg.Main",
+)
+EOF
+ cat >pkg/Main.java <<'EOF'
+package pkg;
+public class Main {
+ public static void main(String[] args) {}
+}
+EOF
+
+ bazel build \
+ --strategy=Javac="${strategy}" \
+ --strategy=JavaResourceJar="${strategy}" \
+ --sandbox_debug \
+ --min_param_file_size=100 \
+ "${@}" \
+ //pkg:java || fail "Build failed"
+}
+
+# Verifies that building a target that uses params files writes those params
+# files to both the execroot and the sandbox.
+function do_test_params_files_not_delayed() {
+ local strategy="${1}"; shift
+
+ local output_base
+ output_base="$(bazel info output_base)" || fail "Cannot get output base"
+
+ # Not passing --noexperimental_delay_virtual_input_materialization on
+ # purpose to ensure that's the current default behavior.
+ build_with_params "${strategy}" \
+ --build # Need a no-op flag to avoid set -u breakage on macOS.
+
+ find -L "${output_base}" -name "*params" >files.txt || true
+ grep -q "${output_base}/execroot" files.txt \
+ || fail "Expected params files not found in execroot"
+ grep -q "${output_base}/sandbox" files.txt \
+ || fail "Expected params files not found in sandbox tree"
+}
+
+# Verifies that building a target that uses params files writes those params
+# files only inside the sandbox when we delay virtual input artifact
+# materialization.
+function do_test_params_files_delayed() {
+ local strategy="${1}"; shift
+
+ local output_base
+ output_base="$(bazel info output_base)" || fail "Cannot get output base"
+
+ build_with_params "${strategy}" \
+ --experimental_delay_virtual_input_materialization
+
+ find -L "${output_base}" -name "*params" >files.txt || true
+ grep -q "${output_base}/execroot" files.txt \
+ && fail "Unexpected params files found in execroot"
+ grep -q "${output_base}/sandbox" files.txt \
+ || fail "Expected params files not found in sandbox tree"
+}
+
+# We expect "sandboxed" to use the system-specific sandbox instead of
+# the processwrapper-sandbox (tested below). But if that's not the case,
+# there is not much we can do here.
+function test_params_files_not_delayed_default_sandbox() {
+ do_test_params_files_not_delayed sandboxed
+}
+function test_params_files_delayed_default_sandbox() {
+ do_test_params_files_delayed sandboxed
+}
+
+function test_params_files_not_delayed_process_wrapper_sandbox() {
+ do_test_params_files_not_delayed processwrapper-sandbox
+}
+function test_params_files_delayed_process_wrapper_sandbox() {
+ do_test_params_files_delayed processwrapper-sandbox
+}
+
+# Worker tests do not really belong in this file, but as we are exercising
+# the same code path used for the sandbox regarding virtual input artifact
+# materialization, we keep them here to reuse the testing logic.
+function test_params_files_not_delayed_worker() {
+ local output_base
+ output_base="$(bazel info output_base)" || fail "Cannot get output base"
+
+ # Not passing --noexperimental_delay_virtual_input_materialization on
+ # purpose to ensure that's the current default behavior.
+ build_with_params worker \
+ --build # Need a no-op flag to avoid set -u breakage on macOS.
+
+ find -L "${output_base}" -name "*params" >files.txt || true
+ grep -q "${output_base}/execroot" files.txt \
+ || fail "Expected params files not found in execroot"
+}
+function test_params_files_delayed_worker() {
+ local output_base
+ output_base="$(bazel info output_base)" || fail "Cannot get output base"
+
+ build_with_params worker \
+ --experimental_delay_virtual_input_materialization
+
+ find -L "${output_base}" -name "*params" >files.txt || true
+ grep -q "${output_base}/execroot" files.txt \
+ || fail "Expected params files not found in execroot"
+}
+
run_suite "sandboxing"
