Delete the sandboxfs sandboxing strategy.
This change deletes the strategy with the goal of simplifying the codebase. We will focus our effort instead on improving the other sandboxing strategies to make them faster.
RELNOTES[INC]: The sandboxfs sandboxing strategy is removed. It hadn't been maintained for a long time, it didn't work for most users and it was not consistently faster while being complex to set up. sandboxfs performance is heavily dependent on the specific setup (setup costs are lower, but you have to pay a penalty for the use of each input) and there are scenarios where it is faster and scenarios where it is slower. Overall it is not worth its weight.
PiperOrigin-RevId: 572977096
Change-Id: Idec5ad988bcdcebb4fca48716246d4b1b5ab2984
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/BUILD b/src/main/java/com/google/devtools/build/lib/sandbox/BUILD
index 119beff..87ac37c 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/BUILD
@@ -82,24 +82,6 @@
)
java_library(
- name = "sandboxfs_sandboxed_spawn",
- srcs = [
- "SandboxfsSandboxedSpawn.java",
- ],
- deps = [
- ":sandbox_helpers",
- ":sandboxed_spawns",
- ":sandboxfs_process",
- "//src/main/java/com/google/devtools/build/lib/exec:tree_deleter",
- "//src/main/java/com/google/devtools/build/lib/vfs",
- "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
- "//third_party:flogger",
- "//third_party:guava",
- "//third_party:jsr305",
- ],
-)
-
-java_library(
name = "sandbox_module",
srcs = ["SandboxModule.java"],
deps = [
@@ -111,7 +93,6 @@
":sandbox_helpers",
":sandbox_options",
":sandboxed_spawns",
- ":sandboxfs_process",
":tree_deleter",
":windows_sandbox",
"//src/main/java/com/google/devtools/build/lib:runtime",
@@ -183,27 +164,6 @@
)
java_library(
- name = "sandboxfs_process",
- srcs = [
- "RealSandboxfs01Process.java",
- "RealSandboxfs02Process.java",
- "RealSandboxfsProcess.java",
- "SandboxfsProcess.java",
- ],
- deps = [
- "//src/main/java/com/google/devtools/build/lib/shell",
- "//src/main/java/com/google/devtools/build/lib/util:os",
- "//src/main/java/com/google/devtools/build/lib/versioning",
- "//src/main/java/com/google/devtools/build/lib/vfs",
- "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
- "//third_party:flogger",
- "//third_party:gson",
- "//third_party:guava",
- "//third_party:jsr305",
- ],
-)
-
-java_library(
name = "process_wrapper_sandbox",
srcs = [
"ProcessWrapperSandboxedSpawnRunner.java",
@@ -213,8 +173,6 @@
":abstract_sandbox_spawn_runner",
":sandbox_helpers",
":sandboxed_spawns",
- ":sandboxfs_process",
- ":sandboxfs_sandboxed_spawn",
"//src/main/java/com/google/devtools/build/lib:runtime",
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/exec:abstract_spawn_strategy",
@@ -226,7 +184,6 @@
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
"//third_party:guava",
- "//third_party:jsr305",
],
)
@@ -245,8 +202,6 @@
":sandbox_helpers",
":sandbox_options",
":sandboxed_spawns",
- ":sandboxfs_process",
- ":sandboxfs_sandboxed_spawn",
"//src/main/java/com/google/devtools/build/lib:runtime",
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/actions:artifacts",
@@ -296,8 +251,6 @@
":abstract_sandbox_spawn_runner",
":sandbox_helpers",
":sandboxed_spawns",
- ":sandboxfs_process",
- ":sandboxfs_sandboxed_spawn",
"//src/main/java/com/google/devtools/build/lib:runtime",
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/exec:abstract_spawn_strategy",
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedSpawnRunner.java
index 264906e..fad5cfa 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedSpawnRunner.java
@@ -107,8 +107,6 @@
private final boolean allowNetwork;
private final ProcessWrapper processWrapper;
private final Path sandboxBase;
- @Nullable private final SandboxfsProcess sandboxfsProcess;
- private final boolean sandboxfsMapSymlinkTargets;
private final TreeDeleter treeDeleter;
/**
@@ -126,16 +124,11 @@
* @param helpers common tools and state across all spawns during sandboxed execution
* @param cmdEnv the command environment to use
* @param sandboxBase path to the sandbox base directory
- * @param sandboxfsProcess instance of the sandboxfs process to use; may be null for none, in
- * which case the runner uses a symlinked sandbox
- * @param sandboxfsMapSymlinkTargets map the targets of symlinks within the sandbox if true
*/
DarwinSandboxedSpawnRunner(
SandboxHelpers helpers,
CommandEnvironment cmdEnv,
Path sandboxBase,
- @Nullable SandboxfsProcess sandboxfsProcess,
- boolean sandboxfsMapSymlinkTargets,
TreeDeleter treeDeleter)
throws IOException, InterruptedException {
super(cmdEnv);
@@ -147,8 +140,6 @@
this.processWrapper = ProcessWrapper.fromCommandEnvironment(cmdEnv);
this.localEnvProvider = LocalEnvProvider.forCurrentOs(cmdEnv.getClientEnv());
this.sandboxBase = sandboxBase;
- this.sandboxfsProcess = sandboxfsProcess;
- this.sandboxfsMapSymlinkTargets = sandboxfsMapSymlinkTargets;
this.treeDeleter = treeDeleter;
}
@@ -267,39 +258,6 @@
allowNetwork
|| Spawns.requiresNetwork(spawn, getSandboxOptions().defaultSandboxAllowNetwork);
- if (sandboxfsProcess != null) {
- return new SandboxfsSandboxedSpawn(
- sandboxfsProcess,
- sandboxPath,
- workspaceName,
- commandLine,
- environment,
- inputs,
- outputs,
- ImmutableSet.of(),
- sandboxfsMapSymlinkTargets,
- treeDeleter,
- spawn.getMnemonic(),
- /* sandboxDebugPath= */ null,
- statisticsPath) {
- @Override
- public void createFileSystem() throws IOException {
- super.createFileSystem();
-
- // The set of writable dirs includes the path to the execroot in the sandbox tree, but not
- // the path to the sibling sandboxfs hierarchy. We must explicitly grant access to this to
- // let builds work when the output tree is not under the default path hanging from tmp.
- writableDirs.add(getSandboxExecRoot());
-
- writeConfig(
- sandboxConfigPath,
- writableDirs,
- getInaccessiblePaths(),
- allowNetworkForThisSpawn,
- statisticsPath);
- }
- };
- } else {
return new SymlinkedSandboxedSpawn(
sandboxPath,
sandboxExecRoot,
@@ -323,7 +281,6 @@
statisticsPath);
}
};
- }
}
private void writeConfig(
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java
index 3b2582f..ab81372 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java
@@ -135,8 +135,6 @@
private final Path inaccessibleHelperDir;
private final LocalEnvProvider localEnvProvider;
private final Duration timeoutKillDelay;
- @Nullable private final SandboxfsProcess sandboxfsProcess;
- private final boolean sandboxfsMapSymlinkTargets;
private final TreeDeleter treeDeleter;
private final Reporter reporter;
private final ImmutableList<Root> packageRoots;
@@ -151,9 +149,6 @@
* @param inaccessibleHelperFile path to a file that is (already) inaccessible
* @param inaccessibleHelperDir path to a directory that is (already) inaccessible
* @param timeoutKillDelay an additional grace period before killing timing out commands
- * @param sandboxfsProcess instance of the sandboxfs process to use; may be null for none, in
- * which case the runner uses a symlinked sandbox
- * @param sandboxfsMapSymlinkTargets map the targets of symlinks within the sandbox if true
*/
LinuxSandboxedSpawnRunner(
SandboxHelpers helpers,
@@ -162,8 +157,6 @@
Path inaccessibleHelperFile,
Path inaccessibleHelperDir,
Duration timeoutKillDelay,
- @Nullable SandboxfsProcess sandboxfsProcess,
- boolean sandboxfsMapSymlinkTargets,
TreeDeleter treeDeleter) {
super(cmdEnv);
this.helpers = helpers;
@@ -176,8 +169,6 @@
this.inaccessibleHelperFile = inaccessibleHelperFile;
this.inaccessibleHelperDir = inaccessibleHelperDir;
this.timeoutKillDelay = timeoutKillDelay;
- this.sandboxfsProcess = sandboxfsProcess;
- this.sandboxfsMapSymlinkTargets = sandboxfsMapSymlinkTargets;
this.localEnvProvider = new PosixLocalEnvProvider(cmdEnv.getClientEnv());
this.treeDeleter = treeDeleter;
this.reporter = cmdEnv.getReporter();
@@ -364,22 +355,7 @@
}
Path statisticsPath = sandboxPath.getRelative("stats.out");
commandLineBuilder.setStatisticsPath(statisticsPath);
- if (sandboxfsProcess != null) {
- return new SandboxfsSandboxedSpawn(
- sandboxfsProcess,
- sandboxPath,
- workspaceName,
- commandLineBuilder.build(),
- environment,
- inputs,
- outputs,
- ImmutableSet.of(),
- sandboxfsMapSymlinkTargets,
- treeDeleter,
- spawn.getMnemonic(),
- sandboxDebugPath,
- statisticsPath);
- } else if (sandboxOptions.useHermetic) {
+ if (sandboxOptions.useHermetic) {
commandLineBuilder.setHermeticSandboxPath(sandboxPath);
return new HardlinkedSandboxedSpawn(
sandboxPath,
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategy.java b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategy.java
index 0077922..4b1ed68 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategy.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategy.java
@@ -22,7 +22,6 @@
import com.google.devtools.build.lib.vfs.Path;
import java.io.IOException;
import java.time.Duration;
-import javax.annotation.Nullable;
/** Strategy that uses sandboxing to execute a process. */
public final class LinuxSandboxedStrategy extends AbstractSpawnStrategy {
@@ -43,17 +42,12 @@
* @param cmdEnv the command environment to use
* @param sandboxBase path to the sandbox base directory
* @param timeoutKillDelay additional grace period before killing timing out commands
- * @param sandboxfsProcess instance of the sandboxfs process to use; may be null for none, in
- * which case the runner uses a symlinked sandbox
- * @param sandboxfsMapSymlinkTargets map the targets of symlinks within the sandbox if true
*/
static LinuxSandboxedSpawnRunner create(
SandboxHelpers helpers,
CommandEnvironment cmdEnv,
Path sandboxBase,
Duration timeoutKillDelay,
- @Nullable SandboxfsProcess sandboxfsProcess,
- boolean sandboxfsMapSymlinkTargets,
TreeDeleter treeDeleter)
throws IOException {
Path inaccessibleHelperFile = LinuxSandboxUtil.getInaccessibleHelperFile(sandboxBase);
@@ -66,8 +60,6 @@
inaccessibleHelperFile,
inaccessibleHelperDir,
timeoutKillDelay,
- sandboxfsProcess,
- sandboxfsMapSymlinkTargets,
treeDeleter);
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedSpawnRunner.java
index 434193a..2f7608d 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedSpawnRunner.java
@@ -16,7 +16,6 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
import com.google.devtools.build.lib.actions.ForbiddenActionInputException;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.exec.TreeDeleter;
@@ -31,7 +30,6 @@
import com.google.devtools.build.lib.vfs.Root;
import java.io.IOException;
import java.time.Duration;
-import javax.annotation.Nullable;
/** Strategy that uses sandboxing to execute a process. */
final class ProcessWrapperSandboxedSpawnRunner extends AbstractSandboxSpawnRunner {
@@ -46,8 +44,6 @@
private final ImmutableList<Root> packageRoots;
private final Path sandboxBase;
private final LocalEnvProvider localEnvProvider;
- @Nullable private final SandboxfsProcess sandboxfsProcess;
- private final boolean sandboxfsMapSymlinkTargets;
private final TreeDeleter treeDeleter;
/**
@@ -56,16 +52,11 @@
* @param helpers common tools and state across all spawns during sandboxed execution
* @param cmdEnv the command environment to use
* @param sandboxBase path to the sandbox base directory
- * @param sandboxfsProcess instance of the sandboxfs process to use; may be null for none, in
- * which case the runner uses a symlinked sandbox
- * @param sandboxfsMapSymlinkTargets map the targets of symlinks within the sandbox if true
*/
ProcessWrapperSandboxedSpawnRunner(
SandboxHelpers helpers,
CommandEnvironment cmdEnv,
Path sandboxBase,
- @Nullable SandboxfsProcess sandboxfsProcess,
- boolean sandboxfsMapSymlinkTargets,
TreeDeleter treeDeleter) {
super(cmdEnv);
this.helpers = helpers;
@@ -74,8 +65,6 @@
this.packageRoots = cmdEnv.getPackageLocator().getPathEntries();
this.localEnvProvider = LocalEnvProvider.forCurrentOs(cmdEnv.getClientEnv());
this.sandboxBase = sandboxBase;
- this.sandboxfsProcess = sandboxfsProcess;
- this.sandboxfsMapSymlinkTargets = sandboxfsMapSymlinkTargets;
this.treeDeleter = treeDeleter;
}
@@ -87,15 +76,13 @@
// so we have to prefix our name to turn it into a globally unique value.
Path sandboxPath =
sandboxBase.getRelative(getName()).getRelative(Integer.toString(context.getId()));
- sandboxPath.getParentDirectory().createDirectory();
- sandboxPath.createDirectory();
+ sandboxPath.createDirectoryAndParents();
// b/64689608: The execroot of the sandboxed process must end with the workspace name, just like
// the normal execroot does.
String workspaceName = execRoot.getBaseName();
Path sandboxExecRoot = sandboxPath.getRelative("execroot").getRelative(workspaceName);
- sandboxExecRoot.getParentDirectory().createDirectory();
- sandboxExecRoot.createDirectory();
+ sandboxExecRoot.createDirectoryAndParents();
ImmutableMap<String, String> environment =
localEnvProvider.rewriteLocalEnv(spawn.getEnvironment(), binTools, "/tmp");
@@ -119,22 +106,6 @@
null);
SandboxOutputs outputs = helpers.getOutputs(spawn);
- if (sandboxfsProcess != null) {
- return new SandboxfsSandboxedSpawn(
- sandboxfsProcess,
- sandboxPath,
- workspaceName,
- commandLineBuilder.build(),
- environment,
- inputs,
- outputs,
- ImmutableSet.of(),
- sandboxfsMapSymlinkTargets,
- treeDeleter,
- spawn.getMnemonic(),
- /* sandboxDebugPath= */ null,
- statisticsPath);
- } else {
return new SymlinkedSandboxedSpawn(
sandboxPath,
sandboxExecRoot,
@@ -147,7 +118,6 @@
/* sandboxDebugPath= */ null,
statisticsPath,
spawn.getMnemonic());
- }
}
@Override
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/RealSandboxfs01Process.java b/src/main/java/com/google/devtools/build/lib/sandbox/RealSandboxfs01Process.java
deleted file mode 100644
index 6eade3e..0000000
--- a/src/main/java/com/google/devtools/build/lib/sandbox/RealSandboxfs01Process.java
+++ /dev/null
@@ -1,138 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.sandbox;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import com.google.devtools.build.lib.shell.Subprocess;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.gson.stream.JsonWriter;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.StringWriter;
-
-/**
- * A sandboxfs implementation that uses an external sandboxfs binary to manage the mount point.
- *
- * <p>This implementation provides support for the reconfiguration protocol available in the 0.1.x
- * series.
- */
-final class RealSandboxfs01Process extends RealSandboxfsProcess {
-
- /**
- * Writer with which to send data to the sandboxfs instance. Null only after {@link #destroy()}
- * has been invoked.
- */
- private final BufferedWriter processStdIn;
-
- /**
- * Reader with which to receive data from the sandboxfs instance. Null only after {@link
- * #destroy()} has been invoked.
- */
- private final BufferedReader processStdOut;
-
- /**
- * Initializes a new sandboxfs process instance.
- *
- * @param process process handle for the already-running sandboxfs instance
- */
- RealSandboxfs01Process(Path mountPoint, Subprocess process) {
- super(mountPoint, process);
-
- this.processStdIn =
- new BufferedWriter(new OutputStreamWriter(process.getOutputStream(), UTF_8));
- this.processStdOut = new BufferedReader(new InputStreamReader(process.getInputStream(), UTF_8));
- }
-
- /**
- * Pushes a new configuration to sandboxfs and waits for acceptance.
- *
- * @param config the configuration chunk to push to sandboxfs
- * @throws IOException if sandboxfs cannot be reconfigured either because of an error in the
- * configuration or because we failed to communicate with the subprocess
- */
- private synchronized void reconfigure(String config) throws IOException {
- processStdIn.write(config);
- processStdIn.write("\n\n");
- processStdIn.flush();
-
- String done = processStdOut.readLine();
- if (done == null) {
- throw new IOException("premature end of output from sandboxfs");
- }
- if (!done.equals("Done")) {
- throw new IOException("received unknown string from sandboxfs: " + done + "; expected Done");
- }
- }
-
- @Override
- @SuppressWarnings("UnnecessaryParentheses")
- public void createSandbox(String name, SandboxCreator creator) throws IOException {
- checkArgument(!PathFragment.containsSeparator(name));
- PathFragment root = PathFragment.create("/").getRelative(name);
-
- StringWriter stringWriter = new StringWriter();
- try (JsonWriter writer = new JsonWriter(stringWriter)) {
- writer.beginArray();
- creator.create(
- ((path, underlyingPath, writable) -> {
- writer.beginObject();
- {
- writer.name("Map");
- writer.beginObject();
- {
- writer.name("Mapping");
- writer.value((root.getRelative(path.toRelative())).getPathString());
- writer.name("Target");
- writer.value(underlyingPath.getPathString());
- writer.name("Writable");
- writer.value(writable);
- }
- writer.endObject();
- }
- writer.endObject();
- }));
- writer.endArray();
- }
- reconfigure(stringWriter.toString());
- }
-
- @Override
- @SuppressWarnings("UnnecessaryParentheses")
- public void destroySandbox(String name) throws IOException {
- checkArgument(!PathFragment.containsSeparator(name));
- PathFragment root = PathFragment.create("/").getRelative(name);
-
- StringWriter stringWriter = new StringWriter();
- try (JsonWriter writer = new JsonWriter(stringWriter)) {
- writer.beginArray();
- {
- writer.beginObject();
- {
- writer.name("Unmap");
- writer.value(root.getPathString());
- }
- writer.endObject();
- }
- writer.endArray();
- }
- reconfigure(stringWriter.toString());
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/RealSandboxfs02Process.java b/src/main/java/com/google/devtools/build/lib/sandbox/RealSandboxfs02Process.java
deleted file mode 100644
index 0153821..0000000
--- a/src/main/java/com/google/devtools/build/lib/sandbox/RealSandboxfs02Process.java
+++ /dev/null
@@ -1,423 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.sandbox;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import com.google.common.flogger.GoogleLogger;
-import com.google.common.util.concurrent.SettableFuture;
-import com.google.devtools.build.lib.shell.Subprocess;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.gson.stream.JsonReader;
-import com.google.gson.stream.JsonToken;
-import com.google.gson.stream.JsonWriter;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutionException;
-import javax.annotation.Nullable;
-import javax.annotation.concurrent.GuardedBy;
-
-/**
- * A sandboxfs implementation that uses an external sandboxfs binary to manage the mount point.
- *
- * <p>This implementation provides support for the reconfiguration protocol introduced in 0.2.0.
- */
-final class RealSandboxfs02Process extends RealSandboxfsProcess {
- private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
-
- /**
- * Writer with which to send data to the sandboxfs instance. Null only after {@link #destroy()}
- * has been invoked.
- */
- @GuardedBy("this")
- private JsonWriter processStdIn;
-
- /**
- * Collection of active reconfiguration requests.
- *
- * <p>Each entry in this map is keyed by the identifier of the sandbox being affected by a
- * reconfiguration request and points to a future that is set when the request completes.
- *
- * <p>New entries can be added to this map at any time, but only before {@link #destroy} is
- * called. Once the sandboxfs instance has been destroyed, we do not expect any new requests to
- * come in. However, existing requests will be drained by the {@link ResponsesReader} thread.
- */
- private final ConcurrentMap<String, SettableFuture<Void>> inFlightRequests =
- new ConcurrentHashMap<>();
-
- /**
- * Thread that reads responses from sandboxfs and dispatches them to the futures maintained by
- * {@link #inFlightRequests}.
- */
- private final Thread responsesReader;
-
- /** Representation of a response returned by sandboxfs. */
- private static class Response {
- /** Identifier given in the request. Null if this carries a fatal error. */
- @Nullable final String id;
-
- /**
- * Error message returned by sandboxfs if not null. If {@link #id} is not null, then this error
- * corresponds to a specific request and is recoverable. Otherwise corresponds to a fatal
- * condition, in which case sandboxfs will have stopped listening for requests.
- */
- @Nullable final String error;
-
- /** Constructs a new response with the given values. */
- Response(@Nullable String id, @Nullable String error) {
- this.id = id;
- this.error = error;
- }
- }
-
- /**
- * A thread that reads responses from the sandboxfs output stream and dispatches them to the
- * futures awaiting for them.
- */
- private static class ResponsesReader extends Thread {
-
- private final JsonReader reader;
- private final ConcurrentMap<String, SettableFuture<Void>> inFlightRequests;
-
- ResponsesReader(
- JsonReader reader, ConcurrentMap<String, SettableFuture<Void>> inFlightRequests) {
- this.reader = reader;
- this.inFlightRequests = inFlightRequests;
- }
-
- /** Waits for responses and dispatches them. */
- private void processResponses() throws IOException {
- while (!Thread.interrupted() && reader.peek() != JsonToken.END_DOCUMENT) {
- Response response = readResponse(reader);
- if (response.id == null) {
- // Non-recoverable error: abort.
- throw new IOException(response.error != null ? response.error : "No error reported");
- }
-
- SettableFuture<Void> future = inFlightRequests.remove(response.id);
- if (future == null) {
- throw new IOException("sandboxfs returned response for unknown id " + response.id);
- }
- if (response.error == null) {
- future.set(null);
- } else {
- future.setException(new IOException(response.error));
- }
- }
- }
-
- @Override
- public void run() {
- try {
- processResponses();
- } catch (EOFException e) {
- // OK, nothing to do.
- } catch (IOException e) {
- logger.atWarning().withCause(e).log("Failed to read responses from sandboxfs");
- }
-
- // sandboxfs has either replied with an unrecoverable error or has stopped providing
- // responses. Either way, we have to clean up any pending in-flight requests to unblock the
- // threads waiting for them.
- //
- // Given that we only get here once destroy() has been called, we do not expect any new
- // requests to show up in the inFlightRequests map. This is why we do not synchronize
- // accesses to the map during the iteration.
- while (!inFlightRequests.isEmpty()) {
- Iterator<Map.Entry<String, SettableFuture<Void>>> iter =
- inFlightRequests.entrySet().iterator();
- while (iter.hasNext()) {
- Map.Entry<String, SettableFuture<Void>> entry = iter.next();
- entry.getValue().cancel(true);
- iter.remove();
- }
- }
- }
- }
-
- /** A pair that represents the identifier and the path of a prefix as it will be serialized. */
- private static class Prefix {
- Integer id;
- String serializedPath;
-
- /**
- * Constructs a new prefix from its components.
- *
- * @param id the numerical identifier of the prefix
- * @param serializedPath the path of the prefix. Given that this is typically comes from a call
- * to {@link Path#getParentDirectory()}, the value may be null or empty.
- */
- Prefix(Integer id, @Nullable PathFragment serializedPath) {
- this.id = id;
- if (serializedPath == null || serializedPath.isEmpty()) {
- this.serializedPath = "/";
- } else {
- this.serializedPath = serializedPath.getPathString();
- }
- }
- }
-
- /** Registers and assigns unique identifiers to path prefixes. */
- private static class Prefixes {
- /** Collection of all known path prefixes to this sandboxfs instance. */
- @GuardedBy("this")
- private final Map<PathFragment, Integer> prefixes = new HashMap<>();
-
- /** Last identifier assigned to a path prefix. */
- @GuardedBy("this")
- private Integer lastPrefixId = 1;
-
- /**
- * Registers the given path as a prefix.
- *
- * @param path the path to register as a prefix, which should always be the return value of a
- * call to {@link Path#getParentDirectory()}. As a result, this should either be an absolute
- * path, or a null or empty path.
- * @param newPrefixes tracker for any new prefixes registered by this function. If the given
- * path is not yet known, it will be added to this collection.
- * @return the identifier for the prefix
- */
- private synchronized Integer registerPrefix(PathFragment path, ArrayList<Prefix> newPrefixes) {
- Integer id = prefixes.get(path);
- if (id != null) {
- return id;
- }
-
- id = lastPrefixId;
- prefixes.put(path, id);
- newPrefixes.add(new Prefix(id, path));
- lastPrefixId++;
-
- return id;
- }
- }
-
- /** Tracker for all known path prefixes to this sandboxfs instance. */
- private final Prefixes allPrefixes = new Prefixes();
-
- /**
- * Initializes a new sandboxfs process instance.
- *
- * @param process process handle for the already-running sandboxfs instance
- */
- RealSandboxfs02Process(Path mountPoint, Subprocess process) {
- super(mountPoint, process);
-
- this.processStdIn =
- new JsonWriter(
- new BufferedWriter(new OutputStreamWriter(process.getOutputStream(), UTF_8)));
- JsonReader processStdOut =
- new JsonReader(new BufferedReader(new InputStreamReader(process.getInputStream(), UTF_8)));
-
- // Must use lenient writing and parsing to accept a stream of separate top-level JSON objects.
- this.processStdIn.setLenient(true);
- processStdOut.setLenient(true);
-
- responsesReader = new ResponsesReader(processStdOut, inFlightRequests);
- responsesReader.start();
- }
-
- @Override
- public synchronized void destroy() {
- super.destroy();
-
- responsesReader.interrupt();
- try {
- responsesReader.join();
- } catch (InterruptedException e) {
- logger.atWarning().withCause(e).log(
- "Interrupted while waiting for responses processor thread");
- Thread.currentThread().interrupt();
- }
-
- processStdIn = null;
- }
-
- /**
- * Waits for a single response from sandboxfs and returns it.
- *
- * @param input the stream connected to sandboxfs's stdout
- * @return the response obtained from the stream
- * @throws IOException if sandboxfs fails to read from the stream for any reason, including EOF
- */
- private static Response readResponse(JsonReader input) throws IOException {
- input.beginObject();
- String id = null;
- String error = null;
- while (input.hasNext()) {
- String name = input.nextName();
- switch (name) {
- case "error":
- if (input.peek() == JsonToken.NULL) {
- input.nextNull();
- } else {
- checkState(error == null);
- error = input.nextString();
- }
- break;
-
- case "id":
- if (input.peek() == JsonToken.NULL) {
- input.nextNull();
- } else {
- checkState(id == null);
- id = input.nextString();
- }
- break;
-
- default:
- throw new IOException("Invalid field name in response: " + name);
- }
- }
- input.endObject();
- return new Response(id, error);
- }
-
- /**
- * Registers a new in-flight operation for the given sandbox identifier.
- *
- * <p>The caller must wait for the returned operation using {@link #waitForRequest}.
- *
- * @param id the identifier of the sandbox for which the request will be issued. There can only be
- * one in-flight request per identifier.
- * @return the future used to wait for the request's completion
- */
- private SettableFuture<Void> newRequest(String id) {
- SettableFuture<Void> future = SettableFuture.create();
- SettableFuture<Void> other = inFlightRequests.put(id, future);
- checkState(other == null, "Cannot have two in-flight requests for sandbox '%s'", id);
- return future;
- }
-
- /**
- * Waits for a request to complete and unregisters its in-flight operation.
- *
- * @param future the value returned by {@link #newRequest}.
- * @throws IOException if the request cannot be waited for or if it raised an error
- */
- private static void waitForRequest(SettableFuture<Void> future) throws IOException {
- try {
- future.get();
- } catch (ExecutionException e) {
- Throwable cause = e.getCause();
- if (cause instanceof IOException) {
- throw (IOException) cause;
- } else {
- throw new AssertionError("Unexpected exception type thrown by readResponse()", cause);
- }
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new IOException("Interrupted while waiting for sandboxfs response");
- }
- }
-
- @Override
- @SuppressWarnings("UnnecessaryParentheses")
- public void createSandbox(String id, SandboxCreator creator) throws IOException {
- checkArgument(!PathFragment.containsSeparator(id));
-
- SettableFuture<Void> future = newRequest(id);
- synchronized (this) {
- processStdIn.beginObject();
- {
- processStdIn.name("C");
- processStdIn.beginObject();
- {
- processStdIn.name("i");
- processStdIn.value(id);
- processStdIn.name("m");
- processStdIn.beginArray();
- ArrayList<Prefix> newPrefixes = new ArrayList<>();
- creator.create(
- (path, underlyingPath, writable) -> {
- Integer pathPrefix =
- allPrefixes.registerPrefix(path.getParentDirectory(), newPrefixes);
- Integer underlyingPathPrefix =
- allPrefixes.registerPrefix(underlyingPath.getParentDirectory(), newPrefixes);
-
- // This synchronized block is theoretically unnecessary because the lock is already
- // held by the caller of this lambda. However, we need to reenter it to appease
- // the @GuardedBy clauses.
- synchronized (this) {
- processStdIn.beginObject();
- {
- processStdIn.name("x");
- processStdIn.value(pathPrefix);
- processStdIn.name("p");
- processStdIn.value(path.getBaseName());
- processStdIn.name("y");
- processStdIn.value(underlyingPathPrefix);
- processStdIn.name("u");
- processStdIn.value(underlyingPath.getBaseName());
- if (writable) {
- processStdIn.name("w");
- processStdIn.value(writable);
- }
- }
- processStdIn.endObject();
- }
- });
- processStdIn.endArray();
- if (!newPrefixes.isEmpty()) {
- processStdIn.name("q");
- processStdIn.beginObject();
- for (Prefix prefix : newPrefixes) {
- processStdIn.name(prefix.id.toString());
- processStdIn.value(prefix.serializedPath);
- }
- processStdIn.endObject();
- }
- }
- processStdIn.endObject();
- }
- processStdIn.endObject();
-
- processStdIn.flush();
- }
- waitForRequest(future);
- }
-
- @Override
- @SuppressWarnings("UnnecessaryParentheses")
- public void destroySandbox(String id) throws IOException {
- checkArgument(!PathFragment.containsSeparator(id));
-
- SettableFuture<Void> future = newRequest(id);
- synchronized (this) {
- processStdIn.beginObject();
- {
- processStdIn.name("D");
- processStdIn.value(id);
- }
- processStdIn.endObject();
-
- processStdIn.flush();
- }
- waitForRequest(future);
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/RealSandboxfsProcess.java b/src/main/java/com/google/devtools/build/lib/sandbox/RealSandboxfsProcess.java
deleted file mode 100644
index d830adec..0000000
--- a/src/main/java/com/google/devtools/build/lib/sandbox/RealSandboxfsProcess.java
+++ /dev/null
@@ -1,189 +0,0 @@
-// Copyright 2018 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.sandbox;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.flogger.GoogleLogger;
-import com.google.devtools.build.lib.shell.Subprocess;
-import com.google.devtools.build.lib.shell.SubprocessBuilder;
-import com.google.devtools.build.lib.util.OS;
-import com.google.devtools.build.lib.versioning.GnuVersionParser;
-import com.google.devtools.build.lib.versioning.ParseException;
-import com.google.devtools.build.lib.versioning.SemVer;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.io.IOException;
-import javax.annotation.Nullable;
-
-/**
- * A sandboxfs implementation that uses an external sandboxfs binary to manage the mount point.
- *
- * <p>This class implements common code to generalize the interactions with sandboxfs, but delegates
- * to its subclassess once the version of sandboxfs in use has been determined. The subclasses
- * implement logic specific to each version to provide compatibility with the different versions of
- * sandboxfs that the user might have installed.
- */
-abstract class RealSandboxfsProcess implements SandboxfsProcess {
-
- private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
-
- /**
- * Contains the {@code --allow} flag to pass to sandboxfs.
- *
- * <p>On macOS, we need to allow users other than self to access the sandboxfs instance. This is
- * necessary because macOS's amfid, which runs as root, has to have access to the binaries within
- * the sandbox in order to validate signatures. See:
- * http://jmmv.dev/2017/10/fighting-execs-sandboxfs-macos.html
- */
- @VisibleForTesting
- static final String ALLOW_FLAG = OS.getCurrent() == OS.DARWIN ? "--allow=other" : "--allow=self";
-
- /** Directory on which the sandboxfs is serving. */
- private final Path mountPoint;
-
- /**
- * Process handle to the sandboxfs instance. Null only after {@link #destroy()} has been invoked.
- */
- @Nullable private Subprocess process;
-
- /**
- * Shutdown hook to stop the sandboxfs instance on abrupt termination. Null only after {@link
- * #destroy()} has been invoked.
- */
- @Nullable private Thread shutdownHook;
-
- /**
- * Initializes a new sandboxfs process instance.
- *
- * @param process process handle for the already-running sandboxfs instance
- */
- RealSandboxfsProcess(Path mountPoint, Subprocess process) {
- this.mountPoint = mountPoint;
- this.process = process;
-
- this.shutdownHook =
- new Thread(
- () -> {
- try {
- this.destroy();
- } catch (Exception e) {
- logger.atWarning().withCause(e).log(
- "Failed to destroy running sandboxfs instance; mount point may have "
- + "been left behind");
- }
- });
- Runtime.getRuntime().addShutdownHook(shutdownHook);
- }
-
- /**
- * Mounts a new sandboxfs instance.
- *
- * <p>The root of the file system instance is left unmapped which means that it remains as
- * read-only throughout the lifetime of this instance. Writable subdirectories can later be mapped
- * via {@link #createSandbox}.
- *
- * @param binary path to the sandboxfs binary. This is a {@link PathFragment} and not a {@link
- * Path} because we want to support "bare" (non-absolute) names for the location of the
- * sandboxfs binary; such names are automatically looked for in the {@code PATH}.
- * @param mountPoint directory on which to mount the sandboxfs instance
- * @param logFile path to the file that will receive all sandboxfs logging output
- * @return a new handle that represents the running process
- * @throws IOException if there is a problem starting the process
- */
- static SandboxfsProcess mount(PathFragment binary, Path mountPoint, Path logFile)
- throws IOException {
- logger.atInfo().log("Mounting sandboxfs (%s) onto %s", binary, mountPoint);
-
- GnuVersionParser<SemVer> parser = new GnuVersionParser<>("sandboxfs", SemVer::parse);
- SemVer version;
- try {
- version = parser.fromProgram(binary);
- } catch (IOException | ParseException e) {
- throw new IOException("Failed to get sandboxfs version from " + binary, e);
- }
-
- ImmutableList.Builder<String> argvBuilder = ImmutableList.builder();
- argvBuilder.add(binary.getPathString());
- argvBuilder.add(ALLOW_FLAG);
-
- // TODO(jmmv): Pass flags to enable sandboxfs' debugging support (--listen_address and --debug)
- // when requested by the user via --sandbox_debug. Tricky because we have to figure out how to
- // deal with port numbers (which sandboxfs can autoassign, but doesn't currently promise a way
- // to tell us back what it picked).
-
- argvBuilder.add(mountPoint.getPathString());
-
- SubprocessBuilder processBuilder = new SubprocessBuilder();
- processBuilder.setArgv(argvBuilder.build());
- processBuilder.setStderr(logFile.getPathFile());
- processBuilder.setEnv(ImmutableMap.of(
- // sandboxfs may need to locate fusermount depending on the FUSE implementation so pass the
- // PATH to the subprocess (which we assume is sufficient).
- "PATH", System.getenv("PATH")));
-
- Subprocess process = processBuilder.start();
- RealSandboxfsProcess sandboxfs;
- if (version.compareTo(SemVer.from(0, 2)) >= 0) {
- sandboxfs = new RealSandboxfs02Process(mountPoint, process);
- } else {
- sandboxfs = new RealSandboxfs01Process(mountPoint, process);
- }
- try {
- // Create an empty sandbox to ensure sandboxfs is successfully serving.
- sandboxfs.createSandbox("empty", (mapper) -> {});
- } catch (IOException e) {
- process.destroyAndWait();
- throw new IOException("sandboxfs failed to start", e);
- }
- return sandboxfs;
- }
-
- @Override
- public Path getMountPoint() {
- return mountPoint;
- }
-
- @Override
- public boolean isAlive() {
- return process != null && !process.finished();
- }
-
- @Override
- public synchronized void destroy() {
- if (shutdownHook != null) {
- Runtime.getRuntime().removeShutdownHook(shutdownHook);
- shutdownHook = null;
- }
-
- if (process != null) {
- try {
- process.getOutputStream().close();
- } catch (IOException e) {
- logger.atWarning().withCause(e).log("Failed to close sandboxfs's stdin pipe");
- }
-
- try {
- process.getInputStream().close();
- } catch (IOException e) {
- logger.atWarning().withCause(e).log("Failed to close sandboxfs's stdout pipe");
- }
-
- process.destroyAndWait();
- process = null;
- }
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxHelpers.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxHelpers.java
index f6b844b..7c140fc 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxHelpers.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxHelpers.java
@@ -76,9 +76,6 @@
/**
* Moves all given outputs from a root to another.
*
- * <p>This is a support function to help with the implementation of {@link
- * SandboxfsSandboxedSpawn#copyOutputs(Path)}.
- *
* @param outputs outputs to move as relative paths to a root
* @param sourceRoot source directory from which to resolve outputs
* @param targetRoot target directory to which to move the resolved outputs from the source
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java
index d005623..0f3b517 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java
@@ -15,7 +15,6 @@
package com.google.devtools.build.lib.sandbox;
import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Preconditions.checkState;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
@@ -25,8 +24,6 @@
import com.google.devtools.build.lib.actions.ForbiddenActionInputException;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.SpawnResult;
-import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent;
-import com.google.devtools.build.lib.buildtool.buildevent.BuildInterruptedEvent;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.exec.ExecutionOptions;
@@ -77,9 +74,6 @@
/** Path to the location of the sandboxes. */
@Nullable private Path sandboxBase;
- /** Instance of the sandboxfs process in use, if enabled. */
- @Nullable private SandboxfsProcess sandboxfsProcess;
-
/**
* Collection of spawn runner instantiated during the executor setup.
*
@@ -220,20 +214,7 @@
}
}
SandboxStash.initialize(env.getWorkspaceName(), env.getOutputBase(), options);
- Path mountPoint = sandboxBase.getRelative("sandboxfs");
- if (sandboxfsProcess != null) {
- if (options.sandboxDebug) {
- env.getReporter()
- .handle(
- Event.info(
- "Unmounting sandboxfs instance left behind on "
- + mountPoint
- + " by a previous command"));
- }
- sandboxfsProcess.destroy();
- sandboxfsProcess = null;
- }
// SpawnExecutionPolicy#getId returns unique base directories for each sandboxed action during
// the life of a Bazel server instance so we don't need to worry about stale directories from
// previous builds. However, on the very first build of an instance of the server, we must
@@ -243,32 +224,7 @@
sandboxBase.deleteTree();
}
firstBuild = false;
-
- PathFragment sandboxfsPath = PathFragment.create(options.sandboxfsPath);
sandboxBase.createDirectoryAndParents();
- if (options.useSandboxfs != TriState.NO) {
- mountPoint.createDirectory();
- Path logFile = sandboxBase.getRelative("sandboxfs.log");
-
- if (sandboxfsProcess == null) {
- if (options.sandboxDebug) {
- env.getReporter().handle(Event.info("Mounting sandboxfs instance on " + mountPoint));
- }
- try (SilentCloseable c = Profiler.instance().profile("mountSandboxfs")) {
- sandboxfsProcess = RealSandboxfsProcess.mount(sandboxfsPath, mountPoint, logFile);
- } catch (IOException e) {
- if (options.sandboxDebug) {
- env.getReporter()
- .handle(
- Event.info(
- "sandboxfs failed to mount due to " + e.getMessage() + "; ignoring"));
- }
- if (options.useSandboxfs == TriState.YES) {
- throw e;
- }
- }
- }
- }
PathFragment windowsSandboxPath = PathFragment.create(options.windowsSandboxPath);
boolean windowsSandboxSupported;
@@ -296,8 +252,6 @@
helpers,
cmdEnv,
sandboxBase,
- sandboxfsProcess,
- options.sandboxfsMapSymlinkTargets,
treeDeleter));
spawnRunners.add(spawnRunner);
builder.registerStrategy(
@@ -350,8 +304,6 @@
cmdEnv,
sandboxBase,
timeoutKillDelay,
- sandboxfsProcess,
- options.sandboxfsMapSymlinkTargets,
treeDeleter));
spawnRunners.add(spawnRunner);
builder.registerStrategy(
@@ -369,8 +321,6 @@
helpers,
cmdEnv,
sandboxBase,
- sandboxfsProcess,
- options.sandboxfsMapSymlinkTargets,
treeDeleter));
spawnRunners.add(spawnRunner);
builder.registerStrategy(
@@ -534,41 +484,6 @@
}
}
- /**
- * Unmounts an existing sandboxfs instance unless the user asked not to by providing the {@code
- * --sandbox_debug} flag.
- */
- private void unmountSandboxfs() {
- if (sandboxfsProcess != null) {
- if (shouldCleanupSandboxBase) {
- sandboxfsProcess.destroy();
- sandboxfsProcess = null;
- } else {
- checkNotNull(env, "env not initialized; was beforeCommand called?");
- env.getReporter()
- .handle(Event.info("Leaving sandboxfs mounted because of --sandbox_debug"));
- }
- }
- }
-
- /** Silently tries to unmount an existing sandboxfs instance, ignoring errors. */
- private void tryUnmountSandboxfsOnShutdown() {
- if (sandboxfsProcess != null) {
- sandboxfsProcess.destroy();
- sandboxfsProcess = null;
- }
- }
-
- @Subscribe
- public void buildComplete(@SuppressWarnings("unused") BuildCompleteEvent event) {
- unmountSandboxfs();
- }
-
- @Subscribe
- public void buildInterrupted(@SuppressWarnings("unused") BuildInterruptedEvent event) {
- unmountSandboxfs();
- }
-
@Subscribe
public void cleanStarting(@SuppressWarnings("unused") CleanStartingEvent event) {
SandboxStash.clean(treeDeleter, env.getOutputBase());
@@ -625,11 +540,6 @@
}
shouldCleanupSandboxBase = false;
- checkState(
- sandboxfsProcess == null,
- "sandboxfs instance should have been shut down at this "
- + "point; were the buildComplete/buildInterrupted events sent?");
-
cleanupSandboxBaseTop(sandboxBase);
// We intentionally keep sandboxBase around, without resetting it to null, in case we have
// asynchronous deletions going on. In that case, we'd still want to retry this during
@@ -643,8 +553,6 @@
}
private void commonShutdown() {
- tryUnmountSandboxfsOnShutdown();
-
// Try to clean up as much garbage as possible, if there happens to be any. This will delay
// server termination but it's the nice thing to do. If the user gets impatient, they can always
// kill us again.
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java
index 79735ed..6be6d9d 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java
@@ -97,10 +97,9 @@
effectTags = {OptionEffectTag.TERMINAL_OUTPUT},
help =
"Enables debugging features for the sandboxing feature. This includes two things: first, "
- + "the sandbox root contents are left untouched after a build (and if sandboxfs is "
- + "in use, the file system is left mounted); and second, prints extra debugging "
- + "information on execution. This can help developers of Bazel or Starlark rules "
- + "with debugging failures due to missing input files, etc.")
+ + "the sandbox root contents are left untouched after a build; and second, prints "
+ + "extra debugging information on execution. This can help developers of Bazel or "
+ + "Starlark rules with debugging failures due to missing input files, etc.")
public boolean sandboxDebug;
@Option(
@@ -187,37 +186,11 @@
public List<ImmutableMap.Entry<String, String>> sandboxAdditionalMounts;
@Option(
- name = "experimental_use_sandboxfs",
- converter = TriStateConverter.class,
- defaultValue = "false",
- documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY,
- effectTags = {OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS, OptionEffectTag.EXECUTION},
- help =
- "Use sandboxfs to create the actions' execroot directories instead of building a symlink "
- + "tree. If \"yes\", the binary provided by --experimental_sandboxfs_path must be "
- + "valid and correspond to a supported version of sandboxfs. If \"auto\", the binary "
- + "may be missing or not compatible.")
- public TriState useSandboxfs;
-
- @Option(
- name = "experimental_sandboxfs_path",
- defaultValue = "sandboxfs",
- documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY,
- effectTags = {OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS, OptionEffectTag.EXECUTION},
- help =
- "Path to the sandboxfs binary to use when --experimental_use_sandboxfs is true. If a "
- + "bare name, use the first binary of that name found in the PATH.")
- public String sandboxfsPath;
-
- @Option(
name = "experimental_sandboxfs_map_symlink_targets",
defaultValue = "false",
documentationCategory = OptionDocumentationCategory.INPUT_STRICTNESS,
effectTags = {OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS, OptionEffectTag.EXECUTION},
- help =
- "If true, maps the targets of symbolic links specified as action inputs into the "
- + "sandbox. This feature exists purely to workaround buggy rules that do not do "
- + "this on their own and should be removed once all such rules are fixed.")
+ help = "No-op")
public boolean sandboxfsMapSymlinkTargets;
@Option(
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxfsProcess.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxfsProcess.java
deleted file mode 100644
index 9c949ed..0000000
--- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxfsProcess.java
+++ /dev/null
@@ -1,93 +0,0 @@
-// Copyright 2018 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.sandbox;
-
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.io.IOException;
-
-/** Interface to interact with a sandboxfs instance. */
-interface SandboxfsProcess {
-
- /** Returns the path to the sandboxfs's mount point. */
- Path getMountPoint();
-
- /** Returns true if the sandboxfs process is still alive. */
- boolean isAlive();
-
- /**
- * Unmounts and stops the sandboxfs process.
- *
- * <p>This function must be idempotent because there can be a race between explicit calls during
- * regular execution and calls from shutdown hooks.
- */
- void destroy();
-
- /** Interface to create a single mapping definition within the sandbox. */
- @FunctionalInterface
- interface SandboxMapper {
- /**
- * Defines a single mapping within the sandbox being constructed.
- *
- * <p>Called only from within the context of a {@link SandboxCreator} instance, which carries
- * the details of the specific sandbox being constructed.
- *
- * @param path the path within the sandbox to define as a mapping. sandboxfs expects this path
- * to be absolute.
- * @param underlyingPath the path to which the mapping points to, which is outside of the
- * sandbox. sandboxfs expects this path to be absolute.
- * @param writable whether the mapping is writable or read-only
- * @throws IOException if the attempt to send the mapping to sandboxfs fails
- */
- void map(PathFragment path, PathFragment underlyingPath, boolean writable) throws IOException;
- }
-
- /** Interface to create all the mappings of a sandbox. */
- @FunctionalInterface
- interface SandboxCreator {
- /**
- * Creates all the mappings of a sandbox.
- *
- * <p>This lambda runs holding a big lock around sandboxfs and is called in the critical path of
- * all running actions. As a result, this lambda should not block (which includes not doing any
- * I/O other than writing to the sandboxfs stream via {@code mapper}).
- *
- * @param mapper a callback to send a single mapping definition to sandboxfs
- * @throws IOException if calls to the mapper fail, or if the lambda desires to raise any other
- * problem during the construction of the sandbox
- */
- void create(SandboxMapper mapper) throws IOException;
- }
-
- /**
- * Creates a top-level directory with the given name and delegates the set up of all mappings
- * within that directory to the given {@code creator} lambda.
- *
- * @param name basename of the top-level directory to create
- * @param creator a callback to populate the sandbox with mappings
- * @throws IOException if sandboxfs cannot be reconfigured either because of an error in the
- * configuration or because we failed to communicate with the subprocess
- */
- void createSandbox(String name, SandboxCreator creator) throws IOException;
-
- /**
- * Destroys a top-level directory and all of its contents.
- *
- * @param name basename of the top-level directory to destroy
- * @throws IOException if sandboxfs cannot be reconfigured either because of an error in the
- * configuration or because we failed to communicate with the subprocess
- */
- void destroySandbox(String name) throws IOException;
-}
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxfsSandboxedSpawn.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxfsSandboxedSpawn.java
deleted file mode 100644
index 6c0e9f6..0000000
--- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxfsSandboxedSpawn.java
+++ /dev/null
@@ -1,400 +0,0 @@
-// Copyright 2018 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.sandbox;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import com.google.common.base.Joiner;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.flogger.GoogleLogger;
-import com.google.devtools.build.lib.exec.TreeDeleter;
-import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxInputs;
-import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxOutputs;
-import com.google.devtools.build.lib.vfs.FileSystemUtils;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.devtools.build.lib.vfs.RootedPath;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-import javax.annotation.Nullable;
-
-/**
- * Creates an execRoot for a Spawn that contains all required input files by mounting a sandboxfs
- * FUSE filesystem on the provided path.
- */
-class SandboxfsSandboxedSpawn implements SandboxedSpawn {
- private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
-
- /** Sequence number to assign a unique subtree to each action within the mount point. */
- private static final AtomicInteger lastId = new AtomicInteger();
-
- /** The sandboxfs instance to use for this spawn. */
- private final SandboxfsProcess process;
-
- /** Arguments to pass to the spawn, including the binary name. */
- private final ImmutableList<String> arguments;
-
- /** Environment variables to pass to the spawn. */
- private final ImmutableMap<String, String> environment;
-
- /** Collection of input files to be made available to the spawn in read-only mode. */
- private final SandboxInputs inputs;
-
- /** Collection of output files to expect from the spawn. */
- private final SandboxOutputs outputs;
-
- /** Collection of directories where the spawn can write files to relative to {@link #execRoot}. */
- private final Set<PathFragment> writableDirs;
-
- /** Map the targets of symlinks within the sandbox if true. */
- private final boolean mapSymlinkTargets;
-
- /** Scheduler for tree deletions. */
- private final TreeDeleter treeDeleter;
-
- /**
- * Writable directory where the spawn runner keeps control files and the execroot outside of the
- * sandboxfs instance.
- */
- private final Path sandboxPath;
-
- /**
- * Writable directory to support the writes performed by the command. This acts as the target
- * of all writable mappings in the sandboxfs instance.
- */
- private final Path sandboxScratchDir;
-
- /** Path to the working directory of the command. */
- private final Path execRoot;
-
- /**
- * Name of the sandbox within the sandboxfs mount point, which is just the basename of the
- * top-level directory where all execroot paths start.
- */
- private final String sandboxName;
-
- /** Path to the execroot within the sandbox. */
- private final PathFragment rootFragment;
-
- /** Flag to track whether the sandbox needs to be unmapped. */
- private boolean sandboxIsMapped;
-
- /** The mnemonic of this spawn. */
- private final String mnemonic;
-
- @Nullable private final Path sandboxDebugPath;
- @Nullable private final Path statisticsPath;
-
- /**
- * Constructs a new sandboxfs-based spawn runner.
- *
- * @param process sandboxfs instance to use for this spawn
- * @param sandboxPath writable directory where the spawn runner keeps control files
- * @param arguments arguments to pass to the spawn, including the binary name
- * @param environment environment variables to pass to the spawn
- * @param inputs input files to be made available to the spawn in read-only mode
- * @param outputs output files to expect from the spawn
- * @param writableDirs directories where the spawn can write files to, relative to the sandbox's
- * dynamically-allocated execroot
- * @param mapSymlinkTargets map the targets of symlinks within the sandbox if true
- * @param treeDeleter scheduler for tree deletions
- * @param mnemonic Mnemonic of this spawn.
- */
- SandboxfsSandboxedSpawn(
- SandboxfsProcess process,
- Path sandboxPath,
- String workspaceName,
- ImmutableList<String> arguments,
- ImmutableMap<String, String> environment,
- SandboxInputs inputs,
- SandboxOutputs outputs,
- Set<PathFragment> writableDirs,
- boolean mapSymlinkTargets,
- TreeDeleter treeDeleter,
- String mnemonic,
- @Nullable Path sandboxDebugPath,
- @Nullable Path statisticsPath) {
- this.process = process;
- this.arguments = arguments;
- this.environment = environment;
- this.inputs = inputs;
- for (PathFragment path : outputs.files().values()) {
- checkArgument(!path.isAbsolute(), "outputs %s must be relative", path);
- }
- for (PathFragment path : outputs.dirs().values()) {
- checkArgument(!path.isAbsolute(), "outputs %s must be relative", path);
- }
- this.outputs = outputs;
- for (PathFragment path : writableDirs) {
- checkArgument(!path.isAbsolute(), "writable directory %s must be relative", path);
- }
- this.writableDirs = writableDirs;
- this.mapSymlinkTargets = mapSymlinkTargets;
- this.treeDeleter = treeDeleter;
- this.mnemonic = mnemonic;
-
- this.sandboxPath = sandboxPath;
- this.sandboxScratchDir = sandboxPath.getRelative("scratch");
-
- int id = lastId.getAndIncrement();
- this.sandboxName = "" + id;
- this.sandboxIsMapped = false;
- this.sandboxDebugPath = sandboxDebugPath;
- this.statisticsPath = statisticsPath;
-
- // b/64689608: The execroot of the sandboxed process must end with the workspace name, just
- // like the normal execroot does. Some tools walk their path hierarchy looking for this
- // component and misbehave if they don't find it.
- this.execRoot =
- process.getMountPoint().getRelative(this.sandboxName).getRelative(workspaceName);
- this.rootFragment = PathFragment.create("/" + workspaceName);
- }
-
- @Override
- public Path getSandboxExecRoot() {
- return execRoot;
- }
-
- @Override
- public ImmutableList<String> getArguments() {
- return arguments;
- }
-
- @Override
- public ImmutableMap<String, String> getEnvironment() {
- return environment;
- }
-
- @Override
- public Path getSandboxDebugPath() {
- return sandboxDebugPath;
- }
-
- @Override
- public Path getStatisticsPath() {
- return statisticsPath;
- }
-
- @Override
- public String getMnemonic() {
- return mnemonic;
- }
-
- @Override
- public void createFileSystem() throws IOException {
- sandboxScratchDir.createDirectory();
-
- Set<PathFragment> dirsToCreate = new HashSet<>(writableDirs);
- for (PathFragment output : outputs.files().values()) {
- dirsToCreate.add(output.getParentDirectory());
- }
- dirsToCreate.addAll(outputs.dirs().values());
- for (PathFragment dir : dirsToCreate) {
- sandboxScratchDir.getRelative(dir).createDirectoryAndParents();
- }
-
- createSandbox(process, sandboxName, rootFragment, sandboxScratchDir, inputs, mapSymlinkTargets);
- sandboxIsMapped = true;
- }
-
- @Override
- public void copyOutputs(Path targetExecRoot) throws IOException {
- // TODO(jmmv): If we knew the targetExecRoot when setting up the spawn, we may be able to
- // configure sandboxfs so that the output files are written directly to their target locations.
- // This would avoid having to move them after-the-fact.
- SandboxHelpers.moveOutputs(outputs, sandboxScratchDir, targetExecRoot);
- }
-
- @Override
- public void delete() {
- // We can only ask sandboxfs to unmap a sandbox if we successfully finished creating it.
- // Otherwise, the request may fail, or we may fail our own checks that validate the lifecycle of
- // the sandboxes.
- if (sandboxIsMapped) {
- try {
- process.destroySandbox(sandboxName);
- } catch (IOException e) {
- // We use independent subdirectories for each action, so a failure to unmap one, while
- // annoying, is not a big deal. The sandboxfs instance will be unmounted anyway after
- // the build, which will cause these to go away anyway.
- logger.atWarning().withCause(e).log("Cannot unmap %s", sandboxName);
- }
- sandboxIsMapped = false;
- }
-
- try {
- treeDeleter.deleteTree(sandboxPath);
- } catch (IOException e) {
- // This usually means that the Spawn itself exited but still has children running that
- // we couldn't wait for, which now block deletion of the sandbox directory. (Those processes
- // may be creating new files in the directories we are trying to delete, preventing the
- // deletion.) On Linux this should never happen: we use PID namespaces when available and the
- // subreaper feature when not to make sure all children have been reliably killed before
- // returning, but on other OSes this might not always work. The SandboxModule will try to
- // delete them again when the build is all done, at which point it hopefully works... so let's
- // just go on here.
- }
- }
-
- /**
- * Maps the targets of relative symlinks into the sandbox.
- *
- * <p>Symlinks with relative targets are tricky business. Consider this simple case: the source
- * tree contains {@code dir/file.h} and {@code dir/symlink.h} where {@code dir/symlink.h}'s target
- * is {@code ./file.h}. If {@code dir/symlink.h} is supplied as an input, we must preserve its
- * target "as is" to avoid confusing any tooling: for example, the C compiler will understand that
- * both {@code dir/file.h} and {@code dir/symlink.h} are the same entity and handle them
- * appropriately. (We did encounter a case where the compiler complained about duplicate symbols
- * because we exposed symlinks as regular files.)
- *
- * <p>However, there is no guarantee that the target of the symlink is mapped in the sandbox. You
- * may think that this is a bug in the rules, and you would probably be right, but until those
- * rules are fixed, we must supply a workaround. Therefore, we must handle these two cases: if the
- * target is explicitly mapped, we do nothing. If it isn't, we have to compute where the target
- * lives within the sandbox and map that as well. Oh, and we have to do this recursively.
- *
- * @param path path to expose within the sandbox
- * @param symlink path to the target of the mapping specified by {@code path}
- * @param mappings mutable collection of mappings to extend with the new symlink entries. Note
- * that the entries added to this map may correspond to explicitly-mapped entries, so the
- * caller must check this to avoid duplicate mappings
- * @throws IOException if we fail to resolve symbolic links
- */
- private static void computeSymlinkMappings(
- PathFragment path, Path symlink, Map<PathFragment, PathFragment> mappings)
- throws IOException {
- for (; ; ) {
- PathFragment symlinkTarget = symlink.readSymbolicLinkUnchecked();
- if (!symlinkTarget.isAbsolute()) {
- PathFragment keyParent = path.getParentDirectory();
- if (keyParent == null) {
- throw new IOException("Cannot resolve " + symlinkTarget + " relative to " + path);
- }
- PathFragment key = keyParent.getRelative(symlinkTarget);
-
- Path valueParent = symlink.getParentDirectory();
- if (valueParent == null) {
- throw new IOException("Cannot resolve " + symlinkTarget + " relative to " + symlink);
- }
- Path value = valueParent.getRelative(symlinkTarget);
- mappings.put(key, value.asFragment());
-
- if (value.isSymbolicLink()) {
- path = key;
- symlink = value;
- continue;
- }
- }
- break;
- }
- }
-
- /**
- * Creates a new set of mappings to sandbox the given inputs.
- *
- * @param process the sandboxfs instance on which to create the sandbox
- * @param sandboxName the name of the sandbox to pass to sandboxfs
- * @param rootFragment path within the sandbox to the execroot to create
- * @param scratchDir writable used as the target for all writable mappings
- * @param inputs collection of paths to expose within the sandbox as read-only mappings, given as
- * a map of mapped path to target path. The target path may be null, in which case an empty
- * read-only file is mapped.
- * @param sandboxfsMapSymlinkTargets map the targets of symlinks within the sandbox if true
- * @throws IOException if we fail to resolve symbolic links
- */
- private static void createSandbox(
- SandboxfsProcess process,
- String sandboxName,
- PathFragment rootFragment,
- Path scratchDir,
- SandboxInputs inputs,
- boolean sandboxfsMapSymlinkTargets)
- throws IOException {
- // Path to the empty file used as the target of mappings that don't provide one. This is
- // lazily created and initialized only when we need such a mapping. It's safe to share the
- // same empty file across all such mappings because this file is exposed as read-only.
- //
- // We cannot use /dev/null, as we used to do in the past, because exposing devices via a
- // FUSE file system (which sandboxfs is) requires root privileges.
- PathFragment emptyFile = null;
-
- // Collection of extra mappings needed to represent the targets of relative symlinks. Lazily
- // created once we encounter the first symlink in the list of inputs.
- Map<PathFragment, PathFragment> symlinks = null;
-
- for (Map.Entry<PathFragment, RootedPath> entry : inputs.getFiles().entrySet()) {
- if (entry.getValue() == null) {
- if (emptyFile == null) {
- Path emptyFilePath = scratchDir.getRelative("empty");
- FileSystemUtils.createEmptyFile(emptyFilePath);
- emptyFile = emptyFilePath.asFragment();
- }
- } else {
- if (sandboxfsMapSymlinkTargets && entry.getValue().asPath().isSymbolicLink()) {
- if (symlinks == null) {
- symlinks = new HashMap<>();
- }
- computeSymlinkMappings(entry.getKey(), entry.getValue().asPath(), symlinks);
- }
- }
- }
-
- // IMPORTANT: Keep the code in the lambda passed to createSandbox() free from any operations
- // that may block. This includes doing any kind of I/O. We used to include the loop above in
- // this call and doing so cost 2-3% of the total build time measured on an iOS build with many
- // actions that have thousands of inputs each.
- @Nullable final PathFragment finalEmptyFile = emptyFile;
- @Nullable final Map<PathFragment, PathFragment> finalSymlinks = symlinks;
- process.createSandbox(
- sandboxName,
- (mapper) -> {
- mapper.map(rootFragment, scratchDir.asFragment(), true);
-
- for (Map.Entry<PathFragment, RootedPath> entry : inputs.getFiles().entrySet()) {
- PathFragment target;
- if (entry.getValue() == null) {
- checkNotNull(finalEmptyFile, "Must have been initialized above by matching logic");
- target = finalEmptyFile;
- } else {
- target = entry.getValue().asPath().asFragment();
- }
- mapper.map(rootFragment.getRelative(entry.getKey()), target, false);
- }
-
- if (finalSymlinks != null) {
- for (Map.Entry<PathFragment, PathFragment> entry : finalSymlinks.entrySet()) {
- if (!inputs.getFiles().containsKey(entry.getKey())) {
- mapper.map(rootFragment.getRelative(entry.getKey()), entry.getValue(), false);
- }
- }
- }
- });
-
- // sandboxfs probably doesn't support symlinks.
- // TODO(jmmv): This claim is simply not true. Figure out why this code snippet was added and
- // address the real problem.
- if (!inputs.getSymlinks().isEmpty()) {
- throw new IOException(
- "sandboxfs sandbox does not support unresolved symlinks "
- + Joiner.on(", ").join(inputs.getSymlinks().keySet()));
- }
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/BUILD b/src/test/java/com/google/devtools/build/lib/sandbox/BUILD
index 318bb6b..61e13a8 100644
--- a/src/test/java/com/google/devtools/build/lib/sandbox/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/sandbox/BUILD
@@ -15,7 +15,6 @@
name = "testutil",
testonly = 1,
srcs = [
- "FakeSandboxfsProcess.java",
"SandboxedSpawnRunnerTestCase.java",
"SpawnRunnerTestUtil.java",
],
@@ -26,10 +25,8 @@
"//src/main/java/com/google/devtools/build/lib:runtime",
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/actions:artifacts",
- "//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity",
"//src/main/java/com/google/devtools/build/lib/exec:spawn_input_expander",
"//src/main/java/com/google/devtools/build/lib/exec:spawn_runner",
- "//src/main/java/com/google/devtools/build/lib/sandbox:sandboxfs_process",
"//src/main/java/com/google/devtools/build/lib/shell",
"//src/main/java/com/google/devtools/build/lib/util/io",
"//src/main/java/com/google/devtools/build/lib/vfs",
@@ -44,19 +41,6 @@
],
)
-java_library(
- name = "sandboxfs-base-tests",
- testonly = 1,
- srcs = ["BaseSandboxfsProcessIntegrationTest.java"],
- deps = [
- "//src/main/java/com/google/devtools/build/lib/sandbox:sandboxfs_process",
- "//src/main/java/com/google/devtools/build/lib/vfs",
- "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
- "//third_party:junit4",
- "//third_party:truth",
- ],
-)
-
java_test(
name = "SmallTests",
size = "small",
@@ -98,40 +82,6 @@
)
java_test(
- name = "SandboxFsTests",
- size = "small",
- srcs =
- [
- "BaseRealSandboxfsProcessTest.java",
- "BaseSandboxfsProcessIntegrationTest.java",
- "FakeSandboxfsProcessIntegrationTest.java",
- "RealSandboxfs01ProcessTest.java",
- "RealSandboxfs02ProcessTest.java",
- "SandboxfsSandboxedSpawnTest.java",
- ],
- tags = ["no_windows"],
- test_class = "com.google.devtools.build.lib.AllTests",
- runtime_deps = [
- "//src/test/java/com/google/devtools/build/lib:test_runner",
- ],
- deps = [
- ":testutil",
- "//src/main/java/com/google/devtools/build/lib/sandbox:sandbox_helpers",
- "//src/main/java/com/google/devtools/build/lib/sandbox:sandboxed_spawns",
- "//src/main/java/com/google/devtools/build/lib/sandbox:sandboxfs_process",
- "//src/main/java/com/google/devtools/build/lib/sandbox:sandboxfs_sandboxed_spawn",
- "//src/main/java/com/google/devtools/build/lib/sandbox:tree_deleter",
- "//src/main/java/com/google/devtools/build/lib/vfs",
- "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
- "//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs",
- "//src/test/java/com/google/devtools/build/lib/testutil:TestUtils",
- "//third_party:guava",
- "//third_party:junit4",
- "//third_party:truth",
- ],
-)
-
-java_test(
name = "SpawnRunnerTests",
size = "medium",
srcs = [
@@ -161,7 +111,6 @@
"//src/main/java/com/google/devtools/build/lib/sandbox:process_wrapper_sandbox",
"//src/main/java/com/google/devtools/build/lib/sandbox:sandbox_helpers",
"//src/main/java/com/google/devtools/build/lib/sandbox:sandboxed_spawns",
- "//src/main/java/com/google/devtools/build/lib/sandbox:sandboxfs_process",
"//src/main/java/com/google/devtools/build/lib/sandbox:tree_deleter",
"//src/main/java/com/google/devtools/build/lib/util:os",
"//src/main/java/com/google/devtools/build/lib/util/io",
@@ -192,25 +141,3 @@
"//third_party:truth",
],
)
-
-java_test(
- name = "sandboxfs-integration-tests",
- srcs = ["RealSandboxfsProcessIntegrationTest.java"],
- data = ["//src/test/java/com/google/devtools/build/lib:embedded_scripts"],
- local = 1,
- tags = [
- "manual", # Test requires: --test_env=SANDBOXFS=/path/to/sandboxfs
- "no-sandbox",
- "no_windows",
- ],
- test_class = "com.google.devtools.build.lib.AllTests",
- runtime_deps = [
- "//src/test/java/com/google/devtools/build/lib:test_runner",
- ],
- deps = [
- ":sandboxfs-base-tests",
- "//src/main/java/com/google/devtools/build/lib/sandbox:sandboxfs_process",
- "//src/main/java/com/google/devtools/build/lib/vfs",
- "//third_party:junit4",
- ],
-)
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/BaseRealSandboxfsProcessTest.java b/src/test/java/com/google/devtools/build/lib/sandbox/BaseRealSandboxfsProcessTest.java
deleted file mode 100644
index 7205d9f..0000000
--- a/src/test/java/com/google/devtools/build/lib/sandbox/BaseRealSandboxfsProcessTest.java
+++ /dev/null
@@ -1,173 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.sandbox;
-
-import static com.google.common.truth.Truth.assertThat;
-
-import com.google.devtools.build.lib.vfs.DigestHashFunction;
-import com.google.devtools.build.lib.vfs.FileSystem;
-import com.google.devtools.build.lib.vfs.FileSystemUtils;
-import com.google.devtools.build.lib.vfs.JavaIoFileSystem;
-import com.google.devtools.build.lib.vfs.Path;
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import org.junit.After;
-import org.junit.Before;
-
-/**
- * Common code to unit test {@link RealSandboxfsProcess}.
- *
- * <p>These tests validate the communication protocol between Bazel and a sandboxfs but do so using
- * golden data. They are meant to smoke-test changes to the Bazel codebase against all supported
- * sandboxfs versions but cannot guarantee that the integration with a real sandboxfs binary work.
- */
-public abstract class BaseRealSandboxfsProcessTest {
-
- /** Path to the mount point passed to sandboxfs. Not accessed, so it's not even created. */
- static final String FAKE_MOUNT_POINT = "/non-existent/mount/point";
-
- /** Sandboxfs version to return to Bazel when queried. */
- private final String version;
-
- /**
- * Expected mount arguments for each {@link #createAndStartFakeSandboxfs} call. Supplied as a
- * single string with all arguments concatenated as seen by sandboxfs.
- */
- private final String expectedArgs;
-
- private FileSystem fileSystem;
- private Path tmpDir;
-
- // Initialized via createAndStartFakeSandboxfs and checked by verifyFakeSandboxfsExecution.
- private Path capturedArgs;
- private Path capturedRequests;
-
- BaseRealSandboxfsProcessTest(String version, String expectedArgs) {
- this.version = version;
- this.expectedArgs = expectedArgs;
- }
-
- @Before
- public void setUp() throws Exception {
- fileSystem = new JavaIoFileSystem(DigestHashFunction.SHA256);
- tmpDir = fileSystem.getPath(System.getenv("TEST_TMPDIR")).getRelative("test");
- tmpDir.createDirectory();
- }
-
- @After
- public void tearDown() throws Exception {
- tmpDir.deleteTree();
- }
-
- /**
- * Starts a sandboxfs instance using a fake binary that captures all received requests and yields
- * mock responses.
- *
- * @param responses the mock responses to return to Bazel when issuing requests, broken down by
- * line printed to stdout
- * @return a sandboxfs process handler
- * @throws IOException if the fake sandboxfs cannot be prepared or started
- */
- SandboxfsProcess createAndStartFakeSandboxfs(List<String> responses) throws IOException {
- capturedArgs = tmpDir.getRelative("captured-args");
- capturedRequests = tmpDir.getRelative("captured-requests");
-
- Path fakeSandboxfs = tmpDir.getRelative("fake-sandboxfs");
- try (PrintWriter writer =
- new PrintWriter(
- new BufferedWriter(
- new OutputStreamWriter(fakeSandboxfs.getOutputStream(), StandardCharsets.UTF_8)))) {
- writer.println("#! /bin/bash");
-
- // Ignore requests for termination. The real sandboxfs process must be sent a SIGTERM to stop
- // serving, but in our case we want to terminate cleanly after waiting for all input to be
- // recorded.
- writer.println("trap '' TERM;");
-
- // Handle a --version invocation and exit quickly, which is a prerequisite for the mount call.
- writer.println("if [ \"${*}\" = \"--version\" ]; then");
- writer.println(" echo sandboxfs " + version + ";");
- writer.println(" exit 0;");
- writer.println("fi;");
-
- // Capture all arguments for later inspection.
- writer.println("for arg in \"${@}\"; do");
- writer.println(" echo \"${arg}\" >>" + capturedArgs + ";");
- writer.println("done;");
-
- // Attempt to "parse" requests coming through stdin by just counting brace pairs, assuming
- // that the input is composed of a stream of JSON objects. Then, for each request, emit one
- // response.
- //
- // We must do this because the unordered response processor required to parse 0.2.0 output
- // expects responses to come only after their requests have been issued. Ideally we'd match
- // our mock responses to specific requests to allow for testing of unordered responses, but
- // for now assume all requests and responses in the test are correctly ordered.
- //
- // TODO(jmmv): This has become pretty awful. Should rethink unit testing.
- for (String response : responses) {
- writer.println("braces=0; started=no");
- writer.println("while read -d '' -n 1 ch; do");
- writer.println(" case \"${ch}\" in");
- writer.println(" '{') braces=$((braces + 1)); started=yes ;;");
- writer.println(" '[') braces=$((braces + 1)); started=yes ;;");
- writer.println(" ']') braces=$((braces - 1)) ;;");
- writer.println(" '}') braces=$((braces - 1)) ;;");
- writer.println(" esac");
- writer.println(" [[ \"${ch}\" != '' ]] || ch='\n'");
- writer.println(" printf '%c' \"${ch}\" >>" + capturedRequests);
- writer.println(" if [[ \"${started}\" = yes && \"${braces}\" -eq 0 ]]; then");
- writer.println(" echo '" + response + "';");
- writer.println(" break;");
- writer.println(" fi");
- writer.println("done");
- }
-
- // Capture any stray requests not expected by the test data.
- writer.println("cat >>" + capturedRequests);
- }
- fakeSandboxfs.setExecutable(true);
-
- return RealSandboxfsProcess.mount(
- fakeSandboxfs.asFragment(),
- fileSystem.getPath(FAKE_MOUNT_POINT),
- tmpDir.getRelative("log"));
- }
-
- /**
- * Checks that the given sandboxfs process behaved as expected.
- *
- * @param process the sandboxfs instance to stop and verify, which must have been previously
- * started by {@link #createAndStartFakeSandboxfs}
- * @param expectedRequests a flat string containing all requests given to sandboxfs (i.e. the raw
- * contents of its stdin)
- * @throws IOException if the fake sandboxfs instance cannot be stopped or if there is a problem
- * reading the captured data
- */
- void verifyFakeSandboxfsExecution(SandboxfsProcess process, String expectedRequests)
- throws IOException {
- process.destroy();
-
- String args = FileSystemUtils.readContent(capturedArgs, StandardCharsets.UTF_8);
- assertThat(args).isEqualTo(expectedArgs);
-
- String requests = FileSystemUtils.readContent(capturedRequests, StandardCharsets.UTF_8);
- assertThat(requests).isEqualTo(expectedRequests);
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/BaseSandboxfsProcessIntegrationTest.java b/src/test/java/com/google/devtools/build/lib/sandbox/BaseSandboxfsProcessIntegrationTest.java
deleted file mode 100644
index 275001b..0000000
--- a/src/test/java/com/google/devtools/build/lib/sandbox/BaseSandboxfsProcessIntegrationTest.java
+++ /dev/null
@@ -1,139 +0,0 @@
-// Copyright 2018 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.sandbox;
-
-import static com.google.common.truth.Truth.assertThat;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.junit.Assert.assertThrows;
-
-import com.google.devtools.build.lib.vfs.FileSystemUtils;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.io.IOException;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Common tests for all implementations of {@link SandboxfsProcess}.
- *
- * <p>Subclasses must define the provided hooks to configure the file system the tests run in (which
- * can be real or virtual), and a mechanism to "mount" a sandboxfs instance.
- *
- * <p>Subclasses inherit and run all the tests in this class.
- */
-abstract class BaseSandboxfsProcessIntegrationTest {
-
- /** Test-specific temporary directory and file system. */
- protected Path tmpDir;
-
- /** Hook to obtain the path to a test-specific temporary directory and file system. */
- abstract Path newTmpDir() throws IOException;
-
- /** Hook to mount a new test-specific sandboxfs instance. */
- abstract SandboxfsProcess mount(Path mountPoint) throws IOException;
-
- @Before
- public void setUp() throws IOException {
- tmpDir = newTmpDir();
- }
-
- @After
- public void tearDown() throws IOException {
- tmpDir.deleteTreesBelow();
- tmpDir = null;
- }
-
- @Test
- public void testMount_missingDirectory() throws IOException {
- IOException expected = assertThrows(
- IOException.class, () -> mount(tmpDir.getRelative("missing")));
- assertThat(expected).hasMessageThat().matches(".*(/missing.*does not exist|failed to start).*");
- }
-
- @Test
- public void testLifeCycle() throws IOException {
- Path mountPoint = tmpDir.getRelative("mnt");
- mountPoint.createDirectory();
- SandboxfsProcess process = mount(mountPoint);
- try {
- assertThat(process.isAlive()).isTrue();
- process.destroy();
- assertThat(process.isAlive()).isFalse();
- process.destroy();
- assertThat(process.isAlive()).isFalse();
- } finally {
- process.destroy();
- }
- }
-
- @Test
- public void testReconfigure() throws IOException {
- Path mountPoint = tmpDir.getRelative("mnt");
- mountPoint.createDirectory();
- SandboxfsProcess process = mount(mountPoint);
- try {
- // Start by ensuring the mount point is empty.
- assertThat(mountPoint.getDirectoryEntries()).isEmpty();
-
- // Create a file outside of the mount point to ensure it's not touched.
- mountPoint.getRelative("../unrelated").createDirectory();
-
- // Create first sandbox.
- Path oneFile = tmpDir.getRelative("one");
- FileSystemUtils.writeContent(oneFile, UTF_8, "One test data");
- process.createSandbox(
- "first",
- (mapper) -> mapper.map(PathFragment.create("/foo"), oneFile.asFragment(), false));
- Path first = mountPoint.getRelative("first");
- assertThat(mountPoint.getDirectoryEntries()).containsExactly(first);
- assertThat(first.getDirectoryEntries()).containsExactly(first.getRelative("foo"));
- assertThat(FileSystemUtils.readContent(first.getRelative("foo"), UTF_8))
- .isEqualTo("One test data");
-
- // Create second sandbox, which is expected to be fully disjoint from the first one.
- Path twoFile = tmpDir.getRelative("two");
- FileSystemUtils.writeContent(twoFile, UTF_8, "Two test data");
- Path longLink = tmpDir.getRelative("long/link");
- longLink.getParentDirectory().createDirectoryAndParents();
- longLink.createSymbolicLink(oneFile); // The target is irrelevant but must exist.
- process.createSandbox(
- "second",
- (mapper) -> {
- mapper.map(PathFragment.create("/foo"), twoFile.asFragment(), false);
- mapper.map(PathFragment.create("/something/complex"), longLink.asFragment(), false);
- });
- Path second = mountPoint.getRelative("second");
- assertThat(mountPoint.getDirectoryEntries()).containsExactly(first, second);
- assertThat(second.getDirectoryEntries())
- .containsExactly(second.getRelative("foo"), second.getRelative("something"));
- assertThat(FileSystemUtils.readContent(first.getRelative("foo"), UTF_8))
- .isEqualTo("One test data");
- assertThat(FileSystemUtils.readContent(second.getRelative("foo"), UTF_8))
- .isEqualTo("Two test data");
- assertThat(FileSystemUtils.readContent(second.getRelative("something/complex"), UTF_8))
- .isEqualTo("One test data");
-
- // Destroy one sandbox and ensure the other remains.
- process.destroySandbox("first");
- assertThat(mountPoint.getDirectoryEntries()).containsExactly(second);
-
- // Ensure that files that should not have been touched throughout the test are still there.
- assertThat(mountPoint.getRelative("../unrelated").exists()).isTrue();
- } finally {
- process.destroy();
- }
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedSpawnRunnerTest.java b/src/test/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedSpawnRunnerTest.java
index 24d252b..cd69081 100644
--- a/src/test/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedSpawnRunnerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedSpawnRunnerTest.java
@@ -128,8 +128,6 @@
new SandboxHelpers(),
commandEnvironment,
sandboxBase,
- /* sandboxfsProcess= */ null,
- /* sandboxfsMapSymlinkTargets= */ false,
treeDeleter);
doSimpleExecutionTest(runner);
}
@@ -141,8 +139,6 @@
new SandboxHelpers(),
commandEnvironment,
sandboxBase,
- /* sandboxfsProcess= */ null,
- /* sandboxfsMapSymlinkTargets= */ false,
treeDeleter);
Spawn spawn =
new SpawnBuilder("cp", "params/param-file", "out")
@@ -169,22 +165,4 @@
.containsExactly("--foo", "--bar");
}
}
-
- @Test
- public void testSimpleExecutionWithSandboxfs() throws Exception {
- Path mountPoint = commandEnvironment.getExecRoot().getRelative("mount");
- mountPoint.createDirectoryAndParents();
- SandboxfsProcess sandboxfsProcess = new FakeSandboxfsProcess(
- mountPoint.getFileSystem(), mountPoint.asFragment());
-
- DarwinSandboxedSpawnRunner runner =
- new DarwinSandboxedSpawnRunner(
- new SandboxHelpers(),
- commandEnvironment,
- sandboxBase,
- sandboxfsProcess,
- /* sandboxfsMapSymlinkTargets= */ false,
- treeDeleter);
- doSimpleExecutionTest(runner);
- }
}
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/FakeSandboxfsProcess.java b/src/test/java/com/google/devtools/build/lib/sandbox/FakeSandboxfsProcess.java
deleted file mode 100644
index adebfea..0000000
--- a/src/test/java/com/google/devtools/build/lib/sandbox/FakeSandboxfsProcess.java
+++ /dev/null
@@ -1,133 +0,0 @@
-// Copyright 2018 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.sandbox;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-
-import com.google.devtools.build.lib.vfs.FileSystem;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
-
-/**
- * A fake in-process sandboxfs implementation that uses symlinks on the Bazel file system API.
- */
-final class FakeSandboxfsProcess implements SandboxfsProcess {
-
- /** File system on which the fake sandboxfs instance operates. */
- private final FileSystem fileSystem;
-
- /** Directory on which the sandboxfs is serving. */
- private final PathFragment mountPoint;
-
- /**
- * Tracker for the sandbox names that currently exist in the sandbox.
- *
- * <p>Used to catch mistakes in creating an already-existing sandbox or deleting a non-existent
- * sandbox.
- */
- private final Set<String> activeSandboxes = new HashSet<>();
-
- /**
- * Whether this "process" is valid or not. Used to better represent the workflow of a real
- * sandboxfs subprocess.
- */
- private boolean alive = true;
-
- /**
- * Initializes a new sandboxfs process instance.
- *
- * <p>To better represent reality, this ensures that the mount point is present and valid.
- *
- * @param fileSystem file system on which the fake sandboxfs instance operates
- * @param mountPoint directory on which the sandboxfs instance is serving
- * @throws IOException if the mount point is missing
- */
- FakeSandboxfsProcess(FileSystem fileSystem, PathFragment mountPoint) throws IOException {
- if (!fileSystem.getPath(mountPoint).exists()) {
- throw new IOException("Mount point " + mountPoint + " does not exist");
- } else if (!fileSystem.getPath(mountPoint).isDirectory()) {
- throw new IOException("Mount point " + mountPoint + " is not a directory");
- }
-
- this.fileSystem = fileSystem;
- this.mountPoint = mountPoint;
- }
-
- @Override
- public Path getMountPoint() {
- return fileSystem.getPath(mountPoint);
- }
-
- @Override
- public synchronized boolean isAlive() {
- return alive;
- }
-
- @Override
- public synchronized void destroy() {
- alive = false;
- }
-
- @Override
- public synchronized void createSandbox(String name, SandboxCreator creator) throws IOException {
- checkState(alive, "Cannot be called after destroy()");
-
- checkArgument(!PathFragment.containsSeparator(name));
- checkArgument(!activeSandboxes.contains(name), "Sandbox %s mapped more than once", name);
- activeSandboxes.add(name);
-
- creator.create(
- (path, underlyingPath, writable) -> {
- checkArgument(
- path.isAbsolute(),
- "Mapping specifications are expected to be absolute but %s is not",
- path);
- Path link =
- fileSystem.getPath(mountPoint).getRelative(name).getRelative(path.toRelative());
- link.getParentDirectory().createDirectoryAndParents();
-
- Path target = fileSystem.getPath(underlyingPath);
- if (!target.exists()) {
- // Not a requirement for the creation of a symbolic link but this reflects the behavior
- // of the real sandboxfs.
- throw new IOException("Target " + underlyingPath + " does not exist");
- }
-
- if (target.isSymbolicLink()) {
- // sandboxfs is able to expose symlinks as they are in the underlying file system.
- // Mimic this behavior by respecting the symlink in that case, instead of just creating
- // a new symlink that points to the actual target.
- link.createSymbolicLink(target.readSymbolicLink());
- } else {
- link.createSymbolicLink(fileSystem.getPath(underlyingPath));
- }
- });
- }
-
- @Override
- public synchronized void destroySandbox(String name) throws IOException {
- checkState(alive, "Cannot be called after destroy()");
-
- checkArgument(!PathFragment.containsSeparator(name));
- checkArgument(activeSandboxes.contains(name), "Sandbox %s not previously created", name);
- activeSandboxes.remove(name);
-
- fileSystem.getPath(mountPoint).getRelative(name).deleteTree();
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/FakeSandboxfsProcessIntegrationTest.java b/src/test/java/com/google/devtools/build/lib/sandbox/FakeSandboxfsProcessIntegrationTest.java
deleted file mode 100644
index 683a277..0000000
--- a/src/test/java/com/google/devtools/build/lib/sandbox/FakeSandboxfsProcessIntegrationTest.java
+++ /dev/null
@@ -1,89 +0,0 @@
-// Copyright 2018 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.sandbox;
-
-import static com.google.common.truth.Truth.assertThat;
-import static org.junit.Assert.assertThrows;
-
-import com.google.devtools.build.lib.vfs.DigestHashFunction;
-import com.google.devtools.build.lib.vfs.FileSystem;
-import com.google.devtools.build.lib.vfs.FileSystemUtils;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
-import java.io.IOException;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link FakeSandboxfsProcess}. */
-@RunWith(JUnit4.class)
-public class FakeSandboxfsProcessIntegrationTest extends BaseSandboxfsProcessIntegrationTest {
-
- @Override
- Path newTmpDir() throws IOException {
- FileSystem fileSystem = new InMemoryFileSystem(DigestHashFunction.SHA256);
- Path tmpDir = fileSystem.getPath("/tmp");
- tmpDir.createDirectory();
- return tmpDir;
- }
-
- @Override
- SandboxfsProcess mount(Path mountPoint) throws IOException {
- return new FakeSandboxfsProcess(mountPoint.getFileSystem(), mountPoint.asFragment());
- }
-
- @Test
- public void testMount_notADirectory() throws IOException {
- FileSystemUtils.createEmptyFile(tmpDir.getRelative("file"));
- IOException expected = assertThrows(
- IOException.class, () -> mount(tmpDir.getRelative("file")));
- assertThat(expected).hasMessageThat().matches(".*/file.*not a directory");
- }
-
- @Test
- public void testSandboxCreate_enforcesNonRepeatedNames() throws IOException {
- Path mountPoint = tmpDir.getRelative("mnt");
- mountPoint.createDirectory();
- SandboxfsProcess process = mount(mountPoint);
-
- process.createSandbox("first", (mapper) -> {});
- process.createSandbox("second", (mapper) -> {});
- assertThrows(
- IllegalArgumentException.class, () -> process.createSandbox("first", (mapper) -> {}));
- }
-
- @Test
- public void testSandboxDestroy_enforcesExistingName() throws IOException {
- Path mountPoint = tmpDir.getRelative("mnt");
- mountPoint.createDirectory();
- SandboxfsProcess process = mount(mountPoint);
-
- process.createSandbox("first", (mapper) -> {});
- process.destroySandbox("first");
- assertThrows(IllegalArgumentException.class, () -> process.destroySandbox("first"));
- }
-
- @Test
- public void testCreateAndDestroySandbox_reuseName() throws IOException {
- Path mountPoint = tmpDir.getRelative("mnt");
- mountPoint.createDirectory();
- SandboxfsProcess process = mount(mountPoint);
-
- process.createSandbox("first", (mapper) -> {});
- process.destroySandbox("first");
- process.createSandbox("first", (mapper) -> {});
- process.destroySandbox("first");
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunnerTest.java b/src/test/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunnerTest.java
index 4f898d9..dd53636 100644
--- a/src/test/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunnerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunnerTest.java
@@ -239,8 +239,6 @@
commandEnvironment,
sandboxBase,
/*timeoutKillDelay=*/ Duration.ofSeconds(2),
- /*sandboxfsProcess=*/ null,
- /*sandboxfsMapSymlinkTargets=*/ false,
treeDeleter);
}
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedSpawnRunnerTest.java b/src/test/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedSpawnRunnerTest.java
index b646fee..8eaff5e 100644
--- a/src/test/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedSpawnRunnerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedSpawnRunnerTest.java
@@ -64,8 +64,6 @@
new SandboxHelpers(),
commandEnvironment,
sandboxBase,
- /* sandboxfsProcess= */ null,
- /* sandboxfsMapSymlinkTargets= */ false,
treeDeleter);
Spawn spawn = new SpawnBuilder("echo", "cooee").build();
@@ -120,8 +118,6 @@
new SandboxHelpers(),
commandEnvironment,
sandboxBase,
- /* sandboxfsProcess= */ null,
- /* sandboxfsMapSymlinkTargets= */ false,
treeDeleter);
Spawn spawn =
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/RealSandboxfs01ProcessTest.java b/src/test/java/com/google/devtools/build/lib/sandbox/RealSandboxfs01ProcessTest.java
deleted file mode 100644
index 11950ae..0000000
--- a/src/test/java/com/google/devtools/build/lib/sandbox/RealSandboxfs01ProcessTest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.sandbox;
-
-import com.google.common.collect.ImmutableList;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.io.IOException;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/**
- * {@inheritDoc}
- *
- * <p><b>Important:</b> If you modify any aspects of the golden data of these tests, you must run
- * the {@link RealSandboxfsProcessIntegrationTest} test against the version of sandboxfs tested here
- * to ensure that the golden data represents reality.
- */
-@RunWith(JUnit4.class)
-public class RealSandboxfs01ProcessTest extends BaseRealSandboxfsProcessTest {
-
- public RealSandboxfs01ProcessTest() {
- super("0.1.0", RealSandboxfsProcess.ALLOW_FLAG + "\n" + FAKE_MOUNT_POINT + "\n");
- }
-
- @Test
- public void testNoActivity() throws IOException {
- SandboxfsProcess process = createAndStartFakeSandboxfs(ImmutableList.of("Done"));
- String expectedRequests = "[]\n\n";
- verifyFakeSandboxfsExecution(process, expectedRequests);
- }
-
- @Test
- public void testCreateAndDestroySandboxes() throws IOException {
- SandboxfsProcess process =
- createAndStartFakeSandboxfs(ImmutableList.of("Done", "Done", "Done", "Done"));
-
- process.createSandbox("sandbox1", (mapper) -> {});
- process.createSandbox(
- "sandbox2",
- (mapper) -> mapper.map(PathFragment.create("/"), PathFragment.create("/some/path"), true));
- process.destroySandbox("sandbox1");
- String expectedRequests =
- "[]\n\n"
- + "[]\n\n"
- + "[{\"Map\":"
- + "{\"Mapping\":\"/sandbox2\",\"Target\":\"/some/path\",\"Writable\":true}}]\n\n"
- + "[{\"Unmap\":\"/sandbox1\"}]\n\n";
-
- verifyFakeSandboxfsExecution(process, expectedRequests);
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/RealSandboxfs02ProcessTest.java b/src/test/java/com/google/devtools/build/lib/sandbox/RealSandboxfs02ProcessTest.java
deleted file mode 100644
index 155fe1b..0000000
--- a/src/test/java/com/google/devtools/build/lib/sandbox/RealSandboxfs02ProcessTest.java
+++ /dev/null
@@ -1,91 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.sandbox;
-
-import com.google.common.collect.ImmutableList;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.io.IOException;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/**
- * {@inheritDoc}
- *
- * <p><b>Important:</b> If you modify any aspects of the golden data of these tests, you must run
- * the {@link RealSandboxfsProcessIntegrationTest} test against the version of sandboxfs tested here
- * to ensure that the golden data represents reality.
- */
-@RunWith(JUnit4.class)
-public class RealSandboxfs02ProcessTest extends BaseRealSandboxfsProcessTest {
-
- public RealSandboxfs02ProcessTest() {
- super("0.2.0", RealSandboxfsProcess.ALLOW_FLAG + "\n" + FAKE_MOUNT_POINT + "\n");
- }
-
- @Test
- public void testNoActivity() throws IOException {
- SandboxfsProcess process = createAndStartFakeSandboxfs(ImmutableList.of("{\"id\":\"empty\"}"));
- String expectedRequests = "{\"C\":{\"i\":\"empty\",\"m\":[]}}";
- verifyFakeSandboxfsExecution(process, expectedRequests);
- }
-
- @Test
- public void testCreateAndDestroySandboxes() throws IOException {
- SandboxfsProcess process =
- createAndStartFakeSandboxfs(
- ImmutableList.of(
- "{\"id\":\"empty\"}",
- "{\"id\":\"sandbox1\"}",
- "{\"id\":\"sandbox2\"}",
- "{\"id\":\"sandbox1\"}",
- "{\"id\":\"sandbox3\"}"));
-
- process.createSandbox("sandbox1", (mapper) -> {});
- process.createSandbox(
- "sandbox2",
- (mapper) -> {
- mapper.map(PathFragment.create("/"), PathFragment.create("/some/path"), true);
- mapper.map(PathFragment.create("/a/b/c"), PathFragment.create("/other"), false);
- });
- process.destroySandbox("sandbox1");
- process.createSandbox(
- "sandbox3",
- (mapper) -> {
- // Reuse a previous prefix.
- mapper.map(PathFragment.create("/a/b/c"), PathFragment.create("/"), true);
- // And create a new prefix to ensure identifiers are not reset.
- mapper.map(PathFragment.create("/a/b"), PathFragment.create("/"), false);
- });
- String expectedRequests =
- "{\"C\":{\"i\":\"empty\",\"m\":[]}}"
- + "{\"C\":{\"i\":\"sandbox1\",\"m\":[]}}"
- + "{\"C\":{\"i\":\"sandbox2\",\"m\":["
- + "{\"x\":1,\"p\":\"\",\"y\":2,\"u\":\"path\",\"w\":true},"
- + "{\"x\":3,\"p\":\"c\",\"y\":4,\"u\":\"other\"}"
- + "],"
- + "\"q\":{\"1\":\"/\",\"2\":\"/some\",\"3\":\"/a/b\",\"4\":\"/\"}"
- + "}}"
- + "{\"D\":\"sandbox1\"}"
- + "{\"C\":{\"i\":\"sandbox3\",\"m\":["
- + "{\"x\":3,\"p\":\"c\",\"y\":1,\"u\":\"\",\"w\":true},"
- + "{\"x\":5,\"p\":\"b\",\"y\":1,\"u\":\"\"}"
- + "],"
- + "\"q\":{\"5\":\"/a\"}"
- + "}}";
-
- verifyFakeSandboxfsExecution(process, expectedRequests);
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/RealSandboxfsProcessIntegrationTest.java b/src/test/java/com/google/devtools/build/lib/sandbox/RealSandboxfsProcessIntegrationTest.java
deleted file mode 100644
index c162449..0000000
--- a/src/test/java/com/google/devtools/build/lib/sandbox/RealSandboxfsProcessIntegrationTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-// Copyright 2018 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.sandbox;
-
-import static junit.framework.TestCase.fail;
-
-import com.google.devtools.build.lib.vfs.DigestHashFunction;
-import com.google.devtools.build.lib.vfs.FileSystem;
-import com.google.devtools.build.lib.vfs.JavaIoFileSystem;
-import com.google.devtools.build.lib.vfs.Path;
-import java.io.IOException;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link RealSandboxfsProcess}. */
-@RunWith(JUnit4.class)
-public class RealSandboxfsProcessIntegrationTest extends BaseSandboxfsProcessIntegrationTest {
-
- @Override
- Path newTmpDir() {
- String rawTmpDir = System.getenv("TEST_TMPDIR");
- if (rawTmpDir == null) {
- fail("Test requires TEST_TMPDIR to be defined in the environment");
- }
-
- FileSystem fileSystem = new JavaIoFileSystem(DigestHashFunction.SHA256);
- Path tmpDir = fileSystem.getPath(rawTmpDir);
- if (!tmpDir.isDirectory()) {
- fail("TEST_TMPDIR must point to a directory");
- }
- return tmpDir;
- }
-
- @Override
- SandboxfsProcess mount(Path mountPoint) throws IOException {
- String rawSandboxfs = System.getenv("SANDBOXFS");
- if (rawSandboxfs == null) {
- fail("Test requires SANDBOXFS to be defined in the environment");
- }
-
- FileSystem fileSystem = new JavaIoFileSystem(DigestHashFunction.SHA256);
- Path sandboxfs = fileSystem.getPath(rawSandboxfs);
- if (!sandboxfs.isExecutable()) {
- fail("SANDBOXFS must point to an executable binary");
- }
- return RealSandboxfsProcess.mount(
- sandboxfs.asFragment(), mountPoint, fileSystem.getPath("/dev/stderr"));
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/SandboxfsSandboxedSpawnTest.java b/src/test/java/com/google/devtools/build/lib/sandbox/SandboxfsSandboxedSpawnTest.java
deleted file mode 100644
index b3b0c4a..0000000
--- a/src/test/java/com/google/devtools/build/lib/sandbox/SandboxfsSandboxedSpawnTest.java
+++ /dev/null
@@ -1,313 +0,0 @@
-// Copyright 2018 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package com.google.devtools.build.lib.sandbox;
-
-import static com.google.common.truth.Truth.assertThat;
-import static org.junit.Assert.assertThrows;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxInputs;
-import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxOutputs;
-import com.google.devtools.build.lib.testutil.TestUtils;
-import com.google.devtools.build.lib.vfs.DigestHashFunction;
-import com.google.devtools.build.lib.vfs.FileSystem;
-import com.google.devtools.build.lib.vfs.FileSystemUtils;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.devtools.build.lib.vfs.Root;
-import com.google.devtools.build.lib.vfs.RootedPath;
-import com.google.devtools.build.lib.vfs.Symlinks;
-import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
-import java.io.IOException;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link SandboxfsSandboxedSpawn}. */
-@RunWith(JUnit4.class)
-public class SandboxfsSandboxedSpawnTest {
- private Path testRoot;
- private Path workspaceDir;
- private Path outerDir;
- private SandboxfsProcess sandboxfs;
-
- @Before
- public final void setupTestDirs() throws IOException {
- FileSystem fileSystem = new InMemoryFileSystem(DigestHashFunction.SHA256);
- testRoot = TestUtils.createUniqueTmpDir(fileSystem);
-
- workspaceDir = testRoot.getRelative("workspace");
- workspaceDir.createDirectory();
- outerDir = testRoot.getRelative("scratch");
- outerDir.createDirectory();
-
- Path mountPoint = testRoot.getRelative("sandbox");
- mountPoint.createDirectory();
- sandboxfs = new FakeSandboxfsProcess(
- mountPoint.getFileSystem(), mountPoint.asFragment());
- }
-
- @Test
- public void testCreateFileSystem() throws Exception {
- RootedPath helloTxt =
- RootedPath.toRootedPath(Root.fromPath(workspaceDir), PathFragment.create("hello.txt"));
- FileSystemUtils.createEmptyFile(helloTxt.asPath());
-
- SandboxedSpawn spawn =
- new SandboxfsSandboxedSpawn(
- sandboxfs,
- outerDir,
- "workspace",
- ImmutableList.of("/bin/true"),
- ImmutableMap.of(),
- new SandboxInputs(
- ImmutableMap.of(PathFragment.create("such/input.txt"), helloTxt),
- ImmutableMap.of(),
- ImmutableMap.of(),
- ImmutableMap.of()),
- SandboxOutputs.create(
- ImmutableSet.of(PathFragment.create("very/output.txt")), ImmutableSet.of()),
- ImmutableSet.of(PathFragment.create("wow/writable")),
- /* mapSymlinkTargets= */ false,
- new SynchronousTreeDeleter(),
- "Mnemonic",
- /* sandboxDebugPath= */ null,
- /* statisticsPath= */ null);
-
- spawn.createFileSystem();
- Path execRoot = spawn.getSandboxExecRoot();
-
- assertThat(execRoot.getRelative("such/input.txt").isSymbolicLink()).isTrue();
- assertThat(execRoot.getRelative("such/input.txt").resolveSymbolicLinks())
- .isEqualTo(helloTxt.asPath());
- assertThat(execRoot.getRelative("very").isDirectory()).isTrue();
- assertThat(execRoot.getRelative("wow/writable").isDirectory()).isTrue();
- }
-
- @Test
- public void testExecRootContainsWorkspaceName() throws Exception {
- Path helloTxt = workspaceDir.getRelative("hello.txt");
- FileSystemUtils.createEmptyFile(helloTxt);
-
- SandboxedSpawn spawn =
- new SandboxfsSandboxedSpawn(
- sandboxfs,
- outerDir,
- "some-workspace-name",
- ImmutableList.of("/bin/true"),
- ImmutableMap.of(),
- new SandboxInputs(
- ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of()),
- SandboxOutputs.create(ImmutableSet.of(), ImmutableSet.of()),
- ImmutableSet.of(),
- /* mapSymlinkTargets= */ false,
- new SynchronousTreeDeleter(),
- "Mnemonic",
- /* sandboxDebugPath= */ null,
- /* statisticsPath= */ null);
- spawn.createFileSystem();
- Path execRoot = spawn.getSandboxExecRoot();
-
- assertThat(execRoot.getPathString()).contains("/some-workspace-name");
- }
-
- @Test
- public void testDelete() throws Exception {
- RootedPath helloTxt =
- RootedPath.toRootedPath(Root.fromPath(workspaceDir), PathFragment.create("hello.txt"));
- FileSystemUtils.createEmptyFile(helloTxt.asPath());
-
- SandboxedSpawn spawn =
- new SandboxfsSandboxedSpawn(
- sandboxfs,
- outerDir,
- "workspace",
- ImmutableList.of("/bin/true"),
- ImmutableMap.of(),
- new SandboxInputs(
- ImmutableMap.of(PathFragment.create("such/input.txt"), helloTxt),
- ImmutableMap.of(),
- ImmutableMap.of(),
- ImmutableMap.of()),
- SandboxOutputs.create(
- ImmutableSet.of(PathFragment.create("very/output.txt")), ImmutableSet.of()),
- ImmutableSet.of(PathFragment.create("wow/writable")),
- /* mapSymlinkTargets= */ false,
- new SynchronousTreeDeleter(),
- "Mnemonic",
- /* sandboxDebugPath= */ null,
- /* statisticsPath= */ null);
- spawn.createFileSystem();
- Path execRoot = spawn.getSandboxExecRoot();
-
- // Pretend to do some work inside the execRoot.
- execRoot.getRelative("tempdir").createDirectory();
- FileSystemUtils.createEmptyFile(execRoot.getRelative("very/output.txt"));
- FileSystemUtils.createEmptyFile(execRoot.getRelative("wow/writable/temp.txt"));
-
- spawn.delete();
-
- assertThat(execRoot.exists()).isFalse();
- }
-
- @Test
- public void testCopyOutputs() throws Exception {
- // These tests are very simple because we just rely on
- // AbstractContainerizingSandboxedSpawnTest.testMoveOutputs to properly verify all corner cases.
- PathFragment outputFile = PathFragment.create("very/output.txt");
-
- SandboxedSpawn spawn =
- new SandboxfsSandboxedSpawn(
- sandboxfs,
- outerDir,
- "workspace",
- ImmutableList.of("/bin/true"),
- ImmutableMap.of(),
- new SandboxInputs(
- ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of()),
- SandboxOutputs.create(ImmutableSet.of(outputFile), ImmutableSet.of()),
- ImmutableSet.of(),
- /* mapSymlinkTargets= */ false,
- new SynchronousTreeDeleter(),
- "Mnemonic",
- /* sandboxDebugPath= */ null,
- /* statisticsPath= */ null);
- spawn.createFileSystem();
- Path execRoot = spawn.getSandboxExecRoot();
-
- FileSystemUtils.createEmptyFile(execRoot.getRelative(outputFile));
-
- Path outputsDir = testRoot.getRelative("outputs");
- outputsDir.getRelative(outputFile.getParentDirectory()).createDirectoryAndParents();
- spawn.copyOutputs(outputsDir);
-
- assertThat(outputsDir.getRelative(outputFile).isFile(Symlinks.NOFOLLOW)).isTrue();
- }
-
- public void testSymlinks(boolean mapSymlinkTargets) throws Exception {
- Root workspaceRoot = Root.fromPath(workspaceDir);
- RootedPath input1 =
- RootedPath.toRootedPath(workspaceRoot, PathFragment.create("dir1/input-1.txt"));
- input1.asPath().getParentDirectory().createDirectory();
- FileSystemUtils.createEmptyFile(input1.asPath());
-
- RootedPath input2 =
- RootedPath.toRootedPath(workspaceRoot, PathFragment.create("dir1/input-2.txt"));
- input2.asPath().getParentDirectory().createDirectory();
- FileSystemUtils.createEmptyFile(input2.asPath());
-
- RootedPath linkToInput1 =
- RootedPath.toRootedPath(workspaceRoot, PathFragment.create("dir2/link-to-input-1"));
- linkToInput1.asPath().getParentDirectory().createDirectory();
- linkToInput1.asPath().createSymbolicLink(PathFragment.create("../dir1/input-1.txt"));
- assertThat(linkToInput1.asPath().readSymbolicLink().isAbsolute()).isFalse();
-
- RootedPath linkToInput2 =
- RootedPath.toRootedPath(workspaceRoot, PathFragment.create("dir2/link-to-input-2"));
- linkToInput2.asPath().getParentDirectory().createDirectory();
- linkToInput2.asPath().createSymbolicLink(PathFragment.create("../dir1/input-2.txt"));
- assertThat(linkToInput2.asPath().readSymbolicLink().isAbsolute()).isFalse();
-
- RootedPath linkToLink =
- RootedPath.toRootedPath(workspaceRoot, PathFragment.create("dir2/link-to-link"));
- linkToLink.asPath().getParentDirectory().createDirectory();
- linkToLink.asPath().createSymbolicLink(PathFragment.create("link-to-input-2"));
- assertThat(linkToLink.asPath().readSymbolicLink().isAbsolute()).isFalse();
-
- RootedPath linkToAbsolutePath =
- RootedPath.toRootedPath(workspaceRoot, PathFragment.create("dir2/link-to-absolute-path"));
- linkToAbsolutePath.getParentDirectory().asPath().createDirectory();
- Path randomPath = workspaceDir.getRelative("/some-random-path");
- FileSystemUtils.createEmptyFile(randomPath);
- linkToAbsolutePath.asPath().createSymbolicLink(randomPath.asFragment());
- assertThat(linkToAbsolutePath.asPath().readSymbolicLink().isAbsolute()).isTrue();
-
- SandboxedSpawn spawn =
- new SandboxfsSandboxedSpawn(
- sandboxfs,
- outerDir,
- "workspace",
- ImmutableList.of("/bin/true"),
- ImmutableMap.of(),
- new SandboxInputs(
- ImmutableMap.of(
- PathFragment.create("dir1/input-1.txt"), input1,
- // input2 and linkToInput2 intentionally left unmapped to verify they are mapped
- // as symlink targets of linktoLink.
- PathFragment.create("such/link-1.txt"), linkToInput1,
- PathFragment.create("such/link-to-link.txt"), linkToLink,
- PathFragment.create("such/abs-link.txt"), linkToAbsolutePath),
- ImmutableMap.of(),
- ImmutableMap.of(),
- ImmutableMap.of()),
- SandboxOutputs.create(
- ImmutableSet.of(PathFragment.create("very/output.txt")), ImmutableSet.of()),
- ImmutableSet.of(),
- mapSymlinkTargets,
- new SynchronousTreeDeleter(),
- "Mnemonic",
- /* sandboxDebugPath= */ null,
- /* statisticsPath= */ null);
-
- spawn.createFileSystem();
- Path execRoot = spawn.getSandboxExecRoot();
-
- // Relative symlinks must be kept as such in the sandbox and they must resolve properly.
- assertThat(execRoot.getRelative("such/link-1.txt").readSymbolicLink())
- .isEqualTo(PathFragment.create("../dir1/input-1.txt"));
- assertThat(execRoot.getRelative("such/link-1.txt").resolveSymbolicLinks())
- .isEqualTo(input1.asPath());
- assertThat(execRoot.getRelative("such/link-to-link.txt").readSymbolicLink())
- .isEqualTo(PathFragment.create("link-to-input-2"));
- if (mapSymlinkTargets) {
- assertThat(execRoot.getRelative("such/link-to-link.txt").resolveSymbolicLinks())
- .isEqualTo(input2.asPath());
- assertThat(execRoot.getRelative("such/link-to-input-2").readSymbolicLink())
- .isEqualTo(PathFragment.create("../dir1/input-2.txt"));
- assertThat(execRoot.getRelative("such/link-to-input-2").resolveSymbolicLinks())
- .isEqualTo(input2.asPath());
- } else {
- assertThrows(
- "Symlink resolution worked, which means the target was mapped when not expected",
- IOException.class,
- () -> execRoot.getRelative("such/link-to-link.txt").resolveSymbolicLinks());
- }
-
- // Targets of symlinks must have been mapped inside the sandbox only when requested.
- assertThat(execRoot.getRelative("dir1/input-1.txt").exists()).isTrue();
- if (mapSymlinkTargets) {
- assertThat(execRoot.getRelative("dir1/input-2.txt").exists()).isTrue();
- } else {
- assertThat(execRoot.getRelative("dir1/input-2.txt").exists()).isFalse();
- }
-
- // Absolute symlinks must be kept as such in the sandbox no matter where they point to.
- assertThat(execRoot.getRelative("such/abs-link.txt").isSymbolicLink()).isTrue();
- assertThat(execRoot.getRelative("such/abs-link.txt").readSymbolicLinkUnchecked())
- .isEqualTo(randomPath.asFragment());
- }
-
- @Test
- public void testSymlinks_targetsMappedIfRequested() throws Exception {
- testSymlinks(true);
- }
-
- @Test
- public void testSymlinks_targetsNotMappedIfNotRequested() throws Exception {
- testSymlinks(false);
- }
-}
diff --git a/src/test/shell/integration/BUILD b/src/test/shell/integration/BUILD
index 8617fe6..251a1e0 100644
--- a/src/test/shell/integration/BUILD
+++ b/src/test/shell/integration/BUILD
@@ -714,14 +714,6 @@
)
sh_test(
- name = "sandboxfs_test",
- size = "medium",
- srcs = ["sandboxfs_test.sh"],
- data = [":test-deps"],
- tags = ["no_windows"],
-)
-
-sh_test(
name = "bazel_java_test",
size = "medium",
srcs = ["bazel_java_test.sh"],
diff --git a/src/test/shell/integration/sandboxfs_test.sh b/src/test/shell/integration/sandboxfs_test.sh
deleted file mode 100755
index 0b6fe49..0000000
--- a/src/test/shell/integration/sandboxfs_test.sh
+++ /dev/null
@@ -1,260 +0,0 @@
-#!/bin/bash
-# Copyright 2018 The Bazel Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-source "${CURRENT_DIR}/../integration_test_setup.sh" \
- || { echo "integration_test_setup.sh not found!" >&2; exit 1; }
-
-# Bazel build arguments to disable the use of the sandbox. We have tests below
-# that configure a fake sandboxfs and they would fail if they were to use it.
-DISABLE_SANDBOX_ARGS=(
- --genrule_strategy=local
- --spawn_strategy=local
-)
-
-# Creates a fake sandboxfs process in "path" that logs interactions with it in
-# the given "log" file and reports the given "version".
-function create_fake_sandboxfs() {
- local path="${1}"; shift
- local log="${1}"; shift
- local version="${1}"; shift
-
- cat >"${path}" <<EOF
-#!/bin/sh
-
-rm -f "${log}"
-trap 'echo "Terminated" >>"${log}"' EXIT TERM
-
-echo "PID: \${$}" >>"${log}"
-echo "ARGS: \${*}" >>"${log}"
-
-if [ "\${1}" = --version ]; then
- echo "${version}"
- exit 0
-fi
-
-while read line; do
- echo "Received: \${line}" >>"${log}"
- if [ -z "\${line}" ]; then
- echo "Done"
- fi
-done
-EOF
- chmod +x "${path}"
-}
-
-function create_hello_package() {
- mkdir -p hello
-
- cat >hello/BUILD <<EOF
-cc_binary(name = "hello", srcs = ["hello.cc"])
-EOF
-
- cat >hello/hello.cc <<EOF
-#include <stdio.h>
-int main(void) { printf("Hello, world!\n"); return 0; }
-EOF
-}
-
-function test_default_sandboxfs_from_path() {
- mkdir -p fake-tools
- create_fake_sandboxfs fake-tools/sandboxfs "$(pwd)/log" "sandboxfs 0.0.0"
- PATH="$(pwd)/fake-tools:${PATH}"; export PATH
-
- create_hello_package
-
- # This test relies on a PATH change that is only recognized when the server
- # first starts up, so ensure there are no Bazel servers left behind.
- #
- # TODO(philwo): This is awful. The testing infrastructure should ensure
- # that tests cannot pollute each other's state, but at the moment that's not
- # the case.
- bazel shutdown
-
- bazel build \
- "${DISABLE_SANDBOX_ARGS[@]}" \
- --experimental_use_sandboxfs \
- //hello >"${TEST_log}" 2>&1 || fail "Build should have succeeded"
-
- # Dump fake sandboxfs' log for debugging.
- sed -e 's,^,SANDBOXFS: ,' log >>"${TEST_log}"
-
- grep -q "Terminated" log \
- || fail "sandboxfs process was not terminated (not executed?)"
-}
-
-function test_explicit_sandboxfs_not_found() {
- create_hello_package
-
- bazel build \
- --experimental_use_sandboxfs \
- --experimental_sandboxfs_path="/non-existent/sandboxfs" \
- //hello >"${TEST_log}" 2>&1 && fail "Build succeeded but should have failed"
-
- expect_log "Failed to get sandboxfs version.*/non-existent/sandboxfs"
-}
-
-function test_explicit_sandboxfs_is_invalid() {
- mkdir -p fake-tools
- create_hello_package
- do_build() {
- bazel build \
- "${DISABLE_SANDBOX_ARGS[@]}" \
- --experimental_use_sandboxfs=yes \
- --experimental_sandboxfs_path="$(pwd)/fake-tools/sandboxfs" \
- //hello
- }
-
- # Try with a binary that prints an invalid version string.
- create_fake_sandboxfs fake-tools/sandboxfs "$(pwd)/log" "not-sandboxfs 0.0.0"
- do_build >"${TEST_log}" 2>&1 && fail "Build should have failed"
-
- # Now try with a valid binary just to ensure our test scenario works.
- create_fake_sandboxfs fake-tools/sandboxfs "$(pwd)/log" "sandboxfs 0.0.0"
- do_build >"${TEST_log}" 2>&1 || fail "Build should have succeeded"
- sed -e 's,^,SANDBOXFS: ,' log >>"${TEST_log}"
-
- grep -q "Terminated" log \
- || fail "sandboxfs process was not terminated (not executed?)"
-}
-
-function test_use_sandboxfs_if_present() {
- # This test relies on a PATH change that is only recognized when the server
- # first starts up, so ensure there are no Bazel servers left behind.
- #
- # TODO(philwo): This is awful. The testing infrastructure should ensure
- # that tests cannot pollute each other's state, but at the moment that's not
- # the case.
- bazel shutdown
-
- mkdir -p fake-tools
- PATH="$(pwd)/fake-tools:${PATH}"; export PATH
- create_hello_package
- do_build() {
- bazel build \
- "${DISABLE_SANDBOX_ARGS[@]}" \
- --experimental_use_sandboxfs=auto \
- //hello
- }
-
- # Try with sandboxfs not in the PATH.
- do_build >"${TEST_log}" 2>&1 || fail "Build should have succeeded"
- [[ ! -f log ]] || echo "sandboxfs was used but should not have"
-
- # Now try with sandboxfs in the PATH to ensure our test scenario works.
- create_fake_sandboxfs fake-tools/sandboxfs "$(pwd)/log" "sandboxfs 0.0.0"
- do_build >"${TEST_log}" 2>&1 || fail "Build should have succeeded"
- sed -e 's,^,SANDBOXFS: ,' log >>"${TEST_log}"
- grep -q "Terminated" log \
- || fail "sandboxfs process was not terminated (not executed?)"
-}
-
-# Runs a build of the given target using a fake sandboxfs that captures its
-# activity and dumps it to the given log file.
-function build_with_fake_sandboxfs() {
- local log="${1}"; shift
-
- create_fake_sandboxfs fake-sandboxfs.sh "${log}" "sandboxfs 0.0.0"
-
- local ret=0
- bazel build \
- "${DISABLE_SANDBOX_ARGS[@]}" \
- --experimental_use_sandboxfs \
- --experimental_sandboxfs_path="$(pwd)/fake-sandboxfs.sh" \
- "${@}" >"${TEST_log}" 2>&1 || ret="${?}"
-
- # Dump fake sandboxfs' log for debugging.
- sed -e 's,^,SANDBOXFS: ,' log >>"${TEST_log}"
-
- return "${ret}"
-}
-
-function test_mount_unmount() {
- create_hello_package
-
- local output_base="$(bazel info output_base)"
- local sandbox_base="${output_base}/sandbox"
-
- build_with_fake_sandboxfs "$(pwd)/log" //hello \
- || fail "Build should have succeeded"
-
- grep -q "ARGS: .*${sandbox_base}/sandboxfs" log \
- || fail "Cannot find expected mount point in sandboxfs mount call"
- grep -q "Terminated" log \
- || fail "sandboxfs process was not terminated (not unmounted?)"
-}
-
-function test_debug_lifecycle() {
- create_hello_package
-
- function sandboxfs_pid() {
- case "$(uname)" in
- Darwin)
- # We cannot use ps to look for the sandbox process because this is
- # not allowed when running with macOS's App Sandboxing.
- grep -q "Terminated" log && return
- grep "^PID:" log | awk '{print $2}'
- ;;
-
- *)
- # We could use the same approach we follow on Darwin to look for the
- # PID of the subprocess, but it's better if we look at the real
- # process table if we are able to.
- ps ax | grep [f]ake-sandboxfs | awk '{print $1}'
- ;;
- esac
- }
-
- # Want sandboxfs to be left mounted after a build with debugging on.
- build_with_fake_sandboxfs "$(pwd)/log" --sandbox_debug //hello
- grep -q "ARGS:" log || fail "sandboxfs was not run"
- grep -q "Terminated" log \
- && fail "sandboxfs process was terminated but should not have been"
- local pid1="$(sandboxfs_pid)"
- [[ -n "${pid1}" ]] || fail "sandboxfs process not found in process table"
-
- # Want sandboxfs to be restarted if the previous build had debugging on.
- build_with_fake_sandboxfs "$(pwd)/log" --sandbox_debug //hello
- local pid2="$(sandboxfs_pid)"
- [[ -n "${pid2}" ]] || fail "sandboxfs process not found in process table"
- [[ "${pid1}" -ne "${pid2}" ]] || fail "sandboxfs was not restarted"
-
- # Want build to finish successfully and to clear the mount point.
- build_with_fake_sandboxfs "$(pwd)/log" --nosandbox_debug //hello
- local pid3="$(sandboxfs_pid)"
- [[ -z "${pid3}" ]] || fail "sandboxfs was not terminated"
-}
-
-function test_always_unmounted_on_exit() {
- create_hello_package
-
- # Want sandboxfs to be left mounted after a build with debugging on.
- build_with_fake_sandboxfs "$(pwd)/log" --sandbox_debug //hello
- grep -q "ARGS:" log || fail "sandboxfs was not run"
- grep -q "Terminated" log \
- && fail "sandboxfs process was terminated but should not have been"
-
- # Want Bazel to unmount the sandboxfs instance on exit no matter what.
- #
- # Note that we do not even tell Bazel where the sandboxfs binary lives
- # but we expect changes to the log of the currently-running sandboxfs
- # binary. This is intentional to verify that the already-mounted
- # instance is the one shut down.
- bazel shutdown
- grep -q "Terminated" log \
- || fail "sandboxfs process was not terminated but should have been"
-}
-
-run_suite "sandboxfs-based sandboxing tests"
