src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java - bazel - Git at Google

 // Copyright 2015 The Bazel Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package com.google.devtools.build.lib.sandbox;

 import static com.google.common.base.Preconditions.checkNotNull;

 import com.google.common.base.Joiner;
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.eventbus.Subscribe;
 import com.google.common.flogger.GoogleLogger;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.exec.ExecutionOptions;
 import com.google.devtools.build.lib.exec.SpawnRunner;
 import com.google.devtools.build.lib.exec.SpawnStrategyRegistry;
 import com.google.devtools.build.lib.exec.TreeDeleter;
 import com.google.devtools.build.lib.exec.local.LocalExecutionOptions;
 import com.google.devtools.build.lib.profiler.Profiler;
 import com.google.devtools.build.lib.profiler.SilentCloseable;
 import com.google.devtools.build.lib.runtime.BlazeModule;
 import com.google.devtools.build.lib.runtime.Command;
 import com.google.devtools.build.lib.runtime.CommandEnvironment;
 import com.google.devtools.build.lib.runtime.commands.events.CleanStartingEvent;
 import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
 import com.google.devtools.build.lib.server.FailureDetails.Sandbox;
 import com.google.devtools.build.lib.util.AbruptExitException;
 import com.google.devtools.build.lib.util.DetailedExitCode;
 import com.google.devtools.build.lib.util.Fingerprint;
 import com.google.devtools.build.lib.util.OS;
 import com.google.devtools.build.lib.vfs.FileSystem;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import com.google.devtools.common.options.OptionsBase;
 import com.google.devtools.common.options.TriState;
 import java.io.File;
 import java.io.IOException;
 import java.time.Duration;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;

 /** This module provides the Sandbox spawn strategy. */
 public final class SandboxModule extends BlazeModule {

   private static final String MAC_INDEX_FILE = ".DS_Store";

   private static final ImmutableSet<String> SANDBOX_BASE_PERSISTENT_DIRS =
       ImmutableSet.of(
           MAC_INDEX_FILE,
           SandboxStash.SANDBOX_STASH_BASE,
           SandboxStash.TEMPORARY_SANDBOX_STASH_BASE,
           AsynchronousTreeDeleter.MOVED_TRASH_DIR);

   private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();

   /** Tracks whether we are issuing the very first build within this Bazel server instance. */
   private static boolean firstBuild = true;

   /** Environment for the running command. */
   @Nullable private CommandEnvironment env;

   /** Path to the location of the sandboxes. */
   @Nullable private Path sandboxBase;

   /**
    * Collection of spawn runner instantiated during the executor setup.
    *
    * <p>We need this information to clean up the heavy subdirectories of the sandbox base on build
    * completion but to avoid wiping the whole sandbox base itself, which could be problematic across
    * builds.
    */
   private final Set<SpawnRunner> spawnRunners = new HashSet<>();

   /**
    * Handler to process expensive tree deletions, potentially outside of the critical path.
    *
    * <p>Sandboxing creates one separate tree for each action, and this tree is used to run the
    * action commands in. These trees are disjoint for all actions and have unique identifiers.
    * Therefore, there is no need for their deletion (which can be very expensive) to happen in the
    * critical path -- so if the user so wishes, we process those deletions asynchronously.
    */
   @Nullable private TreeDeleter treeDeleter;

   /**
    * Whether to remove the sandbox worker directories after a build or not. Useful for debugging to
    * inspect the state of files on failures.
    */
   private boolean shouldCleanupSandboxBase;

   @Override
   public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
     return "build".equals(command.name())
         ? ImmutableList.of(SandboxOptions.class)
         : ImmutableList.of();
   }

   /** Computes the path to the sandbox base tree for the given running command. */
   private static Path computeSandboxBase(SandboxOptions options, CommandEnvironment env)
       throws IOException {
     if (options.sandboxBase.isEmpty()) {
       return env.getOutputBase().getRelative("sandbox");
     } else {
       String dirName =
           String.format(
               "%s-sandbox.%s",
               env.getRuntime().getProductName(),
               Fingerprint.getHexDigest(env.getOutputBase().toString()));
       FileSystem fileSystem = env.getRuntime().getFileSystem();
       if (OS.getCurrent() == OS.DARWIN) {
         // Don't resolve symlinks on macOS: See https://github.com/bazelbuild/bazel/issues/13766
         return fileSystem.getPath(options.sandboxBase).getRelative(dirName);
       }
       Path resolvedSandboxBase = fileSystem.getPath(options.sandboxBase).resolveSymbolicLinks();
       return resolvedSandboxBase.getRelative(dirName);
     }
   }

   @Override
   public void beforeCommand(CommandEnvironment env) {
     // We can't assert that env is null because the Blaze runtime does not guarantee that
     // afterCommand() will be called if the command fails due to, e.g. a syntax error.
     this.env = env;
     env.getEventBus().register(this);

     // Don't attempt cleanup unless the executor is initialized.
     shouldCleanupSandboxBase = false;
   }

   @Override
   public void registerSpawnStrategies(
       SpawnStrategyRegistry.Builder registryBuilder, CommandEnvironment env)
       throws AbruptExitException, InterruptedException {
     checkNotNull(env, "env not initialized; was beforeCommand called?");
     try {
       setup(env, registryBuilder);
     } catch (IOException e) {
       throw new AbruptExitException(
           DetailedExitCode.of(
               FailureDetail.newBuilder()
                   .setMessage(String.format("Failed to initialize sandbox: %s", e.getMessage()))
                   .setSandbox(
                       Sandbox.newBuilder().setCode(Sandbox.Code.INITIALIZATION_FAILURE).build())
                   .build()),
           e);
     }
   }

   /**
    * Returns true if windows-sandbox should be used for this build.
    *
    * <p>Returns true if requested in ["auto", "yes"] and binary is valid. Throws an error if state
    * is "yes" and binary is not valid.
    *
    * @param requested whether windows-sandbox use was requested or not
    * @param binary path of the windows-sandbox binary to use, can be absolute or relative path
    * @return true if windows-sandbox can and should be used; false otherwise
    * @throws IOException if there are problems trying to determine the status of windows-sandbox
    */
   private static boolean shouldUseWindowsSandbox(
       TriState requested, PathFragment binary, ImmutableMap<String, String> clientEnv)
       throws IOException {
     return switch (requested) {
       case AUTO -> WindowsSandboxUtil.isAvailable(binary, clientEnv);
       case NO -> false;
       case YES -> {
         if (!WindowsSandboxUtil.isAvailable(binary, clientEnv)) {
           throw new IOException(
               "windows-sandbox explicitly requested but \""
                   + binary
                   + "\" could not be found or is not valid");
         }
         yield true;
       }
     };
   }

   private void setup(CommandEnvironment cmdEnv, SpawnStrategyRegistry.Builder builder)
       throws IOException, InterruptedException {
     SandboxOptions options = checkNotNull(env.getOptions().getOptions(SandboxOptions.class));
     sandboxBase = computeSandboxBase(options, env);
     Path trashBase = sandboxBase.getRelative(AsynchronousTreeDeleter.MOVED_TRASH_DIR);

     // Do not remove the sandbox base when --sandbox_debug was specified so that people can check
     // out the contents of the generated sandbox directories.
     shouldCleanupSandboxBase = !options.sandboxDebug;

     // If there happens to be any live tree deleter from a previous build and it's different than
     // the one we want now, leave it alone (i.e. don't attempt to wait for pending deletions). Its
     // deletions shouldn't overlap any new directories we create during this build (because the
     // identifiers in the subdirectories will be different).
     if (options.asyncTreeDeleteIdleThreads == 0) {
       if (!(treeDeleter instanceof SynchronousTreeDeleter)) {
         treeDeleter = new SynchronousTreeDeleter();
       }
     } else {
       if (!(treeDeleter instanceof AsynchronousTreeDeleter treeDeleter)
           || !treeDeleter.getTrashBase().equals(trashBase)) {
         if (treeDeleter != null) {
           treeDeleter.shutdown();
         }
         treeDeleter = new AsynchronousTreeDeleter(trashBase);
         firstBuild = true;
       }
     }
     try (SilentCloseable c = Profiler.instance().profile("SandboxStash.initialize")) {
       SandboxStash.initialize(env.getWorkspaceName(), sandboxBase, options, treeDeleter);
     }

     // SpawnExecutionPolicy#getId returns unique base directories for each sandboxed action during
     // the life of a Bazel server instance so we don't need to worry about stale directories from
     // previous builds. However, on the very first build of an instance of the server, we must
     // wipe old contents to avoid reusing stale directories.
     if (firstBuild && sandboxBase.exists()) {
       try (SilentCloseable c = Profiler.instance().profile("clean sandbox on first build")) {
         if (trashBase.exists()) {
           // Delete stale trash from a previous server instance.
           Path staleTrash = getStaleTrashDir(trashBase);
           trashBase.renameTo(staleTrash);
           trashBase.createDirectory();
           treeDeleter.deleteTree(staleTrash);
         } else {
           trashBase.createDirectory();
         }
         // We can delete other dirs asynchronously (if the flag is on).
         for (Path entry : sandboxBase.getDirectoryEntries()) {
           if (entry.getBaseName().equals(AsynchronousTreeDeleter.MOVED_TRASH_DIR)) {
             continue;
           }
           if (entry.getBaseName().equals(SandboxHelpers.INACCESSIBLE_HELPER_DIR)) {
             entry.deleteTree();
           } else if (entry.isDirectory()) {
             treeDeleter.deleteTree(entry);
           } else {
             entry.delete();
           }
         }
       } catch (IOException e) {
         // We have observed asynchronous deletion failing when running Bazel under Docker, see
         // #21719. Different RUN commands with `bazel build` will write to different layers in the
         // docker image. The overlay filesystem is different and the renaming of the directories
         // that we need to do for asynchronous deletion will fail. When that happens we fall back to
         // synchronous deletion here.
         sandboxBase.deleteTree();
       }
     }
     firstBuild = false;
     sandboxBase.createDirectoryAndParents();
     trashBase.createDirectory();

     PathFragment windowsSandboxPath = PathFragment.create(options.windowsSandboxPath);
     boolean windowsSandboxSupported;
     try (SilentCloseable c = Profiler.instance().profile("shouldUseWindowsSandbox")) {
       windowsSandboxSupported =
           shouldUseWindowsSandbox(
               options.useWindowsSandbox, windowsSandboxPath, cmdEnv.getClientEnv());
     }

     Duration timeoutKillDelay =
         cmdEnv.getOptions().getOptions(LocalExecutionOptions.class).getLocalSigkillGraceSeconds();

     boolean processWrapperSupported = ProcessWrapperSandboxedSpawnRunner.isSupported(cmdEnv);
     boolean linuxSandboxSupported = LinuxSandboxedSpawnRunner.isSupported(cmdEnv);
     boolean darwinSandboxSupported = DarwinSandboxedSpawnRunner.isSupported(cmdEnv);

     ExecutionOptions executionOptions =
         checkNotNull(cmdEnv.getOptions().getOptions(ExecutionOptions.class));
     // This works on most platforms, but isn't the best choice, so we put it first and let later
     // platform-specific sandboxing strategies become the default.
     if (processWrapperSupported) {
       SpawnRunner spawnRunner =
           new ProcessWrapperSandboxedSpawnRunner(cmdEnv, sandboxBase, treeDeleter);
       spawnRunners.add(spawnRunner);
       builder.registerStrategy(
           new ProcessWrapperSandboxedStrategy(spawnRunner, executionOptions),
           "sandboxed",
           "processwrapper-sandbox");
     }

     if (options.enableDockerSandbox) {
       // This strategy uses Docker to execute spawns. It should work on all platforms that support
       // Docker.
       Path pathToDocker = getPathToDockerClient(cmdEnv);
       // DockerSandboxedSpawnRunner.isSupported is expensive! It runs docker as a subprocess, and
       // docker hangs sometimes.
       if (pathToDocker != null && DockerSandboxedSpawnRunner.isSupported(cmdEnv, pathToDocker)) {
         String defaultImage = options.dockerImage;
         boolean useCustomizedImages = options.dockerUseCustomizedImages;
         SpawnRunner spawnRunner =
             new DockerSandboxedSpawnRunner(
                 cmdEnv, pathToDocker, sandboxBase, defaultImage, useCustomizedImages, treeDeleter);
         spawnRunners.add(spawnRunner);
         builder.registerStrategy(
             new DockerSandboxedStrategy(spawnRunner, executionOptions), "docker");
       }
     } else if (options.dockerVerbose) {
       cmdEnv
           .getReporter()
           .handle(
               Event.info(
                   "Docker sandboxing disabled. Use the '--experimental_enable_docker_sandbox'"
                       + " command line option to enable it"));
     }

     // This is the preferred sandboxing strategy on Linux.
     if (linuxSandboxSupported) {
       SpawnRunner spawnRunner =
           LinuxSandboxedStrategy.create(
               cmdEnv, sandboxBase, timeoutKillDelay, treeDeleter, options);
       spawnRunners.add(spawnRunner);
       builder.registerStrategy(
           new LinuxSandboxedStrategy(spawnRunner, executionOptions), "sandboxed", "linux-sandbox");
     }

     // This is the preferred sandboxing strategy on macOS.
     if (darwinSandboxSupported) {
       SpawnRunner spawnRunner = new DarwinSandboxedSpawnRunner(cmdEnv, sandboxBase, treeDeleter);
       spawnRunners.add(spawnRunner);
       builder.registerStrategy(
           new DarwinSandboxedStrategy(spawnRunner, executionOptions),
           "sandboxed",
           "darwin-sandbox");
     }

     if (windowsSandboxSupported) {
       SpawnRunner spawnRunner =
           new WindowsSandboxedSpawnRunner(cmdEnv, timeoutKillDelay, windowsSandboxPath);
       spawnRunners.add(spawnRunner);
       builder.registerStrategy(
           new WindowsSandboxedStrategy(spawnRunner, executionOptions),
           "sandboxed",
           "windows-sandbox");
     }

     if (processWrapperSupported
         || linuxSandboxSupported
         || darwinSandboxSupported
         || windowsSandboxSupported) {
       // This makes the "sandboxed" strategy the default Spawn strategy, unless it is
       // overridden by a later BlazeModule.
       builder.setDefaultStrategies(ImmutableList.of("sandboxed"));
     }
   }

   @Nullable
   private static Path getPathToDockerClient(CommandEnvironment cmdEnv) {
     String path = cmdEnv.getClientEnv().getOrDefault("PATH", "");

     // TODO(philwo): Does this return the correct result if one of the elements intentionally ends
     // in white space?
     Splitter pathSplitter =
         Splitter.on(OS.getCurrent() == OS.WINDOWS ? ';' : ':').trimResults().omitEmptyStrings();

     FileSystem fs = cmdEnv.getRuntime().getFileSystem();

     for (String pathElement : pathSplitter.split(path)) {
       // Sometimes the PATH contains the non-absolute entry "." - this resolves it against the
       // current working directory.
       pathElement = new File(pathElement).getAbsolutePath();
       try {
         for (Path dentry : fs.getPath(pathElement).getDirectoryEntries()) {
           if (dentry.getBaseName().replace(".exe", "").equals("docker")) {
             return dentry;
           }
         }
       } catch (IOException e) {
         // Intentionally ignored.
       }
     }

     return null;
   }

   @Subscribe
   public void cleanStarting(@SuppressWarnings("unused") CleanStartingEvent event) {
     if (sandboxBase != null) {
       SandboxStash.clean(treeDeleter, sandboxBase);
     }
   }

   /**
    * If there is anything other than SANDBOX_BASE_PERSISTENT_DIRS in sandboxBase when we hit this
    * precondition then there is a programming error somewhere (or I made a wrong assumption that
    * wasn't caught by any of our tests).
    */
   private static void checkSandboxBaseTopOnlyContainsPersistentDirs(Path sandboxBase) {
     try {
       List<String> directoryEntries =
           sandboxBase.getDirectoryEntries().stream()
               .map(Path::getBaseName)
               .collect(Collectors.toList());
       // If sandbox initialization failed in-between creating the inaccessible dir/file and adding
       // the Linux sandboxing strategy to spawnRunners, then the sandbox base will be in a bad
       // state. We check for that here and clean up.
       if (directoryEntries.contains(SandboxHelpers.INACCESSIBLE_HELPER_DIR)) {
         Path inaccessibleHelperDir = sandboxBase.getChild(SandboxHelpers.INACCESSIBLE_HELPER_DIR);
         inaccessibleHelperDir.chmod(0700);
         directoryEntries.remove(SandboxHelpers.INACCESSIBLE_HELPER_DIR);
         inaccessibleHelperDir.deleteTree();
       }
       if (directoryEntries.contains(SandboxHelpers.INACCESSIBLE_HELPER_FILE)) {
         Path inaccessibleHelperFile = sandboxBase.getChild(SandboxHelpers.INACCESSIBLE_HELPER_FILE);
         directoryEntries.remove(SandboxHelpers.INACCESSIBLE_HELPER_FILE);
         inaccessibleHelperFile.delete();
       }

       if (!SANDBOX_BASE_PERSISTENT_DIRS.containsAll(directoryEntries)) {
         StringBuilder message =
             new StringBuilder(
                 "Found unexpected entries in sandbox base. Please report this in"
                     + " https://github.com/bazelbuild/bazel/issues.");
         message.append(" The entries are: ");
         Joiner.on(", ").appendTo(message, directoryEntries);
         throw new IllegalStateException(message.toString());
       }
     } catch (IOException e) {
       logger.atWarning().withCause(e).log("Failed to clean up sandbox base %s", sandboxBase);
     }
   }

   @Override
   public void afterCommand() {
     checkNotNull(env, "env not initialized; was beforeCommand called?");

     SandboxOptions options = env.getOptions().getOptions(SandboxOptions.class);
     int asyncTreeDeleteThreads = options != null ? options.asyncTreeDeleteIdleThreads : 0;

     // If asynchronous deletions were requested, they may still be ongoing so let them be: trying
     // to delete the base tree synchronously could fail as we can race with those other deletions,
     // and scheduling an asynchronous deletion could race with future builds.
     if (asyncTreeDeleteThreads > 0 && treeDeleter instanceof AsynchronousTreeDeleter treeDeleter) {
       treeDeleter.setThreads(asyncTreeDeleteThreads);
     }
     // `treeDeleter` might not be an AsynchronousTreeDeleter if the user changed the option but
     // then interrupted the build before the start of the execution phase. But that's OK, there
     // will be nothing new to delete. See #13240.

     if (shouldCleanupSandboxBase) {
       try {
         checkNotNull(sandboxBase, "shouldCleanupSandboxBase implies sandboxBase has been set");
         for (SpawnRunner spawnRunner : spawnRunners) {
           spawnRunner.cleanupSandboxBase(sandboxBase, treeDeleter);
           sandboxBase.getChild(spawnRunner.getName()).delete();
         }
       } catch (IOException e) {
         env.getReporter()
             .handle(Event.warn("Failed to delete contents of sandbox " + sandboxBase + ": " + e));
       }
       shouldCleanupSandboxBase = false;

       checkSandboxBaseTopOnlyContainsPersistentDirs(sandboxBase);
       // We intentionally keep sandboxBase around, without resetting it to null, in case we have
       // asynchronous deletions going on. In that case, we'd still want to retry this during
       // shutdown.
     }

     spawnRunners.clear();

     env.getEventBus().unregister(this);
     env = null;
   }

   private void commonShutdown() {
     // Try to clean up as much garbage as possible, if there happens to be any. This will delay
     // server termination but it's the nice thing to do. If the user gets impatient, they can always
     // kill us again.
     if (treeDeleter != null) {
       try {
         treeDeleter.shutdown();
       } finally {
         treeDeleter = null; // Avoid potential reexecution if we crash.
       }
     }

     SandboxStash.shutdown();
   }

   @Override
   public void blazeShutdown() {
     commonShutdown();
   }

   @Override
   public void blazeShutdownOnCrash(DetailedExitCode exitCode) {
     commonShutdown();
   }

   private Path getStaleTrashDir(Path trashBase) {
     int i = 0;
     while (trashBase.getParentDirectory().getChild("stale-trash-" + i++).exists()) {
       ;
     }
     return trashBase.getParentDirectory().getChild("stale-trash-" + --i);
   }
 }
	// Copyright 2015 The Bazel Authors. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package com.google.devtools.build.lib.sandbox;

	import static com.google.common.base.Preconditions.checkNotNull;

	import com.google.common.base.Joiner;
	import com.google.common.base.Splitter;
	import com.google.common.collect.ImmutableList;
	import com.google.common.collect.ImmutableMap;
	import com.google.common.collect.ImmutableSet;
	import com.google.common.eventbus.Subscribe;
	import com.google.common.flogger.GoogleLogger;
	import com.google.devtools.build.lib.events.Event;
	import com.google.devtools.build.lib.exec.ExecutionOptions;
	import com.google.devtools.build.lib.exec.SpawnRunner;
	import com.google.devtools.build.lib.exec.SpawnStrategyRegistry;
	import com.google.devtools.build.lib.exec.TreeDeleter;
	import com.google.devtools.build.lib.exec.local.LocalExecutionOptions;
	import com.google.devtools.build.lib.profiler.Profiler;
	import com.google.devtools.build.lib.profiler.SilentCloseable;
	import com.google.devtools.build.lib.runtime.BlazeModule;
	import com.google.devtools.build.lib.runtime.Command;
	import com.google.devtools.build.lib.runtime.CommandEnvironment;
	import com.google.devtools.build.lib.runtime.commands.events.CleanStartingEvent;
	import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
	import com.google.devtools.build.lib.server.FailureDetails.Sandbox;
	import com.google.devtools.build.lib.util.AbruptExitException;
	import com.google.devtools.build.lib.util.DetailedExitCode;
	import com.google.devtools.build.lib.util.Fingerprint;
	import com.google.devtools.build.lib.util.OS;
	import com.google.devtools.build.lib.vfs.FileSystem;
	import com.google.devtools.build.lib.vfs.Path;
	import com.google.devtools.build.lib.vfs.PathFragment;
	import com.google.devtools.common.options.OptionsBase;
	import com.google.devtools.common.options.TriState;
	import java.io.File;
	import java.io.IOException;
	import java.time.Duration;
	import java.util.HashSet;
	import java.util.List;
	import java.util.Set;
	import java.util.stream.Collectors;
	import javax.annotation.Nullable;

	/** This module provides the Sandbox spawn strategy. */
	public final class SandboxModule extends BlazeModule {

	private static final String MAC_INDEX_FILE = ".DS_Store";

	private static final ImmutableSet<String> SANDBOX_BASE_PERSISTENT_DIRS =
	ImmutableSet.of(
	MAC_INDEX_FILE,
	SandboxStash.SANDBOX_STASH_BASE,
	SandboxStash.TEMPORARY_SANDBOX_STASH_BASE,
	AsynchronousTreeDeleter.MOVED_TRASH_DIR);

	private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();

	/** Tracks whether we are issuing the very first build within this Bazel server instance. */
	private static boolean firstBuild = true;

	/** Environment for the running command. */
	@Nullable private CommandEnvironment env;

	/** Path to the location of the sandboxes. */
	@Nullable private Path sandboxBase;

	/**
	* Collection of spawn runner instantiated during the executor setup.
	*
	* <p>We need this information to clean up the heavy subdirectories of the sandbox base on build
	* completion but to avoid wiping the whole sandbox base itself, which could be problematic across
	* builds.
	*/
	private final Set<SpawnRunner> spawnRunners = new HashSet<>();

	/**
	* Handler to process expensive tree deletions, potentially outside of the critical path.
	*
	* <p>Sandboxing creates one separate tree for each action, and this tree is used to run the
	* action commands in. These trees are disjoint for all actions and have unique identifiers.
	* Therefore, there is no need for their deletion (which can be very expensive) to happen in the
	* critical path -- so if the user so wishes, we process those deletions asynchronously.
	*/
	@Nullable private TreeDeleter treeDeleter;

	/**
	* Whether to remove the sandbox worker directories after a build or not. Useful for debugging to
	* inspect the state of files on failures.
	*/
	private boolean shouldCleanupSandboxBase;

	@Override
	public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
	return "build".equals(command.name())
	? ImmutableList.of(SandboxOptions.class)
	: ImmutableList.of();
	}

	/** Computes the path to the sandbox base tree for the given running command. */
	private static Path computeSandboxBase(SandboxOptions options, CommandEnvironment env)
	throws IOException {
	if (options.sandboxBase.isEmpty()) {
	return env.getOutputBase().getRelative("sandbox");
	} else {
	String dirName =
	String.format(
	"%s-sandbox.%s",
	env.getRuntime().getProductName(),
	Fingerprint.getHexDigest(env.getOutputBase().toString()));
	FileSystem fileSystem = env.getRuntime().getFileSystem();
	if (OS.getCurrent() == OS.DARWIN) {
	// Don't resolve symlinks on macOS: See https://github.com/bazelbuild/bazel/issues/13766
	return fileSystem.getPath(options.sandboxBase).getRelative(dirName);
	}
	Path resolvedSandboxBase = fileSystem.getPath(options.sandboxBase).resolveSymbolicLinks();
	return resolvedSandboxBase.getRelative(dirName);
	}
	}

	@Override
	public void beforeCommand(CommandEnvironment env) {
	// We can't assert that env is null because the Blaze runtime does not guarantee that
	// afterCommand() will be called if the command fails due to, e.g. a syntax error.
	this.env = env;
	env.getEventBus().register(this);

	// Don't attempt cleanup unless the executor is initialized.
	shouldCleanupSandboxBase = false;
	}

	@Override
	public void registerSpawnStrategies(
	SpawnStrategyRegistry.Builder registryBuilder, CommandEnvironment env)
	throws AbruptExitException, InterruptedException {
	checkNotNull(env, "env not initialized; was beforeCommand called?");
	try {
	setup(env, registryBuilder);
	} catch (IOException e) {
	throw new AbruptExitException(
	DetailedExitCode.of(
	FailureDetail.newBuilder()
	.setMessage(String.format("Failed to initialize sandbox: %s", e.getMessage()))
	.setSandbox(
	Sandbox.newBuilder().setCode(Sandbox.Code.INITIALIZATION_FAILURE).build())
	.build()),
	e);
	}
	}

	/**
	* Returns true if windows-sandbox should be used for this build.
	*
	* <p>Returns true if requested in ["auto", "yes"] and binary is valid. Throws an error if state
	* is "yes" and binary is not valid.
	*
	* @param requested whether windows-sandbox use was requested or not
	* @param binary path of the windows-sandbox binary to use, can be absolute or relative path
	* @return true if windows-sandbox can and should be used; false otherwise
	* @throws IOException if there are problems trying to determine the status of windows-sandbox
	*/
	private static boolean shouldUseWindowsSandbox(
	TriState requested, PathFragment binary, ImmutableMap<String, String> clientEnv)
	throws IOException {
	return switch (requested) {
	case AUTO -> WindowsSandboxUtil.isAvailable(binary, clientEnv);
	case NO -> false;
	case YES -> {
	if (!WindowsSandboxUtil.isAvailable(binary, clientEnv)) {
	throw new IOException(
	"windows-sandbox explicitly requested but \""
	+ binary
	+ "\" could not be found or is not valid");
	}
	yield true;
	}
	};
	}

	private void setup(CommandEnvironment cmdEnv, SpawnStrategyRegistry.Builder builder)
	throws IOException, InterruptedException {
	SandboxOptions options = checkNotNull(env.getOptions().getOptions(SandboxOptions.class));
	sandboxBase = computeSandboxBase(options, env);
	Path trashBase = sandboxBase.getRelative(AsynchronousTreeDeleter.MOVED_TRASH_DIR);

	// Do not remove the sandbox base when --sandbox_debug was specified so that people can check
	// out the contents of the generated sandbox directories.
	shouldCleanupSandboxBase = !options.sandboxDebug;

	// If there happens to be any live tree deleter from a previous build and it's different than
	// the one we want now, leave it alone (i.e. don't attempt to wait for pending deletions). Its
	// deletions shouldn't overlap any new directories we create during this build (because the
	// identifiers in the subdirectories will be different).
	if (options.asyncTreeDeleteIdleThreads == 0) {
	if (!(treeDeleter instanceof SynchronousTreeDeleter)) {
	treeDeleter = new SynchronousTreeDeleter();
	}
	} else {
	if (!(treeDeleter instanceof AsynchronousTreeDeleter treeDeleter)
	\|\| !treeDeleter.getTrashBase().equals(trashBase)) {
	if (treeDeleter != null) {
	treeDeleter.shutdown();
	}
	treeDeleter = new AsynchronousTreeDeleter(trashBase);
	firstBuild = true;
	}
	}
	try (SilentCloseable c = Profiler.instance().profile("SandboxStash.initialize")) {
	SandboxStash.initialize(env.getWorkspaceName(), sandboxBase, options, treeDeleter);
	}

	// SpawnExecutionPolicy#getId returns unique base directories for each sandboxed action during
	// the life of a Bazel server instance so we don't need to worry about stale directories from
	// previous builds. However, on the very first build of an instance of the server, we must
	// wipe old contents to avoid reusing stale directories.
	if (firstBuild && sandboxBase.exists()) {
	try (SilentCloseable c = Profiler.instance().profile("clean sandbox on first build")) {
	if (trashBase.exists()) {
	// Delete stale trash from a previous server instance.
	Path staleTrash = getStaleTrashDir(trashBase);
	trashBase.renameTo(staleTrash);
	trashBase.createDirectory();
	treeDeleter.deleteTree(staleTrash);
	} else {
	trashBase.createDirectory();
	}
	// We can delete other dirs asynchronously (if the flag is on).
	for (Path entry : sandboxBase.getDirectoryEntries()) {
	if (entry.getBaseName().equals(AsynchronousTreeDeleter.MOVED_TRASH_DIR)) {
	continue;
	}
	if (entry.getBaseName().equals(SandboxHelpers.INACCESSIBLE_HELPER_DIR)) {
	entry.deleteTree();
	} else if (entry.isDirectory()) {
	treeDeleter.deleteTree(entry);
	} else {
	entry.delete();
	}
	}
	} catch (IOException e) {
	// We have observed asynchronous deletion failing when running Bazel under Docker, see
	// #21719. Different RUN commands with `bazel build` will write to different layers in the
	// docker image. The overlay filesystem is different and the renaming of the directories
	// that we need to do for asynchronous deletion will fail. When that happens we fall back to
	// synchronous deletion here.
	sandboxBase.deleteTree();
	}
	}
	firstBuild = false;
	sandboxBase.createDirectoryAndParents();
	trashBase.createDirectory();

	PathFragment windowsSandboxPath = PathFragment.create(options.windowsSandboxPath);
	boolean windowsSandboxSupported;
	try (SilentCloseable c = Profiler.instance().profile("shouldUseWindowsSandbox")) {
	windowsSandboxSupported =
	shouldUseWindowsSandbox(
	options.useWindowsSandbox, windowsSandboxPath, cmdEnv.getClientEnv());
	}

	Duration timeoutKillDelay =
	cmdEnv.getOptions().getOptions(LocalExecutionOptions.class).getLocalSigkillGraceSeconds();

	boolean processWrapperSupported = ProcessWrapperSandboxedSpawnRunner.isSupported(cmdEnv);
	boolean linuxSandboxSupported = LinuxSandboxedSpawnRunner.isSupported(cmdEnv);
	boolean darwinSandboxSupported = DarwinSandboxedSpawnRunner.isSupported(cmdEnv);

	ExecutionOptions executionOptions =
	checkNotNull(cmdEnv.getOptions().getOptions(ExecutionOptions.class));
	// This works on most platforms, but isn't the best choice, so we put it first and let later
	// platform-specific sandboxing strategies become the default.
	if (processWrapperSupported) {
	SpawnRunner spawnRunner =
	new ProcessWrapperSandboxedSpawnRunner(cmdEnv, sandboxBase, treeDeleter);
	spawnRunners.add(spawnRunner);
	builder.registerStrategy(
	new ProcessWrapperSandboxedStrategy(spawnRunner, executionOptions),
	"sandboxed",
	"processwrapper-sandbox");
	}

	if (options.enableDockerSandbox) {
	// This strategy uses Docker to execute spawns. It should work on all platforms that support
	// Docker.
	Path pathToDocker = getPathToDockerClient(cmdEnv);
	// DockerSandboxedSpawnRunner.isSupported is expensive! It runs docker as a subprocess, and
	// docker hangs sometimes.
	if (pathToDocker != null && DockerSandboxedSpawnRunner.isSupported(cmdEnv, pathToDocker)) {
	String defaultImage = options.dockerImage;
	boolean useCustomizedImages = options.dockerUseCustomizedImages;
	SpawnRunner spawnRunner =
	new DockerSandboxedSpawnRunner(
	cmdEnv, pathToDocker, sandboxBase, defaultImage, useCustomizedImages, treeDeleter);
	spawnRunners.add(spawnRunner);
	builder.registerStrategy(
	new DockerSandboxedStrategy(spawnRunner, executionOptions), "docker");
	}
	} else if (options.dockerVerbose) {
	cmdEnv
	.getReporter()
	.handle(
	Event.info(
	"Docker sandboxing disabled. Use the '--experimental_enable_docker_sandbox'"
	+ " command line option to enable it"));
	}

	// This is the preferred sandboxing strategy on Linux.
	if (linuxSandboxSupported) {
	SpawnRunner spawnRunner =
	LinuxSandboxedStrategy.create(
	cmdEnv, sandboxBase, timeoutKillDelay, treeDeleter, options);
	spawnRunners.add(spawnRunner);
	builder.registerStrategy(
	new LinuxSandboxedStrategy(spawnRunner, executionOptions), "sandboxed", "linux-sandbox");
	}

	// This is the preferred sandboxing strategy on macOS.
	if (darwinSandboxSupported) {
	SpawnRunner spawnRunner = new DarwinSandboxedSpawnRunner(cmdEnv, sandboxBase, treeDeleter);
	spawnRunners.add(spawnRunner);
	builder.registerStrategy(
	new DarwinSandboxedStrategy(spawnRunner, executionOptions),
	"sandboxed",
	"darwin-sandbox");
	}

	if (windowsSandboxSupported) {
	SpawnRunner spawnRunner =
	new WindowsSandboxedSpawnRunner(cmdEnv, timeoutKillDelay, windowsSandboxPath);
	spawnRunners.add(spawnRunner);
	builder.registerStrategy(
	new WindowsSandboxedStrategy(spawnRunner, executionOptions),
	"sandboxed",
	"windows-sandbox");
	}

	if (processWrapperSupported
	\|\| linuxSandboxSupported
	\|\| darwinSandboxSupported
	\|\| windowsSandboxSupported) {
	// This makes the "sandboxed" strategy the default Spawn strategy, unless it is
	// overridden by a later BlazeModule.
	builder.setDefaultStrategies(ImmutableList.of("sandboxed"));
	}
	}

	@Nullable
	private static Path getPathToDockerClient(CommandEnvironment cmdEnv) {
	String path = cmdEnv.getClientEnv().getOrDefault("PATH", "");

	// TODO(philwo): Does this return the correct result if one of the elements intentionally ends
	// in white space?
	Splitter pathSplitter =
	Splitter.on(OS.getCurrent() == OS.WINDOWS ? ';' : ':').trimResults().omitEmptyStrings();

	FileSystem fs = cmdEnv.getRuntime().getFileSystem();

	for (String pathElement : pathSplitter.split(path)) {
	// Sometimes the PATH contains the non-absolute entry "." - this resolves it against the
	// current working directory.
	pathElement = new File(pathElement).getAbsolutePath();
	try {
	for (Path dentry : fs.getPath(pathElement).getDirectoryEntries()) {
	if (dentry.getBaseName().replace(".exe", "").equals("docker")) {
	return dentry;
	}
	}
	} catch (IOException e) {
	// Intentionally ignored.
	}
	}

	return null;
	}

	@Subscribe
	public void cleanStarting(@SuppressWarnings("unused") CleanStartingEvent event) {
	if (sandboxBase != null) {
	SandboxStash.clean(treeDeleter, sandboxBase);
	}
	}

	/**
	* If there is anything other than SANDBOX_BASE_PERSISTENT_DIRS in sandboxBase when we hit this
	* precondition then there is a programming error somewhere (or I made a wrong assumption that
	* wasn't caught by any of our tests).
	*/
	private static void checkSandboxBaseTopOnlyContainsPersistentDirs(Path sandboxBase) {
	try {
	List<String> directoryEntries =
	sandboxBase.getDirectoryEntries().stream()
	.map(Path::getBaseName)
	.collect(Collectors.toList());
	// If sandbox initialization failed in-between creating the inaccessible dir/file and adding
	// the Linux sandboxing strategy to spawnRunners, then the sandbox base will be in a bad
	// state. We check for that here and clean up.
	if (directoryEntries.contains(SandboxHelpers.INACCESSIBLE_HELPER_DIR)) {
	Path inaccessibleHelperDir = sandboxBase.getChild(SandboxHelpers.INACCESSIBLE_HELPER_DIR);
	inaccessibleHelperDir.chmod(0700);
	directoryEntries.remove(SandboxHelpers.INACCESSIBLE_HELPER_DIR);
	inaccessibleHelperDir.deleteTree();
	}
	if (directoryEntries.contains(SandboxHelpers.INACCESSIBLE_HELPER_FILE)) {
	Path inaccessibleHelperFile = sandboxBase.getChild(SandboxHelpers.INACCESSIBLE_HELPER_FILE);
	directoryEntries.remove(SandboxHelpers.INACCESSIBLE_HELPER_FILE);
	inaccessibleHelperFile.delete();
	}

	if (!SANDBOX_BASE_PERSISTENT_DIRS.containsAll(directoryEntries)) {
	StringBuilder message =
	new StringBuilder(
	"Found unexpected entries in sandbox base. Please report this in"
	+ " https://github.com/bazelbuild/bazel/issues.");
	message.append(" The entries are: ");
	Joiner.on(", ").appendTo(message, directoryEntries);
	throw new IllegalStateException(message.toString());
	}
	} catch (IOException e) {
	logger.atWarning().withCause(e).log("Failed to clean up sandbox base %s", sandboxBase);
	}
	}

	@Override
	public void afterCommand() {
	checkNotNull(env, "env not initialized; was beforeCommand called?");

	SandboxOptions options = env.getOptions().getOptions(SandboxOptions.class);
	int asyncTreeDeleteThreads = options != null ? options.asyncTreeDeleteIdleThreads : 0;

	// If asynchronous deletions were requested, they may still be ongoing so let them be: trying
	// to delete the base tree synchronously could fail as we can race with those other deletions,
	// and scheduling an asynchronous deletion could race with future builds.
	if (asyncTreeDeleteThreads > 0 && treeDeleter instanceof AsynchronousTreeDeleter treeDeleter) {
	treeDeleter.setThreads(asyncTreeDeleteThreads);
	}
	// `treeDeleter` might not be an AsynchronousTreeDeleter if the user changed the option but
	// then interrupted the build before the start of the execution phase. But that's OK, there
	// will be nothing new to delete. See #13240.

	if (shouldCleanupSandboxBase) {
	try {
	checkNotNull(sandboxBase, "shouldCleanupSandboxBase implies sandboxBase has been set");
	for (SpawnRunner spawnRunner : spawnRunners) {
	spawnRunner.cleanupSandboxBase(sandboxBase, treeDeleter);
	sandboxBase.getChild(spawnRunner.getName()).delete();
	}
	} catch (IOException e) {
	env.getReporter()
	.handle(Event.warn("Failed to delete contents of sandbox " + sandboxBase + ": " + e));
	}
	shouldCleanupSandboxBase = false;

	checkSandboxBaseTopOnlyContainsPersistentDirs(sandboxBase);
	// We intentionally keep sandboxBase around, without resetting it to null, in case we have
	// asynchronous deletions going on. In that case, we'd still want to retry this during
	// shutdown.
	}

	spawnRunners.clear();

	env.getEventBus().unregister(this);
	env = null;
	}

	private void commonShutdown() {
	// Try to clean up as much garbage as possible, if there happens to be any. This will delay
	// server termination but it's the nice thing to do. If the user gets impatient, they can always
	// kill us again.
	if (treeDeleter != null) {
	try {
	treeDeleter.shutdown();
	} finally {
	treeDeleter = null; // Avoid potential reexecution if we crash.
	}
	}

	SandboxStash.shutdown();
	}

	@Override
	public void blazeShutdown() {
	commonShutdown();
	}

	@Override
	public void blazeShutdownOnCrash(DetailedExitCode exitCode) {
	commonShutdown();
	}

	private Path getStaleTrashDir(Path trashBase) {
	int i = 0;
	while (trashBase.getParentDirectory().getChild("stale-trash-" + i++).exists()) {
	;
	}
	return trashBase.getParentDirectory().getChild("stale-trash-" + --i);
	}
	}