src/main/java/com/google/devtools/build/lib/sandbox/SandboxStash.java - bazel - Git at Google

 // Copyright 2023 The Bazel Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package com.google.devtools.build.lib.sandbox;

 import static com.google.common.base.Preconditions.checkNotNull;

 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.GoogleLogger;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.google.devtools.build.lib.cmdline.Label;
 import com.google.devtools.build.lib.exec.TreeDeleter;
 import com.google.devtools.build.lib.profiler.Profiler;
 import com.google.devtools.build.lib.profiler.SilentCloseable;
 import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxContents;
 import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxOutputs;
 import com.google.devtools.build.lib.vfs.FileSystem;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import javax.annotation.Nullable;

 /**
  * Singleton class for the `--reuse_sandbox_directories` flag: Controls a "stash" of old sandbox
  * directories. When a sandboxed runner needs its directory tree, it first tries to grab a stash by
  * just moving it. They are separated by mnemonic because that makes them much more likely to be
  * able to reuse things common for that mnemonic, e.g. standard libraries.
  */
 public class SandboxStash {

   public static final String SANDBOX_STASH_BASE = "sandbox_stash";

   // Used while we gather all the contents asynchronously.
   public static final String TEMPORARY_SANDBOX_STASH_BASE = "tmp_sandbox_stash";
   private static final String TEST_RUNNER_MNEMONIC = "TestRunner";
   private static final String TEST_SRCDIR = "TEST_SRCDIR";
   private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();

   /** An incrementing count of stashes to avoid filename clashes. */
   static final AtomicInteger stash = new AtomicInteger(0);

   /** If true, we have already warned about an error causing us to turn off reuse. */
   private final AtomicBoolean warnedAboutTurningOffReuse = new AtomicBoolean();

   /**
    * Whether to attempt to reuse previously-created sandboxes. Not final because we may turn it off
    * in case of errors.
    */
   static boolean reuseSandboxDirectories;

   private static SandboxStash instance;
   private final String workspaceName;
   private final Path sandboxBase;

   private final Map<Path, String> stashPathToRunfilesDir = new ConcurrentHashMap<>();

   private static final int POOL_SIZE = Runtime.getRuntime().availableProcessors();
   private final ExecutorService stashFileListingPool =
       Executors.newFixedThreadPool(
           POOL_SIZE,
           new ThreadFactoryBuilder().setNameFormat("stash-file-listing-thread-%d").build());

   public final Map<Path, SandboxContents> pathToContents = new ConcurrentHashMap<>();
   private final Map<Path, Label> sandboxToTarget = new ConcurrentHashMap<>();
   private final Map<Path, Long> pathToLastModified = new ConcurrentHashMap<>();
   private boolean inMemoryStashes;

   public SandboxStash(String workspaceName, Path sandboxBase, boolean inMemoryStashes) {
     this.workspaceName = workspaceName;
     this.sandboxBase = sandboxBase;
     this.inMemoryStashes = inMemoryStashes;
   }

   @Nullable
   @SuppressWarnings("NullableOptional")
   static Optional<SandboxContents> takeStashedSandbox(
       Path sandboxPath,
       String mnemonic,
       Map<String, String> environment,
       SandboxOutputs outputs,
       Label target) {
     if (instance == null) {
       return null;
     }
     return instance.takeStashedSandboxInternal(sandboxPath, mnemonic, environment, outputs, target);
   }

   @Nullable
   @SuppressWarnings("NullableOptional")
   private Optional<SandboxContents> takeStashedSandboxInternal(
       Path sandboxPath,
       String mnemonic,
       Map<String, String> environment,
       SandboxOutputs outputs,
       Label target) {
     try {
       Path sandboxes = getSandboxStashDir(mnemonic, sandboxPath.getFileSystem());
       if (sandboxes == null || isTestXmlGenerationOrCoverageSpawn(mnemonic, outputs)) {
         return null;
       }

       Collection<Path> diskStashes = sandboxes.getDirectoryEntries();
       if (diskStashes.isEmpty()) {
         return null;
       }

       ImmutableList<Path> stashes = sortStashesByMatchingTargetSegments(target, diskStashes);
       // We have to remove the sandbox execroot dir to move a stash there, but it is currently empty
       // and we reinstate it later if we don't get a sandbox. We can't just move the stash dir
       // fully, as we would then lose siblings of the execroot dir, such as hermetic-tmp dirs.
       Path sandboxExecroot = sandboxPath.getChild("execroot");
       sandboxExecroot.deleteTree();
       for (Path stash : stashes) {
         try {
           Path stashExecroot = stash.getChild("execroot");
           stashExecroot.renameTo(sandboxExecroot);
           stash.deleteTree();
           if (isTestAction(mnemonic)) {
             String relativeStashedRunfilesDir = stashPathToRunfilesDir.get(stashExecroot);
             Path stashedRunfilesDir = sandboxExecroot.getRelative(relativeStashedRunfilesDir);
             String relativeCurrentRunfilesDir = getCurrentRunfilesDir(environment);
             Path currentRunfiles = sandboxExecroot.getRelative(relativeCurrentRunfilesDir);
             currentRunfiles.getParentDirectory().createDirectoryAndParents();
             stashedRunfilesDir.renameTo(currentRunfiles);
             stashPathToRunfilesDir.remove(stashExecroot);
             if (useInMemoryStashes() && pathToContents.containsKey(stash)) {
               updateStashContentsAfterRunfilesMove(
                   relativeStashedRunfilesDir,
                   relativeCurrentRunfilesDir,
                   pathToContents.get(stash));
             }
           }
           sandboxToTarget.remove(stash);
           // If we switched the flag experimental_inmemory_sandbox_stashes from false to true
           // without restarting the Bazel server, we may have a stash but not its contents in
           // memory.
           return useInMemoryStashes() && pathToContents.containsKey(stash)
               ? Optional.of(pathToContents.remove(stash))
               : Optional.empty();
         } catch (FileNotFoundException e) {
           // Try the next one, somebody else took this one.
         } catch (IOException e) {
           turnOffReuse("Error renaming sandbox stash %s to %s: %s\n", stash, sandboxPath, e);
           return null;
         }
       }
       return null;
     } catch (IOException e) {
       turnOffReuse("Failed to prepare for reusing stashed sandbox for %s: %s", sandboxPath, e);
       return null;
     }
   }

   /** Atomically moves the sandboxPath directory aside for later reuse. */
   static void stashSandbox(
       Path path,
       String mnemonic,
       Map<String, String> environment,
       SandboxOutputs outputs,
       TreeDeleter treeDeleter,
       Label target) {
     if (instance == null) {
       return;
     }

     Path sandboxes = instance.getSandboxStashDir(mnemonic, path.getFileSystem());
     if (sandboxes == null
         || isTestXmlGenerationOrCoverageSpawn(mnemonic, outputs)
         || !path.exists()) {
       return;
     }
     String stashName = Integer.toString(stash.incrementAndGet());

     if (useInMemoryStashes()) {
       instance.stashSandboxInternalWithInMemoryStashes(
           stashName, sandboxes, path, mnemonic, environment, treeDeleter, target);
     } else {
       instance.stashSandboxInternal(
           stashName, sandboxes, path, mnemonic, environment, treeDeleter, target);
     }
   }

   @SuppressWarnings("FutureReturnValueIgnored")
   private void stashSandboxInternalWithInMemoryStashes(
       String stashName,
       Path sandboxes,
       Path path,
       String mnemonic,
       Map<String, String> environment,
       TreeDeleter treeDeleter,
       Label target) {
     Path temporaryStashes = sandboxBase.getChild(TEMPORARY_SANDBOX_STASH_BASE);
     Path temporaryStash = temporaryStashes.getChild(stashName);
     try {
       temporaryStashes.createDirectory();
       path.getChild("execroot").renameTo(temporaryStash);
     } catch (IOException e) {
       turnOffReuse("Error stashing sandbox at %s: %s", temporaryStash, e);
     }
     stashFileListingPool.submit(
         () -> {
           Path stashPath = sandboxes.getChild(stashName);
           try {
             SandboxContents stashContents = pathToContents.remove(path);
             long lastModified = checkNotNull(pathToLastModified.remove(path));
             SandboxHelpers.updateContentMap(temporaryStash, lastModified, stashContents);
             stashPath.createDirectory();
             Path stashPathExecroot = stashPath.getChild("execroot");
             if (isTestAction(mnemonic)) {
               if (environment.get("TEST_TMPDIR").startsWith("_tmp")) {
                 treeDeleter.deleteTree(
                     temporaryStash.getRelative(environment.get("TEST_WORKSPACE") + "/_tmp"));
               }
               // We do this before the rename operation to avoid a race condition.
               stashPathToRunfilesDir.put(stashPathExecroot, getCurrentRunfilesDir(environment));
             }
             setPathContents(stashPath, stashContents);
             temporaryStash.renameTo(stashPathExecroot);
             if (target != null) {
               sandboxToTarget.put(stashPath, target);
             }
           } catch (InterruptedException e) {
             // Finish the job without stashing the sandbox
           } catch (IOException e) {
             // TODO(bazel-team): Are we sure we don't want to surface this error?
             turnOffReuse("Error stashing sandbox at %s: %s", stashPath, e);
           }
         });
   }

   private void stashSandboxInternal(
       String stashName,
       Path sandboxes,
       Path path,
       String mnemonic,
       Map<String, String> environment,
       TreeDeleter treeDeleter,
       Label target) {
     Path stashPath = sandboxes.getChild(stashName);
     try {
       stashPath.createDirectory();
       Path stashPathExecroot = stashPath.getChild("execroot");
       if (isTestAction(mnemonic)) {
         if (environment.get("TEST_TMPDIR").startsWith("_tmp")) {
           treeDeleter.deleteTree(
               path.getRelative("execroot/" + environment.get("TEST_WORKSPACE") + "/_tmp"));
         }
       }
       if (isTestAction(mnemonic)) {
         // We do this before the rename operation to avoid a race condition.
         stashPathToRunfilesDir.put(stashPathExecroot, getCurrentRunfilesDir(environment));
       }
       path.getChild("execroot").renameTo(stashPathExecroot);
       if (target != null) {
         sandboxToTarget.put(stashPath, target);
       }
     } catch (IOException e) {
       // Since stash names are unique, this IOException indicates some other problem with stashing,
       // so we turn it off.
       turnOffReuse("Error stashing sandbox at %s: %s", stashPath, e);
     }
   }

   /**
    * Returns the sandbox stashing directory appropriate for this mnemonic. In order to maximize
    * reuse, we keep stashed sandboxes separated by mnemonic. May return null if there are errors, in
    * which case sandbox reuse also gets turned off.
    *
    * <p>TODO(bazel-team): Fix integration tests to instantiate FileSystem only once, so that passing
    * it in here (to avoid the cross-filesystem precondition check in renameTo) is no longer
    * necessary.
    */
   @Nullable
   private Path getSandboxStashDir(String mnemonic, FileSystem fileSystem) {
     Path stashDir = getStashBase(fileSystem.getPath(this.sandboxBase.getPathString()));
     try {
       stashDir.createDirectory();
       if (!maybeClearExistingStash(stashDir)) {
         return null;
       }
     } catch (IOException e) {
       turnOffReuse(
           "Error creating sandbox stash dir %s, disabling sandbox reuse: %s\n",
           stashDir, e.getMessage());
       return null;
     }
     Path mnemonicStashDir = stashDir.getChild(mnemonic);
     try {
       mnemonicStashDir.createDirectory();
       return mnemonicStashDir;
     } catch (IOException e) {
       turnOffReuse("Error creating mnemonic stash dir %s: %s\n", mnemonicStashDir, e.getMessage());
       return null;
     }
   }

   private static Path getStashBase(Path sandboxBase) {
     return sandboxBase.getChild(SANDBOX_STASH_BASE);
   }

   /**
    * Clears away existing stash if this is the first access to the stash in this Blaze server
    * instance.
    *
    * @param stashPath Path of the stashes.
    * @return True unless there was an error deleting sandbox stashes.
    */
   private boolean maybeClearExistingStash(Path stashPath) {
     synchronized (stash) {
       if (stash.getAndIncrement() == 0) {
         try {
           for (Path directoryEntry : stashPath.getDirectoryEntries()) {
             directoryEntry.deleteTree();
           }
         } catch (IOException e) {
           turnOffReuse("Unable to clear old sandbox stash %s: %s\n", stashPath, e.getMessage());
           return false;
         }
       }
     }
     return true;
   }

   private void turnOffReuse(String fmt, Object... args) {
     reuseSandboxDirectories = false;
     if (warnedAboutTurningOffReuse.compareAndSet(false, true)) {
       logger.atWarning().logVarargs("Turning off sandbox reuse: " + fmt, args);
     }
   }

   public static void initialize(
       String workspaceName, Path sandboxBase, SandboxOptions options, TreeDeleter treeDeleter) {
     if (options.reuseSandboxDirectories) {
       if (instance == null) {
         instance =
             new SandboxStash(
                 workspaceName, sandboxBase, options.experimentalInMemorySandboxStashes);
       } else {
         if (!Objects.equals(workspaceName, instance.workspaceName)) {
           Path stashBase = getStashBase(instance.sandboxBase);
           try (SilentCloseable c = Profiler.instance().profile("treeDeleter.deleteTree")) {
             for (Path directoryEntry : stashBase.getDirectoryEntries()) {
               treeDeleter.deleteTree(directoryEntry);
             }
           } catch (IOException e) {
             instance.turnOffReuse(
                 "Unable to clear old sandbox stash %s: %s\n", stashBase, e.getMessage());
           }
           instance =
               new SandboxStash(
                   workspaceName, sandboxBase, options.experimentalInMemorySandboxStashes);
         }
         instance.inMemoryStashes = options.experimentalInMemorySandboxStashes;
       }
     } else {
       instance = null;
     }
   }

   public static boolean useInMemoryStashes() {
     Preconditions.checkNotNull(instance);
     return instance.inMemoryStashes;
   }

   public static void setPathContents(Path path, SandboxContents contents) {
     Preconditions.checkNotNull(instance);
     instance.pathToContents.put(path, contents);
   }

   public static void setLastModified(Path path, Long lastModified) {
     if (instance != null) {
       instance.pathToLastModified.put(path, lastModified);
     }
   }

   public static boolean gotInstance() {
     return instance != null;
   }

   public static void shutdown() {
     if (instance != null) {
       instance.stashFileListingPool.shutdown();
     }
   }

   /** Cleans up the entire current stash, if any. Cleaning may be asynchronous. */
   static void clean(TreeDeleter treeDeleter, Path sandboxBase) {
     Path stashDir = getStashBase(sandboxBase);
     if (!stashDir.isDirectory()) {
       return;
     }
     Path stashTrashDir = stashDir.getChild("__trash");
     try {
       stashDir.renameTo(stashTrashDir);
     } catch (IOException e) {
       // If we couldn't move the stashdir away for deletion, we need to delete it synchronously
       // in place, so we can't use the treeDeleter.
       treeDeleter = null;
       stashTrashDir = stashDir;
     }
     try {
       if (treeDeleter != null) {
         treeDeleter.deleteTree(stashTrashDir);
       } else {
         stashTrashDir.deleteTree();
       }
     } catch (IOException e) {
       logger.atWarning().withCause(e).log("Failed to clean sandbox stash %s", stashDir);
     }

     if (instance != null) {
       instance.stashPathToRunfilesDir.clear();
       instance.pathToContents.clear();
       instance.sandboxToTarget.clear();
       instance.pathToLastModified.clear();
     }
   }

   /**
    * Test actions are guaranteed to have a runfiles directory with the test name as part of the
    * name. The path to the directory is unique between tests. If two tests (foo and bar) have the
    * directory <source-root>/pkg/my_runfiles as part of their runfiles and this directory contains
    * 1000 files, we would be symlinking the 1000 files for each test since the paths do not
    * coincide. To make sure we can reuse the runfiles directory we must rename the old runfiles
    * directory for the action that was stashed to the path that is expected by the current test.
    */
   private static boolean isTestAction(String mnemonic) {
     return mnemonic.equals(TEST_RUNNER_MNEMONIC);
   }

   /**
    * Test actions are split in two spawns. The first one runs the test and the second generates the
    * XML output from the test log. We do not want the second spawn to reuse the stash because it
    * doesn't contain the inputs needed to run the test; if it reused it, it would be expensive in
    * two ways: it would have to clean up all the inputs, and it would destroy a valid stash that a
    * different test could potentially use. If we are running coverage, there might be a third spawn
    * for coverage where we apply the same reasoning.
    *
    * <p>We identify the second and third spawn because they have a single output.
    */
   private static boolean isTestXmlGenerationOrCoverageSpawn(
       String mnemonic, SandboxOutputs outputs) {
     return isTestAction(mnemonic) && outputs.files().size() == 1;
   }

   private static String getCurrentRunfilesDir(Map<String, String> environment) {
     return environment.get("TEST_WORKSPACE") + "/" + environment.get(TEST_SRCDIR);
   }

   private ImmutableList<Path> sortStashesByMatchingTargetSegments(
       Label target, Collection<Path> stashes) {
     List<Path> sortedStashes = new ArrayList<>(stashes);
     Map<Path, Integer> countMap = new HashMap<>();
     String[] targetStr = null;
     if (target != null) {
       targetStr = target.getPackageName().split("/");
     }
     for (Path stash : stashes) {
       Label stashTarget = sandboxToTarget.getOrDefault(stash, /* defaultValue= */ null);
       if (target == null) {
         countMap.put(stash, stashTarget == null ? 1 : 0);
       } else {
         countMap.put(
             stash,
             stashTarget == null
                 ? 0
                 : Arrays.mismatch(targetStr, stashTarget.getPackageName().split("/")));
       }
     }
     return ImmutableList.sortedCopyOf(
         Comparator.comparingInt(countMap::get).reversed(), sortedStashes);
   }

   private void updateStashContentsAfterRunfilesMove(
       String stashedRunfiles, String currentRunfiles, SandboxContents stashContents) {
     ImmutableList<String> stashedRunfilesSegments =
         ImmutableList.copyOf(PathFragment.create(stashedRunfiles).segments());
     SandboxContents runfilesStashContents = stashContents;
     for (int i = 0; i < stashedRunfilesSegments.size() - 1; i++) {
       runfilesStashContents =
           Preconditions.checkNotNull(
               runfilesStashContents.dirMap().get(stashedRunfilesSegments.get(i)));
     }
     runfilesStashContents =
         runfilesStashContents.dirMap().remove(stashedRunfilesSegments.getLast());

     ImmutableList<String> currentRunfilesSegments =
         ImmutableList.copyOf(PathFragment.create(currentRunfiles).segments());
     SandboxContents currentStashContents = stashContents;
     for (int i = 0; i < currentRunfilesSegments.size() - 1; i++) {
       String segment = currentRunfilesSegments.get(i);
       currentStashContents.dirMap().putIfAbsent(segment, new SandboxContents());
       currentStashContents = currentStashContents.dirMap().get(segment);
     }
     currentStashContents.dirMap().put(currentRunfilesSegments.getLast(), runfilesStashContents);
   }
 }
	// Copyright 2023 The Bazel Authors. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package com.google.devtools.build.lib.sandbox;

	import static com.google.common.base.Preconditions.checkNotNull;

	import com.google.common.base.Preconditions;
	import com.google.common.collect.ImmutableList;
	import com.google.common.flogger.GoogleLogger;
	import com.google.common.util.concurrent.ThreadFactoryBuilder;
	import com.google.devtools.build.lib.cmdline.Label;
	import com.google.devtools.build.lib.exec.TreeDeleter;
	import com.google.devtools.build.lib.profiler.Profiler;
	import com.google.devtools.build.lib.profiler.SilentCloseable;
	import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxContents;
	import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxOutputs;
	import com.google.devtools.build.lib.vfs.FileSystem;
	import com.google.devtools.build.lib.vfs.Path;
	import com.google.devtools.build.lib.vfs.PathFragment;
	import java.io.FileNotFoundException;
	import java.io.IOException;
	import java.util.ArrayList;
	import java.util.Arrays;
	import java.util.Collection;
	import java.util.Comparator;
	import java.util.HashMap;
	import java.util.List;
	import java.util.Map;
	import java.util.Objects;
	import java.util.Optional;
	import java.util.concurrent.ConcurrentHashMap;
	import java.util.concurrent.ExecutorService;
	import java.util.concurrent.Executors;
	import java.util.concurrent.atomic.AtomicBoolean;
	import java.util.concurrent.atomic.AtomicInteger;
	import javax.annotation.Nullable;

	/**
	* Singleton class for the `--reuse_sandbox_directories` flag: Controls a "stash" of old sandbox
	* directories. When a sandboxed runner needs its directory tree, it first tries to grab a stash by
	* just moving it. They are separated by mnemonic because that makes them much more likely to be
	* able to reuse things common for that mnemonic, e.g. standard libraries.
	*/
	public class SandboxStash {

	public static final String SANDBOX_STASH_BASE = "sandbox_stash";

	// Used while we gather all the contents asynchronously.
	public static final String TEMPORARY_SANDBOX_STASH_BASE = "tmp_sandbox_stash";
	private static final String TEST_RUNNER_MNEMONIC = "TestRunner";
	private static final String TEST_SRCDIR = "TEST_SRCDIR";
	private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();

	/** An incrementing count of stashes to avoid filename clashes. */
	static final AtomicInteger stash = new AtomicInteger(0);

	/** If true, we have already warned about an error causing us to turn off reuse. */
	private final AtomicBoolean warnedAboutTurningOffReuse = new AtomicBoolean();

	/**
	* Whether to attempt to reuse previously-created sandboxes. Not final because we may turn it off
	* in case of errors.
	*/
	static boolean reuseSandboxDirectories;

	private static SandboxStash instance;
	private final String workspaceName;
	private final Path sandboxBase;

	private final Map<Path, String> stashPathToRunfilesDir = new ConcurrentHashMap<>();

	private static final int POOL_SIZE = Runtime.getRuntime().availableProcessors();
	private final ExecutorService stashFileListingPool =
	Executors.newFixedThreadPool(
	POOL_SIZE,
	new ThreadFactoryBuilder().setNameFormat("stash-file-listing-thread-%d").build());

	public final Map<Path, SandboxContents> pathToContents = new ConcurrentHashMap<>();
	private final Map<Path, Label> sandboxToTarget = new ConcurrentHashMap<>();
	private final Map<Path, Long> pathToLastModified = new ConcurrentHashMap<>();
	private boolean inMemoryStashes;

	public SandboxStash(String workspaceName, Path sandboxBase, boolean inMemoryStashes) {
	this.workspaceName = workspaceName;
	this.sandboxBase = sandboxBase;
	this.inMemoryStashes = inMemoryStashes;
	}

	@Nullable
	@SuppressWarnings("NullableOptional")
	static Optional<SandboxContents> takeStashedSandbox(
	Path sandboxPath,
	String mnemonic,
	Map<String, String> environment,
	SandboxOutputs outputs,
	Label target) {
	if (instance == null) {
	return null;
	}
	return instance.takeStashedSandboxInternal(sandboxPath, mnemonic, environment, outputs, target);
	}

	@Nullable
	@SuppressWarnings("NullableOptional")
	private Optional<SandboxContents> takeStashedSandboxInternal(
	Path sandboxPath,
	String mnemonic,
	Map<String, String> environment,
	SandboxOutputs outputs,
	Label target) {
	try {
	Path sandboxes = getSandboxStashDir(mnemonic, sandboxPath.getFileSystem());
	if (sandboxes == null \|\| isTestXmlGenerationOrCoverageSpawn(mnemonic, outputs)) {
	return null;
	}

	Collection<Path> diskStashes = sandboxes.getDirectoryEntries();
	if (diskStashes.isEmpty()) {
	return null;
	}

	ImmutableList<Path> stashes = sortStashesByMatchingTargetSegments(target, diskStashes);
	// We have to remove the sandbox execroot dir to move a stash there, but it is currently empty
	// and we reinstate it later if we don't get a sandbox. We can't just move the stash dir
	// fully, as we would then lose siblings of the execroot dir, such as hermetic-tmp dirs.
	Path sandboxExecroot = sandboxPath.getChild("execroot");
	sandboxExecroot.deleteTree();
	for (Path stash : stashes) {
	try {
	Path stashExecroot = stash.getChild("execroot");
	stashExecroot.renameTo(sandboxExecroot);
	stash.deleteTree();
	if (isTestAction(mnemonic)) {
	String relativeStashedRunfilesDir = stashPathToRunfilesDir.get(stashExecroot);
	Path stashedRunfilesDir = sandboxExecroot.getRelative(relativeStashedRunfilesDir);
	String relativeCurrentRunfilesDir = getCurrentRunfilesDir(environment);
	Path currentRunfiles = sandboxExecroot.getRelative(relativeCurrentRunfilesDir);
	currentRunfiles.getParentDirectory().createDirectoryAndParents();
	stashedRunfilesDir.renameTo(currentRunfiles);
	stashPathToRunfilesDir.remove(stashExecroot);
	if (useInMemoryStashes() && pathToContents.containsKey(stash)) {
	updateStashContentsAfterRunfilesMove(
	relativeStashedRunfilesDir,
	relativeCurrentRunfilesDir,
	pathToContents.get(stash));
	}
	}
	sandboxToTarget.remove(stash);
	// If we switched the flag experimental_inmemory_sandbox_stashes from false to true
	// without restarting the Bazel server, we may have a stash but not its contents in
	// memory.
	return useInMemoryStashes() && pathToContents.containsKey(stash)
	? Optional.of(pathToContents.remove(stash))
	: Optional.empty();
	} catch (FileNotFoundException e) {
	// Try the next one, somebody else took this one.
	} catch (IOException e) {
	turnOffReuse("Error renaming sandbox stash %s to %s: %s\n", stash, sandboxPath, e);
	return null;
	}
	}
	return null;
	} catch (IOException e) {
	turnOffReuse("Failed to prepare for reusing stashed sandbox for %s: %s", sandboxPath, e);
	return null;
	}
	}

	/** Atomically moves the sandboxPath directory aside for later reuse. */
	static void stashSandbox(
	Path path,
	String mnemonic,
	Map<String, String> environment,
	SandboxOutputs outputs,
	TreeDeleter treeDeleter,
	Label target) {
	if (instance == null) {
	return;
	}

	Path sandboxes = instance.getSandboxStashDir(mnemonic, path.getFileSystem());
	if (sandboxes == null
	\|\| isTestXmlGenerationOrCoverageSpawn(mnemonic, outputs)
	\|\| !path.exists()) {
	return;
	}
	String stashName = Integer.toString(stash.incrementAndGet());

	if (useInMemoryStashes()) {
	instance.stashSandboxInternalWithInMemoryStashes(
	stashName, sandboxes, path, mnemonic, environment, treeDeleter, target);
	} else {
	instance.stashSandboxInternal(
	stashName, sandboxes, path, mnemonic, environment, treeDeleter, target);
	}
	}

	@SuppressWarnings("FutureReturnValueIgnored")
	private void stashSandboxInternalWithInMemoryStashes(
	String stashName,
	Path sandboxes,
	Path path,
	String mnemonic,
	Map<String, String> environment,
	TreeDeleter treeDeleter,
	Label target) {
	Path temporaryStashes = sandboxBase.getChild(TEMPORARY_SANDBOX_STASH_BASE);
	Path temporaryStash = temporaryStashes.getChild(stashName);
	try {
	temporaryStashes.createDirectory();
	path.getChild("execroot").renameTo(temporaryStash);
	} catch (IOException e) {
	turnOffReuse("Error stashing sandbox at %s: %s", temporaryStash, e);
	}
	stashFileListingPool.submit(
	() -> {
	Path stashPath = sandboxes.getChild(stashName);
	try {
	SandboxContents stashContents = pathToContents.remove(path);
	long lastModified = checkNotNull(pathToLastModified.remove(path));
	SandboxHelpers.updateContentMap(temporaryStash, lastModified, stashContents);
	stashPath.createDirectory();
	Path stashPathExecroot = stashPath.getChild("execroot");
	if (isTestAction(mnemonic)) {
	if (environment.get("TEST_TMPDIR").startsWith("_tmp")) {
	treeDeleter.deleteTree(
	temporaryStash.getRelative(environment.get("TEST_WORKSPACE") + "/_tmp"));
	}
	// We do this before the rename operation to avoid a race condition.
	stashPathToRunfilesDir.put(stashPathExecroot, getCurrentRunfilesDir(environment));
	}
	setPathContents(stashPath, stashContents);
	temporaryStash.renameTo(stashPathExecroot);
	if (target != null) {
	sandboxToTarget.put(stashPath, target);
	}
	} catch (InterruptedException e) {
	// Finish the job without stashing the sandbox
	} catch (IOException e) {
	// TODO(bazel-team): Are we sure we don't want to surface this error?
	turnOffReuse("Error stashing sandbox at %s: %s", stashPath, e);
	}
	});
	}

	private void stashSandboxInternal(
	String stashName,
	Path sandboxes,
	Path path,
	String mnemonic,
	Map<String, String> environment,
	TreeDeleter treeDeleter,
	Label target) {
	Path stashPath = sandboxes.getChild(stashName);
	try {
	stashPath.createDirectory();
	Path stashPathExecroot = stashPath.getChild("execroot");
	if (isTestAction(mnemonic)) {
	if (environment.get("TEST_TMPDIR").startsWith("_tmp")) {
	treeDeleter.deleteTree(
	path.getRelative("execroot/" + environment.get("TEST_WORKSPACE") + "/_tmp"));
	}
	}
	if (isTestAction(mnemonic)) {
	// We do this before the rename operation to avoid a race condition.
	stashPathToRunfilesDir.put(stashPathExecroot, getCurrentRunfilesDir(environment));
	}
	path.getChild("execroot").renameTo(stashPathExecroot);
	if (target != null) {
	sandboxToTarget.put(stashPath, target);
	}
	} catch (IOException e) {
	// Since stash names are unique, this IOException indicates some other problem with stashing,
	// so we turn it off.
	turnOffReuse("Error stashing sandbox at %s: %s", stashPath, e);
	}
	}

	/**
	* Returns the sandbox stashing directory appropriate for this mnemonic. In order to maximize
	* reuse, we keep stashed sandboxes separated by mnemonic. May return null if there are errors, in
	* which case sandbox reuse also gets turned off.
	*
	* <p>TODO(bazel-team): Fix integration tests to instantiate FileSystem only once, so that passing
	* it in here (to avoid the cross-filesystem precondition check in renameTo) is no longer
	* necessary.
	*/
	@Nullable
	private Path getSandboxStashDir(String mnemonic, FileSystem fileSystem) {
	Path stashDir = getStashBase(fileSystem.getPath(this.sandboxBase.getPathString()));
	try {
	stashDir.createDirectory();
	if (!maybeClearExistingStash(stashDir)) {
	return null;
	}
	} catch (IOException e) {
	turnOffReuse(
	"Error creating sandbox stash dir %s, disabling sandbox reuse: %s\n",
	stashDir, e.getMessage());
	return null;
	}
	Path mnemonicStashDir = stashDir.getChild(mnemonic);
	try {
	mnemonicStashDir.createDirectory();
	return mnemonicStashDir;
	} catch (IOException e) {
	turnOffReuse("Error creating mnemonic stash dir %s: %s\n", mnemonicStashDir, e.getMessage());
	return null;
	}
	}

	private static Path getStashBase(Path sandboxBase) {
	return sandboxBase.getChild(SANDBOX_STASH_BASE);
	}

	/**
	* Clears away existing stash if this is the first access to the stash in this Blaze server
	* instance.
	*
	* @param stashPath Path of the stashes.
	* @return True unless there was an error deleting sandbox stashes.
	*/
	private boolean maybeClearExistingStash(Path stashPath) {
	synchronized (stash) {
	if (stash.getAndIncrement() == 0) {
	try {
	for (Path directoryEntry : stashPath.getDirectoryEntries()) {
	directoryEntry.deleteTree();
	}
	} catch (IOException e) {
	turnOffReuse("Unable to clear old sandbox stash %s: %s\n", stashPath, e.getMessage());
	return false;
	}
	}
	}
	return true;
	}

	private void turnOffReuse(String fmt, Object... args) {
	reuseSandboxDirectories = false;
	if (warnedAboutTurningOffReuse.compareAndSet(false, true)) {
	logger.atWarning().logVarargs("Turning off sandbox reuse: " + fmt, args);
	}
	}

	public static void initialize(
	String workspaceName, Path sandboxBase, SandboxOptions options, TreeDeleter treeDeleter) {
	if (options.reuseSandboxDirectories) {
	if (instance == null) {
	instance =
	new SandboxStash(
	workspaceName, sandboxBase, options.experimentalInMemorySandboxStashes);
	} else {
	if (!Objects.equals(workspaceName, instance.workspaceName)) {
	Path stashBase = getStashBase(instance.sandboxBase);
	try (SilentCloseable c = Profiler.instance().profile("treeDeleter.deleteTree")) {
	for (Path directoryEntry : stashBase.getDirectoryEntries()) {
	treeDeleter.deleteTree(directoryEntry);
	}
	} catch (IOException e) {
	instance.turnOffReuse(
	"Unable to clear old sandbox stash %s: %s\n", stashBase, e.getMessage());
	}
	instance =
	new SandboxStash(
	workspaceName, sandboxBase, options.experimentalInMemorySandboxStashes);
	}
	instance.inMemoryStashes = options.experimentalInMemorySandboxStashes;
	}
	} else {
	instance = null;
	}
	}

	public static boolean useInMemoryStashes() {
	Preconditions.checkNotNull(instance);
	return instance.inMemoryStashes;
	}

	public static void setPathContents(Path path, SandboxContents contents) {
	Preconditions.checkNotNull(instance);
	instance.pathToContents.put(path, contents);
	}

	public static void setLastModified(Path path, Long lastModified) {
	if (instance != null) {
	instance.pathToLastModified.put(path, lastModified);
	}
	}

	public static boolean gotInstance() {
	return instance != null;
	}

	public static void shutdown() {
	if (instance != null) {
	instance.stashFileListingPool.shutdown();
	}
	}

	/** Cleans up the entire current stash, if any. Cleaning may be asynchronous. */
	static void clean(TreeDeleter treeDeleter, Path sandboxBase) {
	Path stashDir = getStashBase(sandboxBase);
	if (!stashDir.isDirectory()) {
	return;
	}
	Path stashTrashDir = stashDir.getChild("__trash");
	try {
	stashDir.renameTo(stashTrashDir);
	} catch (IOException e) {
	// If we couldn't move the stashdir away for deletion, we need to delete it synchronously
	// in place, so we can't use the treeDeleter.
	treeDeleter = null;
	stashTrashDir = stashDir;
	}
	try {
	if (treeDeleter != null) {
	treeDeleter.deleteTree(stashTrashDir);
	} else {
	stashTrashDir.deleteTree();
	}
	} catch (IOException e) {
	logger.atWarning().withCause(e).log("Failed to clean sandbox stash %s", stashDir);
	}

	if (instance != null) {
	instance.stashPathToRunfilesDir.clear();
	instance.pathToContents.clear();
	instance.sandboxToTarget.clear();
	instance.pathToLastModified.clear();
	}
	}

	/**
	* Test actions are guaranteed to have a runfiles directory with the test name as part of the
	* name. The path to the directory is unique between tests. If two tests (foo and bar) have the
	* directory <source-root>/pkg/my_runfiles as part of their runfiles and this directory contains
	* 1000 files, we would be symlinking the 1000 files for each test since the paths do not
	* coincide. To make sure we can reuse the runfiles directory we must rename the old runfiles
	* directory for the action that was stashed to the path that is expected by the current test.
	*/
	private static boolean isTestAction(String mnemonic) {
	return mnemonic.equals(TEST_RUNNER_MNEMONIC);
	}

	/**
	* Test actions are split in two spawns. The first one runs the test and the second generates the
	* XML output from the test log. We do not want the second spawn to reuse the stash because it
	* doesn't contain the inputs needed to run the test; if it reused it, it would be expensive in
	* two ways: it would have to clean up all the inputs, and it would destroy a valid stash that a
	* different test could potentially use. If we are running coverage, there might be a third spawn
	* for coverage where we apply the same reasoning.
	*
	* <p>We identify the second and third spawn because they have a single output.
	*/
	private static boolean isTestXmlGenerationOrCoverageSpawn(
	String mnemonic, SandboxOutputs outputs) {
	return isTestAction(mnemonic) && outputs.files().size() == 1;
	}

	private static String getCurrentRunfilesDir(Map<String, String> environment) {
	return environment.get("TEST_WORKSPACE") + "/" + environment.get(TEST_SRCDIR);
	}

	private ImmutableList<Path> sortStashesByMatchingTargetSegments(
	Label target, Collection<Path> stashes) {
	List<Path> sortedStashes = new ArrayList<>(stashes);
	Map<Path, Integer> countMap = new HashMap<>();
	String[] targetStr = null;
	if (target != null) {
	targetStr = target.getPackageName().split("/");
	}
	for (Path stash : stashes) {
	Label stashTarget = sandboxToTarget.getOrDefault(stash, /* defaultValue= */ null);
	if (target == null) {
	countMap.put(stash, stashTarget == null ? 1 : 0);
	} else {
	countMap.put(
	stash,
	stashTarget == null
	? 0
	: Arrays.mismatch(targetStr, stashTarget.getPackageName().split("/")));
	}
	}
	return ImmutableList.sortedCopyOf(
	Comparator.comparingInt(countMap::get).reversed(), sortedStashes);
	}

	private void updateStashContentsAfterRunfilesMove(
	String stashedRunfiles, String currentRunfiles, SandboxContents stashContents) {
	ImmutableList<String> stashedRunfilesSegments =
	ImmutableList.copyOf(PathFragment.create(stashedRunfiles).segments());
	SandboxContents runfilesStashContents = stashContents;
	for (int i = 0; i < stashedRunfilesSegments.size() - 1; i++) {
	runfilesStashContents =
	Preconditions.checkNotNull(
	runfilesStashContents.dirMap().get(stashedRunfilesSegments.get(i)));
	}
	runfilesStashContents =
	runfilesStashContents.dirMap().remove(stashedRunfilesSegments.getLast());

	ImmutableList<String> currentRunfilesSegments =
	ImmutableList.copyOf(PathFragment.create(currentRunfiles).segments());
	SandboxContents currentStashContents = stashContents;
	for (int i = 0; i < currentRunfilesSegments.size() - 1; i++) {
	String segment = currentRunfilesSegments.get(i);
	currentStashContents.dirMap().putIfAbsent(segment, new SandboxContents());
	currentStashContents = currentStashContents.dirMap().get(segment);
	}
	currentStashContents.dirMap().put(currentRunfilesSegments.getLast(), runfilesStashContents);
	}
	}