src/main/java/com/google/devtools/build/lib/remote/RemoteExternalOverlayFileSystem.java

// Copyright 2025 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.remote;

import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static com.google.common.util.concurrent.Futures.immediateCancelledFuture;
import static com.google.devtools.build.lib.remote.util.Utils.getFromFuture;
import static com.google.devtools.build.lib.remote.util.Utils.waitForBulkTransfer;
import static com.google.devtools.build.lib.util.StringEncoding.unicodeToInternal;

import build.bazel.remote.execution.v2.Digest;
import build.bazel.remote.execution.v2.Directory;
import build.bazel.remote.execution.v2.Tree;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.actions.ActionInputHelper;
import com.google.devtools.build.lib.actions.ActionInputPrefetcher;
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.remote.common.BulkTransferException;
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
import com.google.devtools.build.lib.remote.util.DigestUtil;
import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
import com.google.devtools.build.lib.remote.util.Utils;
import com.google.devtools.build.lib.server.FailureDetails;
import com.google.devtools.build.lib.skyframe.SkyFunctions;
import com.google.devtools.build.lib.vfs.DetailedIOException;
import com.google.devtools.build.lib.vfs.DigestHashFunction;
import com.google.devtools.build.lib.vfs.Dirent;
import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.FileSymlinkLoopException;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.SymlinkTargetType;
import com.google.devtools.build.lib.vfs.Symlinks;
import com.google.devtools.build.skyframe.MemoizingEvaluator;
import com.google.devtools.build.skyframe.SkyFunctionException;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.io.OutputStream;
import java.nio.channels.SeekableByteChannel;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.function.Consumer;
import javax.annotation.Nullable;

/**
 * A file system that overlays the native file system with a {@link RemoteExternalFileSystem} for
 * the "external" directory, which contains the contents of external repositories.
 *
 * <p>Each external repository can either be materialized to the native file system or kept in
 * memory in the {@link RemoteExternalFileSystem}.
 */
public final class RemoteExternalOverlayFileSystem extends FileSystem
    implements SubtreeMaterializer {
  private final PathFragment externalDirectory;
  private final int externalDirectorySegmentCount;
  private final FileSystem nativeFs;
  private final RemoteExternalFileSystem externalFs;
  private final ConcurrentHashMap<String, Future<Void>> materializations =
      new ConcurrentHashMap<>();
  // As long as a repo name appears as a key in this map, the repo contents are available in
  // externalFs.
  private final ConcurrentHashMap<String, String> markerFileContents = new ConcurrentHashMap<>();
  private final Set<String> reposWithLostFiles = ConcurrentHashMap.newKeySet();

  // Per-build information that is set in beforeCommand and cleared in afterCommand.
  @Nullable private CombinedCache cache;
  @Nullable private AbstractActionInputPrefetcher inputPrefetcher;
  @Nullable private Reporter reporter;
  @Nullable private String buildRequestId;
  @Nullable private String commandId;
  @Nullable private MemoizingEvaluator evaluator;
  @Nullable private Duration remoteCacheTtl;
  @Nullable private ExecutorService materializationExecutor;

  public RemoteExternalOverlayFileSystem(PathFragment externalDirectory, FileSystem nativeFs) {
    super(nativeFs.getDigestFunction());
    this.externalDirectory = externalDirectory;
    this.externalDirectorySegmentCount = externalDirectory.segmentCount();
    this.nativeFs = nativeFs;
    this.externalFs = new RemoteExternalFileSystem(nativeFs.getDigestFunction());
  }

  @SuppressWarnings("AllowVirtualThreads")
  public void beforeCommand(
      CombinedCache cache,
      AbstractActionInputPrefetcher inputPrefetcher,
      Reporter reporter,
      String buildRequestId,
      String commandId,
      MemoizingEvaluator evaluator,
      Duration remoteCacheTtl) {
    checkState(
        this.cache == null
            && this.inputPrefetcher == null
            && this.reporter == null
            && this.buildRequestId == null
            && this.commandId == null
            && this.evaluator == null
            && this.remoteCacheTtl == null
            && this.materializationExecutor == null);
    this.cache = cache;
    this.inputPrefetcher = inputPrefetcher;
    this.reporter = reporter;
    this.buildRequestId = buildRequestId;
    this.commandId = commandId;
    this.evaluator = evaluator;
    this.remoteCacheTtl = remoteCacheTtl;
    this.materializationExecutor =
        Executors.newThreadPerTaskExecutor(
            Thread.ofVirtual().name("remote-repo-materialization-", 0).factory());
  }

  public void afterCommand() {
    if (cache == null) {
      // Not all commands cause beforeCommand to be called, but afterCommand is called
      // unconditionally.
      return;
    }
    this.cache = null;
    this.inputPrefetcher = null;
    this.reporter = null;
    this.buildRequestId = null;
    this.commandId = null;
    this.remoteCacheTtl = null;
    // Materializations happen synchronously and upon request by other repo rules, so there is no
    // reason to await their orderly completion in afterCommand.
    materializationExecutor.shutdownNow();
    materializationExecutor = null;
    // Clean up the in-memory contents of materialized repos to save memory, or those that need to
    // be refetched to recover files that the remote cache has lost. This wouldn't be safe to do
    // eagerly as ongoing repo rule evaluations may still refer to the in-memory content and
    // refetching is not atomic.
    materializations.forEach(
        1,
        (repoName, materializationState) ->
            materializationState.state() == Future.State.SUCCESS
                    || reposWithLostFiles.contains(repoName)
                ? repoName
                : null,
        this::evictInMemoryRepo);
    invalidateRepoDirectories(evaluator, reposWithLostFiles);
    reposWithLostFiles.clear();
    this.evaluator = null;
  }

  /** Removes the contents of the given repo from the in-memory overlay file system. */
  private void evictInMemoryRepo(String repoName) {
    try {
      externalFs.deleteTree(externalDirectory.getChild(repoName));
    } catch (IOException e) {
      throw new IllegalStateException("In-memory file system is not expected to throw", e);
    }
    materializations.remove(repoName);
    markerFileContents.remove(repoName);
  }

  /** Invalidates the {@link SkyFunctions#REPOSITORY_DIRECTORY} nodes of the given repos. */
  private static void invalidateRepoDirectories(
      MemoizingEvaluator evaluator, Set<String> repoNames) {
    if (repoNames.isEmpty()) {
      return;
    }
    evaluator.delete(
        k ->
            k.functionName().equals(SkyFunctions.REPOSITORY_DIRECTORY)
                && repoNames.contains(((RepositoryName) k.argument()).getName()));
  }

  /**
   * Injects the given remote contents, possibly prefetching some files, and returns true on
   * success.
   */
  public boolean injectRemoteRepo(RepositoryName repo, Tree remoteContents, String markerFile)
      throws IOException, InterruptedException {
    var repoDir = externalDirectory.getChild(repo.getName());
    deleteTree(repoDir);
    var unused = delete(externalDirectory.getChild(repo.getMarkerFileName()));
    var childMap =
        remoteContents.getChildrenList().stream()
            .collect(
                toImmutableMap(cache.digestUtil::compute, directory -> directory, (a, b) -> a));
    var filesToPrefetch = new ArrayList<PathFragment>();
    injectRecursively(
        externalFs,
        repoDir,
        remoteContents.getRoot(),
        childMap,
        filesToPrefetch::add,
        Instant.now().plus(remoteCacheTtl));
    try {
      // TODO: This prefetches a large number of small files. Investigate whether BatchReadBlobs
      // would be more efficient.
      prefetch(filesToPrefetch);
    } catch (BulkTransferException e) {
      if (e.allCausedByCacheNotFoundException()) {
        // The cache has lost the .bzl files, which should be treated just like a cache miss.
        externalFs.deleteTree(repoDir);
        return false;
      }
      throw e;
    }
    // Create the repo directory on disk so that readdir reflects the overlaid state of the external
    // directory.
    nativeFs.createDirectoryAndParents(repoDir);
    // Keep the marker file contents in memory so that it can be written out when the repo is
    // materialized. This doubles as a presence marker for the in-memory repo contents.
    markerFileContents.put(repo.getName(), markerFile);
    return true;
  }

  private static void injectRecursively(
      RemoteExternalFileSystem fs,
      PathFragment path,
      Directory dir,
      ImmutableMap<Digest, Directory> childMap,
      Consumer<PathFragment> filesToPrefetch,
      Instant expirationTime)
      throws IOException {
    fs.createDirectoryAndParents(path);
    for (var file : dir.getFilesList()) {
      var filePath = path.getRelative(unicodeToInternal(file.getName()));
      if (shouldPrefetch(filePath)) {
        filesToPrefetch.accept(filePath);
      }
      fs.injectFile(
          filePath,
          // Using the *WithMaterializationData variant ensures that the file benefits from the
          // FileContentsProxy optimization to avoid widespread invalidation when it is
          // materialized later, even if expiration times aren't relevant (depends on the usage
          // of the lease extension).
          FileArtifactValue.RemoteFileArtifactValue.createWithMaterializationData(
              DigestUtil.toBinaryDigest(file.getDigest()),
              file.getDigest().getSizeBytes(),
              /* locationIndex= */ 1,
              expirationTime.toEpochMilli(),
              /* materializationExecPath= */ null));
      fs.getPath(filePath).setExecutable(file.getIsExecutable());
      // The RE API does not track whether a file is readable or writable. We choose to make all
      // files readable and not writable to ensure that other repo rules can't accidentally modify
      // the cached repo.
      fs.getPath(filePath).setWritable(false);
    }
    for (var symlink : dir.getSymlinksList()) {
      fs.getPath(path.getRelative(unicodeToInternal(symlink.getName())))
          .createSymbolicLink(PathFragment.create(unicodeToInternal(symlink.getTarget())));
    }
    for (var subdirNode : dir.getDirectoriesList()) {
      var subdirPath = path.getRelative(unicodeToInternal(subdirNode.getName()));
      var subdir = childMap.get(subdirNode.getDigest());
      if (subdir == null) {
        throw new IOException(
            "Directory %s with digest %s not found in tree"
                .formatted(subdirPath, subdirNode.getDigest().getHash()));
      }
      injectRecursively(fs, subdirPath, subdir, childMap, filesToPrefetch, expirationTime);
    }
  }

  /**
   * Materializes the given external repository to the native file system if it hasn't been
   * materialized yet. This method blocks until the materialization is complete.
   *
   * <p>This should only be used for cases in which the given repo is accessed non-hermetically,
   * such as when another repo rule that depends on its files executes a command. Selective reads by
   * Bazel or local actions are handled automatically by the file system or {@link
   * AbstractActionInputPrefetcher}.
   */
  public void ensureMaterialized(RepositoryName repo, ExtendedEventHandler reporter)
      throws IOException, InterruptedException {
    if (!markerFileContents.containsKey(repo.getName())) {
      // The repo has not been injected into the in-memory file system.
      return;
    }
    var unused =
        getFromFuture(
            materializations.computeIfAbsent(
                repo.getName(),
                unusedRepoName ->
                    materializationExecutor.submit(
                        () -> {
                          doMaterialize(repo, reporter);
                          return null;
                        })));
  }

  private void doMaterialize(RepositoryName repo, ExtendedEventHandler reporter)
      throws IOException, InterruptedException {
    reporter.handle(Event.debug("Materializing remote repo %s".formatted(repo)));
    materializeSubtree(externalDirectory.getChild(repo.getName()));

    // After the repo has been copied, atomically materialize the marker file. This ensures that the
    // repo doesn't have to be refetched after the next server restart.
    var markerFile = nativeFs.getPath(externalDirectory.getChild(repo.getMarkerFileName()));
    var markerFileSibling =
        nativeFs.getPath(externalDirectory.getChild(repo.getMarkerFileName() + ".tmp"));
    FileSystemUtils.writeContentAsLatin1(
        markerFileSibling, markerFileContents.remove(repo.getName()));
    markerFileSibling.renameTo(markerFile);
  }

  private void prefetch(Iterable<PathFragment> paths) throws IOException, InterruptedException {
    var unused =
        getFromFuture(
            inputPrefetcher.prefetchFiles(
                /* action= */ null,
                Iterables.transform(paths, ActionInputHelper::fromPath),
                actionInput -> externalFs.getMetadata(actionInput.getExecPath()),
                ActionInputPrefetcher.Priority.CRITICAL,
                ActionInputPrefetcher.Reason.INPUTS));
  }

  /**
   * Informs the FS that no cache is available and in-memory repos can no longer be used.
   *
   * <p>Must not be called while accessing external repos.
   */
  public void notifyNoCacheAvailable(MemoizingEvaluator evaluator) {
    checkState(materializationExecutor == null, "must not be called when active");
    var reposToDiscard = ImmutableSet.copyOf(markerFileContents.keySet());
    reposToDiscard.forEach(this::evictInMemoryRepo);
    invalidateRepoDirectories(evaluator, reposToDiscard);
  }

  /**
   * Materializes the subtree rooted at the given path to the native file system if it lies in a
   * repo whose contents are currently only available in memory.
   *
   * <p>This is used to make the files below a source directory action input available to local
   * actions, which access them through the native file system.
   */
  @Override
  public void ensureSubtreeMaterialized(PathFragment path)
      throws IOException, InterruptedException {
    if (fsForPath(path) != externalFs) {
      return;
    }
    materializeSubtree(path);
  }

  private void materializeSubtree(PathFragment path) throws IOException, InterruptedException {
    var files = new LinkedHashSet<PathFragment>();
    var symlinks = new LinkedHashSet<PathFragment>();
    var root = externalFs.getPath(path);
    if (root.isSymbolicLink()) {
      symlinks.add(path);
      root = root.resolveSymbolicLinks();
    }
    collectAndCreateDirectories(root, files, symlinks, new HashSet<>());
    prefetch(files);
    // Create symlinks last as some platforms don't allow creating a symlink to a non-existent
    // target.
    prefetch(symlinks);
  }

  private void collectAndCreateDirectories(
      Path dir, Set<PathFragment> files, Set<PathFragment> symlinks, Set<PathFragment> visitedDirs)
      throws IOException {
    if (!visitedDirs.add(dir.asFragment())) {
      return;
    }
    nativeFs.createDirectoryAndParents(dir.asFragment());
    for (var dirent : dir.readdir(Symlinks.NOFOLLOW)) {
      var child = dir.getChild(dirent.getName());
      switch (dirent.getType()) {
        case FILE -> files.add(child.asFragment());
        case SYMLINK -> {
          symlinks.add(child.asFragment());
          // The symlink chain is reproduced verbatim on the native file system, but its target may
          // lie outside the materialized subtree and has to be materialized as well so that the
          // chain doesn't dangle.
          Path target;
          try {
            target = child.resolveSymbolicLinks();
          } catch (FileNotFoundException | FileSymlinkLoopException e) {
            // Dangling symlinks and symlink loops are reproduced verbatim.
            continue;
          }
          // TODO(#30160): RepositoryUtils.replantSymlinks currently ensures that all symlinks
          // within a remotely cacheable external repo stay within that repo. If that changes, new
          // logic has to be added here to prefetch such files correctly.
          if (target.isDirectory(Symlinks.NOFOLLOW)) {
            collectAndCreateDirectories(target, files, symlinks, visitedDirs);
          } else {
            files.add(target.asFragment());
          }
        }
        case DIRECTORY -> collectAndCreateDirectories(child, files, symlinks, visitedDirs);
        default -> throw new IOException("Unsupported file type: " + dirent);
      }
    }
  }

  /** Whether the file with the given path should be materialized eagerly when injecting a repo. */
  private static boolean shouldPrefetch(PathFragment path) {
    // .bzl files are typically small and the loads between them can form complex DAGs that can only
    // be discovered layer by layer, so prefetching is worthwhile to reduce the number of sequential
    // cache requests.
    // The REPO.bazel file, if present, is a dependency of any package and will thus have to be
    // fetched anyway.
    return path.getFileExtension().equals("bzl") || path.getBaseName().equals("REPO.bazel");
  }

  @Override
  public FileSystem getHostFileSystem() {
    return nativeFs.getHostFileSystem();
  }

  @Override
  public boolean isFilePathCaseSensitive() {
    return nativeFs.isFilePathCaseSensitive();
  }

  // Always mirror tree deletions to the underlying native file system to support bazel clean and
  // repository refetching.

  @Override
  public void deleteTree(PathFragment path) throws IOException {
    nativeFs.getPath(path).deleteTree();
    externalFs.getPath(path).deleteTree();
  }

  @Override
  public void deleteTreesBelow(PathFragment dir) throws IOException {
    nativeFs.getPath(dir).deleteTreesBelow();
    externalFs.getPath(dir).deleteTreesBelow();
  }

  // All other methods delegate to the file system given by this method. It is important to override
  // each non-final FileSystem method to benefit from optimizations implemented in the respective
  // underlying file systems.
  private FileSystem fsForPath(PathFragment path) {
    if (path.startsWith(externalDirectory) && !path.equals(externalDirectory)) {
      String repoName = path.getSegment(externalDirectorySegmentCount);
      var hasBeenInjected = markerFileContents.containsKey(repoName);
      var hasBeenMaterialized =
          materializations.getOrDefault(repoName, immediateCancelledFuture()).state()
              == Future.State.SUCCESS;
      if (hasBeenInjected && !hasBeenMaterialized) {
        // The repo may have been deleted due to refetching. Clean up in-memory state if that is the
        // case.
        if (externalFs.getPath(externalDirectory.getChild(repoName)).exists()) {
          return externalFs;
        }
        materializations.remove(repoName);
        markerFileContents.remove(repoName);
      }
      // Fall back to the native file system if the repo has been materialized, deleted, or never
      // injected.
    }
    return nativeFs;
  }

  @Override
  public boolean delete(PathFragment path) throws IOException {
    return fsForPath(path).delete(path);
  }

  @Override
  public byte[] getDigest(PathFragment path) throws IOException {
    return fsForPath(path).getDigest(path);
  }

  @Nullable
  @Override
  public byte[] getFastDigest(PathFragment path) throws IOException {
    return fsForPath(path).getFastDigest(path);
  }

  @Override
  public boolean supportsModifications(PathFragment path) {
    return fsForPath(path).supportsModifications(path);
  }

  @Override
  public boolean supportsSymbolicLinksNatively(PathFragment path) {
    return fsForPath(path).supportsSymbolicLinksNatively(path);
  }

  @Override
  public boolean supportsHardLinksNatively(PathFragment path) {
    return fsForPath(path).supportsHardLinksNatively(path);
  }

  @Override
  public boolean createDirectory(PathFragment path) throws IOException {
    return fsForPath(path).createDirectory(path);
  }

  @Override
  public void createDirectoryAndParents(PathFragment path) throws IOException {
    fsForPath(path).createDirectoryAndParents(path);
  }

  @Override
  public long getFileSize(PathFragment path, boolean followSymlinks) throws IOException {
    return fsForPath(path).getFileSize(path, followSymlinks);
  }

  @Override
  public long getLastModifiedTime(PathFragment path, boolean followSymlinks) throws IOException {
    return fsForPath(path).getLastModifiedTime(path, followSymlinks);
  }

  @Override
  public void setLastModifiedTime(PathFragment path, long newTime) throws IOException {
    fsForPath(path).setLastModifiedTime(path, newTime);
  }

  @Override
  public FileStatus stat(PathFragment path, boolean followSymlinks) throws IOException {
    return fsForPath(path).stat(path, followSymlinks);
  }

  @Override
  public void createSymbolicLink(
      PathFragment linkPath, PathFragment targetFragment, SymlinkTargetType hint)
      throws IOException {
    fsForPath(linkPath).createSymbolicLink(linkPath, targetFragment, hint);
  }

  @Override
  public PathFragment readSymbolicLink(PathFragment path) throws IOException {
    return fsForPath(path).readSymbolicLink(path);
  }

  @Override
  public boolean exists(PathFragment path, boolean followSymlinks) {
    return fsForPath(path).exists(path, followSymlinks);
  }

  @Override
  public boolean exists(PathFragment path) {
    return fsForPath(path).exists(path);
  }

  @Override
  public Collection<String> getDirectoryEntries(PathFragment path) throws IOException {
    return fsForPath(path).getDirectoryEntries(path);
  }

  @Override
  public boolean isReadable(PathFragment path) throws IOException {
    return fsForPath(path).isReadable(path);
  }

  @Override
  public void setReadable(PathFragment path, boolean readable) throws IOException {
    fsForPath(path).setReadable(path, readable);
  }

  @Override
  public boolean isWritable(PathFragment path) throws IOException {
    return fsForPath(path).isWritable(path);
  }

  @Override
  public void setWritable(PathFragment path, boolean writable) throws IOException {
    fsForPath(path).setWritable(path, writable);
  }

  @Override
  public boolean isExecutable(PathFragment path) throws IOException {
    return fsForPath(path).isExecutable(path);
  }

  @Override
  public void setExecutable(PathFragment path, boolean executable) throws IOException {
    fsForPath(path).setExecutable(path, executable);
  }

  @Override
  public InputStream getInputStream(PathFragment path) throws IOException {
    return fsForPath(path).getInputStream(path);
  }

  @Override
  public SeekableByteChannel createReadWriteByteChannel(PathFragment path) throws IOException {
    return fsForPath(path).createReadWriteByteChannel(path);
  }

  @Override
  public OutputStream getOutputStream(PathFragment path, boolean append, boolean internal)
      throws IOException {
    return fsForPath(path).getOutputStream(path, append, internal);
  }

  @Override
  public void renameTo(PathFragment sourcePath, PathFragment targetPath) throws IOException {
    fsForPath(sourcePath).renameTo(sourcePath, targetPath);
  }

  @Override
  public void createFSDependentHardLink(PathFragment linkPath, PathFragment originalPath)
      throws IOException {
    fsForPath(originalPath).createFSDependentHardLink(linkPath, originalPath);
  }

  @Override
  public File getIoFile(PathFragment path) {
    return fsForPath(path).getIoFile(path);
  }

  @Override
  public java.nio.file.Path getNioPath(PathFragment path) {
    return fsForPath(path).getNioPath(path);
  }

  @Override
  public String getFileSystemType(PathFragment path) {
    return fsForPath(path).getFileSystemType(path);
  }

  @Override
  public byte[] getxattr(PathFragment path, String name, boolean followSymlinks)
      throws IOException {
    return fsForPath(path).getxattr(path, name, followSymlinks);
  }

  @Override
  public Path resolveSymbolicLinks(PathFragment path) throws IOException {
    // Ensure that the return value doesn't leave the overlay file system.
    return getPath(fsForPath(path).resolveSymbolicLinks(path).asFragment());
  }

  @Nullable
  @Override
  public FileStatus statNullable(PathFragment path, boolean followSymlinks) {
    return fsForPath(path).statNullable(path, followSymlinks);
  }

  @Nullable
  @Override
  public FileStatus statIfFound(PathFragment path, boolean followSymlinks) throws IOException {
    return fsForPath(path).statIfFound(path, followSymlinks);
  }

  @Override
  public boolean isFile(PathFragment path, boolean followSymlinks) {
    return fsForPath(path).isFile(path, followSymlinks);
  }

  @Override
  public boolean isSpecialFile(PathFragment path, boolean followSymlinks) {
    return fsForPath(path).isSpecialFile(path, followSymlinks);
  }

  @Override
  public boolean isSymbolicLink(PathFragment path) {
    return fsForPath(path).isSymbolicLink(path);
  }

  @Override
  public boolean isDirectory(PathFragment path, boolean followSymlinks) {
    return fsForPath(path).isDirectory(path, followSymlinks);
  }

  @Override
  public PathFragment readSymbolicLinkUnchecked(PathFragment path) throws IOException {
    return fsForPath(path).readSymbolicLinkUnchecked(path);
  }

  @Override
  public Collection<Dirent> readdir(PathFragment path, boolean followSymlinks) throws IOException {
    return fsForPath(path).readdir(path, followSymlinks);
  }

  @Override
  public void chmod(PathFragment path, int mode) throws IOException {
    fsForPath(path).chmod(path, mode);
  }

  private final class RemoteExternalFileSystem
      extends RemoteActionFileSystem.RemoteInMemoryFileSystem {

    RemoteExternalFileSystem(DigestHashFunction hashFunction) {
      super(hashFunction);
    }

    private RemoteActionExecutionContext makeRemoteContext(PathFragment relativePath) {
      String repoName = relativePath.subFragment(0, 1).getBaseName();
      var metadata =
          TracingMetadataUtils.buildMetadata(
              buildRequestId, commandId, repoName, /* actionMetadata= */ null);
      // Files in the remote external repo that Bazel reads are worth writing through to the
      // disk cache, as they are likely to be read again on future cold builds.
      return RemoteActionExecutionContext.create(metadata)
          .withReadCachePolicy(RemoteActionExecutionContext.CachePolicy.ANY_CACHE)
          .withWriteCachePolicy(RemoteActionExecutionContext.CachePolicy.ANY_CACHE);
    }

    private FileArtifactValue getMetadata(PathFragment path) throws IOException {
      var status = stat(path, /* followSymlinks= */ false);
      if (!status.isSymbolicLink()) {
        return ((RemoteActionFileSystem.RemoteInMemoryFileInfo) status).getMetadata();
      }
      return FileArtifactValue.createForUnresolvedSymlink(externalFs.getPath(path));
    }

    @Override
    public InputStream getInputStream(PathFragment path) throws IOException {
      // .bzl and REPO.bazel files are prefetched to the native file system during injection, but
      // only if they are regular files, a symlink with such a name is kept in the in-memory overlay
      // only. We thus need to follow symlinks before attempting to read a supposedly prefetched
      // file.
      path = resolveSymbolicLinks(path).asFragment();
      if (shouldPrefetch(path)) {
        return nativeFs.getInputStream(path);
      }
      var relativePath = path.relativeTo(externalDirectory);
      if (!(stat(path, /* followSymlinks= */ true)
          instanceof RemoteActionFileSystem.RemoteInMemoryFileInfo info)) {
        throw Errno.EISDIR.exception(path);
      }
      reporter.post(
          new ExtendedEventHandler.FetchProgress() {
            @Override
            public String getResourceIdentifier() {
              return relativePath.getPathString();
            }

            @Override
            public String getProgress() {
              return "(%s)".formatted(Utils.bytesCountToDisplayString(info.getSize()));
            }

            @Override
            public boolean isFinished() {
              return false;
            }
          });
      var digest = DigestUtil.buildDigest(info.getMetadata().getDigest(), info.getSize());
      try {
        var contentFuture =
            cache.downloadBlob(
                makeRemoteContext(relativePath),
                path.getPathString(),
                /* execPath= */ null,
                digest);
        waitForBulkTransfer(ImmutableList.of(contentFuture));
        return new ByteArrayInputStream(contentFuture.get());
      } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new InterruptedIOException("interrupted while waiting for remote file transfer");
      } catch (BulkTransferException e) {
        if (e.allCausedByCacheNotFoundException()) {
          reposWithLostFiles.add(relativePath.getSegment(0));
          throw new DetailedIOException(
              "%s/%s with digest %s is no longer available in the remote cache"
                  .formatted(
                      externalDirectory.getBaseName(), relativePath, DigestUtil.toString(digest)),
              e,
              FailureDetails.Filesystem.Code.REMOTE_FILE_EVICTED,
              SkyFunctionException.Transience.TRANSIENT);
        }
        throw e;
      } catch (ExecutionException e) {
        throw new IllegalStateException("waitForBulkTransfer should have thrown", e);
      } finally {
        reporter.post(
            new ExtendedEventHandler.FetchProgress() {
              @Override
              public String getResourceIdentifier() {
                return relativePath.getPathString();
              }

              @Override
              public String getProgress() {
                return "";
              }

              @Override
              public boolean isFinished() {
                return true;
              }
            });
      }
    }

    @Override
    public byte[] getDigest(PathFragment path) throws IOException {
      // All regular files in this file system are remote files, whose digest is known in advance
      // and returned by the base implementation of getFastDigest, which also correctly reports
      // errors such as EISDIR for paths that don't resolve to regular files. The base
      // implementation of getDigest would instead download the file contents to hash them.
      return getFastDigest(path);
    }
  }
}