src/main/java/com/google/devtools/build/lib/sandbox/AbstractSandboxSpawnRunner.java

// Copyright 2016 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.sandbox;

import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.stream.Collectors.joining;

import com.google.common.base.Preconditions;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.devtools.build.lib.actions.ActionExecutionMetadata;
import com.google.devtools.build.lib.actions.EnvironmentalExecException;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.ResourceManager;
import com.google.devtools.build.lib.actions.ResourceManager.ResourceHandle;
import com.google.devtools.build.lib.actions.ResourceManager.ResourcePriority;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.SpawnMetrics;
import com.google.devtools.build.lib.actions.SpawnResult;
import com.google.devtools.build.lib.actions.SpawnResult.Status;
import com.google.devtools.build.lib.actions.Spawns;
import com.google.devtools.build.lib.actions.UserExecException;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.EventKind;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.exec.BinTools;
import com.google.devtools.build.lib.exec.ExecutionOptions;
import com.google.devtools.build.lib.exec.SpawnExecutingEvent;
import com.google.devtools.build.lib.exec.SpawnRunner;
import com.google.devtools.build.lib.exec.SpawnSchedulingEvent;
import com.google.devtools.build.lib.exec.TreeDeleter;
import com.google.devtools.build.lib.profiler.Profiler;
import com.google.devtools.build.lib.profiler.ProfilerTask;
import com.google.devtools.build.lib.profiler.SilentCloseable;
import com.google.devtools.build.lib.runtime.CommandEnvironment;
import com.google.devtools.build.lib.server.FailureDetails;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.server.FailureDetails.Sandbox.Code;
import com.google.devtools.build.lib.shell.Subprocess;
import com.google.devtools.build.lib.shell.SubprocessBuilder;
import com.google.devtools.build.lib.shell.TerminationStatus;
import com.google.devtools.build.lib.util.CommandFailureUtils;
import com.google.devtools.build.lib.util.OS;
import com.google.devtools.build.lib.util.io.FileOutErr;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import java.io.IOException;
import java.io.InputStream;
import java.time.Duration;
import java.time.Instant;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Stream;

/** Abstract common ancestor for sandbox spawn runners implementing the common parts. */
abstract class AbstractSandboxSpawnRunner implements SpawnRunner {
  private static final int LOCAL_EXEC_ERROR = -1;

  private record SubprocessResult(
      int exitCode,
      Status status,
      Duration wallTime,
      Instant startTime,
      String failureMessage,
      FailureDetail failureDetail,
      Path statisticsPath) {}

  private final SandboxOptions sandboxOptions;
  private final boolean verboseFailures;
  private final boolean expandParamFiles;
  private final ImmutableSet<Path> inaccessiblePaths;
  protected final BinTools binTools;
  private final Path execRoot;
  private final ResourceManager resourceManager;
  private final Reporter reporter;
  protected final ImmutableMap<String, String> clientEnv;

  public AbstractSandboxSpawnRunner(CommandEnvironment cmdEnv) {
    this.sandboxOptions = cmdEnv.getOptions().getOptions(SandboxOptions.class);
    ExecutionOptions executionOptions = cmdEnv.getOptions().getOptions(ExecutionOptions.class);
    this.verboseFailures = executionOptions.getVerboseFailures();
    this.expandParamFiles = executionOptions.getExpandParamFiles();
    this.inaccessiblePaths =
        sandboxOptions.getInaccessiblePaths(cmdEnv.getRuntime().getFileSystem());
    this.binTools = cmdEnv.getBlazeWorkspace().getBinTools();
    this.execRoot = cmdEnv.getExecRoot();
    this.resourceManager = cmdEnv.getLocalResourceManager();
    this.reporter = cmdEnv.getReporter();
    this.clientEnv = cmdEnv.getClientEnv();
  }

  @Override
  public final SpawnResult exec(Spawn spawn, SpawnExecutionContext context)
      throws ExecException, InterruptedException {
    ActionExecutionMetadata owner = spawn.getResourceOwner();
    context.report(SpawnSchedulingEvent.create(getName()));

    Stopwatch totalTimeStopwatch = Stopwatch.createStarted();
    SpawnMetrics.Builder spawnMetrics = SpawnMetrics.Builder.forLocalExec();

    try {
      Stopwatch prefetchStopwatch = Stopwatch.createStarted();
      try (SilentCloseable c = Profiler.instance().profile("context.prefetchInputs")) {
        context.prefetchInputsAndWait();
      }
      spawnMetrics.addSetupTime(prefetchStopwatch.elapsed());

      Stopwatch queueStopwatch = Stopwatch.createStarted();
      try (ResourceHandle ignored =
          resourceManager.acquireResources(
              owner,
              spawn.getLocalResources(),
              context.speculating()
                  ? ResourcePriority.DYNAMIC_STANDALONE
                  : ResourcePriority.LOCAL)) {
        spawnMetrics.setQueueTime(queueStopwatch.elapsed());
        context.report(SpawnExecutingEvent.create(getName()));

        Stopwatch setupTimeStopwatch = Stopwatch.createStarted();
        SandboxedSpawn sandbox;
        try (SilentCloseable c =
            Profiler.instance().profile(ProfilerTask.SANDBOX_SETUP, "sandbox.prepareSpawn")) {
          sandbox = prepareSpawn(spawn, context);
        }
        return runSpawn(
            spawn, sandbox, context, spawnMetrics, totalTimeStopwatch, setupTimeStopwatch);
      }
    } catch (IOException e) {
      FailureDetail failureDetail =
          SandboxHelpers.createFailureDetail(
              "I/O exception during sandboxed execution", Code.EXECUTION_IO_EXCEPTION);
      throw new UserExecException(e, failureDetail);
    }
  }

  @Override
  public boolean canExec(Spawn spawn) {
    return Spawns.mayBeSandboxed(spawn);
  }

  @Override
  public boolean handlesCaching() {
    return false;
  }

  protected abstract SandboxedSpawn prepareSpawn(Spawn spawn, SpawnExecutionContext context)
      throws IOException, ExecException, InterruptedException;

  private SpawnResult runSpawn(
      Spawn originalSpawn,
      SandboxedSpawn sandbox,
      SpawnExecutionContext context,
      SpawnMetrics.Builder spawnMetrics,
      Stopwatch totalTimeStopwatch,
      Stopwatch setupTimeStopwatch)
      throws ExecException, IOException, InterruptedException {
    try {
      try (SilentCloseable c =
          Profiler.instance().profile(ProfilerTask.SANDBOX_SETUP, "sandbox.createFileSystem")) {
        sandbox.createFileSystem();
      } catch (IOException e) {
        FailureDetail failureDetail =
            SandboxHelpers.createFailureDetail(
                "Could not copy inputs into sandbox", Code.COPY_INPUTS_IO_EXCEPTION);
        throw new EnvironmentalExecException(e, failureDetail);
      }
      spawnMetrics.addSetupTime(setupTimeStopwatch.elapsed());

      SubprocessResult subprocessResult;
      try (SilentCloseable c =
          Profiler.instance()
              .profile(
                  ProfilerTask.SANDBOX_PROCESS_TIME,
                  originalSpawn.getResourceOwner().getMnemonic())) {
        subprocessResult = run(originalSpawn, sandbox, context);
      }
      spawnMetrics.setExecutionWallTime(subprocessResult.wallTime());
      try (SilentCloseable c = Profiler.instance().profile("sandbox.verifyPostCondition")) {
        verifyPostCondition(originalSpawn, sandbox, context);
      }

      Stopwatch processOutputsStopwatch = Stopwatch.createStarted();
      context.lockOutputFiles(
          subprocessResult.exitCode(),
          subprocessResult.failureDetail() != null
              ? subprocessResult.failureDetail().getMessage()
              : "",
          context.getFileOutErr());

      try (SilentCloseable c =
          Profiler.instance().profile(ProfilerTask.SANDBOX_OUTPUTS, "sandbox.copyOutputs")) {
        // We copy the outputs even when the command failed.
        sandbox.copyOutputs(execRoot);
      } catch (IOException e) {
        FailureDetail failureDetail =
            SandboxHelpers.createFailureDetail(
                "Could not copy outputs from sandbox", Code.COPY_OUTPUTS_IO_EXCEPTION);
        throw new EnvironmentalExecException(e, failureDetail);
      }
      spawnMetrics.setProcessOutputsTime(processOutputsStopwatch.elapsed());

      spawnMetrics.setTotalTime(totalTimeStopwatch.elapsed());

      SpawnResult.Builder spawnResultBuilder =
          getSpawnResultBuilder(context)
              .setStatus(subprocessResult.status())
              .setExitCode(subprocessResult.exitCode())
              .setStartTime(subprocessResult.startTime())
              .setWallTimeInMs((int) subprocessResult.wallTime().toMillis())
              .setFailureMessage(subprocessResult.failureMessage())
              .setSpawnMetrics(spawnMetrics.build());
      if (subprocessResult.failureDetail() != null) {
        spawnResultBuilder.setFailureDetail(subprocessResult.failureDetail());
      }
      if (subprocessResult.statisticsPath() != null) {
        spawnResultBuilder.setResourceUsageFromProto(subprocessResult.statisticsPath());
      }
      return spawnResultBuilder.build();
    } finally {
      if (!sandboxOptions.getSandboxDebug()) {
        try (SilentCloseable c = Profiler.instance().profile("sandbox.delete")) {
          sandbox.delete();
        }
      }
    }
  }

  /** Override this method if you need to run a post condition after the action has executed */
  public void verifyPostCondition(
      Spawn originalSpawn, SandboxedSpawn sandbox, SpawnExecutionContext context)
      throws IOException {}

  private String makeFailureMessage(Spawn originalSpawn, SandboxedSpawn sandbox) {
    if (sandboxOptions.getSandboxDebug()) {
      return CommandFailureUtils.describeCommandFailure(
          /* verboseFailures= */ true,
          /* expandParamFiles= */ false,
          sandbox.getSandboxExecRoot().getPathString(),
          sandbox);
    } else {
      return CommandFailureUtils.describeCommandFailure(
          verboseFailures,
          expandParamFiles,
          sandbox.getSandboxExecRoot().getPathString(),
          originalSpawn);
    }
  }

  private SubprocessResult run(
      Spawn originalSpawn, SandboxedSpawn sandbox, SpawnExecutionContext context)
      throws IOException, InterruptedException {
    FileOutErr outErr = context.getFileOutErr();
    Duration timeout = context.getTimeout();

    SubprocessBuilder subprocessBuilder = new SubprocessBuilder(clientEnv);
    subprocessBuilder.setWorkingDirectory(sandbox.getSandboxExecRoot().getPathFile());
    // If the spawn redirects its standard output into one of its outputs, write to that output
    // inside the sandbox instead of the regular stdout capture buffer. The file is a regular
    // output, so no other special handling is necessary.
    Path stdoutPath = Spawns.getStdoutRedirectPath(originalSpawn, sandbox.getSandboxExecRoot());
    subprocessBuilder.setStdout(
        (stdoutPath != null ? stdoutPath : outErr.getOutputPath()).getPathFile());
    subprocessBuilder.setStderr(outErr.getErrorPath().getPathFile());
    subprocessBuilder.setEnv(sandbox.getEnvironment());
    subprocessBuilder.setArgv(sandbox.getArguments());
    boolean useSubprocessTimeout = sandbox.useSubprocessTimeout();
    if (useSubprocessTimeout) {
      subprocessBuilder.setTimeoutMillis(timeout.toMillis());
    }
    Instant startTime = Instant.now();
    Stopwatch executionStopwatch = Stopwatch.createStarted();
    TerminationStatus terminationStatus;
    try {
      Subprocess subprocess = subprocessBuilder.start();
      subprocess.getOutputStream().close();
      try {
        subprocess.waitFor();
        terminationStatus = new TerminationStatus(subprocess.exitValue(), subprocess.timedout());
      } catch (InterruptedException e) {
        subprocess.destroyAndWait();
        throw e;
      }
    } catch (IOException e) {
      String exceptionMsg = e.getMessage() == null ? e.getClass().getName() : e.getMessage();
      String sandboxDebugOutput = getSandboxDebugOutput(sandbox);

      StringBuilder msg = new StringBuilder("Action failed to execute: java.io.IOException: ");
      msg.append(exceptionMsg);
      msg.append("\n");
      if (!sandboxDebugOutput.isEmpty()) {
        msg.append("Sandbox debug output:\n");
        msg.append(sandboxDebugOutput);
        msg.append("\n");
      }

      outErr.getErrorStream().write(msg.toString().getBytes(UTF_8));
      outErr.getErrorStream().flush();
      String message = makeFailureMessage(originalSpawn, sandbox);
      FailureDetail failureDetail =
          SandboxHelpers.createFailureDetail(message, Code.SUBPROCESS_START_FAILED);
      return new SubprocessResult(
          LOCAL_EXEC_ERROR,
          Status.EXECUTION_FAILED,
          Duration.ZERO,
          startTime,
          message,
          failureDetail,
          /* statisticsPath= */ null);
    }

    Duration wallTime = executionStopwatch.elapsed();
    boolean wasTimeout =
        useSubprocessTimeout ? terminationStatus.timedOut() : wasTimeout(timeout, wallTime);

    int exitCode;
    Status status;
    String failureMessage;
    FailureDetail failureDetail;
    if (wasTimeout) {
      exitCode = SpawnResult.POSIX_TIMEOUT_EXIT_CODE;
      status = Status.TIMEOUT;
      failureMessage = makeFailureMessage(originalSpawn, sandbox);
      failureDetail =
          FailureDetail.newBuilder()
              .setMessage(failureMessage)
              .setSpawn(
                  FailureDetails.Spawn.newBuilder().setCode(FailureDetails.Spawn.Code.TIMEOUT))
              .build();
    } else {
      exitCode = terminationStatus.getRawExitCode();
      if (exitCode == 0) {
        status = Status.SUCCESS;
        failureMessage = "";
        failureDetail = null;
      } else {
        status = Status.NON_ZERO_EXIT;
        failureMessage = makeFailureMessage(originalSpawn, sandbox);
        failureDetail =
            FailureDetail.newBuilder()
                .setMessage(failureMessage)
                .setSpawn(
                    FailureDetails.Spawn.newBuilder()
                        .setCode(FailureDetails.Spawn.Code.NON_ZERO_EXIT)
                        .setSpawnExitCode(exitCode))
                .build();
      }
    }

    String sandboxDebugOutput = getSandboxDebugOutput(sandbox);
    if (!sandboxDebugOutput.isEmpty()) {
      reporter.handle(
          Event.of(
              EventKind.DEBUG,
              String.format(
                  "Sandbox debug output for %s %s:\n%s",
                  originalSpawn.getMnemonic(),
                  originalSpawn.getTargetLabel(),
                  sandboxDebugOutput)));
    }

    Path statisticsPath = sandbox.getStatisticsPath();

    return new SubprocessResult(
        exitCode, status, wallTime, startTime, failureMessage, failureDetail, statisticsPath);
  }

  private static String getSandboxDebugOutput(SandboxedSpawn sandbox) throws IOException {
    Optional<String> sandboxDebugOutput = Optional.empty();
    Path sandboxDebugPath = sandbox.getSandboxDebugPath();
    if (sandboxDebugPath != null && sandboxDebugPath.exists()) {
      try (InputStream inputStream = sandboxDebugPath.getInputStream()) {
        String msg = new String(inputStream.readAllBytes(), UTF_8);
        if (!msg.isEmpty()) {
          sandboxDebugOutput = Optional.of(msg);
        }
      }
    }
    Optional<String> interactiveDebugInstructions = sandbox.getInteractiveDebugInstructions();
    return Stream.of(sandboxDebugOutput, interactiveDebugInstructions)
        .flatMap(Optional::stream)
        .collect(joining("\n"));
  }

  private static boolean wasTimeout(Duration timeout, Duration wallTime) {
    return !timeout.isZero() && wallTime.compareTo(timeout) > 0;
  }

  /**
   * Gets the list of directories that the spawn will assume to be writable.
   *
   * @param sandboxExecRoot the exec root of the sandbox
   * @param env the environment of the sandboxed processes
   * @throws IOException because we might resolve symlinks, which throws {@link IOException}.
   */
  protected ImmutableSet<Path> getWritableDirs(Path sandboxExecRoot, Map<String, String> env)
      throws IOException {
    // We have to make the TEST_TMPDIR directory writable if it is specified.
    ImmutableSet.Builder<Path> writablePaths = ImmutableSet.builder();

    // On Windows, sandboxExecRoot is actually the main execroot. We will specify
    // exactly which output path is writable.
    if (OS.getCurrent() != OS.WINDOWS) {
      writablePaths.add(sandboxExecRoot);
    }

    String testTmpdir = env.get("TEST_TMPDIR");
    if (testTmpdir != null) {
      addWritablePath(
          sandboxExecRoot,
          writablePaths,
          testTmpdir,
          "Cannot resolve symlinks in TEST_TMPDIR because it doesn't exist: \"%s\"");
    }
    // As of 2019-07-08:
    // - every caller of `getWritableDirs` passes a LocalEnvProvider-processed environment as
    //   `env`, therefore `env` surely has an entry for TMPDIR on Unix and TEMP/TMP on Windows.
    if (OS.getCurrent() == OS.WINDOWS) {
      addWritablePath(
          sandboxExecRoot,
          writablePaths,
          Preconditions.checkNotNull(env.get("TEMP")),
          "Cannot resolve symlinks in TEMP because it doesn't exist: \"%s\"");
      addWritablePath(
          sandboxExecRoot,
          writablePaths,
          Preconditions.checkNotNull(env.get("TMP")),
          "Cannot resolve symlinks in TMP because it doesn't exist: \"%s\"");
    } else {
      addWritablePath(
          sandboxExecRoot,
          writablePaths,
          Preconditions.checkNotNull(env.get("TMPDIR")),
          "Cannot resolve symlinks in TMPDIR because it doesn't exist: \"%s\"");
    }

    FileSystem fileSystem = sandboxExecRoot.getFileSystem();
    for (PathFragment writablePath : sandboxOptions.getSandboxWritablePath()) {
      Path path = fileSystem.getPath(writablePath);
      writablePaths.add(path);
      // TODO(laszlocsomor): Remove if guard when path.resolveSymbolicLinks supports non-symlink
      // TODO(laszlocsomor): Figure out why OS.getCurrent() != OS.WINDOWS is required, and remove it
      if (OS.getCurrent() != OS.WINDOWS || path.isSymbolicLink()) {
        writablePaths.add(path.resolveSymbolicLinks());
      }
    }

    return writablePaths.build();
  }

  private static void addWritablePath(
      Path sandboxExecRoot,
      ImmutableSet.Builder<Path> writablePaths,
      String pathString,
      String pathDoesNotExistErrorTemplate)
      throws IOException {
    Path path = sandboxExecRoot.getRelative(pathString);
    if (path.startsWith(sandboxExecRoot)) {
      // We add this path even though it is below sandboxExecRoot (and thus already writable as a
      // subpath) to take advantage of the side-effect that SymlinkedExecRoot also creates this
      // needed directory if it doesn't exist yet.
      writablePaths.add(path);
    } else if (path.exists()) {
      // If `path` itself is a symlink, then adding it to `writablePaths` would result in making
      // the symlink itself writable, not what it points to. Therefore we need to resolve symlinks
      // in `path`, however for that we need `path` to exist.
      //
      // TODO(laszlocsomor): Remove if guard when path.resolveSymbolicLinks supports non-symlink
      // TODO(laszlocsomor): Figure out why OS.getCurrent() != OS.WINDOWS is required, and remove it
      if (OS.getCurrent() != OS.WINDOWS || path.isSymbolicLink()) {
        writablePaths.add(path.resolveSymbolicLinks());
      } else {
        writablePaths.add(path);
      }
    } else {
      throw new IOException(String.format(pathDoesNotExistErrorTemplate, path.getPathString()));
    }
  }

  protected ImmutableSet<Path> getInaccessiblePaths() {
    return inaccessiblePaths;
  }

  protected SandboxOptions getSandboxOptions() {
    return sandboxOptions;
  }

  @Override
  public void cleanupSandboxBase(Path sandboxBase, TreeDeleter treeDeleter)
      throws IOException, InterruptedException {
    Path root = sandboxBase.getChild(getName());
    if (root.exists()) {
      for (Path child : root.getDirectoryEntries()) {
        if (Thread.currentThread().isInterrupted()) {
          throw new InterruptedException();
        }
        treeDeleter.deleteTree(child);
      }
      root.delete();
    }
  }
}