| // Copyright 2016 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package com.google.devtools.build.lib.sandbox; |
| |
| import static java.nio.charset.StandardCharsets.UTF_8; |
| import static java.util.stream.Collectors.joining; |
| |
| import com.google.common.base.Preconditions; |
| import com.google.common.collect.ImmutableList; |
| import com.google.common.collect.ImmutableSet; |
| import com.google.devtools.build.lib.actions.ActionExecutionMetadata; |
| import com.google.devtools.build.lib.actions.ExecException; |
| import com.google.devtools.build.lib.actions.ForbiddenActionInputException; |
| import com.google.devtools.build.lib.actions.ResourceManager; |
| import com.google.devtools.build.lib.actions.ResourceManager.ResourceHandle; |
| import com.google.devtools.build.lib.actions.ResourceManager.ResourcePriority; |
| import com.google.devtools.build.lib.actions.Spawn; |
| import com.google.devtools.build.lib.actions.SpawnResult; |
| import com.google.devtools.build.lib.actions.SpawnResult.Status; |
| import com.google.devtools.build.lib.actions.Spawns; |
| import com.google.devtools.build.lib.actions.UserExecException; |
| import com.google.devtools.build.lib.events.Event; |
| import com.google.devtools.build.lib.events.EventKind; |
| import com.google.devtools.build.lib.events.Reporter; |
| import com.google.devtools.build.lib.exec.BinTools; |
| import com.google.devtools.build.lib.exec.ExecutionOptions; |
| import com.google.devtools.build.lib.exec.SpawnExecutingEvent; |
| import com.google.devtools.build.lib.exec.SpawnRunner; |
| import com.google.devtools.build.lib.exec.SpawnSchedulingEvent; |
| import com.google.devtools.build.lib.exec.TreeDeleter; |
| import com.google.devtools.build.lib.profiler.Profiler; |
| import com.google.devtools.build.lib.profiler.SilentCloseable; |
| import com.google.devtools.build.lib.runtime.CommandEnvironment; |
| import com.google.devtools.build.lib.server.FailureDetails; |
| import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; |
| import com.google.devtools.build.lib.server.FailureDetails.Sandbox.Code; |
| import com.google.devtools.build.lib.shell.ExecutionStatistics; |
| import com.google.devtools.build.lib.shell.Subprocess; |
| import com.google.devtools.build.lib.shell.SubprocessBuilder; |
| import com.google.devtools.build.lib.shell.TerminationStatus; |
| import com.google.devtools.build.lib.util.CommandFailureUtils; |
| import com.google.devtools.build.lib.util.OS; |
| import com.google.devtools.build.lib.util.io.FileOutErr; |
| import com.google.devtools.build.lib.vfs.FileSystem; |
| import com.google.devtools.build.lib.vfs.Path; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.time.Duration; |
| import java.time.Instant; |
| import java.util.Map; |
| import java.util.Optional; |
| import java.util.stream.Stream; |
| |
| /** Abstract common ancestor for sandbox spawn runners implementing the common parts. */ |
| abstract class AbstractSandboxSpawnRunner implements SpawnRunner { |
| private static final int LOCAL_EXEC_ERROR = -1; |
| |
| private static final String SANDBOX_DEBUG_SUGGESTION = |
| "\n\nUse --sandbox_debug to see verbose messages from the sandbox " |
| + "and retain the sandbox build root for debugging"; |
| |
| private final SandboxOptions sandboxOptions; |
| private final boolean verboseFailures; |
| private final ImmutableSet<Path> inaccessiblePaths; |
| protected final BinTools binTools; |
| private final Path execRoot; |
| private final ResourceManager resourceManager; |
| private final Reporter reporter; |
| |
| public AbstractSandboxSpawnRunner(CommandEnvironment cmdEnv) { |
| this.sandboxOptions = cmdEnv.getOptions().getOptions(SandboxOptions.class); |
| this.verboseFailures = cmdEnv.getOptions().getOptions(ExecutionOptions.class).verboseFailures; |
| this.inaccessiblePaths = |
| sandboxOptions.getInaccessiblePaths(cmdEnv.getRuntime().getFileSystem()); |
| this.binTools = cmdEnv.getBlazeWorkspace().getBinTools(); |
| this.execRoot = cmdEnv.getExecRoot(); |
| this.resourceManager = cmdEnv.getLocalResourceManager(); |
| this.reporter = cmdEnv.getReporter(); |
| } |
| |
| @Override |
| public final SpawnResult exec(Spawn spawn, SpawnExecutionContext context) |
| throws ExecException, InterruptedException { |
| ActionExecutionMetadata owner = spawn.getResourceOwner(); |
| context.report(SpawnSchedulingEvent.create(getName())); |
| |
| try { |
| try (SilentCloseable c = Profiler.instance().profile("context.prefetchInputs")) { |
| context.prefetchInputsAndWait(); |
| } |
| |
| try (ResourceHandle ignored = |
| resourceManager.acquireResources( |
| owner, |
| spawn.getLocalResources(), |
| context.speculating() |
| ? ResourcePriority.DYNAMIC_STANDALONE |
| : ResourcePriority.LOCAL)) { |
| context.report(SpawnExecutingEvent.create(getName())); |
| SandboxedSpawn sandbox = prepareSpawn(spawn, context); |
| return runSpawn(spawn, sandbox, context); |
| } |
| } catch (IOException e) { |
| FailureDetail failureDetail = |
| SandboxHelpers.createFailureDetail( |
| "I/O exception during sandboxed execution", Code.EXECUTION_IO_EXCEPTION); |
| throw new UserExecException(e, failureDetail); |
| } catch (ForbiddenActionInputException e) { |
| FailureDetail failureDetail = |
| SandboxHelpers.createFailureDetail( |
| "Forbidden input found during sandboxed execution", Code.FORBIDDEN_INPUT); |
| throw new UserExecException(e, failureDetail); |
| } |
| } |
| |
| @Override |
| public boolean canExec(Spawn spawn) { |
| return Spawns.mayBeSandboxed(spawn); |
| } |
| |
| @Override |
| public boolean handlesCaching() { |
| return false; |
| } |
| |
| protected abstract SandboxedSpawn prepareSpawn(Spawn spawn, SpawnExecutionContext context) |
| throws IOException, ExecException, InterruptedException, ForbiddenActionInputException; |
| |
| private SpawnResult runSpawn( |
| Spawn originalSpawn, SandboxedSpawn sandbox, SpawnExecutionContext context) |
| throws IOException, ForbiddenActionInputException, InterruptedException { |
| try { |
| try (SilentCloseable c = Profiler.instance().profile("sandbox.createFileSystem")) { |
| sandbox.createFileSystem(); |
| } |
| SpawnResult result; |
| try (SilentCloseable c = Profiler.instance().profile("subprocess.run")) { |
| result = run(originalSpawn, sandbox, context); |
| } |
| try (SilentCloseable c = Profiler.instance().profile("sandbox.verifyPostCondition")) { |
| verifyPostCondition(originalSpawn, sandbox, context); |
| } |
| |
| context.lockOutputFiles( |
| result.exitCode(), |
| result.failureDetail() != null ? result.failureDetail().getMessage() : "", |
| context.getFileOutErr()); |
| try (SilentCloseable c = Profiler.instance().profile("sandbox.copyOutputs")) { |
| // We copy the outputs even when the command failed. |
| sandbox.copyOutputs(execRoot); |
| } catch (IOException e) { |
| throw new IOException("Could not move output artifacts from sandboxed execution", e); |
| } |
| return result; |
| } finally { |
| if (!sandboxOptions.sandboxDebug) { |
| try (SilentCloseable c = Profiler.instance().profile("sandbox.delete")) { |
| sandbox.delete(); |
| } |
| } |
| } |
| } |
| /** Override this method if you need to run a post condition after the action has executed */ |
| public void verifyPostCondition( |
| Spawn originalSpawn, SandboxedSpawn sandbox, SpawnExecutionContext context) |
| throws IOException, ForbiddenActionInputException {} |
| |
| private String makeFailureMessage(Spawn originalSpawn, SandboxedSpawn sandbox) { |
| if (sandboxOptions.sandboxDebug) { |
| return CommandFailureUtils.describeCommandFailure( |
| true, sandbox.getSandboxExecRoot().getPathString(), sandbox); |
| } else { |
| return CommandFailureUtils.describeCommandFailure( |
| verboseFailures, sandbox.getSandboxExecRoot().getPathString(), originalSpawn) |
| + SANDBOX_DEBUG_SUGGESTION; |
| } |
| } |
| |
| private final SpawnResult run( |
| Spawn originalSpawn, SandboxedSpawn sandbox, SpawnExecutionContext context) |
| throws IOException, InterruptedException { |
| |
| SpawnResult.Builder spawnResultBuilder = getSpawnResultBuilder(context); |
| |
| FileOutErr outErr = context.getFileOutErr(); |
| Duration timeout = context.getTimeout(); |
| |
| SubprocessBuilder subprocessBuilder = new SubprocessBuilder(); |
| subprocessBuilder.setWorkingDirectory(sandbox.getSandboxExecRoot().getPathFile()); |
| subprocessBuilder.setStdout(outErr.getOutputPath().getPathFile()); |
| subprocessBuilder.setStderr(outErr.getErrorPath().getPathFile()); |
| subprocessBuilder.setEnv(sandbox.getEnvironment()); |
| subprocessBuilder.setArgv(ImmutableList.copyOf(sandbox.getArguments())); |
| boolean useSubprocessTimeout = sandbox.useSubprocessTimeout(); |
| if (useSubprocessTimeout) { |
| subprocessBuilder.setTimeoutMillis(timeout.toMillis()); |
| } |
| Instant startTime = Instant.now(); |
| TerminationStatus terminationStatus; |
| try { |
| Subprocess subprocess = subprocessBuilder.start(); |
| subprocess.getOutputStream().close(); |
| try { |
| subprocess.waitFor(); |
| terminationStatus = new TerminationStatus(subprocess.exitValue(), subprocess.timedout()); |
| } catch (InterruptedException e) { |
| subprocess.destroyAndWait(); |
| throw e; |
| } |
| } catch (IOException e) { |
| String exceptionMsg = e.getMessage() == null ? e.getClass().getName() : e.getMessage(); |
| String sandboxDebugOutput = getSandboxDebugOutput(sandbox); |
| |
| StringBuilder msg = new StringBuilder("Action failed to execute: java.io.IOException: "); |
| msg.append(exceptionMsg); |
| msg.append("\n"); |
| if (!sandboxDebugOutput.isEmpty()) { |
| msg.append("Sandbox debug output:\n"); |
| msg.append(sandboxDebugOutput); |
| msg.append("\n"); |
| } |
| |
| outErr.getErrorStream().write(msg.toString().getBytes(UTF_8)); |
| outErr.getErrorStream().flush(); |
| String message = makeFailureMessage(originalSpawn, sandbox); |
| return spawnResultBuilder |
| .setStatus(Status.EXECUTION_FAILED) |
| .setExitCode(LOCAL_EXEC_ERROR) |
| .setFailureMessage(message) |
| .setFailureDetail( |
| SandboxHelpers.createFailureDetail(message, Code.SUBPROCESS_START_FAILED)) |
| .build(); |
| } |
| |
| // TODO(b/62588075): Calculate wall time inside Subprocess instead? |
| Duration wallTime = Duration.between(startTime, Instant.now()); |
| boolean wasTimeout = |
| (useSubprocessTimeout && terminationStatus.timedOut()) |
| || (!useSubprocessTimeout && wasTimeout(timeout, wallTime)); |
| |
| int exitCode; |
| Status status; |
| String failureMessage; |
| FailureDetail failureDetail; |
| if (wasTimeout) { |
| exitCode = SpawnResult.POSIX_TIMEOUT_EXIT_CODE; |
| status = Status.TIMEOUT; |
| failureMessage = makeFailureMessage(originalSpawn, sandbox); |
| failureDetail = |
| FailureDetail.newBuilder() |
| .setMessage(failureMessage) |
| .setSpawn( |
| FailureDetails.Spawn.newBuilder().setCode(FailureDetails.Spawn.Code.TIMEOUT)) |
| .build(); |
| } else { |
| exitCode = terminationStatus.getRawExitCode(); |
| if (exitCode == 0) { |
| status = Status.SUCCESS; |
| failureMessage = ""; |
| failureDetail = null; |
| } else { |
| status = Status.NON_ZERO_EXIT; |
| failureMessage = makeFailureMessage(originalSpawn, sandbox); |
| failureDetail = |
| FailureDetail.newBuilder() |
| .setMessage(failureMessage) |
| .setSpawn( |
| FailureDetails.Spawn.newBuilder() |
| .setCode(FailureDetails.Spawn.Code.NON_ZERO_EXIT) |
| .setSpawnExitCode(exitCode)) |
| .build(); |
| } |
| } |
| |
| spawnResultBuilder |
| .setStatus(status) |
| .setExitCode(exitCode) |
| .setStartTime(startTime) |
| .setWallTimeInMs((int) wallTime.toMillis()) |
| .setFailureMessage(failureMessage); |
| |
| if (failureDetail != null) { |
| spawnResultBuilder.setFailureDetail(failureDetail); |
| } |
| |
| String sandboxDebugOutput = getSandboxDebugOutput(sandbox); |
| if (!sandboxDebugOutput.isEmpty()) { |
| reporter.handle( |
| Event.of( |
| EventKind.DEBUG, |
| String.format( |
| "Sandbox debug output for %s %s:\n%s", |
| originalSpawn.getMnemonic(), |
| originalSpawn.getTargetLabel(), |
| sandboxDebugOutput))); |
| } |
| |
| Path statisticsPath = sandbox.getStatisticsPath(); |
| if (statisticsPath != null) { |
| ExecutionStatistics.getResourceUsage(statisticsPath) |
| .ifPresent( |
| resourceUsage -> { |
| spawnResultBuilder.setUserTimeInMs( |
| (int) resourceUsage.getUserExecutionTime().toMillis()); |
| spawnResultBuilder.setSystemTimeInMs( |
| (int) resourceUsage.getSystemExecutionTime().toMillis()); |
| spawnResultBuilder.setNumBlockOutputOperations( |
| resourceUsage.getBlockOutputOperations()); |
| spawnResultBuilder.setNumBlockInputOperations( |
| resourceUsage.getBlockInputOperations()); |
| spawnResultBuilder.setNumInvoluntaryContextSwitches( |
| resourceUsage.getInvoluntaryContextSwitches()); |
| // The memory usage of the largest child process. For Darwin maxrss returns size in |
| // bytes. |
| if (OS.getCurrent() == OS.DARWIN) { |
| spawnResultBuilder.setMemoryInKb( |
| resourceUsage.getMaximumResidentSetSize() / 1000); |
| } else { |
| spawnResultBuilder.setMemoryInKb(resourceUsage.getMaximumResidentSetSize()); |
| } |
| }); |
| } |
| |
| return spawnResultBuilder.build(); |
| } |
| |
| private String getSandboxDebugOutput(SandboxedSpawn sandbox) throws IOException { |
| Optional<String> sandboxDebugOutput = Optional.empty(); |
| Path sandboxDebugPath = sandbox.getSandboxDebugPath(); |
| if (sandboxDebugPath != null && sandboxDebugPath.exists()) { |
| try (InputStream inputStream = sandboxDebugPath.getInputStream()) { |
| String msg = new String(inputStream.readAllBytes(), UTF_8); |
| if (!msg.isEmpty()) { |
| sandboxDebugOutput = Optional.of(msg); |
| } |
| } |
| } |
| Optional<String> interactiveDebugInstructions = sandbox.getInteractiveDebugInstructions(); |
| return Stream.of(sandboxDebugOutput, interactiveDebugInstructions) |
| .flatMap(Optional::stream) |
| .collect(joining("\n")); |
| } |
| |
| private boolean wasTimeout(Duration timeout, Duration wallTime) { |
| return !timeout.isZero() && wallTime.compareTo(timeout) > 0; |
| } |
| |
| /** |
| * Gets the list of directories that the spawn will assume to be writable. |
| * |
| * @param sandboxExecRoot the exec root of the sandbox from the point of view of the Bazel process |
| * @param withinSandboxExecRoot the exec root from the point of view of the sandboxed processes |
| * @param env the environment of the sandboxed processes |
| * @throws IOException because we might resolve symlinks, which throws {@link IOException}. |
| */ |
| protected ImmutableSet<Path> getWritableDirs( |
| Path sandboxExecRoot, Path withinSandboxExecRoot, Map<String, String> env) |
| throws IOException { |
| // We have to make the TEST_TMPDIR directory writable if it is specified. |
| ImmutableSet.Builder<Path> writablePaths = ImmutableSet.builder(); |
| |
| // On Windows, sandboxExecRoot is actually the main execroot. We will specify |
| // exactly which output path is writable. |
| if (OS.getCurrent() != OS.WINDOWS) { |
| writablePaths.add(withinSandboxExecRoot); |
| } |
| |
| String testTmpdir = env.get("TEST_TMPDIR"); |
| if (testTmpdir != null) { |
| addWritablePath( |
| sandboxExecRoot, |
| writablePaths, |
| testTmpdir, |
| "Cannot resolve symlinks in TEST_TMPDIR because it doesn't exist: \"%s\""); |
| } |
| // As of 2019-07-08: |
| // - every caller of `getWritableDirs` passes a LocalEnvProvider-processed environment as |
| // `env`, therefore `env` surely has an entry for TMPDIR on Unix and TEMP/TMP on Windows. |
| if (OS.getCurrent() == OS.WINDOWS) { |
| addWritablePath( |
| sandboxExecRoot, |
| writablePaths, |
| Preconditions.checkNotNull(env.get("TEMP")), |
| "Cannot resolve symlinks in TEMP because it doesn't exist: \"%s\""); |
| addWritablePath( |
| sandboxExecRoot, |
| writablePaths, |
| Preconditions.checkNotNull(env.get("TMP")), |
| "Cannot resolve symlinks in TMP because it doesn't exist: \"%s\""); |
| } else { |
| addWritablePath( |
| sandboxExecRoot, |
| writablePaths, |
| Preconditions.checkNotNull(env.get("TMPDIR")), |
| "Cannot resolve symlinks in TMPDIR because it doesn't exist: \"%s\""); |
| } |
| |
| FileSystem fileSystem = sandboxExecRoot.getFileSystem(); |
| for (String writablePath : sandboxOptions.sandboxWritablePath) { |
| Path path = fileSystem.getPath(writablePath); |
| writablePaths.add(path); |
| // TODO(laszlocsomor): Remove if guard when path.resolveSymbolicLinks supports non-symlink |
| // TODO(laszlocsomor): Figure out why OS.getCurrent() != OS.WINDOWS is required, and remove it |
| if (OS.getCurrent() != OS.WINDOWS || path.isSymbolicLink()) { |
| writablePaths.add(path.resolveSymbolicLinks()); |
| } |
| } |
| |
| return writablePaths.build(); |
| } |
| |
| private void addWritablePath( |
| Path sandboxExecRoot, |
| ImmutableSet.Builder<Path> writablePaths, |
| String pathString, |
| String pathDoesNotExistErrorTemplate) |
| throws IOException { |
| Path path = sandboxExecRoot.getRelative(pathString); |
| if (path.startsWith(sandboxExecRoot)) { |
| // We add this path even though it is below sandboxExecRoot (and thus already writable as a |
| // subpath) to take advantage of the side-effect that SymlinkedExecRoot also creates this |
| // needed directory if it doesn't exist yet. |
| writablePaths.add(path); |
| } else if (path.exists()) { |
| // If `path` itself is a symlink, then adding it to `writablePaths` would result in making |
| // the symlink itself writable, not what it points to. Therefore we need to resolve symlinks |
| // in `path`, however for that we need `path` to exist. |
| // |
| // TODO(laszlocsomor): Remove if guard when path.resolveSymbolicLinks supports non-symlink |
| // TODO(laszlocsomor): Figure out why OS.getCurrent() != OS.WINDOWS is required, and remove it |
| if (OS.getCurrent() != OS.WINDOWS || path.isSymbolicLink()) { |
| writablePaths.add(path.resolveSymbolicLinks()); |
| } else { |
| writablePaths.add(path); |
| } |
| } else { |
| throw new IOException(String.format(pathDoesNotExistErrorTemplate, path.getPathString())); |
| } |
| } |
| |
| protected ImmutableSet<Path> getInaccessiblePaths() { |
| return inaccessiblePaths; |
| } |
| |
| protected SandboxOptions getSandboxOptions() { |
| return sandboxOptions; |
| } |
| |
| @Override |
| public void cleanupSandboxBase(Path sandboxBase, TreeDeleter treeDeleter) throws IOException { |
| Path root = sandboxBase.getChild(getName()); |
| if (root.exists()) { |
| for (Path child : root.getDirectoryEntries()) { |
| treeDeleter.deleteTree(child); |
| } |
| } |
| } |
| |
| } |
