blob: 3dacaeaa05ee9aacd348c52f7287dc6982dd3377 [file] [log] [blame]
// Copyright 2018 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.sandbox;
import build.bazel.remote.execution.v2.Platform;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.MoreCollectors;
import com.google.common.eventbus.Subscribe;
import com.google.common.io.ByteStreams;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.ForbiddenActionInputException;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.Spawns;
import com.google.devtools.build.lib.actions.UserExecException;
import com.google.devtools.build.lib.analysis.platform.PlatformUtils;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.exec.TreeDeleter;
import com.google.devtools.build.lib.exec.local.LocalEnvProvider;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
import com.google.devtools.build.lib.runtime.CommandCompleteEvent;
import com.google.devtools.build.lib.runtime.CommandEnvironment;
import com.google.devtools.build.lib.runtime.ProcessWrapper;
import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxInputs;
import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxOutputs;
import com.google.devtools.build.lib.server.FailureDetails.Sandbox.Code;
import com.google.devtools.build.lib.shell.Command;
import com.google.devtools.build.lib.shell.CommandException;
import com.google.devtools.build.lib.util.OS;
import com.google.devtools.build.lib.util.ProcessUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicReference;
/** Spawn runner that uses Docker to execute a local subprocess. */
final class DockerSandboxedSpawnRunner extends AbstractSandboxSpawnRunner {
// The name of the container image entry in the Platform proto
// (see third_party/googleapis/devtools/remoteexecution/*/remote_execution.proto and
// remote_default_exec_properties in
// src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java)
private static final String CONTAINER_IMAGE_ENTRY_NAME = "container-image";
private static final String DOCKER_IMAGE_PREFIX = "docker://";
/**
* Returns whether the darwin sandbox is supported on the local machine by running docker info.
* This is expensive, and we have also reports of docker hanging for a long time!
*/
public static boolean isSupported(CommandEnvironment cmdEnv, Path dockerClient)
throws InterruptedException {
boolean verbose = cmdEnv.getOptions().getOptions(SandboxOptions.class).dockerVerbose;
if (ProcessWrapper.fromCommandEnvironment(cmdEnv) == null) {
if (verbose) {
cmdEnv
.getReporter()
.handle(
Event.error(
"Docker sandboxing is disabled because ProcessWrapper is not supported. "
+ "This should never happen - is your Bazel binary corrupted?"));
}
return false;
}
// On Linux we need to know the UID and GID that we're running as, because otherwise Docker will
// create files as 'root' and we can't move them to the execRoot.
if (OS.getCurrent() == OS.LINUX) {
try {
ProcessUtils.getuid();
ProcessUtils.getgid();
} catch (UnsatisfiedLinkError e) {
if (verbose) {
cmdEnv
.getReporter()
.handle(
Event.error(
"Docker sandboxing is disabled, because ProcessUtils.getuid/getgid threw an "
+ "UnsatisfiedLinkError. This means that you're running a Bazel version "
+ "that doesn't have JNI libraries - did you build it correctly?\n"
+ Throwables.getStackTraceAsString(e)));
}
return false;
}
}
Command cmd =
new Command(
new String[] {dockerClient.getPathString(), "info"},
cmdEnv.getClientEnv(),
cmdEnv.getExecRoot().getPathFile());
try {
cmd.execute(ByteStreams.nullOutputStream(), ByteStreams.nullOutputStream());
} catch (CommandException e) {
if (verbose) {
cmdEnv
.getReporter()
.handle(
Event.error(
"Docker sandboxing is disabled, because running 'docker info' failed: "
+ Throwables.getStackTraceAsString(e)));
}
return false;
}
if (verbose) {
cmdEnv.getReporter().handle(Event.info("Docker sandboxing is supported"));
}
return true;
}
private static final ConcurrentHashMap<String, String> imageMap = new ConcurrentHashMap<>();
private final SandboxHelpers helpers;
private final Path execRoot;
private final ImmutableList<Root> packageRoots;
private final boolean allowNetwork;
private final Path dockerClient;
private final ProcessWrapper processWrapper;
private final Path sandboxBase;
private final String defaultImage;
private final LocalEnvProvider localEnvProvider;
private final String commandId;
private final Reporter reporter;
private final boolean useCustomizedImages;
private final TreeDeleter treeDeleter;
private final int uid;
private final int gid;
private final Set<UUID> containersToCleanup;
private final CommandEnvironment cmdEnv;
/**
* Creates a sandboxed spawn runner that uses the {@code linux-sandbox} tool.
*
* @param helpers common tools and state across all spawns during sandboxed execution
* @param cmdEnv the command environment to use
* @param dockerClient path to the `docker` executable
* @param sandboxBase path to the sandbox base directory
* @param defaultImage the Docker image to use if the platform doesn't specify one
* @param useCustomizedImages whether to use customized images for execution
* @param treeDeleter scheduler for tree deletions
*/
DockerSandboxedSpawnRunner(
SandboxHelpers helpers,
CommandEnvironment cmdEnv,
Path dockerClient,
Path sandboxBase,
String defaultImage,
boolean useCustomizedImages,
TreeDeleter treeDeleter) {
super(cmdEnv);
this.helpers = helpers;
this.execRoot = cmdEnv.getExecRoot();
this.packageRoots = cmdEnv.getPackageLocator().getPathEntries();
this.allowNetwork = helpers.shouldAllowNetwork(cmdEnv.getOptions());
this.dockerClient = dockerClient;
this.processWrapper = ProcessWrapper.fromCommandEnvironment(cmdEnv);
this.sandboxBase = sandboxBase;
this.defaultImage = defaultImage;
this.localEnvProvider = LocalEnvProvider.forCurrentOs(cmdEnv.getClientEnv());
this.commandId = cmdEnv.getCommandId().toString();
this.reporter = cmdEnv.getReporter();
this.useCustomizedImages = useCustomizedImages;
this.treeDeleter = treeDeleter;
this.cmdEnv = cmdEnv;
if (OS.getCurrent() == OS.LINUX) {
this.uid = ProcessUtils.getuid();
this.gid = ProcessUtils.getgid();
} else {
this.uid = -1;
this.gid = -1;
}
this.containersToCleanup = Collections.synchronizedSet(new HashSet<>());
cmdEnv.getEventBus().register(this);
}
@Override
protected SandboxedSpawn prepareSpawn(Spawn spawn, SpawnExecutionContext context)
throws IOException, ExecException, InterruptedException, ForbiddenActionInputException {
// Each invocation of "exec" gets its own sandbox base, execroot and temporary directory.
Path sandboxPath =
sandboxBase.getRelative(getName()).getRelative(Integer.toString(context.getId()));
sandboxPath.getParentDirectory().createDirectory();
sandboxPath.createDirectory();
// b/64689608: The execroot of the sandboxed process must end with the workspace name, just like
// the normal execroot does.
Path sandboxExecRoot = sandboxPath.getRelative("execroot").getRelative(execRoot.getBaseName());
sandboxExecRoot.getParentDirectory().createDirectory();
sandboxExecRoot.createDirectory();
ImmutableMap<String, String> environment =
localEnvProvider.rewriteLocalEnv(spawn.getEnvironment(), binTools, "/tmp");
SandboxInputs inputs =
helpers.processInputFiles(
context.getInputMapping(PathFragment.EMPTY_FRAGMENT, /* willAccessRepeatedly= */ true),
execRoot,
execRoot,
packageRoots,
null);
SandboxOutputs outputs = helpers.getOutputs(spawn);
Duration timeout = context.getTimeout();
UUID uuid = UUID.randomUUID();
String baseImageName = dockerContainerFromSpawn(spawn).orElse(this.defaultImage);
if (baseImageName.isEmpty()) {
throw new UserExecException(
SandboxHelpers.createFailureDetail(
String.format(
"Cannot execute %s mnemonic with Docker, because no image could be found in the"
+ " remote_execution_properties of the platform and no default image was set"
+ " via --experimental_docker_image",
spawn.getMnemonic()),
Code.NO_DOCKER_IMAGE));
}
String customizedImageName = getOrCreateCustomizedImage(baseImageName);
DockerCommandLineBuilder cmdLine = new DockerCommandLineBuilder();
cmdLine
.setProcessWrapper(processWrapper)
.setDockerClient(dockerClient)
.setImageName(customizedImageName)
.setCommandArguments(spawn.getArguments())
.setSandboxExecRoot(sandboxExecRoot)
.setAdditionalMounts(getSandboxOptions().sandboxAdditionalMounts)
.setPrivileged(getSandboxOptions().dockerPrivileged)
.setEnvironmentVariables(environment)
.setCreateNetworkNamespace(
!(allowNetwork
|| Spawns.requiresNetwork(spawn, getSandboxOptions().defaultSandboxAllowNetwork)))
.setCommandId(commandId)
.setUuid(uuid);
// If uid / gid are -1, we are on an operating system that doesn't require us to set them on the
// Docker invocation. If they're 0, it means we are running as root and don't need to set them.
if (uid > 0) {
cmdLine.setUid(uid);
}
if (gid > 0) {
cmdLine.setGid(gid);
}
if (!timeout.isZero()) {
cmdLine.setTimeout(timeout);
}
// If we were interrupted, it is possible that "docker run" gets killed in exactly the moment
// between the create and the start call, leaving behind a container that is created but never
// ran. This means that Docker won't automatically clean it up (as --rm only affects the start
// phase and has no effect on the create phase of "docker run").
// We register the container UUID for cleanup, but remove the UUID if the process ran
// successfully.
containersToCleanup.add(uuid);
return new CopyingSandboxedSpawn(
sandboxPath,
sandboxExecRoot,
cmdLine.build(),
cmdEnv.getClientEnv(),
inputs,
outputs,
ImmutableSet.of(),
treeDeleter,
/* sandboxDebugPath= */ null,
/* statisticsPath= */ null,
() -> containersToCleanup.remove(uuid),
spawn.getMnemonic());
}
private String getOrCreateCustomizedImage(String baseImage)
throws UserExecException, InterruptedException {
// TODO(philwo) docker run implicitly does a docker pull if the image does not exist locally.
// Pulling an image can take a long time and a user might not be aware of that. We could check
// if the image exists locally (docker images -q name:tag) and if not, do a docker pull and
// notify the user in a similar way as when we download a http_archive.
//
// This is mostly relevant for the case where we don't build a customized image, as that prints
// a message when it runs.
if (!useCustomizedImages) {
return baseImage;
}
// If we're running as root, we can skip this step, as it's safe to assume that every image
// already has a built-in root user and group.
if (uid == 0 && gid == 0) {
return baseImage;
}
// We only need to create a customized image, if we're running on Linux, as Docker on macOS
// and Windows doesn't map users from the host into the container anyway.
if (OS.getCurrent() != OS.LINUX) {
return baseImage;
}
AtomicReference<UserExecException> thrownUserExecException = new AtomicReference<>();
AtomicReference<InterruptedException> thrownInterruptedException = new AtomicReference<>();
String result =
imageMap.computeIfAbsent(
baseImage,
(image) -> {
reporter.handle(Event.info("Preparing Docker image " + image + " for use..."));
String workDir =
PathFragment.create("/execroot")
.getRelative(execRoot.getBaseName())
.getPathString();
StringBuilder dockerfile = new StringBuilder();
dockerfile.append("ARG image\n");
dockerfile.append("FROM $image\n");
dockerfile.append("ARG work_dir\n");
dockerfile.append("RUN mkdir --parents $work_dir\n"); // could this be a VOLUME?
dockerfile.append("ARG group_name\n");
dockerfile.append("ARG gid\n");
dockerfile.append("RUN groupadd --non-unique --gid $gid $group_name\n");
dockerfile.append("ARG user_name\n");
dockerfile.append("ARG uid\n");
dockerfile.append(
"RUN useradd --non-unique --no-log-init --create-home --gid $gid --home-dir"
+ " $work_dir --no-user-group --uid"
+ " $uid"
+ " $user_name\n"); // we've already created home above?
dockerfile.append(
"RUN chown --recursive $uid:$gid $work_dir\n"); // if we create home with useradd
// it'd already have the right
// ownership
dockerfile.append("USER $user_name:$group_name\n");
dockerfile.append("ENV HOME $work_dir\n");
dockerfile.append("ENV USER $user_name\n");
dockerfile.append("WORKDIR $work_dir\n");
try {
return executeCommand(
ImmutableList.of(
dockerClient.getPathString(),
"build",
"--build-arg",
String.format("image=%s", image),
"--build-arg",
String.format("work_dir=%s", workDir),
"--build-arg",
String.format("group_name=%s", "bazelbuild"),
"--build-arg",
String.format("gid=%d", gid),
"--build-arg",
String.format("user_name=%s", "bazelbuild"),
"--build-arg",
String.format("uid=%d", uid),
"-q",
"-"),
new ByteArrayInputStream(
dockerfile.toString().getBytes(Charset.defaultCharset())));
} catch (UserExecException e) {
thrownUserExecException.set(e);
return null;
} catch (InterruptedException e) {
thrownInterruptedException.set(e);
return null;
}
});
if (thrownUserExecException.get() != null) {
throw thrownUserExecException.get();
}
if (thrownInterruptedException.get() != null) {
throw thrownInterruptedException.get();
}
return result;
}
private String executeCommand(List<String> cmdLine, InputStream stdIn)
throws UserExecException, InterruptedException {
ByteArrayOutputStream stdOut = new ByteArrayOutputStream();
ByteArrayOutputStream stdErr = new ByteArrayOutputStream();
// Docker might need the $HOME and $PATH variables in order to be able to use advanced
// authentication mechanisms (e.g. for Google Cloud), thus we pass in the client env.
Command cmd =
new Command(cmdLine.toArray(new String[0]), cmdEnv.getClientEnv(), execRoot.getPathFile());
try {
cmd.executeAsync(stdIn, stdOut, stdErr, Command.KILL_SUBPROCESS_ON_INTERRUPT).get();
} catch (CommandException e) {
String message = String.format("Running command %s failed: %s", cmd.toDebugString(), stdErr);
throw new UserExecException(
e, SandboxHelpers.createFailureDetail(message, Code.DOCKER_COMMAND_FAILURE));
}
return stdOut.toString().trim();
}
private Optional<String> dockerContainerFromSpawn(Spawn spawn) throws ExecException {
Platform platform =
PlatformUtils.getPlatformProto(spawn, cmdEnv.getOptions().getOptions(RemoteOptions.class));
if (platform != null) {
try {
return platform
.getPropertiesList()
.stream()
.filter(p -> p.getName().equals(CONTAINER_IMAGE_ENTRY_NAME))
.map(p -> p.getValue())
.filter(r -> r.startsWith(DOCKER_IMAGE_PREFIX))
.map(r -> r.substring(DOCKER_IMAGE_PREFIX.length()))
.collect(MoreCollectors.toOptional());
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException(
String.format(
"Platform %s contained multiple container-image entries, but only one is allowed.",
spawn.getExecutionPlatform().label()),
e);
}
} else {
return Optional.empty();
}
}
// Remove all Docker containers that might be stuck in "Created" state and weren't automatically
// cleaned up by Docker itself.
public void cleanup() throws InterruptedException {
if (containersToCleanup == null || containersToCleanup.isEmpty()) {
return;
}
ArrayList<String> cmdLine = new ArrayList<>();
cmdLine.add(dockerClient.getPathString());
cmdLine.add("rm");
cmdLine.add("-fv");
for (UUID uuid : containersToCleanup) {
cmdLine.add(uuid.toString());
}
Command cmd =
new Command(cmdLine.toArray(new String[0]), cmdEnv.getClientEnv(), execRoot.getPathFile());
try {
cmd.execute();
} catch (CommandException e) {
// This is to be expected, as not all UUIDs that we pass to "docker rm" will still be alive
// when this method is called. However, it will successfully remove all the containers that
// *are* still there, even when the command exits with an error.
}
containersToCleanup.clear();
}
@Subscribe
public void commandComplete(@SuppressWarnings("unused") CommandCompleteEvent event) {
try {
cleanup();
} catch (InterruptedException e) {
cmdEnv.getReporter().handle(Event.error("Interrupted while cleaning up docker sandbox"));
Thread.currentThread().interrupt();
}
}
@Override
public String getName() {
return "docker";
}
}