blob: 78fe5c244ee2821319cb72836219d09dfb9ecd96 [file] [log] [blame]
// Copyright 2020 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.sandbox;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static com.google.common.truth.Truth.assertThat;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.concurrent.TimeUnit.SECONDS;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
import com.google.devtools.build.lib.actions.ArtifactRoot;
import com.google.devtools.build.lib.actions.CommandLines.ParamFileActionInput;
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType;
import com.google.devtools.build.lib.actions.PathMapper;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
import com.google.devtools.build.lib.exec.BinTools;
import com.google.devtools.build.lib.exec.util.FakeActionInputFileCache;
import com.google.devtools.build.lib.exec.util.SpawnBuilder;
import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxInputs;
import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxOutputs;
import com.google.devtools.build.lib.skyframe.TreeArtifactValue;
import com.google.devtools.build.lib.testutil.Scratch;
import com.google.devtools.build.lib.testutil.TestUtils;
import com.google.devtools.build.lib.vfs.DigestHashFunction;
import com.google.devtools.build.lib.vfs.Dirent;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.lib.vfs.Symlinks;
import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
import java.io.IOException;
import java.util.Arrays;
import java.util.LinkedHashSet;
import java.util.Set;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import java.util.function.Function;
import javax.annotation.Nullable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Tests for {@link SandboxHelpers}. */
@RunWith(JUnit4.class)
public class SandboxHelpersTest {
private static final byte[] FAKE_DIGEST = new byte[] {1};
private final Scratch scratch = new Scratch();
private Path execRootPath;
private Root execRoot;
@Nullable private ExecutorService executorToCleanup;
@Before
public void createExecRoot() throws IOException {
execRootPath = scratch.dir("/execRoot");
execRoot = Root.fromPath(execRootPath);
}
@After
public void shutdownExecutor() throws InterruptedException {
if (executorToCleanup == null) {
return;
}
executorToCleanup.shutdown();
executorToCleanup.awaitTermination(TestUtils.WAIT_TIMEOUT_SECONDS, SECONDS);
}
private RootedPath execRootedPath(String execPath) {
return RootedPath.toRootedPath(execRoot, PathFragment.create(execPath));
}
@Test
public void processInputFiles_resolvesMaterializationPath_fileArtifact() throws Exception {
ArtifactRoot outputRoot =
ArtifactRoot.asDerivedRoot(execRootPath, ArtifactRoot.RootType.Output, "outputs");
Path sandboxSourceRoot = scratch.dir("/faketmp/sandbox-source-roots");
Artifact input = ActionsTestUtil.createArtifact(outputRoot, "a/a");
FileArtifactValue symlinkTargetMetadata =
FileArtifactValue.createForNormalFile(FAKE_DIGEST, null, 0L);
FileArtifactValue inputMetadata =
FileArtifactValue.createForResolvedSymlink(
PathFragment.create("b/b"), symlinkTargetMetadata, FAKE_DIGEST);
FakeActionInputFileCache inputMetadataProvider = new FakeActionInputFileCache();
inputMetadataProvider.put(input, inputMetadata);
SandboxHelpers sandboxHelpers = new SandboxHelpers();
SandboxInputs inputs =
sandboxHelpers.processInputFiles(
inputMap(input),
inputMetadataProvider,
execRootPath,
execRootPath,
ImmutableList.of(),
sandboxSourceRoot);
assertThat(inputs.getFiles())
.containsEntry(
input.getExecPath(), RootedPath.toRootedPath(execRoot, PathFragment.create("b/b")));
}
@Test
public void processInputFiles_resolvesMaterializationPath_treeArtifact() throws Exception {
ArtifactRoot outputRoot =
ArtifactRoot.asDerivedRoot(execRootPath, ArtifactRoot.RootType.Output, "outputs");
Path sandboxSourceRoot = scratch.dir("/faketmp/sandbox-source-roots");
SpecialArtifact parent =
ActionsTestUtil.createTreeArtifactWithGeneratingAction(
outputRoot, "bin/config/other_dir/subdir");
TreeFileArtifact childA = TreeFileArtifact.createTreeOutput(parent, "a/a");
TreeFileArtifact childB = TreeFileArtifact.createTreeOutput(parent, "b/b");
FileArtifactValue childMetadata = FileArtifactValue.createForNormalFile(FAKE_DIGEST, null, 0L);
TreeArtifactValue parentMetadata =
TreeArtifactValue.newBuilder(parent)
.putChild(childA, childMetadata)
.putChild(childB, childMetadata)
.setMaterializationExecPath(PathFragment.create("materialized"))
.build();
FakeActionInputFileCache inputMetadataProvider = new FakeActionInputFileCache();
inputMetadataProvider.put(parent, parentMetadata.getMetadata());
SandboxHelpers sandboxHelpers = new SandboxHelpers();
SandboxInputs inputs =
sandboxHelpers.processInputFiles(
inputMap(childA, childB),
inputMetadataProvider,
execRootPath,
execRootPath,
ImmutableList.of(),
sandboxSourceRoot);
assertThat(inputs.getFiles())
.containsEntry(
childA.getExecPath(),
RootedPath.toRootedPath(execRoot, PathFragment.create("materialized/a/a")));
assertThat(inputs.getFiles())
.containsEntry(
childB.getExecPath(),
RootedPath.toRootedPath(execRoot, PathFragment.create("materialized/b/b")));
}
@Test
public void processInputFiles_materializesParamFile() throws Exception {
SandboxHelpers sandboxHelpers = new SandboxHelpers();
ParamFileActionInput paramFile =
new ParamFileActionInput(
PathFragment.create("paramFile"),
ImmutableList.of("-a", "-b"),
ParameterFileType.UNQUOTED,
UTF_8);
SandboxInputs inputs =
sandboxHelpers.processInputFiles(
inputMap(paramFile),
new FakeActionInputFileCache(),
execRootPath,
execRootPath,
ImmutableList.of(),
null);
assertThat(inputs.getFiles())
.containsExactly(PathFragment.create("paramFile"), execRootedPath("paramFile"));
assertThat(inputs.getSymlinks()).isEmpty();
assertThat(FileSystemUtils.readLines(execRootPath.getChild("paramFile"), UTF_8))
.containsExactly("-a", "-b")
.inOrder();
assertThat(execRootPath.getChild("paramFile").isExecutable()).isTrue();
}
@Test
public void processInputFiles_materializesBinToolsFile() throws Exception {
SandboxHelpers sandboxHelpers = new SandboxHelpers();
BinTools.PathActionInput tool =
new BinTools.PathActionInput(
scratch.file("tool", "#!/bin/bash", "echo hello"),
PathFragment.create("_bin/say_hello"));
SandboxInputs inputs =
sandboxHelpers.processInputFiles(
inputMap(tool),
new FakeActionInputFileCache(),
execRootPath,
execRootPath,
ImmutableList.of(),
null);
assertThat(inputs.getFiles())
.containsExactly(PathFragment.create("_bin/say_hello"), execRootedPath("_bin/say_hello"));
assertThat(inputs.getSymlinks()).isEmpty();
assertThat(FileSystemUtils.readLines(execRootPath.getRelative("_bin/say_hello"), UTF_8))
.containsExactly("#!/bin/bash", "echo hello")
.inOrder();
assertThat(execRootPath.getRelative("_bin/say_hello").isExecutable()).isTrue();
}
/**
* Test simulating a scenario when 2 parallel writes of the same virtual input both complete write
* of the temp file and then proceed with post-processing steps one-by-one.
*/
@Test
public void sandboxInputMaterializeVirtualInput_parallelWritesForSameInput_writesCorrectFile()
throws Exception {
VirtualActionInput input = ActionsTestUtil.createVirtualActionInput("file", "hello");
executorToCleanup = Executors.newSingleThreadExecutor();
CyclicBarrier bothWroteTempFile = new CyclicBarrier(2);
Semaphore finishProcessingSemaphore = new Semaphore(1);
FileSystem customFs =
new InMemoryFileSystem(DigestHashFunction.SHA1) {
@Override
@SuppressWarnings("UnsynchronizedOverridesSynchronized") // .await() inside
protected void setExecutable(PathFragment path, boolean executable) throws IOException {
try {
bothWroteTempFile.await();
finishProcessingSemaphore.acquire();
} catch (BrokenBarrierException | InterruptedException e) {
throw new IllegalArgumentException(e);
}
super.setExecutable(path, executable);
}
};
Scratch customScratch = new Scratch(customFs);
Path customExecRoot = customScratch.dir("/execroot");
SandboxHelpers sandboxHelpers = new SandboxHelpers();
Future<?> future =
executorToCleanup.submit(
() -> {
try {
var unused =
sandboxHelpers.processInputFiles(
inputMap(input),
new FakeActionInputFileCache(),
customExecRoot,
customExecRoot,
ImmutableList.of(),
null);
finishProcessingSemaphore.release();
} catch (IOException | InterruptedException e) {
throw new IllegalArgumentException(e);
}
});
var unused =
sandboxHelpers.processInputFiles(
inputMap(input),
new FakeActionInputFileCache(),
customExecRoot,
customExecRoot,
ImmutableList.of(),
null);
finishProcessingSemaphore.release();
future.get();
assertThat(customExecRoot.readdir(Symlinks.NOFOLLOW))
.containsExactly(new Dirent("file", Dirent.Type.FILE));
Path outputFile = customExecRoot.getChild("file");
assertThat(FileSystemUtils.readLines(outputFile, UTF_8)).containsExactly("hello");
assertThat(outputFile.isExecutable()).isTrue();
}
private static ImmutableMap<PathFragment, ActionInput> inputMap(ActionInput... inputs) {
return Arrays.stream(inputs)
.collect(toImmutableMap(ActionInput::getExecPath, Function.identity()));
}
@Test
public void atomicallyWriteVirtualInput_writesParamFile() throws Exception {
ParamFileActionInput paramFile =
new ParamFileActionInput(
PathFragment.create("paramFile"),
ImmutableList.of("-a", "-b"),
ParameterFileType.UNQUOTED,
UTF_8);
paramFile.atomicallyWriteRelativeTo(scratch.resolve("/outputs"), "-1234");
assertThat(scratch.resolve("/outputs").readdir(Symlinks.NOFOLLOW))
.containsExactly(new Dirent("paramFile", Dirent.Type.FILE));
Path outputFile = scratch.resolve("/outputs/paramFile");
assertThat(FileSystemUtils.readLines(outputFile, UTF_8)).containsExactly("-a", "-b").inOrder();
assertThat(outputFile.isExecutable()).isTrue();
}
@Test
public void atomicallyWriteVirtualInput_writesBinToolsFile() throws Exception {
BinTools.PathActionInput tool =
new BinTools.PathActionInput(
scratch.file("tool", "tool_code"), PathFragment.create("tools/tool"));
tool.atomicallyWriteRelativeTo(scratch.resolve("/outputs"), "-1234");
assertThat(scratch.resolve("/outputs").readdir(Symlinks.NOFOLLOW))
.containsExactly(new Dirent("tools", Dirent.Type.DIRECTORY));
Path outputFile = scratch.resolve("/outputs/tools/tool");
assertThat(FileSystemUtils.readLines(outputFile, UTF_8)).containsExactly("tool_code");
assertThat(outputFile.isExecutable()).isTrue();
}
@Test
public void atomicallyWriteVirtualInput_writesArbitraryVirtualInput() throws Exception {
VirtualActionInput input = ActionsTestUtil.createVirtualActionInput("file", "hello");
// Store an existing directory at the location where atomicallyWriteTo()
// writes its temporary file. It should be removed prior to the creation of
// the temporary file.
scratch.resolve("/outputs/file-1234").createDirectoryAndParents();
input.atomicallyWriteRelativeTo(scratch.resolve("/outputs"), "-1234");
assertThat(scratch.resolve("/outputs").readdir(Symlinks.NOFOLLOW))
.containsExactly(new Dirent("file", Dirent.Type.FILE));
Path outputFile = scratch.resolve("/outputs/file");
assertThat(FileSystemUtils.readLines(outputFile, UTF_8)).containsExactly("hello");
assertThat(outputFile.isExecutable()).isTrue();
}
@Test
public void cleanExisting_updatesDirs() throws IOException, InterruptedException {
RootedPath inputTxt =
RootedPath.toRootedPath(
Root.fromPath(scratch.getFileSystem().getPath("/")), PathFragment.create("hello.txt"));
Path rootDir = execRootPath.getParentDirectory();
PathFragment input1 = PathFragment.create("existing/directory/with/input1.txt");
PathFragment input2 = PathFragment.create("partial/directory/input2.txt");
PathFragment input3 = PathFragment.create("new/directory/input3.txt");
SandboxInputs inputs =
new SandboxInputs(
ImmutableMap.of(input1, inputTxt, input2, inputTxt, input3, inputTxt),
ImmutableMap.of(),
ImmutableMap.of(),
ImmutableMap.of());
Set<PathFragment> inputsToCreate = new LinkedHashSet<>();
LinkedHashSet<PathFragment> dirsToCreate = new LinkedHashSet<>();
SandboxHelpers.populateInputsAndDirsToCreate(
ImmutableSet.of(),
inputsToCreate,
dirsToCreate,
Iterables.concat(
ImmutableSet.of(), inputs.getFiles().keySet(), inputs.getSymlinks().keySet()),
SandboxOutputs.create(
ImmutableSet.of(PathFragment.create("out/dir/output.txt")), ImmutableSet.of()));
PathFragment inputDir1 = input1.getParentDirectory();
PathFragment inputDir2 = input2.getParentDirectory();
PathFragment inputDir3 = input3.getParentDirectory();
PathFragment outputDir = PathFragment.create("out/dir");
assertThat(dirsToCreate).containsExactly(inputDir1, inputDir2, inputDir3, outputDir);
assertThat(inputsToCreate).containsExactly(input1, input2, input3);
// inputdir1 exists fully
execRootPath.getRelative(inputDir1).createDirectoryAndParents();
// inputdir2 exists partially, should be kept nonetheless.
execRootPath
.getRelative(inputDir2)
.getParentDirectory()
.getRelative("doomedSubdir")
.createDirectoryAndParents();
// inputDir3 just doesn't exist
// outputDir only exists partially
execRootPath.getRelative(outputDir).getParentDirectory().createDirectoryAndParents();
execRootPath.getRelative("justSomeDir/thatIsDoomed").createDirectoryAndParents();
// `thiswillbeafile/output` simulates a directory that was in the stashed dir but whose same
// path is used later for a regular file.
scratch.dir("/execRoot/thiswillbeafile/output");
scratch.file("/execRoot/thiswillbeafile/output/file1");
dirsToCreate.add(PathFragment.create("thiswillbeafile"));
PathFragment input4 = PathFragment.create("thiswillbeafile/output");
SandboxInputs inputs2 =
new SandboxInputs(
ImmutableMap.of(input1, inputTxt, input2, inputTxt, input3, inputTxt, input4, inputTxt),
ImmutableMap.of(),
ImmutableMap.of(),
ImmutableMap.of());
SandboxHelpers.cleanExisting(rootDir, inputs2, inputsToCreate, dirsToCreate, execRootPath);
assertThat(dirsToCreate).containsExactly(inputDir2, inputDir3, outputDir);
assertThat(execRootPath.getRelative("existing/directory/with").exists()).isTrue();
assertThat(execRootPath.getRelative("partial").exists()).isTrue();
assertThat(execRootPath.getRelative("partial/doomedSubdir").exists()).isFalse();
assertThat(execRootPath.getRelative("partial/directory").exists()).isFalse();
assertThat(execRootPath.getRelative("justSomeDir/thatIsDoomed").exists()).isFalse();
assertThat(execRootPath.getRelative("out").exists()).isTrue();
assertThat(execRootPath.getRelative("out/dir").exists()).isFalse();
}
@Test
public void populateInputsAndDirsToCreate_createsMappedDirectories() {
ArtifactRoot outputRoot =
ArtifactRoot.asDerivedRoot(execRootPath, ArtifactRoot.RootType.Output, "outputs");
ActionInput outputFile = ActionsTestUtil.createArtifact(outputRoot, "bin/config/dir/file");
ActionInput outputDir =
ActionsTestUtil.createTreeArtifactWithGeneratingAction(
outputRoot, "bin/config/other_dir/subdir");
PathMapper pathMapper =
execPath -> PathFragment.create(execPath.getPathString().replace("config/", ""));
Spawn spawn =
new SpawnBuilder().withOutputs(outputFile, outputDir).setPathMapper(pathMapper).build();
var sandboxHelpers = new SandboxHelpers();
LinkedHashSet<PathFragment> writableDirs = new LinkedHashSet<>();
LinkedHashSet<PathFragment> inputsToCreate = new LinkedHashSet<>();
LinkedHashSet<PathFragment> dirsToCreate = new LinkedHashSet<>();
SandboxHelpers.populateInputsAndDirsToCreate(
writableDirs,
inputsToCreate,
dirsToCreate,
ImmutableList.of(),
sandboxHelpers.getOutputs(spawn));
assertThat(writableDirs).isEmpty();
assertThat(inputsToCreate).isEmpty();
assertThat(dirsToCreate)
.containsExactly(
PathFragment.create("outputs/bin/dir"),
PathFragment.create("outputs/bin/other_dir/subdir"));
}
@Test
public void moveOutputs_mappedPathMovedToUnmappedPath() throws Exception {
PathFragment unmappedOutputPath = PathFragment.create("bin/config/output");
PathMapper pathMapper =
execPath -> PathFragment.create(execPath.getPathString().replace("config/", ""));
Spawn spawn =
new SpawnBuilder()
.withOutputs(unmappedOutputPath.getPathString())
.setPathMapper(pathMapper)
.build();
var sandboxHelpers = new SandboxHelpers();
Path sandboxBase = execRootPath.getRelative("sandbox");
PathFragment mappedOutputPath = PathFragment.create("bin/output");
sandboxBase.getRelative(mappedOutputPath).getParentDirectory().createDirectoryAndParents();
FileSystemUtils.writeLinesAs(
sandboxBase.getRelative(mappedOutputPath), UTF_8, "hello", "pathmapper");
Path realBase = execRootPath.getRelative("real");
SandboxHelpers.moveOutputs(sandboxHelpers.getOutputs(spawn), sandboxBase, realBase);
assertThat(
FileSystemUtils.readLines(
realBase.getRelative(unmappedOutputPath.getPathString()), UTF_8))
.containsExactly("hello", "pathmapper")
.inOrder();
assertThat(sandboxBase.getRelative(mappedOutputPath).exists()).isFalse();
}
}