blob: 89418d0ba8e405e636938aa5aad2dd8fc9dcc53b [file] [log] [blame]
// Copyright 2016 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertThrows;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.hash.Hashing;
import com.google.devtools.build.lib.actions.Action;
import com.google.devtools.build.lib.actions.ActionAnalysisMetadata;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionExecutionException;
import com.google.devtools.build.lib.actions.ActionKeyContext;
import com.google.devtools.build.lib.actions.ActionLookupData;
import com.google.devtools.build.lib.actions.ActionLookupKey;
import com.google.devtools.build.lib.actions.ActionResult;
import com.google.devtools.build.lib.actions.Actions;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
import com.google.devtools.build.lib.actions.Artifact.SpecialArtifactType;
import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
import com.google.devtools.build.lib.actions.ArtifactRoot;
import com.google.devtools.build.lib.actions.ArtifactRoot.RootType;
import com.google.devtools.build.lib.actions.BuildFailedException;
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue;
import com.google.devtools.build.lib.actions.MetadataProvider;
import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException;
import com.google.devtools.build.lib.actions.cache.MetadataHandler;
import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
import com.google.devtools.build.lib.actions.util.TestAction;
import com.google.devtools.build.lib.actions.util.TestAction.DummyAction;
import com.google.devtools.build.lib.analysis.actions.SpawnActionTemplate;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.collect.nestedset.Order;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.EventCollector;
import com.google.devtools.build.lib.events.EventKind;
import com.google.devtools.build.lib.server.FailureDetails.Crash;
import com.google.devtools.build.lib.server.FailureDetails.Crash.Code;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.skyframe.ActionTemplateExpansionValue.ActionTemplateExpansionKey;
import com.google.devtools.build.lib.skyframe.serialization.testutils.SerializationDepsUtils;
import com.google.devtools.build.lib.skyframe.serialization.testutils.SerializationTester;
import com.google.devtools.build.lib.testutil.TestUtils;
import com.google.devtools.build.lib.util.CrashFailureDetails;
import com.google.devtools.build.lib.util.DetailedExitCode;
import com.google.devtools.build.lib.vfs.DigestHashFunction;
import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.lib.vfs.Symlinks;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Timestamp builder tests for TreeArtifacts. */
@RunWith(JUnit4.class)
public final class TreeArtifactBuildTest extends TimestampBuilderTestCase {
@Test
public void codec() throws Exception {
SpecialArtifact parent = createTreeArtifact("parent");
parent.setGeneratingActionKey(ActionLookupData.create(ACTION_LOOKUP_KEY, 0));
new SerializationTester(parent, TreeFileArtifact.createTreeOutput(parent, "child"))
.addDependency(FileSystem.class, scratch.getFileSystem())
.addDependency(
Root.RootCodecDependencies.class,
new Root.RootCodecDependencies(Root.absoluteRoot(scratch.getFileSystem())))
.addDependencies(SerializationDepsUtils.SERIALIZATION_DEPS_FOR_TEST)
.runTests();
}
/** Simple smoke test. If this isn't passing, something is very wrong... */
@Test
public void treeArtifactSimpleCase() throws Exception {
SpecialArtifact parent = createTreeArtifact("parent");
TouchingTestAction action = new TouchingTestAction(parent, "out1", "out2");
registerAction(action);
TreeArtifactValue result = buildArtifact(parent);
verifyOutputTree(result, parent, "out1", "out2");
}
/** Simple test for the case with dependencies. */
@Test
public void dependentTreeArtifacts() throws Exception {
SpecialArtifact tree1 = createTreeArtifact("tree1");
TouchingTestAction action1 = new TouchingTestAction(tree1, "out1", "out2");
registerAction(action1);
SpecialArtifact tree2 = createTreeArtifact("tree2");
CopyTreeAction action2 = new CopyTreeAction(tree1, tree2);
registerAction(action2);
TreeArtifactValue result = buildArtifact(tree2);
assertThat(tree1.getPath().getRelative("out1").exists()).isTrue();
assertThat(tree1.getPath().getRelative("out2").exists()).isTrue();
verifyOutputTree(result, tree2, "out1", "out2");
}
/** Test for tree artifacts with sub directories. */
@Test
public void treeArtifactWithSubDirectory() throws Exception {
SpecialArtifact parent = createTreeArtifact("parent");
TouchingTestAction action = new TouchingTestAction(parent, "sub1/file1", "sub2/file2");
registerAction(action);
TreeArtifactValue result = buildArtifact(parent);
verifyOutputTree(result, parent, "sub1/file1", "sub2/file2");
}
@Test
public void inputTreeArtifactMetadataProvider() throws Exception {
SpecialArtifact treeArtifactInput = createTreeArtifact("tree");
TouchingTestAction action1 = new TouchingTestAction(treeArtifactInput, "out1", "out2");
registerAction(action1);
Artifact normalOutput = createDerivedArtifact("normal/out");
Action testAction =
new SimpleTestAction(ImmutableList.of(treeArtifactInput), normalOutput) {
@Override
void run(ActionExecutionContext actionExecutionContext) throws IOException {
// Check the metadata provider for input TreeFileArtifacts.
MetadataProvider metadataProvider = actionExecutionContext.getMetadataProvider();
assertThat(
metadataProvider
.getMetadata(TreeFileArtifact.createTreeOutput(treeArtifactInput, "out1"))
.getType()
.isFile())
.isTrue();
assertThat(
metadataProvider
.getMetadata(TreeFileArtifact.createTreeOutput(treeArtifactInput, "out2"))
.getType()
.isFile())
.isTrue();
// Touch the action output.
touchFile(normalOutput);
}
};
registerAction(testAction);
buildArtifact(normalOutput);
}
/** Unchanged TreeArtifact outputs should not cause reexecution. */
@Test
public void cacheCheckingForTreeArtifactsDoesNotCauseReexecution() throws Exception {
SpecialArtifact out1 = createTreeArtifact("out1");
Button button1 = new Button();
SpecialArtifact out2 = createTreeArtifact("out2");
Button button2 = new Button();
TouchingTestAction action1 = new TouchingTestAction(button1, out1, "file_one", "file_two");
registerAction(action1);
CopyTreeAction action2 = new CopyTreeAction(button2, out1, out2);
registerAction(action2);
button1.pressed = false;
button2.pressed = false;
buildArtifact(out2);
assertThat(button1.pressed).isTrue(); // built
assertThat(button2.pressed).isTrue(); // built
button1.pressed = false;
button2.pressed = false;
buildArtifact(out2);
assertThat(button1.pressed).isFalse(); // not built
assertThat(button2.pressed).isFalse(); // not built
}
/** Test rebuilding TreeArtifacts for inputs, outputs, and dependents. Also a test for caching. */
@Test
public void transitiveReexecutionForTreeArtifacts() throws Exception {
Artifact in = createSourceArtifact("input");
writeFile(in, "input content");
Button button1 = new Button();
SpecialArtifact out1 = createTreeArtifact("output1");
WriteInputToFilesAction action1 =
new WriteInputToFilesAction(button1, in, out1, "file1", "file2");
registerAction(action1);
Button button2 = new Button();
SpecialArtifact out2 = createTreeArtifact("output2");
CopyTreeAction action2 = new CopyTreeAction(button2, out1, out2);
registerAction(action2);
button1.pressed = false;
button2.pressed = false;
buildArtifact(out2);
assertThat(button1.pressed).isTrue(); // built
assertThat(button2.pressed).isTrue(); // built
button1.pressed = false;
button2.pressed = false;
writeFile(in, "modified input");
buildArtifact(out2);
assertThat(button1.pressed).isTrue(); // built
assertThat(button2.pressed).isTrue(); // built
button1.pressed = false;
button2.pressed = false;
writeFile(TreeFileArtifact.createTreeOutput(out1, "file1"), "modified output");
buildArtifact(out2);
assertThat(button1.pressed).isTrue(); // built
assertThat(button2.pressed).isFalse(); // should have been cached
button1.pressed = false;
button2.pressed = false;
writeFile(TreeFileArtifact.createTreeOutput(out2, "file1"), "more modified output");
buildArtifact(out2);
assertThat(button1.pressed).isFalse(); // not built
assertThat(button2.pressed).isTrue(); // built
}
/** Tests that changing a TreeArtifact directory should cause reexeuction. */
@Test
public void directoryContentsCachingForTreeArtifacts() throws Exception {
Artifact in = createSourceArtifact("input");
writeFile(in, "input content");
Button button1 = new Button();
SpecialArtifact out1 = createTreeArtifact("output1");
WriteInputToFilesAction action1 =
new WriteInputToFilesAction(button1, in, out1, "file1", "file2");
registerAction(action1);
Button button2 = new Button();
SpecialArtifact out2 = createTreeArtifact("output2");
CopyTreeAction action2 = new CopyTreeAction(button2, out1, out2);
registerAction(action2);
button1.pressed = false;
button2.pressed = false;
buildArtifact(out2);
// just a smoke test--if these aren't built we have bigger problems!
assertThat(button1.pressed).isTrue();
assertThat(button2.pressed).isTrue();
// Adding a file to a directory should cause reexecution.
button1.pressed = false;
button2.pressed = false;
Path spuriousOutputOne = out1.getPath().getRelative("spuriousOutput");
touchFile(spuriousOutputOne);
buildArtifact(out2);
// Should re-execute, and delete spurious output
assertThat(spuriousOutputOne.exists()).isFalse();
assertThat(button1.pressed).isTrue();
assertThat(button2.pressed).isFalse(); // should have been cached
button1.pressed = false;
button2.pressed = false;
Path spuriousOutputTwo = out2.getPath().getRelative("anotherSpuriousOutput");
touchFile(spuriousOutputTwo);
buildArtifact(out2);
assertThat(spuriousOutputTwo.exists()).isFalse();
assertThat(button1.pressed).isFalse();
assertThat(button2.pressed).isTrue();
// Deleting should cause reexecution.
button1.pressed = false;
button2.pressed = false;
TreeFileArtifact out1File1 = TreeFileArtifact.createTreeOutput(out1, "file1");
deleteFile(out1File1);
buildArtifact(out2);
assertThat(out1File1.getPath().exists()).isTrue();
assertThat(button1.pressed).isTrue();
assertThat(button2.pressed).isFalse(); // should have been cached
button1.pressed = false;
button2.pressed = false;
TreeFileArtifact out2File1 = TreeFileArtifact.createTreeOutput(out2, "file1");
deleteFile(out2File1);
buildArtifact(out2);
assertThat(out2File1.getPath().exists()).isTrue();
assertThat(button1.pressed).isFalse();
assertThat(button2.pressed).isTrue();
}
/** TreeArtifacts don't care about mtime, even when the file is empty. */
@Test
public void mTimeForTreeArtifactsDoesNotMatter() throws Exception {
// For this test, we only touch the input file.
Artifact in = createSourceArtifact("touchable_input");
touchFile(in);
Button button1 = new Button();
SpecialArtifact out1 = createTreeArtifact("output1");
WriteInputToFilesAction action1 =
new WriteInputToFilesAction(button1, in, out1, "file1", "file2");
registerAction(action1);
Button button2 = new Button();
SpecialArtifact out2 = createTreeArtifact("output2");
CopyTreeAction action2 = new CopyTreeAction(button2, out1, out2);
registerAction(action2);
button1.pressed = false;
button2.pressed = false;
buildArtifact(out2);
assertThat(button1.pressed).isTrue(); // built
assertThat(button2.pressed).isTrue(); // built
button1.pressed = false;
button2.pressed = false;
touchFile(in);
buildArtifact(out2);
// mtime does not matter.
assertThat(button1.pressed).isFalse();
assertThat(button2.pressed).isFalse();
// None of the below following should result in anything being built.
button1.pressed = false;
button2.pressed = false;
touchFile(TreeFileArtifact.createTreeOutput(out1, "file1"));
buildArtifact(out2);
// Nothing should be built.
assertThat(button1.pressed).isFalse();
assertThat(button2.pressed).isFalse();
button1.pressed = false;
button2.pressed = false;
touchFile(TreeFileArtifact.createTreeOutput(out1, "file2"));
buildArtifact(out2);
// Nothing should be built.
assertThat(button1.pressed).isFalse();
assertThat(button2.pressed).isFalse();
}
private static void checkDirectoryPermissions(Path path) throws IOException {
assertThat(path.isDirectory()).isTrue();
assertThat(path.isExecutable()).isTrue();
assertThat(path.isReadable()).isTrue();
assertThat(path.isWritable()).isFalse();
}
private static void checkFilePermissions(Path path) throws IOException {
assertThat(path.isDirectory()).isFalse();
assertThat(path.isExecutable()).isTrue();
assertThat(path.isReadable()).isTrue();
assertThat(path.isWritable()).isFalse();
}
@Test
public void outputsAreReadOnlyAndExecutable() throws Exception {
SpecialArtifact out = createTreeArtifact("output");
Action action =
new SimpleTestAction(out) {
@Override
void run(ActionExecutionContext context) throws IOException {
writeFile(out.getPath().getChild("one"), "one");
writeFile(out.getPath().getChild("two"), "two");
writeFile(out.getPath().getChild("three").getChild("four"), "three/four");
}
};
registerAction(action);
buildArtifact(out);
checkDirectoryPermissions(out.getPath());
checkFilePermissions(out.getPath().getChild("one"));
checkFilePermissions(out.getPath().getChild("two"));
checkDirectoryPermissions(out.getPath().getChild("three"));
checkFilePermissions(out.getPath().getChild("three").getChild("four"));
}
@Test
public void validRelativeSymlinkAccepted() throws Exception {
SpecialArtifact out = createTreeArtifact("output");
Action action =
new SimpleTestAction(out) {
@Override
void run(ActionExecutionContext actionExecutionContext) throws IOException {
writeFile(out.getPath().getChild("one"), "one");
writeFile(out.getPath().getChild("two"), "two");
FileSystemUtils.ensureSymbolicLink(
out.getPath().getChild("links").getChild("link"), "../one");
}
};
registerAction(action);
buildArtifact(out);
}
@Test
public void invalidSymlinkRejected() {
// Failure expected
EventCollector eventCollector = new EventCollector(EventKind.ERROR);
reporter.removeHandler(failFastHandler);
reporter.addHandler(eventCollector);
SpecialArtifact out = createTreeArtifact("output");
Action action =
new SimpleTestAction(out) {
@Override
void run(ActionExecutionContext actionExecutionContext) throws IOException {
writeFile(out.getPath().getChild("one"), "one");
writeFile(out.getPath().getChild("two"), "two");
FileSystemUtils.ensureSymbolicLink(
out.getPath().getChild("links").getChild("link"), "../invalid");
}
};
registerAction(action);
assertThrows(BuildFailedException.class, () -> buildArtifact(out));
List<Event> errors = ImmutableList.copyOf(eventCollector);
assertThat(errors).hasSize(2);
assertThat(errors.get(0).getMessage()).contains("Failed to resolve relative path links/link");
assertThat(errors.get(1).getMessage()).contains("not all outputs were created or valid");
}
@Test
public void absoluteSymlinkBadTargetRejected() {
// Failure expected
EventCollector eventCollector = new EventCollector(EventKind.ERROR);
reporter.removeHandler(failFastHandler);
reporter.addHandler(eventCollector);
SpecialArtifact out = createTreeArtifact("output");
Action action =
new SimpleTestAction(out) {
@Override
void run(ActionExecutionContext actionExecutionContext) throws IOException {
writeFile(out.getPath().getChild("one"), "one");
writeFile(out.getPath().getChild("two"), "two");
FileSystemUtils.ensureSymbolicLink(
out.getPath().getChild("links").getChild("link"), "/random/pointer");
}
};
registerAction(action);
assertThrows(BuildFailedException.class, () -> buildArtifact(out));
List<Event> errors = ImmutableList.copyOf(eventCollector);
assertThat(errors).hasSize(2);
assertThat(errors.get(0).getMessage()).contains("Failed to resolve relative path links/link");
assertThat(errors.get(1).getMessage()).contains("not all outputs were created or valid");
}
@Test
public void absoluteSymlinkAccepted() throws Exception {
scratch.overwriteFile("/random/pointer");
SpecialArtifact out = createTreeArtifact("output");
Action action =
new SimpleTestAction(out) {
@Override
void run(ActionExecutionContext actionExecutionContext) throws IOException {
writeFile(out.getPath().getChild("one"), "one");
writeFile(out.getPath().getChild("two"), "two");
FileSystemUtils.ensureSymbolicLink(
out.getPath().getChild("links").getChild("link"), "/random/pointer");
}
};
registerAction(action);
buildArtifact(out);
}
@Test
public void relativeSymlinkTraversingOutsideOfTreeArtifactRejected() {
// Failure expected
EventCollector eventCollector = new EventCollector(EventKind.ERROR);
reporter.removeHandler(failFastHandler);
reporter.addHandler(eventCollector);
SpecialArtifact out = createTreeArtifact("output");
Action action =
new SimpleTestAction(out) {
@Override
void run(ActionExecutionContext actionExecutionContext) throws IOException {
writeFile(out.getPath().getChild("one"), "one");
writeFile(out.getPath().getChild("two"), "two");
FileSystemUtils.ensureSymbolicLink(
out.getPath().getChild("links").getChild("link"), "../../output/random/pointer");
}
};
registerAction(action);
assertThrows(BuildFailedException.class, () -> buildArtifact(out));
List<Event> errors = ImmutableList.copyOf(eventCollector);
assertThat(errors).hasSize(2);
assertThat(errors.get(0).getMessage())
.contains(
"A TreeArtifact may not contain relative symlinks whose target paths traverse "
+ "outside of the TreeArtifact");
assertThat(errors.get(1).getMessage()).contains("not all outputs were created or valid");
}
@Test
public void relativeSymlinkTraversingToDirOutsideOfTreeArtifactRejected() throws Exception {
// Failure expected
EventCollector eventCollector = new EventCollector(EventKind.ERROR);
reporter.removeHandler(failFastHandler);
reporter.addHandler(eventCollector);
SpecialArtifact out = createTreeArtifact("output");
// Create a valid directory that can be referenced
scratch.dir(out.getRoot().getRoot().getRelative("some/dir").getPathString());
TestAction action =
new SimpleTestAction(out) {
@Override
void run(ActionExecutionContext actionExecutionContext) throws IOException {
writeFile(out.getPath().getChild("one"), "one");
writeFile(out.getPath().getChild("two"), "two");
FileSystemUtils.ensureSymbolicLink(
out.getPath().getChild("links").getChild("link"), "../../some/dir");
}
};
registerAction(action);
assertThrows(BuildFailedException.class, () -> buildArtifact(out));
List<Event> errors = ImmutableList.copyOf(eventCollector);
assertThat(errors).hasSize(2);
assertThat(errors.get(0).getMessage())
.contains(
"A TreeArtifact may not contain relative symlinks whose target paths traverse "
+ "outside of the TreeArtifact");
assertThat(errors.get(1).getMessage()).contains("not all outputs were created or valid");
}
@Test
public void constructMetadataForDigest() throws Exception {
SpecialArtifact out = createTreeArtifact("output");
Action action =
new SimpleTestAction(out) {
@Override
void run(ActionExecutionContext actionExecutionContext) throws IOException {
TreeFileArtifact child1 = TreeFileArtifact.createTreeOutput(out, "one");
TreeFileArtifact child2 = TreeFileArtifact.createTreeOutput(out, "two");
writeFile(child1, "one");
writeFile(child2, "two");
MetadataHandler md = actionExecutionContext.getMetadataHandler();
FileStatus stat = child1.getPath().stat(Symlinks.NOFOLLOW);
FileArtifactValue metadata1 =
md.constructMetadataForDigest(
child1,
stat,
DigestHashFunction.SHA256.getHashFunction().hashString("one", UTF_8).asBytes());
stat = child2.getPath().stat(Symlinks.NOFOLLOW);
FileArtifactValue metadata2 =
md.constructMetadataForDigest(
child2,
stat,
DigestHashFunction.SHA256.getHashFunction().hashString("two", UTF_8).asBytes());
// The metadata will not be equal to reading from the filesystem since the filesystem
// won't have the digest. However, we should be able to detect that nothing could have
// been modified.
assertThat(
metadata1.couldBeModifiedSince(
FileArtifactValue.createForTesting(child1.getPath())))
.isFalse();
assertThat(
metadata2.couldBeModifiedSince(
FileArtifactValue.createForTesting(child2.getPath())))
.isFalse();
}
};
registerAction(action);
buildArtifact(out);
}
@Test
public void remoteDirectoryInjection() throws Exception {
SpecialArtifact out = createTreeArtifact("output");
RemoteFileArtifactValue remoteFile1 =
new RemoteFileArtifactValue(
Hashing.sha256().hashString("one", UTF_8).asBytes(), /*size=*/ 3, /*locationIndex=*/ 1);
RemoteFileArtifactValue remoteFile2 =
new RemoteFileArtifactValue(
Hashing.sha256().hashString("two", UTF_8).asBytes(), /*size=*/ 3, /*locationIndex=*/ 2);
Action action =
new SimpleTestAction(out) {
@Override
void run(ActionExecutionContext actionExecutionContext) throws IOException {
TreeFileArtifact child1 = TreeFileArtifact.createTreeOutput(out, "one");
TreeFileArtifact child2 = TreeFileArtifact.createTreeOutput(out, "two");
writeFile(child1, "one");
writeFile(child2, "two");
actionExecutionContext
.getMetadataHandler()
.injectTree(
out,
TreeArtifactValue.newBuilder(out)
.putChild(child1, remoteFile1)
.putChild(child2, remoteFile2)
.build());
}
};
registerAction(action);
TreeArtifactValue result = buildArtifact(out);
assertThat(result.getChildValues())
.containsExactly(
TreeFileArtifact.createTreeOutput(out, "one"),
remoteFile1,
TreeFileArtifact.createTreeOutput(out, "two"),
remoteFile2);
}
@Test
public void expandedActionsBuildInActionTemplate() throws Exception {
// artifact1 is a tree artifact generated by a TouchingTestAction.
SpecialArtifact artifact1 = createTreeArtifact("treeArtifact1");
registerAction(new TouchingTestAction(artifact1, "file1", "file2"));
// artifact2 is a tree artifact generated by an action template.
SpecialArtifact artifact2 = createTreeArtifact("treeArtifact2");
SpawnActionTemplate actionTemplate =
ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
registerAction(actionTemplate);
// We mock out the action template function to expand into two actions that just touch the
// output files.
ActionTemplateExpansionKey secondOwner = ActionTemplateExpansionValue.key(ACTION_LOOKUP_KEY, 1);
TreeFileArtifact expectedExpansionOutput1 =
TreeFileArtifact.createTemplateExpansionOutput(artifact2, "child1", secondOwner);
TreeFileArtifact expectedExpansionOutput2 =
TreeFileArtifact.createTemplateExpansionOutput(artifact2, "child2", secondOwner);
Action expandedAction1 =
new DummyAction(
TreeFileArtifact.createTreeOutput(artifact1, "file1"), expectedExpansionOutput1);
Action expandedAction2 =
new DummyAction(
TreeFileArtifact.createTreeOutput(artifact1, "file2"), expectedExpansionOutput2);
actionTemplateExpansionFunction =
new DummyActionTemplateExpansionFunction(
actionKeyContext, ImmutableList.of(expandedAction1, expandedAction2));
TreeArtifactValue result = buildArtifact(artifact2);
assertThat(result.getChildren())
.containsExactly(expectedExpansionOutput1, expectedExpansionOutput2);
}
@Test
public void expandedActionDoesNotGenerateOutputInActionTemplate() {
// expect errors
reporter.removeHandler(failFastHandler);
// artifact1 is a tree artifact generated by a TouchingTestAction.
SpecialArtifact artifact1 = createTreeArtifact("treeArtifact1");
registerAction(new TouchingTestAction(artifact1, "child1", "child2"));
// artifact2 is a tree artifact generated by an action template.
SpecialArtifact artifact2 = createTreeArtifact("treeArtifact2");
SpawnActionTemplate actionTemplate =
ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
registerAction(actionTemplate);
// We mock out the action template function to expand into two actions:
// One Action that touches the output file.
// The other action that does not generate the output file.
ActionTemplateExpansionKey secondOwner = ActionTemplateExpansionKey.of(ACTION_LOOKUP_KEY, 1);
TreeFileArtifact expectedExpansionOutput1 =
TreeFileArtifact.createTemplateExpansionOutput(artifact2, "child1", secondOwner);
TreeFileArtifact expectedExpansionOutput2 =
TreeFileArtifact.createTemplateExpansionOutput(artifact2, "child2", secondOwner);
Action generateOutputAction =
new DummyAction(
TreeFileArtifact.createTreeOutput(artifact1, "child1"), expectedExpansionOutput1);
Action noGenerateOutputAction =
new NoOpDummyAction(
TreeFileArtifact.createTreeOutput(artifact1, "child2"), expectedExpansionOutput2);
actionTemplateExpansionFunction =
new DummyActionTemplateExpansionFunction(
actionKeyContext, ImmutableList.of(generateOutputAction, noGenerateOutputAction));
BuildFailedException e =
assertThrows(BuildFailedException.class, () -> buildArtifact(artifact2));
assertThat(e).hasMessageThat().contains("not all outputs were created or valid");
}
@Test
public void oneExpandedActionThrowsInActionTemplate() {
// expect errors
reporter.removeHandler(failFastHandler);
// artifact1 is a tree artifact generated by a TouchingTestAction.
SpecialArtifact artifact1 = createTreeArtifact("treeArtifact1");
registerAction(new TouchingTestAction(artifact1, "child1", "child2"));
// artifact2 is a tree artifact generated by an action template.
SpecialArtifact artifact2 = createTreeArtifact("treeArtifact2");
SpawnActionTemplate actionTemplate =
ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
registerAction(actionTemplate);
// We mock out the action template function to expand into two actions:
// One Action that touches the output file.
// The other action that just throws when executed.
ActionTemplateExpansionKey secondOwner = ActionTemplateExpansionKey.of(ACTION_LOOKUP_KEY, 1);
TreeFileArtifact expectedExpansionOutput1 =
TreeFileArtifact.createTemplateExpansionOutput(artifact2, "child1", secondOwner);
TreeFileArtifact expectedExpansionOutput2 =
TreeFileArtifact.createTemplateExpansionOutput(artifact2, "child2", secondOwner);
Action generateOutputAction =
new DummyAction(
TreeFileArtifact.createTreeOutput(artifact1, "child1"), expectedExpansionOutput1);
Action throwingAction =
new ThrowingDummyAction(
TreeFileArtifact.createTreeOutput(artifact1, "child2"), expectedExpansionOutput2);
actionTemplateExpansionFunction =
new DummyActionTemplateExpansionFunction(
actionKeyContext, ImmutableList.of(generateOutputAction, throwingAction));
BuildFailedException e =
assertThrows(BuildFailedException.class, () -> buildArtifact(artifact2));
assertThat(e).hasMessageThat().contains("Throwing dummy action");
}
@Test
public void allExpandedActionsThrowInActionTemplate() {
// expect errors
reporter.removeHandler(failFastHandler);
// artifact1 is a tree artifact generated by a TouchingTestAction.
SpecialArtifact artifact1 = createTreeArtifact("treeArtifact1");
registerAction(new TouchingTestAction(artifact1, "child1", "child2"));
// artifact2 is a tree artifact generated by an action template.
SpecialArtifact artifact2 = createTreeArtifact("treeArtifact2");
SpawnActionTemplate actionTemplate =
ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
registerAction(actionTemplate);
// We mock out the action template function to expand into two actions that throw when executed.
ActionTemplateExpansionKey secondOwner = ActionTemplateExpansionKey.of(ACTION_LOOKUP_KEY, 1);
TreeFileArtifact expectedExpansionOutput1 =
TreeFileArtifact.createTemplateExpansionOutput(artifact2, "child1", secondOwner);
TreeFileArtifact expectedExpansionOutput2 =
TreeFileArtifact.createTemplateExpansionOutput(artifact2, "child2", secondOwner);
Action throwingAction =
new ThrowingDummyAction(
TreeFileArtifact.createTreeOutput(artifact1, "child1"), expectedExpansionOutput1);
Action anotherThrowingAction =
new ThrowingDummyAction(
TreeFileArtifact.createTreeOutput(artifact1, "child2"), expectedExpansionOutput2);
actionTemplateExpansionFunction =
new DummyActionTemplateExpansionFunction(
actionKeyContext, ImmutableList.of(throwingAction, anotherThrowingAction));
BuildFailedException e =
assertThrows(BuildFailedException.class, () -> buildArtifact(artifact2));
assertThat(e).hasMessageThat().contains("Throwing dummy action");
}
@Test
public void inputTreeArtifactCreationFailedInActionTemplate() {
// expect errors
reporter.removeHandler(failFastHandler);
// artifact1 is created by a action that throws.
SpecialArtifact artifact1 = createTreeArtifact("treeArtifact1");
registerAction(new ThrowingDummyAction(artifact1));
// artifact2 is a tree artifact generated by an action template.
SpecialArtifact artifact2 = createTreeArtifact("treeArtifact2");
SpawnActionTemplate actionTemplate =
ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
registerAction(actionTemplate);
BuildFailedException e =
assertThrows(BuildFailedException.class, () -> buildArtifact(artifact2));
assertThat(e).hasMessageThat().contains("Throwing dummy action");
}
@Test
public void emptyInputAndOutputTreeArtifactInActionTemplate() throws Exception {
// artifact1 is an empty tree artifact which is generated by a single no-op dummy action.
SpecialArtifact artifact1 = createTreeArtifact("treeArtifact1");
registerAction(new NoOpDummyAction(artifact1));
// artifact2 is a tree artifact generated by an action template that takes artifact1 as input.
SpecialArtifact artifact2 = createTreeArtifact("treeArtifact2");
SpawnActionTemplate actionTemplate =
ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
registerAction(actionTemplate);
buildArtifact(artifact2);
assertThat(artifact1.getPath().exists()).isTrue();
assertThat(artifact1.getPath().getDirectoryEntries()).isEmpty();
assertThat(artifact2.getPath().exists()).isTrue();
assertThat(artifact2.getPath().getDirectoryEntries()).isEmpty();
}
// This happens in the wild. See https://github.com/bazelbuild/bazel/issues/11813.
@Test
public void treeArtifactContainsSymlinkToDirectory() throws Exception {
SpecialArtifact treeArtifact = createTreeArtifact("tree");
registerAction(
new SimpleTestAction(/*output=*/ treeArtifact) {
@Override
void run(ActionExecutionContext context) throws IOException {
PathFragment subdir = PathFragment.create("subdir");
touchFile(treeArtifact.getPath().getRelative(subdir).getRelative("file"));
treeArtifact.getPath().getRelative("link").createSymbolicLink(subdir);
}
});
TreeArtifactValue tree = buildArtifact(treeArtifact);
assertThat(tree.getChildren())
.containsExactly(
TreeFileArtifact.createTreeOutput(treeArtifact, "subdir/file"),
TreeFileArtifact.createTreeOutput(treeArtifact, "link"));
}
private abstract static class SimpleTestAction extends TestAction {
private final Button button;
SimpleTestAction(Artifact output) {
this(/*inputs=*/ ImmutableList.of(), output);
}
SimpleTestAction(Iterable<Artifact> inputs, Artifact output) {
this(new Button(), inputs, output);
}
SimpleTestAction(Button button, Iterable<Artifact> inputs, Artifact output) {
super(NO_EFFECT, NestedSetBuilder.wrap(Order.STABLE_ORDER, inputs), ImmutableSet.of(output));
this.button = button;
}
@Override
public final ActionResult execute(ActionExecutionContext context)
throws ActionExecutionException {
button.pressed = true;
try {
run(context);
} catch (IOException e) {
throw new ActionExecutionException(
e, this, /*catastrophe=*/ false, CrashFailureDetails.detailedExitCodeForThrowable(e));
}
return ActionResult.EMPTY;
}
abstract void run(ActionExecutionContext context) throws IOException;
}
/** An action that touches some output TreeFileArtifacts. Takes no inputs. */
private static final class TouchingTestAction extends SimpleTestAction {
private final ImmutableList<String> outputFiles;
TouchingTestAction(SpecialArtifact output, String... outputFiles) {
this(new Button(), output, outputFiles);
}
TouchingTestAction(Button button, SpecialArtifact output, String... outputFiles) {
super(button, /*inputs=*/ ImmutableList.of(), output);
this.outputFiles = ImmutableList.copyOf(outputFiles);
}
@Override
void run(ActionExecutionContext context) throws IOException {
for (String file : outputFiles) {
touchFile(getPrimaryOutput().getPath().getRelative(file));
}
}
}
/** Takes an input file and populates several copies inside a TreeArtifact. */
private static final class WriteInputToFilesAction extends SimpleTestAction {
private final ImmutableList<String> outputFiles;
WriteInputToFilesAction(
Button button, Artifact input, SpecialArtifact output, String... outputFiles) {
super(button, ImmutableList.of(input), output);
this.outputFiles = ImmutableList.copyOf(outputFiles);
}
@Override
void run(ActionExecutionContext actionExecutionContext) throws IOException {
for (String file : outputFiles) {
Path newOutput = getPrimaryOutput().getPath().getRelative(file);
newOutput.createDirectoryAndParents();
FileSystemUtils.copyFile(getPrimaryInput().getPath(), newOutput);
}
}
}
/** Copies the given TreeFileArtifact inputs to the given outputs, in respective order. */
private static final class CopyTreeAction extends SimpleTestAction {
CopyTreeAction(SpecialArtifact input, SpecialArtifact output) {
this(new Button(), input, output);
}
CopyTreeAction(Button button, SpecialArtifact input, SpecialArtifact output) {
super(button, ImmutableList.of(input), output);
}
@Override
void run(ActionExecutionContext context) throws IOException {
List<Artifact> children = new ArrayList<>();
context.getArtifactExpander().expand(getPrimaryInput(), children);
for (Artifact child : children) {
Path newOutput = getPrimaryOutput().getPath().getRelative(child.getParentRelativePath());
newOutput.createDirectoryAndParents();
FileSystemUtils.copyFile(child.getPath(), newOutput);
}
}
}
private SpecialArtifact createTreeArtifact(String name) {
FileSystem fs = scratch.getFileSystem();
Path execRoot =
fs.getPath(TestUtils.tmpDir()).getRelative("execroot").getRelative("default-exec-root");
PathFragment execPath = PathFragment.create("out").getRelative(name);
return SpecialArtifact.create(
ArtifactRoot.asDerivedRoot(execRoot, RootType.Output, "out"),
execPath,
ACTION_LOOKUP_KEY,
SpecialArtifactType.TREE);
}
private TreeArtifactValue buildArtifact(SpecialArtifact treeArtifact) throws Exception {
Preconditions.checkArgument(treeArtifact.isTreeArtifact(), treeArtifact);
BuilderWithResult builder = cachingBuilder();
buildArtifacts(builder, treeArtifact);
return (TreeArtifactValue) builder.getLatestResult().get(treeArtifact);
}
private void buildArtifact(Artifact normalArtifact) throws Exception {
buildArtifacts(cachingBuilder(), normalArtifact);
}
private static void verifyOutputTree(
TreeArtifactValue result, SpecialArtifact parent, String... expectedChildPaths) {
Preconditions.checkArgument(parent.isTreeArtifact(), parent);
Set<TreeFileArtifact> expectedChildren =
Arrays.stream(expectedChildPaths)
.map(path -> TreeFileArtifact.createTreeOutput(parent, path))
.collect(toImmutableSet());
for (TreeFileArtifact child : expectedChildren) {
assertWithMessage(child + " does not exist").that(child.getPath().exists()).isTrue();
}
assertThat(result.getChildren()).isEqualTo(expectedChildren);
}
private static void writeFile(Path path, String contents) throws IOException {
path.getParentDirectory().createDirectoryAndParents();
// sometimes we write read-only files
if (path.exists()) {
path.setWritable(true);
}
FileSystemUtils.writeContentAsLatin1(path, contents);
}
private static void writeFile(Artifact file, String contents) throws IOException {
writeFile(file.getPath(), contents);
}
private static void touchFile(Path path) throws IOException {
path.getParentDirectory().createDirectoryAndParents();
path.getParentDirectory().setWritable(true);
FileSystemUtils.touchFile(path);
}
private static void touchFile(Artifact file) throws IOException {
touchFile(file.getPath());
}
private static void deleteFile(Artifact file) throws IOException {
Path path = file.getPath();
// sometimes we write read-only files
path.setWritable(true);
// work around the sticky bit (this might depend on the behavior of the OS?)
path.getParentDirectory().setWritable(true);
path.delete();
}
/** A dummy action template expansion function that just returns the injected actions. */
private static final class DummyActionTemplateExpansionFunction implements SkyFunction {
private final ActionKeyContext actionKeyContext;
private final ImmutableList<ActionAnalysisMetadata> actions;
DummyActionTemplateExpansionFunction(
ActionKeyContext actionKeyContext, ImmutableList<ActionAnalysisMetadata> actions) {
this.actionKeyContext = actionKeyContext;
this.actions = actions;
}
@Override
public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException {
try {
return new ActionTemplateExpansionValue(
Actions.assignOwnersAndFilterSharedActionsAndThrowActionConflict(
actionKeyContext,
actions,
(ActionLookupKey) skyKey,
/*outputFiles=*/ null));
} catch (ActionConflictException | Actions.ArtifactGeneratedByOtherRuleException e) {
throw new IllegalStateException(e);
}
}
}
/** No-op action that does not generate the action outputs. */
private static final class NoOpDummyAction extends SimpleTestAction {
NoOpDummyAction(Artifact output) {
super(/*inputs=*/ ImmutableList.of(), output);
}
NoOpDummyAction(Artifact input, Artifact output) {
super(ImmutableList.of(input), output);
}
/** Does nothing. */
@Override
void run(ActionExecutionContext actionExecutionContext) {}
}
/** No-op action that throws when executed. */
private static final class ThrowingDummyAction extends TestAction {
ThrowingDummyAction(Artifact output) {
super(NO_EFFECT, NestedSetBuilder.emptySet(Order.STABLE_ORDER), ImmutableSet.of(output));
}
ThrowingDummyAction(Artifact input, Artifact output) {
super(NO_EFFECT, NestedSetBuilder.create(Order.STABLE_ORDER, input), ImmutableSet.of(output));
}
/** Unconditionally throws. */
@Override
public ActionResult execute(ActionExecutionContext actionExecutionContext)
throws ActionExecutionException {
DetailedExitCode code =
DetailedExitCode.of(
FailureDetail.newBuilder()
.setCrash(Crash.newBuilder().setCode(Code.CRASH_UNKNOWN))
.build());
throw new ActionExecutionException(
"Throwing dummy action", this, /*catastrophe=*/ true, code);
}
}
}