| // Copyright 2016 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| package com.google.devtools.build.lib.skyframe; |
| |
| import static com.google.common.collect.ImmutableSet.toImmutableSet; |
| import static com.google.common.truth.Truth.assertThat; |
| import static com.google.common.truth.Truth.assertWithMessage; |
| import static java.nio.charset.StandardCharsets.UTF_8; |
| import static org.junit.Assert.assertThrows; |
| |
| import com.google.common.base.Preconditions; |
| import com.google.common.collect.ImmutableList; |
| import com.google.common.collect.ImmutableMap; |
| import com.google.common.collect.ImmutableSet; |
| import com.google.common.hash.Hashing; |
| import com.google.devtools.build.lib.actions.Action; |
| import com.google.devtools.build.lib.actions.ActionAnalysisMetadata; |
| import com.google.devtools.build.lib.actions.ActionExecutionContext; |
| import com.google.devtools.build.lib.actions.ActionExecutionException; |
| import com.google.devtools.build.lib.actions.ActionKeyContext; |
| import com.google.devtools.build.lib.actions.ActionLookupData; |
| import com.google.devtools.build.lib.actions.ActionLookupValue.ActionLookupKey; |
| import com.google.devtools.build.lib.actions.ActionResult; |
| import com.google.devtools.build.lib.actions.Actions; |
| import com.google.devtools.build.lib.actions.Artifact; |
| import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact; |
| import com.google.devtools.build.lib.actions.Artifact.SpecialArtifactType; |
| import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact; |
| import com.google.devtools.build.lib.actions.ArtifactRoot; |
| import com.google.devtools.build.lib.actions.BuildFailedException; |
| import com.google.devtools.build.lib.actions.FileArtifactValue; |
| import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue; |
| import com.google.devtools.build.lib.actions.MetadataProvider; |
| import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException; |
| import com.google.devtools.build.lib.actions.cache.MetadataHandler; |
| import com.google.devtools.build.lib.actions.util.ActionsTestUtil; |
| import com.google.devtools.build.lib.actions.util.TestAction; |
| import com.google.devtools.build.lib.actions.util.TestAction.DummyAction; |
| import com.google.devtools.build.lib.analysis.actions.SpawnActionTemplate; |
| import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; |
| import com.google.devtools.build.lib.collect.nestedset.Order; |
| import com.google.devtools.build.lib.events.Event; |
| import com.google.devtools.build.lib.events.EventCollector; |
| import com.google.devtools.build.lib.events.EventKind; |
| import com.google.devtools.build.lib.server.FailureDetails.Crash; |
| import com.google.devtools.build.lib.server.FailureDetails.Crash.Code; |
| import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; |
| import com.google.devtools.build.lib.skyframe.ActionTemplateExpansionValue.ActionTemplateExpansionKey; |
| import com.google.devtools.build.lib.skyframe.serialization.testutils.SerializationDepsUtils; |
| import com.google.devtools.build.lib.skyframe.serialization.testutils.SerializationTester; |
| import com.google.devtools.build.lib.testutil.TestUtils; |
| import com.google.devtools.build.lib.util.CrashFailureDetails; |
| import com.google.devtools.build.lib.util.DetailedExitCode; |
| import com.google.devtools.build.lib.vfs.FileStatus; |
| import com.google.devtools.build.lib.vfs.FileSystem; |
| import com.google.devtools.build.lib.vfs.FileSystemUtils; |
| import com.google.devtools.build.lib.vfs.Path; |
| import com.google.devtools.build.lib.vfs.PathFragment; |
| import com.google.devtools.build.lib.vfs.Root; |
| import com.google.devtools.build.lib.vfs.Symlinks; |
| import com.google.devtools.build.skyframe.SkyFunction; |
| import com.google.devtools.build.skyframe.SkyKey; |
| import com.google.devtools.build.skyframe.SkyValue; |
| import java.io.IOException; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.List; |
| import java.util.Set; |
| import org.junit.Test; |
| import org.junit.runner.RunWith; |
| import org.junit.runners.JUnit4; |
| |
| /** Timestamp builder tests for TreeArtifacts. */ |
| @RunWith(JUnit4.class) |
| public final class TreeArtifactBuildTest extends TimestampBuilderTestCase { |
| |
| @Test |
| public void codec() throws Exception { |
| SpecialArtifact parent = createTreeArtifact("parent"); |
| parent.setGeneratingActionKey(ActionLookupData.create(ACTION_LOOKUP_KEY, 0)); |
| new SerializationTester(parent, TreeFileArtifact.createTreeOutput(parent, "child")) |
| .addDependency(FileSystem.class, scratch.getFileSystem()) |
| .addDependency( |
| Root.RootCodecDependencies.class, |
| new Root.RootCodecDependencies(Root.absoluteRoot(scratch.getFileSystem()))) |
| .addDependencies(SerializationDepsUtils.SERIALIZATION_DEPS_FOR_TEST) |
| .runTests(); |
| } |
| |
| /** Simple smoke test. If this isn't passing, something is very wrong... */ |
| @Test |
| public void treeArtifactSimpleCase() throws Exception { |
| SpecialArtifact parent = createTreeArtifact("parent"); |
| TouchingTestAction action = new TouchingTestAction(parent, "out1", "out2"); |
| registerAction(action); |
| |
| TreeArtifactValue result = buildArtifact(parent); |
| |
| verifyOutputTree(result, parent, "out1", "out2"); |
| } |
| |
| /** Simple test for the case with dependencies. */ |
| @Test |
| public void dependentTreeArtifacts() throws Exception { |
| SpecialArtifact tree1 = createTreeArtifact("tree1"); |
| TouchingTestAction action1 = new TouchingTestAction(tree1, "out1", "out2"); |
| registerAction(action1); |
| |
| SpecialArtifact tree2 = createTreeArtifact("tree2"); |
| CopyTreeAction action2 = new CopyTreeAction(tree1, tree2); |
| registerAction(action2); |
| |
| TreeArtifactValue result = buildArtifact(tree2); |
| |
| assertThat(tree1.getPath().getRelative("out1").exists()).isTrue(); |
| assertThat(tree1.getPath().getRelative("out2").exists()).isTrue(); |
| verifyOutputTree(result, tree2, "out1", "out2"); |
| } |
| |
| /** Test for tree artifacts with sub directories. */ |
| @Test |
| public void treeArtifactWithSubDirectory() throws Exception { |
| SpecialArtifact parent = createTreeArtifact("parent"); |
| TouchingTestAction action = new TouchingTestAction(parent, "sub1/file1", "sub2/file2"); |
| registerAction(action); |
| |
| TreeArtifactValue result = buildArtifact(parent); |
| |
| verifyOutputTree(result, parent, "sub1/file1", "sub2/file2"); |
| } |
| |
| @Test |
| public void inputTreeArtifactMetadataProvider() throws Exception { |
| SpecialArtifact treeArtifactInput = createTreeArtifact("tree"); |
| TouchingTestAction action1 = new TouchingTestAction(treeArtifactInput, "out1", "out2"); |
| registerAction(action1); |
| |
| Artifact normalOutput = createDerivedArtifact("normal/out"); |
| Action testAction = |
| new SimpleTestAction(ImmutableList.of(treeArtifactInput), normalOutput) { |
| @Override |
| void run(ActionExecutionContext actionExecutionContext) throws IOException { |
| // Check the metadata provider for input TreeFileArtifacts. |
| MetadataProvider metadataProvider = actionExecutionContext.getMetadataProvider(); |
| assertThat( |
| metadataProvider |
| .getMetadata(TreeFileArtifact.createTreeOutput(treeArtifactInput, "out1")) |
| .getType() |
| .isFile()) |
| .isTrue(); |
| assertThat( |
| metadataProvider |
| .getMetadata(TreeFileArtifact.createTreeOutput(treeArtifactInput, "out2")) |
| .getType() |
| .isFile()) |
| .isTrue(); |
| |
| // Touch the action output. |
| touchFile(normalOutput); |
| } |
| }; |
| |
| registerAction(testAction); |
| buildArtifact(normalOutput); |
| } |
| |
| /** Unchanged TreeArtifact outputs should not cause reexecution. */ |
| @Test |
| public void cacheCheckingForTreeArtifactsDoesNotCauseReexecution() throws Exception { |
| SpecialArtifact out1 = createTreeArtifact("out1"); |
| Button button1 = new Button(); |
| |
| SpecialArtifact out2 = createTreeArtifact("out2"); |
| Button button2 = new Button(); |
| |
| TouchingTestAction action1 = new TouchingTestAction(button1, out1, "file_one", "file_two"); |
| registerAction(action1); |
| |
| CopyTreeAction action2 = new CopyTreeAction(button2, out1, out2); |
| registerAction(action2); |
| |
| button1.pressed = false; |
| button2.pressed = false; |
| buildArtifact(out2); |
| assertThat(button1.pressed).isTrue(); // built |
| assertThat(button2.pressed).isTrue(); // built |
| |
| button1.pressed = false; |
| button2.pressed = false; |
| buildArtifact(out2); |
| assertThat(button1.pressed).isFalse(); // not built |
| assertThat(button2.pressed).isFalse(); // not built |
| } |
| |
| /** Test rebuilding TreeArtifacts for inputs, outputs, and dependents. Also a test for caching. */ |
| @Test |
| public void transitiveReexecutionForTreeArtifacts() throws Exception { |
| Artifact in = createSourceArtifact("input"); |
| writeFile(in, "input content"); |
| |
| Button button1 = new Button(); |
| SpecialArtifact out1 = createTreeArtifact("output1"); |
| WriteInputToFilesAction action1 = |
| new WriteInputToFilesAction(button1, in, out1, "file1", "file2"); |
| registerAction(action1); |
| |
| Button button2 = new Button(); |
| SpecialArtifact out2 = createTreeArtifact("output2"); |
| CopyTreeAction action2 = new CopyTreeAction(button2, out1, out2); |
| registerAction(action2); |
| |
| button1.pressed = false; |
| button2.pressed = false; |
| buildArtifact(out2); |
| assertThat(button1.pressed).isTrue(); // built |
| assertThat(button2.pressed).isTrue(); // built |
| |
| button1.pressed = false; |
| button2.pressed = false; |
| writeFile(in, "modified input"); |
| buildArtifact(out2); |
| assertThat(button1.pressed).isTrue(); // built |
| assertThat(button2.pressed).isTrue(); // built |
| |
| button1.pressed = false; |
| button2.pressed = false; |
| writeFile(TreeFileArtifact.createTreeOutput(out1, "file1"), "modified output"); |
| buildArtifact(out2); |
| assertThat(button1.pressed).isTrue(); // built |
| assertThat(button2.pressed).isFalse(); // should have been cached |
| |
| button1.pressed = false; |
| button2.pressed = false; |
| writeFile(TreeFileArtifact.createTreeOutput(out2, "file1"), "more modified output"); |
| buildArtifact(out2); |
| assertThat(button1.pressed).isFalse(); // not built |
| assertThat(button2.pressed).isTrue(); // built |
| } |
| |
| /** Tests that changing a TreeArtifact directory should cause reexeuction. */ |
| @Test |
| public void directoryContentsCachingForTreeArtifacts() throws Exception { |
| Artifact in = createSourceArtifact("input"); |
| writeFile(in, "input content"); |
| |
| Button button1 = new Button(); |
| SpecialArtifact out1 = createTreeArtifact("output1"); |
| WriteInputToFilesAction action1 = |
| new WriteInputToFilesAction(button1, in, out1, "file1", "file2"); |
| registerAction(action1); |
| |
| Button button2 = new Button(); |
| SpecialArtifact out2 = createTreeArtifact("output2"); |
| CopyTreeAction action2 = new CopyTreeAction(button2, out1, out2); |
| registerAction(action2); |
| |
| button1.pressed = false; |
| button2.pressed = false; |
| buildArtifact(out2); |
| // just a smoke test--if these aren't built we have bigger problems! |
| assertThat(button1.pressed).isTrue(); |
| assertThat(button2.pressed).isTrue(); |
| |
| // Adding a file to a directory should cause reexecution. |
| button1.pressed = false; |
| button2.pressed = false; |
| Path spuriousOutputOne = out1.getPath().getRelative("spuriousOutput"); |
| touchFile(spuriousOutputOne); |
| buildArtifact(out2); |
| // Should re-execute, and delete spurious output |
| assertThat(spuriousOutputOne.exists()).isFalse(); |
| assertThat(button1.pressed).isTrue(); |
| assertThat(button2.pressed).isFalse(); // should have been cached |
| |
| button1.pressed = false; |
| button2.pressed = false; |
| Path spuriousOutputTwo = out2.getPath().getRelative("anotherSpuriousOutput"); |
| touchFile(spuriousOutputTwo); |
| buildArtifact(out2); |
| assertThat(spuriousOutputTwo.exists()).isFalse(); |
| assertThat(button1.pressed).isFalse(); |
| assertThat(button2.pressed).isTrue(); |
| |
| // Deleting should cause reexecution. |
| button1.pressed = false; |
| button2.pressed = false; |
| TreeFileArtifact out1File1 = TreeFileArtifact.createTreeOutput(out1, "file1"); |
| deleteFile(out1File1); |
| buildArtifact(out2); |
| assertThat(out1File1.getPath().exists()).isTrue(); |
| assertThat(button1.pressed).isTrue(); |
| assertThat(button2.pressed).isFalse(); // should have been cached |
| |
| button1.pressed = false; |
| button2.pressed = false; |
| TreeFileArtifact out2File1 = TreeFileArtifact.createTreeOutput(out2, "file1"); |
| deleteFile(out2File1); |
| buildArtifact(out2); |
| assertThat(out2File1.getPath().exists()).isTrue(); |
| assertThat(button1.pressed).isFalse(); |
| assertThat(button2.pressed).isTrue(); |
| } |
| |
| /** TreeArtifacts don't care about mtime, even when the file is empty. */ |
| @Test |
| public void mTimeForTreeArtifactsDoesNotMatter() throws Exception { |
| // For this test, we only touch the input file. |
| Artifact in = createSourceArtifact("touchable_input"); |
| touchFile(in); |
| |
| Button button1 = new Button(); |
| SpecialArtifact out1 = createTreeArtifact("output1"); |
| WriteInputToFilesAction action1 = |
| new WriteInputToFilesAction(button1, in, out1, "file1", "file2"); |
| registerAction(action1); |
| |
| Button button2 = new Button(); |
| SpecialArtifact out2 = createTreeArtifact("output2"); |
| CopyTreeAction action2 = new CopyTreeAction(button2, out1, out2); |
| registerAction(action2); |
| |
| button1.pressed = false; |
| button2.pressed = false; |
| buildArtifact(out2); |
| assertThat(button1.pressed).isTrue(); // built |
| assertThat(button2.pressed).isTrue(); // built |
| |
| button1.pressed = false; |
| button2.pressed = false; |
| touchFile(in); |
| buildArtifact(out2); |
| // mtime does not matter. |
| assertThat(button1.pressed).isFalse(); |
| assertThat(button2.pressed).isFalse(); |
| |
| // None of the below following should result in anything being built. |
| button1.pressed = false; |
| button2.pressed = false; |
| touchFile(TreeFileArtifact.createTreeOutput(out1, "file1")); |
| buildArtifact(out2); |
| // Nothing should be built. |
| assertThat(button1.pressed).isFalse(); |
| assertThat(button2.pressed).isFalse(); |
| |
| button1.pressed = false; |
| button2.pressed = false; |
| touchFile(TreeFileArtifact.createTreeOutput(out1, "file2")); |
| buildArtifact(out2); |
| // Nothing should be built. |
| assertThat(button1.pressed).isFalse(); |
| assertThat(button2.pressed).isFalse(); |
| } |
| |
| private static void checkDirectoryPermissions(Path path) throws IOException { |
| assertThat(path.isDirectory()).isTrue(); |
| assertThat(path.isExecutable()).isTrue(); |
| assertThat(path.isReadable()).isTrue(); |
| assertThat(path.isWritable()).isFalse(); |
| } |
| |
| private static void checkFilePermissions(Path path) throws IOException { |
| assertThat(path.isDirectory()).isFalse(); |
| assertThat(path.isExecutable()).isTrue(); |
| assertThat(path.isReadable()).isTrue(); |
| assertThat(path.isWritable()).isFalse(); |
| } |
| |
| @Test |
| public void outputsAreReadOnlyAndExecutable() throws Exception { |
| SpecialArtifact out = createTreeArtifact("output"); |
| |
| Action action = |
| new SimpleTestAction(out) { |
| @Override |
| void run(ActionExecutionContext context) throws IOException { |
| writeFile(out.getPath().getChild("one"), "one"); |
| writeFile(out.getPath().getChild("two"), "two"); |
| writeFile(out.getPath().getChild("three").getChild("four"), "three/four"); |
| } |
| }; |
| |
| registerAction(action); |
| buildArtifact(out); |
| |
| checkDirectoryPermissions(out.getPath()); |
| checkFilePermissions(out.getPath().getChild("one")); |
| checkFilePermissions(out.getPath().getChild("two")); |
| checkDirectoryPermissions(out.getPath().getChild("three")); |
| checkFilePermissions(out.getPath().getChild("three").getChild("four")); |
| } |
| |
| @Test |
| public void validRelativeSymlinkAccepted() throws Exception { |
| SpecialArtifact out = createTreeArtifact("output"); |
| |
| Action action = |
| new SimpleTestAction(out) { |
| @Override |
| void run(ActionExecutionContext actionExecutionContext) throws IOException { |
| writeFile(out.getPath().getChild("one"), "one"); |
| writeFile(out.getPath().getChild("two"), "two"); |
| FileSystemUtils.ensureSymbolicLink( |
| out.getPath().getChild("links").getChild("link"), "../one"); |
| } |
| }; |
| |
| registerAction(action); |
| buildArtifact(out); |
| } |
| |
| @Test |
| public void invalidSymlinkRejected() { |
| // Failure expected |
| EventCollector eventCollector = new EventCollector(EventKind.ERROR); |
| reporter.removeHandler(failFastHandler); |
| reporter.addHandler(eventCollector); |
| |
| SpecialArtifact out = createTreeArtifact("output"); |
| |
| Action action = |
| new SimpleTestAction(out) { |
| @Override |
| void run(ActionExecutionContext actionExecutionContext) throws IOException { |
| writeFile(out.getPath().getChild("one"), "one"); |
| writeFile(out.getPath().getChild("two"), "two"); |
| FileSystemUtils.ensureSymbolicLink( |
| out.getPath().getChild("links").getChild("link"), "../invalid"); |
| } |
| }; |
| |
| registerAction(action); |
| assertThrows(BuildFailedException.class, () -> buildArtifact(out)); |
| |
| List<Event> errors = ImmutableList.copyOf(eventCollector); |
| assertThat(errors).hasSize(2); |
| assertThat(errors.get(0).getMessage()).contains("Failed to resolve relative path links/link"); |
| assertThat(errors.get(1).getMessage()).contains("not all outputs were created or valid"); |
| } |
| |
| @Test |
| public void absoluteSymlinkBadTargetRejected() { |
| // Failure expected |
| EventCollector eventCollector = new EventCollector(EventKind.ERROR); |
| reporter.removeHandler(failFastHandler); |
| reporter.addHandler(eventCollector); |
| |
| SpecialArtifact out = createTreeArtifact("output"); |
| |
| Action action = |
| new SimpleTestAction(out) { |
| @Override |
| void run(ActionExecutionContext actionExecutionContext) throws IOException { |
| writeFile(out.getPath().getChild("one"), "one"); |
| writeFile(out.getPath().getChild("two"), "two"); |
| FileSystemUtils.ensureSymbolicLink( |
| out.getPath().getChild("links").getChild("link"), "/random/pointer"); |
| } |
| }; |
| |
| registerAction(action); |
| assertThrows(BuildFailedException.class, () -> buildArtifact(out)); |
| |
| List<Event> errors = ImmutableList.copyOf(eventCollector); |
| assertThat(errors).hasSize(2); |
| assertThat(errors.get(0).getMessage()).contains("Failed to resolve relative path links/link"); |
| assertThat(errors.get(1).getMessage()).contains("not all outputs were created or valid"); |
| } |
| |
| @Test |
| public void absoluteSymlinkAccepted() throws Exception { |
| scratch.overwriteFile("/random/pointer"); |
| |
| SpecialArtifact out = createTreeArtifact("output"); |
| |
| Action action = |
| new SimpleTestAction(out) { |
| @Override |
| void run(ActionExecutionContext actionExecutionContext) throws IOException { |
| writeFile(out.getPath().getChild("one"), "one"); |
| writeFile(out.getPath().getChild("two"), "two"); |
| FileSystemUtils.ensureSymbolicLink( |
| out.getPath().getChild("links").getChild("link"), "/random/pointer"); |
| } |
| }; |
| |
| registerAction(action); |
| buildArtifact(out); |
| } |
| |
| @Test |
| public void relativeSymlinkTraversingOutsideOfTreeArtifactRejected() { |
| // Failure expected |
| EventCollector eventCollector = new EventCollector(EventKind.ERROR); |
| reporter.removeHandler(failFastHandler); |
| reporter.addHandler(eventCollector); |
| |
| SpecialArtifact out = createTreeArtifact("output"); |
| |
| Action action = |
| new SimpleTestAction(out) { |
| @Override |
| void run(ActionExecutionContext actionExecutionContext) throws IOException { |
| writeFile(out.getPath().getChild("one"), "one"); |
| writeFile(out.getPath().getChild("two"), "two"); |
| FileSystemUtils.ensureSymbolicLink( |
| out.getPath().getChild("links").getChild("link"), "../../output/random/pointer"); |
| } |
| }; |
| |
| registerAction(action); |
| |
| assertThrows(BuildFailedException.class, () -> buildArtifact(out)); |
| List<Event> errors = ImmutableList.copyOf(eventCollector); |
| assertThat(errors).hasSize(2); |
| assertThat(errors.get(0).getMessage()) |
| .contains( |
| "A TreeArtifact may not contain relative symlinks whose target paths traverse " |
| + "outside of the TreeArtifact"); |
| assertThat(errors.get(1).getMessage()).contains("not all outputs were created or valid"); |
| } |
| |
| @Test |
| public void relativeSymlinkTraversingToDirOutsideOfTreeArtifactRejected() throws Exception { |
| // Failure expected |
| EventCollector eventCollector = new EventCollector(EventKind.ERROR); |
| reporter.removeHandler(failFastHandler); |
| reporter.addHandler(eventCollector); |
| |
| SpecialArtifact out = createTreeArtifact("output"); |
| |
| // Create a valid directory that can be referenced |
| scratch.dir(out.getRoot().getRoot().getRelative("some/dir").getPathString()); |
| |
| TestAction action = |
| new SimpleTestAction(out) { |
| @Override |
| void run(ActionExecutionContext actionExecutionContext) throws IOException { |
| writeFile(out.getPath().getChild("one"), "one"); |
| writeFile(out.getPath().getChild("two"), "two"); |
| FileSystemUtils.ensureSymbolicLink( |
| out.getPath().getChild("links").getChild("link"), "../../some/dir"); |
| } |
| }; |
| |
| registerAction(action); |
| |
| assertThrows(BuildFailedException.class, () -> buildArtifact(out)); |
| List<Event> errors = ImmutableList.copyOf(eventCollector); |
| assertThat(errors).hasSize(2); |
| assertThat(errors.get(0).getMessage()) |
| .contains( |
| "A TreeArtifact may not contain relative symlinks whose target paths traverse " |
| + "outside of the TreeArtifact"); |
| assertThat(errors.get(1).getMessage()).contains("not all outputs were created or valid"); |
| } |
| |
| @Test |
| public void constructMetadataForDigest() throws Exception { |
| SpecialArtifact out = createTreeArtifact("output"); |
| Action action = |
| new SimpleTestAction(out) { |
| @Override |
| void run(ActionExecutionContext actionExecutionContext) throws IOException { |
| TreeFileArtifact child1 = TreeFileArtifact.createTreeOutput(out, "one"); |
| TreeFileArtifact child2 = TreeFileArtifact.createTreeOutput(out, "two"); |
| writeFile(child1, "one"); |
| writeFile(child2, "two"); |
| |
| MetadataHandler md = actionExecutionContext.getMetadataHandler(); |
| FileStatus stat = child1.getPath().stat(Symlinks.NOFOLLOW); |
| FileArtifactValue metadata1 = |
| md.constructMetadataForDigest( |
| child1, stat, Hashing.sha256().hashString("one", UTF_8).asBytes()); |
| |
| stat = child2.getPath().stat(Symlinks.NOFOLLOW); |
| FileArtifactValue metadata2 = |
| md.constructMetadataForDigest( |
| child2, stat, Hashing.sha256().hashString("two", UTF_8).asBytes()); |
| |
| // The metadata will not be equal to reading from the filesystem since the filesystem |
| // won't have the digest. However, we should be able to detect that nothing could have |
| // been modified. |
| assertThat( |
| metadata1.couldBeModifiedSince( |
| FileArtifactValue.createForTesting(child1.getPath()))) |
| .isFalse(); |
| assertThat( |
| metadata2.couldBeModifiedSince( |
| FileArtifactValue.createForTesting(child2.getPath()))) |
| .isFalse(); |
| } |
| }; |
| |
| registerAction(action); |
| buildArtifact(out); |
| } |
| |
| @Test |
| public void remoteDirectoryInjection() throws Exception { |
| SpecialArtifact out = createTreeArtifact("output"); |
| RemoteFileArtifactValue remoteFile1 = |
| new RemoteFileArtifactValue( |
| Hashing.sha256().hashString("one", UTF_8).asBytes(), /*size=*/ 3, /*locationIndex=*/ 1); |
| RemoteFileArtifactValue remoteFile2 = |
| new RemoteFileArtifactValue( |
| Hashing.sha256().hashString("two", UTF_8).asBytes(), /*size=*/ 3, /*locationIndex=*/ 2); |
| |
| Action action = |
| new SimpleTestAction(out) { |
| @Override |
| void run(ActionExecutionContext actionExecutionContext) throws IOException { |
| TreeFileArtifact child1 = TreeFileArtifact.createTreeOutput(out, "one"); |
| TreeFileArtifact child2 = TreeFileArtifact.createTreeOutput(out, "two"); |
| writeFile(child1, "one"); |
| writeFile(child2, "two"); |
| |
| actionExecutionContext |
| .getMetadataHandler() |
| .injectDirectory(out, ImmutableMap.of(child1, remoteFile1, child2, remoteFile2)); |
| } |
| }; |
| |
| registerAction(action); |
| TreeArtifactValue result = buildArtifact(out); |
| |
| assertThat(result.getChildValues()) |
| .containsExactly( |
| TreeFileArtifact.createTreeOutput(out, "one"), |
| remoteFile1, |
| TreeFileArtifact.createTreeOutput(out, "two"), |
| remoteFile2); |
| } |
| |
| @Test |
| public void expandedActionsBuildInActionTemplate() throws Exception { |
| // artifact1 is a tree artifact generated by a TouchingTestAction. |
| SpecialArtifact artifact1 = createTreeArtifact("treeArtifact1"); |
| registerAction(new TouchingTestAction(artifact1, "file1", "file2")); |
| |
| // artifact2 is a tree artifact generated by an action template. |
| SpecialArtifact artifact2 = createTreeArtifact("treeArtifact2"); |
| SpawnActionTemplate actionTemplate = |
| ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2); |
| registerAction(actionTemplate); |
| |
| // We mock out the action template function to expand into two actions that just touch the |
| // output files. |
| ActionTemplateExpansionKey secondOwner = ActionTemplateExpansionValue.key(ACTION_LOOKUP_KEY, 1); |
| TreeFileArtifact expectedExpansionOutput1 = |
| TreeFileArtifact.createTemplateExpansionOutput(artifact2, "child1", secondOwner); |
| TreeFileArtifact expectedExpansionOutput2 = |
| TreeFileArtifact.createTemplateExpansionOutput(artifact2, "child2", secondOwner); |
| Action expandedAction1 = |
| new DummyAction( |
| TreeFileArtifact.createTreeOutput(artifact1, "file1"), expectedExpansionOutput1); |
| Action expandedAction2 = |
| new DummyAction( |
| TreeFileArtifact.createTreeOutput(artifact1, "file2"), expectedExpansionOutput2); |
| |
| actionTemplateExpansionFunction = |
| new DummyActionTemplateExpansionFunction( |
| actionKeyContext, ImmutableList.of(expandedAction1, expandedAction2)); |
| |
| TreeArtifactValue result = buildArtifact(artifact2); |
| |
| assertThat(result.getChildren()) |
| .containsExactly(expectedExpansionOutput1, expectedExpansionOutput2); |
| } |
| |
| @Test |
| public void expandedActionDoesNotGenerateOutputInActionTemplate() { |
| // expect errors |
| reporter.removeHandler(failFastHandler); |
| |
| // artifact1 is a tree artifact generated by a TouchingTestAction. |
| SpecialArtifact artifact1 = createTreeArtifact("treeArtifact1"); |
| registerAction(new TouchingTestAction(artifact1, "child1", "child2")); |
| |
| // artifact2 is a tree artifact generated by an action template. |
| SpecialArtifact artifact2 = createTreeArtifact("treeArtifact2"); |
| SpawnActionTemplate actionTemplate = |
| ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2); |
| registerAction(actionTemplate); |
| |
| // We mock out the action template function to expand into two actions: |
| // One Action that touches the output file. |
| // The other action that does not generate the output file. |
| ActionTemplateExpansionKey secondOwner = ActionTemplateExpansionKey.of(ACTION_LOOKUP_KEY, 1); |
| TreeFileArtifact expectedExpansionOutput1 = |
| TreeFileArtifact.createTemplateExpansionOutput(artifact2, "child1", secondOwner); |
| TreeFileArtifact expectedExpansionOutput2 = |
| TreeFileArtifact.createTemplateExpansionOutput(artifact2, "child2", secondOwner); |
| Action generateOutputAction = |
| new DummyAction( |
| TreeFileArtifact.createTreeOutput(artifact1, "child1"), expectedExpansionOutput1); |
| Action noGenerateOutputAction = |
| new NoOpDummyAction( |
| TreeFileArtifact.createTreeOutput(artifact1, "child2"), expectedExpansionOutput2); |
| |
| actionTemplateExpansionFunction = |
| new DummyActionTemplateExpansionFunction( |
| actionKeyContext, ImmutableList.of(generateOutputAction, noGenerateOutputAction)); |
| |
| BuildFailedException e = |
| assertThrows(BuildFailedException.class, () -> buildArtifact(artifact2)); |
| assertThat(e).hasMessageThat().contains("not all outputs were created or valid"); |
| } |
| |
| @Test |
| public void oneExpandedActionThrowsInActionTemplate() { |
| // expect errors |
| reporter.removeHandler(failFastHandler); |
| |
| // artifact1 is a tree artifact generated by a TouchingTestAction. |
| SpecialArtifact artifact1 = createTreeArtifact("treeArtifact1"); |
| registerAction(new TouchingTestAction(artifact1, "child1", "child2")); |
| |
| // artifact2 is a tree artifact generated by an action template. |
| SpecialArtifact artifact2 = createTreeArtifact("treeArtifact2"); |
| SpawnActionTemplate actionTemplate = |
| ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2); |
| registerAction(actionTemplate); |
| |
| // We mock out the action template function to expand into two actions: |
| // One Action that touches the output file. |
| // The other action that just throws when executed. |
| ActionTemplateExpansionKey secondOwner = ActionTemplateExpansionKey.of(ACTION_LOOKUP_KEY, 1); |
| TreeFileArtifact expectedExpansionOutput1 = |
| TreeFileArtifact.createTemplateExpansionOutput(artifact2, "child1", secondOwner); |
| TreeFileArtifact expectedExpansionOutput2 = |
| TreeFileArtifact.createTemplateExpansionOutput(artifact2, "child2", secondOwner); |
| Action generateOutputAction = |
| new DummyAction( |
| TreeFileArtifact.createTreeOutput(artifact1, "child1"), expectedExpansionOutput1); |
| Action throwingAction = |
| new ThrowingDummyAction( |
| TreeFileArtifact.createTreeOutput(artifact1, "child2"), expectedExpansionOutput2); |
| |
| actionTemplateExpansionFunction = |
| new DummyActionTemplateExpansionFunction( |
| actionKeyContext, ImmutableList.of(generateOutputAction, throwingAction)); |
| |
| BuildFailedException e = |
| assertThrows(BuildFailedException.class, () -> buildArtifact(artifact2)); |
| assertThat(e).hasMessageThat().contains("Throwing dummy action"); |
| } |
| |
| @Test |
| public void allExpandedActionsThrowInActionTemplate() { |
| // expect errors |
| reporter.removeHandler(failFastHandler); |
| |
| // artifact1 is a tree artifact generated by a TouchingTestAction. |
| SpecialArtifact artifact1 = createTreeArtifact("treeArtifact1"); |
| registerAction(new TouchingTestAction(artifact1, "child1", "child2")); |
| |
| // artifact2 is a tree artifact generated by an action template. |
| SpecialArtifact artifact2 = createTreeArtifact("treeArtifact2"); |
| SpawnActionTemplate actionTemplate = |
| ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2); |
| registerAction(actionTemplate); |
| |
| // We mock out the action template function to expand into two actions that throw when executed. |
| ActionTemplateExpansionKey secondOwner = ActionTemplateExpansionKey.of(ACTION_LOOKUP_KEY, 1); |
| TreeFileArtifact expectedExpansionOutput1 = |
| TreeFileArtifact.createTemplateExpansionOutput(artifact2, "child1", secondOwner); |
| TreeFileArtifact expectedExpansionOutput2 = |
| TreeFileArtifact.createTemplateExpansionOutput(artifact2, "child2", secondOwner); |
| Action throwingAction = |
| new ThrowingDummyAction( |
| TreeFileArtifact.createTreeOutput(artifact1, "child1"), expectedExpansionOutput1); |
| Action anotherThrowingAction = |
| new ThrowingDummyAction( |
| TreeFileArtifact.createTreeOutput(artifact1, "child2"), expectedExpansionOutput2); |
| |
| actionTemplateExpansionFunction = |
| new DummyActionTemplateExpansionFunction( |
| actionKeyContext, ImmutableList.of(throwingAction, anotherThrowingAction)); |
| |
| BuildFailedException e = |
| assertThrows(BuildFailedException.class, () -> buildArtifact(artifact2)); |
| assertThat(e).hasMessageThat().contains("Throwing dummy action"); |
| } |
| |
| @Test |
| public void inputTreeArtifactCreationFailedInActionTemplate() { |
| // expect errors |
| reporter.removeHandler(failFastHandler); |
| |
| // artifact1 is created by a action that throws. |
| SpecialArtifact artifact1 = createTreeArtifact("treeArtifact1"); |
| registerAction(new ThrowingDummyAction(artifact1)); |
| |
| // artifact2 is a tree artifact generated by an action template. |
| SpecialArtifact artifact2 = createTreeArtifact("treeArtifact2"); |
| SpawnActionTemplate actionTemplate = |
| ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2); |
| registerAction(actionTemplate); |
| |
| BuildFailedException e = |
| assertThrows(BuildFailedException.class, () -> buildArtifact(artifact2)); |
| assertThat(e).hasMessageThat().contains("Throwing dummy action"); |
| } |
| |
| @Test |
| public void emptyInputAndOutputTreeArtifactInActionTemplate() throws Exception { |
| // artifact1 is an empty tree artifact which is generated by a single no-op dummy action. |
| SpecialArtifact artifact1 = createTreeArtifact("treeArtifact1"); |
| registerAction(new NoOpDummyAction(artifact1)); |
| |
| // artifact2 is a tree artifact generated by an action template that takes artifact1 as input. |
| SpecialArtifact artifact2 = createTreeArtifact("treeArtifact2"); |
| SpawnActionTemplate actionTemplate = |
| ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2); |
| registerAction(actionTemplate); |
| |
| buildArtifact(artifact2); |
| |
| assertThat(artifact1.getPath().exists()).isTrue(); |
| assertThat(artifact1.getPath().getDirectoryEntries()).isEmpty(); |
| assertThat(artifact2.getPath().exists()).isTrue(); |
| assertThat(artifact2.getPath().getDirectoryEntries()).isEmpty(); |
| } |
| |
| private abstract static class SimpleTestAction extends TestAction { |
| private final Button button; |
| |
| SimpleTestAction(Artifact output) { |
| this(/*inputs=*/ ImmutableList.of(), output); |
| } |
| |
| SimpleTestAction(Iterable<Artifact> inputs, Artifact output) { |
| this(new Button(), inputs, output); |
| } |
| |
| SimpleTestAction(Button button, Iterable<Artifact> inputs, Artifact output) { |
| super(NO_EFFECT, NestedSetBuilder.wrap(Order.STABLE_ORDER, inputs), ImmutableSet.of(output)); |
| this.button = button; |
| } |
| |
| @Override |
| public final ActionResult execute(ActionExecutionContext context) |
| throws ActionExecutionException { |
| button.pressed = true; |
| try { |
| run(context); |
| } catch (IOException e) { |
| throw new ActionExecutionException( |
| e, this, /*catastrophe=*/ false, CrashFailureDetails.detailedExitCodeForThrowable(e)); |
| } |
| return ActionResult.EMPTY; |
| } |
| |
| abstract void run(ActionExecutionContext context) throws IOException; |
| } |
| |
| /** An action that touches some output TreeFileArtifacts. Takes no inputs. */ |
| private static final class TouchingTestAction extends SimpleTestAction { |
| private final ImmutableList<String> outputFiles; |
| |
| TouchingTestAction(SpecialArtifact output, String... outputFiles) { |
| this(new Button(), output, outputFiles); |
| } |
| |
| TouchingTestAction(Button button, SpecialArtifact output, String... outputFiles) { |
| super(button, /*inputs=*/ ImmutableList.of(), output); |
| this.outputFiles = ImmutableList.copyOf(outputFiles); |
| } |
| |
| @Override |
| void run(ActionExecutionContext context) throws IOException { |
| for (String file : outputFiles) { |
| touchFile(getPrimaryOutput().getPath().getRelative(file)); |
| } |
| } |
| } |
| |
| /** Takes an input file and populates several copies inside a TreeArtifact. */ |
| private static final class WriteInputToFilesAction extends SimpleTestAction { |
| private final ImmutableList<String> outputFiles; |
| |
| WriteInputToFilesAction( |
| Button button, Artifact input, SpecialArtifact output, String... outputFiles) { |
| super(button, ImmutableList.of(input), output); |
| this.outputFiles = ImmutableList.copyOf(outputFiles); |
| } |
| |
| @Override |
| void run(ActionExecutionContext actionExecutionContext) throws IOException { |
| for (String file : outputFiles) { |
| Path newOutput = getPrimaryOutput().getPath().getRelative(file); |
| newOutput.createDirectoryAndParents(); |
| FileSystemUtils.copyFile(getPrimaryInput().getPath(), newOutput); |
| } |
| } |
| } |
| |
| /** Copies the given TreeFileArtifact inputs to the given outputs, in respective order. */ |
| private static final class CopyTreeAction extends SimpleTestAction { |
| |
| CopyTreeAction(SpecialArtifact input, SpecialArtifact output) { |
| this(new Button(), input, output); |
| } |
| |
| CopyTreeAction(Button button, SpecialArtifact input, SpecialArtifact output) { |
| super(button, ImmutableList.of(input), output); |
| } |
| |
| @Override |
| void run(ActionExecutionContext context) throws IOException { |
| List<Artifact> children = new ArrayList<>(); |
| context.getArtifactExpander().expand(getPrimaryInput(), children); |
| for (Artifact child : children) { |
| Path newOutput = getPrimaryOutput().getPath().getRelative(child.getParentRelativePath()); |
| newOutput.createDirectoryAndParents(); |
| FileSystemUtils.copyFile(child.getPath(), newOutput); |
| } |
| } |
| } |
| |
| private SpecialArtifact createTreeArtifact(String name) { |
| FileSystem fs = scratch.getFileSystem(); |
| Path execRoot = |
| fs.getPath(TestUtils.tmpDir()).getRelative("execroot").getRelative("default-exec-root"); |
| PathFragment execPath = PathFragment.create("out").getRelative(name); |
| return new SpecialArtifact( |
| ArtifactRoot.asDerivedRoot(execRoot, "out"), |
| execPath, |
| ACTION_LOOKUP_KEY, |
| SpecialArtifactType.TREE); |
| } |
| |
| private TreeArtifactValue buildArtifact(SpecialArtifact treeArtifact) throws Exception { |
| Preconditions.checkArgument(treeArtifact.isTreeArtifact(), treeArtifact); |
| BuilderWithResult builder = cachingBuilder(); |
| buildArtifacts(builder, treeArtifact); |
| return (TreeArtifactValue) builder.getLatestResult().get(treeArtifact); |
| } |
| |
| private void buildArtifact(Artifact normalArtifact) throws Exception { |
| buildArtifacts(cachingBuilder(), normalArtifact); |
| } |
| |
| private static void verifyOutputTree( |
| TreeArtifactValue result, SpecialArtifact parent, String... expectedChildPaths) { |
| Preconditions.checkArgument(parent.isTreeArtifact(), parent); |
| Set<TreeFileArtifact> expectedChildren = |
| Arrays.stream(expectedChildPaths) |
| .map(path -> TreeFileArtifact.createTreeOutput(parent, path)) |
| .collect(toImmutableSet()); |
| for (TreeFileArtifact child : expectedChildren) { |
| assertWithMessage(child + " does not exist").that(child.getPath().exists()).isTrue(); |
| } |
| assertThat(result.getChildren()).isEqualTo(expectedChildren); |
| } |
| |
| private static void writeFile(Path path, String contents) throws IOException { |
| path.getParentDirectory().createDirectoryAndParents(); |
| // sometimes we write read-only files |
| if (path.exists()) { |
| path.setWritable(true); |
| } |
| FileSystemUtils.writeContentAsLatin1(path, contents); |
| } |
| |
| private static void writeFile(Artifact file, String contents) throws IOException { |
| writeFile(file.getPath(), contents); |
| } |
| |
| private static void touchFile(Path path) throws IOException { |
| path.getParentDirectory().createDirectoryAndParents(); |
| path.getParentDirectory().setWritable(true); |
| FileSystemUtils.touchFile(path); |
| } |
| |
| private static void touchFile(Artifact file) throws IOException { |
| touchFile(file.getPath()); |
| } |
| |
| private static void deleteFile(Artifact file) throws IOException { |
| Path path = file.getPath(); |
| // sometimes we write read-only files |
| path.setWritable(true); |
| // work around the sticky bit (this might depend on the behavior of the OS?) |
| path.getParentDirectory().setWritable(true); |
| path.delete(); |
| } |
| |
| /** A dummy action template expansion function that just returns the injected actions. */ |
| private static final class DummyActionTemplateExpansionFunction implements SkyFunction { |
| private final ActionKeyContext actionKeyContext; |
| private final ImmutableList<ActionAnalysisMetadata> actions; |
| |
| DummyActionTemplateExpansionFunction( |
| ActionKeyContext actionKeyContext, ImmutableList<ActionAnalysisMetadata> actions) { |
| this.actionKeyContext = actionKeyContext; |
| this.actions = actions; |
| } |
| |
| @Override |
| public SkyValue compute(SkyKey skyKey, Environment env) { |
| try { |
| return new ActionTemplateExpansionValue( |
| Actions.assignOwnersAndFilterSharedActionsAndThrowActionConflict( |
| actionKeyContext, actions, (ActionLookupKey) skyKey, /*outputFiles=*/ null)); |
| } catch (ActionConflictException e) { |
| throw new IllegalStateException(e); |
| } |
| } |
| |
| @Override |
| public String extractTag(SkyKey skyKey) { |
| return null; |
| } |
| } |
| |
| /** No-op action that does not generate the action outputs. */ |
| private static final class NoOpDummyAction extends SimpleTestAction { |
| |
| NoOpDummyAction(Artifact output) { |
| super(/*inputs=*/ ImmutableList.of(), output); |
| } |
| |
| NoOpDummyAction(Artifact input, Artifact output) { |
| super(ImmutableList.of(input), output); |
| } |
| |
| /** Does nothing. */ |
| @Override |
| void run(ActionExecutionContext actionExecutionContext) {} |
| } |
| |
| /** No-op action that throws when executed. */ |
| private static final class ThrowingDummyAction extends TestAction { |
| |
| ThrowingDummyAction(Artifact output) { |
| super(NO_EFFECT, NestedSetBuilder.emptySet(Order.STABLE_ORDER), ImmutableSet.of(output)); |
| } |
| |
| ThrowingDummyAction(Artifact input, Artifact output) { |
| super(NO_EFFECT, NestedSetBuilder.create(Order.STABLE_ORDER, input), ImmutableSet.of(output)); |
| } |
| |
| /** Unconditionally throws. */ |
| @Override |
| public ActionResult execute(ActionExecutionContext actionExecutionContext) |
| throws ActionExecutionException { |
| DetailedExitCode code = |
| DetailedExitCode.of( |
| FailureDetail.newBuilder() |
| .setCrash(Crash.newBuilder().setCode(Code.CRASH_UNKNOWN)) |
| .build()); |
| throw new ActionExecutionException( |
| "Throwing dummy action", this, /*catastrophe=*/ true, code); |
| } |
| } |
| } |
