Skyframe changes to support SpawnActionTemplate.
1. Adds ActionTemplateExpansion{Function, Value} for ActionTemplate expansion.
2. Changes ArtifactFunction to support evaluating TreeFileArtifacts and TreeArtifacts generated by ActionTemplate.
--
MOS_MIGRATED_REVID=124160939
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java
index d7433ac..789e2b1 100644
--- a/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java
@@ -30,9 +30,14 @@
import com.google.devtools.build.lib.actions.Action;
import com.google.devtools.build.lib.actions.ActionAnalysisMetadata;
import com.google.devtools.build.lib.actions.ActionAnalysisMetadata.MiddlemanType;
+import com.google.devtools.build.lib.actions.ActionInputHelper;
import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
+import com.google.devtools.build.lib.actions.Artifact.SpecialArtifactType;
+import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
import com.google.devtools.build.lib.actions.MissingInputFileException;
import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
import com.google.devtools.build.lib.actions.util.TestAction.DummyAction;
import com.google.devtools.build.lib.events.NullEventHandler;
import com.google.devtools.build.lib.util.Pair;
@@ -169,7 +174,7 @@
new DummyAction(
ImmutableList.of(input1, input2), output, MiddlemanType.AGGREGATING_MIDDLEMAN);
// Overwrite default generating action with this one.
- for (Iterator<Action> it = actions.iterator(); it.hasNext(); ) {
+ for (Iterator<ActionAnalysisMetadata> it = actions.iterator(); it.hasNext(); ) {
if (it.next().getOutputs().contains(output)) {
it.remove();
break;
@@ -324,6 +329,70 @@
.testEquals();
}
+ @Test
+ public void testActionTreeArtifactOutput() throws Throwable {
+ Artifact artifact = createDerivedTreeArtifactWithAction("treeArtifact");
+ TreeFileArtifact treeFileArtifact1 = createFakeTreeFileArtifact(artifact, "child1", "hello1");
+ TreeFileArtifact treeFileArtifact2 = createFakeTreeFileArtifact(artifact, "child2", "hello2");
+
+ TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact);
+ assertNotNull(value.getChildValue(treeFileArtifact1));
+ assertNotNull(value.getChildValue(treeFileArtifact2));
+ assertNotNull(value.getChildValue(treeFileArtifact1).getDigest());
+ assertNotNull(value.getChildValue(treeFileArtifact2).getDigest());
+ }
+
+ @Test
+ public void testSpawnActionTemplate() throws Throwable {
+ // artifact1 is a tree artifact generated by normal action.
+ Artifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1");
+ createFakeTreeFileArtifact(artifact1, "child1", "hello1");
+ createFakeTreeFileArtifact(artifact1, "child2", "hello2");
+
+
+ // artifact2 is a tree artifact generated by action template.
+ Artifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2");
+ TreeFileArtifact treeFileArtifact1 = createFakeTreeFileArtifact(artifact2, "child1", "hello1");
+ TreeFileArtifact treeFileArtifact2 = createFakeTreeFileArtifact(artifact2, "child2", "hello2");
+
+ actions.add(
+ ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2));
+
+ TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact2);
+ assertNotNull(value.getChildValue(treeFileArtifact1));
+ assertNotNull(value.getChildValue(treeFileArtifact2));
+ assertNotNull(value.getChildValue(treeFileArtifact1).getDigest());
+ assertNotNull(value.getChildValue(treeFileArtifact2).getDigest());
+ }
+
+ @Test
+ public void testConsecutiveSpawnActionTemplates() throws Throwable {
+ // artifact1 is a tree artifact generated by normal action.
+ Artifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1");
+ createFakeTreeFileArtifact(artifact1, "child1", "hello1");
+ createFakeTreeFileArtifact(artifact1, "child2", "hello2");
+
+ // artifact2 is a tree artifact generated by action template.
+ Artifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2");
+ createFakeTreeFileArtifact(artifact2, "child1", "hello1");
+ createFakeTreeFileArtifact(artifact2, "child2", "hello2");
+ actions.add(
+ ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2));
+
+ // artifact3 is a tree artifact generated by action template.
+ Artifact artifact3 = createDerivedTreeArtifactOnly("treeArtifact3");
+ TreeFileArtifact treeFileArtifact1 = createFakeTreeFileArtifact(artifact3, "child1", "hello1");
+ TreeFileArtifact treeFileArtifact2 = createFakeTreeFileArtifact(artifact3, "child2", "hello2");
+ actions.add(
+ ActionsTestUtil.createDummySpawnActionTemplate(artifact2, artifact3));
+
+ TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact3);
+ assertNotNull(value.getChildValue(treeFileArtifact1));
+ assertNotNull(value.getChildValue(treeFileArtifact2));
+ assertNotNull(value.getChildValue(treeFileArtifact1).getDigest());
+ assertNotNull(value.getChildValue(treeFileArtifact2).getDigest());
+ }
+
private void file(Path path, String contents) throws Exception {
FileSystemUtils.createDirectoryAndParents(path.getParentDirectory());
writeFile(path, contents);
@@ -343,6 +412,33 @@
return output;
}
+ private Artifact createDerivedTreeArtifactWithAction(String path) {
+ Artifact treeArtifact = createDerivedTreeArtifactOnly(path);
+ actions.add(new DummyAction(ImmutableList.<Artifact>of(), treeArtifact));
+ return treeArtifact;
+ }
+
+ private Artifact createDerivedTreeArtifactOnly(String path) {
+ PathFragment execPath = new PathFragment("out").getRelative(path);
+ Path fullPath = root.getRelative(execPath);
+ return new SpecialArtifact(
+ fullPath,
+ Root.asDerivedRoot(root, root.getRelative("out")),
+ execPath,
+ ALL_OWNER,
+ SpecialArtifactType.TREE);
+ }
+
+ private TreeFileArtifact createFakeTreeFileArtifact(Artifact treeArtifact,
+ String parentRelativePath, String content) throws Exception {
+ TreeFileArtifact treeFileArtifact = ActionInputHelper.treeFileArtifact(
+ treeArtifact, new PathFragment(parentRelativePath));
+ Path path = treeFileArtifact.getPath();
+ FileSystemUtils.createDirectoryAndParents(path.getParentDirectory());
+ writeFile(path, content);
+ return treeFileArtifact;
+ }
+
private void assertValueMatches(FileStatus file, byte[] digest, FileArtifactValue value)
throws IOException {
assertEquals(file.getSize(), value.getSize());
@@ -384,35 +480,46 @@
throws InterruptedException {
setGeneratingActions();
return driver.evaluate(
- Arrays.asList(keys), /*keepGoing=*/
- false,
+ Arrays.asList(keys),
+ /*keepGoing=*/false,
SkyframeExecutor.DEFAULT_THREAD_COUNT,
NullEventHandler.INSTANCE);
}
/** Value Builder for actions that just stats and stores the output file (which must exist). */
- private class SimpleActionExecutionFunction implements SkyFunction {
+ private static class SimpleActionExecutionFunction implements SkyFunction {
@Override
public SkyValue compute(SkyKey skyKey, Environment env) {
Map<Artifact, FileValue> artifactData = new HashMap<>();
+ Map<Artifact, TreeArtifactValue> treeArtifactData = new HashMap<>();
+ Map<Artifact, FileArtifactValue> additionalOutputData = new HashMap<>();
Action action = (Action) skyKey.argument();
Artifact output = Iterables.getOnlyElement(action.getOutputs());
- FileArtifactValue value;
- if (action.getActionType() == MiddlemanType.NORMAL) {
- try {
+
+ try {
+ if (output.isTreeArtifact()) {
+ TreeFileArtifact treeFileArtifact1 = ActionInputHelper.treeFileArtifact(
+ output, new PathFragment("child1"));
+ TreeFileArtifact treeFileArtifact2 = ActionInputHelper.treeFileArtifact(
+ output, new PathFragment("child2"));
+ TreeArtifactValue treeArtifactValue = TreeArtifactValue.create(ImmutableMap.of(
+ treeFileArtifact1, FileArtifactValue.create(treeFileArtifact1),
+ treeFileArtifact2, FileArtifactValue.create(treeFileArtifact2)));
+ treeArtifactData.put(output, treeArtifactValue);
+ } else if (action.getActionType() == MiddlemanType.NORMAL) {
FileValue fileValue = ActionMetadataHandler.fileValueFromArtifact(output, null, null);
artifactData.put(output, fileValue);
- value = FileArtifactValue.create(output, fileValue);
- } catch (IOException e) {
- throw new IllegalStateException(e);
+ additionalOutputData.put(output, FileArtifactValue.create(output, fileValue));
+ } else {
+ additionalOutputData.put(output, FileArtifactValue.DEFAULT_MIDDLEMAN);
}
- } else {
- value = FileArtifactValue.DEFAULT_MIDDLEMAN;
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
}
return new ActionExecutionValue(
artifactData,
- ImmutableMap.<Artifact, TreeArtifactValue>of(),
- ImmutableMap.of(output, value));
+ treeArtifactData,
+ additionalOutputData);
}
@Override
