Automated rollback of commit 3290e22356b59371274849ee51297635b9435285.
*** Reason for rollback ***
Rolling forward with fix: keep passing artifact owner into generated artifact, sacrificing a bit of type safety for actual safety. This effectively reverts a few files from the original CL.
Also fix memory regression: key map of output file artifacts by label, since we can reuse existing labels. Moreover, this allows us to have an empty map for many configured targets: a build of a large internal target had the following numbers of targets per # of output files (apologies for the only partially sorted map):
{0=63649, 1=67148, 2=89175, 3=49347, 4=12712, 5=3027, 261=1, 6=247, 7=9, 775=1, 8=10, 9=82, 10=6, 11=3, 12=54, 13=4, 14=2, 15=24, 655=1, 271=2, 144=1, 17=2, 18=24, 276=1, 21=9, 3990=2, 24=18, 27=11, 156=3, 30=6, 31=2, 33=12, 34=1, 163=1, 36=5, 37=1, 38=2, 294=1, 39=5, 41=1, 42=3, 43=2, 45=9, 302=1, 47=1, 48=4, 561=1, 306=2, 54=2, 183=1, 57=1, 573=2, 318=2, 63=2, 65=1, 66=1, 198=1, 71=1, 72=2, 330=2, 75=2, 204=1, 76=1, 77=2, 80=1, 82=1, 84=3, 87=2, 346=2, 90=4, 480=1, 99=5, 102=1, 231=2, 363=1, 624=2, 117=1, 120=2, 123=2, 252=2}
So those are the sizes of the maps that are actually needed. With the regression fixed, this change actually has a significant progression because we no longer create a duplicate derived artifact (with PathFragments, etc.) per OutputFileConfiguredTarget. Moreover, we don't need to keep a map with every artifact as keys inside ActionLookupValue and RuleConfiguredTarget. The only regression remaining is that we eagerly create ActionLookupData objects for every artifact, versus only on demand during execution. That's not a major issue, and it's ameliorated by no longer needing to intern ActionLookupData objects, which has memory and CPU benefits. Total memory diff for analysis of our large internal target looks very nice (net -125M out of 5.88G, or 2% gain):
objsize chg instances space KB class name
------------------------------------------------------
24 +0 1345958 31545 KB com.google.devtools.build.lib.actions.ActionLookupData
70 +0 -334 1992 KB [Ljava.lang.Object;
16 +0 -19073 -298 KB java.lang.Integer
40 +0 -12795 -499 KB com.google.common.collect.SingletonImmutableBiMap
40 +0 -45433 -1774 KB com.google.common.collect.MapMakerInternalMap$WeakKeyDummyValueEntry
32 -8 0 -1852 KB com.google.devtools.build.lib.analysis.ConfiguredAspect
24 +0 -79943 -1873 KB com.google.common.collect.SingletonImmutableSet
24 +0 -79943 -1873 KB com.google.common.collect.ImmutableEntry
40 +0 -61756 -2412 KB com.google.common.collect.RegularImmutableMap
16 +0 -216644 -3385 KB com.google.common.collect.RegularImmutableList
24 +0 -216650 -5077 KB com.google.common.collect.ImmutableMapEntrySet$RegularEntrySet
44 -4 -61755 -7376 KB [Ljava.util.Map$Entry;
44 -3 -61757 -7409 KB [Lcom.google.common.collect.ImmutableMapEntry;
32 +0 -259112 -8097 KB com.google.devtools.build.lib.actions.Artifact$DerivedArtifact
24 +0 -522911 -12255 KB com.google.devtools.build.lib.vfs.PathFragment
24 +0 -525254 -12310 KB java.lang.String
24 +0 -541285 -12686 KB com.google.common.collect.ImmutableMapEntry$NonTerminalImmutableMapEntry
24 +0 -999865 -23434 KB com.google.common.collect.ImmutableMapEntry
85 +0 -524326 -56276 KB [B
------------------------------------------------------
total change: -125514 KB
PiperOrigin-RevId: 251755552
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java
index f6f8a8b..4ba23c3 100644
--- a/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java
@@ -32,7 +32,6 @@
import com.google.devtools.build.lib.actions.Artifact.SpecialArtifactType;
import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
import com.google.devtools.build.lib.actions.ArtifactFileMetadata;
-import com.google.devtools.build.lib.actions.ArtifactOwner;
import com.google.devtools.build.lib.actions.ArtifactRoot;
import com.google.devtools.build.lib.actions.ArtifactSkyKey;
import com.google.devtools.build.lib.actions.BasicActionLookupValue;
@@ -192,10 +191,8 @@
@Test
public void testActionTreeArtifactOutput() throws Throwable {
SpecialArtifact artifact = createDerivedTreeArtifactWithAction("treeArtifact");
- TreeFileArtifact treeFileArtifact1 =
- createFakeTreeFileArtifact(artifact, ALL_OWNER, "child1", "hello1");
- TreeFileArtifact treeFileArtifact2 =
- createFakeTreeFileArtifact(artifact, ALL_OWNER, "child2", "hello2");
+ TreeFileArtifact treeFileArtifact1 = createFakeTreeFileArtifact(artifact, "child1", "hello1");
+ TreeFileArtifact treeFileArtifact2 = createFakeTreeFileArtifact(artifact, "child2", "hello2");
TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact);
assertThat(value.getChildValues()).containsKey(treeFileArtifact1);
@@ -216,15 +213,11 @@
TreeFileArtifact treeFileArtifact1 =
createFakeTreeFileArtifact(
artifact2,
- ActionTemplateExpansionValue.ActionTemplateExpansionKey.of(
- (ActionLookupValue.ActionLookupKey) artifact2.getArtifactOwner(), 1),
"child1",
"hello1");
TreeFileArtifact treeFileArtifact2 =
createFakeTreeFileArtifact(
artifact2,
- ActionTemplateExpansionValue.ActionTemplateExpansionKey.of(
- (ActionLookupValue.ActionLookupKey) artifact2.getArtifactOwner(), 1),
"child2",
"hello2");
@@ -257,15 +250,11 @@
TreeFileArtifact treeFileArtifact1 =
createFakeTreeFileArtifact(
artifact3,
- ActionTemplateExpansionValue.ActionTemplateExpansionKey.of(
- (ActionLookupValue.ActionLookupKey) artifact2.getArtifactOwner(), 2),
"child1",
"hello1");
TreeFileArtifact treeFileArtifact2 =
createFakeTreeFileArtifact(
artifact3,
- ActionTemplateExpansionValue.ActionTemplateExpansionKey.of(
- (ActionLookupValue.ActionLookupKey) artifact2.getArtifactOwner(), 2),
"child2",
"hello2");
actions.add(
@@ -280,17 +269,24 @@
@Test
public void actionExecutionValueSerialization() throws Exception {
- Artifact artifact1 = createDerivedArtifact("one");
- Artifact artifact2 = createDerivedArtifact("two");
+ ActionLookupData dummyData = ActionLookupData.create(ALL_OWNER, 0);
+ Artifact.DerivedArtifact artifact1 = createDerivedArtifact("one");
+ artifact1.setGeneratingActionKey(dummyData);
+ Artifact.DerivedArtifact artifact2 = createDerivedArtifact("two");
+ artifact2.setGeneratingActionKey(dummyData);
ArtifactFileMetadata metadata1 =
ActionMetadataHandler.fileMetadataFromArtifact(artifact1, null, null);
SpecialArtifact treeArtifact = createDerivedTreeArtifactOnly("tree");
- TreeFileArtifact treeFileArtifact =
- createFakeTreeFileArtifact(treeArtifact, "subpath", "content");
+ treeArtifact.setGeneratingActionKey(dummyData);
+ TreeFileArtifact treeFileArtifact = ActionInputHelper.treeFileArtifact(treeArtifact, "subpath");
+ Path path = treeFileArtifact.getPath();
+ FileSystemUtils.createDirectoryAndParents(path.getParentDirectory());
+ writeFile(path, "contents");
TreeArtifactValue treeArtifactValue =
TreeArtifactValue.create(
ImmutableMap.of(treeFileArtifact, FileArtifactValue.create(treeFileArtifact)));
- Artifact artifact3 = createDerivedArtifact("three");
+ Artifact.DerivedArtifact artifact3 = createDerivedArtifact("three");
+ artifact3.setGeneratingActionKey(dummyData);
FilesetOutputSymlink filesetOutputSymlink =
FilesetOutputSymlink.createForTesting(
PathFragment.EMPTY_FRAGMENT, PathFragment.EMPTY_FRAGMENT, PathFragment.EMPTY_FRAGMENT);
@@ -326,9 +322,9 @@
ArtifactRoot.asSourceRoot(Root.fromPath(root)), PathFragment.create(path));
}
- private Artifact createDerivedArtifact(String path) {
+ private Artifact.DerivedArtifact createDerivedArtifact(String path) {
PathFragment execPath = PathFragment.create("out").getRelative(path);
- Artifact output =
+ Artifact.DerivedArtifact output =
new Artifact.DerivedArtifact(
ArtifactRoot.asDerivedRoot(root, root.getRelative("out")), execPath, ALL_OWNER);
actions.add(new DummyAction(ImmutableList.<Artifact>of(), output));
@@ -358,24 +354,13 @@
}
private TreeFileArtifact createFakeTreeFileArtifact(
- SpecialArtifact treeArtifact, String parentRelativePath, String content) throws Exception {
- return createFakeTreeFileArtifact(
- treeArtifact,
- ActionTemplateExpansionValue.ActionTemplateExpansionKey.of(
- (ActionLookupValue.ActionLookupKey) treeArtifact.getArtifactOwner(), 0),
- parentRelativePath,
- content);
- }
-
- private TreeFileArtifact createFakeTreeFileArtifact(
SpecialArtifact treeArtifact,
- ArtifactOwner artifactOwner,
String parentRelativePath,
String content)
throws Exception {
TreeFileArtifact treeFileArtifact =
- ActionInputHelper.treeFileArtifact(
- treeArtifact, PathFragment.create(parentRelativePath), artifactOwner);
+ ActionsTestUtil.createTreeFileArtifactWithNoGeneratingAction(
+ treeArtifact, parentRelativePath);
Path path = treeFileArtifact.getPath();
FileSystemUtils.createDirectoryAndParents(path.getParentDirectory());
writeFile(path, content);
@@ -416,8 +401,11 @@
ImmutableMap.of(
ALL_OWNER,
new BasicActionLookupValue(
- Actions.filterSharedActionsAndThrowActionConflict(
- actionKeyContext, ImmutableList.copyOf(actions)),
+ Actions.assignOwnersAndFilterSharedActionsAndThrowActionConflict(
+ actionKeyContext,
+ ImmutableList.copyOf(actions),
+ ALL_OWNER,
+ /*outputFiles=*/ null),
/*nonceVersion=*/ null)));
}
}