blob: a727496e98899df0dba0daadf990aa72f4bcd5c3 [file] [log] [blame]
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001// Copyright 2015 The Bazel Authors. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package com.google.devtools.build.lib.skyframe;
Janak Ramakrishnana5578af2017-03-21 17:28:39 +000015
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000016import static com.google.common.truth.Truth.assertThat;
lberki812e6fe2019-07-25 07:50:55 -070017import static com.google.devtools.build.lib.actions.FileArtifactValue.createForTesting;
michajlo660d17f2020-03-27 09:01:57 -070018import static org.junit.Assert.assertThrows;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000019
Googler3b2967d2020-05-18 14:54:46 -070020import com.google.common.base.Preconditions;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000021import com.google.common.collect.ImmutableList;
22import com.google.common.collect.ImmutableMap;
23import com.google.common.collect.ImmutableSet;
24import com.google.common.collect.Iterables;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000025import com.google.devtools.build.lib.actions.Action;
janakr93e3eea2017-03-30 22:09:37 +000026import com.google.devtools.build.lib.actions.ActionLookupData;
27import com.google.devtools.build.lib.actions.ActionLookupValue;
Googler3b2967d2020-05-18 14:54:46 -070028import com.google.devtools.build.lib.actions.ActionTemplate;
janakr0175ce32018-02-26 15:54:57 -080029import com.google.devtools.build.lib.actions.Actions;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000030import com.google.devtools.build.lib.actions.Artifact;
Googler72d648a2020-05-21 11:56:40 -070031import com.google.devtools.build.lib.actions.Artifact.DerivedArtifact;
Rumou Duan73876202016-06-06 18:52:08 +000032import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
33import com.google.devtools.build.lib.actions.Artifact.SpecialArtifactType;
34import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
tomlu1cdcdf92018-01-16 11:07:51 -080035import com.google.devtools.build.lib.actions.ArtifactRoot;
Googlerf0b0c392021-01-27 17:56:52 -080036import com.google.devtools.build.lib.actions.ArtifactRoot.RootType;
cparsonse2d200f2018-03-06 16:15:11 -080037import com.google.devtools.build.lib.actions.BasicActionLookupValue;
shahan602cc852018-06-06 20:09:57 -070038import com.google.devtools.build.lib.actions.FileArtifactValue;
janakre82933c2019-01-02 14:41:50 -080039import com.google.devtools.build.lib.actions.FilesetOutputSymlink;
jhorvitz3daedc32020-07-22 18:33:55 -070040import com.google.devtools.build.lib.actions.MiddlemanType;
janakr0175ce32018-02-26 15:54:57 -080041import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException;
Rumou Duan73876202016-06-06 18:52:08 +000042import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000043import com.google.devtools.build.lib.actions.util.TestAction.DummyAction;
Googler3b2967d2020-05-18 14:54:46 -070044import com.google.devtools.build.lib.analysis.actions.SpawnActionTemplate;
ulfjack1e1a7752019-12-10 21:17:58 -080045import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
46import com.google.devtools.build.lib.collect.nestedset.Order;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000047import com.google.devtools.build.lib.events.NullEventHandler;
janakr8b2f14f2021-04-02 11:17:58 -070048import com.google.devtools.build.lib.skyframe.ArtifactFunction.SourceArtifactException;
janakr40f2f0e2020-06-08 17:03:06 -070049import com.google.devtools.build.lib.skyframe.serialization.testutils.SerializationDepsUtils;
janakre82933c2019-01-02 14:41:50 -080050import com.google.devtools.build.lib.skyframe.serialization.testutils.SerializationTester;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000051import com.google.devtools.build.lib.util.Pair;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000052import com.google.devtools.build.lib.vfs.FileStatus;
lberkif225af12019-08-01 04:21:58 -070053import com.google.devtools.build.lib.vfs.FileStatusWithDigestAdapter;
janakre82933c2019-01-02 14:41:50 -080054import com.google.devtools.build.lib.vfs.FileSystem;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000055import com.google.devtools.build.lib.vfs.Path;
56import com.google.devtools.build.lib.vfs.PathFragment;
tomluee6a6862018-01-17 14:36:26 -080057import com.google.devtools.build.lib.vfs.Root;
lberkif225af12019-08-01 04:21:58 -070058import com.google.devtools.build.lib.vfs.Symlinks;
Googler10028672018-10-25 12:14:34 -070059import com.google.devtools.build.skyframe.EvaluationContext;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000060import com.google.devtools.build.skyframe.EvaluationResult;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000061import com.google.devtools.build.skyframe.SkyFunction;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000062import com.google.devtools.build.skyframe.SkyKey;
63import com.google.devtools.build.skyframe.SkyValue;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000064import java.io.IOException;
buchgrd4d3d502018-08-02 06:47:19 -070065import java.util.ArrayList;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000066import java.util.Arrays;
67import java.util.HashMap;
Googler3b2967d2020-05-18 14:54:46 -070068import java.util.HashSet;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000069import java.util.Map;
Googler3b2967d2020-05-18 14:54:46 -070070import java.util.Set;
Janak Ramakrishnanad77f972016-07-29 20:58:42 +000071import org.junit.Before;
72import org.junit.Test;
73import org.junit.runner.RunWith;
74import org.junit.runners.JUnit4;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000075
76/**
77 * Tests for {@link ArtifactFunction}.
78 */
79// Doesn't actually need any particular Skyframe, but is only relevant to Skyframe full mode.
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +000080@RunWith(JUnit4.class)
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000081public class ArtifactFunctionTest extends ArtifactFunctionTestCase {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000082
Googler3b2967d2020-05-18 14:54:46 -070083 private final Set<Artifact> omittedOutputs = new HashSet<>();
84
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +000085 @Before
Googler3b2967d2020-05-18 14:54:46 -070086 public final void setUp() {
87 delegateActionExecutionFunction = new SimpleActionExecutionFunction(omittedOutputs);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000088 }
89
jhorvitz7f55cb72021-12-16 18:52:24 -080090 private void assertFileArtifactValueMatches() throws Exception {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000091 Artifact output = createDerivedArtifact("output");
92 Path path = output.getPath();
93 file(path, "contents");
jhorvitz7f55cb72021-12-16 18:52:24 -080094 assertValueMatches(path.stat(), path.getDigest(), evaluateFAN(output));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000095 }
96
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +000097 @Test
jhorvitz7f55cb72021-12-16 18:52:24 -080098 public void testBasicArtifact() throws Exception {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000099 fastDigest = false;
jhorvitz7f55cb72021-12-16 18:52:24 -0800100 assertFileArtifactValueMatches();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000101 }
102
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000103 @Test
jhorvitz7f55cb72021-12-16 18:52:24 -0800104 public void testBasicArtifactWithXattr() throws Exception {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000105 fastDigest = true;
jhorvitz7f55cb72021-12-16 18:52:24 -0800106 assertFileArtifactValueMatches();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000107 }
108
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000109 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000110 public void testMissingNonMandatoryArtifact() throws Throwable {
111 Artifact input = createSourceArtifact("input1");
lberki36df7ed2019-06-27 06:32:03 -0700112 assertThat(evaluateArtifactValue(input)).isNotNull();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000113 }
114
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000115 @Test
Michajlo Matijkiw528957e2016-01-19 21:17:45 +0000116 public void testUnreadableInputWithFsWithAvailableDigest() throws Throwable {
Googlere764d562019-10-16 10:56:33 -0700117 final byte[] expectedDigest = {1, 2, 3, 4};
Michajlo Matijkiw528957e2016-01-19 21:17:45 +0000118 setupRoot(
119 new CustomInMemoryFs() {
120 @Override
ajurkowski8883c612021-03-08 08:12:37 -0800121 public byte[] getDigest(PathFragment path) throws IOException {
ccalvarindd9f60e2018-07-23 18:16:18 -0700122 return path.getBaseName().equals("unreadable") ? expectedDigest : super.getDigest(path);
Michajlo Matijkiw528957e2016-01-19 21:17:45 +0000123 }
124 });
125
126 Artifact input = createSourceArtifact("unreadable");
127 Path inputPath = input.getPath();
128 file(inputPath, "dummynotused");
129 inputPath.chmod(0);
130
lberki36df7ed2019-06-27 06:32:03 -0700131 FileArtifactValue value = (FileArtifactValue) evaluateArtifactValue(input);
Michajlo Matijkiw528957e2016-01-19 21:17:45 +0000132
133 FileStatus stat = inputPath.stat();
134 assertThat(value.getSize()).isEqualTo(stat.getSize());
135 assertThat(value.getDigest()).isEqualTo(expectedDigest);
136 }
137
138 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000139 public void testMiddlemanArtifact() throws Throwable {
Janak Ramakrishnana5578af2017-03-21 17:28:39 +0000140 Artifact output = createMiddlemanArtifact("output");
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000141 Artifact input1 = createSourceArtifact("input1");
142 Artifact input2 = createDerivedArtifact("input2");
Benjamin Peterson63748e42018-06-03 22:11:16 -0700143 SpecialArtifact tree = createDerivedTreeArtifactWithAction("treeArtifact");
buchgrd4d3d502018-08-02 06:47:19 -0700144 TreeFileArtifact treeFile1 = createFakeTreeFileArtifact(tree, "child1", "hello1");
145 TreeFileArtifact treeFile2 = createFakeTreeFileArtifact(tree, "child2", "hello2");
146 file(treeFile1.getPath(), "src1");
147 file(treeFile2.getPath(), "src2");
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000148 Action action =
149 new DummyAction(
ulfjack1e1a7752019-12-10 21:17:58 -0800150 NestedSetBuilder.create(Order.STABLE_ORDER, input1, input2, tree),
151 output,
leba40f2e722021-06-30 02:46:15 -0700152 MiddlemanType.RUNFILES_MIDDLEMAN);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000153 actions.add(action);
154 file(input2.getPath(), "contents");
155 file(input1.getPath(), "source contents");
jhorvitz8470e432021-12-14 09:51:17 -0800156 evaluate(Iterables.toArray(Artifact.keys(ImmutableSet.of(input2, input1, tree)), SkyKey.class));
Janak Ramakrishnanad77f972016-07-29 20:58:42 +0000157 SkyValue value = evaluateArtifactValue(output);
buchgrd4d3d502018-08-02 06:47:19 -0700158 ArrayList<Pair<Artifact, ?>> inputs = new ArrayList<>();
leba40f2e722021-06-30 02:46:15 -0700159 inputs.addAll(((RunfilesArtifactValue) value).getFileArtifacts());
160 inputs.addAll(((RunfilesArtifactValue) value).getTreeArtifacts());
buchgrd4d3d502018-08-02 06:47:19 -0700161 assertThat(inputs)
Benjamin Peterson63748e42018-06-03 22:11:16 -0700162 .containsExactly(
lberki812e6fe2019-07-25 07:50:55 -0700163 Pair.of(input1, createForTesting(input1)),
164 Pair.of(input2, createForTesting(input2)),
buchgrd4d3d502018-08-02 06:47:19 -0700165 Pair.of(tree, ((TreeArtifactValue) evaluateArtifactValue(tree))));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000166 }
167
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000168 /**
janakr8b2f14f2021-04-02 11:17:58 -0700169 * Tests that ArtifactFunction rethrows a transitive {@link IOException} as an {@link
170 * SourceArtifactException}.
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000171 */
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000172 @Test
jmmvbfbd95f2020-08-31 13:12:38 -0700173 public void testIOException_endToEnd() throws Throwable {
janakr8b2f14f2021-04-02 11:17:58 -0700174 IOException exception = new IOException("beep");
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000175 setupRoot(
176 new CustomInMemoryFs() {
177 @Override
ajurkowski8883c612021-03-08 08:12:37 -0800178 public FileStatus statIfFound(PathFragment path, boolean followSymlinks)
179 throws IOException {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000180 if (path.getBaseName().equals("bad")) {
181 throw exception;
182 }
fellya205ed82018-09-10 11:52:34 -0700183 return super.statIfFound(path, followSymlinks);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000184 }
185 });
janakr8b2f14f2021-04-02 11:17:58 -0700186 Artifact sourceArtifact = createSourceArtifact("bad");
187 SourceArtifactException e =
188 assertThrows(SourceArtifactException.class, () -> evaluateArtifactValue(sourceArtifact));
189 assertThat(e)
190 .hasMessageThat()
191 .isEqualTo("error reading file '" + sourceArtifact.getExecPathString() + "': beep");
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000192 }
193
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000194 @Test
Rumou Duan73876202016-06-06 18:52:08 +0000195 public void testActionTreeArtifactOutput() throws Throwable {
cpeyserac09f0a2018-02-05 09:33:15 -0800196 SpecialArtifact artifact = createDerivedTreeArtifactWithAction("treeArtifact");
janakrefb3f152019-06-05 17:42:34 -0700197 TreeFileArtifact treeFileArtifact1 = createFakeTreeFileArtifact(artifact, "child1", "hello1");
198 TreeFileArtifact treeFileArtifact2 = createFakeTreeFileArtifact(artifact, "child2", "hello2");
Rumou Duan73876202016-06-06 18:52:08 +0000199
200 TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact);
janakr45b308a2018-06-08 12:51:58 -0700201 assertThat(value.getChildValues()).containsKey(treeFileArtifact1);
202 assertThat(value.getChildValues()).containsKey(treeFileArtifact2);
lberkiaea56b32017-05-30 12:35:33 +0200203 assertThat(value.getChildValues().get(treeFileArtifact1).getDigest()).isNotNull();
204 assertThat(value.getChildValues().get(treeFileArtifact2).getDigest()).isNotNull();
Rumou Duan73876202016-06-06 18:52:08 +0000205 }
206
207 @Test
208 public void testSpawnActionTemplate() throws Throwable {
209 // artifact1 is a tree artifact generated by normal action.
cpeyserac09f0a2018-02-05 09:33:15 -0800210 SpecialArtifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1");
Rumou Duan73876202016-06-06 18:52:08 +0000211 createFakeTreeFileArtifact(artifact1, "child1", "hello1");
212 createFakeTreeFileArtifact(artifact1, "child2", "hello2");
213
Rumou Duan73876202016-06-06 18:52:08 +0000214 // artifact2 is a tree artifact generated by action template.
cpeyserac09f0a2018-02-05 09:33:15 -0800215 SpecialArtifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2");
Googler3b2967d2020-05-18 14:54:46 -0700216 SpawnActionTemplate actionTemplate =
217 ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
218 actions.add(actionTemplate);
janakr45b308a2018-06-08 12:51:58 -0700219 TreeFileArtifact treeFileArtifact1 =
Googler3b2967d2020-05-18 14:54:46 -0700220 createFakeExpansionTreeFileArtifact(actionTemplate, "child1", "hello1");
janakr45b308a2018-06-08 12:51:58 -0700221 TreeFileArtifact treeFileArtifact2 =
Googler3b2967d2020-05-18 14:54:46 -0700222 createFakeExpansionTreeFileArtifact(actionTemplate, "child2", "hello2");
Rumou Duan73876202016-06-06 18:52:08 +0000223
224 TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact2);
janakr45b308a2018-06-08 12:51:58 -0700225 assertThat(value.getChildValues()).containsKey(treeFileArtifact1);
226 assertThat(value.getChildValues()).containsKey(treeFileArtifact2);
lberkiaea56b32017-05-30 12:35:33 +0200227 assertThat(value.getChildValues().get(treeFileArtifact1).getDigest()).isNotNull();
228 assertThat(value.getChildValues().get(treeFileArtifact2).getDigest()).isNotNull();
Rumou Duan73876202016-06-06 18:52:08 +0000229 }
230
231 @Test
232 public void testConsecutiveSpawnActionTemplates() throws Throwable {
233 // artifact1 is a tree artifact generated by normal action.
cpeyserac09f0a2018-02-05 09:33:15 -0800234 SpecialArtifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1");
Rumou Duan73876202016-06-06 18:52:08 +0000235 createFakeTreeFileArtifact(artifact1, "child1", "hello1");
236 createFakeTreeFileArtifact(artifact1, "child2", "hello2");
237
238 // artifact2 is a tree artifact generated by action template.
cpeyserac09f0a2018-02-05 09:33:15 -0800239 SpecialArtifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2");
Googler3b2967d2020-05-18 14:54:46 -0700240 SpawnActionTemplate template2 =
241 ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
242 actions.add(template2);
243 createFakeExpansionTreeFileArtifact(template2, "child1", "hello1");
244 createFakeExpansionTreeFileArtifact(template2, "child2", "hello2");
Rumou Duan73876202016-06-06 18:52:08 +0000245
246 // artifact3 is a tree artifact generated by action template.
cpeyserac09f0a2018-02-05 09:33:15 -0800247 SpecialArtifact artifact3 = createDerivedTreeArtifactOnly("treeArtifact3");
Googler3b2967d2020-05-18 14:54:46 -0700248 SpawnActionTemplate template3 =
249 ActionsTestUtil.createDummySpawnActionTemplate(artifact2, artifact3);
250 actions.add(template3);
janakr45b308a2018-06-08 12:51:58 -0700251 TreeFileArtifact treeFileArtifact1 =
Googler3b2967d2020-05-18 14:54:46 -0700252 createFakeExpansionTreeFileArtifact(template3, "child1", "hello1");
janakr45b308a2018-06-08 12:51:58 -0700253 TreeFileArtifact treeFileArtifact2 =
Googler3b2967d2020-05-18 14:54:46 -0700254 createFakeExpansionTreeFileArtifact(template3, "child2", "hello2");
Rumou Duan73876202016-06-06 18:52:08 +0000255
256 TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact3);
janakr45b308a2018-06-08 12:51:58 -0700257 assertThat(value.getChildValues()).containsKey(treeFileArtifact1);
258 assertThat(value.getChildValues()).containsKey(treeFileArtifact2);
lberkiaea56b32017-05-30 12:35:33 +0200259 assertThat(value.getChildValues().get(treeFileArtifact1).getDigest()).isNotNull();
260 assertThat(value.getChildValues().get(treeFileArtifact2).getDigest()).isNotNull();
Rumou Duan73876202016-06-06 18:52:08 +0000261 }
262
janakre82933c2019-01-02 14:41:50 -0800263 @Test
Googler3b2967d2020-05-18 14:54:46 -0700264 public void actionTemplateExpansionOutputsOmitted() throws Throwable {
265 // artifact1 is a tree artifact generated by normal action.
266 SpecialArtifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1");
267 createFakeTreeFileArtifact(artifact1, "child1", "hello1");
268 createFakeTreeFileArtifact(artifact1, "child2", "hello2");
269
270 // artifact2 is a tree artifact generated by action template.
271 SpecialArtifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2");
272 SpawnActionTemplate actionTemplate =
273 ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
274 actions.add(actionTemplate);
275 TreeFileArtifact treeFileArtifact1 =
276 createFakeExpansionTreeFileArtifact(actionTemplate, "child1", "hello1");
277 TreeFileArtifact treeFileArtifact2 =
278 createFakeExpansionTreeFileArtifact(actionTemplate, "child2", "hello2");
279
280 omittedOutputs.add(treeFileArtifact1);
281 omittedOutputs.add(treeFileArtifact2);
282
283 SkyValue value = evaluateArtifactValue(artifact2);
Googler61a9f572020-06-02 10:28:24 -0700284 assertThat(value).isEqualTo(TreeArtifactValue.OMITTED_TREE_MARKER);
Googler3b2967d2020-05-18 14:54:46 -0700285 }
286
287 @Test
288 public void cannotOmitSomeButNotAllActionTemplateExpansionOutputs() throws Throwable {
289 // artifact1 is a tree artifact generated by normal action.
290 SpecialArtifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1");
291 createFakeTreeFileArtifact(artifact1, "child1", "hello1");
292 createFakeTreeFileArtifact(artifact1, "child2", "hello2");
293
294 // artifact2 is a tree artifact generated by action template.
295 SpecialArtifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2");
296 SpawnActionTemplate actionTemplate =
297 ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
298 actions.add(actionTemplate);
299 TreeFileArtifact treeFileArtifact1 =
300 createFakeExpansionTreeFileArtifact(actionTemplate, "child1", "hello1");
301 TreeFileArtifact treeFileArtifact2 =
302 createFakeExpansionTreeFileArtifact(actionTemplate, "child2", "hello2");
303
304 omittedOutputs.add(treeFileArtifact1);
305
306 Exception e = assertThrows(RuntimeException.class, () -> evaluateArtifactValue(artifact2));
307 assertThat(e).hasCauseThat().isInstanceOf(IllegalStateException.class);
308 assertThat(e)
309 .hasCauseThat()
310 .hasMessageThat()
311 .matches(
312 "Action template expansion has some but not all outputs omitted, present outputs: .*"
313 + treeFileArtifact2.getParentRelativePath()
314 + ".*");
315 }
316
317 @Test
janakre82933c2019-01-02 14:41:50 -0800318 public void actionExecutionValueSerialization() throws Exception {
janakrefb3f152019-06-05 17:42:34 -0700319 ActionLookupData dummyData = ActionLookupData.create(ALL_OWNER, 0);
Googler72d648a2020-05-21 11:56:40 -0700320 DerivedArtifact artifact1 = createDerivedArtifact("one");
lberkif7eee1e2019-07-31 05:49:10 -0700321 FileArtifactValue metadata1 =
322 ActionMetadataHandler.fileArtifactValueFromArtifact(artifact1, null, null);
janakre82933c2019-01-02 14:41:50 -0800323 SpecialArtifact treeArtifact = createDerivedTreeArtifactOnly("tree");
janakrefb3f152019-06-05 17:42:34 -0700324 treeArtifact.setGeneratingActionKey(dummyData);
Googler1d8d1382020-05-18 12:10:49 -0700325 TreeFileArtifact treeFileArtifact = TreeFileArtifact.createTreeOutput(treeArtifact, "subpath");
janakrefb3f152019-06-05 17:42:34 -0700326 Path path = treeFileArtifact.getPath();
Googler1d8d1382020-05-18 12:10:49 -0700327 path.getParentDirectory().createDirectoryAndParents();
janakrefb3f152019-06-05 17:42:34 -0700328 writeFile(path, "contents");
jhorvitzed7ec3b2020-07-24 15:08:03 -0700329 TreeArtifactValue tree =
330 TreeArtifactValue.newBuilder(treeArtifact)
331 .putChild(treeFileArtifact, FileArtifactValue.createForTesting(treeFileArtifact))
332 .build();
Googler72d648a2020-05-21 11:56:40 -0700333 DerivedArtifact artifact3 = createDerivedArtifact("three");
janakre82933c2019-01-02 14:41:50 -0800334 FilesetOutputSymlink filesetOutputSymlink =
335 FilesetOutputSymlink.createForTesting(
336 PathFragment.EMPTY_FRAGMENT, PathFragment.EMPTY_FRAGMENT, PathFragment.EMPTY_FRAGMENT);
337 ActionExecutionValue actionExecutionValue =
jhorvitz7f55cb72021-12-16 18:52:24 -0800338 ActionExecutionValue.createForTesting(
lberkic35878a2019-08-01 02:28:54 -0700339 ImmutableMap.of(artifact1, metadata1, artifact3, FileArtifactValue.DEFAULT_MIDDLEMAN),
jhorvitzed7ec3b2020-07-24 15:08:03 -0700340 ImmutableMap.of(treeArtifact, tree),
jhorvitz7f55cb72021-12-16 18:52:24 -0800341 ImmutableList.of(filesetOutputSymlink));
Googlerce6ad292019-12-20 10:29:21 -0800342 new SerializationTester(actionExecutionValue)
janakre82933c2019-01-02 14:41:50 -0800343 .addDependency(FileSystem.class, root.getFileSystem())
nharmataf93c72c2020-03-13 14:22:10 -0700344 .addDependency(
345 Root.RootCodecDependencies.class,
346 new Root.RootCodecDependencies(Root.absoluteRoot(root.getFileSystem())))
janakr40f2f0e2020-06-08 17:03:06 -0700347 .addDependencies(SerializationDepsUtils.SERIALIZATION_DEPS_FOR_TEST)
janakre82933c2019-01-02 14:41:50 -0800348 .runTests();
349 }
350
ajurkowskie2982912020-04-09 10:32:08 -0700351 private static void file(Path path, String contents) throws Exception {
jhorvitzed7ec3b2020-07-24 15:08:03 -0700352 path.getParentDirectory().createDirectoryAndParents();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000353 writeFile(path, contents);
354 }
355
356 private Artifact createSourceArtifact(String path) {
janakraea05602019-05-22 15:41:29 -0700357 return ActionsTestUtil.createArtifactWithExecPath(
358 ArtifactRoot.asSourceRoot(Root.fromPath(root)), PathFragment.create(path));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000359 }
360
Googler72d648a2020-05-21 11:56:40 -0700361 private DerivedArtifact createDerivedArtifact(String path) {
nharmatab4060b62017-04-04 17:11:39 +0000362 PathFragment execPath = PathFragment.create("out").getRelative(path);
Googler72d648a2020-05-21 11:56:40 -0700363 DerivedArtifact output =
jhorvitzcab340f2021-09-27 09:16:23 -0700364 DerivedArtifact.create(
Googlerf0b0c392021-01-27 17:56:52 -0800365 ArtifactRoot.asDerivedRoot(root, RootType.Output, "out"), execPath, ALL_OWNER);
ulfjack1e1a7752019-12-10 21:17:58 -0800366 actions.add(new DummyAction(NestedSetBuilder.emptySet(Order.STABLE_ORDER), output));
janakr8541f6d2019-06-11 14:40:21 -0700367 output.setGeneratingActionKey(ActionLookupData.create(ALL_OWNER, actions.size() - 1));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000368 return output;
369 }
370
Janak Ramakrishnana5578af2017-03-21 17:28:39 +0000371 private Artifact createMiddlemanArtifact(String path) {
tomlu1cdcdf92018-01-16 11:07:51 -0800372 ArtifactRoot middlemanRoot =
Googlerf0b0c392021-01-27 17:56:52 -0800373 ArtifactRoot.asDerivedRoot(middlemanPath, RootType.Middleman, PathFragment.create("out"));
jhorvitzcab340f2021-09-27 09:16:23 -0700374 return DerivedArtifact.create(
janakr3290e222019-05-29 16:34:22 -0700375 middlemanRoot, middlemanRoot.getExecPath().getRelative(path), ALL_OWNER);
Janak Ramakrishnana5578af2017-03-21 17:28:39 +0000376 }
377
cpeyserac09f0a2018-02-05 09:33:15 -0800378 private SpecialArtifact createDerivedTreeArtifactWithAction(String path) {
379 SpecialArtifact treeArtifact = createDerivedTreeArtifactOnly(path);
ulfjack1e1a7752019-12-10 21:17:58 -0800380 actions.add(new DummyAction(NestedSetBuilder.emptySet(Order.STABLE_ORDER), treeArtifact));
Googler1d8d1382020-05-18 12:10:49 -0700381 treeArtifact.setGeneratingActionKey(ActionLookupData.create(ALL_OWNER, actions.size() - 1));
Rumou Duan73876202016-06-06 18:52:08 +0000382 return treeArtifact;
383 }
384
cpeyserac09f0a2018-02-05 09:33:15 -0800385 private SpecialArtifact createDerivedTreeArtifactOnly(String path) {
nharmatab4060b62017-04-04 17:11:39 +0000386 PathFragment execPath = PathFragment.create("out").getRelative(path);
jhorvitzcab340f2021-09-27 09:16:23 -0700387 return SpecialArtifact.create(
Googlerf0b0c392021-01-27 17:56:52 -0800388 ArtifactRoot.asDerivedRoot(root, RootType.Output, "out"),
Googler147c2872021-01-21 08:47:48 -0800389 execPath,
390 ALL_OWNER,
391 SpecialArtifactType.TREE);
Rumou Duan73876202016-06-06 18:52:08 +0000392 }
393
ajurkowskie2982912020-04-09 10:32:08 -0700394 private static TreeFileArtifact createFakeTreeFileArtifact(
395 SpecialArtifact treeArtifact, String parentRelativePath, String content) throws Exception {
janakr45b308a2018-06-08 12:51:58 -0700396 TreeFileArtifact treeFileArtifact =
Googler1d8d1382020-05-18 12:10:49 -0700397 TreeFileArtifact.createTreeOutput(treeArtifact, parentRelativePath);
Rumou Duan73876202016-06-06 18:52:08 +0000398 Path path = treeFileArtifact.getPath();
Googler1d8d1382020-05-18 12:10:49 -0700399 path.getParentDirectory().createDirectoryAndParents();
400 writeFile(path, content);
401 return treeFileArtifact;
402 }
403
Googler3b2967d2020-05-18 14:54:46 -0700404 private TreeFileArtifact createFakeExpansionTreeFileArtifact(
405 ActionTemplate<?> actionTemplate, String parentRelativePath, String content)
406 throws Exception {
407 int actionIndex = Iterables.indexOf(actions, actionTemplate::equals);
408 Preconditions.checkState(actionIndex >= 0, "%s not registered", actionTemplate);
Googler1d8d1382020-05-18 12:10:49 -0700409 TreeFileArtifact treeFileArtifact =
410 TreeFileArtifact.createTemplateExpansionOutput(
Googler3b2967d2020-05-18 14:54:46 -0700411 actionTemplate.getOutputTreeArtifact(),
Googler1d8d1382020-05-18 12:10:49 -0700412 parentRelativePath,
Googler3b2967d2020-05-18 14:54:46 -0700413 ActionTemplateExpansionValue.key(ALL_OWNER, actionIndex));
Googler1d8d1382020-05-18 12:10:49 -0700414 Path path = treeFileArtifact.getPath();
415 path.getParentDirectory().createDirectoryAndParents();
Rumou Duan73876202016-06-06 18:52:08 +0000416 writeFile(path, content);
417 return treeFileArtifact;
418 }
419
ajurkowskie2982912020-04-09 10:32:08 -0700420 private static void assertValueMatches(FileStatus file, byte[] digest, FileArtifactValue value)
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000421 throws IOException {
lberkiaea56b32017-05-30 12:35:33 +0200422 assertThat(value.getSize()).isEqualTo(file.getSize());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000423 if (digest == null) {
lberkiaea56b32017-05-30 12:35:33 +0200424 assertThat(value.getDigest()).isNull();
425 assertThat(value.getModifiedTime()).isEqualTo(file.getLastModifiedTime());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000426 } else {
lberkiaea56b32017-05-30 12:35:33 +0200427 assertThat(value.getDigest()).isEqualTo(digest);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000428 }
429 }
430
jhorvitz7f55cb72021-12-16 18:52:24 -0800431 private FileArtifactValue evaluateFAN(Artifact artifact) throws Exception {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000432 return ((FileArtifactValue) evaluateArtifactValue(artifact));
433 }
434
jhorvitz7f55cb72021-12-16 18:52:24 -0800435 private SkyValue evaluateArtifactValue(Artifact artifact) throws Exception {
lberki36df7ed2019-06-27 06:32:03 -0700436 SkyKey key = Artifact.key(artifact);
Janak Ramakrishnanad77f972016-07-29 20:58:42 +0000437 EvaluationResult<SkyValue> result = evaluate(ImmutableList.of(key).toArray(new SkyKey[0]));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000438 if (result.hasError()) {
439 throw result.getError().getException();
440 }
janakr8541f6d2019-06-11 14:40:21 -0700441 SkyValue value = result.get(key);
442 if (value instanceof ActionExecutionValue) {
Googleraed41602020-06-02 12:22:53 -0700443 return ((ActionExecutionValue) value).getExistingFileArtifactValue(artifact);
janakr8541f6d2019-06-11 14:40:21 -0700444 }
445 return value;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000446 }
447
janakr0175ce32018-02-26 15:54:57 -0800448 private void setGeneratingActions() throws InterruptedException, ActionConflictException {
janakr573807d2018-01-11 14:02:35 -0800449 if (evaluator.getExistingValue(ALL_OWNER) == null) {
janakr93e3eea2017-03-30 22:09:37 +0000450 differencer.inject(
451 ImmutableMap.of(
janakr573807d2018-01-11 14:02:35 -0800452 ALL_OWNER,
cparsonse2d200f2018-03-06 16:15:11 -0800453 new BasicActionLookupValue(
janakrefb3f152019-06-05 17:42:34 -0700454 Actions.assignOwnersAndFilterSharedActionsAndThrowActionConflict(
455 actionKeyContext,
456 ImmutableList.copyOf(actions),
457 ALL_OWNER,
Googlerce6ad292019-12-20 10:29:21 -0800458 /*outputFiles=*/ null))));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000459 }
460 }
461
462 private <E extends SkyValue> EvaluationResult<E> evaluate(SkyKey... keys)
janakr0175ce32018-02-26 15:54:57 -0800463 throws InterruptedException, ActionConflictException {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000464 setGeneratingActions();
Googler10028672018-10-25 12:14:34 -0700465 EvaluationContext evaluationContext =
466 EvaluationContext.newBuilder()
467 .setKeepGoing(false)
468 .setNumThreads(SkyframeExecutor.DEFAULT_THREAD_COUNT)
michajlo7a485be2020-07-30 11:08:46 -0700469 .setEventHandler(NullEventHandler.INSTANCE)
Googler10028672018-10-25 12:14:34 -0700470 .build();
471 return driver.evaluate(Arrays.asList(keys), evaluationContext);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000472 }
473
Googler3b2967d2020-05-18 14:54:46 -0700474 /**
475 * Value builder for actions that just stats and stores the output file (which must either be
476 * orphaned or exist).
477 */
478 private static final class SimpleActionExecutionFunction implements SkyFunction {
479 private final Set<Artifact> omittedOutputs;
480
481 SimpleActionExecutionFunction(Set<Artifact> omittedOutputs) {
482 this.omittedOutputs = omittedOutputs;
483 }
484
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000485 @Override
janakr93e3eea2017-03-30 22:09:37 +0000486 public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException {
lberkif7eee1e2019-07-31 05:49:10 -0700487 Map<Artifact, FileArtifactValue> artifactData = new HashMap<>();
Rumou Duan73876202016-06-06 18:52:08 +0000488 Map<Artifact, TreeArtifactValue> treeArtifactData = new HashMap<>();
janakr93e3eea2017-03-30 22:09:37 +0000489 ActionLookupData actionLookupData = (ActionLookupData) skyKey.argument();
490 ActionLookupValue actionLookupValue =
janakrbaf52ae2018-02-14 09:03:18 -0800491 (ActionLookupValue) env.getValue(actionLookupData.getActionLookupKey());
janakr93e3eea2017-03-30 22:09:37 +0000492 Action action = actionLookupValue.getAction(actionLookupData.getActionIndex());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000493 Artifact output = Iterables.getOnlyElement(action.getOutputs());
Rumou Duan73876202016-06-06 18:52:08 +0000494
495 try {
Googler3b2967d2020-05-18 14:54:46 -0700496 if (omittedOutputs.contains(output)) {
497 Preconditions.checkState(!output.isTreeArtifact(), "Cannot omit %s", output);
498 artifactData.put(output, FileArtifactValue.OMITTED_FILE_MARKER);
499 } else if (output.isTreeArtifact()) {
jhorvitzed7ec3b2020-07-24 15:08:03 -0700500 SpecialArtifact parent = (SpecialArtifact) output;
Googler1d8d1382020-05-18 12:10:49 -0700501 TreeFileArtifact treeFileArtifact1 =
502 TreeFileArtifact.createTreeOutput((SpecialArtifact) output, "child1");
503 TreeFileArtifact treeFileArtifact2 =
504 TreeFileArtifact.createTreeOutput((SpecialArtifact) output, "child2");
jhorvitzed7ec3b2020-07-24 15:08:03 -0700505 TreeArtifactValue tree =
506 TreeArtifactValue.newBuilder(parent)
507 .putChild(
508 treeFileArtifact1, FileArtifactValue.createForTesting(treeFileArtifact1))
509 .putChild(
510 treeFileArtifact2, FileArtifactValue.createForTesting(treeFileArtifact2))
511 .build();
512 treeArtifactData.put(output, tree);
Rumou Duan73876202016-06-06 18:52:08 +0000513 } else if (action.getActionType() == MiddlemanType.NORMAL) {
lberkif225af12019-08-01 04:21:58 -0700514 Path path = output.getPath();
515 FileArtifactValue noDigest =
516 ActionMetadataHandler.fileArtifactValueFromArtifact(
517 output,
518 FileStatusWithDigestAdapter.adapt(path.statIfFound(Symlinks.NOFOLLOW)),
519 null);
520 FileArtifactValue withDigest =
jhorvitz7f55cb72021-12-16 18:52:24 -0800521 FileArtifactValue.createFromInjectedDigest(noDigest, path.getDigest());
lberkif225af12019-08-01 04:21:58 -0700522 artifactData.put(output, withDigest);
Rumou Duan73876202016-06-06 18:52:08 +0000523 } else {
lberkic35878a2019-08-01 02:28:54 -0700524 artifactData.put(output, FileArtifactValue.DEFAULT_MIDDLEMAN);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000525 }
Rumou Duan73876202016-06-06 18:52:08 +0000526 } catch (IOException e) {
527 throw new IllegalStateException(e);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000528 }
jhorvitz7f55cb72021-12-16 18:52:24 -0800529 return ActionExecutionValue.createForTesting(
jhorvitz8470e432021-12-14 09:51:17 -0800530 ImmutableMap.copyOf(artifactData),
531 ImmutableMap.copyOf(treeArtifactData),
jhorvitz7f55cb72021-12-16 18:52:24 -0800532 /*outputSymlinks=*/ null);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000533 }
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000534 }
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000535}