blob: a483eb7105b07b6ea8ed6e634a615d2f92fdb7c8 [file] [log] [blame]
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001// Copyright 2015 The Bazel Authors. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package com.google.devtools.build.lib.skyframe;
Janak Ramakrishnana5578af2017-03-21 17:28:39 +000015
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000016import static com.google.common.truth.Truth.assertThat;
lberki812e6fe2019-07-25 07:50:55 -070017import static com.google.devtools.build.lib.actions.FileArtifactValue.createForTesting;
michajlo660d17f2020-03-27 09:01:57 -070018import static org.junit.Assert.assertThrows;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000019
Googler3b2967d2020-05-18 14:54:46 -070020import com.google.common.base.Preconditions;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000021import com.google.common.collect.ImmutableList;
22import com.google.common.collect.ImmutableMap;
23import com.google.common.collect.ImmutableSet;
24import com.google.common.collect.Iterables;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000025import com.google.devtools.build.lib.actions.Action;
janakr93e3eea2017-03-30 22:09:37 +000026import com.google.devtools.build.lib.actions.ActionLookupData;
27import com.google.devtools.build.lib.actions.ActionLookupValue;
Googler3b2967d2020-05-18 14:54:46 -070028import com.google.devtools.build.lib.actions.ActionTemplate;
janakr0175ce32018-02-26 15:54:57 -080029import com.google.devtools.build.lib.actions.Actions;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000030import com.google.devtools.build.lib.actions.Artifact;
Googler72d648a2020-05-21 11:56:40 -070031import com.google.devtools.build.lib.actions.Artifact.DerivedArtifact;
Rumou Duan73876202016-06-06 18:52:08 +000032import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
33import com.google.devtools.build.lib.actions.Artifact.SpecialArtifactType;
34import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
tomlu1cdcdf92018-01-16 11:07:51 -080035import com.google.devtools.build.lib.actions.ArtifactRoot;
Googlerf0b0c392021-01-27 17:56:52 -080036import com.google.devtools.build.lib.actions.ArtifactRoot.RootType;
cparsonse2d200f2018-03-06 16:15:11 -080037import com.google.devtools.build.lib.actions.BasicActionLookupValue;
shahan602cc852018-06-06 20:09:57 -070038import com.google.devtools.build.lib.actions.FileArtifactValue;
janakre82933c2019-01-02 14:41:50 -080039import com.google.devtools.build.lib.actions.FilesetOutputSymlink;
jhorvitz3daedc32020-07-22 18:33:55 -070040import com.google.devtools.build.lib.actions.MiddlemanType;
janakr0175ce32018-02-26 15:54:57 -080041import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException;
Rumou Duan73876202016-06-06 18:52:08 +000042import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000043import com.google.devtools.build.lib.actions.util.TestAction.DummyAction;
Googler3b2967d2020-05-18 14:54:46 -070044import com.google.devtools.build.lib.analysis.actions.SpawnActionTemplate;
ulfjack1e1a7752019-12-10 21:17:58 -080045import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
46import com.google.devtools.build.lib.collect.nestedset.Order;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000047import com.google.devtools.build.lib.events.NullEventHandler;
janakr8b2f14f2021-04-02 11:17:58 -070048import com.google.devtools.build.lib.skyframe.ArtifactFunction.SourceArtifactException;
janakr40f2f0e2020-06-08 17:03:06 -070049import com.google.devtools.build.lib.skyframe.serialization.testutils.SerializationDepsUtils;
janakre82933c2019-01-02 14:41:50 -080050import com.google.devtools.build.lib.skyframe.serialization.testutils.SerializationTester;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000051import com.google.devtools.build.lib.util.Pair;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000052import com.google.devtools.build.lib.vfs.FileStatus;
lberkif225af12019-08-01 04:21:58 -070053import com.google.devtools.build.lib.vfs.FileStatusWithDigestAdapter;
janakre82933c2019-01-02 14:41:50 -080054import com.google.devtools.build.lib.vfs.FileSystem;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000055import com.google.devtools.build.lib.vfs.Path;
56import com.google.devtools.build.lib.vfs.PathFragment;
tomluee6a6862018-01-17 14:36:26 -080057import com.google.devtools.build.lib.vfs.Root;
lberkif225af12019-08-01 04:21:58 -070058import com.google.devtools.build.lib.vfs.Symlinks;
Googler10028672018-10-25 12:14:34 -070059import com.google.devtools.build.skyframe.EvaluationContext;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000060import com.google.devtools.build.skyframe.EvaluationResult;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000061import com.google.devtools.build.skyframe.SkyFunction;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000062import com.google.devtools.build.skyframe.SkyKey;
63import com.google.devtools.build.skyframe.SkyValue;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000064import java.io.IOException;
buchgrd4d3d502018-08-02 06:47:19 -070065import java.util.ArrayList;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000066import java.util.Arrays;
67import java.util.HashMap;
Googler3b2967d2020-05-18 14:54:46 -070068import java.util.HashSet;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000069import java.util.Map;
Googler3b2967d2020-05-18 14:54:46 -070070import java.util.Set;
Janak Ramakrishnanad77f972016-07-29 20:58:42 +000071import org.junit.Before;
72import org.junit.Test;
73import org.junit.runner.RunWith;
74import org.junit.runners.JUnit4;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000075
76/**
77 * Tests for {@link ArtifactFunction}.
78 */
79// Doesn't actually need any particular Skyframe, but is only relevant to Skyframe full mode.
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +000080@RunWith(JUnit4.class)
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000081public class ArtifactFunctionTest extends ArtifactFunctionTestCase {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000082
Googler3b2967d2020-05-18 14:54:46 -070083 private final Set<Artifact> omittedOutputs = new HashSet<>();
84
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +000085 @Before
Googler3b2967d2020-05-18 14:54:46 -070086 public final void setUp() {
87 delegateActionExecutionFunction = new SimpleActionExecutionFunction(omittedOutputs);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000088 }
89
90 private void assertFileArtifactValueMatches(boolean expectDigest) throws Throwable {
91 Artifact output = createDerivedArtifact("output");
92 Path path = output.getPath();
93 file(path, "contents");
olaolabfd1d332017-06-19 16:55:24 -040094 assertValueMatches(path.stat(), expectDigest ? path.getDigest() : null, evaluateFAN(output));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000095 }
96
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +000097 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000098 public void testBasicArtifact() throws Throwable {
99 fastDigest = false;
100 assertFileArtifactValueMatches(/*expectDigest=*/ true);
101 }
102
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000103 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000104 public void testBasicArtifactWithXattr() throws Throwable {
105 fastDigest = true;
106 assertFileArtifactValueMatches(/*expectDigest=*/ true);
107 }
108
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000109 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000110 public void testMissingNonMandatoryArtifact() throws Throwable {
111 Artifact input = createSourceArtifact("input1");
lberki36df7ed2019-06-27 06:32:03 -0700112 assertThat(evaluateArtifactValue(input)).isNotNull();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000113 }
114
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000115 @Test
Michajlo Matijkiw528957e2016-01-19 21:17:45 +0000116 public void testUnreadableInputWithFsWithAvailableDigest() throws Throwable {
Googlere764d562019-10-16 10:56:33 -0700117 final byte[] expectedDigest = {1, 2, 3, 4};
Michajlo Matijkiw528957e2016-01-19 21:17:45 +0000118 setupRoot(
119 new CustomInMemoryFs() {
120 @Override
ajurkowski8883c612021-03-08 08:12:37 -0800121 public byte[] getDigest(PathFragment path) throws IOException {
ccalvarindd9f60e2018-07-23 18:16:18 -0700122 return path.getBaseName().equals("unreadable") ? expectedDigest : super.getDigest(path);
Michajlo Matijkiw528957e2016-01-19 21:17:45 +0000123 }
124 });
125
126 Artifact input = createSourceArtifact("unreadable");
127 Path inputPath = input.getPath();
128 file(inputPath, "dummynotused");
129 inputPath.chmod(0);
130
lberki36df7ed2019-06-27 06:32:03 -0700131 FileArtifactValue value = (FileArtifactValue) evaluateArtifactValue(input);
Michajlo Matijkiw528957e2016-01-19 21:17:45 +0000132
133 FileStatus stat = inputPath.stat();
134 assertThat(value.getSize()).isEqualTo(stat.getSize());
135 assertThat(value.getDigest()).isEqualTo(expectedDigest);
136 }
137
138 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000139 public void testMiddlemanArtifact() throws Throwable {
Janak Ramakrishnana5578af2017-03-21 17:28:39 +0000140 Artifact output = createMiddlemanArtifact("output");
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000141 Artifact input1 = createSourceArtifact("input1");
142 Artifact input2 = createDerivedArtifact("input2");
Benjamin Peterson63748e42018-06-03 22:11:16 -0700143 SpecialArtifact tree = createDerivedTreeArtifactWithAction("treeArtifact");
buchgrd4d3d502018-08-02 06:47:19 -0700144 TreeFileArtifact treeFile1 = createFakeTreeFileArtifact(tree, "child1", "hello1");
145 TreeFileArtifact treeFile2 = createFakeTreeFileArtifact(tree, "child2", "hello2");
146 file(treeFile1.getPath(), "src1");
147 file(treeFile2.getPath(), "src2");
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000148 Action action =
149 new DummyAction(
ulfjack1e1a7752019-12-10 21:17:58 -0800150 NestedSetBuilder.create(Order.STABLE_ORDER, input1, input2, tree),
151 output,
leba40f2e722021-06-30 02:46:15 -0700152 MiddlemanType.RUNFILES_MIDDLEMAN);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000153 actions.add(action);
154 file(input2.getPath(), "contents");
155 file(input1.getPath(), "source contents");
janakr8541f6d2019-06-11 14:40:21 -0700156 evaluate(
157 Iterables.toArray(
lberki36df7ed2019-06-27 06:32:03 -0700158 Artifact.keys(ImmutableSet.of(input2, input1, input2, tree)), SkyKey.class));
Janak Ramakrishnanad77f972016-07-29 20:58:42 +0000159 SkyValue value = evaluateArtifactValue(output);
buchgrd4d3d502018-08-02 06:47:19 -0700160 ArrayList<Pair<Artifact, ?>> inputs = new ArrayList<>();
leba40f2e722021-06-30 02:46:15 -0700161 inputs.addAll(((RunfilesArtifactValue) value).getFileArtifacts());
162 inputs.addAll(((RunfilesArtifactValue) value).getTreeArtifacts());
buchgrd4d3d502018-08-02 06:47:19 -0700163 assertThat(inputs)
Benjamin Peterson63748e42018-06-03 22:11:16 -0700164 .containsExactly(
lberki812e6fe2019-07-25 07:50:55 -0700165 Pair.of(input1, createForTesting(input1)),
166 Pair.of(input2, createForTesting(input2)),
buchgrd4d3d502018-08-02 06:47:19 -0700167 Pair.of(tree, ((TreeArtifactValue) evaluateArtifactValue(tree))));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000168 }
169
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000170 /**
janakr8b2f14f2021-04-02 11:17:58 -0700171 * Tests that ArtifactFunction rethrows a transitive {@link IOException} as an {@link
172 * SourceArtifactException}.
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000173 */
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000174 @Test
jmmvbfbd95f2020-08-31 13:12:38 -0700175 public void testIOException_endToEnd() throws Throwable {
janakr8b2f14f2021-04-02 11:17:58 -0700176 IOException exception = new IOException("beep");
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000177 setupRoot(
178 new CustomInMemoryFs() {
179 @Override
ajurkowski8883c612021-03-08 08:12:37 -0800180 public FileStatus statIfFound(PathFragment path, boolean followSymlinks)
181 throws IOException {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000182 if (path.getBaseName().equals("bad")) {
183 throw exception;
184 }
fellya205ed82018-09-10 11:52:34 -0700185 return super.statIfFound(path, followSymlinks);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000186 }
187 });
janakr8b2f14f2021-04-02 11:17:58 -0700188 Artifact sourceArtifact = createSourceArtifact("bad");
189 SourceArtifactException e =
190 assertThrows(SourceArtifactException.class, () -> evaluateArtifactValue(sourceArtifact));
191 assertThat(e)
192 .hasMessageThat()
193 .isEqualTo("error reading file '" + sourceArtifact.getExecPathString() + "': beep");
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000194 }
195
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000196 @Test
Rumou Duan73876202016-06-06 18:52:08 +0000197 public void testActionTreeArtifactOutput() throws Throwable {
cpeyserac09f0a2018-02-05 09:33:15 -0800198 SpecialArtifact artifact = createDerivedTreeArtifactWithAction("treeArtifact");
janakrefb3f152019-06-05 17:42:34 -0700199 TreeFileArtifact treeFileArtifact1 = createFakeTreeFileArtifact(artifact, "child1", "hello1");
200 TreeFileArtifact treeFileArtifact2 = createFakeTreeFileArtifact(artifact, "child2", "hello2");
Rumou Duan73876202016-06-06 18:52:08 +0000201
202 TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact);
janakr45b308a2018-06-08 12:51:58 -0700203 assertThat(value.getChildValues()).containsKey(treeFileArtifact1);
204 assertThat(value.getChildValues()).containsKey(treeFileArtifact2);
lberkiaea56b32017-05-30 12:35:33 +0200205 assertThat(value.getChildValues().get(treeFileArtifact1).getDigest()).isNotNull();
206 assertThat(value.getChildValues().get(treeFileArtifact2).getDigest()).isNotNull();
Rumou Duan73876202016-06-06 18:52:08 +0000207 }
208
209 @Test
210 public void testSpawnActionTemplate() throws Throwable {
211 // artifact1 is a tree artifact generated by normal action.
cpeyserac09f0a2018-02-05 09:33:15 -0800212 SpecialArtifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1");
Rumou Duan73876202016-06-06 18:52:08 +0000213 createFakeTreeFileArtifact(artifact1, "child1", "hello1");
214 createFakeTreeFileArtifact(artifact1, "child2", "hello2");
215
Rumou Duan73876202016-06-06 18:52:08 +0000216 // artifact2 is a tree artifact generated by action template.
cpeyserac09f0a2018-02-05 09:33:15 -0800217 SpecialArtifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2");
Googler3b2967d2020-05-18 14:54:46 -0700218 SpawnActionTemplate actionTemplate =
219 ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
220 actions.add(actionTemplate);
janakr45b308a2018-06-08 12:51:58 -0700221 TreeFileArtifact treeFileArtifact1 =
Googler3b2967d2020-05-18 14:54:46 -0700222 createFakeExpansionTreeFileArtifact(actionTemplate, "child1", "hello1");
janakr45b308a2018-06-08 12:51:58 -0700223 TreeFileArtifact treeFileArtifact2 =
Googler3b2967d2020-05-18 14:54:46 -0700224 createFakeExpansionTreeFileArtifact(actionTemplate, "child2", "hello2");
Rumou Duan73876202016-06-06 18:52:08 +0000225
226 TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact2);
janakr45b308a2018-06-08 12:51:58 -0700227 assertThat(value.getChildValues()).containsKey(treeFileArtifact1);
228 assertThat(value.getChildValues()).containsKey(treeFileArtifact2);
lberkiaea56b32017-05-30 12:35:33 +0200229 assertThat(value.getChildValues().get(treeFileArtifact1).getDigest()).isNotNull();
230 assertThat(value.getChildValues().get(treeFileArtifact2).getDigest()).isNotNull();
Rumou Duan73876202016-06-06 18:52:08 +0000231 }
232
233 @Test
234 public void testConsecutiveSpawnActionTemplates() throws Throwable {
235 // artifact1 is a tree artifact generated by normal action.
cpeyserac09f0a2018-02-05 09:33:15 -0800236 SpecialArtifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1");
Rumou Duan73876202016-06-06 18:52:08 +0000237 createFakeTreeFileArtifact(artifact1, "child1", "hello1");
238 createFakeTreeFileArtifact(artifact1, "child2", "hello2");
239
240 // artifact2 is a tree artifact generated by action template.
cpeyserac09f0a2018-02-05 09:33:15 -0800241 SpecialArtifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2");
Googler3b2967d2020-05-18 14:54:46 -0700242 SpawnActionTemplate template2 =
243 ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
244 actions.add(template2);
245 createFakeExpansionTreeFileArtifact(template2, "child1", "hello1");
246 createFakeExpansionTreeFileArtifact(template2, "child2", "hello2");
Rumou Duan73876202016-06-06 18:52:08 +0000247
248 // artifact3 is a tree artifact generated by action template.
cpeyserac09f0a2018-02-05 09:33:15 -0800249 SpecialArtifact artifact3 = createDerivedTreeArtifactOnly("treeArtifact3");
Googler3b2967d2020-05-18 14:54:46 -0700250 SpawnActionTemplate template3 =
251 ActionsTestUtil.createDummySpawnActionTemplate(artifact2, artifact3);
252 actions.add(template3);
janakr45b308a2018-06-08 12:51:58 -0700253 TreeFileArtifact treeFileArtifact1 =
Googler3b2967d2020-05-18 14:54:46 -0700254 createFakeExpansionTreeFileArtifact(template3, "child1", "hello1");
janakr45b308a2018-06-08 12:51:58 -0700255 TreeFileArtifact treeFileArtifact2 =
Googler3b2967d2020-05-18 14:54:46 -0700256 createFakeExpansionTreeFileArtifact(template3, "child2", "hello2");
Rumou Duan73876202016-06-06 18:52:08 +0000257
258 TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact3);
janakr45b308a2018-06-08 12:51:58 -0700259 assertThat(value.getChildValues()).containsKey(treeFileArtifact1);
260 assertThat(value.getChildValues()).containsKey(treeFileArtifact2);
lberkiaea56b32017-05-30 12:35:33 +0200261 assertThat(value.getChildValues().get(treeFileArtifact1).getDigest()).isNotNull();
262 assertThat(value.getChildValues().get(treeFileArtifact2).getDigest()).isNotNull();
Rumou Duan73876202016-06-06 18:52:08 +0000263 }
264
janakre82933c2019-01-02 14:41:50 -0800265 @Test
Googler3b2967d2020-05-18 14:54:46 -0700266 public void actionTemplateExpansionOutputsOmitted() throws Throwable {
267 // artifact1 is a tree artifact generated by normal action.
268 SpecialArtifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1");
269 createFakeTreeFileArtifact(artifact1, "child1", "hello1");
270 createFakeTreeFileArtifact(artifact1, "child2", "hello2");
271
272 // artifact2 is a tree artifact generated by action template.
273 SpecialArtifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2");
274 SpawnActionTemplate actionTemplate =
275 ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
276 actions.add(actionTemplate);
277 TreeFileArtifact treeFileArtifact1 =
278 createFakeExpansionTreeFileArtifact(actionTemplate, "child1", "hello1");
279 TreeFileArtifact treeFileArtifact2 =
280 createFakeExpansionTreeFileArtifact(actionTemplate, "child2", "hello2");
281
282 omittedOutputs.add(treeFileArtifact1);
283 omittedOutputs.add(treeFileArtifact2);
284
285 SkyValue value = evaluateArtifactValue(artifact2);
Googler61a9f572020-06-02 10:28:24 -0700286 assertThat(value).isEqualTo(TreeArtifactValue.OMITTED_TREE_MARKER);
Googler3b2967d2020-05-18 14:54:46 -0700287 }
288
289 @Test
290 public void cannotOmitSomeButNotAllActionTemplateExpansionOutputs() throws Throwable {
291 // artifact1 is a tree artifact generated by normal action.
292 SpecialArtifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1");
293 createFakeTreeFileArtifact(artifact1, "child1", "hello1");
294 createFakeTreeFileArtifact(artifact1, "child2", "hello2");
295
296 // artifact2 is a tree artifact generated by action template.
297 SpecialArtifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2");
298 SpawnActionTemplate actionTemplate =
299 ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
300 actions.add(actionTemplate);
301 TreeFileArtifact treeFileArtifact1 =
302 createFakeExpansionTreeFileArtifact(actionTemplate, "child1", "hello1");
303 TreeFileArtifact treeFileArtifact2 =
304 createFakeExpansionTreeFileArtifact(actionTemplate, "child2", "hello2");
305
306 omittedOutputs.add(treeFileArtifact1);
307
308 Exception e = assertThrows(RuntimeException.class, () -> evaluateArtifactValue(artifact2));
309 assertThat(e).hasCauseThat().isInstanceOf(IllegalStateException.class);
310 assertThat(e)
311 .hasCauseThat()
312 .hasMessageThat()
313 .matches(
314 "Action template expansion has some but not all outputs omitted, present outputs: .*"
315 + treeFileArtifact2.getParentRelativePath()
316 + ".*");
317 }
318
319 @Test
janakre82933c2019-01-02 14:41:50 -0800320 public void actionExecutionValueSerialization() throws Exception {
janakrefb3f152019-06-05 17:42:34 -0700321 ActionLookupData dummyData = ActionLookupData.create(ALL_OWNER, 0);
Googler72d648a2020-05-21 11:56:40 -0700322 DerivedArtifact artifact1 = createDerivedArtifact("one");
lberkif7eee1e2019-07-31 05:49:10 -0700323 FileArtifactValue metadata1 =
324 ActionMetadataHandler.fileArtifactValueFromArtifact(artifact1, null, null);
janakre82933c2019-01-02 14:41:50 -0800325 SpecialArtifact treeArtifact = createDerivedTreeArtifactOnly("tree");
janakrefb3f152019-06-05 17:42:34 -0700326 treeArtifact.setGeneratingActionKey(dummyData);
Googler1d8d1382020-05-18 12:10:49 -0700327 TreeFileArtifact treeFileArtifact = TreeFileArtifact.createTreeOutput(treeArtifact, "subpath");
janakrefb3f152019-06-05 17:42:34 -0700328 Path path = treeFileArtifact.getPath();
Googler1d8d1382020-05-18 12:10:49 -0700329 path.getParentDirectory().createDirectoryAndParents();
janakrefb3f152019-06-05 17:42:34 -0700330 writeFile(path, "contents");
jhorvitzed7ec3b2020-07-24 15:08:03 -0700331 TreeArtifactValue tree =
332 TreeArtifactValue.newBuilder(treeArtifact)
333 .putChild(treeFileArtifact, FileArtifactValue.createForTesting(treeFileArtifact))
334 .build();
Googler72d648a2020-05-21 11:56:40 -0700335 DerivedArtifact artifact3 = createDerivedArtifact("three");
janakre82933c2019-01-02 14:41:50 -0800336 FilesetOutputSymlink filesetOutputSymlink =
337 FilesetOutputSymlink.createForTesting(
338 PathFragment.EMPTY_FRAGMENT, PathFragment.EMPTY_FRAGMENT, PathFragment.EMPTY_FRAGMENT);
339 ActionExecutionValue actionExecutionValue =
340 ActionExecutionValue.create(
lberkic35878a2019-08-01 02:28:54 -0700341 ImmutableMap.of(artifact1, metadata1, artifact3, FileArtifactValue.DEFAULT_MIDDLEMAN),
jhorvitzed7ec3b2020-07-24 15:08:03 -0700342 ImmutableMap.of(treeArtifact, tree),
janakre82933c2019-01-02 14:41:50 -0800343 ImmutableList.of(filesetOutputSymlink),
344 null,
345 true);
Googlerce6ad292019-12-20 10:29:21 -0800346 new SerializationTester(actionExecutionValue)
janakre82933c2019-01-02 14:41:50 -0800347 .addDependency(FileSystem.class, root.getFileSystem())
nharmataf93c72c2020-03-13 14:22:10 -0700348 .addDependency(
349 Root.RootCodecDependencies.class,
350 new Root.RootCodecDependencies(Root.absoluteRoot(root.getFileSystem())))
janakr40f2f0e2020-06-08 17:03:06 -0700351 .addDependencies(SerializationDepsUtils.SERIALIZATION_DEPS_FOR_TEST)
janakre82933c2019-01-02 14:41:50 -0800352 .runTests();
353 }
354
ajurkowskie2982912020-04-09 10:32:08 -0700355 private static void file(Path path, String contents) throws Exception {
jhorvitzed7ec3b2020-07-24 15:08:03 -0700356 path.getParentDirectory().createDirectoryAndParents();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000357 writeFile(path, contents);
358 }
359
360 private Artifact createSourceArtifact(String path) {
janakraea05602019-05-22 15:41:29 -0700361 return ActionsTestUtil.createArtifactWithExecPath(
362 ArtifactRoot.asSourceRoot(Root.fromPath(root)), PathFragment.create(path));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000363 }
364
Googler72d648a2020-05-21 11:56:40 -0700365 private DerivedArtifact createDerivedArtifact(String path) {
nharmatab4060b62017-04-04 17:11:39 +0000366 PathFragment execPath = PathFragment.create("out").getRelative(path);
Googler72d648a2020-05-21 11:56:40 -0700367 DerivedArtifact output =
jhorvitzcab340f2021-09-27 09:16:23 -0700368 DerivedArtifact.create(
Googlerf0b0c392021-01-27 17:56:52 -0800369 ArtifactRoot.asDerivedRoot(root, RootType.Output, "out"), execPath, ALL_OWNER);
ulfjack1e1a7752019-12-10 21:17:58 -0800370 actions.add(new DummyAction(NestedSetBuilder.emptySet(Order.STABLE_ORDER), output));
janakr8541f6d2019-06-11 14:40:21 -0700371 output.setGeneratingActionKey(ActionLookupData.create(ALL_OWNER, actions.size() - 1));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000372 return output;
373 }
374
Janak Ramakrishnana5578af2017-03-21 17:28:39 +0000375 private Artifact createMiddlemanArtifact(String path) {
tomlu1cdcdf92018-01-16 11:07:51 -0800376 ArtifactRoot middlemanRoot =
Googlerf0b0c392021-01-27 17:56:52 -0800377 ArtifactRoot.asDerivedRoot(middlemanPath, RootType.Middleman, PathFragment.create("out"));
jhorvitzcab340f2021-09-27 09:16:23 -0700378 return DerivedArtifact.create(
janakr3290e222019-05-29 16:34:22 -0700379 middlemanRoot, middlemanRoot.getExecPath().getRelative(path), ALL_OWNER);
Janak Ramakrishnana5578af2017-03-21 17:28:39 +0000380 }
381
cpeyserac09f0a2018-02-05 09:33:15 -0800382 private SpecialArtifact createDerivedTreeArtifactWithAction(String path) {
383 SpecialArtifact treeArtifact = createDerivedTreeArtifactOnly(path);
ulfjack1e1a7752019-12-10 21:17:58 -0800384 actions.add(new DummyAction(NestedSetBuilder.emptySet(Order.STABLE_ORDER), treeArtifact));
Googler1d8d1382020-05-18 12:10:49 -0700385 treeArtifact.setGeneratingActionKey(ActionLookupData.create(ALL_OWNER, actions.size() - 1));
Rumou Duan73876202016-06-06 18:52:08 +0000386 return treeArtifact;
387 }
388
cpeyserac09f0a2018-02-05 09:33:15 -0800389 private SpecialArtifact createDerivedTreeArtifactOnly(String path) {
nharmatab4060b62017-04-04 17:11:39 +0000390 PathFragment execPath = PathFragment.create("out").getRelative(path);
jhorvitzcab340f2021-09-27 09:16:23 -0700391 return SpecialArtifact.create(
Googlerf0b0c392021-01-27 17:56:52 -0800392 ArtifactRoot.asDerivedRoot(root, RootType.Output, "out"),
Googler147c2872021-01-21 08:47:48 -0800393 execPath,
394 ALL_OWNER,
395 SpecialArtifactType.TREE);
Rumou Duan73876202016-06-06 18:52:08 +0000396 }
397
ajurkowskie2982912020-04-09 10:32:08 -0700398 private static TreeFileArtifact createFakeTreeFileArtifact(
399 SpecialArtifact treeArtifact, String parentRelativePath, String content) throws Exception {
janakr45b308a2018-06-08 12:51:58 -0700400 TreeFileArtifact treeFileArtifact =
Googler1d8d1382020-05-18 12:10:49 -0700401 TreeFileArtifact.createTreeOutput(treeArtifact, parentRelativePath);
Rumou Duan73876202016-06-06 18:52:08 +0000402 Path path = treeFileArtifact.getPath();
Googler1d8d1382020-05-18 12:10:49 -0700403 path.getParentDirectory().createDirectoryAndParents();
404 writeFile(path, content);
405 return treeFileArtifact;
406 }
407
Googler3b2967d2020-05-18 14:54:46 -0700408 private TreeFileArtifact createFakeExpansionTreeFileArtifact(
409 ActionTemplate<?> actionTemplate, String parentRelativePath, String content)
410 throws Exception {
411 int actionIndex = Iterables.indexOf(actions, actionTemplate::equals);
412 Preconditions.checkState(actionIndex >= 0, "%s not registered", actionTemplate);
Googler1d8d1382020-05-18 12:10:49 -0700413 TreeFileArtifact treeFileArtifact =
414 TreeFileArtifact.createTemplateExpansionOutput(
Googler3b2967d2020-05-18 14:54:46 -0700415 actionTemplate.getOutputTreeArtifact(),
Googler1d8d1382020-05-18 12:10:49 -0700416 parentRelativePath,
Googler3b2967d2020-05-18 14:54:46 -0700417 ActionTemplateExpansionValue.key(ALL_OWNER, actionIndex));
Googler1d8d1382020-05-18 12:10:49 -0700418 Path path = treeFileArtifact.getPath();
419 path.getParentDirectory().createDirectoryAndParents();
Rumou Duan73876202016-06-06 18:52:08 +0000420 writeFile(path, content);
421 return treeFileArtifact;
422 }
423
ajurkowskie2982912020-04-09 10:32:08 -0700424 private static void assertValueMatches(FileStatus file, byte[] digest, FileArtifactValue value)
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000425 throws IOException {
lberkiaea56b32017-05-30 12:35:33 +0200426 assertThat(value.getSize()).isEqualTo(file.getSize());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000427 if (digest == null) {
lberkiaea56b32017-05-30 12:35:33 +0200428 assertThat(value.getDigest()).isNull();
429 assertThat(value.getModifiedTime()).isEqualTo(file.getLastModifiedTime());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000430 } else {
lberkiaea56b32017-05-30 12:35:33 +0200431 assertThat(value.getDigest()).isEqualTo(digest);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000432 }
433 }
434
435 private FileArtifactValue evaluateFAN(Artifact artifact) throws Throwable {
436 return ((FileArtifactValue) evaluateArtifactValue(artifact));
437 }
438
Janak Ramakrishnanad77f972016-07-29 20:58:42 +0000439 private SkyValue evaluateArtifactValue(Artifact artifact) throws Throwable {
lberki36df7ed2019-06-27 06:32:03 -0700440 SkyKey key = Artifact.key(artifact);
Janak Ramakrishnanad77f972016-07-29 20:58:42 +0000441 EvaluationResult<SkyValue> result = evaluate(ImmutableList.of(key).toArray(new SkyKey[0]));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000442 if (result.hasError()) {
443 throw result.getError().getException();
444 }
janakr8541f6d2019-06-11 14:40:21 -0700445 SkyValue value = result.get(key);
446 if (value instanceof ActionExecutionValue) {
Googleraed41602020-06-02 12:22:53 -0700447 return ((ActionExecutionValue) value).getExistingFileArtifactValue(artifact);
janakr8541f6d2019-06-11 14:40:21 -0700448 }
449 return value;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000450 }
451
janakr0175ce32018-02-26 15:54:57 -0800452 private void setGeneratingActions() throws InterruptedException, ActionConflictException {
janakr573807d2018-01-11 14:02:35 -0800453 if (evaluator.getExistingValue(ALL_OWNER) == null) {
janakr93e3eea2017-03-30 22:09:37 +0000454 differencer.inject(
455 ImmutableMap.of(
janakr573807d2018-01-11 14:02:35 -0800456 ALL_OWNER,
cparsonse2d200f2018-03-06 16:15:11 -0800457 new BasicActionLookupValue(
janakrefb3f152019-06-05 17:42:34 -0700458 Actions.assignOwnersAndFilterSharedActionsAndThrowActionConflict(
459 actionKeyContext,
460 ImmutableList.copyOf(actions),
461 ALL_OWNER,
Googlerce6ad292019-12-20 10:29:21 -0800462 /*outputFiles=*/ null))));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000463 }
464 }
465
466 private <E extends SkyValue> EvaluationResult<E> evaluate(SkyKey... keys)
janakr0175ce32018-02-26 15:54:57 -0800467 throws InterruptedException, ActionConflictException {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000468 setGeneratingActions();
Googler10028672018-10-25 12:14:34 -0700469 EvaluationContext evaluationContext =
470 EvaluationContext.newBuilder()
471 .setKeepGoing(false)
472 .setNumThreads(SkyframeExecutor.DEFAULT_THREAD_COUNT)
michajlo7a485be2020-07-30 11:08:46 -0700473 .setEventHandler(NullEventHandler.INSTANCE)
Googler10028672018-10-25 12:14:34 -0700474 .build();
475 return driver.evaluate(Arrays.asList(keys), evaluationContext);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000476 }
477
Googler3b2967d2020-05-18 14:54:46 -0700478 /**
479 * Value builder for actions that just stats and stores the output file (which must either be
480 * orphaned or exist).
481 */
482 private static final class SimpleActionExecutionFunction implements SkyFunction {
483 private final Set<Artifact> omittedOutputs;
484
485 SimpleActionExecutionFunction(Set<Artifact> omittedOutputs) {
486 this.omittedOutputs = omittedOutputs;
487 }
488
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000489 @Override
janakr93e3eea2017-03-30 22:09:37 +0000490 public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException {
lberkif7eee1e2019-07-31 05:49:10 -0700491 Map<Artifact, FileArtifactValue> artifactData = new HashMap<>();
Rumou Duan73876202016-06-06 18:52:08 +0000492 Map<Artifact, TreeArtifactValue> treeArtifactData = new HashMap<>();
janakr93e3eea2017-03-30 22:09:37 +0000493 ActionLookupData actionLookupData = (ActionLookupData) skyKey.argument();
494 ActionLookupValue actionLookupValue =
janakrbaf52ae2018-02-14 09:03:18 -0800495 (ActionLookupValue) env.getValue(actionLookupData.getActionLookupKey());
janakr93e3eea2017-03-30 22:09:37 +0000496 Action action = actionLookupValue.getAction(actionLookupData.getActionIndex());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000497 Artifact output = Iterables.getOnlyElement(action.getOutputs());
Rumou Duan73876202016-06-06 18:52:08 +0000498
499 try {
Googler3b2967d2020-05-18 14:54:46 -0700500 if (omittedOutputs.contains(output)) {
501 Preconditions.checkState(!output.isTreeArtifact(), "Cannot omit %s", output);
502 artifactData.put(output, FileArtifactValue.OMITTED_FILE_MARKER);
503 } else if (output.isTreeArtifact()) {
jhorvitzed7ec3b2020-07-24 15:08:03 -0700504 SpecialArtifact parent = (SpecialArtifact) output;
Googler1d8d1382020-05-18 12:10:49 -0700505 TreeFileArtifact treeFileArtifact1 =
506 TreeFileArtifact.createTreeOutput((SpecialArtifact) output, "child1");
507 TreeFileArtifact treeFileArtifact2 =
508 TreeFileArtifact.createTreeOutput((SpecialArtifact) output, "child2");
jhorvitzed7ec3b2020-07-24 15:08:03 -0700509 TreeArtifactValue tree =
510 TreeArtifactValue.newBuilder(parent)
511 .putChild(
512 treeFileArtifact1, FileArtifactValue.createForTesting(treeFileArtifact1))
513 .putChild(
514 treeFileArtifact2, FileArtifactValue.createForTesting(treeFileArtifact2))
515 .build();
516 treeArtifactData.put(output, tree);
Rumou Duan73876202016-06-06 18:52:08 +0000517 } else if (action.getActionType() == MiddlemanType.NORMAL) {
lberkif225af12019-08-01 04:21:58 -0700518 Path path = output.getPath();
519 FileArtifactValue noDigest =
520 ActionMetadataHandler.fileArtifactValueFromArtifact(
521 output,
522 FileStatusWithDigestAdapter.adapt(path.statIfFound(Symlinks.NOFOLLOW)),
523 null);
524 FileArtifactValue withDigest =
525 FileArtifactValue.createFromInjectedDigest(
526 noDigest, path.getDigest(), !output.isConstantMetadata());
527 artifactData.put(output, withDigest);
Rumou Duan73876202016-06-06 18:52:08 +0000528 } else {
lberkic35878a2019-08-01 02:28:54 -0700529 artifactData.put(output, FileArtifactValue.DEFAULT_MIDDLEMAN);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000530 }
Rumou Duan73876202016-06-06 18:52:08 +0000531 } catch (IOException e) {
532 throw new IllegalStateException(e);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000533 }
janakrb9d8d582018-06-13 21:57:19 -0700534 return ActionExecutionValue.create(
535 artifactData,
536 treeArtifactData,
janakrb9d8d582018-06-13 21:57:19 -0700537 /*outputSymlinks=*/ null,
shahanef6f4cf2018-06-26 11:24:59 -0700538 /*discoveredModules=*/ null,
janakr9f496f32018-10-24 15:08:09 -0700539 /*actionDependsOnBuildId=*/ false);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000540 }
541
542 @Override
543 public String extractTag(SkyKey skyKey) {
544 return null;
545 }
546 }
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000547}