blob: 51dd6fb097c209ade1c674fcc39bc865a130c344 [file] [log] [blame]
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001// Copyright 2015 The Bazel Authors. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package com.google.devtools.build.lib.skyframe;
Janak Ramakrishnana5578af2017-03-21 17:28:39 +000015
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000016import static com.google.common.truth.Truth.assertThat;
lberki812e6fe2019-07-25 07:50:55 -070017import static com.google.devtools.build.lib.actions.FileArtifactValue.createForTesting;
michajlo660d17f2020-03-27 09:01:57 -070018import static org.junit.Assert.assertThrows;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000019
Googler3b2967d2020-05-18 14:54:46 -070020import com.google.common.base.Preconditions;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000021import com.google.common.collect.ImmutableList;
22import com.google.common.collect.ImmutableMap;
23import com.google.common.collect.ImmutableSet;
24import com.google.common.collect.Iterables;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000025import com.google.devtools.build.lib.actions.Action;
janakr93e3eea2017-03-30 22:09:37 +000026import com.google.devtools.build.lib.actions.ActionLookupData;
27import com.google.devtools.build.lib.actions.ActionLookupValue;
Googler3b2967d2020-05-18 14:54:46 -070028import com.google.devtools.build.lib.actions.ActionTemplate;
janakr0175ce32018-02-26 15:54:57 -080029import com.google.devtools.build.lib.actions.Actions;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000030import com.google.devtools.build.lib.actions.Artifact;
Googler72d648a2020-05-21 11:56:40 -070031import com.google.devtools.build.lib.actions.Artifact.DerivedArtifact;
Rumou Duan73876202016-06-06 18:52:08 +000032import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
33import com.google.devtools.build.lib.actions.Artifact.SpecialArtifactType;
34import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
tomlu1cdcdf92018-01-16 11:07:51 -080035import com.google.devtools.build.lib.actions.ArtifactRoot;
Googlerf0b0c392021-01-27 17:56:52 -080036import com.google.devtools.build.lib.actions.ArtifactRoot.RootType;
cparsonse2d200f2018-03-06 16:15:11 -080037import com.google.devtools.build.lib.actions.BasicActionLookupValue;
shahan602cc852018-06-06 20:09:57 -070038import com.google.devtools.build.lib.actions.FileArtifactValue;
janakre82933c2019-01-02 14:41:50 -080039import com.google.devtools.build.lib.actions.FilesetOutputSymlink;
jhorvitz3daedc32020-07-22 18:33:55 -070040import com.google.devtools.build.lib.actions.MiddlemanType;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000041import com.google.devtools.build.lib.actions.MissingInputFileException;
janakr0175ce32018-02-26 15:54:57 -080042import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException;
Rumou Duan73876202016-06-06 18:52:08 +000043import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000044import com.google.devtools.build.lib.actions.util.TestAction.DummyAction;
Googler3b2967d2020-05-18 14:54:46 -070045import com.google.devtools.build.lib.analysis.actions.SpawnActionTemplate;
ulfjack1e1a7752019-12-10 21:17:58 -080046import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
47import com.google.devtools.build.lib.collect.nestedset.Order;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000048import com.google.devtools.build.lib.events.NullEventHandler;
janakr40f2f0e2020-06-08 17:03:06 -070049import com.google.devtools.build.lib.skyframe.serialization.testutils.SerializationDepsUtils;
janakre82933c2019-01-02 14:41:50 -080050import com.google.devtools.build.lib.skyframe.serialization.testutils.SerializationTester;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000051import com.google.devtools.build.lib.util.Pair;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000052import com.google.devtools.build.lib.vfs.FileStatus;
lberkif225af12019-08-01 04:21:58 -070053import com.google.devtools.build.lib.vfs.FileStatusWithDigestAdapter;
janakre82933c2019-01-02 14:41:50 -080054import com.google.devtools.build.lib.vfs.FileSystem;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000055import com.google.devtools.build.lib.vfs.Path;
56import com.google.devtools.build.lib.vfs.PathFragment;
tomluee6a6862018-01-17 14:36:26 -080057import com.google.devtools.build.lib.vfs.Root;
lberkif225af12019-08-01 04:21:58 -070058import com.google.devtools.build.lib.vfs.Symlinks;
Googler10028672018-10-25 12:14:34 -070059import com.google.devtools.build.skyframe.EvaluationContext;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000060import com.google.devtools.build.skyframe.EvaluationResult;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000061import com.google.devtools.build.skyframe.SkyFunction;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000062import com.google.devtools.build.skyframe.SkyKey;
63import com.google.devtools.build.skyframe.SkyValue;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000064import java.io.IOException;
buchgrd4d3d502018-08-02 06:47:19 -070065import java.util.ArrayList;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000066import java.util.Arrays;
67import java.util.HashMap;
Googler3b2967d2020-05-18 14:54:46 -070068import java.util.HashSet;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000069import java.util.Map;
Googler3b2967d2020-05-18 14:54:46 -070070import java.util.Set;
Janak Ramakrishnanad77f972016-07-29 20:58:42 +000071import org.junit.Before;
72import org.junit.Test;
73import org.junit.runner.RunWith;
74import org.junit.runners.JUnit4;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000075
76/**
77 * Tests for {@link ArtifactFunction}.
78 */
79// Doesn't actually need any particular Skyframe, but is only relevant to Skyframe full mode.
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +000080@RunWith(JUnit4.class)
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000081public class ArtifactFunctionTest extends ArtifactFunctionTestCase {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000082
Googler3b2967d2020-05-18 14:54:46 -070083 private final Set<Artifact> omittedOutputs = new HashSet<>();
84
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +000085 @Before
Googler3b2967d2020-05-18 14:54:46 -070086 public final void setUp() {
87 delegateActionExecutionFunction = new SimpleActionExecutionFunction(omittedOutputs);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000088 }
89
90 private void assertFileArtifactValueMatches(boolean expectDigest) throws Throwable {
91 Artifact output = createDerivedArtifact("output");
92 Path path = output.getPath();
93 file(path, "contents");
olaolabfd1d332017-06-19 16:55:24 -040094 assertValueMatches(path.stat(), expectDigest ? path.getDigest() : null, evaluateFAN(output));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000095 }
96
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +000097 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000098 public void testBasicArtifact() throws Throwable {
99 fastDigest = false;
100 assertFileArtifactValueMatches(/*expectDigest=*/ true);
101 }
102
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000103 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000104 public void testBasicArtifactWithXattr() throws Throwable {
105 fastDigest = true;
106 assertFileArtifactValueMatches(/*expectDigest=*/ true);
107 }
108
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000109 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000110 public void testMissingNonMandatoryArtifact() throws Throwable {
111 Artifact input = createSourceArtifact("input1");
lberki36df7ed2019-06-27 06:32:03 -0700112 assertThat(evaluateArtifactValue(input)).isNotNull();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000113 }
114
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000115 @Test
Michajlo Matijkiw528957e2016-01-19 21:17:45 +0000116 public void testUnreadableInputWithFsWithAvailableDigest() throws Throwable {
Googlere764d562019-10-16 10:56:33 -0700117 final byte[] expectedDigest = {1, 2, 3, 4};
Michajlo Matijkiw528957e2016-01-19 21:17:45 +0000118 setupRoot(
119 new CustomInMemoryFs() {
120 @Override
ajurkowski8883c612021-03-08 08:12:37 -0800121 public byte[] getDigest(PathFragment path) throws IOException {
ccalvarindd9f60e2018-07-23 18:16:18 -0700122 return path.getBaseName().equals("unreadable") ? expectedDigest : super.getDigest(path);
Michajlo Matijkiw528957e2016-01-19 21:17:45 +0000123 }
124 });
125
126 Artifact input = createSourceArtifact("unreadable");
127 Path inputPath = input.getPath();
128 file(inputPath, "dummynotused");
129 inputPath.chmod(0);
130
lberki36df7ed2019-06-27 06:32:03 -0700131 FileArtifactValue value = (FileArtifactValue) evaluateArtifactValue(input);
Michajlo Matijkiw528957e2016-01-19 21:17:45 +0000132
133 FileStatus stat = inputPath.stat();
134 assertThat(value.getSize()).isEqualTo(stat.getSize());
135 assertThat(value.getDigest()).isEqualTo(expectedDigest);
136 }
137
138 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000139 public void testMiddlemanArtifact() throws Throwable {
Janak Ramakrishnana5578af2017-03-21 17:28:39 +0000140 Artifact output = createMiddlemanArtifact("output");
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000141 Artifact input1 = createSourceArtifact("input1");
142 Artifact input2 = createDerivedArtifact("input2");
Benjamin Peterson63748e42018-06-03 22:11:16 -0700143 SpecialArtifact tree = createDerivedTreeArtifactWithAction("treeArtifact");
buchgrd4d3d502018-08-02 06:47:19 -0700144 TreeFileArtifact treeFile1 = createFakeTreeFileArtifact(tree, "child1", "hello1");
145 TreeFileArtifact treeFile2 = createFakeTreeFileArtifact(tree, "child2", "hello2");
146 file(treeFile1.getPath(), "src1");
147 file(treeFile2.getPath(), "src2");
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000148 Action action =
149 new DummyAction(
ulfjack1e1a7752019-12-10 21:17:58 -0800150 NestedSetBuilder.create(Order.STABLE_ORDER, input1, input2, tree),
151 output,
152 MiddlemanType.AGGREGATING_MIDDLEMAN);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000153 actions.add(action);
154 file(input2.getPath(), "contents");
155 file(input1.getPath(), "source contents");
janakr8541f6d2019-06-11 14:40:21 -0700156 evaluate(
157 Iterables.toArray(
lberki36df7ed2019-06-27 06:32:03 -0700158 Artifact.keys(ImmutableSet.of(input2, input1, input2, tree)), SkyKey.class));
Janak Ramakrishnanad77f972016-07-29 20:58:42 +0000159 SkyValue value = evaluateArtifactValue(output);
buchgrd4d3d502018-08-02 06:47:19 -0700160 ArrayList<Pair<Artifact, ?>> inputs = new ArrayList<>();
161 inputs.addAll(((AggregatingArtifactValue) value).getFileArtifacts());
162 inputs.addAll(((AggregatingArtifactValue) value).getTreeArtifacts());
163 assertThat(inputs)
Benjamin Peterson63748e42018-06-03 22:11:16 -0700164 .containsExactly(
lberki812e6fe2019-07-25 07:50:55 -0700165 Pair.of(input1, createForTesting(input1)),
166 Pair.of(input2, createForTesting(input2)),
buchgrd4d3d502018-08-02 06:47:19 -0700167 Pair.of(tree, ((TreeArtifactValue) evaluateArtifactValue(tree))));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000168 }
169
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000170 /**
jmmvbfbd95f2020-08-31 13:12:38 -0700171 * Tests that ArtifactFunction rethrows transitive {@link IOException}s as {@link
172 * MissingInputFileException}s.
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000173 */
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000174 @Test
jmmvbfbd95f2020-08-31 13:12:38 -0700175 public void testIOException_endToEnd() throws Throwable {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000176 final IOException exception = new IOException("beep");
177 setupRoot(
178 new CustomInMemoryFs() {
179 @Override
ajurkowski8883c612021-03-08 08:12:37 -0800180 public FileStatus statIfFound(PathFragment path, boolean followSymlinks)
181 throws IOException {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000182 if (path.getBaseName().equals("bad")) {
183 throw exception;
184 }
fellya205ed82018-09-10 11:52:34 -0700185 return super.statIfFound(path, followSymlinks);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000186 }
187 });
lberki36df7ed2019-06-27 06:32:03 -0700188 IOException e =
189 assertThrows(IOException.class, () -> evaluateArtifactValue(createSourceArtifact("bad")));
jcater83130f42019-04-30 14:29:28 -0700190 assertThat(e).hasMessageThat().contains(exception.getMessage());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000191 }
192
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000193 @Test
Rumou Duan73876202016-06-06 18:52:08 +0000194 public void testActionTreeArtifactOutput() throws Throwable {
cpeyserac09f0a2018-02-05 09:33:15 -0800195 SpecialArtifact artifact = createDerivedTreeArtifactWithAction("treeArtifact");
janakrefb3f152019-06-05 17:42:34 -0700196 TreeFileArtifact treeFileArtifact1 = createFakeTreeFileArtifact(artifact, "child1", "hello1");
197 TreeFileArtifact treeFileArtifact2 = createFakeTreeFileArtifact(artifact, "child2", "hello2");
Rumou Duan73876202016-06-06 18:52:08 +0000198
199 TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact);
janakr45b308a2018-06-08 12:51:58 -0700200 assertThat(value.getChildValues()).containsKey(treeFileArtifact1);
201 assertThat(value.getChildValues()).containsKey(treeFileArtifact2);
lberkiaea56b32017-05-30 12:35:33 +0200202 assertThat(value.getChildValues().get(treeFileArtifact1).getDigest()).isNotNull();
203 assertThat(value.getChildValues().get(treeFileArtifact2).getDigest()).isNotNull();
Rumou Duan73876202016-06-06 18:52:08 +0000204 }
205
206 @Test
207 public void testSpawnActionTemplate() throws Throwable {
208 // artifact1 is a tree artifact generated by normal action.
cpeyserac09f0a2018-02-05 09:33:15 -0800209 SpecialArtifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1");
Rumou Duan73876202016-06-06 18:52:08 +0000210 createFakeTreeFileArtifact(artifact1, "child1", "hello1");
211 createFakeTreeFileArtifact(artifact1, "child2", "hello2");
212
Rumou Duan73876202016-06-06 18:52:08 +0000213 // artifact2 is a tree artifact generated by action template.
cpeyserac09f0a2018-02-05 09:33:15 -0800214 SpecialArtifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2");
Googler3b2967d2020-05-18 14:54:46 -0700215 SpawnActionTemplate actionTemplate =
216 ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
217 actions.add(actionTemplate);
janakr45b308a2018-06-08 12:51:58 -0700218 TreeFileArtifact treeFileArtifact1 =
Googler3b2967d2020-05-18 14:54:46 -0700219 createFakeExpansionTreeFileArtifact(actionTemplate, "child1", "hello1");
janakr45b308a2018-06-08 12:51:58 -0700220 TreeFileArtifact treeFileArtifact2 =
Googler3b2967d2020-05-18 14:54:46 -0700221 createFakeExpansionTreeFileArtifact(actionTemplate, "child2", "hello2");
Rumou Duan73876202016-06-06 18:52:08 +0000222
223 TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact2);
janakr45b308a2018-06-08 12:51:58 -0700224 assertThat(value.getChildValues()).containsKey(treeFileArtifact1);
225 assertThat(value.getChildValues()).containsKey(treeFileArtifact2);
lberkiaea56b32017-05-30 12:35:33 +0200226 assertThat(value.getChildValues().get(treeFileArtifact1).getDigest()).isNotNull();
227 assertThat(value.getChildValues().get(treeFileArtifact2).getDigest()).isNotNull();
Rumou Duan73876202016-06-06 18:52:08 +0000228 }
229
230 @Test
231 public void testConsecutiveSpawnActionTemplates() throws Throwable {
232 // artifact1 is a tree artifact generated by normal action.
cpeyserac09f0a2018-02-05 09:33:15 -0800233 SpecialArtifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1");
Rumou Duan73876202016-06-06 18:52:08 +0000234 createFakeTreeFileArtifact(artifact1, "child1", "hello1");
235 createFakeTreeFileArtifact(artifact1, "child2", "hello2");
236
237 // artifact2 is a tree artifact generated by action template.
cpeyserac09f0a2018-02-05 09:33:15 -0800238 SpecialArtifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2");
Googler3b2967d2020-05-18 14:54:46 -0700239 SpawnActionTemplate template2 =
240 ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
241 actions.add(template2);
242 createFakeExpansionTreeFileArtifact(template2, "child1", "hello1");
243 createFakeExpansionTreeFileArtifact(template2, "child2", "hello2");
Rumou Duan73876202016-06-06 18:52:08 +0000244
245 // artifact3 is a tree artifact generated by action template.
cpeyserac09f0a2018-02-05 09:33:15 -0800246 SpecialArtifact artifact3 = createDerivedTreeArtifactOnly("treeArtifact3");
Googler3b2967d2020-05-18 14:54:46 -0700247 SpawnActionTemplate template3 =
248 ActionsTestUtil.createDummySpawnActionTemplate(artifact2, artifact3);
249 actions.add(template3);
janakr45b308a2018-06-08 12:51:58 -0700250 TreeFileArtifact treeFileArtifact1 =
Googler3b2967d2020-05-18 14:54:46 -0700251 createFakeExpansionTreeFileArtifact(template3, "child1", "hello1");
janakr45b308a2018-06-08 12:51:58 -0700252 TreeFileArtifact treeFileArtifact2 =
Googler3b2967d2020-05-18 14:54:46 -0700253 createFakeExpansionTreeFileArtifact(template3, "child2", "hello2");
Rumou Duan73876202016-06-06 18:52:08 +0000254
255 TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact3);
janakr45b308a2018-06-08 12:51:58 -0700256 assertThat(value.getChildValues()).containsKey(treeFileArtifact1);
257 assertThat(value.getChildValues()).containsKey(treeFileArtifact2);
lberkiaea56b32017-05-30 12:35:33 +0200258 assertThat(value.getChildValues().get(treeFileArtifact1).getDigest()).isNotNull();
259 assertThat(value.getChildValues().get(treeFileArtifact2).getDigest()).isNotNull();
Rumou Duan73876202016-06-06 18:52:08 +0000260 }
261
janakre82933c2019-01-02 14:41:50 -0800262 @Test
Googler3b2967d2020-05-18 14:54:46 -0700263 public void actionTemplateExpansionOutputsOmitted() throws Throwable {
264 // artifact1 is a tree artifact generated by normal action.
265 SpecialArtifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1");
266 createFakeTreeFileArtifact(artifact1, "child1", "hello1");
267 createFakeTreeFileArtifact(artifact1, "child2", "hello2");
268
269 // artifact2 is a tree artifact generated by action template.
270 SpecialArtifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2");
271 SpawnActionTemplate actionTemplate =
272 ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
273 actions.add(actionTemplate);
274 TreeFileArtifact treeFileArtifact1 =
275 createFakeExpansionTreeFileArtifact(actionTemplate, "child1", "hello1");
276 TreeFileArtifact treeFileArtifact2 =
277 createFakeExpansionTreeFileArtifact(actionTemplate, "child2", "hello2");
278
279 omittedOutputs.add(treeFileArtifact1);
280 omittedOutputs.add(treeFileArtifact2);
281
282 SkyValue value = evaluateArtifactValue(artifact2);
Googler61a9f572020-06-02 10:28:24 -0700283 assertThat(value).isEqualTo(TreeArtifactValue.OMITTED_TREE_MARKER);
Googler3b2967d2020-05-18 14:54:46 -0700284 }
285
286 @Test
287 public void cannotOmitSomeButNotAllActionTemplateExpansionOutputs() throws Throwable {
288 // artifact1 is a tree artifact generated by normal action.
289 SpecialArtifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1");
290 createFakeTreeFileArtifact(artifact1, "child1", "hello1");
291 createFakeTreeFileArtifact(artifact1, "child2", "hello2");
292
293 // artifact2 is a tree artifact generated by action template.
294 SpecialArtifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2");
295 SpawnActionTemplate actionTemplate =
296 ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
297 actions.add(actionTemplate);
298 TreeFileArtifact treeFileArtifact1 =
299 createFakeExpansionTreeFileArtifact(actionTemplate, "child1", "hello1");
300 TreeFileArtifact treeFileArtifact2 =
301 createFakeExpansionTreeFileArtifact(actionTemplate, "child2", "hello2");
302
303 omittedOutputs.add(treeFileArtifact1);
304
305 Exception e = assertThrows(RuntimeException.class, () -> evaluateArtifactValue(artifact2));
306 assertThat(e).hasCauseThat().isInstanceOf(IllegalStateException.class);
307 assertThat(e)
308 .hasCauseThat()
309 .hasMessageThat()
310 .matches(
311 "Action template expansion has some but not all outputs omitted, present outputs: .*"
312 + treeFileArtifact2.getParentRelativePath()
313 + ".*");
314 }
315
316 @Test
janakre82933c2019-01-02 14:41:50 -0800317 public void actionExecutionValueSerialization() throws Exception {
janakrefb3f152019-06-05 17:42:34 -0700318 ActionLookupData dummyData = ActionLookupData.create(ALL_OWNER, 0);
Googler72d648a2020-05-21 11:56:40 -0700319 DerivedArtifact artifact1 = createDerivedArtifact("one");
lberkif7eee1e2019-07-31 05:49:10 -0700320 FileArtifactValue metadata1 =
321 ActionMetadataHandler.fileArtifactValueFromArtifact(artifact1, null, null);
janakre82933c2019-01-02 14:41:50 -0800322 SpecialArtifact treeArtifact = createDerivedTreeArtifactOnly("tree");
janakrefb3f152019-06-05 17:42:34 -0700323 treeArtifact.setGeneratingActionKey(dummyData);
Googler1d8d1382020-05-18 12:10:49 -0700324 TreeFileArtifact treeFileArtifact = TreeFileArtifact.createTreeOutput(treeArtifact, "subpath");
janakrefb3f152019-06-05 17:42:34 -0700325 Path path = treeFileArtifact.getPath();
Googler1d8d1382020-05-18 12:10:49 -0700326 path.getParentDirectory().createDirectoryAndParents();
janakrefb3f152019-06-05 17:42:34 -0700327 writeFile(path, "contents");
jhorvitzed7ec3b2020-07-24 15:08:03 -0700328 TreeArtifactValue tree =
329 TreeArtifactValue.newBuilder(treeArtifact)
330 .putChild(treeFileArtifact, FileArtifactValue.createForTesting(treeFileArtifact))
331 .build();
Googler72d648a2020-05-21 11:56:40 -0700332 DerivedArtifact artifact3 = createDerivedArtifact("three");
janakre82933c2019-01-02 14:41:50 -0800333 FilesetOutputSymlink filesetOutputSymlink =
334 FilesetOutputSymlink.createForTesting(
335 PathFragment.EMPTY_FRAGMENT, PathFragment.EMPTY_FRAGMENT, PathFragment.EMPTY_FRAGMENT);
336 ActionExecutionValue actionExecutionValue =
337 ActionExecutionValue.create(
lberkic35878a2019-08-01 02:28:54 -0700338 ImmutableMap.of(artifact1, metadata1, artifact3, FileArtifactValue.DEFAULT_MIDDLEMAN),
jhorvitzed7ec3b2020-07-24 15:08:03 -0700339 ImmutableMap.of(treeArtifact, tree),
janakre82933c2019-01-02 14:41:50 -0800340 ImmutableList.of(filesetOutputSymlink),
341 null,
342 true);
Googlerce6ad292019-12-20 10:29:21 -0800343 new SerializationTester(actionExecutionValue)
janakre82933c2019-01-02 14:41:50 -0800344 .addDependency(FileSystem.class, root.getFileSystem())
nharmataf93c72c2020-03-13 14:22:10 -0700345 .addDependency(
346 Root.RootCodecDependencies.class,
347 new Root.RootCodecDependencies(Root.absoluteRoot(root.getFileSystem())))
janakr40f2f0e2020-06-08 17:03:06 -0700348 .addDependencies(SerializationDepsUtils.SERIALIZATION_DEPS_FOR_TEST)
janakre82933c2019-01-02 14:41:50 -0800349 .runTests();
350 }
351
ajurkowskie2982912020-04-09 10:32:08 -0700352 private static void file(Path path, String contents) throws Exception {
jhorvitzed7ec3b2020-07-24 15:08:03 -0700353 path.getParentDirectory().createDirectoryAndParents();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000354 writeFile(path, contents);
355 }
356
357 private Artifact createSourceArtifact(String path) {
janakraea05602019-05-22 15:41:29 -0700358 return ActionsTestUtil.createArtifactWithExecPath(
359 ArtifactRoot.asSourceRoot(Root.fromPath(root)), PathFragment.create(path));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000360 }
361
Googler72d648a2020-05-21 11:56:40 -0700362 private DerivedArtifact createDerivedArtifact(String path) {
nharmatab4060b62017-04-04 17:11:39 +0000363 PathFragment execPath = PathFragment.create("out").getRelative(path);
Googler72d648a2020-05-21 11:56:40 -0700364 DerivedArtifact output =
Googler660f5b22021-01-25 09:40:17 -0800365 new DerivedArtifact(
Googlerf0b0c392021-01-27 17:56:52 -0800366 ArtifactRoot.asDerivedRoot(root, RootType.Output, "out"), execPath, ALL_OWNER);
ulfjack1e1a7752019-12-10 21:17:58 -0800367 actions.add(new DummyAction(NestedSetBuilder.emptySet(Order.STABLE_ORDER), output));
janakr8541f6d2019-06-11 14:40:21 -0700368 output.setGeneratingActionKey(ActionLookupData.create(ALL_OWNER, actions.size() - 1));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000369 return output;
370 }
371
Janak Ramakrishnana5578af2017-03-21 17:28:39 +0000372 private Artifact createMiddlemanArtifact(String path) {
tomlu1cdcdf92018-01-16 11:07:51 -0800373 ArtifactRoot middlemanRoot =
Googlerf0b0c392021-01-27 17:56:52 -0800374 ArtifactRoot.asDerivedRoot(middlemanPath, RootType.Middleman, PathFragment.create("out"));
Googler72d648a2020-05-21 11:56:40 -0700375 return new DerivedArtifact(
janakr3290e222019-05-29 16:34:22 -0700376 middlemanRoot, middlemanRoot.getExecPath().getRelative(path), ALL_OWNER);
Janak Ramakrishnana5578af2017-03-21 17:28:39 +0000377 }
378
cpeyserac09f0a2018-02-05 09:33:15 -0800379 private SpecialArtifact createDerivedTreeArtifactWithAction(String path) {
380 SpecialArtifact treeArtifact = createDerivedTreeArtifactOnly(path);
ulfjack1e1a7752019-12-10 21:17:58 -0800381 actions.add(new DummyAction(NestedSetBuilder.emptySet(Order.STABLE_ORDER), treeArtifact));
Googler1d8d1382020-05-18 12:10:49 -0700382 treeArtifact.setGeneratingActionKey(ActionLookupData.create(ALL_OWNER, actions.size() - 1));
Rumou Duan73876202016-06-06 18:52:08 +0000383 return treeArtifact;
384 }
385
cpeyserac09f0a2018-02-05 09:33:15 -0800386 private SpecialArtifact createDerivedTreeArtifactOnly(String path) {
nharmatab4060b62017-04-04 17:11:39 +0000387 PathFragment execPath = PathFragment.create("out").getRelative(path);
Rumou Duan73876202016-06-06 18:52:08 +0000388 return new SpecialArtifact(
Googlerf0b0c392021-01-27 17:56:52 -0800389 ArtifactRoot.asDerivedRoot(root, RootType.Output, "out"),
Googler147c2872021-01-21 08:47:48 -0800390 execPath,
391 ALL_OWNER,
392 SpecialArtifactType.TREE);
Rumou Duan73876202016-06-06 18:52:08 +0000393 }
394
ajurkowskie2982912020-04-09 10:32:08 -0700395 private static TreeFileArtifact createFakeTreeFileArtifact(
396 SpecialArtifact treeArtifact, String parentRelativePath, String content) throws Exception {
janakr45b308a2018-06-08 12:51:58 -0700397 TreeFileArtifact treeFileArtifact =
Googler1d8d1382020-05-18 12:10:49 -0700398 TreeFileArtifact.createTreeOutput(treeArtifact, parentRelativePath);
Rumou Duan73876202016-06-06 18:52:08 +0000399 Path path = treeFileArtifact.getPath();
Googler1d8d1382020-05-18 12:10:49 -0700400 path.getParentDirectory().createDirectoryAndParents();
401 writeFile(path, content);
402 return treeFileArtifact;
403 }
404
Googler3b2967d2020-05-18 14:54:46 -0700405 private TreeFileArtifact createFakeExpansionTreeFileArtifact(
406 ActionTemplate<?> actionTemplate, String parentRelativePath, String content)
407 throws Exception {
408 int actionIndex = Iterables.indexOf(actions, actionTemplate::equals);
409 Preconditions.checkState(actionIndex >= 0, "%s not registered", actionTemplate);
Googler1d8d1382020-05-18 12:10:49 -0700410 TreeFileArtifact treeFileArtifact =
411 TreeFileArtifact.createTemplateExpansionOutput(
Googler3b2967d2020-05-18 14:54:46 -0700412 actionTemplate.getOutputTreeArtifact(),
Googler1d8d1382020-05-18 12:10:49 -0700413 parentRelativePath,
Googler3b2967d2020-05-18 14:54:46 -0700414 ActionTemplateExpansionValue.key(ALL_OWNER, actionIndex));
Googler1d8d1382020-05-18 12:10:49 -0700415 Path path = treeFileArtifact.getPath();
416 path.getParentDirectory().createDirectoryAndParents();
Rumou Duan73876202016-06-06 18:52:08 +0000417 writeFile(path, content);
418 return treeFileArtifact;
419 }
420
ajurkowskie2982912020-04-09 10:32:08 -0700421 private static void assertValueMatches(FileStatus file, byte[] digest, FileArtifactValue value)
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000422 throws IOException {
lberkiaea56b32017-05-30 12:35:33 +0200423 assertThat(value.getSize()).isEqualTo(file.getSize());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000424 if (digest == null) {
lberkiaea56b32017-05-30 12:35:33 +0200425 assertThat(value.getDigest()).isNull();
426 assertThat(value.getModifiedTime()).isEqualTo(file.getLastModifiedTime());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000427 } else {
lberkiaea56b32017-05-30 12:35:33 +0200428 assertThat(value.getDigest()).isEqualTo(digest);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000429 }
430 }
431
432 private FileArtifactValue evaluateFAN(Artifact artifact) throws Throwable {
433 return ((FileArtifactValue) evaluateArtifactValue(artifact));
434 }
435
Janak Ramakrishnanad77f972016-07-29 20:58:42 +0000436 private SkyValue evaluateArtifactValue(Artifact artifact) throws Throwable {
lberki36df7ed2019-06-27 06:32:03 -0700437 SkyKey key = Artifact.key(artifact);
Janak Ramakrishnanad77f972016-07-29 20:58:42 +0000438 EvaluationResult<SkyValue> result = evaluate(ImmutableList.of(key).toArray(new SkyKey[0]));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000439 if (result.hasError()) {
440 throw result.getError().getException();
441 }
janakr8541f6d2019-06-11 14:40:21 -0700442 SkyValue value = result.get(key);
443 if (value instanceof ActionExecutionValue) {
Googleraed41602020-06-02 12:22:53 -0700444 return ((ActionExecutionValue) value).getExistingFileArtifactValue(artifact);
janakr8541f6d2019-06-11 14:40:21 -0700445 }
446 return value;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000447 }
448
janakr0175ce32018-02-26 15:54:57 -0800449 private void setGeneratingActions() throws InterruptedException, ActionConflictException {
janakr573807d2018-01-11 14:02:35 -0800450 if (evaluator.getExistingValue(ALL_OWNER) == null) {
janakr93e3eea2017-03-30 22:09:37 +0000451 differencer.inject(
452 ImmutableMap.of(
janakr573807d2018-01-11 14:02:35 -0800453 ALL_OWNER,
cparsonse2d200f2018-03-06 16:15:11 -0800454 new BasicActionLookupValue(
janakrefb3f152019-06-05 17:42:34 -0700455 Actions.assignOwnersAndFilterSharedActionsAndThrowActionConflict(
456 actionKeyContext,
457 ImmutableList.copyOf(actions),
458 ALL_OWNER,
Googlerce6ad292019-12-20 10:29:21 -0800459 /*outputFiles=*/ null))));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000460 }
461 }
462
463 private <E extends SkyValue> EvaluationResult<E> evaluate(SkyKey... keys)
janakr0175ce32018-02-26 15:54:57 -0800464 throws InterruptedException, ActionConflictException {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000465 setGeneratingActions();
Googler10028672018-10-25 12:14:34 -0700466 EvaluationContext evaluationContext =
467 EvaluationContext.newBuilder()
468 .setKeepGoing(false)
469 .setNumThreads(SkyframeExecutor.DEFAULT_THREAD_COUNT)
michajlo7a485be2020-07-30 11:08:46 -0700470 .setEventHandler(NullEventHandler.INSTANCE)
Googler10028672018-10-25 12:14:34 -0700471 .build();
472 return driver.evaluate(Arrays.asList(keys), evaluationContext);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000473 }
474
Googler3b2967d2020-05-18 14:54:46 -0700475 /**
476 * Value builder for actions that just stats and stores the output file (which must either be
477 * orphaned or exist).
478 */
479 private static final class SimpleActionExecutionFunction implements SkyFunction {
480 private final Set<Artifact> omittedOutputs;
481
482 SimpleActionExecutionFunction(Set<Artifact> omittedOutputs) {
483 this.omittedOutputs = omittedOutputs;
484 }
485
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000486 @Override
janakr93e3eea2017-03-30 22:09:37 +0000487 public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException {
lberkif7eee1e2019-07-31 05:49:10 -0700488 Map<Artifact, FileArtifactValue> artifactData = new HashMap<>();
Rumou Duan73876202016-06-06 18:52:08 +0000489 Map<Artifact, TreeArtifactValue> treeArtifactData = new HashMap<>();
janakr93e3eea2017-03-30 22:09:37 +0000490 ActionLookupData actionLookupData = (ActionLookupData) skyKey.argument();
491 ActionLookupValue actionLookupValue =
janakrbaf52ae2018-02-14 09:03:18 -0800492 (ActionLookupValue) env.getValue(actionLookupData.getActionLookupKey());
janakr93e3eea2017-03-30 22:09:37 +0000493 Action action = actionLookupValue.getAction(actionLookupData.getActionIndex());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000494 Artifact output = Iterables.getOnlyElement(action.getOutputs());
Rumou Duan73876202016-06-06 18:52:08 +0000495
496 try {
Googler3b2967d2020-05-18 14:54:46 -0700497 if (omittedOutputs.contains(output)) {
498 Preconditions.checkState(!output.isTreeArtifact(), "Cannot omit %s", output);
499 artifactData.put(output, FileArtifactValue.OMITTED_FILE_MARKER);
500 } else if (output.isTreeArtifact()) {
jhorvitzed7ec3b2020-07-24 15:08:03 -0700501 SpecialArtifact parent = (SpecialArtifact) output;
Googler1d8d1382020-05-18 12:10:49 -0700502 TreeFileArtifact treeFileArtifact1 =
503 TreeFileArtifact.createTreeOutput((SpecialArtifact) output, "child1");
504 TreeFileArtifact treeFileArtifact2 =
505 TreeFileArtifact.createTreeOutput((SpecialArtifact) output, "child2");
jhorvitzed7ec3b2020-07-24 15:08:03 -0700506 TreeArtifactValue tree =
507 TreeArtifactValue.newBuilder(parent)
508 .putChild(
509 treeFileArtifact1, FileArtifactValue.createForTesting(treeFileArtifact1))
510 .putChild(
511 treeFileArtifact2, FileArtifactValue.createForTesting(treeFileArtifact2))
512 .build();
513 treeArtifactData.put(output, tree);
Rumou Duan73876202016-06-06 18:52:08 +0000514 } else if (action.getActionType() == MiddlemanType.NORMAL) {
lberkif225af12019-08-01 04:21:58 -0700515 Path path = output.getPath();
516 FileArtifactValue noDigest =
517 ActionMetadataHandler.fileArtifactValueFromArtifact(
518 output,
519 FileStatusWithDigestAdapter.adapt(path.statIfFound(Symlinks.NOFOLLOW)),
520 null);
521 FileArtifactValue withDigest =
522 FileArtifactValue.createFromInjectedDigest(
523 noDigest, path.getDigest(), !output.isConstantMetadata());
524 artifactData.put(output, withDigest);
Rumou Duan73876202016-06-06 18:52:08 +0000525 } else {
lberkic35878a2019-08-01 02:28:54 -0700526 artifactData.put(output, FileArtifactValue.DEFAULT_MIDDLEMAN);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000527 }
Rumou Duan73876202016-06-06 18:52:08 +0000528 } catch (IOException e) {
529 throw new IllegalStateException(e);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000530 }
janakrb9d8d582018-06-13 21:57:19 -0700531 return ActionExecutionValue.create(
532 artifactData,
533 treeArtifactData,
janakrb9d8d582018-06-13 21:57:19 -0700534 /*outputSymlinks=*/ null,
shahanef6f4cf2018-06-26 11:24:59 -0700535 /*discoveredModules=*/ null,
janakr9f496f32018-10-24 15:08:09 -0700536 /*actionDependsOnBuildId=*/ false);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000537 }
538
539 @Override
540 public String extractTag(SkyKey skyKey) {
541 return null;
542 }
543 }
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000544}