blob: 5607782f7ee1ac1ff030ae40a020b5cf7d028705 [file] [log] [blame]
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001// Copyright 2015 The Bazel Authors. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package com.google.devtools.build.lib.skyframe;
Janak Ramakrishnana5578af2017-03-21 17:28:39 +000015
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000016import static com.google.common.truth.Truth.assertThat;
lberki812e6fe2019-07-25 07:50:55 -070017import static com.google.devtools.build.lib.actions.FileArtifactValue.createForTesting;
michajlo660d17f2020-03-27 09:01:57 -070018import static org.junit.Assert.assertThrows;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000019
Googler3b2967d2020-05-18 14:54:46 -070020import com.google.common.base.Preconditions;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000021import com.google.common.collect.ImmutableList;
22import com.google.common.collect.ImmutableMap;
23import com.google.common.collect.ImmutableSet;
24import com.google.common.collect.Iterables;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000025import com.google.devtools.build.lib.actions.Action;
Rumou Duan33bab462016-04-25 17:55:12 +000026import com.google.devtools.build.lib.actions.ActionAnalysisMetadata.MiddlemanType;
janakr93e3eea2017-03-30 22:09:37 +000027import com.google.devtools.build.lib.actions.ActionLookupData;
28import com.google.devtools.build.lib.actions.ActionLookupValue;
Googler3b2967d2020-05-18 14:54:46 -070029import com.google.devtools.build.lib.actions.ActionTemplate;
janakr0175ce32018-02-26 15:54:57 -080030import com.google.devtools.build.lib.actions.Actions;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000031import com.google.devtools.build.lib.actions.Artifact;
Googler72d648a2020-05-21 11:56:40 -070032import com.google.devtools.build.lib.actions.Artifact.DerivedArtifact;
Rumou Duan73876202016-06-06 18:52:08 +000033import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
34import com.google.devtools.build.lib.actions.Artifact.SpecialArtifactType;
35import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
tomlu1cdcdf92018-01-16 11:07:51 -080036import com.google.devtools.build.lib.actions.ArtifactRoot;
cparsonse2d200f2018-03-06 16:15:11 -080037import com.google.devtools.build.lib.actions.BasicActionLookupValue;
shahan602cc852018-06-06 20:09:57 -070038import com.google.devtools.build.lib.actions.FileArtifactValue;
janakre82933c2019-01-02 14:41:50 -080039import com.google.devtools.build.lib.actions.FilesetOutputSymlink;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000040import com.google.devtools.build.lib.actions.MissingInputFileException;
janakr0175ce32018-02-26 15:54:57 -080041import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException;
Rumou Duan73876202016-06-06 18:52:08 +000042import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000043import com.google.devtools.build.lib.actions.util.TestAction.DummyAction;
Googler3b2967d2020-05-18 14:54:46 -070044import com.google.devtools.build.lib.analysis.actions.SpawnActionTemplate;
ulfjack1e1a7752019-12-10 21:17:58 -080045import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
46import com.google.devtools.build.lib.collect.nestedset.Order;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000047import com.google.devtools.build.lib.events.NullEventHandler;
janakre82933c2019-01-02 14:41:50 -080048import com.google.devtools.build.lib.skyframe.serialization.testutils.SerializationTester;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000049import com.google.devtools.build.lib.util.Pair;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000050import com.google.devtools.build.lib.vfs.FileStatus;
lberkif225af12019-08-01 04:21:58 -070051import com.google.devtools.build.lib.vfs.FileStatusWithDigestAdapter;
janakre82933c2019-01-02 14:41:50 -080052import com.google.devtools.build.lib.vfs.FileSystem;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000053import com.google.devtools.build.lib.vfs.FileSystemUtils;
54import com.google.devtools.build.lib.vfs.Path;
55import com.google.devtools.build.lib.vfs.PathFragment;
tomluee6a6862018-01-17 14:36:26 -080056import com.google.devtools.build.lib.vfs.Root;
lberkif225af12019-08-01 04:21:58 -070057import com.google.devtools.build.lib.vfs.Symlinks;
Googler10028672018-10-25 12:14:34 -070058import com.google.devtools.build.skyframe.EvaluationContext;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000059import com.google.devtools.build.skyframe.EvaluationResult;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000060import com.google.devtools.build.skyframe.SkyFunction;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000061import com.google.devtools.build.skyframe.SkyKey;
62import com.google.devtools.build.skyframe.SkyValue;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000063import java.io.IOException;
buchgrd4d3d502018-08-02 06:47:19 -070064import java.util.ArrayList;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000065import java.util.Arrays;
66import java.util.HashMap;
Googler3b2967d2020-05-18 14:54:46 -070067import java.util.HashSet;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000068import java.util.Map;
Googler3b2967d2020-05-18 14:54:46 -070069import java.util.Set;
Janak Ramakrishnanad77f972016-07-29 20:58:42 +000070import org.junit.Before;
71import org.junit.Test;
72import org.junit.runner.RunWith;
73import org.junit.runners.JUnit4;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000074
75/**
76 * Tests for {@link ArtifactFunction}.
77 */
78// Doesn't actually need any particular Skyframe, but is only relevant to Skyframe full mode.
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +000079@RunWith(JUnit4.class)
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000080public class ArtifactFunctionTest extends ArtifactFunctionTestCase {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000081
Googler3b2967d2020-05-18 14:54:46 -070082 private final Set<Artifact> omittedOutputs = new HashSet<>();
83
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +000084 @Before
Googler3b2967d2020-05-18 14:54:46 -070085 public final void setUp() {
86 delegateActionExecutionFunction = new SimpleActionExecutionFunction(omittedOutputs);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000087 }
88
89 private void assertFileArtifactValueMatches(boolean expectDigest) throws Throwable {
90 Artifact output = createDerivedArtifact("output");
91 Path path = output.getPath();
92 file(path, "contents");
olaolabfd1d332017-06-19 16:55:24 -040093 assertValueMatches(path.stat(), expectDigest ? path.getDigest() : null, evaluateFAN(output));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000094 }
95
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +000096 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000097 public void testBasicArtifact() throws Throwable {
98 fastDigest = false;
99 assertFileArtifactValueMatches(/*expectDigest=*/ true);
100 }
101
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000102 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000103 public void testBasicArtifactWithXattr() throws Throwable {
104 fastDigest = true;
105 assertFileArtifactValueMatches(/*expectDigest=*/ true);
106 }
107
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000108 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000109 public void testMissingNonMandatoryArtifact() throws Throwable {
110 Artifact input = createSourceArtifact("input1");
lberki36df7ed2019-06-27 06:32:03 -0700111 assertThat(evaluateArtifactValue(input)).isNotNull();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000112 }
113
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000114 @Test
Michajlo Matijkiw528957e2016-01-19 21:17:45 +0000115 public void testUnreadableInputWithFsWithAvailableDigest() throws Throwable {
Googlere764d562019-10-16 10:56:33 -0700116 final byte[] expectedDigest = {1, 2, 3, 4};
Michajlo Matijkiw528957e2016-01-19 21:17:45 +0000117 setupRoot(
118 new CustomInMemoryFs() {
119 @Override
ccalvarindd9f60e2018-07-23 18:16:18 -0700120 public byte[] getDigest(Path path) throws IOException {
121 return path.getBaseName().equals("unreadable") ? expectedDigest : super.getDigest(path);
Michajlo Matijkiw528957e2016-01-19 21:17:45 +0000122 }
123 });
124
125 Artifact input = createSourceArtifact("unreadable");
126 Path inputPath = input.getPath();
127 file(inputPath, "dummynotused");
128 inputPath.chmod(0);
129
lberki36df7ed2019-06-27 06:32:03 -0700130 FileArtifactValue value = (FileArtifactValue) evaluateArtifactValue(input);
Michajlo Matijkiw528957e2016-01-19 21:17:45 +0000131
132 FileStatus stat = inputPath.stat();
133 assertThat(value.getSize()).isEqualTo(stat.getSize());
134 assertThat(value.getDigest()).isEqualTo(expectedDigest);
135 }
136
137 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000138 public void testMiddlemanArtifact() throws Throwable {
Janak Ramakrishnana5578af2017-03-21 17:28:39 +0000139 Artifact output = createMiddlemanArtifact("output");
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000140 Artifact input1 = createSourceArtifact("input1");
141 Artifact input2 = createDerivedArtifact("input2");
Benjamin Peterson63748e42018-06-03 22:11:16 -0700142 SpecialArtifact tree = createDerivedTreeArtifactWithAction("treeArtifact");
buchgrd4d3d502018-08-02 06:47:19 -0700143 TreeFileArtifact treeFile1 = createFakeTreeFileArtifact(tree, "child1", "hello1");
144 TreeFileArtifact treeFile2 = createFakeTreeFileArtifact(tree, "child2", "hello2");
145 file(treeFile1.getPath(), "src1");
146 file(treeFile2.getPath(), "src2");
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000147 Action action =
148 new DummyAction(
ulfjack1e1a7752019-12-10 21:17:58 -0800149 NestedSetBuilder.create(Order.STABLE_ORDER, input1, input2, tree),
150 output,
151 MiddlemanType.AGGREGATING_MIDDLEMAN);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000152 actions.add(action);
153 file(input2.getPath(), "contents");
154 file(input1.getPath(), "source contents");
janakr8541f6d2019-06-11 14:40:21 -0700155 evaluate(
156 Iterables.toArray(
lberki36df7ed2019-06-27 06:32:03 -0700157 Artifact.keys(ImmutableSet.of(input2, input1, input2, tree)), SkyKey.class));
Janak Ramakrishnanad77f972016-07-29 20:58:42 +0000158 SkyValue value = evaluateArtifactValue(output);
buchgrd4d3d502018-08-02 06:47:19 -0700159 ArrayList<Pair<Artifact, ?>> inputs = new ArrayList<>();
160 inputs.addAll(((AggregatingArtifactValue) value).getFileArtifacts());
161 inputs.addAll(((AggregatingArtifactValue) value).getTreeArtifacts());
162 assertThat(inputs)
Benjamin Peterson63748e42018-06-03 22:11:16 -0700163 .containsExactly(
lberki812e6fe2019-07-25 07:50:55 -0700164 Pair.of(input1, createForTesting(input1)),
165 Pair.of(input2, createForTesting(input2)),
buchgrd4d3d502018-08-02 06:47:19 -0700166 Pair.of(tree, ((TreeArtifactValue) evaluateArtifactValue(tree))));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000167 }
168
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000169 /**
170 * Tests that ArtifactFunction rethrows transitive {@link IOException}s as
171 * {@link MissingInputFileException}s.
172 */
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000173 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000174 public void testIOException_EndToEnd() throws Throwable {
175 final IOException exception = new IOException("beep");
176 setupRoot(
177 new CustomInMemoryFs() {
178 @Override
fellya205ed82018-09-10 11:52:34 -0700179 public FileStatus statIfFound(Path path, boolean followSymlinks) throws IOException {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000180 if (path.getBaseName().equals("bad")) {
181 throw exception;
182 }
fellya205ed82018-09-10 11:52:34 -0700183 return super.statIfFound(path, followSymlinks);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000184 }
185 });
lberki36df7ed2019-06-27 06:32:03 -0700186 IOException e =
187 assertThrows(IOException.class, () -> evaluateArtifactValue(createSourceArtifact("bad")));
jcater83130f42019-04-30 14:29:28 -0700188 assertThat(e).hasMessageThat().contains(exception.getMessage());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000189 }
190
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000191 @Test
Rumou Duan73876202016-06-06 18:52:08 +0000192 public void testActionTreeArtifactOutput() throws Throwable {
cpeyserac09f0a2018-02-05 09:33:15 -0800193 SpecialArtifact artifact = createDerivedTreeArtifactWithAction("treeArtifact");
janakrefb3f152019-06-05 17:42:34 -0700194 TreeFileArtifact treeFileArtifact1 = createFakeTreeFileArtifact(artifact, "child1", "hello1");
195 TreeFileArtifact treeFileArtifact2 = createFakeTreeFileArtifact(artifact, "child2", "hello2");
Rumou Duan73876202016-06-06 18:52:08 +0000196
197 TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact);
janakr45b308a2018-06-08 12:51:58 -0700198 assertThat(value.getChildValues()).containsKey(treeFileArtifact1);
199 assertThat(value.getChildValues()).containsKey(treeFileArtifact2);
lberkiaea56b32017-05-30 12:35:33 +0200200 assertThat(value.getChildValues().get(treeFileArtifact1).getDigest()).isNotNull();
201 assertThat(value.getChildValues().get(treeFileArtifact2).getDigest()).isNotNull();
Rumou Duan73876202016-06-06 18:52:08 +0000202 }
203
204 @Test
205 public void testSpawnActionTemplate() throws Throwable {
206 // artifact1 is a tree artifact generated by normal action.
cpeyserac09f0a2018-02-05 09:33:15 -0800207 SpecialArtifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1");
Rumou Duan73876202016-06-06 18:52:08 +0000208 createFakeTreeFileArtifact(artifact1, "child1", "hello1");
209 createFakeTreeFileArtifact(artifact1, "child2", "hello2");
210
Rumou Duan73876202016-06-06 18:52:08 +0000211 // artifact2 is a tree artifact generated by action template.
cpeyserac09f0a2018-02-05 09:33:15 -0800212 SpecialArtifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2");
Googler3b2967d2020-05-18 14:54:46 -0700213 SpawnActionTemplate actionTemplate =
214 ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
215 actions.add(actionTemplate);
janakr45b308a2018-06-08 12:51:58 -0700216 TreeFileArtifact treeFileArtifact1 =
Googler3b2967d2020-05-18 14:54:46 -0700217 createFakeExpansionTreeFileArtifact(actionTemplate, "child1", "hello1");
janakr45b308a2018-06-08 12:51:58 -0700218 TreeFileArtifact treeFileArtifact2 =
Googler3b2967d2020-05-18 14:54:46 -0700219 createFakeExpansionTreeFileArtifact(actionTemplate, "child2", "hello2");
Rumou Duan73876202016-06-06 18:52:08 +0000220
221 TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact2);
janakr45b308a2018-06-08 12:51:58 -0700222 assertThat(value.getChildValues()).containsKey(treeFileArtifact1);
223 assertThat(value.getChildValues()).containsKey(treeFileArtifact2);
lberkiaea56b32017-05-30 12:35:33 +0200224 assertThat(value.getChildValues().get(treeFileArtifact1).getDigest()).isNotNull();
225 assertThat(value.getChildValues().get(treeFileArtifact2).getDigest()).isNotNull();
Rumou Duan73876202016-06-06 18:52:08 +0000226 }
227
228 @Test
229 public void testConsecutiveSpawnActionTemplates() throws Throwable {
230 // artifact1 is a tree artifact generated by normal action.
cpeyserac09f0a2018-02-05 09:33:15 -0800231 SpecialArtifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1");
Rumou Duan73876202016-06-06 18:52:08 +0000232 createFakeTreeFileArtifact(artifact1, "child1", "hello1");
233 createFakeTreeFileArtifact(artifact1, "child2", "hello2");
234
235 // artifact2 is a tree artifact generated by action template.
cpeyserac09f0a2018-02-05 09:33:15 -0800236 SpecialArtifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2");
Googler3b2967d2020-05-18 14:54:46 -0700237 SpawnActionTemplate template2 =
238 ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
239 actions.add(template2);
240 createFakeExpansionTreeFileArtifact(template2, "child1", "hello1");
241 createFakeExpansionTreeFileArtifact(template2, "child2", "hello2");
Rumou Duan73876202016-06-06 18:52:08 +0000242
243 // artifact3 is a tree artifact generated by action template.
cpeyserac09f0a2018-02-05 09:33:15 -0800244 SpecialArtifact artifact3 = createDerivedTreeArtifactOnly("treeArtifact3");
Googler3b2967d2020-05-18 14:54:46 -0700245 SpawnActionTemplate template3 =
246 ActionsTestUtil.createDummySpawnActionTemplate(artifact2, artifact3);
247 actions.add(template3);
janakr45b308a2018-06-08 12:51:58 -0700248 TreeFileArtifact treeFileArtifact1 =
Googler3b2967d2020-05-18 14:54:46 -0700249 createFakeExpansionTreeFileArtifact(template3, "child1", "hello1");
janakr45b308a2018-06-08 12:51:58 -0700250 TreeFileArtifact treeFileArtifact2 =
Googler3b2967d2020-05-18 14:54:46 -0700251 createFakeExpansionTreeFileArtifact(template3, "child2", "hello2");
Rumou Duan73876202016-06-06 18:52:08 +0000252
253 TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact3);
janakr45b308a2018-06-08 12:51:58 -0700254 assertThat(value.getChildValues()).containsKey(treeFileArtifact1);
255 assertThat(value.getChildValues()).containsKey(treeFileArtifact2);
lberkiaea56b32017-05-30 12:35:33 +0200256 assertThat(value.getChildValues().get(treeFileArtifact1).getDigest()).isNotNull();
257 assertThat(value.getChildValues().get(treeFileArtifact2).getDigest()).isNotNull();
Rumou Duan73876202016-06-06 18:52:08 +0000258 }
259
janakre82933c2019-01-02 14:41:50 -0800260 @Test
Googler3b2967d2020-05-18 14:54:46 -0700261 public void actionTemplateExpansionOutputsOmitted() throws Throwable {
262 // artifact1 is a tree artifact generated by normal action.
263 SpecialArtifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1");
264 createFakeTreeFileArtifact(artifact1, "child1", "hello1");
265 createFakeTreeFileArtifact(artifact1, "child2", "hello2");
266
267 // artifact2 is a tree artifact generated by action template.
268 SpecialArtifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2");
269 SpawnActionTemplate actionTemplate =
270 ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
271 actions.add(actionTemplate);
272 TreeFileArtifact treeFileArtifact1 =
273 createFakeExpansionTreeFileArtifact(actionTemplate, "child1", "hello1");
274 TreeFileArtifact treeFileArtifact2 =
275 createFakeExpansionTreeFileArtifact(actionTemplate, "child2", "hello2");
276
277 omittedOutputs.add(treeFileArtifact1);
278 omittedOutputs.add(treeFileArtifact2);
279
280 SkyValue value = evaluateArtifactValue(artifact2);
Googlera98fc732020-05-25 12:18:56 -0700281 assertThat(value).isEqualTo(TreeArtifactValue.OMITTED_TREE_MARKER);
Googler3b2967d2020-05-18 14:54:46 -0700282 }
283
284 @Test
285 public void cannotOmitSomeButNotAllActionTemplateExpansionOutputs() throws Throwable {
286 // artifact1 is a tree artifact generated by normal action.
287 SpecialArtifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1");
288 createFakeTreeFileArtifact(artifact1, "child1", "hello1");
289 createFakeTreeFileArtifact(artifact1, "child2", "hello2");
290
291 // artifact2 is a tree artifact generated by action template.
292 SpecialArtifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2");
293 SpawnActionTemplate actionTemplate =
294 ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2);
295 actions.add(actionTemplate);
296 TreeFileArtifact treeFileArtifact1 =
297 createFakeExpansionTreeFileArtifact(actionTemplate, "child1", "hello1");
298 TreeFileArtifact treeFileArtifact2 =
299 createFakeExpansionTreeFileArtifact(actionTemplate, "child2", "hello2");
300
301 omittedOutputs.add(treeFileArtifact1);
302
303 Exception e = assertThrows(RuntimeException.class, () -> evaluateArtifactValue(artifact2));
304 assertThat(e).hasCauseThat().isInstanceOf(IllegalStateException.class);
305 assertThat(e)
306 .hasCauseThat()
307 .hasMessageThat()
308 .matches(
309 "Action template expansion has some but not all outputs omitted, present outputs: .*"
310 + treeFileArtifact2.getParentRelativePath()
311 + ".*");
312 }
313
314 @Test
janakre82933c2019-01-02 14:41:50 -0800315 public void actionExecutionValueSerialization() throws Exception {
janakrefb3f152019-06-05 17:42:34 -0700316 ActionLookupData dummyData = ActionLookupData.create(ALL_OWNER, 0);
Googler72d648a2020-05-21 11:56:40 -0700317 DerivedArtifact artifact1 = createDerivedArtifact("one");
lberkif7eee1e2019-07-31 05:49:10 -0700318 FileArtifactValue metadata1 =
319 ActionMetadataHandler.fileArtifactValueFromArtifact(artifact1, null, null);
janakre82933c2019-01-02 14:41:50 -0800320 SpecialArtifact treeArtifact = createDerivedTreeArtifactOnly("tree");
janakrefb3f152019-06-05 17:42:34 -0700321 treeArtifact.setGeneratingActionKey(dummyData);
Googler1d8d1382020-05-18 12:10:49 -0700322 TreeFileArtifact treeFileArtifact = TreeFileArtifact.createTreeOutput(treeArtifact, "subpath");
janakrefb3f152019-06-05 17:42:34 -0700323 Path path = treeFileArtifact.getPath();
Googler1d8d1382020-05-18 12:10:49 -0700324 path.getParentDirectory().createDirectoryAndParents();
janakrefb3f152019-06-05 17:42:34 -0700325 writeFile(path, "contents");
janakre82933c2019-01-02 14:41:50 -0800326 TreeArtifactValue treeArtifactValue =
327 TreeArtifactValue.create(
lberki812e6fe2019-07-25 07:50:55 -0700328 ImmutableMap.of(
329 treeFileArtifact, FileArtifactValue.createForTesting(treeFileArtifact)));
Googler72d648a2020-05-21 11:56:40 -0700330 DerivedArtifact artifact3 = createDerivedArtifact("three");
janakre82933c2019-01-02 14:41:50 -0800331 FilesetOutputSymlink filesetOutputSymlink =
332 FilesetOutputSymlink.createForTesting(
333 PathFragment.EMPTY_FRAGMENT, PathFragment.EMPTY_FRAGMENT, PathFragment.EMPTY_FRAGMENT);
334 ActionExecutionValue actionExecutionValue =
335 ActionExecutionValue.create(
lberkic35878a2019-08-01 02:28:54 -0700336 ImmutableMap.of(artifact1, metadata1, artifact3, FileArtifactValue.DEFAULT_MIDDLEMAN),
janakre82933c2019-01-02 14:41:50 -0800337 ImmutableMap.of(treeArtifact, treeArtifactValue),
janakre82933c2019-01-02 14:41:50 -0800338 ImmutableList.of(filesetOutputSymlink),
339 null,
340 true);
Googlerce6ad292019-12-20 10:29:21 -0800341 new SerializationTester(actionExecutionValue)
janakre82933c2019-01-02 14:41:50 -0800342 .addDependency(FileSystem.class, root.getFileSystem())
nharmataf93c72c2020-03-13 14:22:10 -0700343 .addDependency(
344 Root.RootCodecDependencies.class,
345 new Root.RootCodecDependencies(Root.absoluteRoot(root.getFileSystem())))
janakre82933c2019-01-02 14:41:50 -0800346 .runTests();
347 }
348
ajurkowskie2982912020-04-09 10:32:08 -0700349 private static void file(Path path, String contents) throws Exception {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000350 FileSystemUtils.createDirectoryAndParents(path.getParentDirectory());
351 writeFile(path, contents);
352 }
353
354 private Artifact createSourceArtifact(String path) {
janakraea05602019-05-22 15:41:29 -0700355 return ActionsTestUtil.createArtifactWithExecPath(
356 ArtifactRoot.asSourceRoot(Root.fromPath(root)), PathFragment.create(path));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000357 }
358
Googler72d648a2020-05-21 11:56:40 -0700359 private DerivedArtifact createDerivedArtifact(String path) {
nharmatab4060b62017-04-04 17:11:39 +0000360 PathFragment execPath = PathFragment.create("out").getRelative(path);
Googler72d648a2020-05-21 11:56:40 -0700361 DerivedArtifact output =
362 new DerivedArtifact(ArtifactRoot.asDerivedRoot(root, "out"), execPath, ALL_OWNER);
ulfjack1e1a7752019-12-10 21:17:58 -0800363 actions.add(new DummyAction(NestedSetBuilder.emptySet(Order.STABLE_ORDER), output));
janakr8541f6d2019-06-11 14:40:21 -0700364 output.setGeneratingActionKey(ActionLookupData.create(ALL_OWNER, actions.size() - 1));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000365 return output;
366 }
367
Janak Ramakrishnana5578af2017-03-21 17:28:39 +0000368 private Artifact createMiddlemanArtifact(String path) {
tomlu1cdcdf92018-01-16 11:07:51 -0800369 ArtifactRoot middlemanRoot =
370 ArtifactRoot.middlemanRoot(middlemanPath, middlemanPath.getRelative("out"));
Googler72d648a2020-05-21 11:56:40 -0700371 return new DerivedArtifact(
janakr3290e222019-05-29 16:34:22 -0700372 middlemanRoot, middlemanRoot.getExecPath().getRelative(path), ALL_OWNER);
Janak Ramakrishnana5578af2017-03-21 17:28:39 +0000373 }
374
cpeyserac09f0a2018-02-05 09:33:15 -0800375 private SpecialArtifact createDerivedTreeArtifactWithAction(String path) {
376 SpecialArtifact treeArtifact = createDerivedTreeArtifactOnly(path);
ulfjack1e1a7752019-12-10 21:17:58 -0800377 actions.add(new DummyAction(NestedSetBuilder.emptySet(Order.STABLE_ORDER), treeArtifact));
Googler1d8d1382020-05-18 12:10:49 -0700378 treeArtifact.setGeneratingActionKey(ActionLookupData.create(ALL_OWNER, actions.size() - 1));
Rumou Duan73876202016-06-06 18:52:08 +0000379 return treeArtifact;
380 }
381
cpeyserac09f0a2018-02-05 09:33:15 -0800382 private SpecialArtifact createDerivedTreeArtifactOnly(String path) {
nharmatab4060b62017-04-04 17:11:39 +0000383 PathFragment execPath = PathFragment.create("out").getRelative(path);
Rumou Duan73876202016-06-06 18:52:08 +0000384 return new SpecialArtifact(
janakr448f1cf2020-03-30 09:12:44 -0700385 ArtifactRoot.asDerivedRoot(root, "out"), execPath, ALL_OWNER, SpecialArtifactType.TREE);
Rumou Duan73876202016-06-06 18:52:08 +0000386 }
387
ajurkowskie2982912020-04-09 10:32:08 -0700388 private static TreeFileArtifact createFakeTreeFileArtifact(
389 SpecialArtifact treeArtifact, String parentRelativePath, String content) throws Exception {
janakr45b308a2018-06-08 12:51:58 -0700390 TreeFileArtifact treeFileArtifact =
Googler1d8d1382020-05-18 12:10:49 -0700391 TreeFileArtifact.createTreeOutput(treeArtifact, parentRelativePath);
Rumou Duan73876202016-06-06 18:52:08 +0000392 Path path = treeFileArtifact.getPath();
Googler1d8d1382020-05-18 12:10:49 -0700393 path.getParentDirectory().createDirectoryAndParents();
394 writeFile(path, content);
395 return treeFileArtifact;
396 }
397
Googler3b2967d2020-05-18 14:54:46 -0700398 private TreeFileArtifact createFakeExpansionTreeFileArtifact(
399 ActionTemplate<?> actionTemplate, String parentRelativePath, String content)
400 throws Exception {
401 int actionIndex = Iterables.indexOf(actions, actionTemplate::equals);
402 Preconditions.checkState(actionIndex >= 0, "%s not registered", actionTemplate);
Googler1d8d1382020-05-18 12:10:49 -0700403 TreeFileArtifact treeFileArtifact =
404 TreeFileArtifact.createTemplateExpansionOutput(
Googler3b2967d2020-05-18 14:54:46 -0700405 actionTemplate.getOutputTreeArtifact(),
Googler1d8d1382020-05-18 12:10:49 -0700406 parentRelativePath,
Googler3b2967d2020-05-18 14:54:46 -0700407 ActionTemplateExpansionValue.key(ALL_OWNER, actionIndex));
Googler1d8d1382020-05-18 12:10:49 -0700408 Path path = treeFileArtifact.getPath();
409 path.getParentDirectory().createDirectoryAndParents();
Rumou Duan73876202016-06-06 18:52:08 +0000410 writeFile(path, content);
411 return treeFileArtifact;
412 }
413
ajurkowskie2982912020-04-09 10:32:08 -0700414 private static void assertValueMatches(FileStatus file, byte[] digest, FileArtifactValue value)
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000415 throws IOException {
lberkiaea56b32017-05-30 12:35:33 +0200416 assertThat(value.getSize()).isEqualTo(file.getSize());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000417 if (digest == null) {
lberkiaea56b32017-05-30 12:35:33 +0200418 assertThat(value.getDigest()).isNull();
419 assertThat(value.getModifiedTime()).isEqualTo(file.getLastModifiedTime());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000420 } else {
lberkiaea56b32017-05-30 12:35:33 +0200421 assertThat(value.getDigest()).isEqualTo(digest);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000422 }
423 }
424
425 private FileArtifactValue evaluateFAN(Artifact artifact) throws Throwable {
426 return ((FileArtifactValue) evaluateArtifactValue(artifact));
427 }
428
Janak Ramakrishnanad77f972016-07-29 20:58:42 +0000429 private SkyValue evaluateArtifactValue(Artifact artifact) throws Throwable {
lberki36df7ed2019-06-27 06:32:03 -0700430 SkyKey key = Artifact.key(artifact);
Janak Ramakrishnanad77f972016-07-29 20:58:42 +0000431 EvaluationResult<SkyValue> result = evaluate(ImmutableList.of(key).toArray(new SkyKey[0]));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000432 if (result.hasError()) {
433 throw result.getError().getException();
434 }
janakr8541f6d2019-06-11 14:40:21 -0700435 SkyValue value = result.get(key);
436 if (value instanceof ActionExecutionValue) {
Googler72d648a2020-05-21 11:56:40 -0700437 return ((ActionExecutionValue) value)
438 .getExistingFileArtifactValue((DerivedArtifact) artifact);
janakr8541f6d2019-06-11 14:40:21 -0700439 }
440 return value;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000441 }
442
janakr0175ce32018-02-26 15:54:57 -0800443 private void setGeneratingActions() throws InterruptedException, ActionConflictException {
janakr573807d2018-01-11 14:02:35 -0800444 if (evaluator.getExistingValue(ALL_OWNER) == null) {
janakr93e3eea2017-03-30 22:09:37 +0000445 differencer.inject(
446 ImmutableMap.of(
janakr573807d2018-01-11 14:02:35 -0800447 ALL_OWNER,
cparsonse2d200f2018-03-06 16:15:11 -0800448 new BasicActionLookupValue(
janakrefb3f152019-06-05 17:42:34 -0700449 Actions.assignOwnersAndFilterSharedActionsAndThrowActionConflict(
450 actionKeyContext,
451 ImmutableList.copyOf(actions),
452 ALL_OWNER,
Googlerce6ad292019-12-20 10:29:21 -0800453 /*outputFiles=*/ null))));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000454 }
455 }
456
457 private <E extends SkyValue> EvaluationResult<E> evaluate(SkyKey... keys)
janakr0175ce32018-02-26 15:54:57 -0800458 throws InterruptedException, ActionConflictException {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000459 setGeneratingActions();
Googler10028672018-10-25 12:14:34 -0700460 EvaluationContext evaluationContext =
461 EvaluationContext.newBuilder()
462 .setKeepGoing(false)
463 .setNumThreads(SkyframeExecutor.DEFAULT_THREAD_COUNT)
464 .setEventHander(NullEventHandler.INSTANCE)
465 .build();
466 return driver.evaluate(Arrays.asList(keys), evaluationContext);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000467 }
468
Googler3b2967d2020-05-18 14:54:46 -0700469 /**
470 * Value builder for actions that just stats and stores the output file (which must either be
471 * orphaned or exist).
472 */
473 private static final class SimpleActionExecutionFunction implements SkyFunction {
474 private final Set<Artifact> omittedOutputs;
475
476 SimpleActionExecutionFunction(Set<Artifact> omittedOutputs) {
477 this.omittedOutputs = omittedOutputs;
478 }
479
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000480 @Override
janakr93e3eea2017-03-30 22:09:37 +0000481 public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException {
lberkif7eee1e2019-07-31 05:49:10 -0700482 Map<Artifact, FileArtifactValue> artifactData = new HashMap<>();
Rumou Duan73876202016-06-06 18:52:08 +0000483 Map<Artifact, TreeArtifactValue> treeArtifactData = new HashMap<>();
janakr93e3eea2017-03-30 22:09:37 +0000484 ActionLookupData actionLookupData = (ActionLookupData) skyKey.argument();
485 ActionLookupValue actionLookupValue =
janakrbaf52ae2018-02-14 09:03:18 -0800486 (ActionLookupValue) env.getValue(actionLookupData.getActionLookupKey());
janakr93e3eea2017-03-30 22:09:37 +0000487 Action action = actionLookupValue.getAction(actionLookupData.getActionIndex());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000488 Artifact output = Iterables.getOnlyElement(action.getOutputs());
Rumou Duan73876202016-06-06 18:52:08 +0000489
490 try {
Googler3b2967d2020-05-18 14:54:46 -0700491 if (omittedOutputs.contains(output)) {
492 Preconditions.checkState(!output.isTreeArtifact(), "Cannot omit %s", output);
493 artifactData.put(output, FileArtifactValue.OMITTED_FILE_MARKER);
494 } else if (output.isTreeArtifact()) {
Googler1d8d1382020-05-18 12:10:49 -0700495 TreeFileArtifact treeFileArtifact1 =
496 TreeFileArtifact.createTreeOutput((SpecialArtifact) output, "child1");
497 TreeFileArtifact treeFileArtifact2 =
498 TreeFileArtifact.createTreeOutput((SpecialArtifact) output, "child2");
lberki812e6fe2019-07-25 07:50:55 -0700499 TreeArtifactValue treeArtifactValue =
500 TreeArtifactValue.create(
501 ImmutableMap.of(
502 treeFileArtifact1, FileArtifactValue.createForTesting(treeFileArtifact1),
503 treeFileArtifact2, FileArtifactValue.createForTesting(treeFileArtifact2)));
Rumou Duan73876202016-06-06 18:52:08 +0000504 treeArtifactData.put(output, treeArtifactValue);
505 } else if (action.getActionType() == MiddlemanType.NORMAL) {
lberkif225af12019-08-01 04:21:58 -0700506 Path path = output.getPath();
507 FileArtifactValue noDigest =
508 ActionMetadataHandler.fileArtifactValueFromArtifact(
509 output,
510 FileStatusWithDigestAdapter.adapt(path.statIfFound(Symlinks.NOFOLLOW)),
511 null);
512 FileArtifactValue withDigest =
513 FileArtifactValue.createFromInjectedDigest(
514 noDigest, path.getDigest(), !output.isConstantMetadata());
515 artifactData.put(output, withDigest);
Rumou Duan73876202016-06-06 18:52:08 +0000516 } else {
lberkic35878a2019-08-01 02:28:54 -0700517 artifactData.put(output, FileArtifactValue.DEFAULT_MIDDLEMAN);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000518 }
Rumou Duan73876202016-06-06 18:52:08 +0000519 } catch (IOException e) {
520 throw new IllegalStateException(e);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000521 }
janakrb9d8d582018-06-13 21:57:19 -0700522 return ActionExecutionValue.create(
523 artifactData,
524 treeArtifactData,
janakrb9d8d582018-06-13 21:57:19 -0700525 /*outputSymlinks=*/ null,
shahanef6f4cf2018-06-26 11:24:59 -0700526 /*discoveredModules=*/ null,
janakr9f496f32018-10-24 15:08:09 -0700527 /*actionDependsOnBuildId=*/ false);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000528 }
529
530 @Override
531 public String extractTag(SkyKey skyKey) {
532 return null;
533 }
534 }
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000535}