blob: cc9bbae44c1923a9764256b0f0a7d85f9d0fbadd [file] [log] [blame]
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001// Copyright 2015 The Bazel Authors. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package com.google.devtools.build.lib.skyframe;
15
Googlerfe72dbb2021-12-29 07:12:53 -080016import static com.google.common.collect.ImmutableList.toImmutableList;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000017import static com.google.common.truth.Truth.assertThat;
lberkiaea56b32017-05-30 12:35:33 +020018import static com.google.common.truth.Truth.assertWithMessage;
janakr933c3012021-07-13 07:44:15 -070019import static java.util.concurrent.TimeUnit.SECONDS;
ajurkowski64795c22022-04-01 10:38:44 -070020import static org.mockito.ArgumentMatchers.anyLong;
21import static org.mockito.Mockito.doNothing;
22import static org.mockito.Mockito.mock;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000023
janakrfc1d79c2022-01-27 13:02:07 -080024import com.google.common.base.Suppliers;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000025import com.google.common.collect.ImmutableList;
26import com.google.common.collect.ImmutableMap;
27import com.google.common.collect.ImmutableSet;
buchgr4992ae22019-03-20 04:23:32 -070028import com.google.common.hash.HashCode;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000029import com.google.common.util.concurrent.Runnables;
30import com.google.devtools.build.lib.actions.Action;
janakrefb3f152019-06-05 17:42:34 -070031import com.google.devtools.build.lib.actions.ActionLookupData;
Googlerfe72dbb2021-12-29 07:12:53 -080032import com.google.devtools.build.lib.actions.ActionLookupKey;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000033import com.google.devtools.build.lib.actions.Artifact;
Googlerfe72dbb2021-12-29 07:12:53 -080034import com.google.devtools.build.lib.actions.Artifact.ArchivedTreeArtifact;
Michael Thvedte4a7b0792016-02-09 12:15:53 +000035import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
Rumou Duana77f32c2016-04-13 21:59:21 +000036import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
tomlu1cdcdf92018-01-16 11:07:51 -080037import com.google.devtools.build.lib.actions.ArtifactRoot;
Googlerf0b0c392021-01-27 17:56:52 -080038import com.google.devtools.build.lib.actions.ArtifactRoot.RootType;
shahan602cc852018-06-06 20:09:57 -070039import com.google.devtools.build.lib.actions.FileArtifactValue;
buchgr4992ae22019-03-20 04:23:32 -070040import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue;
shahan602cc852018-06-06 20:09:57 -070041import com.google.devtools.build.lib.actions.FileStateValue;
42import com.google.devtools.build.lib.actions.FileValue;
janakr13b737a2021-07-02 14:24:25 -070043import com.google.devtools.build.lib.actions.ThreadStateReceiver;
janakraea05602019-05-22 15:41:29 -070044import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000045import com.google.devtools.build.lib.actions.util.TestAction;
Kristina Chodorowf9fdc8d2015-12-08 12:49:31 +000046import com.google.devtools.build.lib.analysis.BlazeDirectories;
janakr3b63a4e2017-09-14 09:55:40 +020047import com.google.devtools.build.lib.analysis.ServerDirectories;
lberki41cfcc72019-08-05 02:55:36 -070048import com.google.devtools.build.lib.clock.BlazeClock;
Googlerfe72dbb2021-12-29 07:12:53 -080049import com.google.devtools.build.lib.cmdline.Label;
ulfjack1e1a7752019-12-10 21:17:58 -080050import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
51import com.google.devtools.build.lib.collect.nestedset.Order;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000052import com.google.devtools.build.lib.events.NullEventHandler;
janakre2af68f2021-03-18 15:11:30 -070053import com.google.devtools.build.lib.io.FileSymlinkCycleUniquenessFunction;
54import com.google.devtools.build.lib.io.FileSymlinkInfiniteExpansionUniquenessFunction;
janakr15e15c22019-01-30 11:24:49 -080055import com.google.devtools.build.lib.packages.WorkspaceFileValue;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000056import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
57import com.google.devtools.build.lib.skyframe.DirtinessCheckerUtils.BasicFilesystemDirtinessChecker;
Nathan Harmatad4f75942016-10-18 08:55:17 +000058import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction;
ajurkowski64795c22022-04-01 10:38:44 -070059import com.google.devtools.build.lib.skyframe.FilesystemValueChecker.ModifiedOutputsReceiver;
jhorvitz1d675d12022-04-28 16:23:21 -070060import com.google.devtools.build.lib.skyframe.PackageFunction.GlobbingStrategy;
John Cater5e9ce942016-10-12 17:23:30 +000061import com.google.devtools.build.lib.skyframe.PackageLookupFunction.CrossRepositoryLabelViolationStrategy;
Googlerfe72dbb2021-12-29 07:12:53 -080062import com.google.devtools.build.lib.testutil.ManualClock;
Luis Fernando Pino Duquebe102182016-05-23 14:03:55 +000063import com.google.devtools.build.lib.testutil.TestConstants;
michajlo8083e322020-03-20 13:32:52 -070064import com.google.devtools.build.lib.testutil.TestPackageFactoryBuilderFactory;
Kristina Chodorowf9fdc8d2015-12-08 12:49:31 +000065import com.google.devtools.build.lib.testutil.TestRuleClassProvider;
janakr933c3012021-07-13 07:44:15 -070066import com.google.devtools.build.lib.testutil.TestUtils;
laszlocsomor29bdf632018-03-13 02:50:46 -070067import com.google.devtools.build.lib.testutil.TimestampGranularityUtils;
Laszlo Csomora278aec2018-03-09 04:07:31 -080068import com.google.devtools.build.lib.util.io.OutErr;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000069import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
70import com.google.devtools.build.lib.vfs.BatchStat;
buchgr4992ae22019-03-20 04:23:32 -070071import com.google.devtools.build.lib.vfs.DigestHashFunction;
nharmatad5339962022-02-11 14:44:08 -080072import com.google.devtools.build.lib.vfs.FileStateKey;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000073import com.google.devtools.build.lib.vfs.FileStatus;
74import com.google.devtools.build.lib.vfs.FileStatusWithDigest;
75import com.google.devtools.build.lib.vfs.FileStatusWithDigestAdapter;
Googlerfe72dbb2021-12-29 07:12:53 -080076import com.google.devtools.build.lib.vfs.FileSystem;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000077import com.google.devtools.build.lib.vfs.FileSystemUtils;
Eric Fellheimere6590722015-11-17 17:07:48 +000078import com.google.devtools.build.lib.vfs.ModifiedFileSet;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000079import com.google.devtools.build.lib.vfs.Path;
80import com.google.devtools.build.lib.vfs.PathFragment;
tomluee6a6862018-01-17 14:36:26 -080081import com.google.devtools.build.lib.vfs.Root;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000082import com.google.devtools.build.lib.vfs.RootedPath;
83import com.google.devtools.build.lib.vfs.Symlinks;
janakrfc1d79c2022-01-27 13:02:07 -080084import com.google.devtools.build.lib.vfs.SyscallCache;
Googlerfe72dbb2021-12-29 07:12:53 -080085import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000086import com.google.devtools.build.skyframe.Differencer.Diff;
Googler10028672018-10-25 12:14:34 -070087import com.google.devtools.build.skyframe.EvaluationContext;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000088import com.google.devtools.build.skyframe.EvaluationResult;
89import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator;
90import com.google.devtools.build.skyframe.MemoizingEvaluator;
91import com.google.devtools.build.skyframe.RecordingDifferencer;
janakr1cde8722017-10-10 03:22:21 +020092import com.google.devtools.build.skyframe.SequencedRecordingDifferencer;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000093import com.google.devtools.build.skyframe.SkyFunction;
94import com.google.devtools.build.skyframe.SkyFunctionName;
95import com.google.devtools.build.skyframe.SkyKey;
96import com.google.devtools.build.skyframe.SkyValue;
ajurkowskibdc60b02021-11-24 08:47:05 -080097import com.google.testing.junit.testparameterinjector.TestParameter;
98import com.google.testing.junit.testparameterinjector.TestParameterInjector;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000099import java.io.IOException;
100import java.util.ArrayList;
101import java.util.Arrays;
102import java.util.Collection;
103import java.util.HashMap;
104import java.util.List;
105import java.util.Map;
106import java.util.UUID;
janakr933c3012021-07-13 07:44:15 -0700107import java.util.concurrent.CountDownLatch;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000108import java.util.concurrent.atomic.AtomicReference;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000109import javax.annotation.Nullable;
Janak Ramakrishnancba16452016-07-29 02:17:02 +0000110import org.junit.Before;
steinman7184cb32020-04-20 14:33:03 -0700111import org.junit.Ignore;
Janak Ramakrishnancba16452016-07-29 02:17:02 +0000112import org.junit.Test;
113import org.junit.runner.RunWith;
ajurkowski64795c22022-04-01 10:38:44 -0700114import org.mockito.ArgumentCaptor;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000115
jhorvitz3daedc32020-07-22 18:33:55 -0700116/** Tests for {@link FilesystemValueChecker}. */
ajurkowskibdc60b02021-11-24 08:47:05 -0800117@RunWith(TestParameterInjector.class)
Googlerfe72dbb2021-12-29 07:12:53 -0800118public final class FilesystemValueCheckerTest {
119 private static final int FSVC_THREADS_FOR_TEST = 200;
120 private static final ActionLookupKey ACTION_LOOKUP_KEY =
121 new ActionLookupKey() {
122 @Override
123 public SkyFunctionName functionName() {
124 return SkyFunctionName.FOR_TESTING;
125 }
126
127 @Nullable
128 @Override
129 public Label getLabel() {
130 return null;
131 }
132
133 @Nullable
134 @Override
135 public BuildConfigurationKey getConfigurationKey() {
136 return null;
137 }
138 };
139 private static final ActionLookupData ACTION_LOOKUP_DATA = actionLookupData(0);
Googler10028672018-10-25 12:14:34 -0700140 private static final EvaluationContext EVALUATION_OPTIONS =
141 EvaluationContext.newBuilder()
142 .setKeepGoing(false)
143 .setNumThreads(SkyframeExecutor.DEFAULT_THREAD_COUNT)
michajlo7a485be2020-07-30 11:08:46 -0700144 .setEventHandler(NullEventHandler.INSTANCE)
Googler10028672018-10-25 12:14:34 -0700145 .build();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000146
Googlerfe72dbb2021-12-29 07:12:53 -0800147 private final MockFileSystem fs = new MockFileSystem();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000148 private RecordingDifferencer differencer;
149 private MemoizingEvaluator evaluator;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000150 private Path pkgRoot;
Googlerfe72dbb2021-12-29 07:12:53 -0800151 @TestParameter private BatchStatMode batchStat;
152
ajurkowski64795c22022-04-01 10:38:44 -0700153 private final ModifiedOutputsReceiver mockModifiedOutputsReceiver =
154 mock(ModifiedOutputsReceiver.class);
155 private final ArgumentCaptor<Artifact> modifiedOutputsCaptor =
156 ArgumentCaptor.forClass(Artifact.class);
157
Googlerfe72dbb2021-12-29 07:12:53 -0800158 private SpecialArtifact createTreeArtifact(String relPath) throws IOException {
159 String outSegment = "bin";
160 Path outputDir = fs.getPath("/" + outSegment);
161 Path outputPath = outputDir.getRelative(relPath);
162 outputDir.createDirectory();
163 ArtifactRoot derivedRoot =
164 ArtifactRoot.asDerivedRoot(fs.getPath("/"), RootType.Output, outSegment);
165 return ActionsTestUtil.createTreeArtifactWithGeneratingAction(
166 derivedRoot,
167 derivedRoot.getExecPath().getRelative(derivedRoot.getRoot().relativize(outputPath)));
168 }
169
170 private static ActionExecutionValue actionValueWithTreeArtifacts(List<TreeFileArtifact> contents)
171 throws IOException {
172 return actionValueWithTreeArtifacts(contents, ImmutableList.of());
173 }
174
175 private static ActionExecutionValue actionValueWithTreeArtifacts(
176 Iterable<TreeFileArtifact> contents, Iterable<ArchivedTreeArtifact> archivedTreeArtifacts)
177 throws IOException {
178 TreeArtifactValue.MultiBuilder treeArtifacts = TreeArtifactValue.newMultiBuilder();
179
180 for (TreeFileArtifact output : contents) {
181 treeArtifacts.putChild(output, createMetadataFromFileSystem(output));
182 }
183
184 for (ArchivedTreeArtifact archivedTreeArtifact : archivedTreeArtifacts) {
185 treeArtifacts.setArchivedRepresentation(
186 archivedTreeArtifact, createMetadataFromFileSystem(archivedTreeArtifact));
187 }
188
189 Map<Artifact, TreeArtifactValue> treeArtifactData = new HashMap<>();
190 treeArtifacts.injectTo(treeArtifactData::put);
191
192 return ActionExecutionValue.createForTesting(
193 /*artifactData=*/ ImmutableMap.of(),
194 ImmutableMap.copyOf(treeArtifactData),
195 /*outputSymlinks=*/ null);
196 }
197
198 private static FileArtifactValue createMetadataFromFileSystem(Artifact artifact)
199 throws IOException {
200 Path path = artifact.getPath();
201 FileArtifactValue noDigest =
202 ActionMetadataHandler.fileArtifactValueFromArtifact(
janakr95278b42022-03-04 13:41:46 -0800203 artifact,
janakr61b3bb22022-03-07 10:11:00 -0800204 FileStatusWithDigestAdapter.maybeAdapt(path.statIfFound(Symlinks.NOFOLLOW)),
janakr95278b42022-03-04 13:41:46 -0800205 SyscallCache.NO_CACHE,
206 null);
Googlerfe72dbb2021-12-29 07:12:53 -0800207 return FileArtifactValue.createFromInjectedDigest(noDigest, path.getDigest());
208 }
209
210 void writeFile(Path path, String... lines) throws IOException {
211 // Make sure we advance the clock to detect modifications which do not change the size, which
212 // rely on ctime.
213 fs.advanceClockMillis(1);
214 FileSystemUtils.writeIsoLatin1(path, lines);
215 }
216
217 private static final class MockFileSystem extends InMemoryFileSystem {
218 boolean statThrowsRuntimeException;
219 boolean readlinkThrowsIoException;
220
221 MockFileSystem() {
222 this(new ManualClock());
223 }
224
225 private MockFileSystem(ManualClock clock) {
226 super(clock, DigestHashFunction.SHA256);
227 }
228
229 @Override
230 public FileStatus statIfFound(PathFragment path, boolean followSymlinks) throws IOException {
231 if (statThrowsRuntimeException) {
232 throw new RuntimeException("bork");
233 }
234 return super.statIfFound(path, followSymlinks);
235 }
236
237 @Override
238 protected PathFragment readSymbolicLink(PathFragment path) throws IOException {
239 if (readlinkThrowsIoException) {
240 throw new IOException("readlink failed");
241 }
242 return super.readSymbolicLink(path);
243 }
244
245 void advanceClockMillis(int millis) {
246 ((ManualClock) clock).advanceMillis(millis);
247 }
248 }
249
250 private enum BatchStatMode {
251 DISABLED {
252 @Nullable
253 @Override
254 BatchStat getBatchStat(FileSystem fileSystem) {
255 return null;
256 }
257 },
258 ENABLED {
259 @Override
260 BatchStat getBatchStat(FileSystem fileSystem) {
261 return (useDigest, includeLinks, paths) -> {
262 List<FileStatusWithDigest> stats = new ArrayList<>();
263 for (PathFragment pathFrag : paths) {
264 stats.add(
janakr61b3bb22022-03-07 10:11:00 -0800265 FileStatusWithDigestAdapter.maybeAdapt(
Googlerfe72dbb2021-12-29 07:12:53 -0800266 fileSystem.getPath("/").getRelative(pathFrag).statIfFound(Symlinks.NOFOLLOW)));
267 }
268 return stats;
269 };
270 }
271 };
272
273 @Nullable
274 abstract BatchStat getBatchStat(FileSystem fileSystem);
275 }
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000276
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000277 @Before
ajurkowskifbd44272021-12-02 16:05:49 -0800278 public void setUp() throws Exception {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000279 ImmutableMap.Builder<SkyFunctionName, SkyFunction> skyFunctions = ImmutableMap.builder();
280
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000281 pkgRoot = fs.getPath("/testroot");
jhorvitz3daedc32020-07-22 18:33:55 -0700282 pkgRoot.createDirectoryAndParents();
Kristina Chodorowf9fdc8d2015-12-08 12:49:31 +0000283 FileSystemUtils.createEmptyFile(pkgRoot.getRelative("WORKSPACE"));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000284
John Catere0d1d0e2017-11-28 20:47:41 -0800285 AtomicReference<PathPackageLocator> pkgLocator =
286 new AtomicReference<>(
287 new PathPackageLocator(
288 fs.getPath("/output_base"),
tomluee6a6862018-01-17 14:36:26 -0800289 ImmutableList.of(Root.fromPath(pkgRoot)),
John Catere0d1d0e2017-11-28 20:47:41 -0800290 BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY));
janakr3b63a4e2017-09-14 09:55:40 +0200291 BlazeDirectories directories =
292 new BlazeDirectories(
cushon849df362018-05-14 01:51:45 -0700293 new ServerDirectories(pkgRoot, pkgRoot, pkgRoot),
294 pkgRoot,
295 /* defaultSystemJavabase= */ null,
296 TestConstants.PRODUCT_NAME);
nharmata3fb7d342018-02-23 11:37:51 -0800297 ExternalFilesHelper externalFilesHelper = ExternalFilesHelper.createForTesting(
Nathan Harmatad4f75942016-10-18 08:55:17 +0000298 pkgLocator, ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS, directories);
shahan602cc852018-06-06 20:09:57 -0700299 skyFunctions.put(
nharmatad5339962022-02-11 14:44:08 -0800300 FileStateKey.FILE_STATE,
shahan602cc852018-06-06 20:09:57 -0700301 new FileStateFunction(
janakrfc1d79c2022-01-27 13:02:07 -0800302 Suppliers.ofInstance(new TimestampGranularityMonitor(BlazeClock.instance())),
janakrb2a94342022-02-05 22:02:14 -0800303 SyscallCache.NO_CACHE,
djasper7faa0ef2019-03-28 10:00:00 -0700304 externalFilesHelper));
shahan602cc852018-06-06 20:09:57 -0700305 skyFunctions.put(FileValue.FILE, new FileFunction(pkgLocator));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000306 skyFunctions.put(
janakre2af68f2021-03-18 15:11:30 -0700307 FileSymlinkCycleUniquenessFunction.NAME, new FileSymlinkCycleUniquenessFunction());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000308 skyFunctions.put(
janakre2af68f2021-03-18 15:11:30 -0700309 FileSymlinkInfiniteExpansionUniquenessFunction.NAME,
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000310 new FileSymlinkInfiniteExpansionUniquenessFunction());
ajurkowski2018b872020-04-14 09:59:00 -0700311 skyFunctions.put(
janakre4fcd472021-05-26 08:47:39 -0700312 SkyFunctions.PACKAGE,
313 new PackageFunction(
314 null,
315 null,
316 null,
317 null,
318 null,
janakre4fcd472021-05-26 08:47:39 -0700319 /*packageProgress=*/ null,
320 PackageFunction.ActionOnIOExceptionReadingBuildFile.UseOriginalIOException.INSTANCE,
jhorvitz1d675d12022-04-28 16:23:21 -0700321 GlobbingStrategy.SKYFRAME_HYBRID,
janakr13b737a2021-07-02 14:24:25 -0700322 k -> ThreadStateReceiver.NULL_INSTANCE));
John Cater5e9ce942016-10-12 17:23:30 +0000323 skyFunctions.put(
324 SkyFunctions.PACKAGE_LOOKUP,
325 new PackageLookupFunction(
jhorvitz3daedc32020-07-22 18:33:55 -0700326 new AtomicReference<>(ImmutableSet.of()),
John Cater0c0735a2016-11-11 01:52:02 +0000327 CrossRepositoryLabelViolationStrategy.ERROR,
ajurkowski2018b872020-04-14 09:59:00 -0700328 BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY,
329 BazelSkyframeExecutorConstants.EXTERNAL_PACKAGE_HELPER));
carmid6a98282018-03-13 19:19:16 -0700330 skyFunctions.put(
janakr15e15c22019-01-30 11:24:49 -0800331 WorkspaceFileValue.WORKSPACE_FILE,
carmid6a98282018-03-13 19:19:16 -0700332 new WorkspaceFileFunction(
333 TestRuleClassProvider.getRuleClassProvider(),
michajlo8083e322020-03-20 13:32:52 -0700334 TestPackageFactoryBuilderFactory.getInstance()
carmid6a98282018-03-13 19:19:16 -0700335 .builder(directories)
nharmatafde0bd2f2018-12-21 10:17:56 -0800336 .build(TestRuleClassProvider.getRuleClassProvider(), fs),
mjhalupkaf0e48112019-01-14 13:01:56 -0800337 directories,
brandjon771a0292020-05-26 12:04:16 -0700338 /*bzlLoadFunctionForInlining=*/ null));
ajurkowski2018b872020-04-14 09:59:00 -0700339 skyFunctions.put(
340 SkyFunctions.EXTERNAL_PACKAGE,
341 new ExternalPackageFunction(BazelSkyframeExecutorConstants.EXTERNAL_PACKAGE_HELPER));
Kristina Chodorowf9fdc8d2015-12-08 12:49:31 +0000342
janakr1cde8722017-10-10 03:22:21 +0200343 differencer = new SequencedRecordingDifferencer();
jhorvitz7f55cb72021-12-16 18:52:24 -0800344 evaluator = new InMemoryMemoizingEvaluator(skyFunctions.buildOrThrow(), differencer);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000345 PrecomputedValue.BUILD_ID.set(differencer, UUID.randomUUID());
Kristina Chodorowf9fdc8d2015-12-08 12:49:31 +0000346 PrecomputedValue.PATH_PACKAGE_LOCATOR.set(differencer, pkgLocator.get());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000347 }
348
ajurkowski64795c22022-04-01 10:38:44 -0700349 @Before
350 public void setupModifiedOutputReceiverMock() {
351 doNothing()
352 .when(mockModifiedOutputsReceiver)
353 .reportModifiedOutputFile(anyLong(), modifiedOutputsCaptor.capture());
354 }
355
Googlerfe72dbb2021-12-29 07:12:53 -0800356 public static ImmutableList<Object[]> batchStatModes() {
357 return Arrays.stream(BatchStatMode.values())
358 .map(mode -> new BatchStatMode[] {mode})
359 .collect(toImmutableList());
360 }
361
362 @Test
363 public void getDirtyActionValues_unchangedEmptyTreeArtifactWithArchivedFile_noDirtyKeys()
364 throws Exception {
365 SpecialArtifact treeArtifact = createTreeArtifact("dir");
366 treeArtifact.getPath().createDirectoryAndParents();
367 ActionExecutionValue actionExecutionValue =
368 actionValueWithTreeArtifacts(
369 ImmutableList.of(),
370 ImmutableList.of(createArchivedTreeArtifactWithContent(treeArtifact)));
371
372 assertThat(getDirtyActionValues(actionExecutionValue)).isEmpty();
373 }
374
375 @Test
376 public void getDirtyActionValues_unchangedTreeArtifactWithArchivedFile_noDirtyKeys()
377 throws Exception {
378 SpecialArtifact treeArtifact = createTreeArtifact("dir");
379 ActionExecutionValue actionExecutionValue =
380 actionValueWithTreeArtifacts(
381 ImmutableList.of(
382 createTreeFileArtifactWithContent(treeArtifact, "file1", "content"),
383 createTreeFileArtifactWithContent(treeArtifact, "file2", "content2")),
384 ImmutableList.of(createArchivedTreeArtifactWithContent(treeArtifact)));
385
386 assertThat(getDirtyActionValues(actionExecutionValue)).isEmpty();
387 }
388
389 @Test
390 public void getDirtyActionValues_editedArchivedFileForEmptyTreeArtifact_reportsChange()
391 throws Exception {
392 SpecialArtifact treeArtifact = createTreeArtifact("dir");
393 treeArtifact.getPath().createDirectoryAndParents();
394 ArchivedTreeArtifact archivedTreeArtifact =
395 createArchivedTreeArtifactWithContent(treeArtifact, "old content");
396 ActionExecutionValue actionExecutionValue =
397 actionValueWithTreeArtifacts(ImmutableList.of(), ImmutableList.of(archivedTreeArtifact));
398
399 writeFile(archivedTreeArtifact.getPath(), "new content");
400 assertThat(getDirtyActionValues(actionExecutionValue)).containsExactly(ACTION_LOOKUP_DATA);
401 }
402
403 @Test
404 public void getDirtyActionValues_editedArchivedFileForTreeArtifact_reportsChange()
405 throws Exception {
406 SpecialArtifact treeArtifact = createTreeArtifact("dir");
407 ArchivedTreeArtifact archivedTreeArtifact =
408 createArchivedTreeArtifactWithContent(treeArtifact, "old content");
409 ActionExecutionValue actionExecutionValue =
410 actionValueWithTreeArtifacts(
411 ImmutableList.of(
412 createTreeFileArtifactWithContent(
413 treeArtifact, /*parentRelativePath=*/ "file1", "content"),
414 createTreeFileArtifactWithContent(
415 treeArtifact, /*parentRelativePath=*/ "file2", "content2")),
416 ImmutableList.of(archivedTreeArtifact));
417
418 writeFile(archivedTreeArtifact.getPath(), "new content");
419 assertThat(getDirtyActionValues(actionExecutionValue)).containsExactly(ACTION_LOOKUP_DATA);
420 }
421
422 @Test
423 public void getDirtyActionValues_deletedArchivedFileForTreeArtifact_reportsChange()
424 throws Exception {
425 SpecialArtifact treeArtifact = createTreeArtifact("dir");
426 ArchivedTreeArtifact archivedTreeArtifact = createArchivedTreeArtifactWithContent(treeArtifact);
427 ActionExecutionValue actionExecutionValue =
428 actionValueWithTreeArtifacts(
429 ImmutableList.of(
430 createTreeFileArtifactWithContent(
431 treeArtifact, /*parentRelativePath=*/ "file1", "content"),
432 createTreeFileArtifactWithContent(
433 treeArtifact, /*parentRelativePath=*/ "file2", "content2")),
434 ImmutableList.of(archivedTreeArtifact));
435
436 archivedTreeArtifact.getPath().delete();
437 assertThat(getDirtyActionValues(actionExecutionValue)).containsExactly(ACTION_LOOKUP_DATA);
438 }
439
440 @Test
441 public void getDirtyActionValues_deletedArchivedFileForEmptyTreeArtifact_reportsChange()
442 throws Exception {
443 SpecialArtifact treeArtifact = createTreeArtifact("dir");
444 ArchivedTreeArtifact archivedTreeArtifact = createArchivedTreeArtifactWithContent(treeArtifact);
445 ActionExecutionValue actionExecutionValue =
446 actionValueWithTreeArtifacts(ImmutableList.of(), ImmutableList.of(archivedTreeArtifact));
447
448 archivedTreeArtifact.getPath().delete();
449 assertThat(getDirtyActionValues(actionExecutionValue)).containsExactly(ACTION_LOOKUP_DATA);
450 }
451
452 @Test
453 public void getDirtyActionValues_editedFileForTreeArtifactWithArchivedFile_reportsChange()
454 throws Exception {
455 SpecialArtifact treeArtifact = createTreeArtifact("dir");
456 TreeFileArtifact child1 =
457 createTreeFileArtifactWithContent(
458 treeArtifact, /*parentRelativePath=*/ "file1", "old content");
459 ActionExecutionValue actionExecutionValue =
460 actionValueWithTreeArtifacts(
461 ImmutableList.of(
462 child1,
463 createTreeFileArtifactWithContent(
464 treeArtifact, /*parentRelativePath=*/ "file2", "content2")),
465 ImmutableList.of(createArchivedTreeArtifactWithContent(treeArtifact)));
466
467 writeFile(child1.getPath(), "new content");
468 assertThat(getDirtyActionValues(actionExecutionValue)).containsExactly(ACTION_LOOKUP_DATA);
469 }
470
471 @Test
472 public void getDirtyActionValues_treeArtifactWithArchivedArtifact_reportsOnlyChangedKey()
473 throws Exception {
474 SpecialArtifact unchangedTreeArtifact = createTreeArtifact("dir1");
475 ActionExecutionValue unchangedValue =
476 actionValueWithTreeArtifacts(
477 ImmutableList.of(createTreeFileArtifactWithContent(unchangedTreeArtifact, "child")),
478 ImmutableList.of(createArchivedTreeArtifactWithContent(unchangedTreeArtifact)));
479 SpecialArtifact changedTreeArtifact = createTreeArtifact("dir2");
480 ArchivedTreeArtifact changedArchivedTreeArtifact =
481 createArchivedTreeArtifactWithContent(changedTreeArtifact, "old content");
482 ActionExecutionValue changedValue =
483 actionValueWithTreeArtifacts(
484 ImmutableList.of(
485 createTreeFileArtifactWithContent(changedTreeArtifact, "file", "content")),
486 ImmutableList.of(changedArchivedTreeArtifact));
487
488 writeFile(changedArchivedTreeArtifact.getPath(), "new content");
489 assertThat(
490 getDirtyActionValues(
491 ImmutableMap.of(
492 actionLookupData(0), unchangedValue, actionLookupData(1), changedValue)))
493 .containsExactly(actionLookupData(1));
494 }
495
496 private Collection<SkyKey> getDirtyActionValues(ActionExecutionValue actionExecutionValue)
497 throws InterruptedException {
498 return getDirtyActionValues(ImmutableMap.of(ACTION_LOOKUP_DATA, actionExecutionValue));
499 }
500
501 private Collection<SkyKey> getDirtyActionValues(ImmutableMap<SkyKey, SkyValue> valuesMap)
502 throws InterruptedException {
ajurkowski64795c22022-04-01 10:38:44 -0700503 return new FilesystemValueChecker(/*tsgm=*/ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
Googlerfe72dbb2021-12-29 07:12:53 -0800504 .getDirtyActionValues(
505 valuesMap,
506 batchStat.getBatchStat(fs),
507 ModifiedFileSet.EVERYTHING_MODIFIED,
ajurkowski64795c22022-04-01 10:38:44 -0700508 /*trustRemoteArtifacts=*/ false,
509 (ignored, ignored2) -> {});
Googlerfe72dbb2021-12-29 07:12:53 -0800510 }
511
512 private TreeFileArtifact createTreeFileArtifactWithContent(
513 SpecialArtifact treeArtifact, String parentRelativePath, String... contentLines)
514 throws IOException {
515 TreeFileArtifact artifact = TreeFileArtifact.createTreeOutput(treeArtifact, parentRelativePath);
516 writeFile(artifact.getPath(), contentLines);
517 return artifact;
518 }
519
520 private ArchivedTreeArtifact createArchivedTreeArtifactWithContent(
521 SpecialArtifact treeArtifact, String... contentLines) throws IOException {
522 ArchivedTreeArtifact artifact = ArchivedTreeArtifact.createForTree(treeArtifact);
523 writeFile(artifact.getPath(), contentLines);
524 return artifact;
525 }
526
527 private static ActionLookupData actionLookupData(int actionIndex) {
528 return ActionLookupData.create(ACTION_LOOKUP_KEY, actionIndex);
529 }
530
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000531 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000532 public void testEmpty() throws Exception {
twerth5aaceb52020-04-07 06:31:56 -0700533 FilesystemValueChecker checker =
534 new FilesystemValueChecker(
janakr873bf0a2022-03-04 13:38:50 -0800535 /*tsgm=*/ null,
536 SyscallCache.NO_CACHE,
janakr873bf0a2022-03-04 13:38:50 -0800537 FSVC_THREADS_FOR_TEST);
Nathan Harmata8cd29782015-11-10 03:24:01 +0000538 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000539 }
540
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000541 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000542 public void testSimple() throws Exception {
twerth5aaceb52020-04-07 06:31:56 -0700543 FilesystemValueChecker checker =
544 new FilesystemValueChecker(
janakr873bf0a2022-03-04 13:38:50 -0800545 /*tsgm=*/ null,
546 SyscallCache.NO_CACHE,
janakr873bf0a2022-03-04 13:38:50 -0800547 FSVC_THREADS_FOR_TEST);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000548
549 Path path = fs.getPath("/foo");
550 FileSystemUtils.createEmptyFile(path);
Nathan Harmata8cd29782015-11-10 03:24:01 +0000551 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000552
tomluee6a6862018-01-17 14:36:26 -0800553 SkyKey skyKey =
554 FileStateValue.key(
tomlu4c9fafd2018-01-18 10:29:11 -0800555 RootedPath.toRootedPath(Root.absoluteRoot(fs), PathFragment.create("/foo")));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000556 EvaluationResult<SkyValue> result =
jhorvitz9ad89182021-12-29 11:57:02 -0800557 evaluator.evaluate(ImmutableList.of(skyKey), EVALUATION_OPTIONS);
lberkiaea56b32017-05-30 12:35:33 +0200558 assertThat(result.hasError()).isFalse();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000559
Nathan Harmata8cd29782015-11-10 03:24:01 +0000560 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000561
562 FileSystemUtils.writeContentAsLatin1(path, "hello");
Nathan Harmata8cd29782015-11-10 03:24:01 +0000563 assertDiffWithNewValues(getDirtyFilesystemKeys(evaluator, checker), skyKey);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000564
565 // The dirty bits are not reset until the FileValues are actually revalidated.
Nathan Harmata8cd29782015-11-10 03:24:01 +0000566 assertDiffWithNewValues(getDirtyFilesystemKeys(evaluator, checker), skyKey);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000567
568 differencer.invalidate(ImmutableList.of(skyKey));
jhorvitz9ad89182021-12-29 11:57:02 -0800569 result = evaluator.evaluate(ImmutableList.of(skyKey), EVALUATION_OPTIONS);
lberkiaea56b32017-05-30 12:35:33 +0200570 assertThat(result.hasError()).isFalse();
Nathan Harmata8cd29782015-11-10 03:24:01 +0000571 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000572 }
573
574 /**
575 * Tests that an already-invalidated value can still be marked changed: symlink points at sym1.
576 * Invalidate symlink by changing sym1 from pointing at path to point to sym2. This only dirties
577 * (rather than changes) symlink because sym2 still points at path, so all symlink stats remain
578 * the same. Then do a null build, change sym1 back to point at path, and change symlink to not be
579 * a symlink anymore. The fact that it is not a symlink should be detected.
580 */
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000581 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000582 public void testDirtySymlink() throws Exception {
twerth5aaceb52020-04-07 06:31:56 -0700583 FilesystemValueChecker checker =
584 new FilesystemValueChecker(
janakr873bf0a2022-03-04 13:38:50 -0800585 /*tsgm=*/ null,
586 SyscallCache.NO_CACHE,
janakr873bf0a2022-03-04 13:38:50 -0800587 FSVC_THREADS_FOR_TEST);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000588
589 Path path = fs.getPath("/foo");
590 FileSystemUtils.writeContentAsLatin1(path, "foo contents");
591 // We need the intermediate sym1 and sym2 so that we can dirty a child of symlink without
592 // actually changing the FileValue calculated for symlink (if we changed the contents of foo,
Laszlo Csomora278aec2018-03-09 04:07:31 -0800593 // the FileValue created for symlink would notice, since it stats foo).
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000594 Path sym1 = fs.getPath("/sym1");
595 Path sym2 = fs.getPath("/sym2");
596 Path symlink = fs.getPath("/bar");
597 FileSystemUtils.ensureSymbolicLink(symlink, sym1);
598 FileSystemUtils.ensureSymbolicLink(sym1, path);
599 FileSystemUtils.ensureSymbolicLink(sym2, path);
600 SkyKey fooKey =
tomlu4c9fafd2018-01-18 10:29:11 -0800601 FileValue.key(RootedPath.toRootedPath(Root.absoluteRoot(fs), PathFragment.create("/foo")));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000602 RootedPath symlinkRootedPath =
tomlu4c9fafd2018-01-18 10:29:11 -0800603 RootedPath.toRootedPath(Root.absoluteRoot(fs), PathFragment.create("/bar"));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000604 SkyKey symlinkKey = FileValue.key(symlinkRootedPath);
605 SkyKey symlinkFileStateKey = FileStateValue.key(symlinkRootedPath);
606 RootedPath sym1RootedPath =
tomlu4c9fafd2018-01-18 10:29:11 -0800607 RootedPath.toRootedPath(Root.absoluteRoot(fs), PathFragment.create("/sym1"));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000608 SkyKey sym1FileStateKey = FileStateValue.key(sym1RootedPath);
609 Iterable<SkyKey> allKeys = ImmutableList.of(symlinkKey, fooKey);
610
611 // First build -- prime the graph.
jhorvitz9ad89182021-12-29 11:57:02 -0800612 EvaluationResult<FileValue> result = evaluator.evaluate(allKeys, EVALUATION_OPTIONS);
lberkiaea56b32017-05-30 12:35:33 +0200613 assertThat(result.hasError()).isFalse();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000614 FileValue symlinkValue = result.get(symlinkKey);
615 FileValue fooValue = result.get(fooKey);
lberkiaea56b32017-05-30 12:35:33 +0200616 assertWithMessage(symlinkValue.toString()).that(symlinkValue.isSymlink()).isTrue();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000617 // Digest is not always available, so use size as a proxy for contents.
lberkiaea56b32017-05-30 12:35:33 +0200618 assertThat(symlinkValue.getSize()).isEqualTo(fooValue.getSize());
Nathan Harmata8cd29782015-11-10 03:24:01 +0000619 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000620
621 // Before second build, move sym1 to point to sym2.
lberkiaea56b32017-05-30 12:35:33 +0200622 assertThat(sym1.delete()).isTrue();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000623 FileSystemUtils.ensureSymbolicLink(sym1, sym2);
Nathan Harmata8cd29782015-11-10 03:24:01 +0000624 assertDiffWithNewValues(getDirtyFilesystemKeys(evaluator, checker), sym1FileStateKey);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000625
626 differencer.invalidate(ImmutableList.of(sym1FileStateKey));
jhorvitz9ad89182021-12-29 11:57:02 -0800627 result = evaluator.evaluate(ImmutableList.of(), EVALUATION_OPTIONS);
lberkiaea56b32017-05-30 12:35:33 +0200628 assertThat(result.hasError()).isFalse();
Nathan Harmata8cd29782015-11-10 03:24:01 +0000629 assertDiffWithNewValues(getDirtyFilesystemKeys(evaluator, checker), sym1FileStateKey);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000630
631 // Before third build, move sym1 back to original (so change pruning will prevent signaling of
632 // its parents, but change symlink for real.
lberkiaea56b32017-05-30 12:35:33 +0200633 assertThat(sym1.delete()).isTrue();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000634 FileSystemUtils.ensureSymbolicLink(sym1, path);
lberkiaea56b32017-05-30 12:35:33 +0200635 assertThat(symlink.delete()).isTrue();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000636 FileSystemUtils.writeContentAsLatin1(symlink, "new symlink contents");
Nathan Harmata8cd29782015-11-10 03:24:01 +0000637 assertDiffWithNewValues(getDirtyFilesystemKeys(evaluator, checker), symlinkFileStateKey);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000638 differencer.invalidate(ImmutableList.of(symlinkFileStateKey));
jhorvitz9ad89182021-12-29 11:57:02 -0800639 result = evaluator.evaluate(allKeys, EVALUATION_OPTIONS);
lberkiaea56b32017-05-30 12:35:33 +0200640 assertThat(result.hasError()).isFalse();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000641 symlinkValue = result.get(symlinkKey);
lberkiaea56b32017-05-30 12:35:33 +0200642 assertWithMessage(symlinkValue.toString()).that(symlinkValue.isSymlink()).isFalse();
643 assertThat(result.get(fooKey)).isEqualTo(fooValue);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000644 assertThat(symlinkValue.getSize()).isNotEqualTo(fooValue.getSize());
Nathan Harmata8cd29782015-11-10 03:24:01 +0000645 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000646 }
647
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000648 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000649 public void testExplicitFiles() throws Exception {
twerth5aaceb52020-04-07 06:31:56 -0700650 FilesystemValueChecker checker =
651 new FilesystemValueChecker(
janakr873bf0a2022-03-04 13:38:50 -0800652 /*tsgm=*/ null,
653 SyscallCache.NO_CACHE,
janakr873bf0a2022-03-04 13:38:50 -0800654 FSVC_THREADS_FOR_TEST);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000655
656 Path path1 = fs.getPath("/foo1");
657 Path path2 = fs.getPath("/foo2");
658 FileSystemUtils.createEmptyFile(path1);
659 FileSystemUtils.createEmptyFile(path2);
Nathan Harmata8cd29782015-11-10 03:24:01 +0000660 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000661
662 SkyKey key1 =
663 FileStateValue.key(
tomlu4c9fafd2018-01-18 10:29:11 -0800664 RootedPath.toRootedPath(Root.absoluteRoot(fs), PathFragment.create("/foo1")));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000665 SkyKey key2 =
666 FileStateValue.key(
tomlu4c9fafd2018-01-18 10:29:11 -0800667 RootedPath.toRootedPath(Root.absoluteRoot(fs), PathFragment.create("/foo2")));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000668 Iterable<SkyKey> skyKeys = ImmutableList.of(key1, key2);
jhorvitz9ad89182021-12-29 11:57:02 -0800669 EvaluationResult<SkyValue> result = evaluator.evaluate(skyKeys, EVALUATION_OPTIONS);
lberkiaea56b32017-05-30 12:35:33 +0200670 assertThat(result.hasError()).isFalse();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000671
Nathan Harmata8cd29782015-11-10 03:24:01 +0000672 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000673
Laszlo Csomora278aec2018-03-09 04:07:31 -0800674 // Wait for the timestamp granularity to elapse, so updating the files will observably advance
675 // their ctime.
laszlocsomor29bdf632018-03-13 02:50:46 -0700676 TimestampGranularityUtils.waitForTimestampGranularity(
677 System.currentTimeMillis(), OutErr.SYSTEM_OUT_ERR);
ajurkowski280bbe22020-08-19 11:26:20 -0700678 // Update path1's contents. This will update the file's ctime with current time indicated by the
679 // clock.
680 fs.advanceClockMillis(1);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000681 FileSystemUtils.writeContentAsLatin1(path1, "hello1");
Laszlo Csomora278aec2018-03-09 04:07:31 -0800682 // Update path2's mtime but not its contents. We expect that an mtime change suffices to update
683 // the ctime.
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000684 path2.setLastModifiedTime(42);
Laszlo Csomora278aec2018-03-09 04:07:31 -0800685 // Assert that both files changed. The change detection relies, among other things, on ctime
686 // change.
Nathan Harmata8cd29782015-11-10 03:24:01 +0000687 assertDiffWithNewValues(getDirtyFilesystemKeys(evaluator, checker), key1, key2);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000688
689 differencer.invalidate(skyKeys);
jhorvitz9ad89182021-12-29 11:57:02 -0800690 result = evaluator.evaluate(skyKeys, EVALUATION_OPTIONS);
lberkiaea56b32017-05-30 12:35:33 +0200691 assertThat(result.hasError()).isFalse();
Nathan Harmata8cd29782015-11-10 03:24:01 +0000692 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000693 }
694
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000695 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000696 public void testFileWithIOExceptionNotConsideredDirty() throws Exception {
697 Path path = fs.getPath("/testroot/foo");
698 path.getParentDirectory().createDirectory();
nharmatab4060b62017-04-04 17:11:39 +0000699 path.createSymbolicLink(PathFragment.create("bar"));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000700
701 fs.readlinkThrowsIoException = true;
tomluee6a6862018-01-17 14:36:26 -0800702 SkyKey fileKey =
703 FileStateValue.key(
704 RootedPath.toRootedPath(Root.fromPath(pkgRoot), PathFragment.create("foo")));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000705 EvaluationResult<SkyValue> result =
jhorvitz9ad89182021-12-29 11:57:02 -0800706 evaluator.evaluate(ImmutableList.of(fileKey), EVALUATION_OPTIONS);
lberkiaea56b32017-05-30 12:35:33 +0200707 assertThat(result.hasError()).isTrue();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000708
709 fs.readlinkThrowsIoException = false;
twerth5aaceb52020-04-07 06:31:56 -0700710 FilesystemValueChecker checker =
711 new FilesystemValueChecker(
janakr873bf0a2022-03-04 13:38:50 -0800712 /*tsgm=*/ null,
713 SyscallCache.NO_CACHE,
janakr873bf0a2022-03-04 13:38:50 -0800714 FSVC_THREADS_FOR_TEST);
Nathan Harmata8cd29782015-11-10 03:24:01 +0000715 Diff diff = getDirtyFilesystemKeys(evaluator, checker);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000716 assertThat(diff.changedKeysWithoutNewValues()).isEmpty();
717 assertThat(diff.changedKeysWithNewValues()).isEmpty();
718 }
719
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000720 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000721 public void testFilesInCycleNotConsideredDirty() throws Exception {
722 Path path1 = pkgRoot.getRelative("foo1");
723 Path path2 = pkgRoot.getRelative("foo2");
724 Path path3 = pkgRoot.getRelative("foo3");
725 FileSystemUtils.ensureSymbolicLink(path1, path2);
726 FileSystemUtils.ensureSymbolicLink(path2, path3);
727 FileSystemUtils.ensureSymbolicLink(path3, path1);
tomluee6a6862018-01-17 14:36:26 -0800728 SkyKey fileKey1 = FileValue.key(RootedPath.toRootedPath(Root.fromPath(pkgRoot), path1));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000729
730 EvaluationResult<SkyValue> result =
jhorvitz9ad89182021-12-29 11:57:02 -0800731 evaluator.evaluate(ImmutableList.of(fileKey1), EVALUATION_OPTIONS);
lberkiaea56b32017-05-30 12:35:33 +0200732 assertThat(result.hasError()).isTrue();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000733
twerth5aaceb52020-04-07 06:31:56 -0700734 FilesystemValueChecker checker =
735 new FilesystemValueChecker(
janakr873bf0a2022-03-04 13:38:50 -0800736 /*tsgm=*/ null,
737 SyscallCache.NO_CACHE,
janakr873bf0a2022-03-04 13:38:50 -0800738 FSVC_THREADS_FOR_TEST);
Nathan Harmata8cd29782015-11-10 03:24:01 +0000739 Diff diff = getDirtyFilesystemKeys(evaluator, checker);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000740 assertThat(diff.changedKeysWithoutNewValues()).isEmpty();
741 assertThat(diff.changedKeysWithNewValues()).isEmpty();
742 }
743
jhorvitz3daedc32020-07-22 18:33:55 -0700744 public void checkDirtyActions(BatchStat batchStatter) throws Exception {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000745 Artifact out1 = createDerivedArtifact("fiz");
746 Artifact out2 = createDerivedArtifact("pop");
747
748 FileSystemUtils.writeContentAsLatin1(out1.getPath(), "hello");
749 FileSystemUtils.writeContentAsLatin1(out2.getPath(), "fizzlepop");
750
lberki41cfcc72019-08-05 02:55:36 -0700751 TimestampGranularityMonitor tsgm = new TimestampGranularityMonitor(BlazeClock.instance());
ajurkowski280bbe22020-08-19 11:26:20 -0700752 SkyKey actionKey1 = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
753 SkyKey actionKey2 = ActionLookupData.create(ACTION_LOOKUP_KEY, 1);
lberki41cfcc72019-08-05 02:55:36 -0700754
755 pretendBuildTwoArtifacts(out1, actionKey1, out2, actionKey2, batchStatter, tsgm);
756
757 // Change the file but not its size
758 FileSystemUtils.writeContentAsLatin1(out1.getPath(), "hallo");
759 checkActionDirtiedByFile(out1, actionKey1, batchStatter, tsgm);
760 pretendBuildTwoArtifacts(out1, actionKey1, out2, actionKey2, batchStatter, tsgm);
761
762 // Now try with a different size
763 FileSystemUtils.writeContentAsLatin1(out1.getPath(), "hallo2");
764 checkActionDirtiedByFile(out1, actionKey1, batchStatter, tsgm);
765 }
766
767 private void pretendBuildTwoArtifacts(
768 Artifact out1,
769 SkyKey actionKey1,
770 Artifact out2,
771 SkyKey actionKey2,
772 BatchStat batchStatter,
773 TimestampGranularityMonitor tsgm)
774 throws InterruptedException {
775 EvaluationContext evaluationContext =
776 EvaluationContext.newBuilder()
777 .setKeepGoing(false)
778 .setNumThreads(1)
michajlo7a485be2020-07-30 11:08:46 -0700779 .setEventHandler(NullEventHandler.INSTANCE)
lberki41cfcc72019-08-05 02:55:36 -0700780 .build();
781
782 tsgm.setCommandStartTime();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000783 differencer.inject(
784 ImmutableMap.<SkyKey, SkyValue>of(
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000785 actionKey1,
786 actionValue(
787 new TestAction(
ulfjack1e1a7752019-12-10 21:17:58 -0800788 Runnables.doNothing(),
789 NestedSetBuilder.emptySet(Order.STABLE_ORDER),
790 ImmutableSet.of(out1))),
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000791 actionKey2,
792 actionValue(
793 new TestAction(
lberkic35878a2019-08-01 02:28:54 -0700794 Runnables.doNothing(),
ulfjack1e1a7752019-12-10 21:17:58 -0800795 NestedSetBuilder.emptySet(Order.STABLE_ORDER),
lberkic35878a2019-08-01 02:28:54 -0700796 ImmutableSet.of(out2)))));
jhorvitz9ad89182021-12-29 11:57:02 -0800797 assertThat(evaluator.evaluate(ImmutableList.of(), evaluationContext).hasError()).isFalse();
Eric Fellheimere6590722015-11-17 17:07:48 +0000798 assertThat(
ajurkowski64795c22022-04-01 10:38:44 -0700799 new FilesystemValueChecker(/*tsgm=*/ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000800 .getDirtyActionValues(
steinman39c00d22020-03-20 15:23:10 -0700801 evaluator.getValues(),
802 batchStatter,
803 ModifiedFileSet.EVERYTHING_MODIFIED,
ajurkowski64795c22022-04-01 10:38:44 -0700804 /* trustRemoteArtifacts= */ false,
805 (ignored, ignored2) -> {}))
lberki41cfcc72019-08-05 02:55:36 -0700806 .isEmpty();
807
808 tsgm.waitForTimestampGranularity(OutErr.SYSTEM_OUT_ERR);
809 }
810
811 private void checkActionDirtiedByFile(
812 Artifact file, SkyKey actionKey, BatchStat batchStatter, TimestampGranularityMonitor tsgm)
813 throws InterruptedException {
Ulf Adamsc73051c62016-03-23 09:18:13 +0000814 assertThat(
ajurkowski64795c22022-04-01 10:38:44 -0700815 new FilesystemValueChecker(tsgm, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
lberki41cfcc72019-08-05 02:55:36 -0700816 .getDirtyActionValues(
steinman39c00d22020-03-20 15:23:10 -0700817 evaluator.getValues(),
818 batchStatter,
819 ModifiedFileSet.EVERYTHING_MODIFIED,
ajurkowski64795c22022-04-01 10:38:44 -0700820 /* trustRemoteArtifacts= */ false,
821 (ignored, ignored2) -> {}))
lberki41cfcc72019-08-05 02:55:36 -0700822 .containsExactly(actionKey);
823 assertThat(
ajurkowski64795c22022-04-01 10:38:44 -0700824 new FilesystemValueChecker(tsgm, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000825 .getDirtyActionValues(
826 evaluator.getValues(),
827 batchStatter,
ajurkowskifbd44272021-12-02 16:05:49 -0800828 ModifiedFileSet.EVERYTHING_DELETED,
ajurkowski64795c22022-04-01 10:38:44 -0700829 /* trustRemoteArtifacts= */ false,
830 (ignored, ignored2) -> {}))
ajurkowskifbd44272021-12-02 16:05:49 -0800831 .containsExactly(actionKey);
832 assertThat(
ajurkowski64795c22022-04-01 10:38:44 -0700833 new FilesystemValueChecker(tsgm, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
ajurkowskifbd44272021-12-02 16:05:49 -0800834 .getDirtyActionValues(
835 evaluator.getValues(),
836 batchStatter,
steinman39c00d22020-03-20 15:23:10 -0700837 new ModifiedFileSet.Builder().modify(file.getExecPath()).build(),
ajurkowski64795c22022-04-01 10:38:44 -0700838 /* trustRemoteArtifacts= */ false,
839 (ignored, ignored2) -> {}))
lberki41cfcc72019-08-05 02:55:36 -0700840 .containsExactly(actionKey);
Ulf Adamsc73051c62016-03-23 09:18:13 +0000841 assertThat(
ajurkowski64795c22022-04-01 10:38:44 -0700842 new FilesystemValueChecker(tsgm, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
lberki41cfcc72019-08-05 02:55:36 -0700843 .getDirtyActionValues(
844 evaluator.getValues(),
845 batchStatter,
846 new ModifiedFileSet.Builder()
847 .modify(file.getExecPath().getParentDirectory())
steinman39c00d22020-03-20 15:23:10 -0700848 .build(),
ajurkowski64795c22022-04-01 10:38:44 -0700849 /* trustRemoteArtifacts= */ false,
850 (ignored, ignored2) -> {}))
lberki41cfcc72019-08-05 02:55:36 -0700851 .isEmpty();
Eric Fellheimere6590722015-11-17 17:07:48 +0000852 assertThat(
ajurkowski64795c22022-04-01 10:38:44 -0700853 new FilesystemValueChecker(tsgm, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
lberki41cfcc72019-08-05 02:55:36 -0700854 .getDirtyActionValues(
steinman39c00d22020-03-20 15:23:10 -0700855 evaluator.getValues(),
856 batchStatter,
857 ModifiedFileSet.NOTHING_MODIFIED,
ajurkowski64795c22022-04-01 10:38:44 -0700858 /* trustRemoteArtifacts= */ false,
859 (ignored, ignored2) -> {}))
lberki41cfcc72019-08-05 02:55:36 -0700860 .isEmpty();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000861 }
862
ajurkowskifbd44272021-12-02 16:05:49 -0800863 enum ModifiedSetReporting {
864 EVERYTHING_MODIFIED {
865 @Override
866 ModifiedFileSet getModifiedFileSet(PathFragment path) {
867 return ModifiedFileSet.EVERYTHING_MODIFIED;
868 }
869 },
870 EVERYTHING_DELETED {
871 @Override
872 ModifiedFileSet getModifiedFileSet(PathFragment path) {
873 return ModifiedFileSet.EVERYTHING_DELETED;
874 }
875 },
876 SINGLE_PATH {
877 @Override
878 ModifiedFileSet getModifiedFileSet(PathFragment path) {
879 return ModifiedFileSet.builder().modify(path).build();
880 }
881 };
882
883 abstract ModifiedFileSet getModifiedFileSet(PathFragment path);
884 }
885
ajurkowskibdc60b02021-11-24 08:47:05 -0800886 @Test
887 public void getDirtyActionValues_touchedTreeDirectory_returnsEmptyDiff(
ajurkowskibdc60b02021-11-24 08:47:05 -0800888 @TestParameter({"", "subdir"}) String touchedTreePath,
ajurkowskifbd44272021-12-02 16:05:49 -0800889 @TestParameter ModifiedSetReporting modifiedSet)
ajurkowskibdc60b02021-11-24 08:47:05 -0800890 throws Exception {
891 SpecialArtifact tree = createTreeArtifact("tree");
892 TreeFileArtifact treeFile = TreeFileArtifact.createTreeOutput(tree, "subdir/file");
893 FileSystemUtils.writeIsoLatin1(treeFile.getPath(), "text");
894 SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
895 differencer.inject(
896 ImmutableMap.of(actionKey, actionValueWithTreeArtifacts(ImmutableList.of(treeFile))));
897 evaluate();
ajurkowskibdc60b02021-11-24 08:47:05 -0800898 FileSystemUtils.touchFile(tree.getPath().getRelative(touchedTreePath));
ajurkowski64795c22022-04-01 10:38:44 -0700899
900 Collection<SkyKey> dirtyActionKeys =
901 new FilesystemValueChecker(/*tsgm=*/ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
902 .getDirtyActionValues(
903 evaluator.getValues(),
904 batchStat.getBatchStat(fs),
905 modifiedSet.getModifiedFileSet(tree.getExecPath()),
906 /*trustRemoteArtifacts=*/ false,
907 mockModifiedOutputsReceiver);
908
909 assertThat(dirtyActionKeys).isEmpty();
910 assertThat(modifiedOutputsCaptor.getAllValues()).isEmpty();
ajurkowskibdc60b02021-11-24 08:47:05 -0800911 }
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000912
ajurkowskibdc60b02021-11-24 08:47:05 -0800913 @Test
914 public void getDirtyActionValues_deleteEmptyTreeDirectory_returnsTreeKey(
Googlerfe72dbb2021-12-29 07:12:53 -0800915 @TestParameter ModifiedSetReporting modifiedSet) throws Exception {
ajurkowskibdc60b02021-11-24 08:47:05 -0800916 SpecialArtifact tree = createTreeArtifact("tree");
917 tree.getPath().createDirectoryAndParents();
918 SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
919 differencer.inject(
920 ImmutableMap.of(actionKey, actionValueWithTreeArtifact(tree, TreeArtifactValue.empty())));
921 evaluate();
ajurkowskibdc60b02021-11-24 08:47:05 -0800922 assertThat(tree.getPath().delete()).isTrue();
ajurkowski64795c22022-04-01 10:38:44 -0700923
924 Collection<SkyKey> dirtyActionKeys =
925 new FilesystemValueChecker(/*tsgm=*/ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
926 .getDirtyActionValues(
927 evaluator.getValues(),
928 batchStat.getBatchStat(fs),
929 modifiedSet.getModifiedFileSet(tree.getExecPath()),
930 /*trustRemoteArtifacts=*/ false,
931 mockModifiedOutputsReceiver);
932
933 assertThat(dirtyActionKeys).containsExactly(actionKey);
934 assertThat(modifiedOutputsCaptor.getAllValues()).containsExactly(tree);
ajurkowskibdc60b02021-11-24 08:47:05 -0800935 }
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000936
ajurkowskibdc60b02021-11-24 08:47:05 -0800937 @Test
Googlerfe72dbb2021-12-29 07:12:53 -0800938 public void getDirtyActionValues_treeDirectoryReplacedWithSymlink_returnsTreeKey()
939 throws Exception {
ajurkowskibdc60b02021-11-24 08:47:05 -0800940 SpecialArtifact tree = createTreeArtifact("tree");
941 tree.getPath().createDirectoryAndParents();
942 SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
943 differencer.inject(
944 ImmutableMap.of(actionKey, actionValueWithTreeArtifact(tree, TreeArtifactValue.empty())));
945 evaluate();
ajurkowskibdc60b02021-11-24 08:47:05 -0800946 Path dummyEmptyDir = fs.getPath("/bin").getRelative("dir");
947 dummyEmptyDir.createDirectoryAndParents();
948 assertThat(tree.getPath().delete()).isTrue();
949 tree.getPath().createSymbolicLink(dummyEmptyDir);
Rumou Duan45e8e572016-06-17 16:43:44 +0000950
ajurkowski64795c22022-04-01 10:38:44 -0700951 Collection<SkyKey> dirtyActionKeys =
952 new FilesystemValueChecker(/*tsgm=*/ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
953 .getDirtyActionValues(
954 evaluator.getValues(),
955 batchStat.getBatchStat(fs),
956 ModifiedFileSet.EVERYTHING_MODIFIED,
957 /* trustRemoteArtifacts= */ false,
958 mockModifiedOutputsReceiver);
959
960 assertThat(dirtyActionKeys).containsExactly(actionKey);
961 assertThat(modifiedOutputsCaptor.getAllValues()).containsExactly(tree);
ajurkowskibdc60b02021-11-24 08:47:05 -0800962 }
963
964 @Test
965 public void getDirtyActionValues_modifiedTreeFile_returnsTreeKey(
Googlerfe72dbb2021-12-29 07:12:53 -0800966 @TestParameter ModifiedSetReporting modifiedSet) throws Exception {
ajurkowskibdc60b02021-11-24 08:47:05 -0800967 SpecialArtifact tree = createTreeArtifact("tree");
968 TreeFileArtifact treeFile = TreeFileArtifact.createTreeOutput(tree, "file");
969 FileSystemUtils.writeIsoLatin1(treeFile.getPath(), "text");
970 SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
971 differencer.inject(
972 ImmutableMap.of(actionKey, actionValueWithTreeArtifacts(ImmutableList.of(treeFile))));
973 evaluate();
ajurkowskibdc60b02021-11-24 08:47:05 -0800974 FileSystemUtils.writeIsoLatin1(treeFile.getPath(), "other text");
ajurkowski64795c22022-04-01 10:38:44 -0700975
976 Collection<SkyKey> dirtyActionKeys =
977 new FilesystemValueChecker(/*tsgm=*/ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
978 .getDirtyActionValues(
979 evaluator.getValues(),
980 batchStat.getBatchStat(fs),
981 modifiedSet.getModifiedFileSet(treeFile.getExecPath()),
982 /*trustRemoteArtifacts=*/ false,
983 mockModifiedOutputsReceiver);
984
985 assertThat(dirtyActionKeys).containsExactly(actionKey);
986 assertThat(modifiedOutputsCaptor.getAllValues()).containsExactly(treeFile);
ajurkowskibdc60b02021-11-24 08:47:05 -0800987 }
988
989 @Test
990 public void getDirtyActionValues_addedTreeFile_returnsTreeKey(
Googlerfe72dbb2021-12-29 07:12:53 -0800991 @TestParameter ModifiedSetReporting modifiedSet) throws Exception {
ajurkowskibdc60b02021-11-24 08:47:05 -0800992 SpecialArtifact tree = createTreeArtifact("tree");
993 TreeFileArtifact treeFile = TreeFileArtifact.createTreeOutput(tree, "file1");
994 FileSystemUtils.writeIsoLatin1(treeFile.getPath());
995 SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
996 differencer.inject(
997 ImmutableMap.of(actionKey, actionValueWithTreeArtifacts(ImmutableList.of(treeFile))));
998 evaluate();
999
1000 TreeFileArtifact newFile = TreeFileArtifact.createTreeOutput(tree, "file2");
1001 FileSystemUtils.writeIsoLatin1(newFile.getPath());
ajurkowski64795c22022-04-01 10:38:44 -07001002 Collection<SkyKey> dirtyActionValues =
1003 new FilesystemValueChecker(/*tsgm=*/ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
1004 .getDirtyActionValues(
1005 evaluator.getValues(),
1006 batchStat.getBatchStat(fs),
1007 modifiedSet.getModifiedFileSet(newFile.getExecPath()),
1008 /*trustRemoteArtifacts=*/ false,
1009 mockModifiedOutputsReceiver);
1010
1011 assertThat(dirtyActionValues).containsExactly(actionKey);
1012 assertThat(modifiedOutputsCaptor.getAllValues()).containsExactly(tree);
ajurkowskibdc60b02021-11-24 08:47:05 -08001013 }
1014
1015 @Test
1016 public void getDirtyActionValues_addedTreeFileToEmptyTree_returnsTreeKey(
Googlerfe72dbb2021-12-29 07:12:53 -08001017 @TestParameter ModifiedSetReporting modifiedSet) throws Exception {
ajurkowskibdc60b02021-11-24 08:47:05 -08001018 SpecialArtifact tree = createTreeArtifact("tree");
1019 tree.getPath().createDirectoryAndParents();
1020 SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
1021 differencer.inject(
1022 ImmutableMap.of(actionKey, actionValueWithTreeArtifact(tree, TreeArtifactValue.empty())));
1023 evaluate();
ajurkowskibdc60b02021-11-24 08:47:05 -08001024 TreeFileArtifact newFile = TreeFileArtifact.createTreeOutput(tree, "file");
1025 FileSystemUtils.writeIsoLatin1(newFile.getPath());
ajurkowski64795c22022-04-01 10:38:44 -07001026
1027 Collection<SkyKey> dirtyActionKeys =
1028 new FilesystemValueChecker(/*tsgm=*/ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
1029 .getDirtyActionValues(
1030 evaluator.getValues(),
1031 batchStat.getBatchStat(fs),
1032 modifiedSet.getModifiedFileSet(newFile.getExecPath()),
1033 /*trustRemoteArtifacts=*/ false,
1034 mockModifiedOutputsReceiver);
1035
1036 assertThat(dirtyActionKeys).containsExactly(actionKey);
1037 assertThat(modifiedOutputsCaptor.getAllValues()).containsExactly(tree);
ajurkowskibdc60b02021-11-24 08:47:05 -08001038 }
1039
1040 @Test
1041 public void getDirtyActionValues_deletedTreeFile_returnsTreeKey(
Googlerfe72dbb2021-12-29 07:12:53 -08001042 @TestParameter ModifiedSetReporting modifiedSet) throws Exception {
ajurkowskibdc60b02021-11-24 08:47:05 -08001043 SpecialArtifact tree = createTreeArtifact("tree");
1044 TreeFileArtifact treeFile = TreeFileArtifact.createTreeOutput(tree, "file");
1045 FileSystemUtils.writeIsoLatin1(treeFile.getPath());
1046 SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
1047 differencer.inject(
1048 ImmutableMap.of(actionKey, actionValueWithTreeArtifacts(ImmutableList.of(treeFile))));
1049 evaluate();
ajurkowskibdc60b02021-11-24 08:47:05 -08001050 assertThat(treeFile.getPath().delete()).isTrue();
ajurkowski64795c22022-04-01 10:38:44 -07001051
1052 Collection<SkyKey> dirtyActionKeys =
1053 new FilesystemValueChecker(/*tsgm=*/ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
1054 .getDirtyActionValues(
1055 evaluator.getValues(),
1056 batchStat.getBatchStat(fs),
1057 modifiedSet.getModifiedFileSet(treeFile.getExecPath()),
1058 /*trustRemoteArtifacts=*/ false,
1059 mockModifiedOutputsReceiver);
1060
1061 assertThat(dirtyActionKeys).containsExactly(actionKey);
1062 assertThat(modifiedOutputsCaptor.getAllValues()).containsExactly(treeFile, tree);
ajurkowskibdc60b02021-11-24 08:47:05 -08001063 }
1064
1065 @Test
Googlerfe72dbb2021-12-29 07:12:53 -08001066 public void getDirtyActionValues_everythingModified_returnsAllKeys() throws Exception {
ajurkowskibdc60b02021-11-24 08:47:05 -08001067 SpecialArtifact tree1 = createTreeArtifact("tree1");
1068 TreeFileArtifact tree1File = TreeFileArtifact.createTreeOutput(tree1, "file");
1069 FileSystemUtils.writeIsoLatin1(tree1File.getPath(), "text");
1070 SpecialArtifact tree2 = createTreeArtifact("tree2");
1071 TreeFileArtifact tree2File = TreeFileArtifact.createTreeOutput(tree2, "file");
1072 FileSystemUtils.writeIsoLatin1(tree2File.getPath());
ajurkowski280bbe22020-08-19 11:26:20 -07001073 SkyKey actionKey1 = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
1074 SkyKey actionKey2 = ActionLookupData.create(ACTION_LOOKUP_KEY, 1);
Michael Thvedte4a7b0792016-02-09 12:15:53 +00001075 differencer.inject(
jhorvitz3daedc32020-07-22 18:33:55 -07001076 ImmutableMap.of(
Janak Ramakrishnan5d152692017-03-20 22:07:14 +00001077 actionKey1,
ajurkowskibdc60b02021-11-24 08:47:05 -08001078 actionValueWithTreeArtifacts(ImmutableList.of(tree1File)),
Janak Ramakrishnan5d152692017-03-20 22:07:14 +00001079 actionKey2,
ajurkowski64795c22022-04-01 10:38:44 -07001080 actionValueWithTreeArtifacts(ImmutableList.of(tree2File))));
ajurkowskibdc60b02021-11-24 08:47:05 -08001081 evaluate();
ajurkowskibdc60b02021-11-24 08:47:05 -08001082 FileSystemUtils.writeIsoLatin1(tree1File.getPath(), "new text");
1083 assertThat(tree2File.getPath().delete()).isTrue();
ajurkowski64795c22022-04-01 10:38:44 -07001084
1085 Collection<SkyKey> dirtyActionKeys =
1086 new FilesystemValueChecker(/*tsgm=*/ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
1087 .getDirtyActionValues(
1088 evaluator.getValues(),
1089 batchStat.getBatchStat(fs),
1090 ModifiedFileSet.EVERYTHING_MODIFIED,
1091 /*trustRemoteArtifacts=*/ false,
1092 mockModifiedOutputsReceiver);
1093
1094 assertThat(dirtyActionKeys).containsExactly(actionKey1, actionKey2);
1095 assertThat(modifiedOutputsCaptor.getAllValues()).containsExactly(tree1File, tree2, tree2File);
ajurkowskibdc60b02021-11-24 08:47:05 -08001096 }
1097
1098 @Test
1099 public void getDirtyActionValues_changedFileNotInModifiedSet_returnsKeysFromSetOnly(
Googlerfe72dbb2021-12-29 07:12:53 -08001100 @TestParameter boolean reportFirst) throws Exception {
ajurkowskibdc60b02021-11-24 08:47:05 -08001101 SpecialArtifact tree1 = createTreeArtifact("tree1");
1102 TreeFileArtifact tree1File = TreeFileArtifact.createTreeOutput(tree1, "file");
1103 FileSystemUtils.writeIsoLatin1(tree1File.getPath(), "text");
1104 SpecialArtifact tree2 = createTreeArtifact("tree2");
1105 TreeFileArtifact tree2File = TreeFileArtifact.createTreeOutput(tree2, "file");
1106 FileSystemUtils.writeIsoLatin1(tree2File.getPath());
1107 SkyKey actionKey1 = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
1108 SkyKey actionKey2 = ActionLookupData.create(ACTION_LOOKUP_KEY, 1);
1109 differencer.inject(
1110 ImmutableMap.of(
1111 actionKey1,
1112 actionValueWithTreeArtifacts(ImmutableList.of(tree1File)),
1113 actionKey2,
1114 actionValueWithTreeArtifacts(ImmutableList.of(tree2File))));
1115 evaluate();
ajurkowskibdc60b02021-11-24 08:47:05 -08001116 FileSystemUtils.writeIsoLatin1(tree1File.getPath(), "new text");
1117 assertThat(tree2File.getPath().delete()).isTrue();
ajurkowski64795c22022-04-01 10:38:44 -07001118
1119 Collection<SkyKey> dirtyActionKeys =
1120 new FilesystemValueChecker(/*tsgm=*/ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
1121 .getDirtyActionValues(
1122 evaluator.getValues(),
1123 batchStat.getBatchStat(fs),
1124 ModifiedFileSet.builder()
1125 .modify((reportFirst ? tree1File : tree2File).getExecPath())
1126 .build(),
1127 /*trustRemoteArtifacts=*/ false,
1128 mockModifiedOutputsReceiver);
1129
1130 assertThat(dirtyActionKeys).containsExactly(reportFirst ? actionKey1 : actionKey2);
1131 assertThat(modifiedOutputsCaptor.getAllValues())
1132 .containsExactlyElementsIn(
1133 reportFirst ? ImmutableList.of(tree1File) : ImmutableList.of(tree2File, tree2));
ajurkowskibdc60b02021-11-24 08:47:05 -08001134 }
1135
1136 @Test
Googlerfe72dbb2021-12-29 07:12:53 -08001137 public void getDirtyActionValues_middleFileSkippedInModifiedFileSet_returnsKeysFromSetOnly()
1138 throws Exception {
ajurkowskibdc60b02021-11-24 08:47:05 -08001139 SpecialArtifact treeA = createTreeArtifact("a_tree");
1140 TreeFileArtifact treeAFile = TreeFileArtifact.createTreeOutput(treeA, "file");
1141 FileSystemUtils.writeIsoLatin1(treeAFile.getPath());
1142 SpecialArtifact treeB = createTreeArtifact("b_tree");
1143 TreeFileArtifact treeBFile = TreeFileArtifact.createTreeOutput(treeB, "file");
1144 FileSystemUtils.writeIsoLatin1(treeBFile.getPath());
1145 SpecialArtifact treeC = createTreeArtifact("c_tree");
1146 TreeFileArtifact treeCFile = TreeFileArtifact.createTreeOutput(treeC, "file");
1147 FileSystemUtils.writeIsoLatin1(treeCFile.getPath());
1148 SkyKey actionKey1 = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
1149 SkyKey actionKey2 = ActionLookupData.create(ACTION_LOOKUP_KEY, 1);
1150 SkyKey actionKey3 = ActionLookupData.create(ACTION_LOOKUP_KEY, 2);
1151 differencer.inject(
1152 ImmutableMap.of(
1153 actionKey1,
1154 actionValueWithTreeArtifacts(ImmutableList.of(treeAFile)),
1155 actionKey2,
1156 actionValueWithTreeArtifacts(ImmutableList.of(treeBFile)),
1157 actionKey3,
1158 actionValueWithTreeArtifacts(ImmutableList.of(treeCFile))));
1159 evaluate();
ajurkowskibdc60b02021-11-24 08:47:05 -08001160 assertThat(treeAFile.getPath().delete()).isTrue();
1161 assertThat(treeBFile.getPath().delete()).isTrue();
1162 assertThat(treeCFile.getPath().delete()).isTrue();
ajurkowski64795c22022-04-01 10:38:44 -07001163
1164 Collection<SkyKey> dirtyActionKeys =
1165 new FilesystemValueChecker(/*tsgm=*/ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
1166 .getDirtyActionValues(
1167 evaluator.getValues(),
1168 batchStat.getBatchStat(fs),
1169 ModifiedFileSet.builder()
1170 .modify(treeAFile.getExecPath())
1171 .modify(treeCFile.getExecPath())
1172 .build(),
1173 /*trustRemoteArtifacts=*/ false,
1174 mockModifiedOutputsReceiver);
1175
1176 assertThat(dirtyActionKeys).containsExactly(actionKey1, actionKey3);
1177 assertThat(modifiedOutputsCaptor.getAllValues())
1178 .containsExactly(treeAFile, treeA, treeCFile, treeC);
ajurkowskibdc60b02021-11-24 08:47:05 -08001179 }
1180
1181 @Test
Googlerfe72dbb2021-12-29 07:12:53 -08001182 public void getDirtyActionValues_nothingModified_returnsEmptyDiff() throws Exception {
ajurkowskibdc60b02021-11-24 08:47:05 -08001183 SpecialArtifact tree = createTreeArtifact("tree");
1184 TreeFileArtifact treeFile = TreeFileArtifact.createTreeOutput(tree, "file");
1185 FileSystemUtils.writeIsoLatin1(treeFile.getPath());
1186 SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
1187 differencer.inject(
1188 ImmutableMap.of(actionKey, actionValueWithTreeArtifacts(ImmutableList.of(treeFile))));
1189 evaluate();
ajurkowskibdc60b02021-11-24 08:47:05 -08001190 assertThat(treeFile.getPath().delete()).isTrue();
ajurkowski64795c22022-04-01 10:38:44 -07001191
1192 Collection<SkyKey> dirtyActionKeys =
1193 new FilesystemValueChecker(/*tsgm=*/ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
1194 .getDirtyActionValues(
1195 evaluator.getValues(),
1196 batchStat.getBatchStat(fs),
1197 ModifiedFileSet.NOTHING_MODIFIED,
1198 /*trustRemoteArtifacts=*/ false,
1199 mockModifiedOutputsReceiver);
1200
1201 assertThat(dirtyActionKeys).isEmpty();
1202 assertThat(modifiedOutputsCaptor.getAllValues()).isEmpty();
ajurkowskibdc60b02021-11-24 08:47:05 -08001203 }
1204
1205 private void evaluate() throws InterruptedException {
Googler10028672018-10-25 12:14:34 -07001206 EvaluationContext evaluationContext =
1207 EvaluationContext.newBuilder()
1208 .setKeepGoing(false)
1209 .setNumThreads(1)
michajlo7a485be2020-07-30 11:08:46 -07001210 .setEventHandler(NullEventHandler.INSTANCE)
Googler10028672018-10-25 12:14:34 -07001211 .build();
jhorvitz9ad89182021-12-29 11:57:02 -08001212 assertThat(evaluator.evaluate(ImmutableList.of(), evaluationContext).hasError()).isFalse();
Michael Thvedte4a7b0792016-02-09 12:15:53 +00001213 }
1214
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001215 private Artifact createDerivedArtifact(String relPath) throws IOException {
janakr448f1cf2020-03-30 09:12:44 -07001216 String outSegment = "bin";
1217 Path outputPath = fs.getPath("/" + outSegment);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001218 outputPath.createDirectory();
janakraea05602019-05-22 15:41:29 -07001219 return ActionsTestUtil.createArtifact(
Googlerf0b0c392021-01-27 17:56:52 -08001220 ArtifactRoot.asDerivedRoot(fs.getPath("/"), RootType.Output, outSegment),
Googler147c2872021-01-21 08:47:48 -08001221 outputPath.getRelative(relPath));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001222 }
1223
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +00001224 @Test
steinman7184cb32020-04-20 14:33:03 -07001225 // TODO(b/154337187): Remove the following annotation to re-enable once this test is de-flaked.
1226 @Ignore
1227 public void testDirtyActions() throws Exception {
jhorvitz3daedc32020-07-22 18:33:55 -07001228 checkDirtyActions(null);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001229 }
1230
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +00001231 @Test
steinman7184cb32020-04-20 14:33:03 -07001232 // TODO(b/154337187): Remove the following annotation to re-enable once this test is de-flaked.
1233 @Ignore
1234 public void testDirtyActionsBatchStat() throws Exception {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001235 checkDirtyActions(
1236 new BatchStat() {
1237 @Override
1238 public List<FileStatusWithDigest> batchStat(
1239 boolean useDigest, boolean includeLinks, Iterable<PathFragment> paths)
1240 throws IOException {
1241 List<FileStatusWithDigest> stats = new ArrayList<>();
1242 for (PathFragment pathFrag : paths) {
1243 stats.add(
janakr61b3bb22022-03-07 10:11:00 -08001244 FileStatusWithDigestAdapter.maybeAdapt(
tomlu6c919062018-01-11 17:32:09 -08001245 fs.getPath("/").getRelative(pathFrag).statIfFound(Symlinks.NOFOLLOW)));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001246 }
1247 return stats;
1248 }
jhorvitz3daedc32020-07-22 18:33:55 -07001249 });
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001250 }
1251
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +00001252 @Test
steinman68e78f52020-04-23 12:30:19 -07001253 // TODO(b/154337187): Remove the following annotation to re-enable once this test is de-flaked.
1254 @Ignore
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001255 public void testDirtyActionsBatchStatWithDigest() throws Exception {
1256 checkDirtyActions(
1257 new BatchStat() {
1258 @Override
1259 public List<FileStatusWithDigest> batchStat(
1260 boolean useDigest, boolean includeLinks, Iterable<PathFragment> paths)
1261 throws IOException {
1262 List<FileStatusWithDigest> stats = new ArrayList<>();
1263 for (PathFragment pathFrag : paths) {
tomlu6c919062018-01-11 17:32:09 -08001264 final Path path = fs.getPath("/").getRelative(pathFrag);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001265 stats.add(statWithDigest(path, path.statIfFound(Symlinks.NOFOLLOW)));
1266 }
1267 return stats;
1268 }
jhorvitz3daedc32020-07-22 18:33:55 -07001269 });
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001270 }
1271
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +00001272 @Test
steinman68e78f52020-04-23 12:30:19 -07001273 // TODO(b/154337187): Remove the following annotation to re-enable once this test is de-flaked.
1274 @Ignore
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001275 public void testDirtyActionsBatchStatFallback() throws Exception {
1276 checkDirtyActions(
1277 new BatchStat() {
1278 @Override
1279 public List<FileStatusWithDigest> batchStat(
1280 boolean useDigest, boolean includeLinks, Iterable<PathFragment> paths)
1281 throws IOException {
1282 throw new IOException("try again");
1283 }
jhorvitz3daedc32020-07-22 18:33:55 -07001284 });
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001285 }
1286
Michael Thvedte4a7b0792016-02-09 12:15:53 +00001287 // TODO(bazel-team): Add some tests for FileSystemValueChecker#changedKeys*() methods.
1288 // Presently these appear to be untested.
1289
ajurkowskie2982912020-04-09 10:32:08 -07001290 private static ActionExecutionValue actionValue(Action action) {
lberkif7eee1e2019-07-31 05:49:10 -07001291 Map<Artifact, FileArtifactValue> artifactData = new HashMap<>();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001292 for (Artifact output : action.getOutputs()) {
1293 try {
1294 Path path = output.getPath();
lberkic35878a2019-08-01 02:28:54 -07001295 FileArtifactValue noDigest =
1296 ActionMetadataHandler.fileArtifactValueFromArtifact(
1297 output,
janakr61b3bb22022-03-07 10:11:00 -08001298 FileStatusWithDigestAdapter.maybeAdapt(path.statIfFound(Symlinks.NOFOLLOW)),
janakr95278b42022-03-04 13:41:46 -08001299 SyscallCache.NO_CACHE,
lberkic35878a2019-08-01 02:28:54 -07001300 null);
1301 FileArtifactValue withDigest =
jhorvitz7f55cb72021-12-16 18:52:24 -08001302 FileArtifactValue.createFromInjectedDigest(noDigest, path.getDigest());
lberkic35878a2019-08-01 02:28:54 -07001303 artifactData.put(output, withDigest);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001304 } catch (IOException e) {
1305 throw new IllegalStateException(e);
1306 }
1307 }
jhorvitz7f55cb72021-12-16 18:52:24 -08001308 return ActionExecutionValue.createForTesting(
jhorvitz8470e432021-12-14 09:51:17 -08001309 ImmutableMap.copyOf(artifactData),
Googler974879d2020-05-27 13:25:52 -07001310 /*treeArtifactData=*/ ImmutableMap.of(),
jhorvitz7f55cb72021-12-16 18:52:24 -08001311 /*outputSymlinks=*/ null);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001312 }
1313
jhorvitzed7ec3b2020-07-24 15:08:03 -07001314 private static ActionExecutionValue actionValueWithTreeArtifact(
1315 SpecialArtifact output, TreeArtifactValue tree) {
jhorvitz7f55cb72021-12-16 18:52:24 -08001316 return ActionExecutionValue.createForTesting(
1317 ImmutableMap.of(), ImmutableMap.of(output, tree), /*outputSymlinks=*/ null);
buchgr4992ae22019-03-20 04:23:32 -07001318 }
1319
ajurkowskie2982912020-04-09 10:32:08 -07001320 private static ActionExecutionValue actionValueWithRemoteArtifact(
buchgr4992ae22019-03-20 04:23:32 -07001321 Artifact output, RemoteFileArtifactValue value) {
jhorvitz7f55cb72021-12-16 18:52:24 -08001322 return ActionExecutionValue.createForTesting(
1323 ImmutableMap.of(output, value), ImmutableMap.of(), /*outputSymlinks=*/ null);
buchgr4992ae22019-03-20 04:23:32 -07001324 }
1325
1326 private RemoteFileArtifactValue createRemoteFileArtifactValue(String contents) {
1327 byte[] data = contents.getBytes();
1328 DigestHashFunction hashFn = fs.getDigestFunction();
1329 HashCode hash = hashFn.getHashFunction().hashBytes(data);
George Gensure3ef8fb92020-05-06 09:49:48 -07001330 return new RemoteFileArtifactValue(hash.asBytes(), data.length, -1, "action-id");
buchgr4992ae22019-03-20 04:23:32 -07001331 }
1332
1333 @Test
1334 public void testRemoteAndLocalArtifacts() throws Exception {
1335 // Test that injected remote artifacts are trusted by the FileSystemValueChecker
steinman39c00d22020-03-20 15:23:10 -07001336 // if it is configured to trust remote artifacts, and that local files always take precedence
1337 // over remote files.
ajurkowski280bbe22020-08-19 11:26:20 -07001338 SkyKey actionKey1 = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
1339 SkyKey actionKey2 = ActionLookupData.create(ACTION_LOOKUP_KEY, 1);
buchgr4992ae22019-03-20 04:23:32 -07001340
1341 Artifact out1 = createDerivedArtifact("foo");
1342 Artifact out2 = createDerivedArtifact("bar");
1343 Map<SkyKey, SkyValue> metadataToInject = new HashMap<>();
1344 metadataToInject.put(
1345 actionKey1,
1346 actionValueWithRemoteArtifact(out1, createRemoteFileArtifactValue("foo-content")));
1347 metadataToInject.put(
1348 actionKey2,
1349 actionValueWithRemoteArtifact(out2, createRemoteFileArtifactValue("bar-content")));
1350 differencer.inject(metadataToInject);
1351
1352 EvaluationContext evaluationContext =
1353 EvaluationContext.newBuilder()
1354 .setKeepGoing(false)
1355 .setNumThreads(1)
michajlo7a485be2020-07-30 11:08:46 -07001356 .setEventHandler(NullEventHandler.INSTANCE)
buchgr4992ae22019-03-20 04:23:32 -07001357 .build();
1358 assertThat(
jhorvitz9ad89182021-12-29 11:57:02 -08001359 evaluator
1360 .evaluate(ImmutableList.of(actionKey1, actionKey2), evaluationContext)
1361 .hasError())
buchgr4992ae22019-03-20 04:23:32 -07001362 .isFalse();
1363 assertThat(
ajurkowski64795c22022-04-01 10:38:44 -07001364 new FilesystemValueChecker(/*tsgm=*/ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
buchgr4992ae22019-03-20 04:23:32 -07001365 .getDirtyActionValues(
1366 evaluator.getValues(),
1367 /* batchStatter= */ null,
steinman39c00d22020-03-20 15:23:10 -07001368 ModifiedFileSet.EVERYTHING_MODIFIED,
ajurkowski64795c22022-04-01 10:38:44 -07001369 /* trustRemoteArtifacts= */ true,
1370 (ignored, ignored2) -> {}))
buchgr4992ae22019-03-20 04:23:32 -07001371 .isEmpty();
1372
1373 // Create the "out1" artifact on the filesystem and test that it invalidates the generating
1374 // action's SkyKey.
1375 FileSystemUtils.writeContentAsLatin1(out1.getPath(), "new-foo-content");
1376 assertThat(
ajurkowski64795c22022-04-01 10:38:44 -07001377 new FilesystemValueChecker(/*tsgm=*/ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
buchgr4992ae22019-03-20 04:23:32 -07001378 .getDirtyActionValues(
1379 evaluator.getValues(),
1380 /* batchStatter= */ null,
steinman39c00d22020-03-20 15:23:10 -07001381 ModifiedFileSet.EVERYTHING_MODIFIED,
ajurkowski64795c22022-04-01 10:38:44 -07001382 /* trustRemoteArtifacts= */ true,
1383 (ignored, ignored2) -> {}))
buchgr4992ae22019-03-20 04:23:32 -07001384 .containsExactly(actionKey1);
1385 }
1386
1387 @Test
1388 public void testRemoteAndLocalTreeArtifacts() throws Exception {
1389 // Test that injected remote tree artifacts are trusted by the FileSystemValueChecker
1390 // and that local files always takes preference over remote files.
ajurkowski280bbe22020-08-19 11:26:20 -07001391 SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
buchgr4992ae22019-03-20 04:23:32 -07001392
1393 SpecialArtifact treeArtifact = createTreeArtifact("dir");
1394 treeArtifact.getPath().createDirectoryAndParents();
jhorvitzed7ec3b2020-07-24 15:08:03 -07001395 TreeArtifactValue tree =
1396 TreeArtifactValue.newBuilder(treeArtifact)
1397 .putChild(
1398 TreeFileArtifact.createTreeOutput(treeArtifact, "foo"),
1399 createRemoteFileArtifactValue("foo-content"))
1400 .putChild(
1401 TreeFileArtifact.createTreeOutput(treeArtifact, "bar"),
1402 createRemoteFileArtifactValue("bar-content"))
1403 .build();
buchgr4992ae22019-03-20 04:23:32 -07001404
jhorvitzed7ec3b2020-07-24 15:08:03 -07001405 differencer.inject(ImmutableMap.of(actionKey, actionValueWithTreeArtifact(treeArtifact, tree)));
buchgr4992ae22019-03-20 04:23:32 -07001406
1407 EvaluationContext evaluationContext =
1408 EvaluationContext.newBuilder()
1409 .setKeepGoing(false)
1410 .setNumThreads(1)
michajlo7a485be2020-07-30 11:08:46 -07001411 .setEventHandler(NullEventHandler.INSTANCE)
buchgr4992ae22019-03-20 04:23:32 -07001412 .build();
jhorvitz9ad89182021-12-29 11:57:02 -08001413 assertThat(evaluator.evaluate(ImmutableList.of(actionKey), evaluationContext).hasError())
buchgr4992ae22019-03-20 04:23:32 -07001414 .isFalse();
1415 assertThat(
ajurkowski64795c22022-04-01 10:38:44 -07001416 new FilesystemValueChecker(/*tsgm=*/ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
buchgr4992ae22019-03-20 04:23:32 -07001417 .getDirtyActionValues(
1418 evaluator.getValues(),
1419 /* batchStatter= */ null,
steinman39c00d22020-03-20 15:23:10 -07001420 ModifiedFileSet.EVERYTHING_MODIFIED,
ajurkowski64795c22022-04-01 10:38:44 -07001421 /* trustRemoteArtifacts= */ false,
1422 (ignored, ignored2) -> {}))
buchgr4992ae22019-03-20 04:23:32 -07001423 .isEmpty();
1424
1425 // Create dir/foo on the local disk and test that it invalidates the associated sky key.
Googler1d8d1382020-05-18 12:10:49 -07001426 TreeFileArtifact fooArtifact = TreeFileArtifact.createTreeOutput(treeArtifact, "foo");
buchgr4992ae22019-03-20 04:23:32 -07001427 FileSystemUtils.writeContentAsLatin1(fooArtifact.getPath(), "new-foo-content");
1428 assertThat(
ajurkowski64795c22022-04-01 10:38:44 -07001429 new FilesystemValueChecker(/*tsgm=*/ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
buchgr4992ae22019-03-20 04:23:32 -07001430 .getDirtyActionValues(
1431 evaluator.getValues(),
1432 /* batchStatter= */ null,
steinman39c00d22020-03-20 15:23:10 -07001433 ModifiedFileSet.EVERYTHING_MODIFIED,
ajurkowski64795c22022-04-01 10:38:44 -07001434 /* trustRemoteArtifacts= */ false,
1435 (ignored, ignored2) -> {}))
buchgr4992ae22019-03-20 04:23:32 -07001436 .containsExactly(actionKey);
1437 }
1438
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +00001439 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001440 public void testPropagatesRuntimeExceptions() throws Exception {
tomluee6a6862018-01-17 14:36:26 -08001441 Collection<SkyKey> values =
1442 ImmutableList.of(
1443 FileValue.key(
1444 RootedPath.toRootedPath(Root.fromPath(pkgRoot), PathFragment.create("foo"))));
jhorvitz9ad89182021-12-29 11:57:02 -08001445 evaluator.evaluate(values, EVALUATION_OPTIONS);
janakrc9f44ec2021-07-01 08:13:09 -07001446 AtomicReference<Throwable> uncaughtRef = new AtomicReference<>();
janakr933c3012021-07-13 07:44:15 -07001447 CountDownLatch throwableCaught = new CountDownLatch(1);
janakrc9f44ec2021-07-01 08:13:09 -07001448 Thread.UncaughtExceptionHandler uncaughtExceptionHandler =
janakr933c3012021-07-13 07:44:15 -07001449 (t, e) -> {
1450 uncaughtRef.compareAndSet(null, e);
1451 throwableCaught.countDown();
1452 };
janakrc9f44ec2021-07-01 08:13:09 -07001453 Thread.setDefaultUncaughtExceptionHandler(uncaughtExceptionHandler);
twerth5aaceb52020-04-07 06:31:56 -07001454 FilesystemValueChecker checker =
1455 new FilesystemValueChecker(
janakr873bf0a2022-03-04 13:38:50 -08001456 /*tsgm=*/ null,
1457 SyscallCache.NO_CACHE,
janakr873bf0a2022-03-04 13:38:50 -08001458 FSVC_THREADS_FOR_TEST);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001459
Nathan Harmata8cd29782015-11-10 03:24:01 +00001460 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001461
1462 fs.statThrowsRuntimeException = true;
janakrc9f44ec2021-07-01 08:13:09 -07001463 getDirtyFilesystemKeys(evaluator, checker);
janakr933c3012021-07-13 07:44:15 -07001464 // Wait for exception handler to trigger (FVC doesn't clean up crashing threads on its own).
1465 assertThat(throwableCaught.await(TestUtils.WAIT_TIMEOUT_SECONDS, SECONDS)).isTrue();
1466 Throwable thrown = uncaughtRef.get();
1467 assertThat(thrown).isNotNull();
1468 assertThat(thrown).hasMessageThat().isEqualTo("bork");
1469 assertThat(thrown).isInstanceOf(RuntimeException.class);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001470 }
1471
1472 private static void assertEmptyDiff(Diff diff) {
1473 assertDiffWithNewValues(diff);
1474 }
1475
1476 private static void assertDiffWithNewValues(Diff diff, SkyKey... keysWithNewValues) {
1477 assertThat(diff.changedKeysWithoutNewValues()).isEmpty();
1478 assertThat(diff.changedKeysWithNewValues().keySet())
1479 .containsExactlyElementsIn(Arrays.asList(keysWithNewValues));
1480 }
1481
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001482 private static FileStatusWithDigest statWithDigest(final Path path, final FileStatus stat) {
1483 return new FileStatusWithDigest() {
1484 @Nullable
1485 @Override
1486 public byte[] getDigest() throws IOException {
olaolabfd1d332017-06-19 16:55:24 -04001487 return path.getDigest();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001488 }
1489
1490 @Override
1491 public boolean isFile() {
1492 return stat.isFile();
1493 }
1494
1495 @Override
1496 public boolean isSpecialFile() {
1497 return stat.isSpecialFile();
1498 }
1499
1500 @Override
1501 public boolean isDirectory() {
1502 return stat.isDirectory();
1503 }
1504
1505 @Override
1506 public boolean isSymbolicLink() {
1507 return stat.isSymbolicLink();
1508 }
1509
1510 @Override
1511 public long getSize() throws IOException {
1512 return stat.getSize();
1513 }
1514
1515 @Override
1516 public long getLastModifiedTime() throws IOException {
1517 return stat.getLastModifiedTime();
1518 }
1519
1520 @Override
1521 public long getLastChangeTime() throws IOException {
1522 return stat.getLastChangeTime();
1523 }
1524
1525 @Override
1526 public long getNodeId() throws IOException {
1527 return stat.getNodeId();
1528 }
1529 };
1530 }
1531
Nathan Harmata8cd29782015-11-10 03:24:01 +00001532 private static Diff getDirtyFilesystemKeys(MemoizingEvaluator evaluator,
1533 FilesystemValueChecker checker) throws InterruptedException {
1534 return checker.getDirtyKeys(evaluator.getValues(), new BasicFilesystemDirtinessChecker());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001535 }
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001536}