blob: 12da7a12a27247c8a373a95eeebb5f453ed9a5ac [file] [log] [blame]
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001// Copyright 2015 The Bazel Authors. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package com.google.devtools.build.lib.skyframe;
15
16import static com.google.common.truth.Truth.assertThat;
lberkiaea56b32017-05-30 12:35:33 +020017import static com.google.common.truth.Truth.assertWithMessage;
michajlo660d17f2020-03-27 09:01:57 -070018import static org.junit.Assert.assertThrows;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000019
20import com.google.common.collect.ImmutableList;
21import com.google.common.collect.ImmutableMap;
22import com.google.common.collect.ImmutableSet;
buchgr4992ae22019-03-20 04:23:32 -070023import com.google.common.hash.HashCode;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000024import com.google.common.util.concurrent.Runnables;
25import com.google.devtools.build.lib.actions.Action;
buchgr4992ae22019-03-20 04:23:32 -070026import com.google.devtools.build.lib.actions.ActionInputHelper;
janakrefb3f152019-06-05 17:42:34 -070027import com.google.devtools.build.lib.actions.ActionLookupData;
janakr93e3eea2017-03-30 22:09:37 +000028import com.google.devtools.build.lib.actions.ActionLookupValue.ActionLookupKey;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000029import com.google.devtools.build.lib.actions.Artifact;
Michael Thvedte4a7b0792016-02-09 12:15:53 +000030import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
31import com.google.devtools.build.lib.actions.Artifact.SpecialArtifactType;
Rumou Duana77f32c2016-04-13 21:59:21 +000032import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
tomlu1cdcdf92018-01-16 11:07:51 -080033import com.google.devtools.build.lib.actions.ArtifactRoot;
shahan602cc852018-06-06 20:09:57 -070034import com.google.devtools.build.lib.actions.FileArtifactValue;
buchgr4992ae22019-03-20 04:23:32 -070035import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue;
shahan602cc852018-06-06 20:09:57 -070036import com.google.devtools.build.lib.actions.FileStateValue;
37import com.google.devtools.build.lib.actions.FileValue;
janakraea05602019-05-22 15:41:29 -070038import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000039import com.google.devtools.build.lib.actions.util.TestAction;
Kristina Chodorowf9fdc8d2015-12-08 12:49:31 +000040import com.google.devtools.build.lib.analysis.BlazeDirectories;
janakr3b63a4e2017-09-14 09:55:40 +020041import com.google.devtools.build.lib.analysis.ServerDirectories;
lberki41cfcc72019-08-05 02:55:36 -070042import com.google.devtools.build.lib.clock.BlazeClock;
Kristina Chodorowf9fdc8d2015-12-08 12:49:31 +000043import com.google.devtools.build.lib.cmdline.PackageIdentifier;
ulfjack1e1a7752019-12-10 21:17:58 -080044import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
45import com.google.devtools.build.lib.collect.nestedset.Order;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000046import com.google.devtools.build.lib.events.NullEventHandler;
janakr15e15c22019-01-30 11:24:49 -080047import com.google.devtools.build.lib.packages.WorkspaceFileValue;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000048import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
49import com.google.devtools.build.lib.skyframe.DirtinessCheckerUtils.BasicFilesystemDirtinessChecker;
Nathan Harmatad4f75942016-10-18 08:55:17 +000050import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction;
John Cater5e9ce942016-10-12 17:23:30 +000051import com.google.devtools.build.lib.skyframe.PackageLookupFunction.CrossRepositoryLabelViolationStrategy;
Luis Fernando Pino Duquebe102182016-05-23 14:03:55 +000052import com.google.devtools.build.lib.testutil.TestConstants;
michajlo8083e322020-03-20 13:32:52 -070053import com.google.devtools.build.lib.testutil.TestPackageFactoryBuilderFactory;
Kristina Chodorowf9fdc8d2015-12-08 12:49:31 +000054import com.google.devtools.build.lib.testutil.TestRuleClassProvider;
laszlocsomor29bdf632018-03-13 02:50:46 -070055import com.google.devtools.build.lib.testutil.TimestampGranularityUtils;
Laszlo Csomora278aec2018-03-09 04:07:31 -080056import com.google.devtools.build.lib.util.io.OutErr;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000057import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
58import com.google.devtools.build.lib.vfs.BatchStat;
buchgr4992ae22019-03-20 04:23:32 -070059import com.google.devtools.build.lib.vfs.DigestHashFunction;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000060import com.google.devtools.build.lib.vfs.FileStatus;
61import com.google.devtools.build.lib.vfs.FileStatusWithDigest;
62import com.google.devtools.build.lib.vfs.FileStatusWithDigestAdapter;
63import com.google.devtools.build.lib.vfs.FileSystemUtils;
Eric Fellheimere6590722015-11-17 17:07:48 +000064import com.google.devtools.build.lib.vfs.ModifiedFileSet;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000065import com.google.devtools.build.lib.vfs.Path;
66import com.google.devtools.build.lib.vfs.PathFragment;
tomluee6a6862018-01-17 14:36:26 -080067import com.google.devtools.build.lib.vfs.Root;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000068import com.google.devtools.build.lib.vfs.RootedPath;
69import com.google.devtools.build.lib.vfs.Symlinks;
djasper7faa0ef2019-03-28 10:00:00 -070070import com.google.devtools.build.lib.vfs.UnixGlob;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000071import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
72import com.google.devtools.build.skyframe.Differencer.Diff;
Googler10028672018-10-25 12:14:34 -070073import com.google.devtools.build.skyframe.EvaluationContext;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000074import com.google.devtools.build.skyframe.EvaluationResult;
75import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator;
76import com.google.devtools.build.skyframe.MemoizingEvaluator;
77import com.google.devtools.build.skyframe.RecordingDifferencer;
janakr1cde8722017-10-10 03:22:21 +020078import com.google.devtools.build.skyframe.SequencedRecordingDifferencer;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000079import com.google.devtools.build.skyframe.SequentialBuildDriver;
80import com.google.devtools.build.skyframe.SkyFunction;
81import com.google.devtools.build.skyframe.SkyFunctionName;
82import com.google.devtools.build.skyframe.SkyKey;
83import com.google.devtools.build.skyframe.SkyValue;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000084import java.io.IOException;
85import java.util.ArrayList;
86import java.util.Arrays;
87import java.util.Collection;
buchgr4992ae22019-03-20 04:23:32 -070088import java.util.Collections;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000089import java.util.HashMap;
90import java.util.List;
91import java.util.Map;
92import java.util.UUID;
93import java.util.concurrent.atomic.AtomicReference;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000094import javax.annotation.Nullable;
Janak Ramakrishnancba16452016-07-29 02:17:02 +000095import org.junit.Before;
96import org.junit.Test;
97import org.junit.runner.RunWith;
98import org.junit.runners.JUnit4;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +000099
100/**
101 * Tests for {@link FilesystemValueChecker}.
102 */
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000103@RunWith(JUnit4.class)
104public class FilesystemValueCheckerTest {
Googler10028672018-10-25 12:14:34 -0700105 private static final EvaluationContext EVALUATION_OPTIONS =
106 EvaluationContext.newBuilder()
107 .setKeepGoing(false)
108 .setNumThreads(SkyframeExecutor.DEFAULT_THREAD_COUNT)
109 .setEventHander(NullEventHandler.INSTANCE)
110 .build();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000111
112 private RecordingDifferencer differencer;
113 private MemoizingEvaluator evaluator;
114 private SequentialBuildDriver driver;
115 private MockFileSystem fs;
116 private Path pkgRoot;
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000117
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000118 @Before
Florian Weikert92b22362015-12-03 10:17:18 +0000119 public final void setUp() throws Exception {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000120 ImmutableMap.Builder<SkyFunctionName, SkyFunction> skyFunctions = ImmutableMap.builder();
121
122 fs = new MockFileSystem();
123 pkgRoot = fs.getPath("/testroot");
Kristina Chodorowf9fdc8d2015-12-08 12:49:31 +0000124 FileSystemUtils.createDirectoryAndParents(pkgRoot);
125 FileSystemUtils.createEmptyFile(pkgRoot.getRelative("WORKSPACE"));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000126
John Catere0d1d0e2017-11-28 20:47:41 -0800127 AtomicReference<PathPackageLocator> pkgLocator =
128 new AtomicReference<>(
129 new PathPackageLocator(
130 fs.getPath("/output_base"),
tomluee6a6862018-01-17 14:36:26 -0800131 ImmutableList.of(Root.fromPath(pkgRoot)),
John Catere0d1d0e2017-11-28 20:47:41 -0800132 BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY));
janakr3b63a4e2017-09-14 09:55:40 +0200133 BlazeDirectories directories =
134 new BlazeDirectories(
cushon849df362018-05-14 01:51:45 -0700135 new ServerDirectories(pkgRoot, pkgRoot, pkgRoot),
136 pkgRoot,
137 /* defaultSystemJavabase= */ null,
138 TestConstants.PRODUCT_NAME);
nharmata3fb7d342018-02-23 11:37:51 -0800139 ExternalFilesHelper externalFilesHelper = ExternalFilesHelper.createForTesting(
Nathan Harmatad4f75942016-10-18 08:55:17 +0000140 pkgLocator, ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS, directories);
shahan602cc852018-06-06 20:09:57 -0700141 skyFunctions.put(
142 FileStateValue.FILE_STATE,
143 new FileStateFunction(
djasper7faa0ef2019-03-28 10:00:00 -0700144 new AtomicReference<TimestampGranularityMonitor>(),
145 new AtomicReference<>(UnixGlob.DEFAULT_SYSCALLS),
146 externalFilesHelper));
shahan602cc852018-06-06 20:09:57 -0700147 skyFunctions.put(FileValue.FILE, new FileFunction(pkgLocator));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000148 skyFunctions.put(
149 SkyFunctions.FILE_SYMLINK_CYCLE_UNIQUENESS, new FileSymlinkCycleUniquenessFunction());
150 skyFunctions.put(
151 SkyFunctions.FILE_SYMLINK_INFINITE_EXPANSION_UNIQUENESS,
152 new FileSymlinkInfiniteExpansionUniquenessFunction());
Kristina Chodorowf9fdc8d2015-12-08 12:49:31 +0000153 skyFunctions.put(SkyFunctions.PACKAGE,
154 new PackageFunction(null, null, null, null, null, null, null));
John Cater5e9ce942016-10-12 17:23:30 +0000155 skyFunctions.put(
156 SkyFunctions.PACKAGE_LOOKUP,
157 new PackageLookupFunction(
158 new AtomicReference<>(ImmutableSet.<PackageIdentifier>of()),
John Cater0c0735a2016-11-11 01:52:02 +0000159 CrossRepositoryLabelViolationStrategy.ERROR,
John Catere0d1d0e2017-11-28 20:47:41 -0800160 BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY));
Damien Martin-Guillerez4ecfe512016-01-22 11:03:23 +0000161 skyFunctions.put(SkyFunctions.WORKSPACE_AST,
162 new WorkspaceASTFunction(TestRuleClassProvider.getRuleClassProvider()));
carmid6a98282018-03-13 19:19:16 -0700163 skyFunctions.put(
janakr15e15c22019-01-30 11:24:49 -0800164 WorkspaceFileValue.WORKSPACE_FILE,
carmid6a98282018-03-13 19:19:16 -0700165 new WorkspaceFileFunction(
166 TestRuleClassProvider.getRuleClassProvider(),
michajlo8083e322020-03-20 13:32:52 -0700167 TestPackageFactoryBuilderFactory.getInstance()
carmid6a98282018-03-13 19:19:16 -0700168 .builder(directories)
nharmatafde0bd2f2018-12-21 10:17:56 -0800169 .build(TestRuleClassProvider.getRuleClassProvider(), fs),
mjhalupkaf0e48112019-01-14 13:01:56 -0800170 directories,
171 /*skylarkImportLookupFunctionForInlining=*/ null));
Damien Martin-Guillerezbc8b5e02016-02-05 22:09:09 +0000172 skyFunctions.put(SkyFunctions.EXTERNAL_PACKAGE, new ExternalPackageFunction());
Kristina Chodorowf9fdc8d2015-12-08 12:49:31 +0000173
janakr1cde8722017-10-10 03:22:21 +0200174 differencer = new SequencedRecordingDifferencer();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000175 evaluator = new InMemoryMemoizingEvaluator(skyFunctions.build(), differencer);
176 driver = new SequentialBuildDriver(evaluator);
177 PrecomputedValue.BUILD_ID.set(differencer, UUID.randomUUID());
Kristina Chodorowf9fdc8d2015-12-08 12:49:31 +0000178 PrecomputedValue.PATH_PACKAGE_LOCATOR.set(differencer, pkgLocator.get());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000179 }
180
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000181 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000182 public void testEmpty() throws Exception {
Ulf Adamsc73051c62016-03-23 09:18:13 +0000183 FilesystemValueChecker checker = new FilesystemValueChecker(null, null);
Nathan Harmata8cd29782015-11-10 03:24:01 +0000184 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000185 }
186
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000187 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000188 public void testSimple() throws Exception {
Ulf Adamsc73051c62016-03-23 09:18:13 +0000189 FilesystemValueChecker checker = new FilesystemValueChecker(null, null);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000190
191 Path path = fs.getPath("/foo");
192 FileSystemUtils.createEmptyFile(path);
Nathan Harmata8cd29782015-11-10 03:24:01 +0000193 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000194
tomluee6a6862018-01-17 14:36:26 -0800195 SkyKey skyKey =
196 FileStateValue.key(
tomlu4c9fafd2018-01-18 10:29:11 -0800197 RootedPath.toRootedPath(Root.absoluteRoot(fs), PathFragment.create("/foo")));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000198 EvaluationResult<SkyValue> result =
Googler10028672018-10-25 12:14:34 -0700199 driver.evaluate(ImmutableList.of(skyKey), EVALUATION_OPTIONS);
lberkiaea56b32017-05-30 12:35:33 +0200200 assertThat(result.hasError()).isFalse();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000201
Nathan Harmata8cd29782015-11-10 03:24:01 +0000202 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000203
204 FileSystemUtils.writeContentAsLatin1(path, "hello");
Nathan Harmata8cd29782015-11-10 03:24:01 +0000205 assertDiffWithNewValues(getDirtyFilesystemKeys(evaluator, checker), skyKey);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000206
207 // The dirty bits are not reset until the FileValues are actually revalidated.
Nathan Harmata8cd29782015-11-10 03:24:01 +0000208 assertDiffWithNewValues(getDirtyFilesystemKeys(evaluator, checker), skyKey);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000209
210 differencer.invalidate(ImmutableList.of(skyKey));
Googler10028672018-10-25 12:14:34 -0700211 result = driver.evaluate(ImmutableList.of(skyKey), EVALUATION_OPTIONS);
lberkiaea56b32017-05-30 12:35:33 +0200212 assertThat(result.hasError()).isFalse();
Nathan Harmata8cd29782015-11-10 03:24:01 +0000213 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000214 }
215
216 /**
217 * Tests that an already-invalidated value can still be marked changed: symlink points at sym1.
218 * Invalidate symlink by changing sym1 from pointing at path to point to sym2. This only dirties
219 * (rather than changes) symlink because sym2 still points at path, so all symlink stats remain
220 * the same. Then do a null build, change sym1 back to point at path, and change symlink to not be
221 * a symlink anymore. The fact that it is not a symlink should be detected.
222 */
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000223 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000224 public void testDirtySymlink() throws Exception {
Ulf Adamsc73051c62016-03-23 09:18:13 +0000225 FilesystemValueChecker checker = new FilesystemValueChecker(null, null);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000226
227 Path path = fs.getPath("/foo");
228 FileSystemUtils.writeContentAsLatin1(path, "foo contents");
229 // We need the intermediate sym1 and sym2 so that we can dirty a child of symlink without
230 // actually changing the FileValue calculated for symlink (if we changed the contents of foo,
Laszlo Csomora278aec2018-03-09 04:07:31 -0800231 // the FileValue created for symlink would notice, since it stats foo).
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000232 Path sym1 = fs.getPath("/sym1");
233 Path sym2 = fs.getPath("/sym2");
234 Path symlink = fs.getPath("/bar");
235 FileSystemUtils.ensureSymbolicLink(symlink, sym1);
236 FileSystemUtils.ensureSymbolicLink(sym1, path);
237 FileSystemUtils.ensureSymbolicLink(sym2, path);
238 SkyKey fooKey =
tomlu4c9fafd2018-01-18 10:29:11 -0800239 FileValue.key(RootedPath.toRootedPath(Root.absoluteRoot(fs), PathFragment.create("/foo")));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000240 RootedPath symlinkRootedPath =
tomlu4c9fafd2018-01-18 10:29:11 -0800241 RootedPath.toRootedPath(Root.absoluteRoot(fs), PathFragment.create("/bar"));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000242 SkyKey symlinkKey = FileValue.key(symlinkRootedPath);
243 SkyKey symlinkFileStateKey = FileStateValue.key(symlinkRootedPath);
244 RootedPath sym1RootedPath =
tomlu4c9fafd2018-01-18 10:29:11 -0800245 RootedPath.toRootedPath(Root.absoluteRoot(fs), PathFragment.create("/sym1"));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000246 SkyKey sym1FileStateKey = FileStateValue.key(sym1RootedPath);
247 Iterable<SkyKey> allKeys = ImmutableList.of(symlinkKey, fooKey);
248
249 // First build -- prime the graph.
Googler10028672018-10-25 12:14:34 -0700250 EvaluationResult<FileValue> result = driver.evaluate(allKeys, EVALUATION_OPTIONS);
lberkiaea56b32017-05-30 12:35:33 +0200251 assertThat(result.hasError()).isFalse();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000252 FileValue symlinkValue = result.get(symlinkKey);
253 FileValue fooValue = result.get(fooKey);
lberkiaea56b32017-05-30 12:35:33 +0200254 assertWithMessage(symlinkValue.toString()).that(symlinkValue.isSymlink()).isTrue();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000255 // Digest is not always available, so use size as a proxy for contents.
lberkiaea56b32017-05-30 12:35:33 +0200256 assertThat(symlinkValue.getSize()).isEqualTo(fooValue.getSize());
Nathan Harmata8cd29782015-11-10 03:24:01 +0000257 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000258
259 // Before second build, move sym1 to point to sym2.
lberkiaea56b32017-05-30 12:35:33 +0200260 assertThat(sym1.delete()).isTrue();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000261 FileSystemUtils.ensureSymbolicLink(sym1, sym2);
Nathan Harmata8cd29782015-11-10 03:24:01 +0000262 assertDiffWithNewValues(getDirtyFilesystemKeys(evaluator, checker), sym1FileStateKey);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000263
264 differencer.invalidate(ImmutableList.of(sym1FileStateKey));
Googler10028672018-10-25 12:14:34 -0700265 result = driver.evaluate(ImmutableList.<SkyKey>of(), EVALUATION_OPTIONS);
lberkiaea56b32017-05-30 12:35:33 +0200266 assertThat(result.hasError()).isFalse();
Nathan Harmata8cd29782015-11-10 03:24:01 +0000267 assertDiffWithNewValues(getDirtyFilesystemKeys(evaluator, checker), sym1FileStateKey);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000268
269 // Before third build, move sym1 back to original (so change pruning will prevent signaling of
270 // its parents, but change symlink for real.
lberkiaea56b32017-05-30 12:35:33 +0200271 assertThat(sym1.delete()).isTrue();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000272 FileSystemUtils.ensureSymbolicLink(sym1, path);
lberkiaea56b32017-05-30 12:35:33 +0200273 assertThat(symlink.delete()).isTrue();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000274 FileSystemUtils.writeContentAsLatin1(symlink, "new symlink contents");
Nathan Harmata8cd29782015-11-10 03:24:01 +0000275 assertDiffWithNewValues(getDirtyFilesystemKeys(evaluator, checker), symlinkFileStateKey);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000276 differencer.invalidate(ImmutableList.of(symlinkFileStateKey));
Googler10028672018-10-25 12:14:34 -0700277 result = driver.evaluate(allKeys, EVALUATION_OPTIONS);
lberkiaea56b32017-05-30 12:35:33 +0200278 assertThat(result.hasError()).isFalse();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000279 symlinkValue = result.get(symlinkKey);
lberkiaea56b32017-05-30 12:35:33 +0200280 assertWithMessage(symlinkValue.toString()).that(symlinkValue.isSymlink()).isFalse();
281 assertThat(result.get(fooKey)).isEqualTo(fooValue);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000282 assertThat(symlinkValue.getSize()).isNotEqualTo(fooValue.getSize());
Nathan Harmata8cd29782015-11-10 03:24:01 +0000283 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000284 }
285
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000286 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000287 public void testExplicitFiles() throws Exception {
Ulf Adamsc73051c62016-03-23 09:18:13 +0000288 FilesystemValueChecker checker = new FilesystemValueChecker(null, null);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000289
290 Path path1 = fs.getPath("/foo1");
291 Path path2 = fs.getPath("/foo2");
292 FileSystemUtils.createEmptyFile(path1);
293 FileSystemUtils.createEmptyFile(path2);
Nathan Harmata8cd29782015-11-10 03:24:01 +0000294 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000295
296 SkyKey key1 =
297 FileStateValue.key(
tomlu4c9fafd2018-01-18 10:29:11 -0800298 RootedPath.toRootedPath(Root.absoluteRoot(fs), PathFragment.create("/foo1")));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000299 SkyKey key2 =
300 FileStateValue.key(
tomlu4c9fafd2018-01-18 10:29:11 -0800301 RootedPath.toRootedPath(Root.absoluteRoot(fs), PathFragment.create("/foo2")));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000302 Iterable<SkyKey> skyKeys = ImmutableList.of(key1, key2);
Googler10028672018-10-25 12:14:34 -0700303 EvaluationResult<SkyValue> result = driver.evaluate(skyKeys, EVALUATION_OPTIONS);
lberkiaea56b32017-05-30 12:35:33 +0200304 assertThat(result.hasError()).isFalse();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000305
Nathan Harmata8cd29782015-11-10 03:24:01 +0000306 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000307
Laszlo Csomora278aec2018-03-09 04:07:31 -0800308 // Wait for the timestamp granularity to elapse, so updating the files will observably advance
309 // their ctime.
laszlocsomor29bdf632018-03-13 02:50:46 -0700310 TimestampGranularityUtils.waitForTimestampGranularity(
311 System.currentTimeMillis(), OutErr.SYSTEM_OUT_ERR);
Laszlo Csomora278aec2018-03-09 04:07:31 -0800312 // Update path1's contents and mtime. This will update the file's ctime.
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000313 FileSystemUtils.writeContentAsLatin1(path1, "hello1");
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000314 path1.setLastModifiedTime(27);
Laszlo Csomora278aec2018-03-09 04:07:31 -0800315 // Update path2's mtime but not its contents. We expect that an mtime change suffices to update
316 // the ctime.
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000317 path2.setLastModifiedTime(42);
Laszlo Csomora278aec2018-03-09 04:07:31 -0800318 // Assert that both files changed. The change detection relies, among other things, on ctime
319 // change.
Nathan Harmata8cd29782015-11-10 03:24:01 +0000320 assertDiffWithNewValues(getDirtyFilesystemKeys(evaluator, checker), key1, key2);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000321
322 differencer.invalidate(skyKeys);
Googler10028672018-10-25 12:14:34 -0700323 result = driver.evaluate(skyKeys, EVALUATION_OPTIONS);
lberkiaea56b32017-05-30 12:35:33 +0200324 assertThat(result.hasError()).isFalse();
Nathan Harmata8cd29782015-11-10 03:24:01 +0000325 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000326 }
327
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000328 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000329 public void testFileWithIOExceptionNotConsideredDirty() throws Exception {
330 Path path = fs.getPath("/testroot/foo");
331 path.getParentDirectory().createDirectory();
nharmatab4060b62017-04-04 17:11:39 +0000332 path.createSymbolicLink(PathFragment.create("bar"));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000333
334 fs.readlinkThrowsIoException = true;
tomluee6a6862018-01-17 14:36:26 -0800335 SkyKey fileKey =
336 FileStateValue.key(
337 RootedPath.toRootedPath(Root.fromPath(pkgRoot), PathFragment.create("foo")));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000338 EvaluationResult<SkyValue> result =
Googler10028672018-10-25 12:14:34 -0700339 driver.evaluate(ImmutableList.of(fileKey), EVALUATION_OPTIONS);
lberkiaea56b32017-05-30 12:35:33 +0200340 assertThat(result.hasError()).isTrue();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000341
342 fs.readlinkThrowsIoException = false;
Ulf Adamsc73051c62016-03-23 09:18:13 +0000343 FilesystemValueChecker checker = new FilesystemValueChecker(null, null);
Nathan Harmata8cd29782015-11-10 03:24:01 +0000344 Diff diff = getDirtyFilesystemKeys(evaluator, checker);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000345 assertThat(diff.changedKeysWithoutNewValues()).isEmpty();
346 assertThat(diff.changedKeysWithNewValues()).isEmpty();
347 }
348
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000349 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000350 public void testFilesInCycleNotConsideredDirty() throws Exception {
351 Path path1 = pkgRoot.getRelative("foo1");
352 Path path2 = pkgRoot.getRelative("foo2");
353 Path path3 = pkgRoot.getRelative("foo3");
354 FileSystemUtils.ensureSymbolicLink(path1, path2);
355 FileSystemUtils.ensureSymbolicLink(path2, path3);
356 FileSystemUtils.ensureSymbolicLink(path3, path1);
tomluee6a6862018-01-17 14:36:26 -0800357 SkyKey fileKey1 = FileValue.key(RootedPath.toRootedPath(Root.fromPath(pkgRoot), path1));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000358
359 EvaluationResult<SkyValue> result =
Googler10028672018-10-25 12:14:34 -0700360 driver.evaluate(ImmutableList.of(fileKey1), EVALUATION_OPTIONS);
lberkiaea56b32017-05-30 12:35:33 +0200361 assertThat(result.hasError()).isTrue();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000362
Ulf Adamsc73051c62016-03-23 09:18:13 +0000363 FilesystemValueChecker checker = new FilesystemValueChecker(null, null);
Nathan Harmata8cd29782015-11-10 03:24:01 +0000364 Diff diff = getDirtyFilesystemKeys(evaluator, checker);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000365 assertThat(diff.changedKeysWithoutNewValues()).isEmpty();
366 assertThat(diff.changedKeysWithNewValues()).isEmpty();
367 }
368
369 public void checkDirtyActions(BatchStat batchStatter, boolean forceDigests) throws Exception {
370 Artifact out1 = createDerivedArtifact("fiz");
371 Artifact out2 = createDerivedArtifact("pop");
372
373 FileSystemUtils.writeContentAsLatin1(out1.getPath(), "hello");
374 FileSystemUtils.writeContentAsLatin1(out2.getPath(), "fizzlepop");
375
lberki41cfcc72019-08-05 02:55:36 -0700376 TimestampGranularityMonitor tsgm = new TimestampGranularityMonitor(BlazeClock.instance());
janakrbaf52ae2018-02-14 09:03:18 -0800377 ActionLookupKey actionLookupKey =
janakr573807d2018-01-11 14:02:35 -0800378 new ActionLookupKey() {
379 @Override
380 public SkyFunctionName functionName() {
381 return SkyFunctionName.FOR_TESTING;
382 }
383 };
janakrefb3f152019-06-05 17:42:34 -0700384 SkyKey actionKey1 = ActionLookupData.create(actionLookupKey, 0);
385 SkyKey actionKey2 = ActionLookupData.create(actionLookupKey, 1);
lberki41cfcc72019-08-05 02:55:36 -0700386
387 pretendBuildTwoArtifacts(out1, actionKey1, out2, actionKey2, batchStatter, tsgm);
388
389 // Change the file but not its size
390 FileSystemUtils.writeContentAsLatin1(out1.getPath(), "hallo");
391 checkActionDirtiedByFile(out1, actionKey1, batchStatter, tsgm);
392 pretendBuildTwoArtifacts(out1, actionKey1, out2, actionKey2, batchStatter, tsgm);
393
394 // Now try with a different size
395 FileSystemUtils.writeContentAsLatin1(out1.getPath(), "hallo2");
396 checkActionDirtiedByFile(out1, actionKey1, batchStatter, tsgm);
397 }
398
399 private void pretendBuildTwoArtifacts(
400 Artifact out1,
401 SkyKey actionKey1,
402 Artifact out2,
403 SkyKey actionKey2,
404 BatchStat batchStatter,
405 TimestampGranularityMonitor tsgm)
406 throws InterruptedException {
407 EvaluationContext evaluationContext =
408 EvaluationContext.newBuilder()
409 .setKeepGoing(false)
410 .setNumThreads(1)
411 .setEventHander(NullEventHandler.INSTANCE)
412 .build();
413
414 tsgm.setCommandStartTime();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000415 differencer.inject(
416 ImmutableMap.<SkyKey, SkyValue>of(
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000417 actionKey1,
418 actionValue(
419 new TestAction(
ulfjack1e1a7752019-12-10 21:17:58 -0800420 Runnables.doNothing(),
421 NestedSetBuilder.emptySet(Order.STABLE_ORDER),
422 ImmutableSet.of(out1))),
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000423 actionKey2,
424 actionValue(
425 new TestAction(
lberkic35878a2019-08-01 02:28:54 -0700426 Runnables.doNothing(),
ulfjack1e1a7752019-12-10 21:17:58 -0800427 NestedSetBuilder.emptySet(Order.STABLE_ORDER),
lberkic35878a2019-08-01 02:28:54 -0700428 ImmutableSet.of(out2)))));
Googler10028672018-10-25 12:14:34 -0700429 assertThat(driver.evaluate(ImmutableList.<SkyKey>of(), evaluationContext).hasError()).isFalse();
Eric Fellheimere6590722015-11-17 17:07:48 +0000430 assertThat(
lberki41cfcc72019-08-05 02:55:36 -0700431 new FilesystemValueChecker(tsgm, null)
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000432 .getDirtyActionValues(
steinman39c00d22020-03-20 15:23:10 -0700433 evaluator.getValues(),
434 batchStatter,
435 ModifiedFileSet.EVERYTHING_MODIFIED,
436 /* trustRemoteArtifacts= */ false))
lberki41cfcc72019-08-05 02:55:36 -0700437 .isEmpty();
438
439 tsgm.waitForTimestampGranularity(OutErr.SYSTEM_OUT_ERR);
440 }
441
442 private void checkActionDirtiedByFile(
443 Artifact file, SkyKey actionKey, BatchStat batchStatter, TimestampGranularityMonitor tsgm)
444 throws InterruptedException {
Ulf Adamsc73051c62016-03-23 09:18:13 +0000445 assertThat(
lberki41cfcc72019-08-05 02:55:36 -0700446 new FilesystemValueChecker(tsgm, null)
447 .getDirtyActionValues(
steinman39c00d22020-03-20 15:23:10 -0700448 evaluator.getValues(),
449 batchStatter,
450 ModifiedFileSet.EVERYTHING_MODIFIED,
451 /* trustRemoteArtifacts= */ false))
lberki41cfcc72019-08-05 02:55:36 -0700452 .containsExactly(actionKey);
453 assertThat(
454 new FilesystemValueChecker(tsgm, null)
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000455 .getDirtyActionValues(
456 evaluator.getValues(),
457 batchStatter,
steinman39c00d22020-03-20 15:23:10 -0700458 new ModifiedFileSet.Builder().modify(file.getExecPath()).build(),
459 /* trustRemoteArtifacts= */ false))
lberki41cfcc72019-08-05 02:55:36 -0700460 .containsExactly(actionKey);
Ulf Adamsc73051c62016-03-23 09:18:13 +0000461 assertThat(
lberki41cfcc72019-08-05 02:55:36 -0700462 new FilesystemValueChecker(tsgm, null)
463 .getDirtyActionValues(
464 evaluator.getValues(),
465 batchStatter,
466 new ModifiedFileSet.Builder()
467 .modify(file.getExecPath().getParentDirectory())
steinman39c00d22020-03-20 15:23:10 -0700468 .build(),
469 /* trustRemoteArtifacts= */ false))
lberki41cfcc72019-08-05 02:55:36 -0700470 .isEmpty();
Eric Fellheimere6590722015-11-17 17:07:48 +0000471 assertThat(
lberki41cfcc72019-08-05 02:55:36 -0700472 new FilesystemValueChecker(tsgm, null)
473 .getDirtyActionValues(
steinman39c00d22020-03-20 15:23:10 -0700474 evaluator.getValues(),
475 batchStatter,
476 ModifiedFileSet.NOTHING_MODIFIED,
477 /* trustRemoteArtifacts= */ false))
lberki41cfcc72019-08-05 02:55:36 -0700478 .isEmpty();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000479 }
480
janakrefb3f152019-06-05 17:42:34 -0700481 private void checkDirtyTreeArtifactActions(BatchStat batchStatter) throws Exception {
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000482 // Normally, an Action specifies the contents of a TreeArtifact when it executes.
483 // To decouple FileSystemValueTester checking from Action execution, we inject TreeArtifact
484 // contents into ActionExecutionValues.
485
cpeyserac09f0a2018-02-05 09:33:15 -0800486 SpecialArtifact out1 = createTreeArtifact("one");
janakrefb3f152019-06-05 17:42:34 -0700487 TreeFileArtifact file11 =
488 ActionsTestUtil.createTreeFileArtifactWithNoGeneratingAction(out1, "fizz");
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000489 FileSystemUtils.createDirectoryAndParents(out1.getPath());
490 FileSystemUtils.writeContentAsLatin1(file11.getPath(), "buzz");
491
cpeyserac09f0a2018-02-05 09:33:15 -0800492 SpecialArtifact out2 = createTreeArtifact("two");
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000493 FileSystemUtils.createDirectoryAndParents(out2.getPath().getChild("subdir"));
janakrefb3f152019-06-05 17:42:34 -0700494 TreeFileArtifact file21 =
495 ActionsTestUtil.createTreeFileArtifactWithNoGeneratingAction(out2, "moony");
496 TreeFileArtifact file22 =
497 ActionsTestUtil.createTreeFileArtifactWithNoGeneratingAction(out2, "subdir/wormtail");
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000498 FileSystemUtils.writeContentAsLatin1(file21.getPath(), "padfoot");
499 FileSystemUtils.writeContentAsLatin1(file22.getPath(), "prongs");
500
cpeyserac09f0a2018-02-05 09:33:15 -0800501 SpecialArtifact outEmpty = createTreeArtifact("empty");
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000502 FileSystemUtils.createDirectoryAndParents(outEmpty.getPath());
503
cpeyserac09f0a2018-02-05 09:33:15 -0800504 SpecialArtifact outUnchanging = createTreeArtifact("untouched");
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000505 FileSystemUtils.createDirectoryAndParents(outUnchanging.getPath());
506
cpeyserac09f0a2018-02-05 09:33:15 -0800507 SpecialArtifact last = createTreeArtifact("zzzzzzzzzz");
Rumou Duan45e8e572016-06-17 16:43:44 +0000508 FileSystemUtils.createDirectoryAndParents(last.getPath());
509
janakrbaf52ae2018-02-14 09:03:18 -0800510 ActionLookupKey actionLookupKey =
janakr573807d2018-01-11 14:02:35 -0800511 new ActionLookupKey() {
512 @Override
513 public SkyFunctionName functionName() {
514 return SkyFunctionName.FOR_TESTING;
515 }
516 };
janakrefb3f152019-06-05 17:42:34 -0700517 SkyKey actionKey1 = ActionLookupData.create(actionLookupKey, 0);
518 SkyKey actionKey2 = ActionLookupData.create(actionLookupKey, 1);
519 SkyKey actionKeyEmpty = ActionLookupData.create(actionLookupKey, 2);
520 SkyKey actionKeyUnchanging = ActionLookupData.create(actionLookupKey, 3);
521 SkyKey actionKeyLast = ActionLookupData.create(actionLookupKey, 4);
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000522 differencer.inject(
523 ImmutableMap.<SkyKey, SkyValue>of(
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000524 actionKey1,
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000525 actionValueWithTreeArtifacts(ImmutableList.of(file11)),
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000526 actionKey2,
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000527 actionValueWithTreeArtifacts(ImmutableList.of(file21, file22)),
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000528 actionKeyEmpty,
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000529 actionValueWithEmptyDirectory(outEmpty),
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000530 actionKeyUnchanging,
Rumou Duan45e8e572016-06-17 16:43:44 +0000531 actionValueWithEmptyDirectory(outUnchanging),
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000532 actionKeyLast,
Rumou Duan45e8e572016-06-17 16:43:44 +0000533 actionValueWithEmptyDirectory(last)));
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000534
Googler10028672018-10-25 12:14:34 -0700535 EvaluationContext evaluationContext =
536 EvaluationContext.newBuilder()
537 .setKeepGoing(false)
538 .setNumThreads(1)
539 .setEventHander(NullEventHandler.INSTANCE)
540 .build();
541 assertThat(driver.evaluate(ImmutableList.<SkyKey>of(), evaluationContext).hasError()).isFalse();
steinman39c00d22020-03-20 15:23:10 -0700542 assertThat(
543 new FilesystemValueChecker(null, null)
544 .getDirtyActionValues(
545 evaluator.getValues(),
546 batchStatter,
547 ModifiedFileSet.EVERYTHING_MODIFIED,
548 /* trustRemoteArtifacts= */ false))
549 .isEmpty();
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000550
551 // Touching the TreeArtifact directory should have no effect
552 FileSystemUtils.touchFile(out1.getPath());
553 assertThat(
steinman39c00d22020-03-20 15:23:10 -0700554 new FilesystemValueChecker(null, null)
555 .getDirtyActionValues(
556 evaluator.getValues(),
557 batchStatter,
558 ModifiedFileSet.EVERYTHING_MODIFIED,
559 /* trustRemoteArtifacts= */ false))
560 .isEmpty();
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000561 // Neither should touching a subdirectory.
562 FileSystemUtils.touchFile(out2.getPath().getChild("subdir"));
563 assertThat(
steinman39c00d22020-03-20 15:23:10 -0700564 new FilesystemValueChecker(null, null)
565 .getDirtyActionValues(
566 evaluator.getValues(),
567 batchStatter,
568 ModifiedFileSet.EVERYTHING_MODIFIED,
569 /* trustRemoteArtifacts= */ false))
570 .isEmpty();
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000571
572 /* **** Tests for directories **** */
573
574 // Removing a directory (even if empty) should have an effect
575 outEmpty.getPath().delete();
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000576 assertThat(
577 new FilesystemValueChecker(null, null)
578 .getDirtyActionValues(
579 evaluator.getValues(),
580 batchStatter,
steinman39c00d22020-03-20 15:23:10 -0700581 new ModifiedFileSet.Builder().modify(outEmpty.getExecPath()).build(),
582 /* trustRemoteArtifacts= */ false))
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000583 .containsExactly(actionKeyEmpty);
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000584 // Symbolic links should count as dirty
585 Path dummyEmptyDir = fs.getPath("/bin").getRelative("symlink");
586 FileSystemUtils.createDirectoryAndParents(dummyEmptyDir);
587 FileSystemUtils.ensureSymbolicLink(outEmpty.getPath(), dummyEmptyDir);
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000588 assertThat(
589 new FilesystemValueChecker(null, null)
590 .getDirtyActionValues(
591 evaluator.getValues(),
592 batchStatter,
steinman39c00d22020-03-20 15:23:10 -0700593 new ModifiedFileSet.Builder().modify(outEmpty.getExecPath()).build(),
594 /* trustRemoteArtifacts= */ false))
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000595 .containsExactly(actionKeyEmpty);
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000596
597 // We're done fiddling with this... restore the original state
598 outEmpty.getPath().delete();
jmmv5cc1f652019-03-20 09:34:08 -0700599 dummyEmptyDir.deleteTree();
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000600 FileSystemUtils.createDirectoryAndParents(outEmpty.getPath());
601
602 /* **** Tests for files and directory contents ****/
603
604 // Test that file contents matter. This is covered by existing tests already,
605 // so it's just a sanity check.
606 FileSystemUtils.writeContentAsLatin1(file11.getPath(), "goodbye");
Ulf Adamsc73051c62016-03-23 09:18:13 +0000607 assertThat(
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000608 new FilesystemValueChecker(null, null)
609 .getDirtyActionValues(
610 evaluator.getValues(),
611 batchStatter,
steinman39c00d22020-03-20 15:23:10 -0700612 new ModifiedFileSet.Builder().modify(file11.getExecPath()).build(),
613 /* trustRemoteArtifacts= */ false))
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000614 .containsExactly(actionKey1);
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000615
616 // Test that directory contents (and nested contents) matter
janakrefb3f152019-06-05 17:42:34 -0700617 Artifact out1new =
618 ActionsTestUtil.createTreeFileArtifactWithNoGeneratingAction(out1, "julius/caesar");
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000619 FileSystemUtils.createDirectoryAndParents(out1.getPath().getChild("julius"));
620 FileSystemUtils.writeContentAsLatin1(out1new.getPath(), "octavian");
621 // even for empty directories
janakrefb3f152019-06-05 17:42:34 -0700622 Artifact outEmptyNew =
623 ActionsTestUtil.createTreeFileArtifactWithNoGeneratingAction(outEmpty, "marcus");
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000624 FileSystemUtils.writeContentAsLatin1(outEmptyNew.getPath(), "aurelius");
625 // so does removing
626 file21.getPath().delete();
627 // now, let's test our changes are actually visible
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000628 assertThat(
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000629 new FilesystemValueChecker(null, null)
630 .getDirtyActionValues(
steinman39c00d22020-03-20 15:23:10 -0700631 evaluator.getValues(),
632 batchStatter,
633 ModifiedFileSet.EVERYTHING_MODIFIED,
634 /* trustRemoteArtifacts= */ false))
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000635 .containsExactly(actionKey1, actionKey2, actionKeyEmpty);
636 assertThat(
637 new FilesystemValueChecker(null, null)
638 .getDirtyActionValues(
639 evaluator.getValues(),
640 batchStatter,
641 new ModifiedFileSet.Builder()
642 .modify(file21.getExecPath())
643 .modify(out1new.getExecPath())
644 .modify(outEmptyNew.getExecPath())
steinman39c00d22020-03-20 15:23:10 -0700645 .build(),
646 /* trustRemoteArtifacts= */ false))
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000647 .containsExactly(actionKey1, actionKey2, actionKeyEmpty);
Rumou Duan45e8e572016-06-17 16:43:44 +0000648 // We also check that if the modified file set does not contain our modified files on disk,
649 // we are not going to check and return them.
650 assertThat(
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000651 new FilesystemValueChecker(null, null)
652 .getDirtyActionValues(
653 evaluator.getValues(),
654 batchStatter,
655 new ModifiedFileSet.Builder()
656 .modify(file21.getExecPath())
657 .modify(outEmptyNew.getExecPath())
steinman39c00d22020-03-20 15:23:10 -0700658 .build(),
659 /* trustRemoteArtifacts= */ false))
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000660 .containsExactly(actionKey2, actionKeyEmpty);
Rumou Duan45e8e572016-06-17 16:43:44 +0000661 assertThat(
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000662 new FilesystemValueChecker(null, null)
663 .getDirtyActionValues(
664 evaluator.getValues(),
665 batchStatter,
666 new ModifiedFileSet.Builder()
667 .modify(file21.getExecPath())
668 .modify(out1new.getExecPath())
steinman39c00d22020-03-20 15:23:10 -0700669 .build(),
670 /* trustRemoteArtifacts= */ false))
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000671 .containsExactly(actionKey1, actionKey2);
Rumou Duan45e8e572016-06-17 16:43:44 +0000672 // Check modifying the last (lexicographically) tree artifact.
673 last.getPath().delete();
674 assertThat(
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000675 new FilesystemValueChecker(null, null)
676 .getDirtyActionValues(
677 evaluator.getValues(),
678 batchStatter,
679 new ModifiedFileSet.Builder()
680 .modify(file21.getExecPath())
681 .modify(out1new.getExecPath())
682 .modify(last.getExecPath())
steinman39c00d22020-03-20 15:23:10 -0700683 .build(),
684 /* trustRemoteArtifacts= */ false))
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000685 .containsExactly(actionKey1, actionKey2, actionKeyLast);
Rumou Duan45e8e572016-06-17 16:43:44 +0000686 // Check ModifiedFileSet without the last (lexicographically) tree artifact.
687 assertThat(
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000688 new FilesystemValueChecker(null, null)
689 .getDirtyActionValues(
690 evaluator.getValues(),
691 batchStatter,
692 new ModifiedFileSet.Builder()
693 .modify(file21.getExecPath())
694 .modify(out1new.getExecPath())
steinman39c00d22020-03-20 15:23:10 -0700695 .build(),
696 /* trustRemoteArtifacts= */ false))
Janak Ramakrishnan5d152692017-03-20 22:07:14 +0000697 .containsExactly(actionKey1, actionKey2);
Rumou Duan45e8e572016-06-17 16:43:44 +0000698 // Restore
699 last.getPath().delete();
700 FileSystemUtils.createDirectoryAndParents(last.getPath());
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000701 // We add a test for NOTHING_MODIFIED, because FileSystemValueChecker doesn't
702 // pay attention to file sets for TreeArtifact directory listings.
703 assertThat(
steinman39c00d22020-03-20 15:23:10 -0700704 new FilesystemValueChecker(null, null)
705 .getDirtyActionValues(
706 evaluator.getValues(),
707 batchStatter,
708 ModifiedFileSet.NOTHING_MODIFIED,
709 /* trustRemoteArtifacts= */ false))
710 .isEmpty();
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000711 }
712
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000713 private Artifact createDerivedArtifact(String relPath) throws IOException {
janakr448f1cf2020-03-30 09:12:44 -0700714 String outSegment = "bin";
715 Path outputPath = fs.getPath("/" + outSegment);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000716 outputPath.createDirectory();
janakraea05602019-05-22 15:41:29 -0700717 return ActionsTestUtil.createArtifact(
janakr448f1cf2020-03-30 09:12:44 -0700718 ArtifactRoot.asDerivedRoot(fs.getPath("/"), outSegment), outputPath.getRelative(relPath));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000719 }
720
cpeyserac09f0a2018-02-05 09:33:15 -0800721 private SpecialArtifact createTreeArtifact(String relPath) throws IOException {
janakr448f1cf2020-03-30 09:12:44 -0700722 String outSegment = "bin";
723 Path outputDir = fs.getPath("/" + outSegment);
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000724 Path outputPath = outputDir.getRelative(relPath);
725 outputDir.createDirectory();
janakr448f1cf2020-03-30 09:12:44 -0700726 ArtifactRoot derivedRoot = ArtifactRoot.asDerivedRoot(fs.getPath("/"), outSegment);
tomluee6a6862018-01-17 14:36:26 -0800727 return new SpecialArtifact(
tomluee6a6862018-01-17 14:36:26 -0800728 derivedRoot,
729 derivedRoot.getExecPath().getRelative(derivedRoot.getRoot().relativize(outputPath)),
janakrefb3f152019-06-05 17:42:34 -0700730 ActionsTestUtil.NULL_ARTIFACT_OWNER,
tomluee6a6862018-01-17 14:36:26 -0800731 SpecialArtifactType.TREE);
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000732 }
733
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000734 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000735 public void testDirtyActions() throws Exception {
736 checkDirtyActions(null, false);
737 }
738
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000739 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000740 public void testDirtyActionsBatchStat() throws Exception {
741 checkDirtyActions(
742 new BatchStat() {
743 @Override
744 public List<FileStatusWithDigest> batchStat(
745 boolean useDigest, boolean includeLinks, Iterable<PathFragment> paths)
746 throws IOException {
747 List<FileStatusWithDigest> stats = new ArrayList<>();
748 for (PathFragment pathFrag : paths) {
749 stats.add(
750 FileStatusWithDigestAdapter.adapt(
tomlu6c919062018-01-11 17:32:09 -0800751 fs.getPath("/").getRelative(pathFrag).statIfFound(Symlinks.NOFOLLOW)));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000752 }
753 return stats;
754 }
755 },
756 false);
757 }
758
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000759 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000760 public void testDirtyActionsBatchStatWithDigest() throws Exception {
761 checkDirtyActions(
762 new BatchStat() {
763 @Override
764 public List<FileStatusWithDigest> batchStat(
765 boolean useDigest, boolean includeLinks, Iterable<PathFragment> paths)
766 throws IOException {
767 List<FileStatusWithDigest> stats = new ArrayList<>();
768 for (PathFragment pathFrag : paths) {
tomlu6c919062018-01-11 17:32:09 -0800769 final Path path = fs.getPath("/").getRelative(pathFrag);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000770 stats.add(statWithDigest(path, path.statIfFound(Symlinks.NOFOLLOW)));
771 }
772 return stats;
773 }
774 },
775 true);
776 }
777
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +0000778 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000779 public void testDirtyActionsBatchStatFallback() throws Exception {
780 checkDirtyActions(
781 new BatchStat() {
782 @Override
783 public List<FileStatusWithDigest> batchStat(
784 boolean useDigest, boolean includeLinks, Iterable<PathFragment> paths)
785 throws IOException {
786 throw new IOException("try again");
787 }
788 },
789 false);
790 }
791
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000792 @Test
793 public void testDirtyTreeArtifactActions() throws Exception {
794 checkDirtyTreeArtifactActions(null);
795 }
796
797 @Test
798 public void testDirtyTreeArtifactActionsBatchStat() throws Exception {
799 checkDirtyTreeArtifactActions(
800 new BatchStat() {
801 @Override
802 public List<FileStatusWithDigest> batchStat(
803 boolean useDigest, boolean includeLinks, Iterable<PathFragment> paths)
804 throws IOException {
805 List<FileStatusWithDigest> stats = new ArrayList<>();
806 for (PathFragment pathFrag : paths) {
807 stats.add(
808 FileStatusWithDigestAdapter.adapt(
tomlu6c919062018-01-11 17:32:09 -0800809 fs.getPath("/").getRelative(pathFrag).statIfFound(Symlinks.NOFOLLOW)));
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000810 }
811 return stats;
812 }
813 });
814 }
815
816 // TODO(bazel-team): Add some tests for FileSystemValueChecker#changedKeys*() methods.
817 // Presently these appear to be untested.
818
lberkic35878a2019-08-01 02:28:54 -0700819 private ActionExecutionValue actionValue(Action action) {
lberkif7eee1e2019-07-31 05:49:10 -0700820 Map<Artifact, FileArtifactValue> artifactData = new HashMap<>();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000821 for (Artifact output : action.getOutputs()) {
822 try {
823 Path path = output.getPath();
lberkic35878a2019-08-01 02:28:54 -0700824 FileArtifactValue noDigest =
825 ActionMetadataHandler.fileArtifactValueFromArtifact(
826 output,
827 FileStatusWithDigestAdapter.adapt(path.statIfFound(Symlinks.NOFOLLOW)),
828 null);
829 FileArtifactValue withDigest =
830 FileArtifactValue.createFromInjectedDigest(
831 noDigest, path.getDigest(), !output.isConstantMetadata());
832 artifactData.put(output, withDigest);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000833 } catch (IOException e) {
834 throw new IllegalStateException(e);
835 }
836 }
janakrb9d8d582018-06-13 21:57:19 -0700837 return ActionExecutionValue.create(
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000838 artifactData,
839 ImmutableMap.<Artifact, TreeArtifactValue>of(),
janakrb9d8d582018-06-13 21:57:19 -0700840 /*outputSymlinks=*/ null,
shahanef6f4cf2018-06-26 11:24:59 -0700841 /*discoveredModules=*/ null,
janakr9f496f32018-10-24 15:08:09 -0700842 /*actionDependsOnBuildId=*/ false);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +0000843 }
844
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000845 private ActionExecutionValue actionValueWithEmptyDirectory(Artifact emptyDir) {
846 TreeArtifactValue emptyValue = TreeArtifactValue.create
Rumou Duana77f32c2016-04-13 21:59:21 +0000847 (ImmutableMap.<TreeFileArtifact, FileArtifactValue>of());
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000848
janakrb9d8d582018-06-13 21:57:19 -0700849 return ActionExecutionValue.create(
janakr0c42fc82018-09-14 10:37:25 -0700850 ImmutableMap.of(),
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000851 ImmutableMap.of(emptyDir, emptyValue),
janakrb9d8d582018-06-13 21:57:19 -0700852 /*outputSymlinks=*/ null,
shahanef6f4cf2018-06-26 11:24:59 -0700853 /*discoveredModules=*/ null,
janakr9f496f32018-10-24 15:08:09 -0700854 /*actionDependsOnBuildId=*/ false);
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000855 }
856
Rumou Duana77f32c2016-04-13 21:59:21 +0000857 private ActionExecutionValue actionValueWithTreeArtifacts(List<TreeFileArtifact> contents) {
lberkif7eee1e2019-07-31 05:49:10 -0700858 Map<Artifact, FileArtifactValue> fileData = new HashMap<>();
Rumou Duana77f32c2016-04-13 21:59:21 +0000859 Map<Artifact, Map<TreeFileArtifact, FileArtifactValue>> directoryData = new HashMap<>();
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000860
Rumou Duana77f32c2016-04-13 21:59:21 +0000861 for (TreeFileArtifact output : contents) {
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000862 try {
Rumou Duana77f32c2016-04-13 21:59:21 +0000863 Map<TreeFileArtifact, FileArtifactValue> dirDatum =
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000864 directoryData.get(output.getParent());
865 if (dirDatum == null) {
866 dirDatum = new HashMap<>();
867 directoryData.put(output.getParent(), dirDatum);
868 }
lberkic35878a2019-08-01 02:28:54 -0700869 Path path = output.getPath();
870 FileArtifactValue noDigest =
871 ActionMetadataHandler.fileArtifactValueFromArtifact(
872 output,
873 FileStatusWithDigestAdapter.adapt(path.statIfFound(Symlinks.NOFOLLOW)),
874 null);
875 FileArtifactValue withDigest =
876 FileArtifactValue.createFromInjectedDigest(
877 noDigest, path.getDigest(), !output.isConstantMetadata());
878 dirDatum.put(output, withDigest);
879 fileData.put(output, withDigest);
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000880 } catch (IOException e) {
881 throw new IllegalStateException(e);
882 }
883 }
884
885 Map<Artifact, TreeArtifactValue> treeArtifactData = new HashMap<>();
Rumou Duana77f32c2016-04-13 21:59:21 +0000886 for (Map.Entry<Artifact, Map<TreeFileArtifact, FileArtifactValue>> dirDatum :
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000887 directoryData.entrySet()) {
Rumou Duana77f32c2016-04-13 21:59:21 +0000888 treeArtifactData.put(dirDatum.getKey(), TreeArtifactValue.create(dirDatum.getValue()));
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000889 }
890
janakrb9d8d582018-06-13 21:57:19 -0700891 return ActionExecutionValue.create(
kush2ce45a22018-05-02 14:15:37 -0700892 fileData,
893 treeArtifactData,
janakrb9d8d582018-06-13 21:57:19 -0700894 /*outputSymlinks=*/ null,
shahanef6f4cf2018-06-26 11:24:59 -0700895 /*discoveredModules=*/ null,
janakr9f496f32018-10-24 15:08:09 -0700896 /*actionDependsOnBuildId=*/ false);
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000897 }
898
buchgr4992ae22019-03-20 04:23:32 -0700899 private ActionExecutionValue actionValueWithRemoteTreeArtifact(
900 SpecialArtifact output, Map<PathFragment, RemoteFileArtifactValue> children) {
901 ImmutableMap.Builder<TreeFileArtifact, FileArtifactValue> childFileValues =
902 ImmutableMap.builder();
903 for (Map.Entry<PathFragment, RemoteFileArtifactValue> child : children.entrySet()) {
904 childFileValues.put(
janakrefb3f152019-06-05 17:42:34 -0700905 ActionInputHelper.treeFileArtifactWithNoGeneratingActionSet(
906 output, child.getKey(), output.getArtifactOwner()),
907 child.getValue());
buchgr4992ae22019-03-20 04:23:32 -0700908 }
909 TreeArtifactValue treeArtifactValue = TreeArtifactValue.create(childFileValues.build());
910 return ActionExecutionValue.create(
buchgr4992ae22019-03-20 04:23:32 -0700911 ImmutableMap.of(),
lberkic35878a2019-08-01 02:28:54 -0700912 Collections.singletonMap(output, treeArtifactValue),
buchgr4992ae22019-03-20 04:23:32 -0700913 /* outputSymlinks= */ null,
914 /* discoveredModules= */ null,
915 /* actionDependsOnBuildId= */ false);
916 }
917
918 private ActionExecutionValue actionValueWithRemoteArtifact(
919 Artifact output, RemoteFileArtifactValue value) {
920 return ActionExecutionValue.create(
buchgr4992ae22019-03-20 04:23:32 -0700921 Collections.singletonMap(output, value),
lberkic35878a2019-08-01 02:28:54 -0700922 ImmutableMap.of(),
buchgr4992ae22019-03-20 04:23:32 -0700923 /* outputSymlinks= */ null,
924 /* discoveredModules= */ null,
925 /* actionDependsOnBuildId= */ false);
926 }
927
928 private RemoteFileArtifactValue createRemoteFileArtifactValue(String contents) {
929 byte[] data = contents.getBytes();
930 DigestHashFunction hashFn = fs.getDigestFunction();
931 HashCode hash = hashFn.getHashFunction().hashBytes(data);
932 return new RemoteFileArtifactValue(hash.asBytes(), data.length, -1);
933 }
934
935 @Test
936 public void testRemoteAndLocalArtifacts() throws Exception {
937 // Test that injected remote artifacts are trusted by the FileSystemValueChecker
steinman39c00d22020-03-20 15:23:10 -0700938 // if it is configured to trust remote artifacts, and that local files always take precedence
939 // over remote files.
buchgr4992ae22019-03-20 04:23:32 -0700940 ActionLookupKey actionLookupKey =
941 new ActionLookupKey() {
942 @Override
943 public SkyFunctionName functionName() {
944 return SkyFunctionName.FOR_TESTING;
945 }
946 };
janakrefb3f152019-06-05 17:42:34 -0700947 SkyKey actionKey1 = ActionLookupData.create(actionLookupKey, 0);
948 SkyKey actionKey2 = ActionLookupData.create(actionLookupKey, 1);
buchgr4992ae22019-03-20 04:23:32 -0700949
950 Artifact out1 = createDerivedArtifact("foo");
951 Artifact out2 = createDerivedArtifact("bar");
952 Map<SkyKey, SkyValue> metadataToInject = new HashMap<>();
953 metadataToInject.put(
954 actionKey1,
955 actionValueWithRemoteArtifact(out1, createRemoteFileArtifactValue("foo-content")));
956 metadataToInject.put(
957 actionKey2,
958 actionValueWithRemoteArtifact(out2, createRemoteFileArtifactValue("bar-content")));
959 differencer.inject(metadataToInject);
960
961 EvaluationContext evaluationContext =
962 EvaluationContext.newBuilder()
963 .setKeepGoing(false)
964 .setNumThreads(1)
965 .setEventHander(NullEventHandler.INSTANCE)
966 .build();
967 assertThat(
968 driver.evaluate(ImmutableList.of(actionKey1, actionKey2), evaluationContext).hasError())
969 .isFalse();
970 assertThat(
971 new FilesystemValueChecker(/* tsgm= */ null, /* lastExecutionTimeRange= */ null)
972 .getDirtyActionValues(
973 evaluator.getValues(),
974 /* batchStatter= */ null,
steinman39c00d22020-03-20 15:23:10 -0700975 ModifiedFileSet.EVERYTHING_MODIFIED,
976 /* trustRemoteArtifacts= */ true))
buchgr4992ae22019-03-20 04:23:32 -0700977 .isEmpty();
978
979 // Create the "out1" artifact on the filesystem and test that it invalidates the generating
980 // action's SkyKey.
981 FileSystemUtils.writeContentAsLatin1(out1.getPath(), "new-foo-content");
982 assertThat(
983 new FilesystemValueChecker(/* tsgm= */ null, /* lastExecutionTimeRange= */ null)
984 .getDirtyActionValues(
985 evaluator.getValues(),
986 /* batchStatter= */ null,
steinman39c00d22020-03-20 15:23:10 -0700987 ModifiedFileSet.EVERYTHING_MODIFIED,
988 /* trustRemoteArtifacts= */ true))
buchgr4992ae22019-03-20 04:23:32 -0700989 .containsExactly(actionKey1);
990 }
991
992 @Test
993 public void testRemoteAndLocalTreeArtifacts() throws Exception {
994 // Test that injected remote tree artifacts are trusted by the FileSystemValueChecker
995 // and that local files always takes preference over remote files.
996 ActionLookupKey actionLookupKey =
997 new ActionLookupKey() {
998 @Override
999 public SkyFunctionName functionName() {
1000 return SkyFunctionName.FOR_TESTING;
1001 }
1002 };
janakrefb3f152019-06-05 17:42:34 -07001003 SkyKey actionKey = ActionLookupData.create(actionLookupKey, 0);
buchgr4992ae22019-03-20 04:23:32 -07001004
1005 SpecialArtifact treeArtifact = createTreeArtifact("dir");
1006 treeArtifact.getPath().createDirectoryAndParents();
1007 Map<PathFragment, RemoteFileArtifactValue> treeArtifactMetadata = new HashMap<>();
1008 treeArtifactMetadata.put(
1009 PathFragment.create("foo"), createRemoteFileArtifactValue("foo-content"));
1010 treeArtifactMetadata.put(
1011 PathFragment.create("bar"), createRemoteFileArtifactValue("bar-content"));
1012
1013 Map<SkyKey, SkyValue> metadataToInject = new HashMap<>();
1014 metadataToInject.put(
1015 actionKey, actionValueWithRemoteTreeArtifact(treeArtifact, treeArtifactMetadata));
1016 differencer.inject(metadataToInject);
1017
1018 EvaluationContext evaluationContext =
1019 EvaluationContext.newBuilder()
1020 .setKeepGoing(false)
1021 .setNumThreads(1)
1022 .setEventHander(NullEventHandler.INSTANCE)
1023 .build();
1024 assertThat(driver.evaluate(ImmutableList.of(actionKey), evaluationContext).hasError())
1025 .isFalse();
1026 assertThat(
1027 new FilesystemValueChecker(/* tsgm= */ null, /* lastExecutionTimeRange= */ null)
1028 .getDirtyActionValues(
1029 evaluator.getValues(),
1030 /* batchStatter= */ null,
steinman39c00d22020-03-20 15:23:10 -07001031 ModifiedFileSet.EVERYTHING_MODIFIED,
1032 /* trustRemoteArtifacts= */ false))
buchgr4992ae22019-03-20 04:23:32 -07001033 .isEmpty();
1034
1035 // Create dir/foo on the local disk and test that it invalidates the associated sky key.
janakrefb3f152019-06-05 17:42:34 -07001036 TreeFileArtifact fooArtifact =
1037 ActionsTestUtil.createTreeFileArtifactWithNoGeneratingAction(treeArtifact, "foo");
buchgr4992ae22019-03-20 04:23:32 -07001038 FileSystemUtils.writeContentAsLatin1(fooArtifact.getPath(), "new-foo-content");
1039 assertThat(
1040 new FilesystemValueChecker(/* tsgm= */ null, /* lastExecutionTimeRange= */ null)
1041 .getDirtyActionValues(
1042 evaluator.getValues(),
1043 /* batchStatter= */ null,
steinman39c00d22020-03-20 15:23:10 -07001044 ModifiedFileSet.EVERYTHING_MODIFIED,
1045 /* trustRemoteArtifacts= */ false))
buchgr4992ae22019-03-20 04:23:32 -07001046 .containsExactly(actionKey);
1047 }
1048
Han-Wen Nienhuys3b2eae32015-10-28 16:35:08 +00001049 @Test
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001050 public void testPropagatesRuntimeExceptions() throws Exception {
tomluee6a6862018-01-17 14:36:26 -08001051 Collection<SkyKey> values =
1052 ImmutableList.of(
1053 FileValue.key(
1054 RootedPath.toRootedPath(Root.fromPath(pkgRoot), PathFragment.create("foo"))));
Googler10028672018-10-25 12:14:34 -07001055 driver.evaluate(values, EVALUATION_OPTIONS);
Ulf Adamsc73051c62016-03-23 09:18:13 +00001056 FilesystemValueChecker checker = new FilesystemValueChecker(null, null);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001057
Nathan Harmata8cd29782015-11-10 03:24:01 +00001058 assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001059
1060 fs.statThrowsRuntimeException = true;
jcater83130f42019-04-30 14:29:28 -07001061 RuntimeException e =
1062 assertThrows(RuntimeException.class, () -> getDirtyFilesystemKeys(evaluator, checker));
1063 assertThat(e).hasMessageThat().isEqualTo("bork");
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001064 }
1065
1066 private static void assertEmptyDiff(Diff diff) {
1067 assertDiffWithNewValues(diff);
1068 }
1069
1070 private static void assertDiffWithNewValues(Diff diff, SkyKey... keysWithNewValues) {
1071 assertThat(diff.changedKeysWithoutNewValues()).isEmpty();
1072 assertThat(diff.changedKeysWithNewValues().keySet())
1073 .containsExactlyElementsIn(Arrays.asList(keysWithNewValues));
1074 }
1075
1076 private class MockFileSystem extends InMemoryFileSystem {
1077
1078 boolean statThrowsRuntimeException;
1079 boolean readlinkThrowsIoException;
1080
1081 MockFileSystem() {
ccalvarinc9efd062018-07-27 12:46:46 -07001082 super();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001083 }
1084
1085 @Override
fellya205ed82018-09-10 11:52:34 -07001086 public FileStatus statIfFound(Path path, boolean followSymlinks) throws IOException {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001087 if (statThrowsRuntimeException) {
1088 throw new RuntimeException("bork");
1089 }
fellya205ed82018-09-10 11:52:34 -07001090 return super.statIfFound(path, followSymlinks);
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001091 }
1092
1093 @Override
aehligc801c392017-12-19 07:12:25 -08001094 protected PathFragment readSymbolicLink(Path path) throws IOException {
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001095 if (readlinkThrowsIoException) {
1096 throw new IOException("readlink failed");
1097 }
1098 return super.readSymbolicLink(path);
1099 }
1100 }
1101
1102 private static FileStatusWithDigest statWithDigest(final Path path, final FileStatus stat) {
1103 return new FileStatusWithDigest() {
1104 @Nullable
1105 @Override
1106 public byte[] getDigest() throws IOException {
olaolabfd1d332017-06-19 16:55:24 -04001107 return path.getDigest();
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001108 }
1109
1110 @Override
1111 public boolean isFile() {
1112 return stat.isFile();
1113 }
1114
1115 @Override
1116 public boolean isSpecialFile() {
1117 return stat.isSpecialFile();
1118 }
1119
1120 @Override
1121 public boolean isDirectory() {
1122 return stat.isDirectory();
1123 }
1124
1125 @Override
1126 public boolean isSymbolicLink() {
1127 return stat.isSymbolicLink();
1128 }
1129
1130 @Override
1131 public long getSize() throws IOException {
1132 return stat.getSize();
1133 }
1134
1135 @Override
1136 public long getLastModifiedTime() throws IOException {
1137 return stat.getLastModifiedTime();
1138 }
1139
1140 @Override
1141 public long getLastChangeTime() throws IOException {
1142 return stat.getLastChangeTime();
1143 }
1144
1145 @Override
1146 public long getNodeId() throws IOException {
1147 return stat.getNodeId();
1148 }
1149 };
1150 }
1151
Nathan Harmata8cd29782015-11-10 03:24:01 +00001152 private static Diff getDirtyFilesystemKeys(MemoizingEvaluator evaluator,
1153 FilesystemValueChecker checker) throws InterruptedException {
1154 return checker.getDirtyKeys(evaluator.getValues(), new BasicFilesystemDirtinessChecker());
Han-Wen Nienhuys81b90832015-10-26 16:57:27 +00001155 }
1156}