blob: 9e3932eb8cc55c0d9938329c786b8d55f93a1da8 [file] [log] [blame]
Damien Martin-Guillerezf88f4d82015-09-25 13:56:55 +00001// Copyright 2014 The Bazel Authors. All rights reserved.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +01002//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package com.google.devtools.build.lib.skyframe;
15
tomlua155b532017-11-08 20:12:47 +010016import com.google.common.base.Preconditions;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010017import com.google.common.base.Predicate;
Rumou Duan45e8e572016-06-17 16:43:44 +000018import com.google.common.base.Supplier;
19import com.google.common.base.Suppliers;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010020import com.google.common.base.Throwables;
21import com.google.common.collect.ImmutableList;
Eric Fellheimere6590722015-11-17 17:07:48 +000022import com.google.common.collect.ImmutableSet;
Rumou Duan45e8e572016-06-17 16:43:44 +000023import com.google.common.collect.ImmutableSortedSet;
Miguel Alcon Pinto7cf23652015-03-10 21:27:48 +000024import com.google.common.collect.Range;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010025import com.google.common.collect.Sets;
janakrc3bcb982020-04-14 06:50:08 -070026import com.google.common.flogger.GoogleLogger;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010027import com.google.common.util.concurrent.ThreadFactoryBuilder;
28import com.google.devtools.build.lib.actions.Artifact;
Googler974879d2020-05-27 13:25:52 -070029import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
buchgr4992ae22019-03-20 04:23:32 -070030import com.google.devtools.build.lib.actions.FileArtifactValue;
lberkif7eee1e2019-07-31 05:49:10 -070031import com.google.devtools.build.lib.actions.FileStateType;
Eric Fellheimer6a9d7e52015-06-18 22:08:32 +000032import com.google.devtools.build.lib.concurrent.ExecutorUtil;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010033import com.google.devtools.build.lib.concurrent.Sharder;
34import com.google.devtools.build.lib.concurrent.ThrowableRecordingRunnableWrapper;
Nathan Harmatae1b61d02015-10-06 00:09:42 +000035import com.google.devtools.build.lib.profiler.AutoProfiler;
36import com.google.devtools.build.lib.profiler.AutoProfiler.ElapsedTimeReceiver;
twerth646dfd12018-07-04 01:58:40 -070037import com.google.devtools.build.lib.profiler.Profiler;
38import com.google.devtools.build.lib.profiler.SilentCloseable;
Janak Ramakrishnan6ddbb6e2015-07-28 21:39:22 +000039import com.google.devtools.build.lib.skyframe.SkyValueDirtinessChecker.DirtyResult;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010040import com.google.devtools.build.lib.util.LoggingUtil;
41import com.google.devtools.build.lib.util.Pair;
42import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
43import com.google.devtools.build.lib.vfs.BatchStat;
44import com.google.devtools.build.lib.vfs.FileStatusWithDigest;
Eric Fellheimere6590722015-11-17 17:07:48 +000045import com.google.devtools.build.lib.vfs.ModifiedFileSet;
Michael Thvedte4a7b0792016-02-09 12:15:53 +000046import com.google.devtools.build.lib.vfs.Path;
Eric Fellheimere6590722015-11-17 17:07:48 +000047import com.google.devtools.build.lib.vfs.PathFragment;
janakr0c42fc82018-09-14 10:37:25 -070048import com.google.devtools.build.lib.vfs.Symlinks;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010049import com.google.devtools.build.skyframe.Differencer;
nharmataacccd422020-01-27 13:38:23 -080050import com.google.devtools.build.skyframe.Differencer.DiffWithDelta.Delta;
janakre54491e2018-07-11 16:29:13 -070051import com.google.devtools.build.skyframe.FunctionHermeticity;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010052import com.google.devtools.build.skyframe.SkyFunctionName;
53import com.google.devtools.build.skyframe.SkyKey;
54import com.google.devtools.build.skyframe.SkyValue;
Nathan Harmata8cd29782015-11-10 03:24:01 +000055import com.google.devtools.build.skyframe.WalkableGraph;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010056import java.io.IOException;
57import java.util.Collection;
58import java.util.Collections;
59import java.util.HashMap;
Nathan Harmata8cd29782015-11-10 03:24:01 +000060import java.util.HashSet;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010061import java.util.List;
62import java.util.Map;
Rumou Duan45e8e572016-06-17 16:43:44 +000063import java.util.NavigableSet;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010064import java.util.Set;
65import java.util.concurrent.ConcurrentHashMap;
66import java.util.concurrent.ExecutorService;
67import java.util.concurrent.Executors;
cushon4d70fae2017-04-11 01:01:13 +000068import java.util.concurrent.Future;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010069import java.util.concurrent.atomic.AtomicInteger;
70import java.util.logging.Level;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010071import javax.annotation.Nullable;
72
73/**
74 * A helper class to find dirty values by accessing the filesystem directly (contrast with
75 * {@link DiffAwareness}).
76 */
Nathan Harmata8cd29782015-11-10 03:24:01 +000077public class FilesystemValueChecker {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010078
janakrc3bcb982020-04-14 06:50:08 -070079 private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010080
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010081 private static final Predicate<SkyKey> ACTION_FILTER =
82 SkyFunctionName.functionIs(SkyFunctions.ACTION_EXECUTION);
83
buchgr4992ae22019-03-20 04:23:32 -070084 @Nullable private final TimestampGranularityMonitor tsgm;
twerth5aaceb52020-04-07 06:31:56 -070085 @Nullable private final Range<Long> lastExecutionTimeRange;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010086 private AtomicInteger modifiedOutputFilesCounter = new AtomicInteger(0);
Miguel Alcon Pinto7cf23652015-03-10 21:27:48 +000087 private AtomicInteger modifiedOutputFilesIntraBuildCounter = new AtomicInteger(0);
twerth5aaceb52020-04-07 06:31:56 -070088 private final int numThreads;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010089
steinman39c00d22020-03-20 15:23:10 -070090 public FilesystemValueChecker(
twerth5aaceb52020-04-07 06:31:56 -070091 @Nullable TimestampGranularityMonitor tsgm,
92 @Nullable Range<Long> lastExecutionTimeRange,
93 int numThreads) {
Nathan Harmata2ff0a6d2015-08-12 21:10:56 +000094 this.tsgm = tsgm;
95 this.lastExecutionTimeRange = lastExecutionTimeRange;
twerth5aaceb52020-04-07 06:31:56 -070096 this.numThreads = numThreads;
Nathan Harmata2ff0a6d2015-08-12 21:10:56 +000097 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010098 /**
Nathan Harmata8cd29782015-11-10 03:24:01 +000099 * Returns a {@link Differencer.DiffWithDelta} containing keys from the give map that are dirty
100 * according to the passed-in {@code dirtinessChecker}.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100101 */
Nathan Harmata8cd29782015-11-10 03:24:01 +0000102 // TODO(bazel-team): Refactor these methods so that FilesystemValueChecker only operates on a
103 // WalkableGraph.
nharmataacccd422020-01-27 13:38:23 -0800104 ImmutableBatchDirtyResult getDirtyKeys(
105 Map<SkyKey, SkyValue> valuesMap, SkyValueDirtinessChecker dirtinessChecker)
106 throws InterruptedException {
Nathan Harmata8cd29782015-11-10 03:24:01 +0000107 return getDirtyValues(new MapBackedValueFetcher(valuesMap), valuesMap.keySet(),
108 dirtinessChecker, /*checkMissingValues=*/false);
Nathan Harmata2ff0a6d2015-08-12 21:10:56 +0000109 }
110
111 /**
112 * Returns a {@link Differencer.DiffWithDelta} containing keys that are dirty according to the
113 * passed-in {@code dirtinessChecker}.
114 */
nharmataacccd422020-01-27 13:38:23 -0800115 public ImmutableBatchDirtyResult getNewAndOldValues(
116 Map<SkyKey, SkyValue> valuesMap,
Benjamin Peterson4024e162020-03-06 02:28:29 -0800117 Collection<SkyKey> keys,
nharmataacccd422020-01-27 13:38:23 -0800118 SkyValueDirtinessChecker dirtinessChecker)
119 throws InterruptedException {
Nathan Harmata8cd29782015-11-10 03:24:01 +0000120 return getDirtyValues(new MapBackedValueFetcher(valuesMap), keys,
121 dirtinessChecker, /*checkMissingValues=*/true);
122 }
123
124 /**
125 * Returns a {@link Differencer.DiffWithDelta} containing keys that are dirty according to the
126 * passed-in {@code dirtinessChecker}.
127 */
Benjamin Peterson4024e162020-03-06 02:28:29 -0800128 public Differencer.DiffWithDelta getNewAndOldValues(
129 WalkableGraph walkableGraph,
130 Collection<SkyKey> keys,
131 SkyValueDirtinessChecker dirtinessChecker)
132 throws InterruptedException {
Nathan Harmata8cd29782015-11-10 03:24:01 +0000133 return getDirtyValues(new WalkableGraphBackedValueFetcher(walkableGraph), keys,
134 dirtinessChecker, /*checkMissingValues=*/true);
135 }
136
Janak Ramakrishnan3c0adb22016-08-15 21:54:55 +0000137 private interface ValueFetcher {
Nathan Harmata8cd29782015-11-10 03:24:01 +0000138 @Nullable
Janak Ramakrishnan3c0adb22016-08-15 21:54:55 +0000139 SkyValue get(SkyKey key) throws InterruptedException;
Nathan Harmata8cd29782015-11-10 03:24:01 +0000140 }
141
142 private static class WalkableGraphBackedValueFetcher implements ValueFetcher {
143 private final WalkableGraph walkableGraph;
144
145 private WalkableGraphBackedValueFetcher(WalkableGraph walkableGraph) {
146 this.walkableGraph = walkableGraph;
147 }
148
149 @Override
150 @Nullable
Janak Ramakrishnan3c0adb22016-08-15 21:54:55 +0000151 public SkyValue get(SkyKey key) throws InterruptedException {
Janak Ramakrishnan112840b2016-12-29 21:49:56 +0000152 return walkableGraph.getValue(key);
Nathan Harmata8cd29782015-11-10 03:24:01 +0000153 }
154 }
155
156 private static class MapBackedValueFetcher implements ValueFetcher {
157 private final Map<SkyKey, SkyValue> valuesMap;
158
159 private MapBackedValueFetcher(Map<SkyKey, SkyValue> valuesMap) {
160 this.valuesMap = valuesMap;
161 }
162
163 @Override
164 @Nullable
165 public SkyValue get(SkyKey key) {
166 return valuesMap.get(key);
167 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100168 }
169
170 /**
steinman39c00d22020-03-20 15:23:10 -0700171 * Return a collection of action values which have output files that are not in-sync with the
172 * on-disk file value (were modified externally).
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100173 */
steinman39c00d22020-03-20 15:23:10 -0700174 Collection<SkyKey> getDirtyActionValues(
175 Map<SkyKey, SkyValue> valuesMap,
176 @Nullable final BatchStat batchStatter,
177 ModifiedFileSet modifiedOutputFiles,
178 boolean trustRemoteArtifacts)
179 throws InterruptedException {
Eric Fellheimere6590722015-11-17 17:07:48 +0000180 if (modifiedOutputFiles == ModifiedFileSet.NOTHING_MODIFIED) {
janakrc3bcb982020-04-14 06:50:08 -0700181 logger.atInfo().log("Not checking for dirty actions since nothing was modified");
Eric Fellheimere6590722015-11-17 17:07:48 +0000182 return ImmutableList.of();
183 }
janakrc3bcb982020-04-14 06:50:08 -0700184 logger.atInfo().log("Accumulating dirty actions");
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100185 final int numOutputJobs = Runtime.getRuntime().availableProcessors() * 4;
Nathan Harmata8cd29782015-11-10 03:24:01 +0000186 final Set<SkyKey> actionSkyKeys = new HashSet<>();
twerth646dfd12018-07-04 01:58:40 -0700187 try (SilentCloseable c = Profiler.instance().profile("getDirtyActionValues.filter_actions")) {
188 for (SkyKey key : valuesMap.keySet()) {
189 if (ACTION_FILTER.apply(key)) {
190 actionSkyKeys.add(key);
191 }
Nathan Harmata8cd29782015-11-10 03:24:01 +0000192 }
193 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100194 final Sharder<Pair<SkyKey, ActionExecutionValue>> outputShards =
195 new Sharder<>(numOutputJobs, actionSkyKeys.size());
196
197 for (SkyKey key : actionSkyKeys) {
Nathan Harmata8cd29782015-11-10 03:24:01 +0000198 outputShards.add(Pair.of(key, (ActionExecutionValue) valuesMap.get(key)));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100199 }
janakrc3bcb982020-04-14 06:50:08 -0700200 logger.atInfo().log("Sharded action values for batching");
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100201
202 ExecutorService executor = Executors.newFixedThreadPool(
203 numOutputJobs,
204 new ThreadFactoryBuilder().setNameFormat("FileSystem Output File Invalidator %d").build());
205
206 Collection<SkyKey> dirtyKeys = Sets.newConcurrentHashSet();
207 ThrowableRecordingRunnableWrapper wrapper =
208 new ThrowableRecordingRunnableWrapper("FileSystemValueChecker#getDirtyActionValues");
209
210 modifiedOutputFilesCounter.set(0);
Miguel Alcon Pinto7cf23652015-03-10 21:27:48 +0000211 modifiedOutputFilesIntraBuildCounter.set(0);
Rumou Duan45e8e572016-06-17 16:43:44 +0000212 final ImmutableSet<PathFragment> knownModifiedOutputFiles =
Eric Fellheimere6590722015-11-17 17:07:48 +0000213 modifiedOutputFiles == ModifiedFileSet.EVERYTHING_MODIFIED
214 ? null
215 : modifiedOutputFiles.modifiedSourceFiles();
Rumou Duan45e8e572016-06-17 16:43:44 +0000216
217 // Initialized lazily through a supplier because it is only used to check modified
218 // TreeArtifacts, which are not frequently used in builds.
219 Supplier<NavigableSet<PathFragment>> sortedKnownModifiedOutputFiles =
220 Suppliers.memoize(new Supplier<NavigableSet<PathFragment>>() {
221 @Override
222 public NavigableSet<PathFragment> get() {
223 if (knownModifiedOutputFiles == null) {
224 return null;
225 } else {
226 return ImmutableSortedSet.copyOf(knownModifiedOutputFiles);
227 }
228 }
229 });
230
twerth646dfd12018-07-04 01:58:40 -0700231 boolean interrupted;
232 try (SilentCloseable c = Profiler.instance().profile("getDirtyActionValues.stat_files")) {
233 for (List<Pair<SkyKey, ActionExecutionValue>> shard : outputShards) {
234 Runnable job =
235 (batchStatter == null)
236 ? outputStatJob(
steinman39c00d22020-03-20 15:23:10 -0700237 dirtyKeys,
238 shard,
239 knownModifiedOutputFiles,
240 sortedKnownModifiedOutputFiles,
241 trustRemoteArtifacts)
twerth646dfd12018-07-04 01:58:40 -0700242 : batchStatJob(
243 dirtyKeys,
244 shard,
245 batchStatter,
246 knownModifiedOutputFiles,
steinman39c00d22020-03-20 15:23:10 -0700247 sortedKnownModifiedOutputFiles,
248 trustRemoteArtifacts);
twerth646dfd12018-07-04 01:58:40 -0700249 Future<?> unused = executor.submit(wrapper.wrap(job));
250 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100251
twerth646dfd12018-07-04 01:58:40 -0700252 interrupted = ExecutorUtil.interruptibleShutdown(executor);
253 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100254 Throwables.propagateIfPossible(wrapper.getFirstThrownError());
janakrc3bcb982020-04-14 06:50:08 -0700255 logger.atInfo().log("Completed output file stat checks");
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100256 if (interrupted) {
257 throw new InterruptedException();
258 }
259 return dirtyKeys;
260 }
261
steinman39c00d22020-03-20 15:23:10 -0700262 private Runnable batchStatJob(
263 final Collection<SkyKey> dirtyKeys,
264 final List<Pair<SkyKey, ActionExecutionValue>> shard,
265 final BatchStat batchStatter,
266 final ImmutableSet<PathFragment> knownModifiedOutputFiles,
267 final Supplier<NavigableSet<PathFragment>> sortedKnownModifiedOutputFiles,
268 boolean trustRemoteArtifacts) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100269 return new Runnable() {
270 @Override
271 public void run() {
Rumou Duana77f32c2016-04-13 21:59:21 +0000272 Map<Artifact, Pair<SkyKey, ActionExecutionValue>> fileToKeyAndValue = new HashMap<>();
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000273 Map<Artifact, Pair<SkyKey, ActionExecutionValue>> treeArtifactsToKeyAndValue =
274 new HashMap<>();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100275 for (Pair<SkyKey, ActionExecutionValue> keyAndValue : shard) {
276 ActionExecutionValue actionValue = keyAndValue.getSecond();
277 if (actionValue == null) {
278 dirtyKeys.add(keyAndValue.getFirst());
279 } else {
Rumou Duana77f32c2016-04-13 21:59:21 +0000280 for (Artifact artifact : actionValue.getAllFileValues().keySet()) {
lberkic63bafa2019-08-01 10:23:49 -0700281 if (!artifact.isMiddlemanArtifact()
282 && shouldCheckFile(knownModifiedOutputFiles, artifact)) {
Rumou Duana77f32c2016-04-13 21:59:21 +0000283 fileToKeyAndValue.put(artifact, keyAndValue);
Eric Fellheimere6590722015-11-17 17:07:48 +0000284 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100285 }
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000286
Googler974879d2020-05-27 13:25:52 -0700287 for (Map.Entry<Artifact, TreeArtifactValue> entry :
288 actionValue.getAllTreeArtifactValues().entrySet()) {
289 Artifact treeArtifact = entry.getKey();
290 TreeArtifactValue tree = entry.getValue();
291 for (TreeFileArtifact child : tree.getChildren()) {
292 if (shouldCheckFile(knownModifiedOutputFiles, child)) {
293 fileToKeyAndValue.put(child, keyAndValue);
294 }
295 }
296 if (shouldCheckTreeArtifact(sortedKnownModifiedOutputFiles.get(), treeArtifact)) {
297 treeArtifactsToKeyAndValue.put(treeArtifact, keyAndValue);
Rumou Duan45e8e572016-06-17 16:43:44 +0000298 }
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000299 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100300 }
301 }
302
Rumou Duana77f32c2016-04-13 21:59:21 +0000303 List<Artifact> artifacts = ImmutableList.copyOf(fileToKeyAndValue.keySet());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100304 List<FileStatusWithDigest> stats;
305 try {
lberki97abb522017-09-04 18:51:57 +0200306 stats =
307 batchStatter.batchStat(
308 /*includeDigest=*/ true,
309 /*includeLinks=*/ true,
310 Artifact.asPathFragments(artifacts));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100311 } catch (IOException e) {
312 // Batch stat did not work. Log an exception and fall back on system calls.
313 LoggingUtil.logToRemote(Level.WARNING, "Unable to process batch stat", e);
janakrc3bcb982020-04-14 06:50:08 -0700314 logger.atWarning().withCause(e).log("Unable to process batch stat");
steinman39c00d22020-03-20 15:23:10 -0700315 outputStatJob(
316 dirtyKeys,
317 shard,
318 knownModifiedOutputFiles,
319 sortedKnownModifiedOutputFiles,
320 trustRemoteArtifacts)
Rumou Duan45e8e572016-06-17 16:43:44 +0000321 .run();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100322 return;
323 } catch (InterruptedException e) {
324 // We handle interrupt in the main thread.
325 return;
326 }
327
lberki97abb522017-09-04 18:51:57 +0200328 Preconditions.checkState(
329 artifacts.size() == stats.size(),
330 "artifacts.size() == %s stats.size() == %s",
331 artifacts.size(),
332 stats.size());
Rumou Duana77f32c2016-04-13 21:59:21 +0000333 for (int i = 0; i < artifacts.size(); i++) {
334 Artifact artifact = artifacts.get(i);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100335 FileStatusWithDigest stat = stats.get(i);
Rumou Duana77f32c2016-04-13 21:59:21 +0000336 Pair<SkyKey, ActionExecutionValue> keyAndValue = fileToKeyAndValue.get(artifact);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100337 ActionExecutionValue actionValue = keyAndValue.getSecond();
338 SkyKey key = keyAndValue.getFirst();
Googleraed41602020-06-02 12:22:53 -0700339 FileArtifactValue lastKnownData = actionValue.getExistingFileArtifactValue(artifact);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100340 try {
lberkif7eee1e2019-07-31 05:49:10 -0700341 FileArtifactValue newData =
342 ActionMetadataHandler.fileArtifactValueFromArtifact(artifact, stat, tsgm);
lberkic35878a2019-08-01 02:28:54 -0700343 if (newData.couldBeModifiedSince(lastKnownData)) {
lberki299e3f02019-07-31 04:49:21 -0700344 updateIntraBuildModifiedCounter(stat != null ? stat.getLastChangeTime() : -1);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100345 modifiedOutputFilesCounter.getAndIncrement();
346 dirtyKeys.add(key);
347 }
348 } catch (IOException e) {
349 // This is an unexpected failure getting a digest or symlink target.
350 modifiedOutputFilesCounter.getAndIncrement();
351 dirtyKeys.add(key);
352 }
353 }
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000354
355 // Unfortunately, there exists no facility to batch list directories.
356 // We must use direct filesystem calls.
357 for (Map.Entry<Artifact, Pair<SkyKey, ActionExecutionValue>> entry :
358 treeArtifactsToKeyAndValue.entrySet()) {
359 Artifact artifact = entry.getKey();
360 if (treeArtifactIsDirty(
361 entry.getKey(), entry.getValue().getSecond().getTreeArtifactValue(artifact))) {
362 Path path = artifact.getPath();
363 // Count the changed directory as one "file".
364 // TODO(bazel-team): There are no tests for this codepath.
365 try {
lberki299e3f02019-07-31 04:49:21 -0700366 updateIntraBuildModifiedCounter(path.exists() ? path.getLastModifiedTime() : -1);
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000367 } catch (IOException e) {
368 // Do nothing here.
369 }
370
371 modifiedOutputFilesCounter.getAndIncrement();
372 dirtyKeys.add(entry.getValue().getFirst());
373 }
374 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100375 }
376 };
377 }
378
lberki299e3f02019-07-31 04:49:21 -0700379 private void updateIntraBuildModifiedCounter(long time) {
380 if (lastExecutionTimeRange != null && lastExecutionTimeRange.contains(time)) {
Miguel Alcon Pinto7cf23652015-03-10 21:27:48 +0000381 modifiedOutputFilesIntraBuildCounter.incrementAndGet();
382 }
383 }
384
steinman39c00d22020-03-20 15:23:10 -0700385 private Runnable outputStatJob(
386 final Collection<SkyKey> dirtyKeys,
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000387 final List<Pair<SkyKey, ActionExecutionValue>> shard,
Rumou Duan45e8e572016-06-17 16:43:44 +0000388 final ImmutableSet<PathFragment> knownModifiedOutputFiles,
steinman39c00d22020-03-20 15:23:10 -0700389 final Supplier<NavigableSet<PathFragment>> sortedKnownModifiedOutputFiles,
390 boolean trustRemoteArtifacts) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100391 return new Runnable() {
392 @Override
393 public void run() {
394 for (Pair<SkyKey, ActionExecutionValue> keyAndValue : shard) {
395 ActionExecutionValue value = keyAndValue.getSecond();
Eric Fellheimere6590722015-11-17 17:07:48 +0000396 if (value == null
Rumou Duan45e8e572016-06-17 16:43:44 +0000397 || actionValueIsDirtyWithDirectSystemCalls(
steinman39c00d22020-03-20 15:23:10 -0700398 value,
399 knownModifiedOutputFiles,
400 sortedKnownModifiedOutputFiles,
401 trustRemoteArtifacts)) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100402 dirtyKeys.add(keyAndValue.getFirst());
403 }
404 }
405 }
406 };
407 }
408
409 /**
Miguel Alcon Pinto7cf23652015-03-10 21:27:48 +0000410 * Returns the number of modified output files inside of dirty actions.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100411 */
412 int getNumberOfModifiedOutputFiles() {
413 return modifiedOutputFilesCounter.get();
414 }
415
Janak Ramakrishnan6ddbb6e2015-07-28 21:39:22 +0000416 /** Returns the number of modified output files that occur during the previous build. */
417 int getNumberOfModifiedOutputFilesDuringPreviousBuild() {
Miguel Alcon Pinto7cf23652015-03-10 21:27:48 +0000418 return modifiedOutputFilesIntraBuildCounter.get();
419 }
420
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000421 private boolean treeArtifactIsDirty(Artifact artifact, TreeArtifactValue value) {
422 if (artifact.getPath().isSymbolicLink()) {
423 // TreeArtifacts may not be symbolic links.
424 return true;
425 }
426
427 // There doesn't appear to be any facility to batch list directories... we must
428 // do things the 'slow' way.
429 try {
felly09efb3f2018-07-26 07:46:15 -0700430 Set<PathFragment> currentDirectoryValue =
431 TreeArtifactValue.explodeDirectory(artifact.getPath());
buchgr4992ae22019-03-20 04:23:32 -0700432 return !(currentDirectoryValue.isEmpty() && value.isRemote())
433 && !currentDirectoryValue.equals(value.getChildPaths());
Rumou Duan9ad28cd2016-10-19 19:28:06 +0000434 } catch (IOException e) {
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000435 return true;
436 }
437 }
438
Googler974879d2020-05-27 13:25:52 -0700439 private boolean artifactIsDirtyWithDirectSystemCalls(
440 ImmutableSet<PathFragment> knownModifiedOutputFiles,
441 boolean trustRemoteArtifacts,
442 Map.Entry<? extends Artifact, FileArtifactValue> entry) {
443 Artifact file = entry.getKey();
444 FileArtifactValue lastKnownData = entry.getValue();
445 if (file.isMiddlemanArtifact() || !shouldCheckFile(knownModifiedOutputFiles, file)) {
446 return false;
447 }
448 try {
449 FileArtifactValue fileMetadata =
450 ActionMetadataHandler.fileArtifactValueFromArtifact(file, null, tsgm);
451 boolean trustRemoteValue =
452 fileMetadata.getType() == FileStateType.NONEXISTENT
453 && lastKnownData.isRemote()
454 && trustRemoteArtifacts;
455 if (!trustRemoteValue && fileMetadata.couldBeModifiedSince(lastKnownData)) {
456 updateIntraBuildModifiedCounter(
457 fileMetadata.getType() != FileStateType.NONEXISTENT
458 ? file.getPath().getLastModifiedTime(Symlinks.FOLLOW)
459 : -1);
460 modifiedOutputFilesCounter.getAndIncrement();
461 return true;
462 }
463 return false;
464 } catch (IOException e) {
465 // This is an unexpected failure getting a digest or symlink target.
466 modifiedOutputFilesCounter.getAndIncrement();
467 return true;
468 }
469 }
470
steinman39c00d22020-03-20 15:23:10 -0700471 private boolean actionValueIsDirtyWithDirectSystemCalls(
472 ActionExecutionValue actionValue,
Rumou Duan45e8e572016-06-17 16:43:44 +0000473 ImmutableSet<PathFragment> knownModifiedOutputFiles,
steinman39c00d22020-03-20 15:23:10 -0700474 Supplier<NavigableSet<PathFragment>> sortedKnownModifiedOutputFiles,
475 boolean trustRemoteArtifacts) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100476 boolean isDirty = false;
lberkif7eee1e2019-07-31 05:49:10 -0700477 for (Map.Entry<Artifact, FileArtifactValue> entry : actionValue.getAllFileValues().entrySet()) {
Googler974879d2020-05-27 13:25:52 -0700478 if (artifactIsDirtyWithDirectSystemCalls(
479 knownModifiedOutputFiles, trustRemoteArtifacts, entry)) {
480 isDirty = true;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100481 }
482 }
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000483
484 for (Map.Entry<Artifact, TreeArtifactValue> entry :
485 actionValue.getAllTreeArtifactValues().entrySet()) {
Googler974879d2020-05-27 13:25:52 -0700486 TreeArtifactValue tree = entry.getValue();
Rumou Duan45e8e572016-06-17 16:43:44 +0000487
Googler974879d2020-05-27 13:25:52 -0700488 if (!tree.isRemote()) {
489 for (Map.Entry<TreeFileArtifact, FileArtifactValue> childEntry :
490 tree.getChildValues().entrySet()) {
491 if (artifactIsDirtyWithDirectSystemCalls(
492 knownModifiedOutputFiles, trustRemoteArtifacts, childEntry)) {
493 isDirty = true;
494 }
495 }
496 }
497
498 Artifact treeArtifact = entry.getKey();
499
500 if (shouldCheckTreeArtifact(sortedKnownModifiedOutputFiles.get(), treeArtifact)
501 && treeArtifactIsDirty(treeArtifact, entry.getValue())) {
502 Path path = treeArtifact.getPath();
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000503 // Count the changed directory as one "file".
504 try {
lberki299e3f02019-07-31 04:49:21 -0700505 updateIntraBuildModifiedCounter(path.exists() ? path.getLastModifiedTime() : -1);
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000506 } catch (IOException e) {
507 // Do nothing here.
508 }
509
510 modifiedOutputFilesCounter.getAndIncrement();
511 isDirty = true;
512 }
513 }
514
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100515 return isDirty;
516 }
517
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000518 private static boolean shouldCheckFile(ImmutableSet<PathFragment> knownModifiedOutputFiles,
Rumou Duana77f32c2016-04-13 21:59:21 +0000519 Artifact artifact) {
Eric Fellheimere6590722015-11-17 17:07:48 +0000520 return knownModifiedOutputFiles == null
Rumou Duana77f32c2016-04-13 21:59:21 +0000521 || knownModifiedOutputFiles.contains(artifact.getExecPath());
Eric Fellheimere6590722015-11-17 17:07:48 +0000522 }
523
Rumou Duan45e8e572016-06-17 16:43:44 +0000524 private static boolean shouldCheckTreeArtifact(
525 @Nullable NavigableSet<PathFragment> knownModifiedOutputFiles, Artifact treeArtifact) {
526 // If null, everything needs to be checked.
527 if (knownModifiedOutputFiles == null) {
528 return true;
529 }
530
531 // Here we do the following to see whether a TreeArtifact is modified:
532 // 1. Sort the set of modified file paths in lexicographical order using TreeSet.
533 // 2. Get the first modified output file path that is greater than or equal to the exec path of
534 // the TreeArtifact to check.
535 // 3. Check whether the returned file path contains the exec path of the TreeArtifact as a
536 // prefix path.
537 PathFragment artifactExecPath = treeArtifact.getExecPath();
538 PathFragment headPath = knownModifiedOutputFiles.ceiling(artifactExecPath);
539
540 return headPath != null && headPath.startsWith(artifactExecPath);
541 }
542
nharmataacccd422020-01-27 13:38:23 -0800543 private ImmutableBatchDirtyResult getDirtyValues(
544 ValueFetcher fetcher,
Benjamin Peterson4024e162020-03-06 02:28:29 -0800545 Collection<SkyKey> keys,
nharmataacccd422020-01-27 13:38:23 -0800546 final SkyValueDirtinessChecker checker,
547 final boolean checkMissingValues)
548 throws InterruptedException {
Janak Ramakrishnan6ddbb6e2015-07-28 21:39:22 +0000549 ExecutorService executor =
550 Executors.newFixedThreadPool(
twerth5aaceb52020-04-07 06:31:56 -0700551 numThreads,
Janak Ramakrishnan6ddbb6e2015-07-28 21:39:22 +0000552 new ThreadFactoryBuilder().setNameFormat("FileSystem Value Invalidator %d").build());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100553
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100554 ThrowableRecordingRunnableWrapper wrapper =
555 new ThrowableRecordingRunnableWrapper("FilesystemValueChecker#getDirtyValues");
Nathan Harmatae1b61d02015-10-06 00:09:42 +0000556 final AtomicInteger numKeysChecked = new AtomicInteger(0);
nharmataacccd422020-01-27 13:38:23 -0800557 MutableBatchDirtyResult batchResult = new MutableBatchDirtyResult(numKeysChecked);
lberki97abb522017-09-04 18:51:57 +0200558 ElapsedTimeReceiver elapsedTimeReceiver =
fellyb6f86422018-02-20 19:01:35 -0800559 elapsedTimeNanos -> {
560 if (elapsedTimeNanos > 0) {
janakrc3bcb982020-04-14 06:50:08 -0700561 logger.atInfo().log(
562 "Spent %d nanoseconds checking %d filesystem nodes (%d scanned)",
563 elapsedTimeNanos, numKeysChecked.get(), keys.size());
Nathan Harmatae1b61d02015-10-06 00:09:42 +0000564 }
lberki97abb522017-09-04 18:51:57 +0200565 };
Nathan Harmatae1b61d02015-10-06 00:09:42 +0000566 try (AutoProfiler prof = AutoProfiler.create(elapsedTimeReceiver)) {
Nathan Harmata8cd29782015-11-10 03:24:01 +0000567 for (final SkyKey key : keys) {
Nathan Harmata3a509bd2015-10-06 01:00:47 +0000568 if (!checker.applies(key)) {
569 continue;
570 }
janakre54491e2018-07-11 16:29:13 -0700571 Preconditions.checkState(
572 key.functionName().getHermeticity() == FunctionHermeticity.NONHERMETIC,
573 "Only non-hermetic keys can be dirty roots: %s",
574 key);
Nathan Harmatae1b61d02015-10-06 00:09:42 +0000575 executor.execute(
576 wrapper.wrap(
fellyb6f86422018-02-20 19:01:35 -0800577 () -> {
578 SkyValue value;
579 try {
580 value = fetcher.get(key);
581 } catch (InterruptedException e) {
582 // Exit fast. Interrupt is handled below on the main thread.
583 return;
584 }
585 if (!checkMissingValues && value == null) {
586 return;
587 }
588
589 numKeysChecked.incrementAndGet();
590 DirtyResult result = checker.check(key, value, tsgm);
591 if (result.isDirty()) {
592 batchResult.add(key, value, result.getNewValue());
Janak Ramakrishnan6ddbb6e2015-07-28 21:39:22 +0000593 }
Nathan Harmatae1b61d02015-10-06 00:09:42 +0000594 }));
595 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100596
Nathan Harmatae1b61d02015-10-06 00:09:42 +0000597 boolean interrupted = ExecutorUtil.interruptibleShutdown(executor);
598 Throwables.propagateIfPossible(wrapper.getFirstThrownError());
599 if (interrupted) {
600 throw new InterruptedException();
601 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100602 }
nharmataacccd422020-01-27 13:38:23 -0800603 return batchResult.toImmutable();
604 }
605
606 static class ImmutableBatchDirtyResult implements Differencer.DiffWithDelta {
607 private final Collection<SkyKey> dirtyKeysWithoutNewValues;
608 private final Map<SkyKey, Delta> dirtyKeysWithNewAndOldValues;
609 private final int numKeysChecked;
610
611 private ImmutableBatchDirtyResult(
612 Collection<SkyKey> dirtyKeysWithoutNewValues,
613 Map<SkyKey, Delta> dirtyKeysWithNewAndOldValues,
614 int numKeysChecked) {
615 this.dirtyKeysWithoutNewValues = dirtyKeysWithoutNewValues;
616 this.dirtyKeysWithNewAndOldValues = dirtyKeysWithNewAndOldValues;
617 this.numKeysChecked = numKeysChecked;
618 }
619
620 @Override
621 public Collection<SkyKey> changedKeysWithoutNewValues() {
622 return dirtyKeysWithoutNewValues;
623 }
624
625 @Override
626 public Map<SkyKey, Delta> changedKeysWithNewAndOldValues() {
627 return dirtyKeysWithNewAndOldValues;
628 }
629
630 @Override
631 public Map<SkyKey, SkyValue> changedKeysWithNewValues() {
632 return Delta.newValues(dirtyKeysWithNewAndOldValues);
633 }
634
635 int getNumKeysChecked() {
636 return numKeysChecked;
637 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100638 }
639
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100640 /**
nharmataacccd422020-01-27 13:38:23 -0800641 * Result of a batch call to {@link SkyValueDirtinessChecker#check}. Partitions the dirty values
642 * based on whether we have a new value available for them or not.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100643 */
nharmataacccd422020-01-27 13:38:23 -0800644 private static class MutableBatchDirtyResult {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100645 private final Set<SkyKey> concurrentDirtyKeysWithoutNewValues =
646 Collections.newSetFromMap(new ConcurrentHashMap<SkyKey, Boolean>());
Nathan Harmata2ff0a6d2015-08-12 21:10:56 +0000647 private final ConcurrentHashMap<SkyKey, Delta> concurrentDirtyKeysWithNewAndOldValues =
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100648 new ConcurrentHashMap<>();
nharmataacccd422020-01-27 13:38:23 -0800649 private final AtomicInteger numChecked;
650
651 private MutableBatchDirtyResult(AtomicInteger numChecked) {
652 this.numChecked = numChecked;
653 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100654
Nathan Harmata2ff0a6d2015-08-12 21:10:56 +0000655 private void add(SkyKey key, @Nullable SkyValue oldValue, @Nullable SkyValue newValue) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100656 if (newValue == null) {
657 concurrentDirtyKeysWithoutNewValues.add(key);
658 } else {
Nathan Harmata2ff0a6d2015-08-12 21:10:56 +0000659 if (oldValue == null) {
660 concurrentDirtyKeysWithNewAndOldValues.put(key, new Delta(newValue));
661 } else {
662 concurrentDirtyKeysWithNewAndOldValues.put(key, new Delta(oldValue, newValue));
663 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100664 }
665 }
666
nharmataacccd422020-01-27 13:38:23 -0800667 private ImmutableBatchDirtyResult toImmutable() {
668 return new ImmutableBatchDirtyResult(
669 concurrentDirtyKeysWithoutNewValues,
670 concurrentDirtyKeysWithNewAndOldValues,
671 numChecked.get());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100672 }
673 }
674
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100675}