blob: 6fef55c5b941ec5b97860bf029d34a89628dd2fc [file] [log] [blame]
Damien Martin-Guillerezf88f4d82015-09-25 13:56:55 +00001// Copyright 2014 The Bazel Authors. All rights reserved.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +01002//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package com.google.devtools.build.lib.skyframe;
15
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010016import com.google.common.base.Predicate;
Rumou Duan45e8e572016-06-17 16:43:44 +000017import com.google.common.base.Supplier;
18import com.google.common.base.Suppliers;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010019import com.google.common.base.Throwables;
20import com.google.common.collect.ImmutableList;
Eric Fellheimere6590722015-11-17 17:07:48 +000021import com.google.common.collect.ImmutableSet;
Rumou Duan45e8e572016-06-17 16:43:44 +000022import com.google.common.collect.ImmutableSortedSet;
Miguel Alcon Pinto7cf23652015-03-10 21:27:48 +000023import com.google.common.collect.Range;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010024import com.google.common.collect.Sets;
25import com.google.common.util.concurrent.ThreadFactoryBuilder;
26import com.google.devtools.build.lib.actions.Artifact;
Eric Fellheimer6a9d7e52015-06-18 22:08:32 +000027import com.google.devtools.build.lib.concurrent.ExecutorUtil;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010028import com.google.devtools.build.lib.concurrent.Sharder;
29import com.google.devtools.build.lib.concurrent.ThrowableRecordingRunnableWrapper;
Nathan Harmatae1b61d02015-10-06 00:09:42 +000030import com.google.devtools.build.lib.profiler.AutoProfiler;
31import com.google.devtools.build.lib.profiler.AutoProfiler.ElapsedTimeReceiver;
Janak Ramakrishnan6ddbb6e2015-07-28 21:39:22 +000032import com.google.devtools.build.lib.skyframe.SkyValueDirtinessChecker.DirtyResult;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010033import com.google.devtools.build.lib.util.LoggingUtil;
34import com.google.devtools.build.lib.util.Pair;
Mark Schaller6df81792015-12-10 18:47:47 +000035import com.google.devtools.build.lib.util.Preconditions;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010036import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
37import com.google.devtools.build.lib.vfs.BatchStat;
38import com.google.devtools.build.lib.vfs.FileStatusWithDigest;
Eric Fellheimere6590722015-11-17 17:07:48 +000039import com.google.devtools.build.lib.vfs.ModifiedFileSet;
Michael Thvedte4a7b0792016-02-09 12:15:53 +000040import com.google.devtools.build.lib.vfs.Path;
Eric Fellheimere6590722015-11-17 17:07:48 +000041import com.google.devtools.build.lib.vfs.PathFragment;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010042import com.google.devtools.build.skyframe.Differencer;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010043import com.google.devtools.build.skyframe.SkyFunctionName;
44import com.google.devtools.build.skyframe.SkyKey;
45import com.google.devtools.build.skyframe.SkyValue;
Nathan Harmata8cd29782015-11-10 03:24:01 +000046import com.google.devtools.build.skyframe.WalkableGraph;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010047import java.io.IOException;
48import java.util.Collection;
49import java.util.Collections;
50import java.util.HashMap;
Nathan Harmata8cd29782015-11-10 03:24:01 +000051import java.util.HashSet;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010052import java.util.List;
53import java.util.Map;
Rumou Duan45e8e572016-06-17 16:43:44 +000054import java.util.NavigableSet;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010055import java.util.Set;
56import java.util.concurrent.ConcurrentHashMap;
57import java.util.concurrent.ExecutorService;
58import java.util.concurrent.Executors;
Nathan Harmatae1b61d02015-10-06 00:09:42 +000059import java.util.concurrent.TimeUnit;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010060import java.util.concurrent.atomic.AtomicInteger;
61import java.util.logging.Level;
62import java.util.logging.Logger;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010063import javax.annotation.Nullable;
64
65/**
66 * A helper class to find dirty values by accessing the filesystem directly (contrast with
67 * {@link DiffAwareness}).
68 */
Nathan Harmata8cd29782015-11-10 03:24:01 +000069public class FilesystemValueChecker {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010070
Nathan Harmata3a509bd2015-10-06 01:00:47 +000071 private static final int DIRTINESS_CHECK_THREADS = 200;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010072 private static final Logger LOG = Logger.getLogger(FilesystemValueChecker.class.getName());
73
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010074 private static final Predicate<SkyKey> ACTION_FILTER =
75 SkyFunctionName.functionIs(SkyFunctions.ACTION_EXECUTION);
76
77 private final TimestampGranularityMonitor tsgm;
Nathan Harmata9b38b2c2015-08-27 16:11:07 +000078 @Nullable
Miguel Alcon Pinto7cf23652015-03-10 21:27:48 +000079 private final Range<Long> lastExecutionTimeRange;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010080 private AtomicInteger modifiedOutputFilesCounter = new AtomicInteger(0);
Miguel Alcon Pinto7cf23652015-03-10 21:27:48 +000081 private AtomicInteger modifiedOutputFilesIntraBuildCounter = new AtomicInteger(0);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010082
Nathan Harmata8cd29782015-11-10 03:24:01 +000083 public FilesystemValueChecker(@Nullable TimestampGranularityMonitor tsgm,
84 @Nullable Range<Long> lastExecutionTimeRange) {
Nathan Harmata2ff0a6d2015-08-12 21:10:56 +000085 this.tsgm = tsgm;
86 this.lastExecutionTimeRange = lastExecutionTimeRange;
87 }
88
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010089 /**
Nathan Harmata8cd29782015-11-10 03:24:01 +000090 * Returns a {@link Differencer.DiffWithDelta} containing keys from the give map that are dirty
91 * according to the passed-in {@code dirtinessChecker}.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010092 */
Nathan Harmata8cd29782015-11-10 03:24:01 +000093 // TODO(bazel-team): Refactor these methods so that FilesystemValueChecker only operates on a
94 // WalkableGraph.
95 Differencer.DiffWithDelta getDirtyKeys(Map<SkyKey, SkyValue> valuesMap,
96 SkyValueDirtinessChecker dirtinessChecker) throws InterruptedException {
97 return getDirtyValues(new MapBackedValueFetcher(valuesMap), valuesMap.keySet(),
98 dirtinessChecker, /*checkMissingValues=*/false);
Nathan Harmata2ff0a6d2015-08-12 21:10:56 +000099 }
100
101 /**
102 * Returns a {@link Differencer.DiffWithDelta} containing keys that are dirty according to the
103 * passed-in {@code dirtinessChecker}.
104 */
Nathan Harmata8cd29782015-11-10 03:24:01 +0000105 public Differencer.DiffWithDelta getNewAndOldValues(Map<SkyKey, SkyValue> valuesMap,
106 Iterable<SkyKey> keys, SkyValueDirtinessChecker dirtinessChecker)
107 throws InterruptedException {
108 return getDirtyValues(new MapBackedValueFetcher(valuesMap), keys,
109 dirtinessChecker, /*checkMissingValues=*/true);
110 }
111
112 /**
113 * Returns a {@link Differencer.DiffWithDelta} containing keys that are dirty according to the
114 * passed-in {@code dirtinessChecker}.
115 */
116 public Differencer.DiffWithDelta getNewAndOldValues(WalkableGraph walkableGraph,
117 Iterable<SkyKey> keys, SkyValueDirtinessChecker dirtinessChecker)
118 throws InterruptedException {
119 return getDirtyValues(new WalkableGraphBackedValueFetcher(walkableGraph), keys,
120 dirtinessChecker, /*checkMissingValues=*/true);
121 }
122
Janak Ramakrishnan3c0adb22016-08-15 21:54:55 +0000123 private interface ValueFetcher {
Nathan Harmata8cd29782015-11-10 03:24:01 +0000124 @Nullable
Janak Ramakrishnan3c0adb22016-08-15 21:54:55 +0000125 SkyValue get(SkyKey key) throws InterruptedException;
Nathan Harmata8cd29782015-11-10 03:24:01 +0000126 }
127
128 private static class WalkableGraphBackedValueFetcher implements ValueFetcher {
129 private final WalkableGraph walkableGraph;
130
131 private WalkableGraphBackedValueFetcher(WalkableGraph walkableGraph) {
132 this.walkableGraph = walkableGraph;
133 }
134
135 @Override
136 @Nullable
Janak Ramakrishnan3c0adb22016-08-15 21:54:55 +0000137 public SkyValue get(SkyKey key) throws InterruptedException {
Janak Ramakrishnan112840b2016-12-29 21:49:56 +0000138 return walkableGraph.getValue(key);
Nathan Harmata8cd29782015-11-10 03:24:01 +0000139 }
140 }
141
142 private static class MapBackedValueFetcher implements ValueFetcher {
143 private final Map<SkyKey, SkyValue> valuesMap;
144
145 private MapBackedValueFetcher(Map<SkyKey, SkyValue> valuesMap) {
146 this.valuesMap = valuesMap;
147 }
148
149 @Override
150 @Nullable
151 public SkyValue get(SkyKey key) {
152 return valuesMap.get(key);
153 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100154 }
155
156 /**
157 * Return a collection of action values which have output files that are not in-sync with
158 * the on-disk file value (were modified externally).
159 */
Nathan Harmata8cd29782015-11-10 03:24:01 +0000160 Collection<SkyKey> getDirtyActionValues(Map<SkyKey, SkyValue> valuesMap,
Eric Fellheimere6590722015-11-17 17:07:48 +0000161 @Nullable final BatchStat batchStatter, ModifiedFileSet modifiedOutputFiles)
162 throws InterruptedException {
163 if (modifiedOutputFiles == ModifiedFileSet.NOTHING_MODIFIED) {
164 LOG.info("Not checking for dirty actions since nothing was modified");
165 return ImmutableList.of();
166 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100167 LOG.info("Accumulating dirty actions");
168 final int numOutputJobs = Runtime.getRuntime().availableProcessors() * 4;
Nathan Harmata8cd29782015-11-10 03:24:01 +0000169 final Set<SkyKey> actionSkyKeys = new HashSet<>();
170 for (SkyKey key : valuesMap.keySet()) {
171 if (ACTION_FILTER.apply(key)) {
172 actionSkyKeys.add(key);
173 }
174 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100175 final Sharder<Pair<SkyKey, ActionExecutionValue>> outputShards =
176 new Sharder<>(numOutputJobs, actionSkyKeys.size());
177
178 for (SkyKey key : actionSkyKeys) {
Nathan Harmata8cd29782015-11-10 03:24:01 +0000179 outputShards.add(Pair.of(key, (ActionExecutionValue) valuesMap.get(key)));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100180 }
181 LOG.info("Sharded action values for batching");
182
183 ExecutorService executor = Executors.newFixedThreadPool(
184 numOutputJobs,
185 new ThreadFactoryBuilder().setNameFormat("FileSystem Output File Invalidator %d").build());
186
187 Collection<SkyKey> dirtyKeys = Sets.newConcurrentHashSet();
188 ThrowableRecordingRunnableWrapper wrapper =
189 new ThrowableRecordingRunnableWrapper("FileSystemValueChecker#getDirtyActionValues");
190
191 modifiedOutputFilesCounter.set(0);
Miguel Alcon Pinto7cf23652015-03-10 21:27:48 +0000192 modifiedOutputFilesIntraBuildCounter.set(0);
Rumou Duan45e8e572016-06-17 16:43:44 +0000193 final ImmutableSet<PathFragment> knownModifiedOutputFiles =
Eric Fellheimere6590722015-11-17 17:07:48 +0000194 modifiedOutputFiles == ModifiedFileSet.EVERYTHING_MODIFIED
195 ? null
196 : modifiedOutputFiles.modifiedSourceFiles();
Rumou Duan45e8e572016-06-17 16:43:44 +0000197
198 // Initialized lazily through a supplier because it is only used to check modified
199 // TreeArtifacts, which are not frequently used in builds.
200 Supplier<NavigableSet<PathFragment>> sortedKnownModifiedOutputFiles =
201 Suppliers.memoize(new Supplier<NavigableSet<PathFragment>>() {
202 @Override
203 public NavigableSet<PathFragment> get() {
204 if (knownModifiedOutputFiles == null) {
205 return null;
206 } else {
207 return ImmutableSortedSet.copyOf(knownModifiedOutputFiles);
208 }
209 }
210 });
211
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100212 for (List<Pair<SkyKey, ActionExecutionValue>> shard : outputShards) {
213 Runnable job = (batchStatter == null)
Rumou Duan45e8e572016-06-17 16:43:44 +0000214 ? outputStatJob(dirtyKeys, shard, knownModifiedOutputFiles,
215 sortedKnownModifiedOutputFiles)
216 : batchStatJob(dirtyKeys, shard, batchStatter, knownModifiedOutputFiles,
217 sortedKnownModifiedOutputFiles);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100218 executor.submit(wrapper.wrap(job));
219 }
220
Eric Fellheimer6a9d7e52015-06-18 22:08:32 +0000221 boolean interrupted = ExecutorUtil.interruptibleShutdown(executor);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100222 Throwables.propagateIfPossible(wrapper.getFirstThrownError());
223 LOG.info("Completed output file stat checks");
224 if (interrupted) {
225 throw new InterruptedException();
226 }
227 return dirtyKeys;
228 }
229
230 private Runnable batchStatJob(final Collection<SkyKey> dirtyKeys,
Eric Fellheimere6590722015-11-17 17:07:48 +0000231 final List<Pair<SkyKey, ActionExecutionValue>> shard,
Rumou Duan45e8e572016-06-17 16:43:44 +0000232 final BatchStat batchStatter, final ImmutableSet<PathFragment> knownModifiedOutputFiles,
233 final Supplier<NavigableSet<PathFragment>> sortedKnownModifiedOutputFiles) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100234 return new Runnable() {
235 @Override
236 public void run() {
Rumou Duana77f32c2016-04-13 21:59:21 +0000237 Map<Artifact, Pair<SkyKey, ActionExecutionValue>> fileToKeyAndValue = new HashMap<>();
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000238 Map<Artifact, Pair<SkyKey, ActionExecutionValue>> treeArtifactsToKeyAndValue =
239 new HashMap<>();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100240 for (Pair<SkyKey, ActionExecutionValue> keyAndValue : shard) {
241 ActionExecutionValue actionValue = keyAndValue.getSecond();
242 if (actionValue == null) {
243 dirtyKeys.add(keyAndValue.getFirst());
244 } else {
Rumou Duana77f32c2016-04-13 21:59:21 +0000245 for (Artifact artifact : actionValue.getAllFileValues().keySet()) {
246 if (shouldCheckFile(knownModifiedOutputFiles, artifact)) {
247 fileToKeyAndValue.put(artifact, keyAndValue);
Eric Fellheimere6590722015-11-17 17:07:48 +0000248 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100249 }
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000250
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000251 for (Artifact artifact : actionValue.getAllTreeArtifactValues().keySet()) {
Rumou Duan45e8e572016-06-17 16:43:44 +0000252 if (shouldCheckTreeArtifact(sortedKnownModifiedOutputFiles.get(), artifact)) {
253 treeArtifactsToKeyAndValue.put(artifact, keyAndValue);
254 }
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000255 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100256 }
257 }
258
Rumou Duana77f32c2016-04-13 21:59:21 +0000259 List<Artifact> artifacts = ImmutableList.copyOf(fileToKeyAndValue.keySet());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100260 List<FileStatusWithDigest> stats;
261 try {
262 stats = batchStatter.batchStat(/*includeDigest=*/true, /*includeLinks=*/true,
Rumou Duana77f32c2016-04-13 21:59:21 +0000263 Artifact.asPathFragments(artifacts));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100264 } catch (IOException e) {
265 // Batch stat did not work. Log an exception and fall back on system calls.
266 LoggingUtil.logToRemote(Level.WARNING, "Unable to process batch stat", e);
Rumou Duan45e8e572016-06-17 16:43:44 +0000267 outputStatJob(dirtyKeys, shard, knownModifiedOutputFiles, sortedKnownModifiedOutputFiles)
268 .run();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100269 return;
270 } catch (InterruptedException e) {
271 // We handle interrupt in the main thread.
272 return;
273 }
274
Rumou Duana77f32c2016-04-13 21:59:21 +0000275 Preconditions.checkState(artifacts.size() == stats.size(),
276 "artifacts.size() == %s stats.size() == %s", artifacts.size(), stats.size());
277 for (int i = 0; i < artifacts.size(); i++) {
278 Artifact artifact = artifacts.get(i);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100279 FileStatusWithDigest stat = stats.get(i);
Rumou Duana77f32c2016-04-13 21:59:21 +0000280 Pair<SkyKey, ActionExecutionValue> keyAndValue = fileToKeyAndValue.get(artifact);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100281 ActionExecutionValue actionValue = keyAndValue.getSecond();
282 SkyKey key = keyAndValue.getFirst();
Rumou Duana77f32c2016-04-13 21:59:21 +0000283 FileValue lastKnownData = actionValue.getAllFileValues().get(artifact);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100284 try {
Rumou Duana77f32c2016-04-13 21:59:21 +0000285 FileValue newData = ActionMetadataHandler.fileValueFromArtifact(artifact, stat,
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000286 tsgm);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100287 if (!newData.equals(lastKnownData)) {
Miguel Alcon Pintobdfdd092015-04-10 19:04:49 +0000288 updateIntraBuildModifiedCounter(stat != null ? stat.getLastChangeTime() : -1,
289 lastKnownData.isSymlink(), newData.isSymlink());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100290 modifiedOutputFilesCounter.getAndIncrement();
291 dirtyKeys.add(key);
292 }
293 } catch (IOException e) {
294 // This is an unexpected failure getting a digest or symlink target.
295 modifiedOutputFilesCounter.getAndIncrement();
296 dirtyKeys.add(key);
297 }
298 }
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000299
300 // Unfortunately, there exists no facility to batch list directories.
301 // We must use direct filesystem calls.
302 for (Map.Entry<Artifact, Pair<SkyKey, ActionExecutionValue>> entry :
303 treeArtifactsToKeyAndValue.entrySet()) {
304 Artifact artifact = entry.getKey();
305 if (treeArtifactIsDirty(
306 entry.getKey(), entry.getValue().getSecond().getTreeArtifactValue(artifact))) {
307 Path path = artifact.getPath();
308 // Count the changed directory as one "file".
309 // TODO(bazel-team): There are no tests for this codepath.
310 try {
311 updateIntraBuildModifiedCounter(path.exists()
312 ? path.getLastModifiedTime()
313 : -1, false, path.isSymbolicLink());
314 } catch (IOException e) {
315 // Do nothing here.
316 }
317
318 modifiedOutputFilesCounter.getAndIncrement();
319 dirtyKeys.add(entry.getValue().getFirst());
320 }
321 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100322 }
323 };
324 }
325
Miguel Alcon Pintobdfdd092015-04-10 19:04:49 +0000326 private void updateIntraBuildModifiedCounter(long time, boolean oldWasSymlink,
Nathan Harmata7a344272015-06-05 20:52:38 +0000327 boolean newIsSymlink) {
Miguel Alcon Pintobdfdd092015-04-10 19:04:49 +0000328 if (lastExecutionTimeRange != null
329 && lastExecutionTimeRange.contains(time)
330 && !(oldWasSymlink && newIsSymlink)) {
Miguel Alcon Pinto7cf23652015-03-10 21:27:48 +0000331 modifiedOutputFilesIntraBuildCounter.incrementAndGet();
332 }
333 }
334
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100335 private Runnable outputStatJob(final Collection<SkyKey> dirtyKeys,
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000336 final List<Pair<SkyKey, ActionExecutionValue>> shard,
Rumou Duan45e8e572016-06-17 16:43:44 +0000337 final ImmutableSet<PathFragment> knownModifiedOutputFiles,
338 final Supplier<NavigableSet<PathFragment>> sortedKnownModifiedOutputFiles) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100339 return new Runnable() {
340 @Override
341 public void run() {
342 for (Pair<SkyKey, ActionExecutionValue> keyAndValue : shard) {
343 ActionExecutionValue value = keyAndValue.getSecond();
Eric Fellheimere6590722015-11-17 17:07:48 +0000344 if (value == null
Rumou Duan45e8e572016-06-17 16:43:44 +0000345 || actionValueIsDirtyWithDirectSystemCalls(
346 value, knownModifiedOutputFiles, sortedKnownModifiedOutputFiles)) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100347 dirtyKeys.add(keyAndValue.getFirst());
348 }
349 }
350 }
351 };
352 }
353
354 /**
Miguel Alcon Pinto7cf23652015-03-10 21:27:48 +0000355 * Returns the number of modified output files inside of dirty actions.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100356 */
357 int getNumberOfModifiedOutputFiles() {
358 return modifiedOutputFilesCounter.get();
359 }
360
Janak Ramakrishnan6ddbb6e2015-07-28 21:39:22 +0000361 /** Returns the number of modified output files that occur during the previous build. */
362 int getNumberOfModifiedOutputFilesDuringPreviousBuild() {
Miguel Alcon Pinto7cf23652015-03-10 21:27:48 +0000363 return modifiedOutputFilesIntraBuildCounter.get();
364 }
365
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000366 private boolean treeArtifactIsDirty(Artifact artifact, TreeArtifactValue value) {
367 if (artifact.getPath().isSymbolicLink()) {
368 // TreeArtifacts may not be symbolic links.
369 return true;
370 }
371
372 // There doesn't appear to be any facility to batch list directories... we must
373 // do things the 'slow' way.
374 try {
375 Set<PathFragment> currentDirectoryValue = TreeArtifactValue.explodeDirectory(artifact);
376 Set<PathFragment> valuePaths = value.getChildPaths();
377 return !currentDirectoryValue.equals(valuePaths);
Rumou Duan9ad28cd2016-10-19 19:28:06 +0000378 } catch (IOException e) {
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000379 return true;
380 }
381 }
382
Eric Fellheimere6590722015-11-17 17:07:48 +0000383 private boolean actionValueIsDirtyWithDirectSystemCalls(ActionExecutionValue actionValue,
Rumou Duan45e8e572016-06-17 16:43:44 +0000384 ImmutableSet<PathFragment> knownModifiedOutputFiles,
385 Supplier<NavigableSet<PathFragment>> sortedKnownModifiedOutputFiles) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100386 boolean isDirty = false;
Rumou Duana77f32c2016-04-13 21:59:21 +0000387 for (Map.Entry<Artifact, FileValue> entry : actionValue.getAllFileValues().entrySet()) {
388 Artifact file = entry.getKey();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100389 FileValue lastKnownData = entry.getValue();
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000390 if (shouldCheckFile(knownModifiedOutputFiles, file)) {
Eric Fellheimere6590722015-11-17 17:07:48 +0000391 try {
Rumou Duana77f32c2016-04-13 21:59:21 +0000392 FileValue fileValue = ActionMetadataHandler.fileValueFromArtifact(file, null,
393 tsgm);
Eric Fellheimere6590722015-11-17 17:07:48 +0000394 if (!fileValue.equals(lastKnownData)) {
395 updateIntraBuildModifiedCounter(fileValue.exists()
396 ? fileValue.realRootedPath().asPath().getLastModifiedTime()
397 : -1, lastKnownData.isSymlink(), fileValue.isSymlink());
398 modifiedOutputFilesCounter.getAndIncrement();
399 isDirty = true;
400 }
401 } catch (IOException e) {
402 // This is an unexpected failure getting a digest or symlink target.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100403 modifiedOutputFilesCounter.getAndIncrement();
404 isDirty = true;
405 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100406 }
407 }
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000408
409 for (Map.Entry<Artifact, TreeArtifactValue> entry :
410 actionValue.getAllTreeArtifactValues().entrySet()) {
411 Artifact artifact = entry.getKey();
Rumou Duan45e8e572016-06-17 16:43:44 +0000412
413 if (shouldCheckTreeArtifact(sortedKnownModifiedOutputFiles.get(), artifact)
414 && treeArtifactIsDirty(artifact, entry.getValue())) {
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000415 Path path = artifact.getPath();
416 // Count the changed directory as one "file".
417 try {
418 updateIntraBuildModifiedCounter(path.exists()
419 ? path.getLastModifiedTime()
420 : -1, false, path.isSymbolicLink());
421 } catch (IOException e) {
422 // Do nothing here.
423 }
424
425 modifiedOutputFilesCounter.getAndIncrement();
426 isDirty = true;
427 }
428 }
429
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100430 return isDirty;
431 }
432
Michael Thvedte4a7b0792016-02-09 12:15:53 +0000433 private static boolean shouldCheckFile(ImmutableSet<PathFragment> knownModifiedOutputFiles,
Rumou Duana77f32c2016-04-13 21:59:21 +0000434 Artifact artifact) {
Eric Fellheimere6590722015-11-17 17:07:48 +0000435 return knownModifiedOutputFiles == null
Rumou Duana77f32c2016-04-13 21:59:21 +0000436 || knownModifiedOutputFiles.contains(artifact.getExecPath());
Eric Fellheimere6590722015-11-17 17:07:48 +0000437 }
438
Rumou Duan45e8e572016-06-17 16:43:44 +0000439 private static boolean shouldCheckTreeArtifact(
440 @Nullable NavigableSet<PathFragment> knownModifiedOutputFiles, Artifact treeArtifact) {
441 // If null, everything needs to be checked.
442 if (knownModifiedOutputFiles == null) {
443 return true;
444 }
445
446 // Here we do the following to see whether a TreeArtifact is modified:
447 // 1. Sort the set of modified file paths in lexicographical order using TreeSet.
448 // 2. Get the first modified output file path that is greater than or equal to the exec path of
449 // the TreeArtifact to check.
450 // 3. Check whether the returned file path contains the exec path of the TreeArtifact as a
451 // prefix path.
452 PathFragment artifactExecPath = treeArtifact.getExecPath();
453 PathFragment headPath = knownModifiedOutputFiles.ceiling(artifactExecPath);
454
455 return headPath != null && headPath.startsWith(artifactExecPath);
456 }
457
Nathan Harmata8cd29782015-11-10 03:24:01 +0000458 private BatchDirtyResult getDirtyValues(ValueFetcher fetcher,
459 Iterable<SkyKey> keys, final SkyValueDirtinessChecker checker,
Nathan Harmata2ff0a6d2015-08-12 21:10:56 +0000460 final boolean checkMissingValues) throws InterruptedException {
Janak Ramakrishnan6ddbb6e2015-07-28 21:39:22 +0000461 ExecutorService executor =
462 Executors.newFixedThreadPool(
463 DIRTINESS_CHECK_THREADS,
464 new ThreadFactoryBuilder().setNameFormat("FileSystem Value Invalidator %d").build());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100465
466 final BatchDirtyResult batchResult = new BatchDirtyResult();
467 ThrowableRecordingRunnableWrapper wrapper =
468 new ThrowableRecordingRunnableWrapper("FilesystemValueChecker#getDirtyValues");
Nathan Harmatae1b61d02015-10-06 00:09:42 +0000469 final AtomicInteger numKeysScanned = new AtomicInteger(0);
470 final AtomicInteger numKeysChecked = new AtomicInteger(0);
471 ElapsedTimeReceiver elapsedTimeReceiver = new ElapsedTimeReceiver() {
472 @Override
473 public void accept(long elapsedTimeNanos) {
474 if (elapsedTimeNanos > 0) {
475 LOG.info(String.format("Spent %d ms checking %d filesystem nodes (%d scanned)",
476 TimeUnit.MILLISECONDS.convert(elapsedTimeNanos, TimeUnit.NANOSECONDS),
477 numKeysChecked.get(),
478 numKeysScanned.get()));
479 }
480 }
481 };
482 try (AutoProfiler prof = AutoProfiler.create(elapsedTimeReceiver)) {
Nathan Harmata8cd29782015-11-10 03:24:01 +0000483 for (final SkyKey key : keys) {
Nathan Harmata3a509bd2015-10-06 01:00:47 +0000484 numKeysScanned.incrementAndGet();
485 if (!checker.applies(key)) {
486 continue;
487 }
Nathan Harmata8cd29782015-11-10 03:24:01 +0000488 final SkyValue value = fetcher.get(key);
489 if (!checkMissingValues && value == null) {
490 continue;
491 }
Nathan Harmatae1b61d02015-10-06 00:09:42 +0000492 executor.execute(
493 wrapper.wrap(
494 new Runnable() {
495 @Override
496 public void run() {
Nathan Harmata8cd29782015-11-10 03:24:01 +0000497 numKeysChecked.incrementAndGet();
498 DirtyResult result = checker.check(key, value, tsgm);
499 if (result.isDirty()) {
500 batchResult.add(key, value, result.getNewValue());
Janak Ramakrishnan6ddbb6e2015-07-28 21:39:22 +0000501 }
502 }
Nathan Harmatae1b61d02015-10-06 00:09:42 +0000503 }));
504 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100505
Nathan Harmatae1b61d02015-10-06 00:09:42 +0000506 boolean interrupted = ExecutorUtil.interruptibleShutdown(executor);
507 Throwables.propagateIfPossible(wrapper.getFirstThrownError());
508 if (interrupted) {
509 throw new InterruptedException();
510 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100511 }
512 return batchResult;
513 }
514
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100515 /**
Nathan Harmata8cd29782015-11-10 03:24:01 +0000516 * Result of a batch call to {@link SkyValueDirtinessChecker#check}. Partitions the dirty
Janak Ramakrishnan6ddbb6e2015-07-28 21:39:22 +0000517 * values based on whether we have a new value available for them or not.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100518 */
Nathan Harmata2ff0a6d2015-08-12 21:10:56 +0000519 private static class BatchDirtyResult implements Differencer.DiffWithDelta {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100520
521 private final Set<SkyKey> concurrentDirtyKeysWithoutNewValues =
522 Collections.newSetFromMap(new ConcurrentHashMap<SkyKey, Boolean>());
Nathan Harmata2ff0a6d2015-08-12 21:10:56 +0000523 private final ConcurrentHashMap<SkyKey, Delta> concurrentDirtyKeysWithNewAndOldValues =
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100524 new ConcurrentHashMap<>();
525
Nathan Harmata2ff0a6d2015-08-12 21:10:56 +0000526 private void add(SkyKey key, @Nullable SkyValue oldValue, @Nullable SkyValue newValue) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100527 if (newValue == null) {
528 concurrentDirtyKeysWithoutNewValues.add(key);
529 } else {
Nathan Harmata2ff0a6d2015-08-12 21:10:56 +0000530 if (oldValue == null) {
531 concurrentDirtyKeysWithNewAndOldValues.put(key, new Delta(newValue));
532 } else {
533 concurrentDirtyKeysWithNewAndOldValues.put(key, new Delta(oldValue, newValue));
534 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100535 }
536 }
537
538 @Override
Michajlo Matijkiw2b71efe2015-06-19 19:23:16 +0000539 public Collection<SkyKey> changedKeysWithoutNewValues() {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100540 return concurrentDirtyKeysWithoutNewValues;
541 }
542
543 @Override
Nathan Harmata2ff0a6d2015-08-12 21:10:56 +0000544 public Map<SkyKey, Delta> changedKeysWithNewAndOldValues() {
545 return concurrentDirtyKeysWithNewAndOldValues;
546 }
547
548 @Override
549 public Map<SkyKey, SkyValue> changedKeysWithNewValues() {
550 return Delta.newValues(concurrentDirtyKeysWithNewAndOldValues);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100551 }
552 }
553
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100554}