Damien Martin-Guillerez | f88f4d8 | 2015-09-25 13:56:55 +0000 | [diff] [blame] | 1 | // Copyright 2014 The Bazel Authors. All rights reserved. |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | package com.google.devtools.build.lib.skyframe; |
| 15 | |
tomlu | a155b53 | 2017-11-08 20:12:47 +0100 | [diff] [blame] | 16 | import com.google.common.base.Preconditions; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 17 | import com.google.common.base.Predicate; |
Rumou Duan | 45e8e57 | 2016-06-17 16:43:44 +0000 | [diff] [blame] | 18 | import com.google.common.base.Supplier; |
| 19 | import com.google.common.base.Suppliers; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 20 | import com.google.common.base.Throwables; |
| 21 | import com.google.common.collect.ImmutableList; |
Eric Fellheimer | e659072 | 2015-11-17 17:07:48 +0000 | [diff] [blame] | 22 | import com.google.common.collect.ImmutableSet; |
Rumou Duan | 45e8e57 | 2016-06-17 16:43:44 +0000 | [diff] [blame] | 23 | import com.google.common.collect.ImmutableSortedSet; |
Miguel Alcon Pinto | 7cf2365 | 2015-03-10 21:27:48 +0000 | [diff] [blame] | 24 | import com.google.common.collect.Range; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 25 | import com.google.common.collect.Sets; |
janakr | c3bcb98 | 2020-04-14 06:50:08 -0700 | [diff] [blame] | 26 | import com.google.common.flogger.GoogleLogger; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 27 | import com.google.common.util.concurrent.ThreadFactoryBuilder; |
| 28 | import com.google.devtools.build.lib.actions.Artifact; |
Googler | 974879d | 2020-05-27 13:25:52 -0700 | [diff] [blame] | 29 | import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact; |
buchgr | 4992ae2 | 2019-03-20 04:23:32 -0700 | [diff] [blame] | 30 | import com.google.devtools.build.lib.actions.FileArtifactValue; |
lberki | f7eee1e | 2019-07-31 05:49:10 -0700 | [diff] [blame] | 31 | import com.google.devtools.build.lib.actions.FileStateType; |
Eric Fellheimer | 6a9d7e5 | 2015-06-18 22:08:32 +0000 | [diff] [blame] | 32 | import com.google.devtools.build.lib.concurrent.ExecutorUtil; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 33 | import com.google.devtools.build.lib.concurrent.Sharder; |
| 34 | import com.google.devtools.build.lib.concurrent.ThrowableRecordingRunnableWrapper; |
Nathan Harmata | e1b61d0 | 2015-10-06 00:09:42 +0000 | [diff] [blame] | 35 | import com.google.devtools.build.lib.profiler.AutoProfiler; |
| 36 | import com.google.devtools.build.lib.profiler.AutoProfiler.ElapsedTimeReceiver; |
twerth | 646dfd1 | 2018-07-04 01:58:40 -0700 | [diff] [blame] | 37 | import com.google.devtools.build.lib.profiler.Profiler; |
| 38 | import com.google.devtools.build.lib.profiler.SilentCloseable; |
Janak Ramakrishnan | 6ddbb6e | 2015-07-28 21:39:22 +0000 | [diff] [blame] | 39 | import com.google.devtools.build.lib.skyframe.SkyValueDirtinessChecker.DirtyResult; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 40 | import com.google.devtools.build.lib.util.LoggingUtil; |
| 41 | import com.google.devtools.build.lib.util.Pair; |
| 42 | import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; |
| 43 | import com.google.devtools.build.lib.vfs.BatchStat; |
| 44 | import com.google.devtools.build.lib.vfs.FileStatusWithDigest; |
Eric Fellheimer | e659072 | 2015-11-17 17:07:48 +0000 | [diff] [blame] | 45 | import com.google.devtools.build.lib.vfs.ModifiedFileSet; |
Michael Thvedt | e4a7b079 | 2016-02-09 12:15:53 +0000 | [diff] [blame] | 46 | import com.google.devtools.build.lib.vfs.Path; |
Eric Fellheimer | e659072 | 2015-11-17 17:07:48 +0000 | [diff] [blame] | 47 | import com.google.devtools.build.lib.vfs.PathFragment; |
janakr | 0c42fc8 | 2018-09-14 10:37:25 -0700 | [diff] [blame] | 48 | import com.google.devtools.build.lib.vfs.Symlinks; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 49 | import com.google.devtools.build.skyframe.Differencer; |
nharmata | acccd42 | 2020-01-27 13:38:23 -0800 | [diff] [blame] | 50 | import com.google.devtools.build.skyframe.Differencer.DiffWithDelta.Delta; |
janakr | e54491e | 2018-07-11 16:29:13 -0700 | [diff] [blame] | 51 | import com.google.devtools.build.skyframe.FunctionHermeticity; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 52 | import com.google.devtools.build.skyframe.SkyFunctionName; |
| 53 | import com.google.devtools.build.skyframe.SkyKey; |
| 54 | import com.google.devtools.build.skyframe.SkyValue; |
Nathan Harmata | 8cd2978 | 2015-11-10 03:24:01 +0000 | [diff] [blame] | 55 | import com.google.devtools.build.skyframe.WalkableGraph; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 56 | import java.io.IOException; |
| 57 | import java.util.Collection; |
| 58 | import java.util.Collections; |
| 59 | import java.util.HashMap; |
Nathan Harmata | 8cd2978 | 2015-11-10 03:24:01 +0000 | [diff] [blame] | 60 | import java.util.HashSet; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 61 | import java.util.List; |
| 62 | import java.util.Map; |
Rumou Duan | 45e8e57 | 2016-06-17 16:43:44 +0000 | [diff] [blame] | 63 | import java.util.NavigableSet; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 64 | import java.util.Set; |
| 65 | import java.util.concurrent.ConcurrentHashMap; |
| 66 | import java.util.concurrent.ExecutorService; |
| 67 | import java.util.concurrent.Executors; |
cushon | 4d70fae | 2017-04-11 01:01:13 +0000 | [diff] [blame] | 68 | import java.util.concurrent.Future; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 69 | import java.util.concurrent.atomic.AtomicInteger; |
| 70 | import java.util.logging.Level; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 71 | import javax.annotation.Nullable; |
| 72 | |
| 73 | /** |
| 74 | * A helper class to find dirty values by accessing the filesystem directly (contrast with |
| 75 | * {@link DiffAwareness}). |
| 76 | */ |
Nathan Harmata | 8cd2978 | 2015-11-10 03:24:01 +0000 | [diff] [blame] | 77 | public class FilesystemValueChecker { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 78 | |
janakr | c3bcb98 | 2020-04-14 06:50:08 -0700 | [diff] [blame] | 79 | private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 80 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 81 | private static final Predicate<SkyKey> ACTION_FILTER = |
| 82 | SkyFunctionName.functionIs(SkyFunctions.ACTION_EXECUTION); |
| 83 | |
buchgr | 4992ae2 | 2019-03-20 04:23:32 -0700 | [diff] [blame] | 84 | @Nullable private final TimestampGranularityMonitor tsgm; |
twerth | 5aaceb5 | 2020-04-07 06:31:56 -0700 | [diff] [blame] | 85 | @Nullable private final Range<Long> lastExecutionTimeRange; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 86 | private AtomicInteger modifiedOutputFilesCounter = new AtomicInteger(0); |
Miguel Alcon Pinto | 7cf2365 | 2015-03-10 21:27:48 +0000 | [diff] [blame] | 87 | private AtomicInteger modifiedOutputFilesIntraBuildCounter = new AtomicInteger(0); |
twerth | 5aaceb5 | 2020-04-07 06:31:56 -0700 | [diff] [blame] | 88 | private final int numThreads; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 89 | |
steinman | 39c00d2 | 2020-03-20 15:23:10 -0700 | [diff] [blame] | 90 | public FilesystemValueChecker( |
twerth | 5aaceb5 | 2020-04-07 06:31:56 -0700 | [diff] [blame] | 91 | @Nullable TimestampGranularityMonitor tsgm, |
| 92 | @Nullable Range<Long> lastExecutionTimeRange, |
| 93 | int numThreads) { |
Nathan Harmata | 2ff0a6d | 2015-08-12 21:10:56 +0000 | [diff] [blame] | 94 | this.tsgm = tsgm; |
| 95 | this.lastExecutionTimeRange = lastExecutionTimeRange; |
twerth | 5aaceb5 | 2020-04-07 06:31:56 -0700 | [diff] [blame] | 96 | this.numThreads = numThreads; |
Nathan Harmata | 2ff0a6d | 2015-08-12 21:10:56 +0000 | [diff] [blame] | 97 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 98 | /** |
Nathan Harmata | 8cd2978 | 2015-11-10 03:24:01 +0000 | [diff] [blame] | 99 | * Returns a {@link Differencer.DiffWithDelta} containing keys from the give map that are dirty |
| 100 | * according to the passed-in {@code dirtinessChecker}. |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 101 | */ |
Nathan Harmata | 8cd2978 | 2015-11-10 03:24:01 +0000 | [diff] [blame] | 102 | // TODO(bazel-team): Refactor these methods so that FilesystemValueChecker only operates on a |
| 103 | // WalkableGraph. |
nharmata | acccd42 | 2020-01-27 13:38:23 -0800 | [diff] [blame] | 104 | ImmutableBatchDirtyResult getDirtyKeys( |
| 105 | Map<SkyKey, SkyValue> valuesMap, SkyValueDirtinessChecker dirtinessChecker) |
| 106 | throws InterruptedException { |
Nathan Harmata | 8cd2978 | 2015-11-10 03:24:01 +0000 | [diff] [blame] | 107 | return getDirtyValues(new MapBackedValueFetcher(valuesMap), valuesMap.keySet(), |
| 108 | dirtinessChecker, /*checkMissingValues=*/false); |
Nathan Harmata | 2ff0a6d | 2015-08-12 21:10:56 +0000 | [diff] [blame] | 109 | } |
| 110 | |
| 111 | /** |
| 112 | * Returns a {@link Differencer.DiffWithDelta} containing keys that are dirty according to the |
| 113 | * passed-in {@code dirtinessChecker}. |
| 114 | */ |
nharmata | acccd42 | 2020-01-27 13:38:23 -0800 | [diff] [blame] | 115 | public ImmutableBatchDirtyResult getNewAndOldValues( |
| 116 | Map<SkyKey, SkyValue> valuesMap, |
Benjamin Peterson | 4024e16 | 2020-03-06 02:28:29 -0800 | [diff] [blame] | 117 | Collection<SkyKey> keys, |
nharmata | acccd42 | 2020-01-27 13:38:23 -0800 | [diff] [blame] | 118 | SkyValueDirtinessChecker dirtinessChecker) |
| 119 | throws InterruptedException { |
Nathan Harmata | 8cd2978 | 2015-11-10 03:24:01 +0000 | [diff] [blame] | 120 | return getDirtyValues(new MapBackedValueFetcher(valuesMap), keys, |
| 121 | dirtinessChecker, /*checkMissingValues=*/true); |
| 122 | } |
| 123 | |
| 124 | /** |
| 125 | * Returns a {@link Differencer.DiffWithDelta} containing keys that are dirty according to the |
| 126 | * passed-in {@code dirtinessChecker}. |
| 127 | */ |
Benjamin Peterson | 4024e16 | 2020-03-06 02:28:29 -0800 | [diff] [blame] | 128 | public Differencer.DiffWithDelta getNewAndOldValues( |
| 129 | WalkableGraph walkableGraph, |
| 130 | Collection<SkyKey> keys, |
| 131 | SkyValueDirtinessChecker dirtinessChecker) |
| 132 | throws InterruptedException { |
Nathan Harmata | 8cd2978 | 2015-11-10 03:24:01 +0000 | [diff] [blame] | 133 | return getDirtyValues(new WalkableGraphBackedValueFetcher(walkableGraph), keys, |
| 134 | dirtinessChecker, /*checkMissingValues=*/true); |
| 135 | } |
| 136 | |
Janak Ramakrishnan | 3c0adb2 | 2016-08-15 21:54:55 +0000 | [diff] [blame] | 137 | private interface ValueFetcher { |
Nathan Harmata | 8cd2978 | 2015-11-10 03:24:01 +0000 | [diff] [blame] | 138 | @Nullable |
Janak Ramakrishnan | 3c0adb2 | 2016-08-15 21:54:55 +0000 | [diff] [blame] | 139 | SkyValue get(SkyKey key) throws InterruptedException; |
Nathan Harmata | 8cd2978 | 2015-11-10 03:24:01 +0000 | [diff] [blame] | 140 | } |
| 141 | |
| 142 | private static class WalkableGraphBackedValueFetcher implements ValueFetcher { |
| 143 | private final WalkableGraph walkableGraph; |
| 144 | |
| 145 | private WalkableGraphBackedValueFetcher(WalkableGraph walkableGraph) { |
| 146 | this.walkableGraph = walkableGraph; |
| 147 | } |
| 148 | |
| 149 | @Override |
| 150 | @Nullable |
Janak Ramakrishnan | 3c0adb2 | 2016-08-15 21:54:55 +0000 | [diff] [blame] | 151 | public SkyValue get(SkyKey key) throws InterruptedException { |
Janak Ramakrishnan | 112840b | 2016-12-29 21:49:56 +0000 | [diff] [blame] | 152 | return walkableGraph.getValue(key); |
Nathan Harmata | 8cd2978 | 2015-11-10 03:24:01 +0000 | [diff] [blame] | 153 | } |
| 154 | } |
| 155 | |
| 156 | private static class MapBackedValueFetcher implements ValueFetcher { |
| 157 | private final Map<SkyKey, SkyValue> valuesMap; |
| 158 | |
| 159 | private MapBackedValueFetcher(Map<SkyKey, SkyValue> valuesMap) { |
| 160 | this.valuesMap = valuesMap; |
| 161 | } |
| 162 | |
| 163 | @Override |
| 164 | @Nullable |
| 165 | public SkyValue get(SkyKey key) { |
| 166 | return valuesMap.get(key); |
| 167 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 168 | } |
| 169 | |
| 170 | /** |
steinman | 39c00d2 | 2020-03-20 15:23:10 -0700 | [diff] [blame] | 171 | * Return a collection of action values which have output files that are not in-sync with the |
| 172 | * on-disk file value (were modified externally). |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 173 | */ |
steinman | 39c00d2 | 2020-03-20 15:23:10 -0700 | [diff] [blame] | 174 | Collection<SkyKey> getDirtyActionValues( |
| 175 | Map<SkyKey, SkyValue> valuesMap, |
| 176 | @Nullable final BatchStat batchStatter, |
| 177 | ModifiedFileSet modifiedOutputFiles, |
| 178 | boolean trustRemoteArtifacts) |
| 179 | throws InterruptedException { |
Eric Fellheimer | e659072 | 2015-11-17 17:07:48 +0000 | [diff] [blame] | 180 | if (modifiedOutputFiles == ModifiedFileSet.NOTHING_MODIFIED) { |
janakr | c3bcb98 | 2020-04-14 06:50:08 -0700 | [diff] [blame] | 181 | logger.atInfo().log("Not checking for dirty actions since nothing was modified"); |
Eric Fellheimer | e659072 | 2015-11-17 17:07:48 +0000 | [diff] [blame] | 182 | return ImmutableList.of(); |
| 183 | } |
janakr | c3bcb98 | 2020-04-14 06:50:08 -0700 | [diff] [blame] | 184 | logger.atInfo().log("Accumulating dirty actions"); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 185 | final int numOutputJobs = Runtime.getRuntime().availableProcessors() * 4; |
Nathan Harmata | 8cd2978 | 2015-11-10 03:24:01 +0000 | [diff] [blame] | 186 | final Set<SkyKey> actionSkyKeys = new HashSet<>(); |
twerth | 646dfd1 | 2018-07-04 01:58:40 -0700 | [diff] [blame] | 187 | try (SilentCloseable c = Profiler.instance().profile("getDirtyActionValues.filter_actions")) { |
| 188 | for (SkyKey key : valuesMap.keySet()) { |
| 189 | if (ACTION_FILTER.apply(key)) { |
| 190 | actionSkyKeys.add(key); |
| 191 | } |
Nathan Harmata | 8cd2978 | 2015-11-10 03:24:01 +0000 | [diff] [blame] | 192 | } |
| 193 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 194 | final Sharder<Pair<SkyKey, ActionExecutionValue>> outputShards = |
| 195 | new Sharder<>(numOutputJobs, actionSkyKeys.size()); |
| 196 | |
| 197 | for (SkyKey key : actionSkyKeys) { |
Nathan Harmata | 8cd2978 | 2015-11-10 03:24:01 +0000 | [diff] [blame] | 198 | outputShards.add(Pair.of(key, (ActionExecutionValue) valuesMap.get(key))); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 199 | } |
janakr | c3bcb98 | 2020-04-14 06:50:08 -0700 | [diff] [blame] | 200 | logger.atInfo().log("Sharded action values for batching"); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 201 | |
| 202 | ExecutorService executor = Executors.newFixedThreadPool( |
| 203 | numOutputJobs, |
| 204 | new ThreadFactoryBuilder().setNameFormat("FileSystem Output File Invalidator %d").build()); |
| 205 | |
| 206 | Collection<SkyKey> dirtyKeys = Sets.newConcurrentHashSet(); |
| 207 | ThrowableRecordingRunnableWrapper wrapper = |
| 208 | new ThrowableRecordingRunnableWrapper("FileSystemValueChecker#getDirtyActionValues"); |
| 209 | |
| 210 | modifiedOutputFilesCounter.set(0); |
Miguel Alcon Pinto | 7cf2365 | 2015-03-10 21:27:48 +0000 | [diff] [blame] | 211 | modifiedOutputFilesIntraBuildCounter.set(0); |
Rumou Duan | 45e8e57 | 2016-06-17 16:43:44 +0000 | [diff] [blame] | 212 | final ImmutableSet<PathFragment> knownModifiedOutputFiles = |
Eric Fellheimer | e659072 | 2015-11-17 17:07:48 +0000 | [diff] [blame] | 213 | modifiedOutputFiles == ModifiedFileSet.EVERYTHING_MODIFIED |
| 214 | ? null |
| 215 | : modifiedOutputFiles.modifiedSourceFiles(); |
Rumou Duan | 45e8e57 | 2016-06-17 16:43:44 +0000 | [diff] [blame] | 216 | |
| 217 | // Initialized lazily through a supplier because it is only used to check modified |
| 218 | // TreeArtifacts, which are not frequently used in builds. |
| 219 | Supplier<NavigableSet<PathFragment>> sortedKnownModifiedOutputFiles = |
| 220 | Suppliers.memoize(new Supplier<NavigableSet<PathFragment>>() { |
| 221 | @Override |
| 222 | public NavigableSet<PathFragment> get() { |
| 223 | if (knownModifiedOutputFiles == null) { |
| 224 | return null; |
| 225 | } else { |
| 226 | return ImmutableSortedSet.copyOf(knownModifiedOutputFiles); |
| 227 | } |
| 228 | } |
| 229 | }); |
| 230 | |
twerth | 646dfd1 | 2018-07-04 01:58:40 -0700 | [diff] [blame] | 231 | boolean interrupted; |
| 232 | try (SilentCloseable c = Profiler.instance().profile("getDirtyActionValues.stat_files")) { |
| 233 | for (List<Pair<SkyKey, ActionExecutionValue>> shard : outputShards) { |
| 234 | Runnable job = |
| 235 | (batchStatter == null) |
| 236 | ? outputStatJob( |
steinman | 39c00d2 | 2020-03-20 15:23:10 -0700 | [diff] [blame] | 237 | dirtyKeys, |
| 238 | shard, |
| 239 | knownModifiedOutputFiles, |
| 240 | sortedKnownModifiedOutputFiles, |
| 241 | trustRemoteArtifacts) |
twerth | 646dfd1 | 2018-07-04 01:58:40 -0700 | [diff] [blame] | 242 | : batchStatJob( |
| 243 | dirtyKeys, |
| 244 | shard, |
| 245 | batchStatter, |
| 246 | knownModifiedOutputFiles, |
steinman | 39c00d2 | 2020-03-20 15:23:10 -0700 | [diff] [blame] | 247 | sortedKnownModifiedOutputFiles, |
| 248 | trustRemoteArtifacts); |
twerth | 646dfd1 | 2018-07-04 01:58:40 -0700 | [diff] [blame] | 249 | Future<?> unused = executor.submit(wrapper.wrap(job)); |
| 250 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 251 | |
twerth | 646dfd1 | 2018-07-04 01:58:40 -0700 | [diff] [blame] | 252 | interrupted = ExecutorUtil.interruptibleShutdown(executor); |
| 253 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 254 | Throwables.propagateIfPossible(wrapper.getFirstThrownError()); |
janakr | c3bcb98 | 2020-04-14 06:50:08 -0700 | [diff] [blame] | 255 | logger.atInfo().log("Completed output file stat checks"); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 256 | if (interrupted) { |
| 257 | throw new InterruptedException(); |
| 258 | } |
| 259 | return dirtyKeys; |
| 260 | } |
| 261 | |
steinman | 39c00d2 | 2020-03-20 15:23:10 -0700 | [diff] [blame] | 262 | private Runnable batchStatJob( |
| 263 | final Collection<SkyKey> dirtyKeys, |
| 264 | final List<Pair<SkyKey, ActionExecutionValue>> shard, |
| 265 | final BatchStat batchStatter, |
| 266 | final ImmutableSet<PathFragment> knownModifiedOutputFiles, |
| 267 | final Supplier<NavigableSet<PathFragment>> sortedKnownModifiedOutputFiles, |
| 268 | boolean trustRemoteArtifacts) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 269 | return new Runnable() { |
| 270 | @Override |
| 271 | public void run() { |
Rumou Duan | a77f32c | 2016-04-13 21:59:21 +0000 | [diff] [blame] | 272 | Map<Artifact, Pair<SkyKey, ActionExecutionValue>> fileToKeyAndValue = new HashMap<>(); |
Michael Thvedt | e4a7b079 | 2016-02-09 12:15:53 +0000 | [diff] [blame] | 273 | Map<Artifact, Pair<SkyKey, ActionExecutionValue>> treeArtifactsToKeyAndValue = |
| 274 | new HashMap<>(); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 275 | for (Pair<SkyKey, ActionExecutionValue> keyAndValue : shard) { |
| 276 | ActionExecutionValue actionValue = keyAndValue.getSecond(); |
| 277 | if (actionValue == null) { |
| 278 | dirtyKeys.add(keyAndValue.getFirst()); |
| 279 | } else { |
Rumou Duan | a77f32c | 2016-04-13 21:59:21 +0000 | [diff] [blame] | 280 | for (Artifact artifact : actionValue.getAllFileValues().keySet()) { |
lberki | c63bafa | 2019-08-01 10:23:49 -0700 | [diff] [blame] | 281 | if (!artifact.isMiddlemanArtifact() |
| 282 | && shouldCheckFile(knownModifiedOutputFiles, artifact)) { |
Rumou Duan | a77f32c | 2016-04-13 21:59:21 +0000 | [diff] [blame] | 283 | fileToKeyAndValue.put(artifact, keyAndValue); |
Eric Fellheimer | e659072 | 2015-11-17 17:07:48 +0000 | [diff] [blame] | 284 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 285 | } |
Michael Thvedt | e4a7b079 | 2016-02-09 12:15:53 +0000 | [diff] [blame] | 286 | |
Googler | 974879d | 2020-05-27 13:25:52 -0700 | [diff] [blame] | 287 | for (Map.Entry<Artifact, TreeArtifactValue> entry : |
| 288 | actionValue.getAllTreeArtifactValues().entrySet()) { |
| 289 | Artifact treeArtifact = entry.getKey(); |
| 290 | TreeArtifactValue tree = entry.getValue(); |
| 291 | for (TreeFileArtifact child : tree.getChildren()) { |
| 292 | if (shouldCheckFile(knownModifiedOutputFiles, child)) { |
| 293 | fileToKeyAndValue.put(child, keyAndValue); |
| 294 | } |
| 295 | } |
| 296 | if (shouldCheckTreeArtifact(sortedKnownModifiedOutputFiles.get(), treeArtifact)) { |
| 297 | treeArtifactsToKeyAndValue.put(treeArtifact, keyAndValue); |
Rumou Duan | 45e8e57 | 2016-06-17 16:43:44 +0000 | [diff] [blame] | 298 | } |
Michael Thvedt | e4a7b079 | 2016-02-09 12:15:53 +0000 | [diff] [blame] | 299 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 300 | } |
| 301 | } |
| 302 | |
Rumou Duan | a77f32c | 2016-04-13 21:59:21 +0000 | [diff] [blame] | 303 | List<Artifact> artifacts = ImmutableList.copyOf(fileToKeyAndValue.keySet()); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 304 | List<FileStatusWithDigest> stats; |
| 305 | try { |
lberki | 97abb52 | 2017-09-04 18:51:57 +0200 | [diff] [blame] | 306 | stats = |
| 307 | batchStatter.batchStat( |
| 308 | /*includeDigest=*/ true, |
| 309 | /*includeLinks=*/ true, |
| 310 | Artifact.asPathFragments(artifacts)); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 311 | } catch (IOException e) { |
| 312 | // Batch stat did not work. Log an exception and fall back on system calls. |
| 313 | LoggingUtil.logToRemote(Level.WARNING, "Unable to process batch stat", e); |
janakr | c3bcb98 | 2020-04-14 06:50:08 -0700 | [diff] [blame] | 314 | logger.atWarning().withCause(e).log("Unable to process batch stat"); |
steinman | 39c00d2 | 2020-03-20 15:23:10 -0700 | [diff] [blame] | 315 | outputStatJob( |
| 316 | dirtyKeys, |
| 317 | shard, |
| 318 | knownModifiedOutputFiles, |
| 319 | sortedKnownModifiedOutputFiles, |
| 320 | trustRemoteArtifacts) |
Rumou Duan | 45e8e57 | 2016-06-17 16:43:44 +0000 | [diff] [blame] | 321 | .run(); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 322 | return; |
| 323 | } catch (InterruptedException e) { |
| 324 | // We handle interrupt in the main thread. |
| 325 | return; |
| 326 | } |
| 327 | |
lberki | 97abb52 | 2017-09-04 18:51:57 +0200 | [diff] [blame] | 328 | Preconditions.checkState( |
| 329 | artifacts.size() == stats.size(), |
| 330 | "artifacts.size() == %s stats.size() == %s", |
| 331 | artifacts.size(), |
| 332 | stats.size()); |
Rumou Duan | a77f32c | 2016-04-13 21:59:21 +0000 | [diff] [blame] | 333 | for (int i = 0; i < artifacts.size(); i++) { |
| 334 | Artifact artifact = artifacts.get(i); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 335 | FileStatusWithDigest stat = stats.get(i); |
Rumou Duan | a77f32c | 2016-04-13 21:59:21 +0000 | [diff] [blame] | 336 | Pair<SkyKey, ActionExecutionValue> keyAndValue = fileToKeyAndValue.get(artifact); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 337 | ActionExecutionValue actionValue = keyAndValue.getSecond(); |
| 338 | SkyKey key = keyAndValue.getFirst(); |
Googler | aed4160 | 2020-06-02 12:22:53 -0700 | [diff] [blame^] | 339 | FileArtifactValue lastKnownData = actionValue.getExistingFileArtifactValue(artifact); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 340 | try { |
lberki | f7eee1e | 2019-07-31 05:49:10 -0700 | [diff] [blame] | 341 | FileArtifactValue newData = |
| 342 | ActionMetadataHandler.fileArtifactValueFromArtifact(artifact, stat, tsgm); |
lberki | c35878a | 2019-08-01 02:28:54 -0700 | [diff] [blame] | 343 | if (newData.couldBeModifiedSince(lastKnownData)) { |
lberki | 299e3f0 | 2019-07-31 04:49:21 -0700 | [diff] [blame] | 344 | updateIntraBuildModifiedCounter(stat != null ? stat.getLastChangeTime() : -1); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 345 | modifiedOutputFilesCounter.getAndIncrement(); |
| 346 | dirtyKeys.add(key); |
| 347 | } |
| 348 | } catch (IOException e) { |
| 349 | // This is an unexpected failure getting a digest or symlink target. |
| 350 | modifiedOutputFilesCounter.getAndIncrement(); |
| 351 | dirtyKeys.add(key); |
| 352 | } |
| 353 | } |
Michael Thvedt | e4a7b079 | 2016-02-09 12:15:53 +0000 | [diff] [blame] | 354 | |
| 355 | // Unfortunately, there exists no facility to batch list directories. |
| 356 | // We must use direct filesystem calls. |
| 357 | for (Map.Entry<Artifact, Pair<SkyKey, ActionExecutionValue>> entry : |
| 358 | treeArtifactsToKeyAndValue.entrySet()) { |
| 359 | Artifact artifact = entry.getKey(); |
| 360 | if (treeArtifactIsDirty( |
| 361 | entry.getKey(), entry.getValue().getSecond().getTreeArtifactValue(artifact))) { |
| 362 | Path path = artifact.getPath(); |
| 363 | // Count the changed directory as one "file". |
| 364 | // TODO(bazel-team): There are no tests for this codepath. |
| 365 | try { |
lberki | 299e3f0 | 2019-07-31 04:49:21 -0700 | [diff] [blame] | 366 | updateIntraBuildModifiedCounter(path.exists() ? path.getLastModifiedTime() : -1); |
Michael Thvedt | e4a7b079 | 2016-02-09 12:15:53 +0000 | [diff] [blame] | 367 | } catch (IOException e) { |
| 368 | // Do nothing here. |
| 369 | } |
| 370 | |
| 371 | modifiedOutputFilesCounter.getAndIncrement(); |
| 372 | dirtyKeys.add(entry.getValue().getFirst()); |
| 373 | } |
| 374 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 375 | } |
| 376 | }; |
| 377 | } |
| 378 | |
lberki | 299e3f0 | 2019-07-31 04:49:21 -0700 | [diff] [blame] | 379 | private void updateIntraBuildModifiedCounter(long time) { |
| 380 | if (lastExecutionTimeRange != null && lastExecutionTimeRange.contains(time)) { |
Miguel Alcon Pinto | 7cf2365 | 2015-03-10 21:27:48 +0000 | [diff] [blame] | 381 | modifiedOutputFilesIntraBuildCounter.incrementAndGet(); |
| 382 | } |
| 383 | } |
| 384 | |
steinman | 39c00d2 | 2020-03-20 15:23:10 -0700 | [diff] [blame] | 385 | private Runnable outputStatJob( |
| 386 | final Collection<SkyKey> dirtyKeys, |
Michael Thvedt | e4a7b079 | 2016-02-09 12:15:53 +0000 | [diff] [blame] | 387 | final List<Pair<SkyKey, ActionExecutionValue>> shard, |
Rumou Duan | 45e8e57 | 2016-06-17 16:43:44 +0000 | [diff] [blame] | 388 | final ImmutableSet<PathFragment> knownModifiedOutputFiles, |
steinman | 39c00d2 | 2020-03-20 15:23:10 -0700 | [diff] [blame] | 389 | final Supplier<NavigableSet<PathFragment>> sortedKnownModifiedOutputFiles, |
| 390 | boolean trustRemoteArtifacts) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 391 | return new Runnable() { |
| 392 | @Override |
| 393 | public void run() { |
| 394 | for (Pair<SkyKey, ActionExecutionValue> keyAndValue : shard) { |
| 395 | ActionExecutionValue value = keyAndValue.getSecond(); |
Eric Fellheimer | e659072 | 2015-11-17 17:07:48 +0000 | [diff] [blame] | 396 | if (value == null |
Rumou Duan | 45e8e57 | 2016-06-17 16:43:44 +0000 | [diff] [blame] | 397 | || actionValueIsDirtyWithDirectSystemCalls( |
steinman | 39c00d2 | 2020-03-20 15:23:10 -0700 | [diff] [blame] | 398 | value, |
| 399 | knownModifiedOutputFiles, |
| 400 | sortedKnownModifiedOutputFiles, |
| 401 | trustRemoteArtifacts)) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 402 | dirtyKeys.add(keyAndValue.getFirst()); |
| 403 | } |
| 404 | } |
| 405 | } |
| 406 | }; |
| 407 | } |
| 408 | |
| 409 | /** |
Miguel Alcon Pinto | 7cf2365 | 2015-03-10 21:27:48 +0000 | [diff] [blame] | 410 | * Returns the number of modified output files inside of dirty actions. |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 411 | */ |
| 412 | int getNumberOfModifiedOutputFiles() { |
| 413 | return modifiedOutputFilesCounter.get(); |
| 414 | } |
| 415 | |
Janak Ramakrishnan | 6ddbb6e | 2015-07-28 21:39:22 +0000 | [diff] [blame] | 416 | /** Returns the number of modified output files that occur during the previous build. */ |
| 417 | int getNumberOfModifiedOutputFilesDuringPreviousBuild() { |
Miguel Alcon Pinto | 7cf2365 | 2015-03-10 21:27:48 +0000 | [diff] [blame] | 418 | return modifiedOutputFilesIntraBuildCounter.get(); |
| 419 | } |
| 420 | |
Michael Thvedt | e4a7b079 | 2016-02-09 12:15:53 +0000 | [diff] [blame] | 421 | private boolean treeArtifactIsDirty(Artifact artifact, TreeArtifactValue value) { |
| 422 | if (artifact.getPath().isSymbolicLink()) { |
| 423 | // TreeArtifacts may not be symbolic links. |
| 424 | return true; |
| 425 | } |
| 426 | |
| 427 | // There doesn't appear to be any facility to batch list directories... we must |
| 428 | // do things the 'slow' way. |
| 429 | try { |
felly | 09efb3f | 2018-07-26 07:46:15 -0700 | [diff] [blame] | 430 | Set<PathFragment> currentDirectoryValue = |
| 431 | TreeArtifactValue.explodeDirectory(artifact.getPath()); |
buchgr | 4992ae2 | 2019-03-20 04:23:32 -0700 | [diff] [blame] | 432 | return !(currentDirectoryValue.isEmpty() && value.isRemote()) |
| 433 | && !currentDirectoryValue.equals(value.getChildPaths()); |
Rumou Duan | 9ad28cd | 2016-10-19 19:28:06 +0000 | [diff] [blame] | 434 | } catch (IOException e) { |
Michael Thvedt | e4a7b079 | 2016-02-09 12:15:53 +0000 | [diff] [blame] | 435 | return true; |
| 436 | } |
| 437 | } |
| 438 | |
Googler | 974879d | 2020-05-27 13:25:52 -0700 | [diff] [blame] | 439 | private boolean artifactIsDirtyWithDirectSystemCalls( |
| 440 | ImmutableSet<PathFragment> knownModifiedOutputFiles, |
| 441 | boolean trustRemoteArtifacts, |
| 442 | Map.Entry<? extends Artifact, FileArtifactValue> entry) { |
| 443 | Artifact file = entry.getKey(); |
| 444 | FileArtifactValue lastKnownData = entry.getValue(); |
| 445 | if (file.isMiddlemanArtifact() || !shouldCheckFile(knownModifiedOutputFiles, file)) { |
| 446 | return false; |
| 447 | } |
| 448 | try { |
| 449 | FileArtifactValue fileMetadata = |
| 450 | ActionMetadataHandler.fileArtifactValueFromArtifact(file, null, tsgm); |
| 451 | boolean trustRemoteValue = |
| 452 | fileMetadata.getType() == FileStateType.NONEXISTENT |
| 453 | && lastKnownData.isRemote() |
| 454 | && trustRemoteArtifacts; |
| 455 | if (!trustRemoteValue && fileMetadata.couldBeModifiedSince(lastKnownData)) { |
| 456 | updateIntraBuildModifiedCounter( |
| 457 | fileMetadata.getType() != FileStateType.NONEXISTENT |
| 458 | ? file.getPath().getLastModifiedTime(Symlinks.FOLLOW) |
| 459 | : -1); |
| 460 | modifiedOutputFilesCounter.getAndIncrement(); |
| 461 | return true; |
| 462 | } |
| 463 | return false; |
| 464 | } catch (IOException e) { |
| 465 | // This is an unexpected failure getting a digest or symlink target. |
| 466 | modifiedOutputFilesCounter.getAndIncrement(); |
| 467 | return true; |
| 468 | } |
| 469 | } |
| 470 | |
steinman | 39c00d2 | 2020-03-20 15:23:10 -0700 | [diff] [blame] | 471 | private boolean actionValueIsDirtyWithDirectSystemCalls( |
| 472 | ActionExecutionValue actionValue, |
Rumou Duan | 45e8e57 | 2016-06-17 16:43:44 +0000 | [diff] [blame] | 473 | ImmutableSet<PathFragment> knownModifiedOutputFiles, |
steinman | 39c00d2 | 2020-03-20 15:23:10 -0700 | [diff] [blame] | 474 | Supplier<NavigableSet<PathFragment>> sortedKnownModifiedOutputFiles, |
| 475 | boolean trustRemoteArtifacts) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 476 | boolean isDirty = false; |
lberki | f7eee1e | 2019-07-31 05:49:10 -0700 | [diff] [blame] | 477 | for (Map.Entry<Artifact, FileArtifactValue> entry : actionValue.getAllFileValues().entrySet()) { |
Googler | 974879d | 2020-05-27 13:25:52 -0700 | [diff] [blame] | 478 | if (artifactIsDirtyWithDirectSystemCalls( |
| 479 | knownModifiedOutputFiles, trustRemoteArtifacts, entry)) { |
| 480 | isDirty = true; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 481 | } |
| 482 | } |
Michael Thvedt | e4a7b079 | 2016-02-09 12:15:53 +0000 | [diff] [blame] | 483 | |
| 484 | for (Map.Entry<Artifact, TreeArtifactValue> entry : |
| 485 | actionValue.getAllTreeArtifactValues().entrySet()) { |
Googler | 974879d | 2020-05-27 13:25:52 -0700 | [diff] [blame] | 486 | TreeArtifactValue tree = entry.getValue(); |
Rumou Duan | 45e8e57 | 2016-06-17 16:43:44 +0000 | [diff] [blame] | 487 | |
Googler | 974879d | 2020-05-27 13:25:52 -0700 | [diff] [blame] | 488 | if (!tree.isRemote()) { |
| 489 | for (Map.Entry<TreeFileArtifact, FileArtifactValue> childEntry : |
| 490 | tree.getChildValues().entrySet()) { |
| 491 | if (artifactIsDirtyWithDirectSystemCalls( |
| 492 | knownModifiedOutputFiles, trustRemoteArtifacts, childEntry)) { |
| 493 | isDirty = true; |
| 494 | } |
| 495 | } |
| 496 | } |
| 497 | |
| 498 | Artifact treeArtifact = entry.getKey(); |
| 499 | |
| 500 | if (shouldCheckTreeArtifact(sortedKnownModifiedOutputFiles.get(), treeArtifact) |
| 501 | && treeArtifactIsDirty(treeArtifact, entry.getValue())) { |
| 502 | Path path = treeArtifact.getPath(); |
Michael Thvedt | e4a7b079 | 2016-02-09 12:15:53 +0000 | [diff] [blame] | 503 | // Count the changed directory as one "file". |
| 504 | try { |
lberki | 299e3f0 | 2019-07-31 04:49:21 -0700 | [diff] [blame] | 505 | updateIntraBuildModifiedCounter(path.exists() ? path.getLastModifiedTime() : -1); |
Michael Thvedt | e4a7b079 | 2016-02-09 12:15:53 +0000 | [diff] [blame] | 506 | } catch (IOException e) { |
| 507 | // Do nothing here. |
| 508 | } |
| 509 | |
| 510 | modifiedOutputFilesCounter.getAndIncrement(); |
| 511 | isDirty = true; |
| 512 | } |
| 513 | } |
| 514 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 515 | return isDirty; |
| 516 | } |
| 517 | |
Michael Thvedt | e4a7b079 | 2016-02-09 12:15:53 +0000 | [diff] [blame] | 518 | private static boolean shouldCheckFile(ImmutableSet<PathFragment> knownModifiedOutputFiles, |
Rumou Duan | a77f32c | 2016-04-13 21:59:21 +0000 | [diff] [blame] | 519 | Artifact artifact) { |
Eric Fellheimer | e659072 | 2015-11-17 17:07:48 +0000 | [diff] [blame] | 520 | return knownModifiedOutputFiles == null |
Rumou Duan | a77f32c | 2016-04-13 21:59:21 +0000 | [diff] [blame] | 521 | || knownModifiedOutputFiles.contains(artifact.getExecPath()); |
Eric Fellheimer | e659072 | 2015-11-17 17:07:48 +0000 | [diff] [blame] | 522 | } |
| 523 | |
Rumou Duan | 45e8e57 | 2016-06-17 16:43:44 +0000 | [diff] [blame] | 524 | private static boolean shouldCheckTreeArtifact( |
| 525 | @Nullable NavigableSet<PathFragment> knownModifiedOutputFiles, Artifact treeArtifact) { |
| 526 | // If null, everything needs to be checked. |
| 527 | if (knownModifiedOutputFiles == null) { |
| 528 | return true; |
| 529 | } |
| 530 | |
| 531 | // Here we do the following to see whether a TreeArtifact is modified: |
| 532 | // 1. Sort the set of modified file paths in lexicographical order using TreeSet. |
| 533 | // 2. Get the first modified output file path that is greater than or equal to the exec path of |
| 534 | // the TreeArtifact to check. |
| 535 | // 3. Check whether the returned file path contains the exec path of the TreeArtifact as a |
| 536 | // prefix path. |
| 537 | PathFragment artifactExecPath = treeArtifact.getExecPath(); |
| 538 | PathFragment headPath = knownModifiedOutputFiles.ceiling(artifactExecPath); |
| 539 | |
| 540 | return headPath != null && headPath.startsWith(artifactExecPath); |
| 541 | } |
| 542 | |
nharmata | acccd42 | 2020-01-27 13:38:23 -0800 | [diff] [blame] | 543 | private ImmutableBatchDirtyResult getDirtyValues( |
| 544 | ValueFetcher fetcher, |
Benjamin Peterson | 4024e16 | 2020-03-06 02:28:29 -0800 | [diff] [blame] | 545 | Collection<SkyKey> keys, |
nharmata | acccd42 | 2020-01-27 13:38:23 -0800 | [diff] [blame] | 546 | final SkyValueDirtinessChecker checker, |
| 547 | final boolean checkMissingValues) |
| 548 | throws InterruptedException { |
Janak Ramakrishnan | 6ddbb6e | 2015-07-28 21:39:22 +0000 | [diff] [blame] | 549 | ExecutorService executor = |
| 550 | Executors.newFixedThreadPool( |
twerth | 5aaceb5 | 2020-04-07 06:31:56 -0700 | [diff] [blame] | 551 | numThreads, |
Janak Ramakrishnan | 6ddbb6e | 2015-07-28 21:39:22 +0000 | [diff] [blame] | 552 | new ThreadFactoryBuilder().setNameFormat("FileSystem Value Invalidator %d").build()); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 553 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 554 | ThrowableRecordingRunnableWrapper wrapper = |
| 555 | new ThrowableRecordingRunnableWrapper("FilesystemValueChecker#getDirtyValues"); |
Nathan Harmata | e1b61d0 | 2015-10-06 00:09:42 +0000 | [diff] [blame] | 556 | final AtomicInteger numKeysChecked = new AtomicInteger(0); |
nharmata | acccd42 | 2020-01-27 13:38:23 -0800 | [diff] [blame] | 557 | MutableBatchDirtyResult batchResult = new MutableBatchDirtyResult(numKeysChecked); |
lberki | 97abb52 | 2017-09-04 18:51:57 +0200 | [diff] [blame] | 558 | ElapsedTimeReceiver elapsedTimeReceiver = |
felly | b6f8642 | 2018-02-20 19:01:35 -0800 | [diff] [blame] | 559 | elapsedTimeNanos -> { |
| 560 | if (elapsedTimeNanos > 0) { |
janakr | c3bcb98 | 2020-04-14 06:50:08 -0700 | [diff] [blame] | 561 | logger.atInfo().log( |
| 562 | "Spent %d nanoseconds checking %d filesystem nodes (%d scanned)", |
| 563 | elapsedTimeNanos, numKeysChecked.get(), keys.size()); |
Nathan Harmata | e1b61d0 | 2015-10-06 00:09:42 +0000 | [diff] [blame] | 564 | } |
lberki | 97abb52 | 2017-09-04 18:51:57 +0200 | [diff] [blame] | 565 | }; |
Nathan Harmata | e1b61d0 | 2015-10-06 00:09:42 +0000 | [diff] [blame] | 566 | try (AutoProfiler prof = AutoProfiler.create(elapsedTimeReceiver)) { |
Nathan Harmata | 8cd2978 | 2015-11-10 03:24:01 +0000 | [diff] [blame] | 567 | for (final SkyKey key : keys) { |
Nathan Harmata | 3a509bd | 2015-10-06 01:00:47 +0000 | [diff] [blame] | 568 | if (!checker.applies(key)) { |
| 569 | continue; |
| 570 | } |
janakr | e54491e | 2018-07-11 16:29:13 -0700 | [diff] [blame] | 571 | Preconditions.checkState( |
| 572 | key.functionName().getHermeticity() == FunctionHermeticity.NONHERMETIC, |
| 573 | "Only non-hermetic keys can be dirty roots: %s", |
| 574 | key); |
Nathan Harmata | e1b61d0 | 2015-10-06 00:09:42 +0000 | [diff] [blame] | 575 | executor.execute( |
| 576 | wrapper.wrap( |
felly | b6f8642 | 2018-02-20 19:01:35 -0800 | [diff] [blame] | 577 | () -> { |
| 578 | SkyValue value; |
| 579 | try { |
| 580 | value = fetcher.get(key); |
| 581 | } catch (InterruptedException e) { |
| 582 | // Exit fast. Interrupt is handled below on the main thread. |
| 583 | return; |
| 584 | } |
| 585 | if (!checkMissingValues && value == null) { |
| 586 | return; |
| 587 | } |
| 588 | |
| 589 | numKeysChecked.incrementAndGet(); |
| 590 | DirtyResult result = checker.check(key, value, tsgm); |
| 591 | if (result.isDirty()) { |
| 592 | batchResult.add(key, value, result.getNewValue()); |
Janak Ramakrishnan | 6ddbb6e | 2015-07-28 21:39:22 +0000 | [diff] [blame] | 593 | } |
Nathan Harmata | e1b61d0 | 2015-10-06 00:09:42 +0000 | [diff] [blame] | 594 | })); |
| 595 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 596 | |
Nathan Harmata | e1b61d0 | 2015-10-06 00:09:42 +0000 | [diff] [blame] | 597 | boolean interrupted = ExecutorUtil.interruptibleShutdown(executor); |
| 598 | Throwables.propagateIfPossible(wrapper.getFirstThrownError()); |
| 599 | if (interrupted) { |
| 600 | throw new InterruptedException(); |
| 601 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 602 | } |
nharmata | acccd42 | 2020-01-27 13:38:23 -0800 | [diff] [blame] | 603 | return batchResult.toImmutable(); |
| 604 | } |
| 605 | |
| 606 | static class ImmutableBatchDirtyResult implements Differencer.DiffWithDelta { |
| 607 | private final Collection<SkyKey> dirtyKeysWithoutNewValues; |
| 608 | private final Map<SkyKey, Delta> dirtyKeysWithNewAndOldValues; |
| 609 | private final int numKeysChecked; |
| 610 | |
| 611 | private ImmutableBatchDirtyResult( |
| 612 | Collection<SkyKey> dirtyKeysWithoutNewValues, |
| 613 | Map<SkyKey, Delta> dirtyKeysWithNewAndOldValues, |
| 614 | int numKeysChecked) { |
| 615 | this.dirtyKeysWithoutNewValues = dirtyKeysWithoutNewValues; |
| 616 | this.dirtyKeysWithNewAndOldValues = dirtyKeysWithNewAndOldValues; |
| 617 | this.numKeysChecked = numKeysChecked; |
| 618 | } |
| 619 | |
| 620 | @Override |
| 621 | public Collection<SkyKey> changedKeysWithoutNewValues() { |
| 622 | return dirtyKeysWithoutNewValues; |
| 623 | } |
| 624 | |
| 625 | @Override |
| 626 | public Map<SkyKey, Delta> changedKeysWithNewAndOldValues() { |
| 627 | return dirtyKeysWithNewAndOldValues; |
| 628 | } |
| 629 | |
| 630 | @Override |
| 631 | public Map<SkyKey, SkyValue> changedKeysWithNewValues() { |
| 632 | return Delta.newValues(dirtyKeysWithNewAndOldValues); |
| 633 | } |
| 634 | |
| 635 | int getNumKeysChecked() { |
| 636 | return numKeysChecked; |
| 637 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 638 | } |
| 639 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 640 | /** |
nharmata | acccd42 | 2020-01-27 13:38:23 -0800 | [diff] [blame] | 641 | * Result of a batch call to {@link SkyValueDirtinessChecker#check}. Partitions the dirty values |
| 642 | * based on whether we have a new value available for them or not. |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 643 | */ |
nharmata | acccd42 | 2020-01-27 13:38:23 -0800 | [diff] [blame] | 644 | private static class MutableBatchDirtyResult { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 645 | private final Set<SkyKey> concurrentDirtyKeysWithoutNewValues = |
| 646 | Collections.newSetFromMap(new ConcurrentHashMap<SkyKey, Boolean>()); |
Nathan Harmata | 2ff0a6d | 2015-08-12 21:10:56 +0000 | [diff] [blame] | 647 | private final ConcurrentHashMap<SkyKey, Delta> concurrentDirtyKeysWithNewAndOldValues = |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 648 | new ConcurrentHashMap<>(); |
nharmata | acccd42 | 2020-01-27 13:38:23 -0800 | [diff] [blame] | 649 | private final AtomicInteger numChecked; |
| 650 | |
| 651 | private MutableBatchDirtyResult(AtomicInteger numChecked) { |
| 652 | this.numChecked = numChecked; |
| 653 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 654 | |
Nathan Harmata | 2ff0a6d | 2015-08-12 21:10:56 +0000 | [diff] [blame] | 655 | private void add(SkyKey key, @Nullable SkyValue oldValue, @Nullable SkyValue newValue) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 656 | if (newValue == null) { |
| 657 | concurrentDirtyKeysWithoutNewValues.add(key); |
| 658 | } else { |
Nathan Harmata | 2ff0a6d | 2015-08-12 21:10:56 +0000 | [diff] [blame] | 659 | if (oldValue == null) { |
| 660 | concurrentDirtyKeysWithNewAndOldValues.put(key, new Delta(newValue)); |
| 661 | } else { |
| 662 | concurrentDirtyKeysWithNewAndOldValues.put(key, new Delta(oldValue, newValue)); |
| 663 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 664 | } |
| 665 | } |
| 666 | |
nharmata | acccd42 | 2020-01-27 13:38:23 -0800 | [diff] [blame] | 667 | private ImmutableBatchDirtyResult toImmutable() { |
| 668 | return new ImmutableBatchDirtyResult( |
| 669 | concurrentDirtyKeysWithoutNewValues, |
| 670 | concurrentDirtyKeysWithNewAndOldValues, |
| 671 | numChecked.get()); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 672 | } |
| 673 | } |
| 674 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 675 | } |