blob: 971d84df61a62f2a70b4e9bb4c85f2aa578b5886 [file] [log] [blame]
Damien Martin-Guillerezf88f4d82015-09-25 13:56:55 +00001// Copyright 2014 The Bazel Authors. All rights reserved.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +01002//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package com.google.devtools.build.lib.skyframe;
15
16import com.google.common.annotations.VisibleForTesting;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010017import com.google.common.collect.ImmutableList;
18import com.google.common.collect.ImmutableSet;
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000019import com.google.common.collect.Maps;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010020import com.google.common.collect.Sets;
Michajlo Matijkiwa09f7662015-06-12 17:41:21 +000021import com.google.common.io.BaseEncoding;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010022import com.google.devtools.build.lib.actions.ActionInput;
Googlerece75722016-02-11 17:55:41 +000023import com.google.devtools.build.lib.actions.ActionInputHelper;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010024import com.google.devtools.build.lib.actions.Artifact;
Rumou Duana77f32c2016-04-13 21:59:21 +000025import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
Shreya Bhattarai141b6c22016-08-22 22:00:24 +000026import com.google.devtools.build.lib.actions.cache.Md5Digest;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010027import com.google.devtools.build.lib.actions.cache.Metadata;
28import com.google.devtools.build.lib.actions.cache.MetadataHandler;
Mark Schaller6df81792015-12-10 18:47:47 +000029import com.google.devtools.build.lib.util.Preconditions;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010030import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
31import com.google.devtools.build.lib.vfs.FileStatus;
32import com.google.devtools.build.lib.vfs.FileStatusWithDigest;
33import com.google.devtools.build.lib.vfs.FileStatusWithDigestAdapter;
34import com.google.devtools.build.lib.vfs.Path;
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000035import com.google.devtools.build.lib.vfs.PathFragment;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010036import com.google.devtools.build.lib.vfs.RootedPath;
37import com.google.devtools.build.lib.vfs.Symlinks;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010038import java.io.FileNotFoundException;
39import java.io.IOException;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010040import java.util.Arrays;
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000041import java.util.Collection;
42import java.util.Collections;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010043import java.util.Map;
44import java.util.Set;
45import java.util.concurrent.ConcurrentHashMap;
46import java.util.concurrent.ConcurrentMap;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010047import javax.annotation.Nullable;
48
49/**
50 * Cache provided by an {@link ActionExecutionFunction}, allowing Blaze to obtain data from the
51 * graph and to inject data (e.g. file digests) back into the graph.
52 *
53 * <p>Data for the action's inputs is injected into this cache on construction, using the graph as
54 * the source of truth.
55 *
56 * <p>As well, this cache collects data about the action's output files, which is used in three
57 * ways. First, it is served as requested during action execution, primarily by the {@code
58 * ActionCacheChecker} when determining if the action must be rerun, and then after the action is
Janak Ramakrishnanad77f972016-07-29 20:58:42 +000059 * run, to gather information about the outputs. Second, it is accessed by {@link ArtifactFunction}s
60 * in order to construct {@link FileArtifactValue}s, and by this class itself to generate {@link
61 * TreeArtifactValue}s. Third, the {@link FilesystemValueChecker} uses it to determine the set of
62 * output files to check for inter-build modifications. Because all these use cases are slightly
63 * different, we must occasionally store two versions of the data for a value. See {@link
64 * #getAdditionalOutputData} for elaboration on the difference between these cases, and see the
65 * javadoc for the various internal maps to see what is stored where.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010066 */
67@VisibleForTesting
Janak Ramakrishnana5c1f962015-04-03 23:06:31 +000068public class ActionMetadataHandler implements MetadataHandler {
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000069
70 /**
71 * Data for input artifacts. Immutable.
72 *
73 * <p>This should never be read directly. Use {@link #getInputFileArtifactValue} instead.</p>
74 */
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010075 private final Map<Artifact, FileArtifactValue> inputArtifactData;
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000076
Rumou Duana77f32c2016-04-13 21:59:21 +000077 /** FileValues for each output Artifact. */
78 private final ConcurrentMap<Artifact, FileValue> outputArtifactData =
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010079 new ConcurrentHashMap<>();
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000080
81 /**
82 * Maps output TreeArtifacts to their contents. These maps are either injected or read
83 * directly from the filesystem.
84 * If the value is null, this means nothing was injected, and the output TreeArtifact
85 * is to have its values read from disk instead.
86 */
Rumou Duana77f32c2016-04-13 21:59:21 +000087 private final ConcurrentMap<Artifact, Set<TreeFileArtifact>> outputDirectoryListings =
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000088 new ConcurrentHashMap<>();
89
90 /** Outputs that are to be omitted. */
Michajlo Matijkiw13459b42015-03-12 19:43:20 +000091 private final Set<Artifact> omittedOutputs = Sets.newConcurrentHashSet();
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000092
93 /**
94 * Contains RealArtifactValues when those values must be stored separately.
95 * See {@link #getAdditionalOutputData()} for details.
96 */
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010097 private final ConcurrentMap<Artifact, FileArtifactValue> additionalOutputData =
98 new ConcurrentHashMap<>();
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000099
100 /**
Rumou Duana77f32c2016-04-13 21:59:21 +0000101 * Data for TreeArtifactValues, constructed from outputArtifactData and
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000102 * additionalOutputFileData.
103 */
104 private final ConcurrentMap<Artifact, TreeArtifactValue> outputTreeArtifactData =
105 new ConcurrentHashMap<>();
106
Rumou Duana77f32c2016-04-13 21:59:21 +0000107 /** Tracks which Artifacts have had metadata injected. */
108 private final Set<Artifact> injectedFiles = Sets.newConcurrentHashSet();
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000109
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100110 private final ImmutableSet<Artifact> outputs;
Julio Merino4d80aa72016-11-01 20:58:16 +0000111
112 /**
113 * The timestamp granularity monitor for this build.
114 * Use {@link #getTimestampGranularityMonitor(Artifact)} to fetch this member.
115 */
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100116 private final TimestampGranularityMonitor tsgm;
117
Janak Ramakrishnan90f3d342015-03-27 19:45:18 +0000118 @VisibleForTesting
Janak Ramakrishnana5c1f962015-04-03 23:06:31 +0000119 public ActionMetadataHandler(Map<Artifact, FileArtifactValue> inputArtifactData,
Janak Ramakrishnan90f3d342015-03-27 19:45:18 +0000120 Iterable<Artifact> outputs,
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100121 TimestampGranularityMonitor tsgm) {
122 this.inputArtifactData = Preconditions.checkNotNull(inputArtifactData);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100123 this.outputs = ImmutableSet.copyOf(outputs);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100124 this.tsgm = tsgm;
125 }
126
127 @Override
128 public Metadata getMetadataMaybe(Artifact artifact) {
129 try {
130 return getMetadata(artifact);
131 } catch (IOException e) {
132 return null;
133 }
134 }
135
Julio Merino4d80aa72016-11-01 20:58:16 +0000136 /**
137 * Gets the {@link TimestampGranularityMonitor} to use for a given artifact.
138 *
139 * <p>If the artifact is of type "constant metadata", this returns null so that changes to such
140 * artifacts do not tickle the timestamp granularity monitor, delaying the build for no reason.
141 *
142 * @param artifact the artifact for which to fetch the timestamp granularity monitor
143 * @return the timestamp granularity monitor to use, which may be null
144 */
145 private TimestampGranularityMonitor getTimestampGranularityMonitor(Artifact artifact) {
146 return artifact.isConstantMetadata() ? null : tsgm;
147 }
148
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100149 private static Metadata metadataFromValue(FileArtifactValue value) throws FileNotFoundException {
Michajlo Matijkiw597d55e2015-03-25 17:17:52 +0000150 if (value == FileArtifactValue.MISSING_FILE_MARKER
151 || value == FileArtifactValue.OMITTED_FILE_MARKER) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100152 throw new FileNotFoundException();
153 }
Janak Ramakrishnan406c0872016-07-28 21:32:01 +0000154 // If the file is a directory, we need to return the mtime because the action cache uses mtime
155 // to determine if this artifact has changed. We want this code path to go away somehow
156 // for directories (maybe by implementing FileSet in Skyframe).
157 return value.isFile() ? new Metadata(value.getDigest()) : new Metadata(value.getModifiedTime());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100158 }
159
160 @Override
161 public Metadata getMetadata(Artifact artifact) throws IOException {
162 Metadata metadata = getRealMetadata(artifact);
163 return artifact.isConstantMetadata() ? Metadata.CONSTANT_METADATA : metadata;
164 }
165
166 @Nullable
Rumou Duana33bea32016-10-07 16:51:20 +0000167 private FileArtifactValue getInputFileArtifactValue(Artifact input) {
168 if (outputs.contains(input)) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100169 return null;
170 }
Rumou Duana33bea32016-10-07 16:51:20 +0000171
172 if (input.hasParent() && outputs.contains(input.getParent())) {
173 return null;
174 }
175
Janak Ramakrishnan90f3d342015-03-27 19:45:18 +0000176 return Preconditions.checkNotNull(inputArtifactData.get(input), input);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100177 }
178
179 /**
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000180 * Get the real (viz. on-disk) metadata for an Artifact.
181 * A key assumption is that getRealMetadata() will be called for every Artifact in this
182 * ActionMetadataHandler, to populate additionalOutputData and outputTreeArtifactData.
183 *
184 * <p>We cache data for constant-metadata artifacts, even though it is technically unnecessary,
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100185 * because the data stored in this cache is consumed by various parts of Blaze via the {@link
186 * ActionExecutionValue} (for now, {@link FilesystemValueChecker} and {@link ArtifactFunction}).
187 * It is simpler for those parts if every output of the action is present in the cache. However,
188 * we must not return the actual metadata for a constant-metadata artifact.
189 */
190 private Metadata getRealMetadata(Artifact artifact) throws IOException {
191 FileArtifactValue value = getInputFileArtifactValue(artifact);
192 if (value != null) {
193 return metadataFromValue(value);
194 }
195 if (artifact.isSourceArtifact()) {
196 // A discovered input we didn't have data for.
197 // TODO(bazel-team): Change this to an assertion once Skyframe has native input discovery, so
198 // all inputs will already have metadata known.
199 return null;
200 } else if (artifact.isMiddlemanArtifact()) {
201 // A middleman artifact's data was either already injected from the action cache checker using
202 // #setDigestForVirtualArtifact, or it has the default middleman value.
203 value = additionalOutputData.get(artifact);
204 if (value != null) {
205 return metadataFromValue(value);
206 }
207 value = FileArtifactValue.DEFAULT_MIDDLEMAN;
208 FileArtifactValue oldValue = additionalOutputData.putIfAbsent(artifact, value);
209 checkInconsistentData(artifact, oldValue, value);
210 return metadataFromValue(value);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000211 } else if (artifact.isTreeArtifact()) {
212 TreeArtifactValue setValue = getTreeArtifactValue(artifact);
213 if (setValue != null && setValue != TreeArtifactValue.MISSING_TREE_ARTIFACT) {
214 return setValue.getMetadata();
215 }
216 // We use FileNotFoundExceptions to determine if an Artifact was or wasn't found.
217 // Calling code depends on this particular exception.
218 throw new FileNotFoundException(artifact + " not found");
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100219 }
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000220 // It's an ordinary artifact.
Rumou Duana77f32c2016-04-13 21:59:21 +0000221 FileValue fileValue = outputArtifactData.get(artifact);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100222 if (fileValue != null) {
223 // Non-middleman artifacts should only have additionalOutputData if they have
224 // outputArtifactData. We don't assert this because of concurrency possibilities, but at least
225 // we don't check additionalOutputData unless we expect that we might see the artifact there.
226 value = additionalOutputData.get(artifact);
227 // If additional output data is present for this artifact, we use it in preference to the
228 // usual calculation.
229 if (value != null) {
230 return metadataFromValue(value);
231 }
232 if (!fileValue.exists()) {
233 throw new FileNotFoundException(artifact.prettyPrint() + " does not exist");
234 }
235 return new Metadata(Preconditions.checkNotNull(fileValue.getDigest(), artifact));
236 }
237 // We do not cache exceptions besides nonexistence here, because it is unlikely that the file
238 // will be requested from this cache too many times.
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000239 fileValue = constructFileValue(artifact, null);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100240 return maybeStoreAdditionalData(artifact, fileValue, null);
241 }
242
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100243 /**
Rumou Duana77f32c2016-04-13 21:59:21 +0000244 * Check that the new {@code data} we just calculated for an {@link Artifact} agrees with the
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100245 * {@code oldData} (presumably calculated concurrently), if it was present.
246 */
247 // Not private only because used by SkyframeActionExecutor's metadata handler.
Rumou Duana77f32c2016-04-13 21:59:21 +0000248 static void checkInconsistentData(Artifact artifact,
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100249 @Nullable Object oldData, Object data) throws IOException {
250 if (oldData != null && !oldData.equals(data)) {
251 // Another thread checked this file since we looked at the map, and got a different answer
252 // than we did. Presumably the user modified the file between reads.
Rumou Duana77f32c2016-04-13 21:59:21 +0000253 throw new IOException("Data for " + artifact.prettyPrint() + " changed to " + data
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100254 + " after it was calculated as " + oldData);
255 }
256 }
257
258 /**
259 * See {@link #getAdditionalOutputData} for why we sometimes need to store additional data, even
260 * for normal (non-middleman) artifacts.
261 */
262 @Nullable
Rumou Duana77f32c2016-04-13 21:59:21 +0000263 private Metadata maybeStoreAdditionalData(Artifact artifact, FileValue data,
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100264 @Nullable byte[] injectedDigest) throws IOException {
265 if (!data.exists()) {
266 // Nonexistent files should only occur before executing an action.
Rumou Duana77f32c2016-04-13 21:59:21 +0000267 throw new FileNotFoundException(artifact.prettyPrint() + " does not exist");
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100268 }
Janak Ramakrishnancba16452016-07-29 02:17:02 +0000269 boolean isFile = data.isFile();
270 if (isFile && !artifact.hasParent() && data.getDigest() != null) {
271 // We do not need to store the FileArtifactValue separately -- the digest is in the file value
272 // and that is all that is needed for this file's metadata.
273 return new Metadata(data.getDigest());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100274 }
Janak Ramakrishnancba16452016-07-29 02:17:02 +0000275 // Unfortunately, the FileValue does not contain enough information for us to calculate the
276 // corresponding FileArtifactValue -- either the metadata must use the modified time, which we
277 // do not expose in the FileValue, or the FileValue didn't store the digest So we store the
278 // metadata separately.
279 // Use the FileValue's digest if no digest was injected, or if the file can't be digested.
280 injectedDigest = injectedDigest != null || !isFile ? injectedDigest : data.getDigest();
281 FileArtifactValue value =
282 FileArtifactValue.create(artifact, isFile, isFile ? data.getSize() : 0, injectedDigest);
283 FileArtifactValue oldValue = additionalOutputData.putIfAbsent(artifact, value);
284 checkInconsistentData(artifact, oldValue, value);
285 return metadataFromValue(value);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100286 }
287
288 @Override
Shreya Bhattarai141b6c22016-08-22 22:00:24 +0000289 public void setDigestForVirtualArtifact(Artifact artifact, Md5Digest md5Digest) {
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000290 Preconditions.checkArgument(artifact.isMiddlemanArtifact(), artifact);
Shreya Bhattarai141b6c22016-08-22 22:00:24 +0000291 Preconditions.checkNotNull(md5Digest, artifact);
292 additionalOutputData.put(
293 artifact, FileArtifactValue.createProxy(md5Digest.getDigestBytesUnsafe()));
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000294 }
295
Rumou Duana77f32c2016-04-13 21:59:21 +0000296 private Set<TreeFileArtifact> getTreeArtifactContents(Artifact artifact) {
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000297 Preconditions.checkArgument(artifact.isTreeArtifact(), artifact);
Rumou Duana77f32c2016-04-13 21:59:21 +0000298 Set<TreeFileArtifact> contents = outputDirectoryListings.get(artifact);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000299 if (contents == null) {
300 // Unfortunately, there is no such thing as a ConcurrentHashSet.
Rumou Duana77f32c2016-04-13 21:59:21 +0000301 contents = Collections.newSetFromMap(new ConcurrentHashMap<TreeFileArtifact, Boolean>());
302 Set<TreeFileArtifact> oldContents = outputDirectoryListings.putIfAbsent(artifact, contents);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000303 // Avoid a race condition.
304 if (oldContents != null) {
305 contents = oldContents;
306 }
307 }
308 return contents;
309 }
310
311 private TreeArtifactValue getTreeArtifactValue(Artifact artifact) throws IOException {
312 TreeArtifactValue value = outputTreeArtifactData.get(artifact);
313 if (value != null) {
314 return value;
315 }
316
Rumou Duana77f32c2016-04-13 21:59:21 +0000317 Set<TreeFileArtifact> registeredContents = outputDirectoryListings.get(artifact);
Googlerece75722016-02-11 17:55:41 +0000318 if (registeredContents != null) {
319 // Check that our registered outputs matches on-disk outputs. Only perform this check
320 // when contents were explicitly registered.
321 // TODO(bazel-team): Provide a way for actions to register empty TreeArtifacts.
322
323 // By the time we're constructing TreeArtifactValues, use of the metadata handler
324 // should be single threaded and there should be no race condition.
325 // The current design of ActionMetadataHandler makes this hard to enforce.
326 Set<PathFragment> paths = null;
Rumou Duan9ad28cd2016-10-19 19:28:06 +0000327 paths = TreeArtifactValue.explodeDirectory(artifact);
Rumou Duana77f32c2016-04-13 21:59:21 +0000328 Set<TreeFileArtifact> diskFiles = ActionInputHelper.asTreeFileArtifacts(artifact, paths);
Googlerece75722016-02-11 17:55:41 +0000329 if (!diskFiles.equals(registeredContents)) {
330 // There might be more than one error here. We first look for missing output files.
Rumou Duana77f32c2016-04-13 21:59:21 +0000331 Set<TreeFileArtifact> missingFiles = Sets.difference(registeredContents, diskFiles);
Googlerece75722016-02-11 17:55:41 +0000332 if (!missingFiles.isEmpty()) {
333 // Don't throw IOException--getMetadataMaybe() eats them.
334 // TODO(bazel-team): Report this error in a better way when called by checkOutputs()
335 // Currently it's hard to report this error without refactoring, since checkOutputs()
336 // likes to substitute its own error messages upon catching IOException, and falls
337 // through to unrecoverable error behavior on any other exception.
Rumou Duan9ad28cd2016-10-19 19:28:06 +0000338 throw new IOException("Output file " + missingFiles.iterator().next()
Googlerece75722016-02-11 17:55:41 +0000339 + " was registered, but not present on disk");
340 }
341
Rumou Duana77f32c2016-04-13 21:59:21 +0000342 Set<TreeFileArtifact> extraFiles = Sets.difference(diskFiles, registeredContents);
Googlerece75722016-02-11 17:55:41 +0000343 // extraFiles cannot be empty
Rumou Duan9ad28cd2016-10-19 19:28:06 +0000344 throw new IOException(
Googlerece75722016-02-11 17:55:41 +0000345 "File " + extraFiles.iterator().next().getParentRelativePath()
346 + ", present in TreeArtifact " + artifact + ", was not registered");
347 }
348
349 value = constructTreeArtifactValue(registeredContents);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000350 } else {
Googlerece75722016-02-11 17:55:41 +0000351 value = constructTreeArtifactValueFromFilesystem(artifact);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000352 }
353
354 TreeArtifactValue oldValue = outputTreeArtifactData.putIfAbsent(artifact, value);
355 checkInconsistentData(artifact, oldValue, value);
356 return value;
357 }
358
Rumou Duana77f32c2016-04-13 21:59:21 +0000359 private TreeArtifactValue constructTreeArtifactValue(Collection<TreeFileArtifact> contents)
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000360 throws IOException {
Rumou Duana77f32c2016-04-13 21:59:21 +0000361 Map<TreeFileArtifact, FileArtifactValue> values =
362 Maps.newHashMapWithExpectedSize(contents.size());
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000363
Rumou Duana77f32c2016-04-13 21:59:21 +0000364 for (TreeFileArtifact treeFileArtifact : contents) {
Rumou Duan73876202016-06-06 18:52:08 +0000365 FileArtifactValue cachedValue = additionalOutputData.get(treeFileArtifact);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000366 if (cachedValue == null) {
Rumou Duana77f32c2016-04-13 21:59:21 +0000367 FileValue fileValue = outputArtifactData.get(treeFileArtifact);
368 // This is similar to what's present in getRealMetadataForArtifact, except
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000369 // we get back the FileValue, not the metadata.
370 // We do not cache exceptions besides nonexistence here, because it is unlikely that the
371 // file will be requested from this cache too many times.
372 if (fileValue == null) {
Rumou Duan11730a02016-10-24 20:27:58 +0000373 try {
374 fileValue = constructFileValue(treeFileArtifact, /*statNoFollow=*/ null);
375 } catch (FileNotFoundException e) {
376 String errorMessage = String.format(
377 "Failed to resolve relative path %s inside TreeArtifact %s. "
378 + "The associated file is either missing or is an invalid symlink.",
379 treeFileArtifact.getParentRelativePath(),
380 treeFileArtifact.getParent().getExecPathString());
381 throw new IOException(errorMessage, e);
382 }
383
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000384 // A minor hack: maybeStoreAdditionalData will force the data to be stored
Rumou Duan73876202016-06-06 18:52:08 +0000385 // in additionalOutputData.
Rumou Duana77f32c2016-04-13 21:59:21 +0000386 maybeStoreAdditionalData(treeFileArtifact, fileValue, null);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000387 }
Rumou Duana77f32c2016-04-13 21:59:21 +0000388 cachedValue = Preconditions.checkNotNull(
Rumou Duan73876202016-06-06 18:52:08 +0000389 additionalOutputData.get(treeFileArtifact), treeFileArtifact);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000390 }
391
Rumou Duana77f32c2016-04-13 21:59:21 +0000392 values.put(treeFileArtifact, cachedValue);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000393 }
394
395 return TreeArtifactValue.create(values);
396 }
397
Googlerece75722016-02-11 17:55:41 +0000398 private TreeArtifactValue constructTreeArtifactValueFromFilesystem(Artifact artifact)
399 throws IOException {
400 Preconditions.checkState(artifact.isTreeArtifact(), artifact);
401
402 if (!artifact.getPath().isDirectory() || artifact.getPath().isSymbolicLink()) {
403 return TreeArtifactValue.MISSING_TREE_ARTIFACT;
404 }
405
406 Set<PathFragment> paths = null;
Rumou Duan9ad28cd2016-10-19 19:28:06 +0000407 paths = TreeArtifactValue.explodeDirectory(artifact);
Googlerece75722016-02-11 17:55:41 +0000408 // If you're reading tree artifacts from disk while outputDirectoryListings are being injected,
409 // something has gone terribly wrong.
410 Object previousDirectoryListing =
411 outputDirectoryListings.put(artifact,
Rumou Duana77f32c2016-04-13 21:59:21 +0000412 Collections.newSetFromMap(new ConcurrentHashMap<TreeFileArtifact, Boolean>()));
Googlerece75722016-02-11 17:55:41 +0000413 Preconditions.checkState(previousDirectoryListing == null,
414 "Race condition while constructing TreArtifactValue: %s, %s",
415 artifact, previousDirectoryListing);
Rumou Duana77f32c2016-04-13 21:59:21 +0000416 return constructTreeArtifactValue(ActionInputHelper.asTreeFileArtifacts(artifact, paths));
Googlerece75722016-02-11 17:55:41 +0000417 }
418
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000419 @Override
Rumou Duana77f32c2016-04-13 21:59:21 +0000420 public void addExpandedTreeOutput(TreeFileArtifact output) {
421 Set<TreeFileArtifact> values = getTreeArtifactContents(output.getParent());
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000422 values.add(output);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100423 }
424
425 @Override
Rumou Duana33bea32016-10-07 16:51:20 +0000426 public Iterable<TreeFileArtifact> getExpandedOutputs(Artifact artifact) {
427 return ImmutableSet.copyOf(getTreeArtifactContents(artifact));
428 }
429
430 @Override
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100431 public void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest) {
Rumou Duana77f32c2016-04-13 21:59:21 +0000432 // Assumption: any non-Artifact output is 'virtual' and should be ignored here.
433 if (output instanceof Artifact) {
434 final Artifact artifact = (Artifact) output;
435 Preconditions.checkState(injectedFiles.add(artifact), artifact);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100436 FileValue fileValue;
437 try {
438 // This call may do an unnecessary call to Path#getFastDigest to see if the digest is
439 // readily available. We cannot pass the digest in, though, because if it is not available
440 // from the filesystem, this FileValue will not compare equal to another one created for the
441 // same file, because the other one will be missing its digest.
Rumou Duana77f32c2016-04-13 21:59:21 +0000442 fileValue = fileValueFromArtifact(artifact,
Julio Merino4d80aa72016-11-01 20:58:16 +0000443 FileStatusWithDigestAdapter.adapt(statNoFollow),
444 getTimestampGranularityMonitor(artifact));
Michajlo Matijkiwa09f7662015-06-12 17:41:21 +0000445 // Ensure the digest supplied matches the actual digest if it exists.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100446 byte[] fileDigest = fileValue.getDigest();
Michajlo Matijkiwa09f7662015-06-12 17:41:21 +0000447 if (fileDigest != null && !Arrays.equals(digest, fileDigest)) {
448 BaseEncoding base16 = BaseEncoding.base16();
449 String digestString = (digest != null) ? base16.encode(digest) : "null";
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000450 String fileDigestString = base16.encode(fileDigest);
Michajlo Matijkiwa09f7662015-06-12 17:41:21 +0000451 throw new IllegalStateException("Expected digest " + digestString + " for artifact "
Rumou Duana77f32c2016-04-13 21:59:21 +0000452 + artifact + ", but got " + fileDigestString + " (" + fileValue + ")");
Michajlo Matijkiwa09f7662015-06-12 17:41:21 +0000453 }
Rumou Duana77f32c2016-04-13 21:59:21 +0000454 outputArtifactData.put(artifact, fileValue);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100455 } catch (IOException e) {
456 // Do nothing - we just failed to inject metadata. Real error handling will be done later,
457 // when somebody will try to access that file.
458 return;
459 }
460 // If needed, insert additional data. Note that this can only be true if the file is empty or
461 // the filesystem does not support fast digests. Since we usually only inject digests when
462 // running with a filesystem that supports fast digests, this is fairly unlikely.
463 try {
Rumou Duana77f32c2016-04-13 21:59:21 +0000464 maybeStoreAdditionalData(artifact, fileValue, digest);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100465 } catch (IOException e) {
466 if (fileValue.getSize() != 0) {
467 // Empty files currently have their mtimes examined, and so could throw. No other files
468 // should throw, since all filesystem access has already been done.
469 throw new IllegalStateException(
470 "Filesystem should not have been accessed while injecting data for "
Rumou Duana77f32c2016-04-13 21:59:21 +0000471 + artifact.prettyPrint(), e);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100472 }
473 // Ignore exceptions for empty files, as above.
474 }
475 }
476 }
477
478 @Override
Michajlo Matijkiw13459b42015-03-12 19:43:20 +0000479 public void markOmitted(ActionInput output) {
480 if (output instanceof Artifact) {
481 Artifact artifact = (Artifact) output;
482 Preconditions.checkState(omittedOutputs.add(artifact), artifact);
483 additionalOutputData.put(artifact, FileArtifactValue.OMITTED_FILE_MARKER);
484 }
485 }
486
487 @Override
488 public boolean artifactOmitted(Artifact artifact) {
489 return omittedOutputs.contains(artifact);
490 }
491
492 @Override
Janak Ramakrishnan73055be2015-04-13 18:32:49 +0000493 public void discardOutputMetadata() {
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000494 Preconditions.checkState(injectedFiles.isEmpty(),
495 "Files cannot be injected before action execution: %s", injectedFiles);
Janak Ramakrishnan73055be2015-04-13 18:32:49 +0000496 Preconditions.checkState(omittedOutputs.isEmpty(),
497 "Artifacts cannot be marked omitted before action execution: %s", omittedOutputs);
Rumou Duana77f32c2016-04-13 21:59:21 +0000498 outputArtifactData.clear();
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000499 outputDirectoryListings.clear();
500 outputTreeArtifactData.clear();
Janak Ramakrishnan73055be2015-04-13 18:32:49 +0000501 additionalOutputData.clear();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100502 }
503
504 @Override
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100505 public boolean isRegularFile(Artifact artifact) {
506 // Currently this method is used only for genrule input directory checks. If we need to call
507 // this on output artifacts too, this could be more efficient.
508 FileArtifactValue value = getInputFileArtifactValue(artifact);
Michajlo Matijkiwdee781c2015-05-22 23:25:34 +0000509 if (value != null && value.isFile()) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100510 return true;
511 }
512 return artifact.getPath().isFile();
513 }
514
515 @Override
Rumou Duana77f32c2016-04-13 21:59:21 +0000516 public boolean isInjected(Artifact file) {
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000517 return injectedFiles.contains(file);
518 }
519
520 /** @return data for output files that was computed during execution. */
Rumou Duana77f32c2016-04-13 21:59:21 +0000521 Map<Artifact, FileValue> getOutputArtifactData() {
522 return outputArtifactData;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100523 }
524
525 /**
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000526 * @return data for TreeArtifacts that was computed during execution. May contain copies of
527 * {@link TreeArtifactValue#MISSING_TREE_ARTIFACT}.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100528 */
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000529 Map<Artifact, TreeArtifactValue> getOutputTreeArtifactData() {
530 return outputTreeArtifactData;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100531 }
532
533 /**
534 * Returns data for any output files whose metadata was not computable from the corresponding
Rumou Duana77f32c2016-04-13 21:59:21 +0000535 * entry in {@link #getOutputArtifactData}.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100536 *
537 * <p>There are three reasons why we might not be able to compute metadata for an artifact from
538 * the FileValue. First, middleman artifacts have no corresponding FileValues. Second, if
539 * computing a file's digest is not fast, the FileValue does not do so, so a file on a filesystem
540 * without fast digests has to have its metadata stored separately. Third, some files' metadata
541 * (directories, empty files) contain their mtimes, which the FileValue does not expose, so that
542 * has to be stored separately.
543 *
544 * <p>Note that for files that need digests, we can't easily inject the digest in the FileValue
545 * because it would complicate equality-checking on subsequent builds -- if our filesystem doesn't
546 * do fast digests, the comparison value would not have a digest.
547 */
548 Map<Artifact, FileArtifactValue> getAdditionalOutputData() {
549 return additionalOutputData;
550 }
551
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000552 /** Constructs a new FileValue, saves it, and checks inconsistent data. */
Rumou Duana77f32c2016-04-13 21:59:21 +0000553 FileValue constructFileValue(Artifact artifact, @Nullable FileStatusWithDigest statNoFollow)
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000554 throws IOException {
Julio Merino4d80aa72016-11-01 20:58:16 +0000555 FileValue value = fileValueFromArtifact(artifact, statNoFollow,
556 getTimestampGranularityMonitor(artifact));
Rumou Duana77f32c2016-04-13 21:59:21 +0000557 FileValue oldFsValue = outputArtifactData.putIfAbsent(artifact, value);
558 checkInconsistentData(artifact, oldFsValue, null);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000559 return value;
560 }
561
562 @VisibleForTesting
Rumou Duana77f32c2016-04-13 21:59:21 +0000563 static FileValue fileValueFromArtifact(Artifact artifact,
Ulf Adamsc73051c62016-03-23 09:18:13 +0000564 @Nullable FileStatusWithDigest statNoFollow, @Nullable TimestampGranularityMonitor tsgm)
Janak Ramakrishnana5c1f962015-04-03 23:06:31 +0000565 throws IOException {
Rumou Duana77f32c2016-04-13 21:59:21 +0000566 Path path = artifact.getPath();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100567 RootedPath rootedPath =
Rumou Duana77f32c2016-04-13 21:59:21 +0000568 RootedPath.toRootedPath(artifact.getRoot().getPath(), artifact.getRootRelativePath());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100569 if (statNoFollow == null) {
570 statNoFollow = FileStatusWithDigestAdapter.adapt(path.statIfFound(Symlinks.NOFOLLOW));
571 if (statNoFollow == null) {
572 return FileValue.value(rootedPath, FileStateValue.NONEXISTENT_FILE_STATE_NODE,
573 rootedPath, FileStateValue.NONEXISTENT_FILE_STATE_NODE);
574 }
575 }
576 Path realPath = path;
577 // We use FileStatus#isSymbolicLink over Path#isSymbolicLink to avoid the unnecessary stat
578 // done by the latter.
579 if (statNoFollow.isSymbolicLink()) {
580 realPath = path.resolveSymbolicLinks();
581 // We need to protect against symlink cycles since FileValue#value assumes it's dealing with a
582 // file that's not in a symlink cycle.
583 if (realPath.equals(path)) {
584 throw new IOException("symlink cycle");
585 }
586 }
587 RootedPath realRootedPath = RootedPath.toRootedPathMaybeUnderRoot(realPath,
Rumou Duana77f32c2016-04-13 21:59:21 +0000588 ImmutableList.of(artifact.getRoot().getPath()));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100589 FileStateValue fileStateValue;
590 FileStateValue realFileStateValue;
591 try {
592 fileStateValue = FileStateValue.createWithStatNoFollow(rootedPath, statNoFollow, tsgm);
593 // TODO(bazel-team): consider avoiding a 'stat' here when the symlink target hasn't changed
594 // and is a source file (since changes to those are checked separately).
595 realFileStateValue = realPath.equals(path) ? fileStateValue
596 : FileStateValue.create(realRootedPath, tsgm);
597 } catch (InconsistentFilesystemException e) {
598 throw new IOException(e);
599 }
600 return FileValue.value(rootedPath, fileStateValue, realRootedPath, realFileStateValue);
601 }
602}