blob: 087e97a5f10538c9879a8c90e0382961284e82bf [file] [log] [blame]
Damien Martin-Guillerezf88f4d82015-09-25 13:56:55 +00001// Copyright 2014 The Bazel Authors. All rights reserved.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +01002//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package com.google.devtools.build.lib.skyframe;
15
16import com.google.common.annotations.VisibleForTesting;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010017import com.google.common.collect.ImmutableList;
18import com.google.common.collect.ImmutableSet;
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000019import com.google.common.collect.Maps;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010020import com.google.common.collect.Sets;
Michajlo Matijkiwa09f7662015-06-12 17:41:21 +000021import com.google.common.io.BaseEncoding;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010022import com.google.devtools.build.lib.actions.ActionInput;
Googlerece75722016-02-11 17:55:41 +000023import com.google.devtools.build.lib.actions.ActionInputHelper;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010024import com.google.devtools.build.lib.actions.Artifact;
Rumou Duana77f32c2016-04-13 21:59:21 +000025import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
Shreya Bhattarai141b6c22016-08-22 22:00:24 +000026import com.google.devtools.build.lib.actions.cache.Md5Digest;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010027import com.google.devtools.build.lib.actions.cache.Metadata;
28import com.google.devtools.build.lib.actions.cache.MetadataHandler;
Googlerece75722016-02-11 17:55:41 +000029import com.google.devtools.build.lib.skyframe.TreeArtifactValue.TreeArtifactException;
Mark Schaller6df81792015-12-10 18:47:47 +000030import com.google.devtools.build.lib.util.Preconditions;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010031import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
32import com.google.devtools.build.lib.vfs.FileStatus;
33import com.google.devtools.build.lib.vfs.FileStatusWithDigest;
34import com.google.devtools.build.lib.vfs.FileStatusWithDigestAdapter;
35import com.google.devtools.build.lib.vfs.Path;
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000036import com.google.devtools.build.lib.vfs.PathFragment;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010037import com.google.devtools.build.lib.vfs.RootedPath;
38import com.google.devtools.build.lib.vfs.Symlinks;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010039import java.io.FileNotFoundException;
40import java.io.IOException;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010041import java.util.Arrays;
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000042import java.util.Collection;
43import java.util.Collections;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010044import java.util.Map;
45import java.util.Set;
46import java.util.concurrent.ConcurrentHashMap;
47import java.util.concurrent.ConcurrentMap;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010048import javax.annotation.Nullable;
49
50/**
51 * Cache provided by an {@link ActionExecutionFunction}, allowing Blaze to obtain data from the
52 * graph and to inject data (e.g. file digests) back into the graph.
53 *
54 * <p>Data for the action's inputs is injected into this cache on construction, using the graph as
55 * the source of truth.
56 *
57 * <p>As well, this cache collects data about the action's output files, which is used in three
58 * ways. First, it is served as requested during action execution, primarily by the {@code
59 * ActionCacheChecker} when determining if the action must be rerun, and then after the action is
Janak Ramakrishnanad77f972016-07-29 20:58:42 +000060 * run, to gather information about the outputs. Second, it is accessed by {@link ArtifactFunction}s
61 * in order to construct {@link FileArtifactValue}s, and by this class itself to generate {@link
62 * TreeArtifactValue}s. Third, the {@link FilesystemValueChecker} uses it to determine the set of
63 * output files to check for inter-build modifications. Because all these use cases are slightly
64 * different, we must occasionally store two versions of the data for a value. See {@link
65 * #getAdditionalOutputData} for elaboration on the difference between these cases, and see the
66 * javadoc for the various internal maps to see what is stored where.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010067 */
68@VisibleForTesting
Janak Ramakrishnana5c1f962015-04-03 23:06:31 +000069public class ActionMetadataHandler implements MetadataHandler {
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000070
71 /**
72 * Data for input artifacts. Immutable.
73 *
74 * <p>This should never be read directly. Use {@link #getInputFileArtifactValue} instead.</p>
75 */
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010076 private final Map<Artifact, FileArtifactValue> inputArtifactData;
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000077
Rumou Duana77f32c2016-04-13 21:59:21 +000078 /** FileValues for each output Artifact. */
79 private final ConcurrentMap<Artifact, FileValue> outputArtifactData =
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010080 new ConcurrentHashMap<>();
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000081
82 /**
83 * Maps output TreeArtifacts to their contents. These maps are either injected or read
84 * directly from the filesystem.
85 * If the value is null, this means nothing was injected, and the output TreeArtifact
86 * is to have its values read from disk instead.
87 */
Rumou Duana77f32c2016-04-13 21:59:21 +000088 private final ConcurrentMap<Artifact, Set<TreeFileArtifact>> outputDirectoryListings =
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000089 new ConcurrentHashMap<>();
90
91 /** Outputs that are to be omitted. */
Michajlo Matijkiw13459b42015-03-12 19:43:20 +000092 private final Set<Artifact> omittedOutputs = Sets.newConcurrentHashSet();
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +000093
94 /**
95 * Contains RealArtifactValues when those values must be stored separately.
96 * See {@link #getAdditionalOutputData()} for details.
97 */
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010098 private final ConcurrentMap<Artifact, FileArtifactValue> additionalOutputData =
99 new ConcurrentHashMap<>();
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000100
101 /**
Rumou Duana77f32c2016-04-13 21:59:21 +0000102 * Data for TreeArtifactValues, constructed from outputArtifactData and
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000103 * additionalOutputFileData.
104 */
105 private final ConcurrentMap<Artifact, TreeArtifactValue> outputTreeArtifactData =
106 new ConcurrentHashMap<>();
107
Rumou Duana77f32c2016-04-13 21:59:21 +0000108 /** Tracks which Artifacts have had metadata injected. */
109 private final Set<Artifact> injectedFiles = Sets.newConcurrentHashSet();
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000110
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100111 private final ImmutableSet<Artifact> outputs;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100112 private final TimestampGranularityMonitor tsgm;
113
Janak Ramakrishnan90f3d342015-03-27 19:45:18 +0000114 @VisibleForTesting
Janak Ramakrishnana5c1f962015-04-03 23:06:31 +0000115 public ActionMetadataHandler(Map<Artifact, FileArtifactValue> inputArtifactData,
Janak Ramakrishnan90f3d342015-03-27 19:45:18 +0000116 Iterable<Artifact> outputs,
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100117 TimestampGranularityMonitor tsgm) {
118 this.inputArtifactData = Preconditions.checkNotNull(inputArtifactData);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100119 this.outputs = ImmutableSet.copyOf(outputs);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100120 this.tsgm = tsgm;
121 }
122
123 @Override
124 public Metadata getMetadataMaybe(Artifact artifact) {
125 try {
126 return getMetadata(artifact);
127 } catch (IOException e) {
128 return null;
129 }
130 }
131
132 private static Metadata metadataFromValue(FileArtifactValue value) throws FileNotFoundException {
Michajlo Matijkiw597d55e2015-03-25 17:17:52 +0000133 if (value == FileArtifactValue.MISSING_FILE_MARKER
134 || value == FileArtifactValue.OMITTED_FILE_MARKER) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100135 throw new FileNotFoundException();
136 }
Janak Ramakrishnan406c0872016-07-28 21:32:01 +0000137 // If the file is a directory, we need to return the mtime because the action cache uses mtime
138 // to determine if this artifact has changed. We want this code path to go away somehow
139 // for directories (maybe by implementing FileSet in Skyframe).
140 return value.isFile() ? new Metadata(value.getDigest()) : new Metadata(value.getModifiedTime());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100141 }
142
143 @Override
144 public Metadata getMetadata(Artifact artifact) throws IOException {
145 Metadata metadata = getRealMetadata(artifact);
146 return artifact.isConstantMetadata() ? Metadata.CONSTANT_METADATA : metadata;
147 }
148
149 @Nullable
150 private FileArtifactValue getInputFileArtifactValue(ActionInput input) {
Janak Ramakrishnan90f3d342015-03-27 19:45:18 +0000151 if (outputs.contains(input) || !(input instanceof Artifact)) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100152 return null;
153 }
Janak Ramakrishnan90f3d342015-03-27 19:45:18 +0000154 return Preconditions.checkNotNull(inputArtifactData.get(input), input);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100155 }
156
157 /**
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000158 * Get the real (viz. on-disk) metadata for an Artifact.
159 * A key assumption is that getRealMetadata() will be called for every Artifact in this
160 * ActionMetadataHandler, to populate additionalOutputData and outputTreeArtifactData.
161 *
162 * <p>We cache data for constant-metadata artifacts, even though it is technically unnecessary,
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100163 * because the data stored in this cache is consumed by various parts of Blaze via the {@link
164 * ActionExecutionValue} (for now, {@link FilesystemValueChecker} and {@link ArtifactFunction}).
165 * It is simpler for those parts if every output of the action is present in the cache. However,
166 * we must not return the actual metadata for a constant-metadata artifact.
167 */
168 private Metadata getRealMetadata(Artifact artifact) throws IOException {
169 FileArtifactValue value = getInputFileArtifactValue(artifact);
170 if (value != null) {
171 return metadataFromValue(value);
172 }
173 if (artifact.isSourceArtifact()) {
174 // A discovered input we didn't have data for.
175 // TODO(bazel-team): Change this to an assertion once Skyframe has native input discovery, so
176 // all inputs will already have metadata known.
177 return null;
178 } else if (artifact.isMiddlemanArtifact()) {
179 // A middleman artifact's data was either already injected from the action cache checker using
180 // #setDigestForVirtualArtifact, or it has the default middleman value.
181 value = additionalOutputData.get(artifact);
182 if (value != null) {
183 return metadataFromValue(value);
184 }
185 value = FileArtifactValue.DEFAULT_MIDDLEMAN;
186 FileArtifactValue oldValue = additionalOutputData.putIfAbsent(artifact, value);
187 checkInconsistentData(artifact, oldValue, value);
188 return metadataFromValue(value);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000189 } else if (artifact.isTreeArtifact()) {
190 TreeArtifactValue setValue = getTreeArtifactValue(artifact);
191 if (setValue != null && setValue != TreeArtifactValue.MISSING_TREE_ARTIFACT) {
192 return setValue.getMetadata();
193 }
194 // We use FileNotFoundExceptions to determine if an Artifact was or wasn't found.
195 // Calling code depends on this particular exception.
196 throw new FileNotFoundException(artifact + " not found");
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100197 }
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000198 // It's an ordinary artifact.
Rumou Duana77f32c2016-04-13 21:59:21 +0000199 FileValue fileValue = outputArtifactData.get(artifact);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100200 if (fileValue != null) {
201 // Non-middleman artifacts should only have additionalOutputData if they have
202 // outputArtifactData. We don't assert this because of concurrency possibilities, but at least
203 // we don't check additionalOutputData unless we expect that we might see the artifact there.
204 value = additionalOutputData.get(artifact);
205 // If additional output data is present for this artifact, we use it in preference to the
206 // usual calculation.
207 if (value != null) {
208 return metadataFromValue(value);
209 }
210 if (!fileValue.exists()) {
211 throw new FileNotFoundException(artifact.prettyPrint() + " does not exist");
212 }
213 return new Metadata(Preconditions.checkNotNull(fileValue.getDigest(), artifact));
214 }
215 // We do not cache exceptions besides nonexistence here, because it is unlikely that the file
216 // will be requested from this cache too many times.
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000217 fileValue = constructFileValue(artifact, null);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100218 return maybeStoreAdditionalData(artifact, fileValue, null);
219 }
220
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100221 /**
Rumou Duana77f32c2016-04-13 21:59:21 +0000222 * Check that the new {@code data} we just calculated for an {@link Artifact} agrees with the
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100223 * {@code oldData} (presumably calculated concurrently), if it was present.
224 */
225 // Not private only because used by SkyframeActionExecutor's metadata handler.
Rumou Duana77f32c2016-04-13 21:59:21 +0000226 static void checkInconsistentData(Artifact artifact,
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100227 @Nullable Object oldData, Object data) throws IOException {
228 if (oldData != null && !oldData.equals(data)) {
229 // Another thread checked this file since we looked at the map, and got a different answer
230 // than we did. Presumably the user modified the file between reads.
Rumou Duana77f32c2016-04-13 21:59:21 +0000231 throw new IOException("Data for " + artifact.prettyPrint() + " changed to " + data
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100232 + " after it was calculated as " + oldData);
233 }
234 }
235
236 /**
237 * See {@link #getAdditionalOutputData} for why we sometimes need to store additional data, even
238 * for normal (non-middleman) artifacts.
239 */
240 @Nullable
Rumou Duana77f32c2016-04-13 21:59:21 +0000241 private Metadata maybeStoreAdditionalData(Artifact artifact, FileValue data,
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100242 @Nullable byte[] injectedDigest) throws IOException {
243 if (!data.exists()) {
244 // Nonexistent files should only occur before executing an action.
Rumou Duana77f32c2016-04-13 21:59:21 +0000245 throw new FileNotFoundException(artifact.prettyPrint() + " does not exist");
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100246 }
Janak Ramakrishnancba16452016-07-29 02:17:02 +0000247 boolean isFile = data.isFile();
248 if (isFile && !artifact.hasParent() && data.getDigest() != null) {
249 // We do not need to store the FileArtifactValue separately -- the digest is in the file value
250 // and that is all that is needed for this file's metadata.
251 return new Metadata(data.getDigest());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100252 }
Janak Ramakrishnancba16452016-07-29 02:17:02 +0000253 // Unfortunately, the FileValue does not contain enough information for us to calculate the
254 // corresponding FileArtifactValue -- either the metadata must use the modified time, which we
255 // do not expose in the FileValue, or the FileValue didn't store the digest So we store the
256 // metadata separately.
257 // Use the FileValue's digest if no digest was injected, or if the file can't be digested.
258 injectedDigest = injectedDigest != null || !isFile ? injectedDigest : data.getDigest();
259 FileArtifactValue value =
260 FileArtifactValue.create(artifact, isFile, isFile ? data.getSize() : 0, injectedDigest);
261 FileArtifactValue oldValue = additionalOutputData.putIfAbsent(artifact, value);
262 checkInconsistentData(artifact, oldValue, value);
263 return metadataFromValue(value);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100264 }
265
266 @Override
Shreya Bhattarai141b6c22016-08-22 22:00:24 +0000267 public void setDigestForVirtualArtifact(Artifact artifact, Md5Digest md5Digest) {
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000268 Preconditions.checkArgument(artifact.isMiddlemanArtifact(), artifact);
Shreya Bhattarai141b6c22016-08-22 22:00:24 +0000269 Preconditions.checkNotNull(md5Digest, artifact);
270 additionalOutputData.put(
271 artifact, FileArtifactValue.createProxy(md5Digest.getDigestBytesUnsafe()));
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000272 }
273
Rumou Duana77f32c2016-04-13 21:59:21 +0000274 private Set<TreeFileArtifact> getTreeArtifactContents(Artifact artifact) {
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000275 Preconditions.checkArgument(artifact.isTreeArtifact(), artifact);
Rumou Duana77f32c2016-04-13 21:59:21 +0000276 Set<TreeFileArtifact> contents = outputDirectoryListings.get(artifact);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000277 if (contents == null) {
278 // Unfortunately, there is no such thing as a ConcurrentHashSet.
Rumou Duana77f32c2016-04-13 21:59:21 +0000279 contents = Collections.newSetFromMap(new ConcurrentHashMap<TreeFileArtifact, Boolean>());
280 Set<TreeFileArtifact> oldContents = outputDirectoryListings.putIfAbsent(artifact, contents);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000281 // Avoid a race condition.
282 if (oldContents != null) {
283 contents = oldContents;
284 }
285 }
286 return contents;
287 }
288
289 private TreeArtifactValue getTreeArtifactValue(Artifact artifact) throws IOException {
290 TreeArtifactValue value = outputTreeArtifactData.get(artifact);
291 if (value != null) {
292 return value;
293 }
294
Rumou Duana77f32c2016-04-13 21:59:21 +0000295 Set<TreeFileArtifact> registeredContents = outputDirectoryListings.get(artifact);
Googlerece75722016-02-11 17:55:41 +0000296 if (registeredContents != null) {
297 // Check that our registered outputs matches on-disk outputs. Only perform this check
298 // when contents were explicitly registered.
299 // TODO(bazel-team): Provide a way for actions to register empty TreeArtifacts.
300
301 // By the time we're constructing TreeArtifactValues, use of the metadata handler
302 // should be single threaded and there should be no race condition.
303 // The current design of ActionMetadataHandler makes this hard to enforce.
304 Set<PathFragment> paths = null;
305 try {
306 paths = TreeArtifactValue.explodeDirectory(artifact);
307 } catch (TreeArtifactException e) {
308 throw new IllegalStateException(e);
309 }
Rumou Duana77f32c2016-04-13 21:59:21 +0000310 Set<TreeFileArtifact> diskFiles = ActionInputHelper.asTreeFileArtifacts(artifact, paths);
Googlerece75722016-02-11 17:55:41 +0000311 if (!diskFiles.equals(registeredContents)) {
312 // There might be more than one error here. We first look for missing output files.
Rumou Duana77f32c2016-04-13 21:59:21 +0000313 Set<TreeFileArtifact> missingFiles = Sets.difference(registeredContents, diskFiles);
Googlerece75722016-02-11 17:55:41 +0000314 if (!missingFiles.isEmpty()) {
315 // Don't throw IOException--getMetadataMaybe() eats them.
316 // TODO(bazel-team): Report this error in a better way when called by checkOutputs()
317 // Currently it's hard to report this error without refactoring, since checkOutputs()
318 // likes to substitute its own error messages upon catching IOException, and falls
319 // through to unrecoverable error behavior on any other exception.
320 throw new IllegalStateException("Output file " + missingFiles.iterator().next()
321 + " was registered, but not present on disk");
322 }
323
Rumou Duana77f32c2016-04-13 21:59:21 +0000324 Set<TreeFileArtifact> extraFiles = Sets.difference(diskFiles, registeredContents);
Googlerece75722016-02-11 17:55:41 +0000325 // extraFiles cannot be empty
326 throw new IllegalStateException(
327 "File " + extraFiles.iterator().next().getParentRelativePath()
328 + ", present in TreeArtifact " + artifact + ", was not registered");
329 }
330
331 value = constructTreeArtifactValue(registeredContents);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000332 } else {
Googlerece75722016-02-11 17:55:41 +0000333 value = constructTreeArtifactValueFromFilesystem(artifact);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000334 }
335
336 TreeArtifactValue oldValue = outputTreeArtifactData.putIfAbsent(artifact, value);
337 checkInconsistentData(artifact, oldValue, value);
338 return value;
339 }
340
Rumou Duana77f32c2016-04-13 21:59:21 +0000341 private TreeArtifactValue constructTreeArtifactValue(Collection<TreeFileArtifact> contents)
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000342 throws IOException {
Rumou Duana77f32c2016-04-13 21:59:21 +0000343 Map<TreeFileArtifact, FileArtifactValue> values =
344 Maps.newHashMapWithExpectedSize(contents.size());
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000345
Rumou Duana77f32c2016-04-13 21:59:21 +0000346 for (TreeFileArtifact treeFileArtifact : contents) {
Rumou Duan73876202016-06-06 18:52:08 +0000347 FileArtifactValue cachedValue = additionalOutputData.get(treeFileArtifact);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000348 if (cachedValue == null) {
Rumou Duana77f32c2016-04-13 21:59:21 +0000349 FileValue fileValue = outputArtifactData.get(treeFileArtifact);
350 // This is similar to what's present in getRealMetadataForArtifact, except
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000351 // we get back the FileValue, not the metadata.
352 // We do not cache exceptions besides nonexistence here, because it is unlikely that the
353 // file will be requested from this cache too many times.
354 if (fileValue == null) {
Rumou Duana77f32c2016-04-13 21:59:21 +0000355 fileValue = constructFileValue(treeFileArtifact, /*statNoFollow=*/ null);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000356 // A minor hack: maybeStoreAdditionalData will force the data to be stored
Rumou Duan73876202016-06-06 18:52:08 +0000357 // in additionalOutputData.
Rumou Duana77f32c2016-04-13 21:59:21 +0000358 maybeStoreAdditionalData(treeFileArtifact, fileValue, null);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000359 }
Rumou Duana77f32c2016-04-13 21:59:21 +0000360 cachedValue = Preconditions.checkNotNull(
Rumou Duan73876202016-06-06 18:52:08 +0000361 additionalOutputData.get(treeFileArtifact), treeFileArtifact);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000362 }
363
Rumou Duana77f32c2016-04-13 21:59:21 +0000364 values.put(treeFileArtifact, cachedValue);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000365 }
366
367 return TreeArtifactValue.create(values);
368 }
369
Googlerece75722016-02-11 17:55:41 +0000370 private TreeArtifactValue constructTreeArtifactValueFromFilesystem(Artifact artifact)
371 throws IOException {
372 Preconditions.checkState(artifact.isTreeArtifact(), artifact);
373
374 if (!artifact.getPath().isDirectory() || artifact.getPath().isSymbolicLink()) {
375 return TreeArtifactValue.MISSING_TREE_ARTIFACT;
376 }
377
378 Set<PathFragment> paths = null;
379 try {
380 paths = TreeArtifactValue.explodeDirectory(artifact);
381 } catch (TreeArtifactException e) {
382 throw new IllegalStateException(e);
383 }
384 // If you're reading tree artifacts from disk while outputDirectoryListings are being injected,
385 // something has gone terribly wrong.
386 Object previousDirectoryListing =
387 outputDirectoryListings.put(artifact,
Rumou Duana77f32c2016-04-13 21:59:21 +0000388 Collections.newSetFromMap(new ConcurrentHashMap<TreeFileArtifact, Boolean>()));
Googlerece75722016-02-11 17:55:41 +0000389 Preconditions.checkState(previousDirectoryListing == null,
390 "Race condition while constructing TreArtifactValue: %s, %s",
391 artifact, previousDirectoryListing);
Rumou Duana77f32c2016-04-13 21:59:21 +0000392 return constructTreeArtifactValue(ActionInputHelper.asTreeFileArtifacts(artifact, paths));
Googlerece75722016-02-11 17:55:41 +0000393 }
394
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000395 @Override
Rumou Duana77f32c2016-04-13 21:59:21 +0000396 public void addExpandedTreeOutput(TreeFileArtifact output) {
397 Set<TreeFileArtifact> values = getTreeArtifactContents(output.getParent());
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000398 values.add(output);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100399 }
400
401 @Override
402 public void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest) {
Rumou Duana77f32c2016-04-13 21:59:21 +0000403 // Assumption: any non-Artifact output is 'virtual' and should be ignored here.
404 if (output instanceof Artifact) {
405 final Artifact artifact = (Artifact) output;
406 Preconditions.checkState(injectedFiles.add(artifact), artifact);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100407 FileValue fileValue;
408 try {
409 // This call may do an unnecessary call to Path#getFastDigest to see if the digest is
410 // readily available. We cannot pass the digest in, though, because if it is not available
411 // from the filesystem, this FileValue will not compare equal to another one created for the
412 // same file, because the other one will be missing its digest.
Rumou Duana77f32c2016-04-13 21:59:21 +0000413 fileValue = fileValueFromArtifact(artifact,
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000414 FileStatusWithDigestAdapter.adapt(statNoFollow), tsgm);
Michajlo Matijkiwa09f7662015-06-12 17:41:21 +0000415 // Ensure the digest supplied matches the actual digest if it exists.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100416 byte[] fileDigest = fileValue.getDigest();
Michajlo Matijkiwa09f7662015-06-12 17:41:21 +0000417 if (fileDigest != null && !Arrays.equals(digest, fileDigest)) {
418 BaseEncoding base16 = BaseEncoding.base16();
419 String digestString = (digest != null) ? base16.encode(digest) : "null";
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000420 String fileDigestString = base16.encode(fileDigest);
Michajlo Matijkiwa09f7662015-06-12 17:41:21 +0000421 throw new IllegalStateException("Expected digest " + digestString + " for artifact "
Rumou Duana77f32c2016-04-13 21:59:21 +0000422 + artifact + ", but got " + fileDigestString + " (" + fileValue + ")");
Michajlo Matijkiwa09f7662015-06-12 17:41:21 +0000423 }
Rumou Duana77f32c2016-04-13 21:59:21 +0000424 outputArtifactData.put(artifact, fileValue);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100425 } catch (IOException e) {
426 // Do nothing - we just failed to inject metadata. Real error handling will be done later,
427 // when somebody will try to access that file.
428 return;
429 }
430 // If needed, insert additional data. Note that this can only be true if the file is empty or
431 // the filesystem does not support fast digests. Since we usually only inject digests when
432 // running with a filesystem that supports fast digests, this is fairly unlikely.
433 try {
Rumou Duana77f32c2016-04-13 21:59:21 +0000434 maybeStoreAdditionalData(artifact, fileValue, digest);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100435 } catch (IOException e) {
436 if (fileValue.getSize() != 0) {
437 // Empty files currently have their mtimes examined, and so could throw. No other files
438 // should throw, since all filesystem access has already been done.
439 throw new IllegalStateException(
440 "Filesystem should not have been accessed while injecting data for "
Rumou Duana77f32c2016-04-13 21:59:21 +0000441 + artifact.prettyPrint(), e);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100442 }
443 // Ignore exceptions for empty files, as above.
444 }
445 }
446 }
447
448 @Override
Michajlo Matijkiw13459b42015-03-12 19:43:20 +0000449 public void markOmitted(ActionInput output) {
450 if (output instanceof Artifact) {
451 Artifact artifact = (Artifact) output;
452 Preconditions.checkState(omittedOutputs.add(artifact), artifact);
453 additionalOutputData.put(artifact, FileArtifactValue.OMITTED_FILE_MARKER);
454 }
455 }
456
457 @Override
458 public boolean artifactOmitted(Artifact artifact) {
459 return omittedOutputs.contains(artifact);
460 }
461
462 @Override
Janak Ramakrishnan73055be2015-04-13 18:32:49 +0000463 public void discardOutputMetadata() {
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000464 Preconditions.checkState(injectedFiles.isEmpty(),
465 "Files cannot be injected before action execution: %s", injectedFiles);
Janak Ramakrishnan73055be2015-04-13 18:32:49 +0000466 Preconditions.checkState(omittedOutputs.isEmpty(),
467 "Artifacts cannot be marked omitted before action execution: %s", omittedOutputs);
Rumou Duana77f32c2016-04-13 21:59:21 +0000468 outputArtifactData.clear();
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000469 outputDirectoryListings.clear();
470 outputTreeArtifactData.clear();
Janak Ramakrishnan73055be2015-04-13 18:32:49 +0000471 additionalOutputData.clear();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100472 }
473
474 @Override
475 public boolean artifactExists(Artifact artifact) {
Michajlo Matijkiw13459b42015-03-12 19:43:20 +0000476 Preconditions.checkState(!artifactOmitted(artifact), artifact);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100477 return getMetadataMaybe(artifact) != null;
478 }
479
480 @Override
481 public boolean isRegularFile(Artifact artifact) {
482 // Currently this method is used only for genrule input directory checks. If we need to call
483 // this on output artifacts too, this could be more efficient.
484 FileArtifactValue value = getInputFileArtifactValue(artifact);
Michajlo Matijkiwdee781c2015-05-22 23:25:34 +0000485 if (value != null && value.isFile()) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100486 return true;
487 }
488 return artifact.getPath().isFile();
489 }
490
491 @Override
Rumou Duana77f32c2016-04-13 21:59:21 +0000492 public boolean isInjected(Artifact file) {
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000493 return injectedFiles.contains(file);
494 }
495
496 /** @return data for output files that was computed during execution. */
Rumou Duana77f32c2016-04-13 21:59:21 +0000497 Map<Artifact, FileValue> getOutputArtifactData() {
498 return outputArtifactData;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100499 }
500
501 /**
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000502 * @return data for TreeArtifacts that was computed during execution. May contain copies of
503 * {@link TreeArtifactValue#MISSING_TREE_ARTIFACT}.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100504 */
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000505 Map<Artifact, TreeArtifactValue> getOutputTreeArtifactData() {
506 return outputTreeArtifactData;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100507 }
508
509 /**
510 * Returns data for any output files whose metadata was not computable from the corresponding
Rumou Duana77f32c2016-04-13 21:59:21 +0000511 * entry in {@link #getOutputArtifactData}.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100512 *
513 * <p>There are three reasons why we might not be able to compute metadata for an artifact from
514 * the FileValue. First, middleman artifacts have no corresponding FileValues. Second, if
515 * computing a file's digest is not fast, the FileValue does not do so, so a file on a filesystem
516 * without fast digests has to have its metadata stored separately. Third, some files' metadata
517 * (directories, empty files) contain their mtimes, which the FileValue does not expose, so that
518 * has to be stored separately.
519 *
520 * <p>Note that for files that need digests, we can't easily inject the digest in the FileValue
521 * because it would complicate equality-checking on subsequent builds -- if our filesystem doesn't
522 * do fast digests, the comparison value would not have a digest.
523 */
524 Map<Artifact, FileArtifactValue> getAdditionalOutputData() {
525 return additionalOutputData;
526 }
527
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000528 /** Constructs a new FileValue, saves it, and checks inconsistent data. */
Rumou Duana77f32c2016-04-13 21:59:21 +0000529 FileValue constructFileValue(Artifact artifact, @Nullable FileStatusWithDigest statNoFollow)
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000530 throws IOException {
Rumou Duana77f32c2016-04-13 21:59:21 +0000531 FileValue value = fileValueFromArtifact(artifact, statNoFollow, tsgm);
532 FileValue oldFsValue = outputArtifactData.putIfAbsent(artifact, value);
533 checkInconsistentData(artifact, oldFsValue, null);
Michael Thvedt8d5a7bb2016-02-09 03:06:34 +0000534 return value;
535 }
536
537 @VisibleForTesting
Rumou Duana77f32c2016-04-13 21:59:21 +0000538 static FileValue fileValueFromArtifact(Artifact artifact,
Ulf Adamsc73051c62016-03-23 09:18:13 +0000539 @Nullable FileStatusWithDigest statNoFollow, @Nullable TimestampGranularityMonitor tsgm)
Janak Ramakrishnana5c1f962015-04-03 23:06:31 +0000540 throws IOException {
Rumou Duana77f32c2016-04-13 21:59:21 +0000541 Path path = artifact.getPath();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100542 RootedPath rootedPath =
Rumou Duana77f32c2016-04-13 21:59:21 +0000543 RootedPath.toRootedPath(artifact.getRoot().getPath(), artifact.getRootRelativePath());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100544 if (statNoFollow == null) {
545 statNoFollow = FileStatusWithDigestAdapter.adapt(path.statIfFound(Symlinks.NOFOLLOW));
546 if (statNoFollow == null) {
547 return FileValue.value(rootedPath, FileStateValue.NONEXISTENT_FILE_STATE_NODE,
548 rootedPath, FileStateValue.NONEXISTENT_FILE_STATE_NODE);
549 }
550 }
551 Path realPath = path;
552 // We use FileStatus#isSymbolicLink over Path#isSymbolicLink to avoid the unnecessary stat
553 // done by the latter.
554 if (statNoFollow.isSymbolicLink()) {
555 realPath = path.resolveSymbolicLinks();
556 // We need to protect against symlink cycles since FileValue#value assumes it's dealing with a
557 // file that's not in a symlink cycle.
558 if (realPath.equals(path)) {
559 throw new IOException("symlink cycle");
560 }
561 }
562 RootedPath realRootedPath = RootedPath.toRootedPathMaybeUnderRoot(realPath,
Rumou Duana77f32c2016-04-13 21:59:21 +0000563 ImmutableList.of(artifact.getRoot().getPath()));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100564 FileStateValue fileStateValue;
565 FileStateValue realFileStateValue;
566 try {
567 fileStateValue = FileStateValue.createWithStatNoFollow(rootedPath, statNoFollow, tsgm);
568 // TODO(bazel-team): consider avoiding a 'stat' here when the symlink target hasn't changed
569 // and is a source file (since changes to those are checked separately).
570 realFileStateValue = realPath.equals(path) ? fileStateValue
571 : FileStateValue.create(realRootedPath, tsgm);
572 } catch (InconsistentFilesystemException e) {
573 throw new IOException(e);
574 }
575 return FileValue.value(rootedPath, fileStateValue, realRootedPath, realFileStateValue);
576 }
577}