blob: baad566f5e5de7a5264dda2c23d5bb40e602bae1 [file] [log] [blame]
Ulf Adamsc0a84442017-03-21 10:08:03 +00001// Copyright 2017 The Bazel Authors. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package com.google.devtools.build.lib.exec;
15
Ulf Adamsc0a84442017-03-21 10:08:03 +000016import com.google.common.annotations.VisibleForTesting;
17import com.google.common.base.Preconditions;
kush2ce45a22018-05-02 14:15:37 -070018import com.google.common.collect.ImmutableList;
Ulf Adamsc0a84442017-03-21 10:08:03 +000019import com.google.devtools.build.lib.actions.ActionInput;
Ulf Adamsc0a84442017-03-21 10:08:03 +000020import com.google.devtools.build.lib.actions.ActionInputHelper;
21import com.google.devtools.build.lib.actions.Artifact;
22import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander;
buchgrd4d3d502018-08-02 06:47:19 -070023import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
felly56fd4fe2018-09-05 10:54:54 -070024import com.google.devtools.build.lib.actions.ArtifactPathResolver;
buchgrd4d3d502018-08-02 06:47:19 -070025import com.google.devtools.build.lib.actions.FileArtifactValue;
kush2ce45a22018-05-02 14:15:37 -070026import com.google.devtools.build.lib.actions.FilesetOutputSymlink;
shahan499503b2018-06-07 18:57:07 -070027import com.google.devtools.build.lib.actions.MetadataProvider;
Ulf Adamsc0a84442017-03-21 10:08:03 +000028import com.google.devtools.build.lib.actions.RunfilesSupplier;
29import com.google.devtools.build.lib.actions.Spawn;
ulfjack3e288682018-01-08 09:31:17 -080030import com.google.devtools.build.lib.actions.cache.VirtualActionInput.EmptyActionInput;
ulfjack1973be42018-05-30 03:25:35 -070031import com.google.devtools.build.lib.exec.FilesetManifest.RelativeSymlinkBehavior;
tomlu1a19b622018-01-11 15:17:28 -080032import com.google.devtools.build.lib.vfs.Path;
Ulf Adamsc0a84442017-03-21 10:08:03 +000033import com.google.devtools.build.lib.vfs.PathFragment;
34import java.io.IOException;
buchgrd4d3d502018-08-02 06:47:19 -070035import java.util.Collections;
Ulf Adamsc0a84442017-03-21 10:08:03 +000036import java.util.List;
37import java.util.Map;
Ulf Adamsc0a84442017-03-21 10:08:03 +000038import java.util.SortedMap;
39import java.util.TreeMap;
40
41/**
42 * A helper class for spawn strategies to turn runfiles suppliers into input mappings. This class
43 * performs no I/O operations, but only rearranges the files according to how the runfiles should be
44 * laid out.
45 */
46public class SpawnInputExpander {
buchgrd4d3d502018-08-02 06:47:19 -070047 @VisibleForTesting static final ActionInput EMPTY_FILE = new EmptyActionInput("/dev/null");
Ulf Adamsc0a84442017-03-21 10:08:03 +000048
tomlu1a19b622018-01-11 15:17:28 -080049 private final Path execRoot;
Ulf Adamsc0a84442017-03-21 10:08:03 +000050 private final boolean strict;
ulfjack1973be42018-05-30 03:25:35 -070051 private final RelativeSymlinkBehavior relSymlinkBehavior;
Ulf Adamsc0a84442017-03-21 10:08:03 +000052
53 /**
54 * Creates a new instance. If strict is true, then the expander checks for directories in runfiles
55 * and throws an exception if it finds any. Otherwise it silently ignores directories in runfiles
tomlu1a19b622018-01-11 15:17:28 -080056 * and adds a mapping for them. At this time, directories in filesets are always silently added as
57 * mappings.
Ulf Adamsc0a84442017-03-21 10:08:03 +000058 *
59 * <p>Directories in inputs are a correctness issue: Bazel only tracks dependencies at the action
60 * level, and it does not track dependencies on directories. Making a directory available to a
61 * spawn even though it's contents are not tracked as dependencies leads to incorrect incremental
62 * builds, since changes to the contents do not trigger action invalidation.
63 *
64 * <p>As such, all spawn strategies should always be strict and not make directories available to
65 * the subprocess. However, that's a breaking change, and therefore we make it depend on this flag
66 * for now.
67 */
tomlu1a19b622018-01-11 15:17:28 -080068 public SpawnInputExpander(Path execRoot, boolean strict) {
ulfjack1973be42018-05-30 03:25:35 -070069 this(execRoot, strict, RelativeSymlinkBehavior.ERROR);
70 }
71
72 /**
73 * Creates a new instance. If strict is true, then the expander checks for directories in runfiles
74 * and throws an exception if it finds any. Otherwise it silently ignores directories in runfiles
75 * and adds a mapping for them. At this time, directories in filesets are always silently added as
76 * mappings.
77 *
78 * <p>Directories in inputs are a correctness issue: Bazel only tracks dependencies at the action
79 * level, and it does not track dependencies on directories. Making a directory available to a
80 * spawn even though it's contents are not tracked as dependencies leads to incorrect incremental
81 * builds, since changes to the contents do not trigger action invalidation.
82 *
83 * <p>As such, all spawn strategies should always be strict and not make directories available to
84 * the subprocess. However, that's a breaking change, and therefore we make it depend on this flag
85 * for now.
86 */
87 public SpawnInputExpander(
88 Path execRoot, boolean strict, RelativeSymlinkBehavior relSymlinkBehavior) {
tomlu1a19b622018-01-11 15:17:28 -080089 this.execRoot = execRoot;
Ulf Adamsc0a84442017-03-21 10:08:03 +000090 this.strict = strict;
ulfjack1973be42018-05-30 03:25:35 -070091 this.relSymlinkBehavior = relSymlinkBehavior;
Ulf Adamsc0a84442017-03-21 10:08:03 +000092 }
93
94 private void addMapping(
95 Map<PathFragment, ActionInput> inputMappings,
96 PathFragment targetLocation,
97 ActionInput input) {
98 Preconditions.checkArgument(!targetLocation.isAbsolute(), targetLocation);
99 if (!inputMappings.containsKey(targetLocation)) {
100 inputMappings.put(targetLocation, input);
101 }
102 }
103
104 /** Adds runfiles inputs from runfilesSupplier to inputMappings. */
105 @VisibleForTesting
106 void addRunfilesToInputs(
107 Map<PathFragment, ActionInput> inputMap,
108 RunfilesSupplier runfilesSupplier,
buchgrd4d3d502018-08-02 06:47:19 -0700109 MetadataProvider actionFileCache,
philwo9ed9d8a2018-09-03 07:30:26 -0700110 ArtifactExpander artifactExpander,
felly56fd4fe2018-09-05 10:54:54 -0700111 ArtifactPathResolver pathResolver,
philwo9ed9d8a2018-09-03 07:30:26 -0700112 boolean expandTreeArtifactsInRunfiles)
shahan499503b2018-06-07 18:57:07 -0700113 throws IOException {
buchgrd4d3d502018-08-02 06:47:19 -0700114 Map<PathFragment, Map<PathFragment, Artifact>> rootsAndMappings =
felly56fd4fe2018-09-05 10:54:54 -0700115 runfilesSupplier.getMappings(pathResolver);
Ulf Adamsc0a84442017-03-21 10:08:03 +0000116
jcater36745912018-05-01 13:20:00 -0700117 for (Map.Entry<PathFragment, Map<PathFragment, Artifact>> rootAndMappings :
Ulf Adamsc0a84442017-03-21 10:08:03 +0000118 rootsAndMappings.entrySet()) {
119 PathFragment root = rootAndMappings.getKey();
120 Preconditions.checkState(!root.isAbsolute(), root);
jcater36745912018-05-01 13:20:00 -0700121 for (Map.Entry<PathFragment, Artifact> mapping : rootAndMappings.getValue().entrySet()) {
Ulf Adamsc0a84442017-03-21 10:08:03 +0000122 PathFragment location = root.getRelative(mapping.getKey());
123 Artifact localArtifact = mapping.getValue();
124 if (localArtifact != null) {
buchgrd4d3d502018-08-02 06:47:19 -0700125 Preconditions.checkState(!localArtifact.isMiddlemanArtifact());
philwo9ed9d8a2018-09-03 07:30:26 -0700126 if (expandTreeArtifactsInRunfiles && localArtifact.isTreeArtifact()) {
buchgrd4d3d502018-08-02 06:47:19 -0700127 List<ActionInput> expandedInputs =
128 ActionInputHelper.expandArtifacts(
129 Collections.singletonList(localArtifact), artifactExpander);
130 for (ActionInput input : expandedInputs) {
131 addMapping(
132 inputMap,
133 location.getRelative(((TreeFileArtifact) input).getParentRelativePath()),
134 input);
135 }
136 } else {
137 if (strict) {
138 failIfDirectory(actionFileCache, localArtifact);
139 }
140 addMapping(inputMap, location, localArtifact);
Ulf Adamsc0a84442017-03-21 10:08:03 +0000141 }
Ulf Adamsc0a84442017-03-21 10:08:03 +0000142 } else {
143 addMapping(inputMap, location, EMPTY_FILE);
144 }
145 }
146 }
147 }
148
buchgrd4d3d502018-08-02 06:47:19 -0700149 private static void failIfDirectory(MetadataProvider actionFileCache, ActionInput input)
150 throws IOException {
151 FileArtifactValue metadata = actionFileCache.getMetadata(input);
152 if (metadata != null && !metadata.getType().isFile()) {
153 throw new IOException("Not a file: " + input.getExecPathString());
154 }
155 }
156
ulfjack1973be42018-05-30 03:25:35 -0700157 @VisibleForTesting
158 void addFilesetManifests(
tomlu73eccc22018-09-06 08:02:37 -0700159 Map<Artifact, ImmutableList<FilesetOutputSymlink>> filesetMappings,
kush2ce45a22018-05-02 14:15:37 -0700160 Map<PathFragment, ActionInput> inputMappings)
161 throws IOException {
tomlu73eccc22018-09-06 08:02:37 -0700162 for (Artifact fileset : filesetMappings.keySet()) {
163 ImmutableList<FilesetOutputSymlink> outputSymlinks = filesetMappings.get(fileset);
kush2ce45a22018-05-02 14:15:37 -0700164 FilesetManifest filesetManifest =
ulfjack1973be42018-05-30 03:25:35 -0700165 FilesetManifest.constructFilesetManifest(
Googler9dc3e792018-10-17 05:16:47 -0700166 outputSymlinks, fileset.getExecPath(), relSymlinkBehavior);
kush2ce45a22018-05-02 14:15:37 -0700167
168 for (Map.Entry<PathFragment, String> mapping : filesetManifest.getEntries().entrySet()) {
169 String value = mapping.getValue();
Googler9dc3e792018-10-17 05:16:47 -0700170 ActionInput artifact =
171 value == null
172 ? EMPTY_FILE
173 : ActionInputHelper.fromPath(execRoot.getRelative(value).getPathString());
kush2ce45a22018-05-02 14:15:37 -0700174 addMapping(inputMappings, mapping.getKey(), artifact);
175 }
176 }
177 }
178
Ulf Adamsc0a84442017-03-21 10:08:03 +0000179 private void addInputs(
180 Map<PathFragment, ActionInput> inputMap, Spawn spawn, ArtifactExpander artifactExpander) {
181 List<ActionInput> inputs =
182 ActionInputHelper.expandArtifacts(spawn.getInputFiles(), artifactExpander);
183 for (ActionInput input : inputs) {
184 addMapping(inputMap, input.getExecPath(), input);
185 }
186 }
187
188 /**
buchgrd4d3d502018-08-02 06:47:19 -0700189 * Convert the inputs and runfiles of the given spawn to a map from exec-root relative paths to
philwo9ed9d8a2018-09-03 07:30:26 -0700190 * {@link ActionInput}s. The returned map does not contain tree artifacts as they are expanded to
191 * file artifacts.
buchgrd4d3d502018-08-02 06:47:19 -0700192 *
193 * <p>The returned map never contains {@code null} values; it uses {@link #EMPTY_FILE} for empty
kush2ce45a22018-05-02 14:15:37 -0700194 * files, which is an instance of {@link
195 * com.google.devtools.build.lib.actions.cache.VirtualActionInput}.
buchgrd4d3d502018-08-02 06:47:19 -0700196 *
197 * <p>The returned map contains all runfiles, but not the {@code MANIFEST}.
Ulf Adamsc0a84442017-03-21 10:08:03 +0000198 */
199 public SortedMap<PathFragment, ActionInput> getInputMapping(
philwo9ed9d8a2018-09-03 07:30:26 -0700200 Spawn spawn,
201 ArtifactExpander artifactExpander,
felly56fd4fe2018-09-05 10:54:54 -0700202 ArtifactPathResolver pathResolver,
philwo9ed9d8a2018-09-03 07:30:26 -0700203 MetadataProvider actionInputFileCache,
204 boolean expandTreeArtifactsInRunfiles)
kush2ce45a22018-05-02 14:15:37 -0700205 throws IOException {
Jakob Buchgraber4b3c2eb2019-04-04 02:01:48 -0700206
Ulf Adamsc0a84442017-03-21 10:08:03 +0000207 TreeMap<PathFragment, ActionInput> inputMap = new TreeMap<>();
208 addInputs(inputMap, spawn, artifactExpander);
209 addRunfilesToInputs(
philwo9ed9d8a2018-09-03 07:30:26 -0700210 inputMap,
211 spawn.getRunfilesSupplier(),
212 actionInputFileCache,
213 artifactExpander,
felly56fd4fe2018-09-05 10:54:54 -0700214 pathResolver,
philwo9ed9d8a2018-09-03 07:30:26 -0700215 expandTreeArtifactsInRunfiles);
kush2ce45a22018-05-02 14:15:37 -0700216 addFilesetManifests(spawn.getFilesetMappings(), inputMap);
Ulf Adamsc0a84442017-03-21 10:08:03 +0000217 return inputMap;
218 }
219}