Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 1 | // Copyright 2017 The Bazel Authors. All rights reserved. |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | package com.google.devtools.build.lib.exec; |
| 15 | |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 16 | import com.google.common.annotations.VisibleForTesting; |
| 17 | import com.google.common.base.Preconditions; |
kush | 2ce45a2 | 2018-05-02 14:15:37 -0700 | [diff] [blame] | 18 | import com.google.common.collect.ImmutableList; |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 19 | import com.google.devtools.build.lib.actions.ActionInput; |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 20 | import com.google.devtools.build.lib.actions.ActionInputHelper; |
| 21 | import com.google.devtools.build.lib.actions.Artifact; |
| 22 | import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander; |
buchgr | d4d3d50 | 2018-08-02 06:47:19 -0700 | [diff] [blame] | 23 | import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact; |
felly | 56fd4fe | 2018-09-05 10:54:54 -0700 | [diff] [blame] | 24 | import com.google.devtools.build.lib.actions.ArtifactPathResolver; |
buchgr | d4d3d50 | 2018-08-02 06:47:19 -0700 | [diff] [blame] | 25 | import com.google.devtools.build.lib.actions.FileArtifactValue; |
kush | 2ce45a2 | 2018-05-02 14:15:37 -0700 | [diff] [blame] | 26 | import com.google.devtools.build.lib.actions.FilesetOutputSymlink; |
shahan | 499503b | 2018-06-07 18:57:07 -0700 | [diff] [blame] | 27 | import com.google.devtools.build.lib.actions.MetadataProvider; |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 28 | import com.google.devtools.build.lib.actions.RunfilesSupplier; |
| 29 | import com.google.devtools.build.lib.actions.Spawn; |
ulfjack | 3e28868 | 2018-01-08 09:31:17 -0800 | [diff] [blame] | 30 | import com.google.devtools.build.lib.actions.cache.VirtualActionInput.EmptyActionInput; |
ulfjack | 1973be4 | 2018-05-30 03:25:35 -0700 | [diff] [blame] | 31 | import com.google.devtools.build.lib.exec.FilesetManifest.RelativeSymlinkBehavior; |
tomlu | 1a19b62 | 2018-01-11 15:17:28 -0800 | [diff] [blame] | 32 | import com.google.devtools.build.lib.vfs.Path; |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 33 | import com.google.devtools.build.lib.vfs.PathFragment; |
| 34 | import java.io.IOException; |
buchgr | d4d3d50 | 2018-08-02 06:47:19 -0700 | [diff] [blame] | 35 | import java.util.Collections; |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 36 | import java.util.List; |
| 37 | import java.util.Map; |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 38 | import java.util.SortedMap; |
| 39 | import java.util.TreeMap; |
| 40 | |
| 41 | /** |
| 42 | * A helper class for spawn strategies to turn runfiles suppliers into input mappings. This class |
| 43 | * performs no I/O operations, but only rearranges the files according to how the runfiles should be |
| 44 | * laid out. |
| 45 | */ |
| 46 | public class SpawnInputExpander { |
buchgr | d4d3d50 | 2018-08-02 06:47:19 -0700 | [diff] [blame] | 47 | @VisibleForTesting static final ActionInput EMPTY_FILE = new EmptyActionInput("/dev/null"); |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 48 | |
tomlu | 1a19b62 | 2018-01-11 15:17:28 -0800 | [diff] [blame] | 49 | private final Path execRoot; |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 50 | private final boolean strict; |
ulfjack | 1973be4 | 2018-05-30 03:25:35 -0700 | [diff] [blame] | 51 | private final RelativeSymlinkBehavior relSymlinkBehavior; |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 52 | |
| 53 | /** |
| 54 | * Creates a new instance. If strict is true, then the expander checks for directories in runfiles |
| 55 | * and throws an exception if it finds any. Otherwise it silently ignores directories in runfiles |
tomlu | 1a19b62 | 2018-01-11 15:17:28 -0800 | [diff] [blame] | 56 | * and adds a mapping for them. At this time, directories in filesets are always silently added as |
| 57 | * mappings. |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 58 | * |
| 59 | * <p>Directories in inputs are a correctness issue: Bazel only tracks dependencies at the action |
| 60 | * level, and it does not track dependencies on directories. Making a directory available to a |
| 61 | * spawn even though it's contents are not tracked as dependencies leads to incorrect incremental |
| 62 | * builds, since changes to the contents do not trigger action invalidation. |
| 63 | * |
| 64 | * <p>As such, all spawn strategies should always be strict and not make directories available to |
| 65 | * the subprocess. However, that's a breaking change, and therefore we make it depend on this flag |
| 66 | * for now. |
| 67 | */ |
tomlu | 1a19b62 | 2018-01-11 15:17:28 -0800 | [diff] [blame] | 68 | public SpawnInputExpander(Path execRoot, boolean strict) { |
ulfjack | 1973be4 | 2018-05-30 03:25:35 -0700 | [diff] [blame] | 69 | this(execRoot, strict, RelativeSymlinkBehavior.ERROR); |
| 70 | } |
| 71 | |
| 72 | /** |
| 73 | * Creates a new instance. If strict is true, then the expander checks for directories in runfiles |
| 74 | * and throws an exception if it finds any. Otherwise it silently ignores directories in runfiles |
| 75 | * and adds a mapping for them. At this time, directories in filesets are always silently added as |
| 76 | * mappings. |
| 77 | * |
| 78 | * <p>Directories in inputs are a correctness issue: Bazel only tracks dependencies at the action |
| 79 | * level, and it does not track dependencies on directories. Making a directory available to a |
| 80 | * spawn even though it's contents are not tracked as dependencies leads to incorrect incremental |
| 81 | * builds, since changes to the contents do not trigger action invalidation. |
| 82 | * |
| 83 | * <p>As such, all spawn strategies should always be strict and not make directories available to |
| 84 | * the subprocess. However, that's a breaking change, and therefore we make it depend on this flag |
| 85 | * for now. |
| 86 | */ |
| 87 | public SpawnInputExpander( |
| 88 | Path execRoot, boolean strict, RelativeSymlinkBehavior relSymlinkBehavior) { |
tomlu | 1a19b62 | 2018-01-11 15:17:28 -0800 | [diff] [blame] | 89 | this.execRoot = execRoot; |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 90 | this.strict = strict; |
ulfjack | 1973be4 | 2018-05-30 03:25:35 -0700 | [diff] [blame] | 91 | this.relSymlinkBehavior = relSymlinkBehavior; |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 92 | } |
| 93 | |
| 94 | private void addMapping( |
| 95 | Map<PathFragment, ActionInput> inputMappings, |
| 96 | PathFragment targetLocation, |
| 97 | ActionInput input) { |
| 98 | Preconditions.checkArgument(!targetLocation.isAbsolute(), targetLocation); |
| 99 | if (!inputMappings.containsKey(targetLocation)) { |
| 100 | inputMappings.put(targetLocation, input); |
| 101 | } |
| 102 | } |
| 103 | |
| 104 | /** Adds runfiles inputs from runfilesSupplier to inputMappings. */ |
| 105 | @VisibleForTesting |
| 106 | void addRunfilesToInputs( |
| 107 | Map<PathFragment, ActionInput> inputMap, |
| 108 | RunfilesSupplier runfilesSupplier, |
buchgr | d4d3d50 | 2018-08-02 06:47:19 -0700 | [diff] [blame] | 109 | MetadataProvider actionFileCache, |
philwo | 9ed9d8a | 2018-09-03 07:30:26 -0700 | [diff] [blame] | 110 | ArtifactExpander artifactExpander, |
felly | 56fd4fe | 2018-09-05 10:54:54 -0700 | [diff] [blame] | 111 | ArtifactPathResolver pathResolver, |
philwo | 9ed9d8a | 2018-09-03 07:30:26 -0700 | [diff] [blame] | 112 | boolean expandTreeArtifactsInRunfiles) |
shahan | 499503b | 2018-06-07 18:57:07 -0700 | [diff] [blame] | 113 | throws IOException { |
buchgr | d4d3d50 | 2018-08-02 06:47:19 -0700 | [diff] [blame] | 114 | Map<PathFragment, Map<PathFragment, Artifact>> rootsAndMappings = |
felly | 56fd4fe | 2018-09-05 10:54:54 -0700 | [diff] [blame] | 115 | runfilesSupplier.getMappings(pathResolver); |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 116 | |
jcater | 3674591 | 2018-05-01 13:20:00 -0700 | [diff] [blame] | 117 | for (Map.Entry<PathFragment, Map<PathFragment, Artifact>> rootAndMappings : |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 118 | rootsAndMappings.entrySet()) { |
| 119 | PathFragment root = rootAndMappings.getKey(); |
| 120 | Preconditions.checkState(!root.isAbsolute(), root); |
jcater | 3674591 | 2018-05-01 13:20:00 -0700 | [diff] [blame] | 121 | for (Map.Entry<PathFragment, Artifact> mapping : rootAndMappings.getValue().entrySet()) { |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 122 | PathFragment location = root.getRelative(mapping.getKey()); |
| 123 | Artifact localArtifact = mapping.getValue(); |
| 124 | if (localArtifact != null) { |
buchgr | d4d3d50 | 2018-08-02 06:47:19 -0700 | [diff] [blame] | 125 | Preconditions.checkState(!localArtifact.isMiddlemanArtifact()); |
philwo | 9ed9d8a | 2018-09-03 07:30:26 -0700 | [diff] [blame] | 126 | if (expandTreeArtifactsInRunfiles && localArtifact.isTreeArtifact()) { |
buchgr | d4d3d50 | 2018-08-02 06:47:19 -0700 | [diff] [blame] | 127 | List<ActionInput> expandedInputs = |
| 128 | ActionInputHelper.expandArtifacts( |
| 129 | Collections.singletonList(localArtifact), artifactExpander); |
| 130 | for (ActionInput input : expandedInputs) { |
| 131 | addMapping( |
| 132 | inputMap, |
| 133 | location.getRelative(((TreeFileArtifact) input).getParentRelativePath()), |
| 134 | input); |
| 135 | } |
| 136 | } else { |
| 137 | if (strict) { |
| 138 | failIfDirectory(actionFileCache, localArtifact); |
| 139 | } |
| 140 | addMapping(inputMap, location, localArtifact); |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 141 | } |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 142 | } else { |
| 143 | addMapping(inputMap, location, EMPTY_FILE); |
| 144 | } |
| 145 | } |
| 146 | } |
| 147 | } |
| 148 | |
buchgr | d4d3d50 | 2018-08-02 06:47:19 -0700 | [diff] [blame] | 149 | private static void failIfDirectory(MetadataProvider actionFileCache, ActionInput input) |
| 150 | throws IOException { |
| 151 | FileArtifactValue metadata = actionFileCache.getMetadata(input); |
| 152 | if (metadata != null && !metadata.getType().isFile()) { |
| 153 | throw new IOException("Not a file: " + input.getExecPathString()); |
| 154 | } |
| 155 | } |
| 156 | |
ulfjack | 1973be4 | 2018-05-30 03:25:35 -0700 | [diff] [blame] | 157 | @VisibleForTesting |
| 158 | void addFilesetManifests( |
tomlu | 73eccc2 | 2018-09-06 08:02:37 -0700 | [diff] [blame] | 159 | Map<Artifact, ImmutableList<FilesetOutputSymlink>> filesetMappings, |
kush | 2ce45a2 | 2018-05-02 14:15:37 -0700 | [diff] [blame] | 160 | Map<PathFragment, ActionInput> inputMappings) |
| 161 | throws IOException { |
tomlu | 73eccc2 | 2018-09-06 08:02:37 -0700 | [diff] [blame] | 162 | for (Artifact fileset : filesetMappings.keySet()) { |
| 163 | ImmutableList<FilesetOutputSymlink> outputSymlinks = filesetMappings.get(fileset); |
kush | 2ce45a2 | 2018-05-02 14:15:37 -0700 | [diff] [blame] | 164 | FilesetManifest filesetManifest = |
ulfjack | 1973be4 | 2018-05-30 03:25:35 -0700 | [diff] [blame] | 165 | FilesetManifest.constructFilesetManifest( |
Googler | 9dc3e79 | 2018-10-17 05:16:47 -0700 | [diff] [blame] | 166 | outputSymlinks, fileset.getExecPath(), relSymlinkBehavior); |
kush | 2ce45a2 | 2018-05-02 14:15:37 -0700 | [diff] [blame] | 167 | |
| 168 | for (Map.Entry<PathFragment, String> mapping : filesetManifest.getEntries().entrySet()) { |
| 169 | String value = mapping.getValue(); |
Googler | 9dc3e79 | 2018-10-17 05:16:47 -0700 | [diff] [blame] | 170 | ActionInput artifact = |
| 171 | value == null |
| 172 | ? EMPTY_FILE |
| 173 | : ActionInputHelper.fromPath(execRoot.getRelative(value).getPathString()); |
kush | 2ce45a2 | 2018-05-02 14:15:37 -0700 | [diff] [blame] | 174 | addMapping(inputMappings, mapping.getKey(), artifact); |
| 175 | } |
| 176 | } |
| 177 | } |
| 178 | |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 179 | private void addInputs( |
| 180 | Map<PathFragment, ActionInput> inputMap, Spawn spawn, ArtifactExpander artifactExpander) { |
| 181 | List<ActionInput> inputs = |
| 182 | ActionInputHelper.expandArtifacts(spawn.getInputFiles(), artifactExpander); |
| 183 | for (ActionInput input : inputs) { |
| 184 | addMapping(inputMap, input.getExecPath(), input); |
| 185 | } |
| 186 | } |
| 187 | |
| 188 | /** |
buchgr | d4d3d50 | 2018-08-02 06:47:19 -0700 | [diff] [blame] | 189 | * Convert the inputs and runfiles of the given spawn to a map from exec-root relative paths to |
philwo | 9ed9d8a | 2018-09-03 07:30:26 -0700 | [diff] [blame] | 190 | * {@link ActionInput}s. The returned map does not contain tree artifacts as they are expanded to |
| 191 | * file artifacts. |
buchgr | d4d3d50 | 2018-08-02 06:47:19 -0700 | [diff] [blame] | 192 | * |
| 193 | * <p>The returned map never contains {@code null} values; it uses {@link #EMPTY_FILE} for empty |
kush | 2ce45a2 | 2018-05-02 14:15:37 -0700 | [diff] [blame] | 194 | * files, which is an instance of {@link |
| 195 | * com.google.devtools.build.lib.actions.cache.VirtualActionInput}. |
buchgr | d4d3d50 | 2018-08-02 06:47:19 -0700 | [diff] [blame] | 196 | * |
| 197 | * <p>The returned map contains all runfiles, but not the {@code MANIFEST}. |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 198 | */ |
| 199 | public SortedMap<PathFragment, ActionInput> getInputMapping( |
philwo | 9ed9d8a | 2018-09-03 07:30:26 -0700 | [diff] [blame] | 200 | Spawn spawn, |
| 201 | ArtifactExpander artifactExpander, |
felly | 56fd4fe | 2018-09-05 10:54:54 -0700 | [diff] [blame] | 202 | ArtifactPathResolver pathResolver, |
philwo | 9ed9d8a | 2018-09-03 07:30:26 -0700 | [diff] [blame] | 203 | MetadataProvider actionInputFileCache, |
| 204 | boolean expandTreeArtifactsInRunfiles) |
kush | 2ce45a2 | 2018-05-02 14:15:37 -0700 | [diff] [blame] | 205 | throws IOException { |
Jakob Buchgraber | 4b3c2eb | 2019-04-04 02:01:48 -0700 | [diff] [blame] | 206 | |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 207 | TreeMap<PathFragment, ActionInput> inputMap = new TreeMap<>(); |
| 208 | addInputs(inputMap, spawn, artifactExpander); |
| 209 | addRunfilesToInputs( |
philwo | 9ed9d8a | 2018-09-03 07:30:26 -0700 | [diff] [blame] | 210 | inputMap, |
| 211 | spawn.getRunfilesSupplier(), |
| 212 | actionInputFileCache, |
| 213 | artifactExpander, |
felly | 56fd4fe | 2018-09-05 10:54:54 -0700 | [diff] [blame] | 214 | pathResolver, |
philwo | 9ed9d8a | 2018-09-03 07:30:26 -0700 | [diff] [blame] | 215 | expandTreeArtifactsInRunfiles); |
kush | 2ce45a2 | 2018-05-02 14:15:37 -0700 | [diff] [blame] | 216 | addFilesetManifests(spawn.getFilesetMappings(), inputMap); |
Ulf Adams | c0a8444 | 2017-03-21 10:08:03 +0000 | [diff] [blame] | 217 | return inputMap; |
| 218 | } |
| 219 | } |