Damien Martin-Guillerez | f88f4d8 | 2015-09-25 13:56:55 +0000 | [diff] [blame] | 1 | // Copyright 2014 The Bazel Authors. All rights reserved. |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | package com.google.devtools.build.lib.skyframe; |
| 15 | |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 16 | import static java.nio.charset.StandardCharsets.UTF_8; |
| 17 | |
Googler | 5284e6c | 2019-10-30 07:43:54 -0700 | [diff] [blame] | 18 | import com.google.common.annotations.VisibleForTesting; |
tomlu | a155b53 | 2017-11-08 20:12:47 +0100 | [diff] [blame] | 19 | import com.google.common.base.Preconditions; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 20 | import com.google.common.base.Verify; |
| 21 | import com.google.common.collect.Collections2; |
Googler | c8acdbd | 2018-11-08 10:47:15 -0800 | [diff] [blame] | 22 | import com.google.common.collect.ImmutableList; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 23 | import com.google.devtools.build.lib.actions.Artifact; |
Googler | c8acdbd | 2018-11-08 10:47:15 -0800 | [diff] [blame] | 24 | import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact; |
shahan | 602cc85 | 2018-06-06 20:09:57 -0700 | [diff] [blame] | 25 | import com.google.devtools.build.lib.actions.FileArtifactValue; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 26 | import com.google.devtools.build.lib.actions.FileStateType; |
| 27 | import com.google.devtools.build.lib.actions.FileStateValue; |
Googler | 5284e6c | 2019-10-30 07:43:54 -0700 | [diff] [blame] | 28 | import com.google.devtools.build.lib.actions.FileStateValue.RegularFileStateValue; |
shahan | 602cc85 | 2018-06-06 20:09:57 -0700 | [diff] [blame] | 29 | import com.google.devtools.build.lib.actions.FileValue; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 30 | import com.google.devtools.build.lib.actions.HasDigest; |
Laszlo Csomor | 207140f | 2015-12-07 15:07:33 +0000 | [diff] [blame] | 31 | import com.google.devtools.build.lib.collect.nestedset.NestedSet; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 32 | import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; |
| 33 | import com.google.devtools.build.lib.events.Event; |
janakr | e2af68f | 2021-03-18 15:11:30 -0700 | [diff] [blame] | 34 | import com.google.devtools.build.lib.io.FileSymlinkException; |
| 35 | import com.google.devtools.build.lib.io.FileSymlinkInfiniteExpansionException; |
| 36 | import com.google.devtools.build.lib.io.FileSymlinkInfiniteExpansionUniquenessFunction; |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 37 | import com.google.devtools.build.lib.packages.BuildFileNotFoundException; |
twerth | 45d82ca | 2021-02-24 02:28:54 -0800 | [diff] [blame] | 38 | import com.google.devtools.build.lib.profiler.Profiler; |
| 39 | import com.google.devtools.build.lib.profiler.ProfilerTask; |
| 40 | import com.google.devtools.build.lib.profiler.SilentCloseable; |
janakr | a4a564a | 2021-03-18 12:40:11 -0700 | [diff] [blame] | 41 | import com.google.devtools.build.lib.server.FailureDetails; |
jcater | b11c6e2 | 2020-04-03 11:40:16 -0700 | [diff] [blame] | 42 | import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.FileType; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 43 | import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFile; |
Laszlo Csomor | b4d482b | 2015-12-04 13:23:54 +0000 | [diff] [blame] | 44 | import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFileFactory; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 45 | import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.TraversalRequest; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 46 | import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; |
Googler | 0129a24 | 2021-12-01 09:04:45 -0800 | [diff] [blame] | 47 | import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 48 | import com.google.devtools.build.lib.util.Fingerprint; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 49 | import com.google.devtools.build.lib.vfs.Dirent; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 50 | import com.google.devtools.build.lib.vfs.FileStatus; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 51 | import com.google.devtools.build.lib.vfs.Path; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 52 | import com.google.devtools.build.lib.vfs.PathFragment; |
tomlu | ee6a686 | 2018-01-17 14:36:26 -0800 | [diff] [blame] | 53 | import com.google.devtools.build.lib.vfs.Root; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 54 | import com.google.devtools.build.lib.vfs.RootedPath; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 55 | import com.google.devtools.build.lib.vfs.Symlinks; |
janakr | fc1d79c | 2022-01-27 13:02:07 -0800 | [diff] [blame] | 56 | import com.google.devtools.build.lib.vfs.SyscallCache; |
janakr | e285322 | 2022-03-08 10:52:18 -0800 | [diff] [blame] | 57 | import com.google.devtools.build.lib.vfs.XattrProvider; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 58 | import com.google.devtools.build.skyframe.SkyFunction; |
| 59 | import com.google.devtools.build.skyframe.SkyFunctionException; |
| 60 | import com.google.devtools.build.skyframe.SkyKey; |
| 61 | import com.google.devtools.build.skyframe.SkyValue; |
kush | 95bf7c8 | 2017-08-30 00:27:35 +0200 | [diff] [blame] | 62 | import java.io.IOException; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 63 | import java.math.BigInteger; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 64 | import java.util.ArrayList; |
| 65 | import java.util.Collection; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 66 | import java.util.Collections; |
| 67 | import java.util.HashMap; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 68 | import java.util.List; |
| 69 | import java.util.Map; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 70 | import javax.annotation.Nullable; |
| 71 | |
| 72 | /** A {@link SkyFunction} to build {@link RecursiveFilesystemTraversalValue}s. */ |
| 73 | public final class RecursiveFilesystemTraversalFunction implements SkyFunction { |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 74 | private static final byte[] MISSING_FINGERPRINT = |
| 75 | new BigInteger(1, "NonexistentFileStateValue".getBytes(UTF_8)).toByteArray(); |
| 76 | |
Googler | 0129a24 | 2021-12-01 09:04:45 -0800 | [diff] [blame] | 77 | @SerializationConstant @AutoCodec.VisibleForSerialization |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 78 | static final HasDigest NON_EXISTENT_HAS_DIGEST = () -> MISSING_FINGERPRINT; |
| 79 | |
| 80 | private static final FileInfo NON_EXISTENT_FILE_INFO = |
| 81 | new FileInfo(FileType.NONEXISTENT, NON_EXISTENT_HAS_DIGEST, null, null); |
| 82 | |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 83 | /** The exception that {@link RecursiveFilesystemTraversalFunctionException} wraps. */ |
| 84 | public static class RecursiveFilesystemTraversalException extends Exception { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 85 | |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 86 | /** |
| 87 | * Categories of errors that prevent normal {@link RecursiveFilesystemTraversalFunction} |
| 88 | * evaluation. |
| 89 | */ |
| 90 | public enum Type { |
| 91 | /** |
| 92 | * The traversal encountered a subdirectory with a BUILD file but is not allowed to recurse |
| 93 | * into it. See {@code PackageBoundaryMode#REPORT_ERROR}. |
| 94 | */ |
| 95 | CANNOT_CROSS_PACKAGE_BOUNDARY, |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 96 | |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 97 | /** A dangling symlink was dereferenced. */ |
| 98 | DANGLING_SYMLINK, |
| 99 | |
| 100 | /** A file operation failed. */ |
| 101 | FILE_OPERATION_FAILURE, |
| 102 | |
| 103 | /** A generated directory's root-relative path conflicts with a package's path. */ |
| 104 | GENERATED_PATH_CONFLICT, |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 105 | |
| 106 | /** A file/directory visited was part of a symlink cycle or infinite expansion. */ |
| 107 | SYMLINK_CYCLE_OR_INFINITE_EXPANSION, |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 108 | } |
| 109 | |
| 110 | private final Type type; |
| 111 | |
| 112 | RecursiveFilesystemTraversalException(String message, Type type) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 113 | super(message); |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 114 | this.type = type; |
| 115 | } |
| 116 | |
| 117 | public Type getType() { |
| 118 | return type; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 119 | } |
| 120 | } |
| 121 | |
| 122 | /** |
| 123 | * Thrown when a dangling symlink is attempted to be dereferenced. |
| 124 | * |
| 125 | * <p>Note: this class is not identical to the one in com.google.devtools.build.lib.view.fileset |
| 126 | * and it's not easy to merge the two because of the dependency structure. The other one will |
| 127 | * probably be removed along with the rest of the legacy Fileset code. |
| 128 | */ |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 129 | static final class DanglingSymlinkException extends RecursiveFilesystemTraversalException { |
| 130 | DanglingSymlinkException(String path, String unresolvedLink) { |
Laszlo Csomor | 0ad729f | 2015-12-02 15:20:35 +0000 | [diff] [blame] | 131 | super( |
| 132 | String.format( |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 133 | "Found dangling symlink: %s, unresolved path: \"%s\"", path, unresolvedLink), |
| 134 | Type.DANGLING_SYMLINK); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 135 | Preconditions.checkArgument(path != null && !path.isEmpty()); |
| 136 | Preconditions.checkArgument(unresolvedLink != null && !unresolvedLink.isEmpty()); |
kush | 95bf7c8 | 2017-08-30 00:27:35 +0200 | [diff] [blame] | 137 | } |
| 138 | } |
| 139 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 140 | /** Exception type thrown by {@link RecursiveFilesystemTraversalFunction#compute}. */ |
| 141 | private static final class RecursiveFilesystemTraversalFunctionException extends |
| 142 | SkyFunctionException { |
| 143 | RecursiveFilesystemTraversalFunctionException(RecursiveFilesystemTraversalException e) { |
| 144 | super(e, Transience.PERSISTENT); |
| 145 | } |
| 146 | } |
| 147 | |
janakr | b2a9434 | 2022-02-05 22:02:14 -0800 | [diff] [blame] | 148 | private final SyscallCache syscallCache; |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 149 | |
janakr | b2a9434 | 2022-02-05 22:02:14 -0800 | [diff] [blame] | 150 | RecursiveFilesystemTraversalFunction(SyscallCache syscallCache) { |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 151 | this.syscallCache = syscallCache; |
| 152 | } |
| 153 | |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 154 | @Nullable |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 155 | @Override |
| 156 | public SkyValue compute(SkyKey skyKey, Environment env) |
Janak Ramakrishnan | 3c0adb2 | 2016-08-15 21:54:55 +0000 | [diff] [blame] | 157 | throws RecursiveFilesystemTraversalFunctionException, InterruptedException { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 158 | TraversalRequest traversal = (TraversalRequest) skyKey.argument(); |
twerth | 45d82ca | 2021-02-24 02:28:54 -0800 | [diff] [blame] | 159 | try (SilentCloseable c = |
| 160 | Profiler.instance() |
| 161 | .profile(ProfilerTask.FILESYSTEM_TRAVERSAL, traversal.getRoot().toString())) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 162 | // Stat the traversal root. |
janakr | b2a9434 | 2022-02-05 22:02:14 -0800 | [diff] [blame] | 163 | FileInfo rootInfo = lookUpFileInfo(env, traversal, syscallCache); |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 164 | if (rootInfo == null) { |
| 165 | return null; |
| 166 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 167 | |
| 168 | if (!rootInfo.type.exists()) { |
| 169 | // May be a dangling symlink or a non-existent file. Handle gracefully. |
| 170 | if (rootInfo.type.isSymlink()) { |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 171 | return resultForDanglingSymlink(traversal.root.asRootedPath(), rootInfo); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 172 | } else { |
| 173 | return RecursiveFilesystemTraversalValue.EMPTY; |
| 174 | } |
| 175 | } |
| 176 | |
| 177 | if (rootInfo.type.isFile()) { |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 178 | return resultForFileRoot(traversal.root.asRootedPath(), rootInfo); |
Googler | c8acdbd | 2018-11-08 10:47:15 -0800 | [diff] [blame] | 179 | } else if (rootInfo.type.isDirectory() && rootInfo.metadata instanceof TreeArtifactValue) { |
| 180 | final TreeArtifactValue value = (TreeArtifactValue) rootInfo.metadata; |
| 181 | ImmutableList.Builder<RecursiveFilesystemTraversalValue> list = ImmutableList.builder(); |
| 182 | for (Map.Entry<TreeFileArtifact, FileArtifactValue> entry |
| 183 | : value.getChildValues().entrySet()) { |
| 184 | RootedPath path = |
| 185 | RootedPath.toRootedPath( |
| 186 | traversal.root.asRootedPath().getRoot(), entry.getKey().getPath()); |
jcater | b11c6e2 | 2020-04-03 11:40:16 -0700 | [diff] [blame] | 187 | list.add( |
| 188 | resultForFileRoot( |
| 189 | path, |
| 190 | // TreeArtifact can't have symbolic inside. So the assumption for FileType.FILE |
| 191 | // is always true. |
| 192 | new FileInfo(FileType.FILE, entry.getValue(), path, null))); |
Googler | c8acdbd | 2018-11-08 10:47:15 -0800 | [diff] [blame] | 193 | } |
| 194 | return resultForDirectory(traversal, rootInfo, list.build()); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 195 | } |
| 196 | |
| 197 | // Otherwise the root is a directory or a symlink to one. |
janakr | b2a9434 | 2022-02-05 22:02:14 -0800 | [diff] [blame] | 198 | PkgLookupResult pkgLookupResult = checkIfPackage(env, traversal, rootInfo, syscallCache); |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 199 | if (pkgLookupResult == null) { |
| 200 | return null; |
| 201 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 202 | traversal = pkgLookupResult.traversal; |
| 203 | |
| 204 | if (pkgLookupResult.isConflicting()) { |
| 205 | // The traversal was requested for an output directory whose root-relative path conflicts |
| 206 | // with a source package. We can't handle that, bail out. |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 207 | throw createGeneratedPathConflictException(traversal); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 208 | } else if (pkgLookupResult.isPackage() && !traversal.skipTestingForSubpackage) { |
| 209 | // The traversal was requested for a directory that defines a package. |
tomlu | 8cc5dcf | 2018-01-19 09:28:06 -0800 | [diff] [blame] | 210 | String msg = |
| 211 | traversal.errorInfo |
| 212 | + " crosses package boundary into package rooted at " |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 213 | + traversal.root.asRootedPath().getRootRelativePath().getPathString(); |
Laszlo Csomor | f04efcc | 2015-02-12 17:08:06 +0000 | [diff] [blame] | 214 | switch (traversal.crossPkgBoundaries) { |
| 215 | case CROSS: |
| 216 | // We are free to traverse the subpackage but we need to display a warning. |
Ulf Adams | 760e709 | 2016-04-21 08:09:51 +0000 | [diff] [blame] | 217 | env.getListener().handle(Event.warn(null, msg)); |
Laszlo Csomor | f04efcc | 2015-02-12 17:08:06 +0000 | [diff] [blame] | 218 | break; |
| 219 | case DONT_CROSS: |
| 220 | // We cannot traverse the subpackage and should skip it silently. Return empty results. |
| 221 | return RecursiveFilesystemTraversalValue.EMPTY; |
| 222 | case REPORT_ERROR: |
| 223 | // We cannot traverse the subpackage and should complain loudly (display an error). |
| 224 | throw new RecursiveFilesystemTraversalFunctionException( |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 225 | new RecursiveFilesystemTraversalException( |
| 226 | msg, RecursiveFilesystemTraversalException.Type.CANNOT_CROSS_PACKAGE_BOUNDARY)); |
Laszlo Csomor | f04efcc | 2015-02-12 17:08:06 +0000 | [diff] [blame] | 227 | default: |
| 228 | throw new IllegalStateException(traversal.toString()); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 229 | } |
| 230 | } |
| 231 | |
| 232 | // We are free to traverse this directory. |
| 233 | Collection<SkyKey> dependentKeys = createRecursiveTraversalKeys(env, traversal); |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 234 | if (dependentKeys == null) { |
| 235 | return null; |
| 236 | } |
| 237 | Collection<RecursiveFilesystemTraversalValue> subdirTraversals = |
| 238 | traverseChildren(env, dependentKeys, /*inline=*/ traversal.isRootGenerated); |
| 239 | if (subdirTraversals == null) { |
| 240 | return null; |
| 241 | } |
| 242 | return resultForDirectory(traversal, rootInfo, subdirTraversals); |
| 243 | } catch (IOException | BuildFileNotFoundException e) { |
| 244 | String message = |
| 245 | String.format( |
| 246 | "Error while traversing directory %s: %s", |
| 247 | traversal.root.getRelativePart(), e.getMessage()); |
janakr | a4a564a | 2021-03-18 12:40:11 -0700 | [diff] [blame] | 248 | // Trying to stat the starting point of this root may have failed with a symlink cycle or |
| 249 | // trying to get a package lookup value may have failed due to a symlink cycle. |
| 250 | RecursiveFilesystemTraversalException.Type exceptionType = |
| 251 | RecursiveFilesystemTraversalException.Type.FILE_OPERATION_FAILURE; |
| 252 | if (e instanceof FileSymlinkException) { |
| 253 | exceptionType = |
| 254 | RecursiveFilesystemTraversalException.Type.SYMLINK_CYCLE_OR_INFINITE_EXPANSION; |
| 255 | } |
| 256 | if (e instanceof DetailedException) { |
| 257 | FailureDetails.PackageLoading.Code code = |
| 258 | ((DetailedException) e) |
| 259 | .getDetailedExitCode() |
| 260 | .getFailureDetail() |
| 261 | .getPackageLoading() |
| 262 | .getCode(); |
| 263 | if (code == FailureDetails.PackageLoading.Code.SYMLINK_CYCLE_OR_INFINITE_EXPANSION) { |
| 264 | exceptionType = |
| 265 | RecursiveFilesystemTraversalException.Type.SYMLINK_CYCLE_OR_INFINITE_EXPANSION; |
| 266 | } |
| 267 | } |
kush | 95bf7c8 | 2017-08-30 00:27:35 +0200 | [diff] [blame] | 268 | throw new RecursiveFilesystemTraversalFunctionException( |
janakr | a4a564a | 2021-03-18 12:40:11 -0700 | [diff] [blame] | 269 | new RecursiveFilesystemTraversalException(message, exceptionType)); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 270 | } |
| 271 | } |
| 272 | |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 273 | private static RecursiveFilesystemTraversalFunctionException createGeneratedPathConflictException( |
| 274 | TraversalRequest traversal) { |
| 275 | String message = |
| 276 | String.format( |
| 277 | "Generated directory %s conflicts with package under the same path. " |
| 278 | + "Additional info: %s", |
| 279 | traversal.root.asRootedPath().getRootRelativePath().getPathString(), |
| 280 | traversal.errorInfo != null ? traversal.errorInfo : traversal.toString()); |
| 281 | return new RecursiveFilesystemTraversalFunctionException( |
| 282 | new RecursiveFilesystemTraversalException( |
| 283 | message, RecursiveFilesystemTraversalException.Type.GENERATED_PATH_CONFLICT)); |
| 284 | } |
| 285 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 286 | private static final class FileInfo { |
| 287 | final FileType type; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 288 | final HasDigest metadata; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 289 | @Nullable final RootedPath realPath; |
| 290 | @Nullable final PathFragment unresolvedSymlinkTarget; |
| 291 | |
kush | 4b120e7 | 2018-07-11 16:21:27 -0700 | [diff] [blame] | 292 | FileInfo( |
| 293 | FileType type, |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 294 | HasDigest metadata, |
kush | 4b120e7 | 2018-07-11 16:21:27 -0700 | [diff] [blame] | 295 | @Nullable RootedPath realPath, |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 296 | @Nullable PathFragment unresolvedSymlinkTarget) { |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 297 | Preconditions.checkNotNull(metadata.getDigest(), metadata); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 298 | this.type = Preconditions.checkNotNull(type); |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 299 | this.metadata = metadata; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 300 | this.realPath = realPath; |
| 301 | this.unresolvedSymlinkTarget = unresolvedSymlinkTarget; |
| 302 | } |
| 303 | |
| 304 | @Override |
| 305 | public String toString() { |
| 306 | if (type.isSymlink()) { |
| 307 | return String.format("(%s: link_value=%s, real_path=%s)", type, |
| 308 | unresolvedSymlinkTarget.getPathString(), realPath); |
| 309 | } else { |
| 310 | return String.format("(%s: real_path=%s)", type, realPath); |
| 311 | } |
| 312 | } |
| 313 | } |
| 314 | |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 315 | @Nullable |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 316 | private static FileInfo lookUpFileInfo( |
| 317 | Environment env, TraversalRequest traversal, SyscallCache syscallCache) |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 318 | throws IOException, InterruptedException { |
janakr | 5fb2a48 | 2018-03-02 17:48:57 -0800 | [diff] [blame] | 319 | if (traversal.isRootGenerated) { |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 320 | HasDigest fsVal = null; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 321 | if (traversal.root.getOutputArtifact() != null) { |
| 322 | Artifact artifact = traversal.root.getOutputArtifact(); |
lberki | 36df7ed | 2019-06-27 06:32:03 -0700 | [diff] [blame] | 323 | SkyKey artifactKey = Artifact.key(artifact); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 324 | SkyValue value = env.getValue(artifactKey); |
| 325 | if (env.valuesMissing()) { |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 326 | return null; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 327 | } |
kush | 95bf7c8 | 2017-08-30 00:27:35 +0200 | [diff] [blame] | 328 | |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 329 | if (value instanceof FileArtifactValue || value instanceof TreeArtifactValue) { |
| 330 | fsVal = (HasDigest) value; |
janakr | 8541f6d | 2019-06-11 14:40:21 -0700 | [diff] [blame] | 331 | } else if (value instanceof ActionExecutionValue) { |
Googler | aed4160 | 2020-06-02 12:22:53 -0700 | [diff] [blame] | 332 | fsVal = ((ActionExecutionValue) value).getExistingFileArtifactValue(artifact); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 333 | } else { |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 334 | return NON_EXISTENT_FILE_INFO; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 335 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 336 | } |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 337 | RootedPath realPath = traversal.root.asRootedPath(); |
felly | 6c399d6 | 2018-07-18 15:55:59 -0700 | [diff] [blame] | 338 | if (traversal.strictOutputFiles) { |
| 339 | Preconditions.checkNotNull(fsVal, "Strict Fileset output tree has null FileArtifactValue"); |
Googler | c8acdbd | 2018-11-08 10:47:15 -0800 | [diff] [blame] | 340 | return new FileInfo( |
| 341 | (fsVal instanceof TreeArtifactValue ? FileType.DIRECTORY : FileType.FILE), |
jcater | b11c6e2 | 2020-04-03 11:40:16 -0700 | [diff] [blame] | 342 | fsVal, |
| 343 | realPath, |
| 344 | null); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 345 | } else { |
felly | 6c399d6 | 2018-07-18 15:55:59 -0700 | [diff] [blame] | 346 | // FileArtifactValue does not currently track symlinks. If it did, we could potentially |
| 347 | // remove some of the filesystem operations we're doing here. |
| 348 | Path path = traversal.root.asRootedPath().asPath(); |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 349 | FileStateValue fileState = |
janakr | 841c04a | 2022-02-22 14:36:37 -0800 | [diff] [blame] | 350 | FileStateValue.create(traversal.root.asRootedPath(), syscallCache, null); |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 351 | if (fileState.getType() == FileStateType.NONEXISTENT) { |
| 352 | throw new IOException("Missing file: " + path); |
| 353 | } |
felly | 6c399d6 | 2018-07-18 15:55:59 -0700 | [diff] [blame] | 354 | FileStatus followStat = path.statIfFound(Symlinks.FOLLOW); |
| 355 | FileType type; |
| 356 | PathFragment unresolvedLinkTarget = null; |
| 357 | if (followStat == null) { |
| 358 | type = FileType.DANGLING_SYMLINK; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 359 | if (fileState.getType() != FileStateType.SYMLINK) { |
| 360 | throw new IOException("Expected symlink for " + path + ", but got: " + fileState); |
felly | 6c399d6 | 2018-07-18 15:55:59 -0700 | [diff] [blame] | 361 | } |
| 362 | unresolvedLinkTarget = path.readSymbolicLink(); |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 363 | } else if (fileState.getType() == FileStateType.REGULAR_FILE) { |
felly | 6c399d6 | 2018-07-18 15:55:59 -0700 | [diff] [blame] | 364 | type = FileType.FILE; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 365 | } else if (fileState.getType() == FileStateType.DIRECTORY) { |
felly | 6c399d6 | 2018-07-18 15:55:59 -0700 | [diff] [blame] | 366 | type = FileType.DIRECTORY; |
| 367 | } else { |
| 368 | unresolvedLinkTarget = path.readSymbolicLink(); |
| 369 | realPath = |
| 370 | RootedPath.toRootedPath( |
| 371 | Root.absoluteRoot(path.getFileSystem()), path.resolveSymbolicLinks()); |
| 372 | type = followStat.isFile() ? FileType.SYMLINK_TO_FILE : FileType.SYMLINK_TO_DIRECTORY; |
| 373 | } |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 374 | if (fsVal == null) { |
| 375 | fsVal = fileState; |
| 376 | } |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 377 | return new FileInfo( |
| 378 | type, withDigest(fsVal, path, syscallCache), realPath, unresolvedLinkTarget); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 379 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 380 | } else { |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 381 | // Stat the file. |
| 382 | FileValue fileValue = |
| 383 | (FileValue) env.getValueOrThrow( |
| 384 | FileValue.key(traversal.root.asRootedPath()), IOException.class); |
| 385 | |
| 386 | if (env.valuesMissing()) { |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 387 | return null; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 388 | } |
lberki | 7598bc6 | 2020-10-02 01:33:14 -0700 | [diff] [blame] | 389 | if (fileValue.unboundedAncestorSymlinkExpansionChain() != null) { |
| 390 | SkyKey uniquenessKey = |
| 391 | FileSymlinkInfiniteExpansionUniquenessFunction.key( |
| 392 | fileValue.unboundedAncestorSymlinkExpansionChain()); |
| 393 | env.getValue(uniquenessKey); |
| 394 | if (env.valuesMissing()) { |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 395 | return null; |
lberki | 7598bc6 | 2020-10-02 01:33:14 -0700 | [diff] [blame] | 396 | } |
| 397 | |
| 398 | throw new FileSymlinkInfiniteExpansionException( |
| 399 | fileValue.pathToUnboundedAncestorSymlinkExpansionChain(), |
| 400 | fileValue.unboundedAncestorSymlinkExpansionChain()); |
| 401 | } |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 402 | if (fileValue.exists()) { |
| 403 | // If it exists, it may either be a symlink or a file/directory. |
| 404 | PathFragment unresolvedLinkTarget = null; |
| 405 | FileType type; |
| 406 | if (fileValue.isSymlink()) { |
| 407 | unresolvedLinkTarget = fileValue.getUnresolvedLinkTarget(); |
| 408 | type = fileValue.isDirectory() ? FileType.SYMLINK_TO_DIRECTORY : FileType.SYMLINK_TO_FILE; |
| 409 | } else { |
| 410 | type = fileValue.isDirectory() ? FileType.DIRECTORY : FileType.FILE; |
| 411 | } |
Googler | 5284e6c | 2019-10-30 07:43:54 -0700 | [diff] [blame] | 412 | Path path = traversal.root.asRootedPath().asPath(); |
kush | 4b120e7 | 2018-07-11 16:21:27 -0700 | [diff] [blame] | 413 | return new FileInfo( |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 414 | type, |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 415 | withDigest(fileValue.realFileStateValue(), path, syscallCache), |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 416 | fileValue.realRootedPath(), |
| 417 | unresolvedLinkTarget); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 418 | } else { |
| 419 | // If it doesn't exist, or it's a dangling symlink, we still want to handle that gracefully. |
| 420 | return new FileInfo( |
| 421 | fileValue.isSymlink() ? FileType.DANGLING_SYMLINK : FileType.NONEXISTENT, |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 422 | withDigest(fileValue.realFileStateValue(), null, syscallCache), |
kush | 4b120e7 | 2018-07-11 16:21:27 -0700 | [diff] [blame] | 423 | null, |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 424 | fileValue.isSymlink() ? fileValue.getUnresolvedLinkTarget() : null); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 425 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 426 | } |
| 427 | } |
| 428 | |
Googler | 5284e6c | 2019-10-30 07:43:54 -0700 | [diff] [blame] | 429 | /** |
| 430 | * Transform the HasDigest to the appropriate type based on the current state of the digest. If |
| 431 | * fsVal is type RegularFileStateValue or FileArtifactValue and has a valid digest value, then we |
| 432 | * want to convert it to a new FileArtifactValue type. Otherwise if they are of the two |
| 433 | * forementioned types but do not have a digest, then we will create a FileArtifactValue using its |
| 434 | * {@link Path}. Otherwise we will fingerprint the digest and return it as a new {@link |
| 435 | * HasDigest.ByteStringDigest} object. |
| 436 | * |
| 437 | * @param fsVal - the HasDigest value that was in the graph. |
| 438 | * @param path - the Path of the digest. |
| 439 | * @return transformed HasDigest value based on the digest field and object type. |
| 440 | */ |
| 441 | @VisibleForTesting |
janakr | e285322 | 2022-03-08 10:52:18 -0800 | [diff] [blame] | 442 | static HasDigest withDigest(HasDigest fsVal, Path path, XattrProvider syscallCache) |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 443 | throws IOException { |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 444 | if (fsVal instanceof FileStateValue) { |
Googler | 5284e6c | 2019-10-30 07:43:54 -0700 | [diff] [blame] | 445 | FileStateValue fsv = (FileStateValue) fsVal; |
| 446 | if (fsv instanceof RegularFileStateValue) { |
| 447 | RegularFileStateValue rfsv = (RegularFileStateValue) fsv; |
| 448 | return rfsv.getDigest() != null |
| 449 | // If we have the digest, then simply convert it with the digest value. |
| 450 | ? FileArtifactValue.createForVirtualActionInput(rfsv.getDigest(), rfsv.getSize()) |
| 451 | // Otherwise, create a file FileArtifactValue (RegularFileArtifactValue) based on the |
| 452 | // path and size. |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 453 | : FileArtifactValue.createForNormalFileUsingPath(path, rfsv.getSize(), syscallCache); |
Googler | 5284e6c | 2019-10-30 07:43:54 -0700 | [diff] [blame] | 454 | } |
janakr | e430dc0 | 2021-04-28 12:28:57 -0700 | [diff] [blame] | 455 | return new HasDigest.ByteStringDigest(fsv.getValueFingerprint()); |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 456 | } else if (fsVal instanceof FileArtifactValue) { |
Googler | 5284e6c | 2019-10-30 07:43:54 -0700 | [diff] [blame] | 457 | FileArtifactValue fav = ((FileArtifactValue) fsVal); |
| 458 | if (fav.getDigest() != null) { |
| 459 | return fav; |
| 460 | } |
| 461 | |
| 462 | // In the case there is a directory, the HasDigest value should not be converted. Otherwise, |
| 463 | // if the HasDigest value is a file, convert it using the Path and size values. |
| 464 | return fav.getType().isFile() |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 465 | ? FileArtifactValue.createForNormalFileUsingPath(path, fav.getSize(), syscallCache) |
janakr | e430dc0 | 2021-04-28 12:28:57 -0700 | [diff] [blame] | 466 | : new HasDigest.ByteStringDigest(fav.getValueFingerprint()); |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 467 | } |
| 468 | return fsVal; |
| 469 | } |
| 470 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 471 | private static final class PkgLookupResult { |
| 472 | private enum Type { |
| 473 | CONFLICT, DIRECTORY, PKG |
| 474 | } |
| 475 | |
| 476 | private final Type type; |
| 477 | final TraversalRequest traversal; |
| 478 | final FileInfo rootInfo; |
| 479 | |
| 480 | /** Result for a generated directory that conflicts with a source package. */ |
| 481 | static PkgLookupResult conflict(TraversalRequest traversal, FileInfo rootInfo) { |
| 482 | return new PkgLookupResult(Type.CONFLICT, traversal, rootInfo); |
| 483 | } |
| 484 | |
| 485 | /** Result for a source or generated directory (not a package). */ |
| 486 | static PkgLookupResult directory(TraversalRequest traversal, FileInfo rootInfo) { |
| 487 | return new PkgLookupResult(Type.DIRECTORY, traversal, rootInfo); |
| 488 | } |
| 489 | |
| 490 | /** Result for a package, i.e. a directory with a BUILD file. */ |
| 491 | static PkgLookupResult pkg(TraversalRequest traversal, FileInfo rootInfo) { |
| 492 | return new PkgLookupResult(Type.PKG, traversal, rootInfo); |
| 493 | } |
| 494 | |
| 495 | private PkgLookupResult(Type type, TraversalRequest traversal, FileInfo rootInfo) { |
| 496 | this.type = Preconditions.checkNotNull(type); |
| 497 | this.traversal = Preconditions.checkNotNull(traversal); |
| 498 | this.rootInfo = Preconditions.checkNotNull(rootInfo); |
| 499 | } |
| 500 | |
| 501 | boolean isPackage() { |
| 502 | return type == Type.PKG; |
| 503 | } |
| 504 | |
| 505 | boolean isConflicting() { |
| 506 | return type == Type.CONFLICT; |
| 507 | } |
| 508 | |
| 509 | @Override |
| 510 | public String toString() { |
| 511 | return String.format("(%s: info=%s, traversal=%s)", type, rootInfo, traversal); |
| 512 | } |
| 513 | } |
| 514 | |
| 515 | /** |
| 516 | * Checks whether the {@code traversal}'s path refers to a package directory. |
| 517 | * |
| 518 | * @return the result of the lookup; it contains potentially new {@link TraversalRequest} and |
| 519 | * {@link FileInfo} so the caller should use these instead of the old ones (this happens when |
| 520 | * a package is found, but under a different root than expected) |
| 521 | */ |
Janak Ramakrishnan | 3c0adb2 | 2016-08-15 21:54:55 +0000 | [diff] [blame] | 522 | private static PkgLookupResult checkIfPackage( |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 523 | Environment env, TraversalRequest traversal, FileInfo rootInfo, SyscallCache syscallCache) |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 524 | throws IOException, InterruptedException, BuildFileNotFoundException { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 525 | Preconditions.checkArgument(rootInfo.type.exists() && !rootInfo.type.isFile(), |
| 526 | "{%s} {%s}", traversal, rootInfo); |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 527 | // PackageLookupFunction/dependencies can only throw IOException, BuildFileNotFoundException, |
| 528 | // and RepositoryFetchException, and RepositoryFetchException is not in play here. Note that |
| 529 | // run-of-the-mill circular symlinks will *not* throw here, and will trigger later errors during |
| 530 | // the recursive traversal. |
tomlu | 8cc5dcf | 2018-01-19 09:28:06 -0800 | [diff] [blame] | 531 | PackageLookupValue pkgLookup = |
| 532 | (PackageLookupValue) |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 533 | env.getValueOrThrow( |
| 534 | PackageLookupValue.key(traversal.root.asRootedPath().getRootRelativePath()), |
| 535 | BuildFileNotFoundException.class, |
| 536 | IOException.class); |
| 537 | if (env.valuesMissing()) { |
| 538 | return null; |
| 539 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 540 | |
| 541 | if (pkgLookup.packageExists()) { |
janakr | 5fb2a48 | 2018-03-02 17:48:57 -0800 | [diff] [blame] | 542 | if (traversal.isRootGenerated) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 543 | // The traversal's root was a generated directory, but its root-relative path conflicts with |
| 544 | // an existing package. |
| 545 | return PkgLookupResult.conflict(traversal, rootInfo); |
| 546 | } else { |
| 547 | // The traversal's root was a source directory and it defines a package. |
tomlu | ee6a686 | 2018-01-17 14:36:26 -0800 | [diff] [blame] | 548 | Root pkgRoot = pkgLookup.getRoot(); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 549 | if (!pkgRoot.equals(traversal.root.asRootedPath().getRoot())) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 550 | // However the root of this package is different from what we expected. stat() the real |
| 551 | // BUILD file of that package. |
| 552 | traversal = traversal.forChangedRootPath(pkgRoot); |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 553 | rootInfo = lookUpFileInfo(env, traversal, syscallCache); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 554 | Verify.verify(rootInfo.type.exists(), "{%s} {%s}", traversal, rootInfo); |
| 555 | } |
| 556 | return PkgLookupResult.pkg(traversal, rootInfo); |
| 557 | } |
| 558 | } else { |
| 559 | // The traversal's root was a directory (source or generated one), no package exists under the |
| 560 | // same root-relative path. |
| 561 | return PkgLookupResult.directory(traversal, rootInfo); |
| 562 | } |
| 563 | } |
| 564 | |
| 565 | /** |
| 566 | * List the directory and create {@code SkyKey}s to request contents of its children recursively. |
| 567 | * |
| 568 | * <p>The returned keys are of type {@link SkyFunctions#RECURSIVE_FILESYSTEM_TRAVERSAL}. |
| 569 | */ |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 570 | @Nullable |
Janak Ramakrishnan | 3c0adb2 | 2016-08-15 21:54:55 +0000 | [diff] [blame] | 571 | private static Collection<SkyKey> createRecursiveTraversalKeys( |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 572 | Environment env, TraversalRequest traversal) throws InterruptedException, IOException { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 573 | // Use the traversal's path, even if it's a symlink. The contents of the directory, as listed |
| 574 | // in the result, must be relative to it. |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 575 | Iterable<Dirent> dirents; |
janakr | 5fb2a48 | 2018-03-02 17:48:57 -0800 | [diff] [blame] | 576 | if (traversal.isRootGenerated) { |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 577 | // If we're dealing with an output file, read the directory directly instead of creating |
| 578 | // filesystem nodes under the output tree. |
| 579 | List<Dirent> direntsCollection = |
| 580 | new ArrayList<>( |
| 581 | traversal.root.asRootedPath().asPath().readdir(Symlinks.FOLLOW)); |
| 582 | Collections.sort(direntsCollection); |
| 583 | dirents = direntsCollection; |
| 584 | } else { |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 585 | DirectoryListingValue dirListingValue = |
| 586 | (DirectoryListingValue) |
| 587 | env.getValueOrThrow( |
| 588 | DirectoryListingValue.key(traversal.root.asRootedPath()), IOException.class); |
| 589 | if (dirListingValue == null) { |
| 590 | return null; |
| 591 | } |
| 592 | dirents = dirListingValue.getDirents(); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 593 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 594 | |
| 595 | List<SkyKey> result = new ArrayList<>(); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 596 | for (Dirent dirent : dirents) { |
tomlu | 8cc5dcf | 2018-01-19 09:28:06 -0800 | [diff] [blame] | 597 | RootedPath childPath = |
| 598 | RootedPath.toRootedPath( |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 599 | traversal.root.asRootedPath().getRoot(), |
| 600 | traversal.root.asRootedPath().getRootRelativePath().getRelative(dirent.getName())); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 601 | TraversalRequest childTraversal = traversal.forChildEntry(childPath); |
janakr | 5fb2a48 | 2018-03-02 17:48:57 -0800 | [diff] [blame] | 602 | result.add(childTraversal); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 603 | } |
| 604 | return result; |
| 605 | } |
| 606 | |
| 607 | /** |
| 608 | * Creates result for a dangling symlink. |
| 609 | * |
| 610 | * @param linkName path to the symbolic link |
| 611 | * @param info the {@link FileInfo} associated with the link file |
| 612 | */ |
| 613 | private static RecursiveFilesystemTraversalValue resultForDanglingSymlink(RootedPath linkName, |
| 614 | FileInfo info) { |
| 615 | Preconditions.checkState(info.type.isSymlink() && !info.type.exists(), "{%s} {%s}", linkName, |
| 616 | info.type); |
| 617 | return RecursiveFilesystemTraversalValue.of( |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 618 | ResolvedFileFactory.danglingSymlink(linkName, info.unresolvedSymlinkTarget, info.metadata)); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 619 | } |
| 620 | |
| 621 | /** |
| 622 | * Creates results for a file or for a symlink that points to one. |
| 623 | * |
| 624 | * <p>A symlink may be direct (points to a file) or transitive (points at a direct or transitive |
| 625 | * symlink). |
| 626 | */ |
| 627 | private static RecursiveFilesystemTraversalValue resultForFileRoot(RootedPath path, |
| 628 | FileInfo info) { |
| 629 | Preconditions.checkState(info.type.isFile() && info.type.exists(), "{%s} {%s}", path, |
| 630 | info.type); |
| 631 | if (info.type.isSymlink()) { |
Laszlo Csomor | b4d482b | 2015-12-04 13:23:54 +0000 | [diff] [blame] | 632 | return RecursiveFilesystemTraversalValue.of( |
| 633 | ResolvedFileFactory.symlinkToFile( |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 634 | info.realPath, path, info.unresolvedSymlinkTarget, info.metadata)); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 635 | } else { |
Laszlo Csomor | b4d482b | 2015-12-04 13:23:54 +0000 | [diff] [blame] | 636 | return RecursiveFilesystemTraversalValue.of( |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 637 | ResolvedFileFactory.regularFile(path, info.metadata)); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 638 | } |
| 639 | } |
| 640 | |
| 641 | private static RecursiveFilesystemTraversalValue resultForDirectory(TraversalRequest traversal, |
| 642 | FileInfo rootInfo, Collection<RecursiveFilesystemTraversalValue> subdirTraversals) { |
| 643 | // Collect transitive closure of files in subdirectories. |
| 644 | NestedSetBuilder<ResolvedFile> paths = NestedSetBuilder.stableOrder(); |
| 645 | for (RecursiveFilesystemTraversalValue child : subdirTraversals) { |
| 646 | paths.addTransitive(child.getTransitiveFiles()); |
| 647 | } |
| 648 | ResolvedFile root; |
| 649 | if (rootInfo.type.isSymlink()) { |
Laszlo Csomor | 207140f | 2015-12-07 15:07:33 +0000 | [diff] [blame] | 650 | NestedSet<ResolvedFile> children = paths.build(); |
Laszlo Csomor | b4d482b | 2015-12-04 13:23:54 +0000 | [diff] [blame] | 651 | root = |
| 652 | ResolvedFileFactory.symlinkToDirectory( |
| 653 | rootInfo.realPath, |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 654 | traversal.root.asRootedPath(), |
Laszlo Csomor | b4d482b | 2015-12-04 13:23:54 +0000 | [diff] [blame] | 655 | rootInfo.unresolvedSymlinkTarget, |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 656 | hashDirectorySymlink(children, rootInfo.metadata)); |
Laszlo Csomor | 207140f | 2015-12-07 15:07:33 +0000 | [diff] [blame] | 657 | paths = NestedSetBuilder.<ResolvedFile>stableOrder().addTransitive(children).add(root); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 658 | } else { |
Laszlo Csomor | b4d482b | 2015-12-04 13:23:54 +0000 | [diff] [blame] | 659 | root = ResolvedFileFactory.directory(rootInfo.realPath); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 660 | } |
| 661 | return RecursiveFilesystemTraversalValue.of(root, paths.build()); |
| 662 | } |
| 663 | |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 664 | private static HasDigest hashDirectorySymlink( |
ulfjack | 61216fd | 2019-12-18 12:31:03 -0800 | [diff] [blame] | 665 | NestedSet<ResolvedFile> children, HasDigest metadata) { |
Laszlo Csomor | 207140f | 2015-12-07 15:07:33 +0000 | [diff] [blame] | 666 | // If the root is a directory symlink, the associated FileStateValue does not change when the |
| 667 | // linked directory's contents change, so we can't use the FileStateValue as metadata like we |
| 668 | // do with other ResolvedFile kinds. Instead we compute a metadata hash from the child |
| 669 | // elements and return that as the ResolvedFile's metadata hash. |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 670 | Fingerprint fp = new Fingerprint(); |
| 671 | fp.addBytes(metadata.getDigest()); |
ulfjack | 61216fd | 2019-12-18 12:31:03 -0800 | [diff] [blame] | 672 | for (ResolvedFile file : children.toList()) { |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 673 | fp.addPath(file.getNameInSymlinkTree()); |
| 674 | fp.addBytes(file.getMetadata().getDigest()); |
Laszlo Csomor | 207140f | 2015-12-07 15:07:33 +0000 | [diff] [blame] | 675 | } |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 676 | byte[] result = fp.digestAndReset(); |
| 677 | return new HasDigest.ByteStringDigest(result); |
Laszlo Csomor | 207140f | 2015-12-07 15:07:33 +0000 | [diff] [blame] | 678 | } |
| 679 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 680 | /** |
| 681 | * Requests Skyframe to compute the dependent values and returns them. |
| 682 | * |
| 683 | * <p>The keys must all be {@link SkyFunctions#RECURSIVE_FILESYSTEM_TRAVERSAL} keys. |
| 684 | */ |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 685 | @Nullable |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 686 | private Collection<RecursiveFilesystemTraversalValue> traverseChildren( |
| 687 | Environment env, Iterable<SkyKey> keys, boolean inline) |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 688 | throws InterruptedException, RecursiveFilesystemTraversalFunctionException { |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 689 | Map<SkyKey, SkyValue> values; |
| 690 | if (inline) { |
| 691 | // Don't create Skyframe nodes for a recursive traversal over the output tree. |
| 692 | // Instead, inline the recursion in the top-level request. |
| 693 | values = new HashMap<>(); |
| 694 | for (SkyKey depKey : keys) { |
| 695 | values.put(depKey, compute(depKey, env)); |
| 696 | } |
| 697 | } else { |
| 698 | values = env.getValues(keys); |
| 699 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 700 | if (env.valuesMissing()) { |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 701 | return null; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 702 | } |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 703 | |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 704 | return Collections2.transform(values.values(), RecursiveFilesystemTraversalValue.class::cast); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 705 | } |
| 706 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 707 | } |