Damien Martin-Guillerez | f88f4d8 | 2015-09-25 13:56:55 +0000 | [diff] [blame] | 1 | // Copyright 2014 The Bazel Authors. All rights reserved. |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | package com.google.devtools.build.lib.skyframe; |
| 15 | |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 16 | import static com.google.common.base.Preconditions.checkArgument; |
| 17 | import static com.google.common.base.Preconditions.checkNotNull; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 18 | import static java.nio.charset.StandardCharsets.UTF_8; |
| 19 | |
Googler | 5284e6c | 2019-10-30 07:43:54 -0700 | [diff] [blame] | 20 | import com.google.common.annotations.VisibleForTesting; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 21 | import com.google.common.base.Verify; |
Googler | c8acdbd | 2018-11-08 10:47:15 -0800 | [diff] [blame] | 22 | import com.google.common.collect.ImmutableList; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 23 | import com.google.devtools.build.lib.actions.Artifact; |
Googler | c8acdbd | 2018-11-08 10:47:15 -0800 | [diff] [blame] | 24 | import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact; |
shahan | 602cc85 | 2018-06-06 20:09:57 -0700 | [diff] [blame] | 25 | import com.google.devtools.build.lib.actions.FileArtifactValue; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 26 | import com.google.devtools.build.lib.actions.FileStateType; |
| 27 | import com.google.devtools.build.lib.actions.FileStateValue; |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 28 | import com.google.devtools.build.lib.actions.FileStateValue.RegularFileStateValueWithContentsProxy; |
| 29 | import com.google.devtools.build.lib.actions.FileStateValue.RegularFileStateValueWithDigest; |
shahan | 602cc85 | 2018-06-06 20:09:57 -0700 | [diff] [blame] | 30 | import com.google.devtools.build.lib.actions.FileValue; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 31 | import com.google.devtools.build.lib.actions.HasDigest; |
Laszlo Csomor | 207140f | 2015-12-07 15:07:33 +0000 | [diff] [blame] | 32 | import com.google.devtools.build.lib.collect.nestedset.NestedSet; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 33 | import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; |
| 34 | import com.google.devtools.build.lib.events.Event; |
janakr | e2af68f | 2021-03-18 15:11:30 -0700 | [diff] [blame] | 35 | import com.google.devtools.build.lib.io.FileSymlinkException; |
| 36 | import com.google.devtools.build.lib.io.FileSymlinkInfiniteExpansionException; |
| 37 | import com.google.devtools.build.lib.io.FileSymlinkInfiniteExpansionUniquenessFunction; |
Googler | 0d4739e | 2023-10-30 10:00:55 -0700 | [diff] [blame] | 38 | import com.google.devtools.build.lib.io.InconsistentFilesystemException; |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 39 | import com.google.devtools.build.lib.packages.BuildFileNotFoundException; |
twerth | 45d82ca | 2021-02-24 02:28:54 -0800 | [diff] [blame] | 40 | import com.google.devtools.build.lib.profiler.Profiler; |
| 41 | import com.google.devtools.build.lib.profiler.ProfilerTask; |
| 42 | import com.google.devtools.build.lib.profiler.SilentCloseable; |
janakr | a4a564a | 2021-03-18 12:40:11 -0700 | [diff] [blame] | 43 | import com.google.devtools.build.lib.server.FailureDetails; |
jcater | b11c6e2 | 2020-04-03 11:40:16 -0700 | [diff] [blame] | 44 | import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.FileType; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 45 | import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFile; |
Laszlo Csomor | b4d482b | 2015-12-04 13:23:54 +0000 | [diff] [blame] | 46 | import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFileFactory; |
Googler | de9d1f5 | 2023-11-06 09:28:56 -0800 | [diff] [blame] | 47 | import com.google.devtools.build.lib.skyframe.serialization.VisibleForSerialization; |
Googler | 0129a24 | 2021-12-01 09:04:45 -0800 | [diff] [blame] | 48 | import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 49 | import com.google.devtools.build.lib.util.Fingerprint; |
Googler | 225d0a8 | 2024-10-18 08:26:22 -0700 | [diff] [blame] | 50 | import com.google.devtools.build.lib.vfs.DetailedIOException; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 51 | import com.google.devtools.build.lib.vfs.Dirent; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 52 | import com.google.devtools.build.lib.vfs.FileStatus; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 53 | import com.google.devtools.build.lib.vfs.Path; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 54 | import com.google.devtools.build.lib.vfs.PathFragment; |
tomlu | ee6a686 | 2018-01-17 14:36:26 -0800 | [diff] [blame] | 55 | import com.google.devtools.build.lib.vfs.Root; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 56 | import com.google.devtools.build.lib.vfs.RootedPath; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 57 | import com.google.devtools.build.lib.vfs.Symlinks; |
janakr | fc1d79c | 2022-01-27 13:02:07 -0800 | [diff] [blame] | 58 | import com.google.devtools.build.lib.vfs.SyscallCache; |
janakr | e285322 | 2022-03-08 10:52:18 -0800 | [diff] [blame] | 59 | import com.google.devtools.build.lib.vfs.XattrProvider; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 60 | import com.google.devtools.build.skyframe.SkyFunction; |
| 61 | import com.google.devtools.build.skyframe.SkyFunctionException; |
| 62 | import com.google.devtools.build.skyframe.SkyKey; |
| 63 | import com.google.devtools.build.skyframe.SkyValue; |
Googler | b5052b8 | 2022-11-03 11:33:56 -0700 | [diff] [blame] | 64 | import com.google.devtools.build.skyframe.SkyframeLookupResult; |
kush | 95bf7c8 | 2017-08-30 00:27:35 +0200 | [diff] [blame] | 65 | import java.io.IOException; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 66 | import java.math.BigInteger; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 67 | import java.util.ArrayList; |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 68 | import java.util.Collection; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 69 | import java.util.Collections; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 70 | import java.util.List; |
| 71 | import java.util.Map; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 72 | import javax.annotation.Nullable; |
| 73 | |
| 74 | /** A {@link SkyFunction} to build {@link RecursiveFilesystemTraversalValue}s. */ |
| 75 | public final class RecursiveFilesystemTraversalFunction implements SkyFunction { |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 76 | private static final byte[] MISSING_FINGERPRINT = |
| 77 | new BigInteger(1, "NonexistentFileStateValue".getBytes(UTF_8)).toByteArray(); |
| 78 | |
Googler | de9d1f5 | 2023-11-06 09:28:56 -0800 | [diff] [blame] | 79 | @SerializationConstant @VisibleForSerialization |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 80 | static final HasDigest NON_EXISTENT_HAS_DIGEST = () -> MISSING_FINGERPRINT; |
| 81 | |
| 82 | private static final FileInfo NON_EXISTENT_FILE_INFO = |
| 83 | new FileInfo(FileType.NONEXISTENT, NON_EXISTENT_HAS_DIGEST, null, null); |
| 84 | |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 85 | /** The exception that {@link RecursiveFilesystemTraversalFunctionException} wraps. */ |
| 86 | public static class RecursiveFilesystemTraversalException extends Exception { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 87 | |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 88 | /** |
| 89 | * Categories of errors that prevent normal {@link RecursiveFilesystemTraversalFunction} |
| 90 | * evaluation. |
| 91 | */ |
| 92 | public enum Type { |
| 93 | /** |
| 94 | * The traversal encountered a subdirectory with a BUILD file but is not allowed to recurse |
| 95 | * into it. See {@code PackageBoundaryMode#REPORT_ERROR}. |
| 96 | */ |
| 97 | CANNOT_CROSS_PACKAGE_BOUNDARY, |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 98 | |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 99 | /** A dangling symlink was dereferenced. */ |
| 100 | DANGLING_SYMLINK, |
| 101 | |
| 102 | /** A file operation failed. */ |
| 103 | FILE_OPERATION_FAILURE, |
| 104 | |
| 105 | /** A generated directory's root-relative path conflicts with a package's path. */ |
| 106 | GENERATED_PATH_CONFLICT, |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 107 | |
| 108 | /** A file/directory visited was part of a symlink cycle or infinite expansion. */ |
| 109 | SYMLINK_CYCLE_OR_INFINITE_EXPANSION, |
Googler | 0d4739e | 2023-10-30 10:00:55 -0700 | [diff] [blame] | 110 | |
| 111 | /** The filesystem told us inconsistent information. */ |
| 112 | INCONSISTENT_FILESYSTEM, |
Googler | 225d0a8 | 2024-10-18 08:26:22 -0700 | [diff] [blame] | 113 | |
| 114 | /** The filesystem threw a {@link DetailedIOException}. */ |
| 115 | DETAILED_IO_EXCEPTION, |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 116 | } |
| 117 | |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 118 | private final RecursiveFilesystemTraversalException.Type type; |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 119 | |
Googler | 225d0a8 | 2024-10-18 08:26:22 -0700 | [diff] [blame] | 120 | RecursiveFilesystemTraversalException(String message, DetailedIOException cause) { |
| 121 | super(message, cause); |
| 122 | this.type = RecursiveFilesystemTraversalException.Type.DETAILED_IO_EXCEPTION; |
| 123 | } |
| 124 | |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 125 | RecursiveFilesystemTraversalException( |
| 126 | String message, RecursiveFilesystemTraversalException.Type type) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 127 | super(message); |
Googler | 225d0a8 | 2024-10-18 08:26:22 -0700 | [diff] [blame] | 128 | checkArgument(type != Type.DETAILED_IO_EXCEPTION); |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 129 | this.type = type; |
| 130 | } |
| 131 | |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 132 | public RecursiveFilesystemTraversalException.Type getType() { |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 133 | return type; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 134 | } |
| 135 | } |
| 136 | |
| 137 | /** |
| 138 | * Thrown when a dangling symlink is attempted to be dereferenced. |
| 139 | * |
| 140 | * <p>Note: this class is not identical to the one in com.google.devtools.build.lib.view.fileset |
| 141 | * and it's not easy to merge the two because of the dependency structure. The other one will |
| 142 | * probably be removed along with the rest of the legacy Fileset code. |
| 143 | */ |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 144 | static final class DanglingSymlinkException extends RecursiveFilesystemTraversalException { |
| 145 | DanglingSymlinkException(String path, String unresolvedLink) { |
Laszlo Csomor | 0ad729f | 2015-12-02 15:20:35 +0000 | [diff] [blame] | 146 | super( |
| 147 | String.format( |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 148 | "Found dangling symlink: %s, unresolved path: \"%s\"", path, unresolvedLink), |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 149 | RecursiveFilesystemTraversalException.Type.DANGLING_SYMLINK); |
| 150 | checkArgument(path != null && !path.isEmpty()); |
| 151 | checkArgument(unresolvedLink != null && !unresolvedLink.isEmpty()); |
kush | 95bf7c8 | 2017-08-30 00:27:35 +0200 | [diff] [blame] | 152 | } |
| 153 | } |
| 154 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 155 | /** Exception type thrown by {@link RecursiveFilesystemTraversalFunction#compute}. */ |
| 156 | private static final class RecursiveFilesystemTraversalFunctionException extends |
| 157 | SkyFunctionException { |
| 158 | RecursiveFilesystemTraversalFunctionException(RecursiveFilesystemTraversalException e) { |
| 159 | super(e, Transience.PERSISTENT); |
| 160 | } |
| 161 | } |
| 162 | |
janakr | b2a9434 | 2022-02-05 22:02:14 -0800 | [diff] [blame] | 163 | private final SyscallCache syscallCache; |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 164 | |
janakr | b2a9434 | 2022-02-05 22:02:14 -0800 | [diff] [blame] | 165 | RecursiveFilesystemTraversalFunction(SyscallCache syscallCache) { |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 166 | this.syscallCache = syscallCache; |
| 167 | } |
| 168 | |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 169 | @Nullable |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 170 | @Override |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 171 | public RecursiveFilesystemTraversalValue compute(SkyKey skyKey, Environment env) |
Janak Ramakrishnan | 3c0adb2 | 2016-08-15 21:54:55 +0000 | [diff] [blame] | 172 | throws RecursiveFilesystemTraversalFunctionException, InterruptedException { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 173 | TraversalRequest traversal = (TraversalRequest) skyKey.argument(); |
twerth | 45d82ca | 2021-02-24 02:28:54 -0800 | [diff] [blame] | 174 | try (SilentCloseable c = |
| 175 | Profiler.instance() |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 176 | .profile(ProfilerTask.FILESYSTEM_TRAVERSAL, traversal.root().toString())) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 177 | // Stat the traversal root. |
janakr | b2a9434 | 2022-02-05 22:02:14 -0800 | [diff] [blame] | 178 | FileInfo rootInfo = lookUpFileInfo(env, traversal, syscallCache); |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 179 | if (rootInfo == null) { |
| 180 | return null; |
| 181 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 182 | |
| 183 | if (!rootInfo.type.exists()) { |
| 184 | // May be a dangling symlink or a non-existent file. Handle gracefully. |
| 185 | if (rootInfo.type.isSymlink()) { |
Googler | 0d4739e | 2023-10-30 10:00:55 -0700 | [diff] [blame] | 186 | return RecursiveFilesystemTraversalValue.of( |
| 187 | ResolvedFileFactory.danglingSymlink( |
| 188 | traversal.root().asRootedPath(), |
| 189 | rootInfo.unresolvedSymlinkTarget, |
| 190 | rootInfo.metadata)); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 191 | } else { |
| 192 | return RecursiveFilesystemTraversalValue.EMPTY; |
| 193 | } |
| 194 | } |
| 195 | |
| 196 | if (rootInfo.type.isFile()) { |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 197 | return resultForFileRoot(traversal.root().asRootedPath(), rootInfo); |
| 198 | } |
Googler | 6f48f1c | 2024-04-16 14:29:09 -0700 | [diff] [blame] | 199 | if (rootInfo.type.isDirectory() && rootInfo.metadata instanceof TreeArtifactValue value) { |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 200 | ImmutableList.Builder<RecursiveFilesystemTraversalValue> traversalValues = |
| 201 | ImmutableList.builderWithExpectedSize(value.getChildValues().size()); |
Googler | c8acdbd | 2018-11-08 10:47:15 -0800 | [diff] [blame] | 202 | for (Map.Entry<TreeFileArtifact, FileArtifactValue> entry |
| 203 | : value.getChildValues().entrySet()) { |
| 204 | RootedPath path = |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 205 | RootedPath.toRootedPath(traversal.root().getRootPart(), entry.getKey().getPath()); |
| 206 | traversalValues.add( |
jcater | b11c6e2 | 2020-04-03 11:40:16 -0700 | [diff] [blame] | 207 | resultForFileRoot( |
| 208 | path, |
| 209 | // TreeArtifact can't have symbolic inside. So the assumption for FileType.FILE |
| 210 | // is always true. |
| 211 | new FileInfo(FileType.FILE, entry.getValue(), path, null))); |
Googler | c8acdbd | 2018-11-08 10:47:15 -0800 | [diff] [blame] | 212 | } |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 213 | return resultForDirectory(traversal, rootInfo, traversalValues.build()); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 214 | } |
| 215 | |
| 216 | // Otherwise the root is a directory or a symlink to one. |
janakr | b2a9434 | 2022-02-05 22:02:14 -0800 | [diff] [blame] | 217 | PkgLookupResult pkgLookupResult = checkIfPackage(env, traversal, rootInfo, syscallCache); |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 218 | if (pkgLookupResult == null) { |
| 219 | return null; |
| 220 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 221 | traversal = pkgLookupResult.traversal; |
| 222 | |
| 223 | if (pkgLookupResult.isConflicting()) { |
| 224 | // The traversal was requested for an output directory whose root-relative path conflicts |
| 225 | // with a source package. We can't handle that, bail out. |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 226 | throw createGeneratedPathConflictException(traversal); |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 227 | } else if (pkgLookupResult.isPackage() && !traversal.skipTestingForSubpackage()) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 228 | // The traversal was requested for a directory that defines a package. |
tomlu | 8cc5dcf | 2018-01-19 09:28:06 -0800 | [diff] [blame] | 229 | String msg = |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 230 | traversal.errorInfo() |
tomlu | 8cc5dcf | 2018-01-19 09:28:06 -0800 | [diff] [blame] | 231 | + " crosses package boundary into package rooted at " |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 232 | + traversal.root().getRelativePart().getPathString(); |
| 233 | switch (traversal.crossPkgBoundaries()) { |
Laszlo Csomor | f04efcc | 2015-02-12 17:08:06 +0000 | [diff] [blame] | 234 | case CROSS: |
| 235 | // We are free to traverse the subpackage but we need to display a warning. |
Ulf Adams | 760e709 | 2016-04-21 08:09:51 +0000 | [diff] [blame] | 236 | env.getListener().handle(Event.warn(null, msg)); |
Laszlo Csomor | f04efcc | 2015-02-12 17:08:06 +0000 | [diff] [blame] | 237 | break; |
| 238 | case DONT_CROSS: |
| 239 | // We cannot traverse the subpackage and should skip it silently. Return empty results. |
| 240 | return RecursiveFilesystemTraversalValue.EMPTY; |
| 241 | case REPORT_ERROR: |
| 242 | // We cannot traverse the subpackage and should complain loudly (display an error). |
| 243 | throw new RecursiveFilesystemTraversalFunctionException( |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 244 | new RecursiveFilesystemTraversalException( |
| 245 | msg, RecursiveFilesystemTraversalException.Type.CANNOT_CROSS_PACKAGE_BOUNDARY)); |
Laszlo Csomor | f04efcc | 2015-02-12 17:08:06 +0000 | [diff] [blame] | 246 | default: |
| 247 | throw new IllegalStateException(traversal.toString()); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 248 | } |
| 249 | } |
| 250 | |
| 251 | // We are free to traverse this directory. |
emilyguo | 305fcc3 | 2022-03-29 10:10:07 -0700 | [diff] [blame] | 252 | ImmutableList<RecursiveFilesystemTraversalValue> subdirTraversals = |
emilyguo | c07e66d | 2022-03-23 18:57:24 -0700 | [diff] [blame] | 253 | traverseChildren(env, traversal); |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 254 | if (subdirTraversals == null) { |
| 255 | return null; |
| 256 | } |
| 257 | return resultForDirectory(traversal, rootInfo, subdirTraversals); |
| 258 | } catch (IOException | BuildFileNotFoundException e) { |
| 259 | String message = |
| 260 | String.format( |
| 261 | "Error while traversing directory %s: %s", |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 262 | traversal.root().getRelativePart(), e.getMessage()); |
Googler | 225d0a8 | 2024-10-18 08:26:22 -0700 | [diff] [blame] | 263 | |
| 264 | if (e instanceof DetailedIOException detailedException) { |
| 265 | throw new RecursiveFilesystemTraversalFunctionException( |
| 266 | new RecursiveFilesystemTraversalException(message, detailedException)); |
| 267 | } |
| 268 | |
janakr | a4a564a | 2021-03-18 12:40:11 -0700 | [diff] [blame] | 269 | // Trying to stat the starting point of this root may have failed with a symlink cycle or |
| 270 | // trying to get a package lookup value may have failed due to a symlink cycle. |
| 271 | RecursiveFilesystemTraversalException.Type exceptionType = |
| 272 | RecursiveFilesystemTraversalException.Type.FILE_OPERATION_FAILURE; |
Googler | 0d4739e | 2023-10-30 10:00:55 -0700 | [diff] [blame] | 273 | if (e instanceof InconsistentFilesystemException) { |
| 274 | exceptionType = RecursiveFilesystemTraversalException.Type.INCONSISTENT_FILESYSTEM; |
| 275 | } |
janakr | a4a564a | 2021-03-18 12:40:11 -0700 | [diff] [blame] | 276 | if (e instanceof FileSymlinkException) { |
| 277 | exceptionType = |
| 278 | RecursiveFilesystemTraversalException.Type.SYMLINK_CYCLE_OR_INFINITE_EXPANSION; |
| 279 | } |
| 280 | if (e instanceof DetailedException) { |
| 281 | FailureDetails.PackageLoading.Code code = |
| 282 | ((DetailedException) e) |
| 283 | .getDetailedExitCode() |
| 284 | .getFailureDetail() |
| 285 | .getPackageLoading() |
| 286 | .getCode(); |
| 287 | if (code == FailureDetails.PackageLoading.Code.SYMLINK_CYCLE_OR_INFINITE_EXPANSION) { |
| 288 | exceptionType = |
| 289 | RecursiveFilesystemTraversalException.Type.SYMLINK_CYCLE_OR_INFINITE_EXPANSION; |
| 290 | } |
| 291 | } |
kush | 95bf7c8 | 2017-08-30 00:27:35 +0200 | [diff] [blame] | 292 | throw new RecursiveFilesystemTraversalFunctionException( |
janakr | a4a564a | 2021-03-18 12:40:11 -0700 | [diff] [blame] | 293 | new RecursiveFilesystemTraversalException(message, exceptionType)); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 294 | } |
| 295 | } |
| 296 | |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 297 | private static RecursiveFilesystemTraversalFunctionException createGeneratedPathConflictException( |
| 298 | TraversalRequest traversal) { |
| 299 | String message = |
| 300 | String.format( |
| 301 | "Generated directory %s conflicts with package under the same path. " |
| 302 | + "Additional info: %s", |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 303 | traversal.root().getRelativePart().getPathString(), traversal.errorInfo()); |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 304 | return new RecursiveFilesystemTraversalFunctionException( |
| 305 | new RecursiveFilesystemTraversalException( |
| 306 | message, RecursiveFilesystemTraversalException.Type.GENERATED_PATH_CONFLICT)); |
| 307 | } |
| 308 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 309 | private static final class FileInfo { |
| 310 | final FileType type; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 311 | final HasDigest metadata; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 312 | @Nullable final RootedPath realPath; |
| 313 | @Nullable final PathFragment unresolvedSymlinkTarget; |
| 314 | |
kush | 4b120e7 | 2018-07-11 16:21:27 -0700 | [diff] [blame] | 315 | FileInfo( |
| 316 | FileType type, |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 317 | HasDigest metadata, |
kush | 4b120e7 | 2018-07-11 16:21:27 -0700 | [diff] [blame] | 318 | @Nullable RootedPath realPath, |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 319 | @Nullable PathFragment unresolvedSymlinkTarget) { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 320 | checkNotNull(metadata.getDigest(), metadata); |
| 321 | this.type = checkNotNull(type); |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 322 | this.metadata = metadata; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 323 | this.realPath = realPath; |
| 324 | this.unresolvedSymlinkTarget = unresolvedSymlinkTarget; |
| 325 | } |
| 326 | |
| 327 | @Override |
| 328 | public String toString() { |
| 329 | if (type.isSymlink()) { |
| 330 | return String.format("(%s: link_value=%s, real_path=%s)", type, |
| 331 | unresolvedSymlinkTarget.getPathString(), realPath); |
| 332 | } else { |
| 333 | return String.format("(%s: real_path=%s)", type, realPath); |
| 334 | } |
| 335 | } |
| 336 | } |
| 337 | |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 338 | @Nullable |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 339 | private static FileInfo lookUpFileInfo( |
| 340 | Environment env, TraversalRequest traversal, SyscallCache syscallCache) |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 341 | throws IOException, InterruptedException { |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 342 | if (traversal.isRootGenerated()) { |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 343 | HasDigest fsVal = null; |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 344 | if (traversal.root().getOutputArtifact() != null) { |
| 345 | Artifact artifact = traversal.root().getOutputArtifact(); |
lberki | 36df7ed | 2019-06-27 06:32:03 -0700 | [diff] [blame] | 346 | SkyKey artifactKey = Artifact.key(artifact); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 347 | SkyValue value = env.getValue(artifactKey); |
| 348 | if (env.valuesMissing()) { |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 349 | return null; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 350 | } |
kush | 95bf7c8 | 2017-08-30 00:27:35 +0200 | [diff] [blame] | 351 | |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 352 | if (value instanceof FileArtifactValue || value instanceof TreeArtifactValue) { |
| 353 | fsVal = (HasDigest) value; |
Googler | 3beaaaf | 2024-04-25 18:37:43 -0700 | [diff] [blame] | 354 | } else if (value instanceof ActionExecutionValue actionExecutionValue) { |
| 355 | fsVal = actionExecutionValue.getExistingFileArtifactValue(artifact); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 356 | } else { |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 357 | return NON_EXISTENT_FILE_INFO; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 358 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 359 | } |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 360 | RootedPath realPath = traversal.root().asRootedPath(); |
| 361 | if (traversal.strictOutputFiles()) { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 362 | checkNotNull(fsVal, "Strict Fileset output tree has null FileArtifactValue"); |
Googler | c8acdbd | 2018-11-08 10:47:15 -0800 | [diff] [blame] | 363 | return new FileInfo( |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 364 | fsVal instanceof TreeArtifactValue ? FileType.DIRECTORY : FileType.FILE, |
jcater | b11c6e2 | 2020-04-03 11:40:16 -0700 | [diff] [blame] | 365 | fsVal, |
| 366 | realPath, |
| 367 | null); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 368 | } else { |
felly | 6c399d6 | 2018-07-18 15:55:59 -0700 | [diff] [blame] | 369 | // FileArtifactValue does not currently track symlinks. If it did, we could potentially |
| 370 | // remove some of the filesystem operations we're doing here. |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 371 | Path path = traversal.root().asPath(); |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 372 | FileStateValue fileState = |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 373 | FileStateValue.create(traversal.root().asRootedPath(), syscallCache, null); |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 374 | if (fileState.getType() == FileStateType.NONEXISTENT) { |
| 375 | throw new IOException("Missing file: " + path); |
| 376 | } |
felly | 6c399d6 | 2018-07-18 15:55:59 -0700 | [diff] [blame] | 377 | FileStatus followStat = path.statIfFound(Symlinks.FOLLOW); |
| 378 | FileType type; |
| 379 | PathFragment unresolvedLinkTarget = null; |
| 380 | if (followStat == null) { |
| 381 | type = FileType.DANGLING_SYMLINK; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 382 | if (fileState.getType() != FileStateType.SYMLINK) { |
| 383 | throw new IOException("Expected symlink for " + path + ", but got: " + fileState); |
felly | 6c399d6 | 2018-07-18 15:55:59 -0700 | [diff] [blame] | 384 | } |
| 385 | unresolvedLinkTarget = path.readSymbolicLink(); |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 386 | } else if (fileState.getType() == FileStateType.REGULAR_FILE) { |
felly | 6c399d6 | 2018-07-18 15:55:59 -0700 | [diff] [blame] | 387 | type = FileType.FILE; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 388 | } else if (fileState.getType() == FileStateType.DIRECTORY) { |
felly | 6c399d6 | 2018-07-18 15:55:59 -0700 | [diff] [blame] | 389 | type = FileType.DIRECTORY; |
| 390 | } else { |
| 391 | unresolvedLinkTarget = path.readSymbolicLink(); |
| 392 | realPath = |
| 393 | RootedPath.toRootedPath( |
| 394 | Root.absoluteRoot(path.getFileSystem()), path.resolveSymbolicLinks()); |
| 395 | type = followStat.isFile() ? FileType.SYMLINK_TO_FILE : FileType.SYMLINK_TO_DIRECTORY; |
| 396 | } |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 397 | if (fsVal == null) { |
| 398 | fsVal = fileState; |
| 399 | } |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 400 | return new FileInfo( |
| 401 | type, withDigest(fsVal, path, syscallCache), realPath, unresolvedLinkTarget); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 402 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 403 | } else { |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 404 | // Stat the file. |
Googler | bf9a3d4 | 2024-07-24 18:52:40 -0700 | [diff] [blame] | 405 | RootedPath rootedPath = traversal.root().asRootedPath(); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 406 | FileValue fileValue = |
Googler | bf9a3d4 | 2024-07-24 18:52:40 -0700 | [diff] [blame] | 407 | (FileValue) env.getValueOrThrow(FileValue.key(rootedPath), IOException.class); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 408 | |
| 409 | if (env.valuesMissing()) { |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 410 | return null; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 411 | } |
Googler | bf9a3d4 | 2024-07-24 18:52:40 -0700 | [diff] [blame] | 412 | return toFileInfo(rootedPath, fileValue, env, traversal.root().asPath(), syscallCache); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 413 | } |
| 414 | } |
| 415 | |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 416 | @Nullable |
| 417 | private static FileInfo toFileInfo( |
Googler | bf9a3d4 | 2024-07-24 18:52:40 -0700 | [diff] [blame] | 418 | RootedPath rootedPath, |
| 419 | FileValue fileValue, |
| 420 | Environment env, |
| 421 | Path path, |
| 422 | SyscallCache syscallCache) |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 423 | throws IOException, InterruptedException { |
| 424 | if (fileValue.unboundedAncestorSymlinkExpansionChain() != null) { |
Googler | 9fb98e3 | 2023-07-13 07:40:38 -0700 | [diff] [blame] | 425 | SkyKey uniquenessKey = |
| 426 | FileSymlinkInfiniteExpansionUniquenessFunction.key( |
| 427 | fileValue.unboundedAncestorSymlinkExpansionChain()); |
| 428 | env.getValue(uniquenessKey); |
| 429 | if (env.valuesMissing()) { |
| 430 | return null; |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 431 | } |
| 432 | |
| 433 | throw new FileSymlinkInfiniteExpansionException( |
| 434 | fileValue.pathToUnboundedAncestorSymlinkExpansionChain(), |
| 435 | fileValue.unboundedAncestorSymlinkExpansionChain()); |
| 436 | } |
| 437 | |
| 438 | if (!fileValue.exists()) { |
| 439 | // If it doesn't exist, or it's a dangling symlink, we still want to handle that gracefully. |
| 440 | return new FileInfo( |
| 441 | fileValue.isSymlink() ? FileType.DANGLING_SYMLINK : FileType.NONEXISTENT, |
| 442 | withDigest(fileValue.realFileStateValue(), null, syscallCache), |
| 443 | null, |
| 444 | fileValue.isSymlink() ? fileValue.getUnresolvedLinkTarget() : null); |
| 445 | } |
| 446 | |
| 447 | // If it exists, it may either be a symlink or a file/directory. |
| 448 | PathFragment unresolvedLinkTarget = null; |
| 449 | FileType type; |
| 450 | if (fileValue.isSymlink()) { |
| 451 | unresolvedLinkTarget = fileValue.getUnresolvedLinkTarget(); |
| 452 | type = fileValue.isDirectory() ? FileType.SYMLINK_TO_DIRECTORY : FileType.SYMLINK_TO_FILE; |
| 453 | } else { |
| 454 | type = fileValue.isDirectory() ? FileType.DIRECTORY : FileType.FILE; |
| 455 | } |
| 456 | return new FileInfo( |
| 457 | type, |
| 458 | withDigest(fileValue.realFileStateValue(), path, syscallCache), |
Googler | bf9a3d4 | 2024-07-24 18:52:40 -0700 | [diff] [blame] | 459 | fileValue.realRootedPath(rootedPath), |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 460 | unresolvedLinkTarget); |
| 461 | } |
| 462 | |
Googler | 5284e6c | 2019-10-30 07:43:54 -0700 | [diff] [blame] | 463 | /** |
| 464 | * Transform the HasDigest to the appropriate type based on the current state of the digest. If |
| 465 | * fsVal is type RegularFileStateValue or FileArtifactValue and has a valid digest value, then we |
| 466 | * want to convert it to a new FileArtifactValue type. Otherwise if they are of the two |
| 467 | * forementioned types but do not have a digest, then we will create a FileArtifactValue using its |
| 468 | * {@link Path}. Otherwise we will fingerprint the digest and return it as a new {@link |
| 469 | * HasDigest.ByteStringDigest} object. |
| 470 | * |
| 471 | * @param fsVal - the HasDigest value that was in the graph. |
| 472 | * @param path - the Path of the digest. |
| 473 | * @return transformed HasDigest value based on the digest field and object type. |
| 474 | */ |
| 475 | @VisibleForTesting |
janakr | e285322 | 2022-03-08 10:52:18 -0800 | [diff] [blame] | 476 | static HasDigest withDigest(HasDigest fsVal, Path path, XattrProvider syscallCache) |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 477 | throws IOException { |
Googler | 6f48f1c | 2024-04-16 14:29:09 -0700 | [diff] [blame] | 478 | if (fsVal instanceof FileStateValue fsv) { |
| 479 | if (fsv instanceof RegularFileStateValueWithDigest rfsv) { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 480 | return FileArtifactValue.createForVirtualActionInput(rfsv.getDigest(), rfsv.getSize()); |
Googler | 6f48f1c | 2024-04-16 14:29:09 -0700 | [diff] [blame] | 481 | } else if (fsv instanceof RegularFileStateValueWithContentsProxy rfsv) { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 482 | return FileArtifactValue.createForNormalFileUsingPath(path, rfsv.getSize(), syscallCache); |
Googler | 5284e6c | 2019-10-30 07:43:54 -0700 | [diff] [blame] | 483 | } |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 484 | |
janakr | e430dc0 | 2021-04-28 12:28:57 -0700 | [diff] [blame] | 485 | return new HasDigest.ByteStringDigest(fsv.getValueFingerprint()); |
Googler | 6f48f1c | 2024-04-16 14:29:09 -0700 | [diff] [blame] | 486 | } else if (fsVal instanceof FileArtifactValue fav) { |
Googler | 5284e6c | 2019-10-30 07:43:54 -0700 | [diff] [blame] | 487 | if (fav.getDigest() != null) { |
| 488 | return fav; |
| 489 | } |
| 490 | |
| 491 | // In the case there is a directory, the HasDigest value should not be converted. Otherwise, |
| 492 | // if the HasDigest value is a file, convert it using the Path and size values. |
| 493 | return fav.getType().isFile() |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 494 | ? FileArtifactValue.createForNormalFileUsingPath(path, fav.getSize(), syscallCache) |
janakr | e430dc0 | 2021-04-28 12:28:57 -0700 | [diff] [blame] | 495 | : new HasDigest.ByteStringDigest(fav.getValueFingerprint()); |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 496 | } |
| 497 | return fsVal; |
| 498 | } |
| 499 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 500 | private static final class PkgLookupResult { |
| 501 | private enum Type { |
| 502 | CONFLICT, DIRECTORY, PKG |
| 503 | } |
| 504 | |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 505 | private final PkgLookupResult.Type type; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 506 | final TraversalRequest traversal; |
| 507 | final FileInfo rootInfo; |
| 508 | |
| 509 | /** Result for a generated directory that conflicts with a source package. */ |
| 510 | static PkgLookupResult conflict(TraversalRequest traversal, FileInfo rootInfo) { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 511 | return new PkgLookupResult(PkgLookupResult.Type.CONFLICT, traversal, rootInfo); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 512 | } |
| 513 | |
| 514 | /** Result for a source or generated directory (not a package). */ |
| 515 | static PkgLookupResult directory(TraversalRequest traversal, FileInfo rootInfo) { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 516 | return new PkgLookupResult(PkgLookupResult.Type.DIRECTORY, traversal, rootInfo); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 517 | } |
| 518 | |
| 519 | /** Result for a package, i.e. a directory with a BUILD file. */ |
| 520 | static PkgLookupResult pkg(TraversalRequest traversal, FileInfo rootInfo) { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 521 | return new PkgLookupResult(PkgLookupResult.Type.PKG, traversal, rootInfo); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 522 | } |
| 523 | |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 524 | private PkgLookupResult( |
| 525 | PkgLookupResult.Type type, TraversalRequest traversal, FileInfo rootInfo) { |
| 526 | this.type = checkNotNull(type); |
| 527 | this.traversal = checkNotNull(traversal); |
| 528 | this.rootInfo = checkNotNull(rootInfo); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 529 | } |
| 530 | |
| 531 | boolean isPackage() { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 532 | return type == PkgLookupResult.Type.PKG; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 533 | } |
| 534 | |
| 535 | boolean isConflicting() { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 536 | return type == PkgLookupResult.Type.CONFLICT; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 537 | } |
| 538 | |
| 539 | @Override |
| 540 | public String toString() { |
| 541 | return String.format("(%s: info=%s, traversal=%s)", type, rootInfo, traversal); |
| 542 | } |
| 543 | } |
| 544 | |
| 545 | /** |
| 546 | * Checks whether the {@code traversal}'s path refers to a package directory. |
| 547 | * |
| 548 | * @return the result of the lookup; it contains potentially new {@link TraversalRequest} and |
| 549 | * {@link FileInfo} so the caller should use these instead of the old ones (this happens when |
| 550 | * a package is found, but under a different root than expected) |
| 551 | */ |
Googler | 5cc598c | 2022-07-06 03:29:45 -0700 | [diff] [blame] | 552 | @Nullable |
Janak Ramakrishnan | 3c0adb2 | 2016-08-15 21:54:55 +0000 | [diff] [blame] | 553 | private static PkgLookupResult checkIfPackage( |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 554 | Environment env, TraversalRequest traversal, FileInfo rootInfo, SyscallCache syscallCache) |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 555 | throws IOException, InterruptedException, BuildFileNotFoundException { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 556 | checkArgument( |
| 557 | rootInfo.type.exists() && !rootInfo.type.isFile(), "{%s} {%s}", traversal, rootInfo); |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 558 | // PackageLookupFunction/dependencies can only throw IOException, BuildFileNotFoundException, |
| 559 | // and RepositoryFetchException, and RepositoryFetchException is not in play here. Note that |
| 560 | // run-of-the-mill circular symlinks will *not* throw here, and will trigger later errors during |
| 561 | // the recursive traversal. |
tomlu | 8cc5dcf | 2018-01-19 09:28:06 -0800 | [diff] [blame] | 562 | PackageLookupValue pkgLookup = |
| 563 | (PackageLookupValue) |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 564 | env.getValueOrThrow( |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 565 | PackageLookupValue.key(traversal.root().getRelativePart()), |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 566 | BuildFileNotFoundException.class, |
| 567 | IOException.class); |
| 568 | if (env.valuesMissing()) { |
| 569 | return null; |
| 570 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 571 | |
| 572 | if (pkgLookup.packageExists()) { |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 573 | if (traversal.isRootGenerated()) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 574 | // The traversal's root was a generated directory, but its root-relative path conflicts with |
| 575 | // an existing package. |
| 576 | return PkgLookupResult.conflict(traversal, rootInfo); |
| 577 | } else { |
| 578 | // The traversal's root was a source directory and it defines a package. |
tomlu | ee6a686 | 2018-01-17 14:36:26 -0800 | [diff] [blame] | 579 | Root pkgRoot = pkgLookup.getRoot(); |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 580 | if (!pkgRoot.equals(traversal.root().getRootPart())) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 581 | // However the root of this package is different from what we expected. stat() the real |
| 582 | // BUILD file of that package. |
| 583 | traversal = traversal.forChangedRootPath(pkgRoot); |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 584 | rootInfo = lookUpFileInfo(env, traversal, syscallCache); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 585 | Verify.verify(rootInfo.type.exists(), "{%s} {%s}", traversal, rootInfo); |
| 586 | } |
| 587 | return PkgLookupResult.pkg(traversal, rootInfo); |
| 588 | } |
| 589 | } else { |
| 590 | // The traversal's root was a directory (source or generated one), no package exists under the |
| 591 | // same root-relative path. |
| 592 | return PkgLookupResult.directory(traversal, rootInfo); |
| 593 | } |
| 594 | } |
| 595 | |
| 596 | /** |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 597 | * Creates results for a file or for a symlink that points to one. |
| 598 | * |
| 599 | * <p>A symlink may be direct (points to a file) or transitive (points at a direct or transitive |
| 600 | * symlink). |
| 601 | */ |
Googler | 0d4739e | 2023-10-30 10:00:55 -0700 | [diff] [blame] | 602 | private static RecursiveFilesystemTraversalValue resultForFileRoot(RootedPath path, FileInfo info) |
| 603 | throws InconsistentFilesystemException { |
| 604 | if (!info.type.isFile() || !info.type.exists()) { |
| 605 | throw new InconsistentFilesystemException( |
| 606 | String.format( |
| 607 | "We were previously told %s was an existing file but it's actually %s", path, info)); |
| 608 | } |
| 609 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 610 | if (info.type.isSymlink()) { |
Laszlo Csomor | b4d482b | 2015-12-04 13:23:54 +0000 | [diff] [blame] | 611 | return RecursiveFilesystemTraversalValue.of( |
| 612 | ResolvedFileFactory.symlinkToFile( |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 613 | info.realPath, path, info.unresolvedSymlinkTarget, info.metadata)); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 614 | } else { |
Laszlo Csomor | b4d482b | 2015-12-04 13:23:54 +0000 | [diff] [blame] | 615 | return RecursiveFilesystemTraversalValue.of( |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 616 | ResolvedFileFactory.regularFile(path, info.metadata)); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 617 | } |
| 618 | } |
| 619 | |
emilyguo | c07e66d | 2022-03-23 18:57:24 -0700 | [diff] [blame] | 620 | private static RecursiveFilesystemTraversalValue resultForDirectory( |
| 621 | TraversalRequest traversal, |
| 622 | FileInfo rootInfo, |
emilyguo | 305fcc3 | 2022-03-29 10:10:07 -0700 | [diff] [blame] | 623 | ImmutableList<RecursiveFilesystemTraversalValue> subdirTraversals) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 624 | // Collect transitive closure of files in subdirectories. |
| 625 | NestedSetBuilder<ResolvedFile> paths = NestedSetBuilder.stableOrder(); |
| 626 | for (RecursiveFilesystemTraversalValue child : subdirTraversals) { |
| 627 | paths.addTransitive(child.getTransitiveFiles()); |
| 628 | } |
| 629 | ResolvedFile root; |
| 630 | if (rootInfo.type.isSymlink()) { |
Laszlo Csomor | 207140f | 2015-12-07 15:07:33 +0000 | [diff] [blame] | 631 | NestedSet<ResolvedFile> children = paths.build(); |
Laszlo Csomor | b4d482b | 2015-12-04 13:23:54 +0000 | [diff] [blame] | 632 | root = |
| 633 | ResolvedFileFactory.symlinkToDirectory( |
| 634 | rootInfo.realPath, |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 635 | traversal.root().asRootedPath(), |
Laszlo Csomor | b4d482b | 2015-12-04 13:23:54 +0000 | [diff] [blame] | 636 | rootInfo.unresolvedSymlinkTarget, |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 637 | hashDirectorySymlink(children, rootInfo.metadata)); |
Laszlo Csomor | 207140f | 2015-12-07 15:07:33 +0000 | [diff] [blame] | 638 | paths = NestedSetBuilder.<ResolvedFile>stableOrder().addTransitive(children).add(root); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 639 | } else { |
Laszlo Csomor | b4d482b | 2015-12-04 13:23:54 +0000 | [diff] [blame] | 640 | root = ResolvedFileFactory.directory(rootInfo.realPath); |
Fabian Meumertzheim | fbfab66 | 2022-08-19 16:08:17 -0700 | [diff] [blame] | 641 | if (traversal.emitEmptyDirectoryNodes() && paths.isEmpty()) { |
| 642 | paths.add(root); |
| 643 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 644 | } |
| 645 | return RecursiveFilesystemTraversalValue.of(root, paths.build()); |
| 646 | } |
| 647 | |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 648 | private static HasDigest hashDirectorySymlink( |
ulfjack | 61216fd | 2019-12-18 12:31:03 -0800 | [diff] [blame] | 649 | NestedSet<ResolvedFile> children, HasDigest metadata) { |
Laszlo Csomor | 207140f | 2015-12-07 15:07:33 +0000 | [diff] [blame] | 650 | // If the root is a directory symlink, the associated FileStateValue does not change when the |
| 651 | // linked directory's contents change, so we can't use the FileStateValue as metadata like we |
| 652 | // do with other ResolvedFile kinds. Instead we compute a metadata hash from the child |
| 653 | // elements and return that as the ResolvedFile's metadata hash. |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 654 | Fingerprint fp = new Fingerprint(); |
| 655 | fp.addBytes(metadata.getDigest()); |
ulfjack | 61216fd | 2019-12-18 12:31:03 -0800 | [diff] [blame] | 656 | for (ResolvedFile file : children.toList()) { |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 657 | fp.addPath(file.getNameInSymlinkTree()); |
| 658 | fp.addBytes(file.getMetadata().getDigest()); |
Laszlo Csomor | 207140f | 2015-12-07 15:07:33 +0000 | [diff] [blame] | 659 | } |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 660 | byte[] result = fp.digestAndReset(); |
| 661 | return new HasDigest.ByteStringDigest(result); |
Laszlo Csomor | 207140f | 2015-12-07 15:07:33 +0000 | [diff] [blame] | 662 | } |
| 663 | |
emilyguo | c07e66d | 2022-03-23 18:57:24 -0700 | [diff] [blame] | 664 | /** Requests Skyframe to compute the dependent values and returns them. */ |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 665 | @Nullable |
emilyguo | 305fcc3 | 2022-03-29 10:10:07 -0700 | [diff] [blame] | 666 | private ImmutableList<RecursiveFilesystemTraversalValue> traverseChildren( |
emilyguo | c07e66d | 2022-03-23 18:57:24 -0700 | [diff] [blame] | 667 | Environment env, TraversalRequest traversal) |
| 668 | throws InterruptedException, RecursiveFilesystemTraversalFunctionException, IOException { |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 669 | return traversal.isRootGenerated() |
| 670 | ? traverseGeneratedChildren(env, traversal) |
| 671 | : traverseSourceChildren(env, traversal); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 672 | } |
| 673 | |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 674 | @Nullable |
| 675 | private ImmutableList<RecursiveFilesystemTraversalValue> traverseGeneratedChildren( |
| 676 | Environment env, TraversalRequest traversal) |
| 677 | throws InterruptedException, RecursiveFilesystemTraversalFunctionException, IOException { |
| 678 | // If we're dealing with an output file, read the directory directly instead of creating |
| 679 | // filesystem nodes under the output tree. |
| 680 | Collection<Dirent> dirents = traversal.root().asPath().readdir(Symlinks.FOLLOW); |
| 681 | if (dirents.isEmpty()) { |
| 682 | return ImmutableList.of(); |
| 683 | } |
| 684 | List<Dirent> sortedDirents = new ArrayList<>(dirents); |
| 685 | Collections.sort(sortedDirents); |
| 686 | |
| 687 | ImmutableList.Builder<RecursiveFilesystemTraversalValue> childValues = |
| 688 | ImmutableList.builderWithExpectedSize(dirents.size()); |
| 689 | for (Dirent dirent : sortedDirents) { |
| 690 | RecursiveFilesystemTraversalValue childValue = |
| 691 | compute(traversal.forChildEntry(dirent.getName()), env); |
| 692 | if (childValue != null) { |
| 693 | childValues.add(childValue); |
| 694 | } |
| 695 | } |
| 696 | return env.valuesMissing() ? null : childValues.build(); |
| 697 | } |
| 698 | |
| 699 | @Nullable |
| 700 | private ImmutableList<RecursiveFilesystemTraversalValue> traverseSourceChildren( |
| 701 | Environment env, TraversalRequest traversal) throws InterruptedException, IOException { |
| 702 | DirectoryListingValue dirListingValue = |
| 703 | (DirectoryListingValue) |
| 704 | env.getValueOrThrow( |
| 705 | DirectoryListingValue.key(traversal.root().asRootedPath()), IOException.class); |
| 706 | if (dirListingValue == null) { |
| 707 | return null; |
| 708 | } |
| 709 | Dirents dirents = dirListingValue.getDirents(); |
| 710 | if (dirents.size() == 0) { |
| 711 | return ImmutableList.of(); |
| 712 | } |
| 713 | |
| 714 | List<SkyKey> childKeys = new ArrayList<>(dirents.size()); |
| 715 | for (Dirent dirent : dirents) { |
| 716 | TraversalRequest childRequest = traversal.forChildEntry(dirent.getName()); |
| 717 | // For source files, request the FileValue directly instead of another TraversalRequest. This |
| 718 | // makes the base case of the recursive traversal a directory with no subdirectories instead |
| 719 | // of a file, greatly reducing the number of skyframe nodes for directories with many files. |
| 720 | if (dirent.getType() == Dirent.Type.FILE) { |
| 721 | childKeys.add(FileValue.key(childRequest.root().asRootedPath())); |
| 722 | } else { |
| 723 | childKeys.add(childRequest); |
| 724 | } |
| 725 | } |
| 726 | |
Googler | b5052b8 | 2022-11-03 11:33:56 -0700 | [diff] [blame] | 727 | SkyframeLookupResult result = env.getValuesAndExceptions(childKeys); |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 728 | ImmutableList.Builder<RecursiveFilesystemTraversalValue> childValues = |
| 729 | ImmutableList.builderWithExpectedSize(childKeys.size()); |
| 730 | for (SkyKey key : childKeys) { |
Googler | b5052b8 | 2022-11-03 11:33:56 -0700 | [diff] [blame] | 731 | SkyValue value = result.get(key); |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 732 | if (value == null) { |
| 733 | continue; |
| 734 | } |
Googler | 6f48f1c | 2024-04-16 14:29:09 -0700 | [diff] [blame] | 735 | if (key instanceof FileValue.Key fileKey) { |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 736 | FileInfo fileInfo = |
Googler | bf9a3d4 | 2024-07-24 18:52:40 -0700 | [diff] [blame] | 737 | toFileInfo( |
| 738 | fileKey.argument(), |
| 739 | (FileValue) value, |
| 740 | env, |
| 741 | fileKey.argument().asPath(), |
| 742 | syscallCache); |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 743 | if (fileInfo != null) { |
| 744 | childValues.add(resultForFileRoot(fileKey.argument(), fileInfo)); |
| 745 | } |
| 746 | } else { |
| 747 | childValues.add((RecursiveFilesystemTraversalValue) value); |
| 748 | } |
| 749 | } |
| 750 | return env.valuesMissing() ? null : childValues.build(); |
| 751 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 752 | } |