Damien Martin-Guillerez | f88f4d8 | 2015-09-25 13:56:55 +0000 | [diff] [blame] | 1 | // Copyright 2014 The Bazel Authors. All rights reserved. |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | package com.google.devtools.build.lib.skyframe; |
| 15 | |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 16 | import static com.google.common.base.Preconditions.checkArgument; |
| 17 | import static com.google.common.base.Preconditions.checkNotNull; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 18 | import static java.nio.charset.StandardCharsets.UTF_8; |
| 19 | |
Googler | 5284e6c | 2019-10-30 07:43:54 -0700 | [diff] [blame] | 20 | import com.google.common.annotations.VisibleForTesting; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 21 | import com.google.common.base.Verify; |
Googler | c8acdbd | 2018-11-08 10:47:15 -0800 | [diff] [blame] | 22 | import com.google.common.collect.ImmutableList; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 23 | import com.google.devtools.build.lib.actions.Artifact; |
Googler | c8acdbd | 2018-11-08 10:47:15 -0800 | [diff] [blame] | 24 | import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact; |
shahan | 602cc85 | 2018-06-06 20:09:57 -0700 | [diff] [blame] | 25 | import com.google.devtools.build.lib.actions.FileArtifactValue; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 26 | import com.google.devtools.build.lib.actions.FileStateType; |
| 27 | import com.google.devtools.build.lib.actions.FileStateValue; |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 28 | import com.google.devtools.build.lib.actions.FileStateValue.RegularFileStateValueWithContentsProxy; |
| 29 | import com.google.devtools.build.lib.actions.FileStateValue.RegularFileStateValueWithDigest; |
shahan | 602cc85 | 2018-06-06 20:09:57 -0700 | [diff] [blame] | 30 | import com.google.devtools.build.lib.actions.FileValue; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 31 | import com.google.devtools.build.lib.actions.HasDigest; |
Laszlo Csomor | 207140f | 2015-12-07 15:07:33 +0000 | [diff] [blame] | 32 | import com.google.devtools.build.lib.collect.nestedset.NestedSet; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 33 | import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; |
| 34 | import com.google.devtools.build.lib.events.Event; |
janakr | e2af68f | 2021-03-18 15:11:30 -0700 | [diff] [blame] | 35 | import com.google.devtools.build.lib.io.FileSymlinkException; |
| 36 | import com.google.devtools.build.lib.io.FileSymlinkInfiniteExpansionException; |
| 37 | import com.google.devtools.build.lib.io.FileSymlinkInfiniteExpansionUniquenessFunction; |
Googler | 0d4739e | 2023-10-30 10:00:55 -0700 | [diff] [blame] | 38 | import com.google.devtools.build.lib.io.InconsistentFilesystemException; |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 39 | import com.google.devtools.build.lib.packages.BuildFileNotFoundException; |
twerth | 45d82ca | 2021-02-24 02:28:54 -0800 | [diff] [blame] | 40 | import com.google.devtools.build.lib.profiler.Profiler; |
| 41 | import com.google.devtools.build.lib.profiler.ProfilerTask; |
| 42 | import com.google.devtools.build.lib.profiler.SilentCloseable; |
janakr | a4a564a | 2021-03-18 12:40:11 -0700 | [diff] [blame] | 43 | import com.google.devtools.build.lib.server.FailureDetails; |
jcater | b11c6e2 | 2020-04-03 11:40:16 -0700 | [diff] [blame] | 44 | import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.FileType; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 45 | import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFile; |
Laszlo Csomor | b4d482b | 2015-12-04 13:23:54 +0000 | [diff] [blame] | 46 | import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFileFactory; |
Googler | de9d1f5 | 2023-11-06 09:28:56 -0800 | [diff] [blame] | 47 | import com.google.devtools.build.lib.skyframe.serialization.VisibleForSerialization; |
Googler | 0129a24 | 2021-12-01 09:04:45 -0800 | [diff] [blame] | 48 | import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 49 | import com.google.devtools.build.lib.util.Fingerprint; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 50 | import com.google.devtools.build.lib.vfs.Dirent; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 51 | import com.google.devtools.build.lib.vfs.FileStatus; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 52 | import com.google.devtools.build.lib.vfs.Path; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 53 | import com.google.devtools.build.lib.vfs.PathFragment; |
tomlu | ee6a686 | 2018-01-17 14:36:26 -0800 | [diff] [blame] | 54 | import com.google.devtools.build.lib.vfs.Root; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 55 | import com.google.devtools.build.lib.vfs.RootedPath; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 56 | import com.google.devtools.build.lib.vfs.Symlinks; |
janakr | fc1d79c | 2022-01-27 13:02:07 -0800 | [diff] [blame] | 57 | import com.google.devtools.build.lib.vfs.SyscallCache; |
janakr | e285322 | 2022-03-08 10:52:18 -0800 | [diff] [blame] | 58 | import com.google.devtools.build.lib.vfs.XattrProvider; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 59 | import com.google.devtools.build.skyframe.SkyFunction; |
| 60 | import com.google.devtools.build.skyframe.SkyFunctionException; |
| 61 | import com.google.devtools.build.skyframe.SkyKey; |
| 62 | import com.google.devtools.build.skyframe.SkyValue; |
Googler | b5052b8 | 2022-11-03 11:33:56 -0700 | [diff] [blame] | 63 | import com.google.devtools.build.skyframe.SkyframeLookupResult; |
kush | 95bf7c8 | 2017-08-30 00:27:35 +0200 | [diff] [blame] | 64 | import java.io.IOException; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 65 | import java.math.BigInteger; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 66 | import java.util.ArrayList; |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 67 | import java.util.Collection; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 68 | import java.util.Collections; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 69 | import java.util.List; |
| 70 | import java.util.Map; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 71 | import javax.annotation.Nullable; |
| 72 | |
| 73 | /** A {@link SkyFunction} to build {@link RecursiveFilesystemTraversalValue}s. */ |
| 74 | public final class RecursiveFilesystemTraversalFunction implements SkyFunction { |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 75 | private static final byte[] MISSING_FINGERPRINT = |
| 76 | new BigInteger(1, "NonexistentFileStateValue".getBytes(UTF_8)).toByteArray(); |
| 77 | |
Googler | de9d1f5 | 2023-11-06 09:28:56 -0800 | [diff] [blame] | 78 | @SerializationConstant @VisibleForSerialization |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 79 | static final HasDigest NON_EXISTENT_HAS_DIGEST = () -> MISSING_FINGERPRINT; |
| 80 | |
| 81 | private static final FileInfo NON_EXISTENT_FILE_INFO = |
| 82 | new FileInfo(FileType.NONEXISTENT, NON_EXISTENT_HAS_DIGEST, null, null); |
| 83 | |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 84 | /** The exception that {@link RecursiveFilesystemTraversalFunctionException} wraps. */ |
| 85 | public static class RecursiveFilesystemTraversalException extends Exception { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 86 | |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 87 | /** |
| 88 | * Categories of errors that prevent normal {@link RecursiveFilesystemTraversalFunction} |
| 89 | * evaluation. |
| 90 | */ |
| 91 | public enum Type { |
| 92 | /** |
| 93 | * The traversal encountered a subdirectory with a BUILD file but is not allowed to recurse |
| 94 | * into it. See {@code PackageBoundaryMode#REPORT_ERROR}. |
| 95 | */ |
| 96 | CANNOT_CROSS_PACKAGE_BOUNDARY, |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 97 | |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 98 | /** A dangling symlink was dereferenced. */ |
| 99 | DANGLING_SYMLINK, |
| 100 | |
| 101 | /** A file operation failed. */ |
| 102 | FILE_OPERATION_FAILURE, |
| 103 | |
| 104 | /** A generated directory's root-relative path conflicts with a package's path. */ |
| 105 | GENERATED_PATH_CONFLICT, |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 106 | |
| 107 | /** A file/directory visited was part of a symlink cycle or infinite expansion. */ |
| 108 | SYMLINK_CYCLE_OR_INFINITE_EXPANSION, |
Googler | 0d4739e | 2023-10-30 10:00:55 -0700 | [diff] [blame] | 109 | |
| 110 | /** The filesystem told us inconsistent information. */ |
| 111 | INCONSISTENT_FILESYSTEM, |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 112 | } |
| 113 | |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 114 | private final RecursiveFilesystemTraversalException.Type type; |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 115 | |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 116 | RecursiveFilesystemTraversalException( |
| 117 | String message, RecursiveFilesystemTraversalException.Type type) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 118 | super(message); |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 119 | this.type = type; |
| 120 | } |
| 121 | |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 122 | public RecursiveFilesystemTraversalException.Type getType() { |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 123 | return type; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 124 | } |
| 125 | } |
| 126 | |
| 127 | /** |
| 128 | * Thrown when a dangling symlink is attempted to be dereferenced. |
| 129 | * |
| 130 | * <p>Note: this class is not identical to the one in com.google.devtools.build.lib.view.fileset |
| 131 | * and it's not easy to merge the two because of the dependency structure. The other one will |
| 132 | * probably be removed along with the rest of the legacy Fileset code. |
| 133 | */ |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 134 | static final class DanglingSymlinkException extends RecursiveFilesystemTraversalException { |
| 135 | DanglingSymlinkException(String path, String unresolvedLink) { |
Laszlo Csomor | 0ad729f | 2015-12-02 15:20:35 +0000 | [diff] [blame] | 136 | super( |
| 137 | String.format( |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 138 | "Found dangling symlink: %s, unresolved path: \"%s\"", path, unresolvedLink), |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 139 | RecursiveFilesystemTraversalException.Type.DANGLING_SYMLINK); |
| 140 | checkArgument(path != null && !path.isEmpty()); |
| 141 | checkArgument(unresolvedLink != null && !unresolvedLink.isEmpty()); |
kush | 95bf7c8 | 2017-08-30 00:27:35 +0200 | [diff] [blame] | 142 | } |
| 143 | } |
| 144 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 145 | /** Exception type thrown by {@link RecursiveFilesystemTraversalFunction#compute}. */ |
| 146 | private static final class RecursiveFilesystemTraversalFunctionException extends |
| 147 | SkyFunctionException { |
| 148 | RecursiveFilesystemTraversalFunctionException(RecursiveFilesystemTraversalException e) { |
| 149 | super(e, Transience.PERSISTENT); |
| 150 | } |
| 151 | } |
| 152 | |
janakr | b2a9434 | 2022-02-05 22:02:14 -0800 | [diff] [blame] | 153 | private final SyscallCache syscallCache; |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 154 | |
janakr | b2a9434 | 2022-02-05 22:02:14 -0800 | [diff] [blame] | 155 | RecursiveFilesystemTraversalFunction(SyscallCache syscallCache) { |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 156 | this.syscallCache = syscallCache; |
| 157 | } |
| 158 | |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 159 | @Nullable |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 160 | @Override |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 161 | public RecursiveFilesystemTraversalValue compute(SkyKey skyKey, Environment env) |
Janak Ramakrishnan | 3c0adb2 | 2016-08-15 21:54:55 +0000 | [diff] [blame] | 162 | throws RecursiveFilesystemTraversalFunctionException, InterruptedException { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 163 | TraversalRequest traversal = (TraversalRequest) skyKey.argument(); |
twerth | 45d82ca | 2021-02-24 02:28:54 -0800 | [diff] [blame] | 164 | try (SilentCloseable c = |
| 165 | Profiler.instance() |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 166 | .profile(ProfilerTask.FILESYSTEM_TRAVERSAL, traversal.root().toString())) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 167 | // Stat the traversal root. |
janakr | b2a9434 | 2022-02-05 22:02:14 -0800 | [diff] [blame] | 168 | FileInfo rootInfo = lookUpFileInfo(env, traversal, syscallCache); |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 169 | if (rootInfo == null) { |
| 170 | return null; |
| 171 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 172 | |
| 173 | if (!rootInfo.type.exists()) { |
| 174 | // May be a dangling symlink or a non-existent file. Handle gracefully. |
| 175 | if (rootInfo.type.isSymlink()) { |
Googler | 0d4739e | 2023-10-30 10:00:55 -0700 | [diff] [blame] | 176 | return RecursiveFilesystemTraversalValue.of( |
| 177 | ResolvedFileFactory.danglingSymlink( |
| 178 | traversal.root().asRootedPath(), |
| 179 | rootInfo.unresolvedSymlinkTarget, |
| 180 | rootInfo.metadata)); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 181 | } else { |
| 182 | return RecursiveFilesystemTraversalValue.EMPTY; |
| 183 | } |
| 184 | } |
| 185 | |
| 186 | if (rootInfo.type.isFile()) { |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 187 | return resultForFileRoot(traversal.root().asRootedPath(), rootInfo); |
| 188 | } |
Googler | 6f48f1c | 2024-04-16 14:29:09 -0700 | [diff] [blame] | 189 | if (rootInfo.type.isDirectory() && rootInfo.metadata instanceof TreeArtifactValue value) { |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 190 | ImmutableList.Builder<RecursiveFilesystemTraversalValue> traversalValues = |
| 191 | ImmutableList.builderWithExpectedSize(value.getChildValues().size()); |
Googler | c8acdbd | 2018-11-08 10:47:15 -0800 | [diff] [blame] | 192 | for (Map.Entry<TreeFileArtifact, FileArtifactValue> entry |
| 193 | : value.getChildValues().entrySet()) { |
| 194 | RootedPath path = |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 195 | RootedPath.toRootedPath(traversal.root().getRootPart(), entry.getKey().getPath()); |
| 196 | traversalValues.add( |
jcater | b11c6e2 | 2020-04-03 11:40:16 -0700 | [diff] [blame] | 197 | resultForFileRoot( |
| 198 | path, |
| 199 | // TreeArtifact can't have symbolic inside. So the assumption for FileType.FILE |
| 200 | // is always true. |
| 201 | new FileInfo(FileType.FILE, entry.getValue(), path, null))); |
Googler | c8acdbd | 2018-11-08 10:47:15 -0800 | [diff] [blame] | 202 | } |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 203 | return resultForDirectory(traversal, rootInfo, traversalValues.build()); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 204 | } |
| 205 | |
| 206 | // Otherwise the root is a directory or a symlink to one. |
janakr | b2a9434 | 2022-02-05 22:02:14 -0800 | [diff] [blame] | 207 | PkgLookupResult pkgLookupResult = checkIfPackage(env, traversal, rootInfo, syscallCache); |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 208 | if (pkgLookupResult == null) { |
| 209 | return null; |
| 210 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 211 | traversal = pkgLookupResult.traversal; |
| 212 | |
| 213 | if (pkgLookupResult.isConflicting()) { |
| 214 | // The traversal was requested for an output directory whose root-relative path conflicts |
| 215 | // with a source package. We can't handle that, bail out. |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 216 | throw createGeneratedPathConflictException(traversal); |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 217 | } else if (pkgLookupResult.isPackage() && !traversal.skipTestingForSubpackage()) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 218 | // The traversal was requested for a directory that defines a package. |
tomlu | 8cc5dcf | 2018-01-19 09:28:06 -0800 | [diff] [blame] | 219 | String msg = |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 220 | traversal.errorInfo() |
tomlu | 8cc5dcf | 2018-01-19 09:28:06 -0800 | [diff] [blame] | 221 | + " crosses package boundary into package rooted at " |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 222 | + traversal.root().getRelativePart().getPathString(); |
| 223 | switch (traversal.crossPkgBoundaries()) { |
Laszlo Csomor | f04efcc | 2015-02-12 17:08:06 +0000 | [diff] [blame] | 224 | case CROSS: |
| 225 | // We are free to traverse the subpackage but we need to display a warning. |
Ulf Adams | 760e709 | 2016-04-21 08:09:51 +0000 | [diff] [blame] | 226 | env.getListener().handle(Event.warn(null, msg)); |
Laszlo Csomor | f04efcc | 2015-02-12 17:08:06 +0000 | [diff] [blame] | 227 | break; |
| 228 | case DONT_CROSS: |
| 229 | // We cannot traverse the subpackage and should skip it silently. Return empty results. |
| 230 | return RecursiveFilesystemTraversalValue.EMPTY; |
| 231 | case REPORT_ERROR: |
| 232 | // We cannot traverse the subpackage and should complain loudly (display an error). |
| 233 | throw new RecursiveFilesystemTraversalFunctionException( |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 234 | new RecursiveFilesystemTraversalException( |
| 235 | msg, RecursiveFilesystemTraversalException.Type.CANNOT_CROSS_PACKAGE_BOUNDARY)); |
Laszlo Csomor | f04efcc | 2015-02-12 17:08:06 +0000 | [diff] [blame] | 236 | default: |
| 237 | throw new IllegalStateException(traversal.toString()); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 238 | } |
| 239 | } |
| 240 | |
| 241 | // We are free to traverse this directory. |
emilyguo | 305fcc3 | 2022-03-29 10:10:07 -0700 | [diff] [blame] | 242 | ImmutableList<RecursiveFilesystemTraversalValue> subdirTraversals = |
emilyguo | c07e66d | 2022-03-23 18:57:24 -0700 | [diff] [blame] | 243 | traverseChildren(env, traversal); |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 244 | if (subdirTraversals == null) { |
| 245 | return null; |
| 246 | } |
| 247 | return resultForDirectory(traversal, rootInfo, subdirTraversals); |
| 248 | } catch (IOException | BuildFileNotFoundException e) { |
| 249 | String message = |
| 250 | String.format( |
| 251 | "Error while traversing directory %s: %s", |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 252 | traversal.root().getRelativePart(), e.getMessage()); |
janakr | a4a564a | 2021-03-18 12:40:11 -0700 | [diff] [blame] | 253 | // Trying to stat the starting point of this root may have failed with a symlink cycle or |
| 254 | // trying to get a package lookup value may have failed due to a symlink cycle. |
| 255 | RecursiveFilesystemTraversalException.Type exceptionType = |
| 256 | RecursiveFilesystemTraversalException.Type.FILE_OPERATION_FAILURE; |
Googler | 0d4739e | 2023-10-30 10:00:55 -0700 | [diff] [blame] | 257 | if (e instanceof InconsistentFilesystemException) { |
| 258 | exceptionType = RecursiveFilesystemTraversalException.Type.INCONSISTENT_FILESYSTEM; |
| 259 | } |
janakr | a4a564a | 2021-03-18 12:40:11 -0700 | [diff] [blame] | 260 | if (e instanceof FileSymlinkException) { |
| 261 | exceptionType = |
| 262 | RecursiveFilesystemTraversalException.Type.SYMLINK_CYCLE_OR_INFINITE_EXPANSION; |
| 263 | } |
| 264 | if (e instanceof DetailedException) { |
| 265 | FailureDetails.PackageLoading.Code code = |
| 266 | ((DetailedException) e) |
| 267 | .getDetailedExitCode() |
| 268 | .getFailureDetail() |
| 269 | .getPackageLoading() |
| 270 | .getCode(); |
| 271 | if (code == FailureDetails.PackageLoading.Code.SYMLINK_CYCLE_OR_INFINITE_EXPANSION) { |
| 272 | exceptionType = |
| 273 | RecursiveFilesystemTraversalException.Type.SYMLINK_CYCLE_OR_INFINITE_EXPANSION; |
| 274 | } |
| 275 | } |
kush | 95bf7c8 | 2017-08-30 00:27:35 +0200 | [diff] [blame] | 276 | throw new RecursiveFilesystemTraversalFunctionException( |
janakr | a4a564a | 2021-03-18 12:40:11 -0700 | [diff] [blame] | 277 | new RecursiveFilesystemTraversalException(message, exceptionType)); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 278 | } |
| 279 | } |
| 280 | |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 281 | private static RecursiveFilesystemTraversalFunctionException createGeneratedPathConflictException( |
| 282 | TraversalRequest traversal) { |
| 283 | String message = |
| 284 | String.format( |
| 285 | "Generated directory %s conflicts with package under the same path. " |
| 286 | + "Additional info: %s", |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 287 | traversal.root().getRelativePart().getPathString(), traversal.errorInfo()); |
mschaller | c12c044 | 2020-06-08 12:06:35 -0700 | [diff] [blame] | 288 | return new RecursiveFilesystemTraversalFunctionException( |
| 289 | new RecursiveFilesystemTraversalException( |
| 290 | message, RecursiveFilesystemTraversalException.Type.GENERATED_PATH_CONFLICT)); |
| 291 | } |
| 292 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 293 | private static final class FileInfo { |
| 294 | final FileType type; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 295 | final HasDigest metadata; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 296 | @Nullable final RootedPath realPath; |
| 297 | @Nullable final PathFragment unresolvedSymlinkTarget; |
| 298 | |
kush | 4b120e7 | 2018-07-11 16:21:27 -0700 | [diff] [blame] | 299 | FileInfo( |
| 300 | FileType type, |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 301 | HasDigest metadata, |
kush | 4b120e7 | 2018-07-11 16:21:27 -0700 | [diff] [blame] | 302 | @Nullable RootedPath realPath, |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 303 | @Nullable PathFragment unresolvedSymlinkTarget) { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 304 | checkNotNull(metadata.getDigest(), metadata); |
| 305 | this.type = checkNotNull(type); |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 306 | this.metadata = metadata; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 307 | this.realPath = realPath; |
| 308 | this.unresolvedSymlinkTarget = unresolvedSymlinkTarget; |
| 309 | } |
| 310 | |
| 311 | @Override |
| 312 | public String toString() { |
| 313 | if (type.isSymlink()) { |
| 314 | return String.format("(%s: link_value=%s, real_path=%s)", type, |
| 315 | unresolvedSymlinkTarget.getPathString(), realPath); |
| 316 | } else { |
| 317 | return String.format("(%s: real_path=%s)", type, realPath); |
| 318 | } |
| 319 | } |
| 320 | } |
| 321 | |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 322 | @Nullable |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 323 | private static FileInfo lookUpFileInfo( |
| 324 | Environment env, TraversalRequest traversal, SyscallCache syscallCache) |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 325 | throws IOException, InterruptedException { |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 326 | if (traversal.isRootGenerated()) { |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 327 | HasDigest fsVal = null; |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 328 | if (traversal.root().getOutputArtifact() != null) { |
| 329 | Artifact artifact = traversal.root().getOutputArtifact(); |
lberki | 36df7ed | 2019-06-27 06:32:03 -0700 | [diff] [blame] | 330 | SkyKey artifactKey = Artifact.key(artifact); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 331 | SkyValue value = env.getValue(artifactKey); |
| 332 | if (env.valuesMissing()) { |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 333 | return null; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 334 | } |
kush | 95bf7c8 | 2017-08-30 00:27:35 +0200 | [diff] [blame] | 335 | |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 336 | if (value instanceof FileArtifactValue || value instanceof TreeArtifactValue) { |
| 337 | fsVal = (HasDigest) value; |
janakr | 8541f6d | 2019-06-11 14:40:21 -0700 | [diff] [blame] | 338 | } else if (value instanceof ActionExecutionValue) { |
Googler | aed4160 | 2020-06-02 12:22:53 -0700 | [diff] [blame] | 339 | fsVal = ((ActionExecutionValue) value).getExistingFileArtifactValue(artifact); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 340 | } else { |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 341 | return NON_EXISTENT_FILE_INFO; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 342 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 343 | } |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 344 | RootedPath realPath = traversal.root().asRootedPath(); |
| 345 | if (traversal.strictOutputFiles()) { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 346 | checkNotNull(fsVal, "Strict Fileset output tree has null FileArtifactValue"); |
Googler | c8acdbd | 2018-11-08 10:47:15 -0800 | [diff] [blame] | 347 | return new FileInfo( |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 348 | fsVal instanceof TreeArtifactValue ? FileType.DIRECTORY : FileType.FILE, |
jcater | b11c6e2 | 2020-04-03 11:40:16 -0700 | [diff] [blame] | 349 | fsVal, |
| 350 | realPath, |
| 351 | null); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 352 | } else { |
felly | 6c399d6 | 2018-07-18 15:55:59 -0700 | [diff] [blame] | 353 | // FileArtifactValue does not currently track symlinks. If it did, we could potentially |
| 354 | // remove some of the filesystem operations we're doing here. |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 355 | Path path = traversal.root().asPath(); |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 356 | FileStateValue fileState = |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 357 | FileStateValue.create(traversal.root().asRootedPath(), syscallCache, null); |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 358 | if (fileState.getType() == FileStateType.NONEXISTENT) { |
| 359 | throw new IOException("Missing file: " + path); |
| 360 | } |
felly | 6c399d6 | 2018-07-18 15:55:59 -0700 | [diff] [blame] | 361 | FileStatus followStat = path.statIfFound(Symlinks.FOLLOW); |
| 362 | FileType type; |
| 363 | PathFragment unresolvedLinkTarget = null; |
| 364 | if (followStat == null) { |
| 365 | type = FileType.DANGLING_SYMLINK; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 366 | if (fileState.getType() != FileStateType.SYMLINK) { |
| 367 | throw new IOException("Expected symlink for " + path + ", but got: " + fileState); |
felly | 6c399d6 | 2018-07-18 15:55:59 -0700 | [diff] [blame] | 368 | } |
| 369 | unresolvedLinkTarget = path.readSymbolicLink(); |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 370 | } else if (fileState.getType() == FileStateType.REGULAR_FILE) { |
felly | 6c399d6 | 2018-07-18 15:55:59 -0700 | [diff] [blame] | 371 | type = FileType.FILE; |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 372 | } else if (fileState.getType() == FileStateType.DIRECTORY) { |
felly | 6c399d6 | 2018-07-18 15:55:59 -0700 | [diff] [blame] | 373 | type = FileType.DIRECTORY; |
| 374 | } else { |
| 375 | unresolvedLinkTarget = path.readSymbolicLink(); |
| 376 | realPath = |
| 377 | RootedPath.toRootedPath( |
| 378 | Root.absoluteRoot(path.getFileSystem()), path.resolveSymbolicLinks()); |
| 379 | type = followStat.isFile() ? FileType.SYMLINK_TO_FILE : FileType.SYMLINK_TO_DIRECTORY; |
| 380 | } |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 381 | if (fsVal == null) { |
| 382 | fsVal = fileState; |
| 383 | } |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 384 | return new FileInfo( |
| 385 | type, withDigest(fsVal, path, syscallCache), realPath, unresolvedLinkTarget); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 386 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 387 | } else { |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 388 | // Stat the file. |
| 389 | FileValue fileValue = |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 390 | (FileValue) |
| 391 | env.getValueOrThrow( |
| 392 | FileValue.key(traversal.root().asRootedPath()), IOException.class); |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 393 | |
| 394 | if (env.valuesMissing()) { |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 395 | return null; |
felly | 5be4dd6 | 2018-02-05 11:11:53 -0800 | [diff] [blame] | 396 | } |
Googler | 9fb98e3 | 2023-07-13 07:40:38 -0700 | [diff] [blame] | 397 | return toFileInfo(fileValue, env, traversal.root().asPath(), syscallCache); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 398 | } |
| 399 | } |
| 400 | |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 401 | @Nullable |
| 402 | private static FileInfo toFileInfo( |
Googler | 9fb98e3 | 2023-07-13 07:40:38 -0700 | [diff] [blame] | 403 | FileValue fileValue, Environment env, Path path, SyscallCache syscallCache) |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 404 | throws IOException, InterruptedException { |
| 405 | if (fileValue.unboundedAncestorSymlinkExpansionChain() != null) { |
Googler | 9fb98e3 | 2023-07-13 07:40:38 -0700 | [diff] [blame] | 406 | SkyKey uniquenessKey = |
| 407 | FileSymlinkInfiniteExpansionUniquenessFunction.key( |
| 408 | fileValue.unboundedAncestorSymlinkExpansionChain()); |
| 409 | env.getValue(uniquenessKey); |
| 410 | if (env.valuesMissing()) { |
| 411 | return null; |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 412 | } |
| 413 | |
| 414 | throw new FileSymlinkInfiniteExpansionException( |
| 415 | fileValue.pathToUnboundedAncestorSymlinkExpansionChain(), |
| 416 | fileValue.unboundedAncestorSymlinkExpansionChain()); |
| 417 | } |
| 418 | |
| 419 | if (!fileValue.exists()) { |
| 420 | // If it doesn't exist, or it's a dangling symlink, we still want to handle that gracefully. |
| 421 | return new FileInfo( |
| 422 | fileValue.isSymlink() ? FileType.DANGLING_SYMLINK : FileType.NONEXISTENT, |
| 423 | withDigest(fileValue.realFileStateValue(), null, syscallCache), |
| 424 | null, |
| 425 | fileValue.isSymlink() ? fileValue.getUnresolvedLinkTarget() : null); |
| 426 | } |
| 427 | |
| 428 | // If it exists, it may either be a symlink or a file/directory. |
| 429 | PathFragment unresolvedLinkTarget = null; |
| 430 | FileType type; |
| 431 | if (fileValue.isSymlink()) { |
| 432 | unresolvedLinkTarget = fileValue.getUnresolvedLinkTarget(); |
| 433 | type = fileValue.isDirectory() ? FileType.SYMLINK_TO_DIRECTORY : FileType.SYMLINK_TO_FILE; |
| 434 | } else { |
| 435 | type = fileValue.isDirectory() ? FileType.DIRECTORY : FileType.FILE; |
| 436 | } |
| 437 | return new FileInfo( |
| 438 | type, |
| 439 | withDigest(fileValue.realFileStateValue(), path, syscallCache), |
| 440 | fileValue.realRootedPath(), |
| 441 | unresolvedLinkTarget); |
| 442 | } |
| 443 | |
Googler | 5284e6c | 2019-10-30 07:43:54 -0700 | [diff] [blame] | 444 | /** |
| 445 | * Transform the HasDigest to the appropriate type based on the current state of the digest. If |
| 446 | * fsVal is type RegularFileStateValue or FileArtifactValue and has a valid digest value, then we |
| 447 | * want to convert it to a new FileArtifactValue type. Otherwise if they are of the two |
| 448 | * forementioned types but do not have a digest, then we will create a FileArtifactValue using its |
| 449 | * {@link Path}. Otherwise we will fingerprint the digest and return it as a new {@link |
| 450 | * HasDigest.ByteStringDigest} object. |
| 451 | * |
| 452 | * @param fsVal - the HasDigest value that was in the graph. |
| 453 | * @param path - the Path of the digest. |
| 454 | * @return transformed HasDigest value based on the digest field and object type. |
| 455 | */ |
| 456 | @VisibleForTesting |
janakr | e285322 | 2022-03-08 10:52:18 -0800 | [diff] [blame] | 457 | static HasDigest withDigest(HasDigest fsVal, Path path, XattrProvider syscallCache) |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 458 | throws IOException { |
Googler | 6f48f1c | 2024-04-16 14:29:09 -0700 | [diff] [blame] | 459 | if (fsVal instanceof FileStateValue fsv) { |
| 460 | if (fsv instanceof RegularFileStateValueWithDigest rfsv) { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 461 | return FileArtifactValue.createForVirtualActionInput(rfsv.getDigest(), rfsv.getSize()); |
Googler | 6f48f1c | 2024-04-16 14:29:09 -0700 | [diff] [blame] | 462 | } else if (fsv instanceof RegularFileStateValueWithContentsProxy rfsv) { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 463 | return FileArtifactValue.createForNormalFileUsingPath(path, rfsv.getSize(), syscallCache); |
Googler | 5284e6c | 2019-10-30 07:43:54 -0700 | [diff] [blame] | 464 | } |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 465 | |
janakr | e430dc0 | 2021-04-28 12:28:57 -0700 | [diff] [blame] | 466 | return new HasDigest.ByteStringDigest(fsv.getValueFingerprint()); |
Googler | 6f48f1c | 2024-04-16 14:29:09 -0700 | [diff] [blame] | 467 | } else if (fsVal instanceof FileArtifactValue fav) { |
Googler | 5284e6c | 2019-10-30 07:43:54 -0700 | [diff] [blame] | 468 | if (fav.getDigest() != null) { |
| 469 | return fav; |
| 470 | } |
| 471 | |
| 472 | // In the case there is a directory, the HasDigest value should not be converted. Otherwise, |
| 473 | // if the HasDigest value is a file, convert it using the Path and size values. |
| 474 | return fav.getType().isFile() |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 475 | ? FileArtifactValue.createForNormalFileUsingPath(path, fav.getSize(), syscallCache) |
janakr | e430dc0 | 2021-04-28 12:28:57 -0700 | [diff] [blame] | 476 | : new HasDigest.ByteStringDigest(fav.getValueFingerprint()); |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 477 | } |
| 478 | return fsVal; |
| 479 | } |
| 480 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 481 | private static final class PkgLookupResult { |
| 482 | private enum Type { |
| 483 | CONFLICT, DIRECTORY, PKG |
| 484 | } |
| 485 | |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 486 | private final PkgLookupResult.Type type; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 487 | final TraversalRequest traversal; |
| 488 | final FileInfo rootInfo; |
| 489 | |
| 490 | /** Result for a generated directory that conflicts with a source package. */ |
| 491 | static PkgLookupResult conflict(TraversalRequest traversal, FileInfo rootInfo) { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 492 | return new PkgLookupResult(PkgLookupResult.Type.CONFLICT, traversal, rootInfo); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 493 | } |
| 494 | |
| 495 | /** Result for a source or generated directory (not a package). */ |
| 496 | static PkgLookupResult directory(TraversalRequest traversal, FileInfo rootInfo) { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 497 | return new PkgLookupResult(PkgLookupResult.Type.DIRECTORY, traversal, rootInfo); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 498 | } |
| 499 | |
| 500 | /** Result for a package, i.e. a directory with a BUILD file. */ |
| 501 | static PkgLookupResult pkg(TraversalRequest traversal, FileInfo rootInfo) { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 502 | return new PkgLookupResult(PkgLookupResult.Type.PKG, traversal, rootInfo); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 503 | } |
| 504 | |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 505 | private PkgLookupResult( |
| 506 | PkgLookupResult.Type type, TraversalRequest traversal, FileInfo rootInfo) { |
| 507 | this.type = checkNotNull(type); |
| 508 | this.traversal = checkNotNull(traversal); |
| 509 | this.rootInfo = checkNotNull(rootInfo); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 510 | } |
| 511 | |
| 512 | boolean isPackage() { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 513 | return type == PkgLookupResult.Type.PKG; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 514 | } |
| 515 | |
| 516 | boolean isConflicting() { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 517 | return type == PkgLookupResult.Type.CONFLICT; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 518 | } |
| 519 | |
| 520 | @Override |
| 521 | public String toString() { |
| 522 | return String.format("(%s: info=%s, traversal=%s)", type, rootInfo, traversal); |
| 523 | } |
| 524 | } |
| 525 | |
| 526 | /** |
| 527 | * Checks whether the {@code traversal}'s path refers to a package directory. |
| 528 | * |
| 529 | * @return the result of the lookup; it contains potentially new {@link TraversalRequest} and |
| 530 | * {@link FileInfo} so the caller should use these instead of the old ones (this happens when |
| 531 | * a package is found, but under a different root than expected) |
| 532 | */ |
Googler | 5cc598c | 2022-07-06 03:29:45 -0700 | [diff] [blame] | 533 | @Nullable |
Janak Ramakrishnan | 3c0adb2 | 2016-08-15 21:54:55 +0000 | [diff] [blame] | 534 | private static PkgLookupResult checkIfPackage( |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 535 | Environment env, TraversalRequest traversal, FileInfo rootInfo, SyscallCache syscallCache) |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 536 | throws IOException, InterruptedException, BuildFileNotFoundException { |
Googler | 8b68efe | 2023-03-15 14:57:50 -0700 | [diff] [blame] | 537 | checkArgument( |
| 538 | rootInfo.type.exists() && !rootInfo.type.isFile(), "{%s} {%s}", traversal, rootInfo); |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 539 | // PackageLookupFunction/dependencies can only throw IOException, BuildFileNotFoundException, |
| 540 | // and RepositoryFetchException, and RepositoryFetchException is not in play here. Note that |
| 541 | // run-of-the-mill circular symlinks will *not* throw here, and will trigger later errors during |
| 542 | // the recursive traversal. |
tomlu | 8cc5dcf | 2018-01-19 09:28:06 -0800 | [diff] [blame] | 543 | PackageLookupValue pkgLookup = |
| 544 | (PackageLookupValue) |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 545 | env.getValueOrThrow( |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 546 | PackageLookupValue.key(traversal.root().getRelativePart()), |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 547 | BuildFileNotFoundException.class, |
| 548 | IOException.class); |
| 549 | if (env.valuesMissing()) { |
| 550 | return null; |
| 551 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 552 | |
| 553 | if (pkgLookup.packageExists()) { |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 554 | if (traversal.isRootGenerated()) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 555 | // The traversal's root was a generated directory, but its root-relative path conflicts with |
| 556 | // an existing package. |
| 557 | return PkgLookupResult.conflict(traversal, rootInfo); |
| 558 | } else { |
| 559 | // The traversal's root was a source directory and it defines a package. |
tomlu | ee6a686 | 2018-01-17 14:36:26 -0800 | [diff] [blame] | 560 | Root pkgRoot = pkgLookup.getRoot(); |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 561 | if (!pkgRoot.equals(traversal.root().getRootPart())) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 562 | // However the root of this package is different from what we expected. stat() the real |
| 563 | // BUILD file of that package. |
| 564 | traversal = traversal.forChangedRootPath(pkgRoot); |
janakr | 491e441 | 2022-01-28 15:35:34 -0800 | [diff] [blame] | 565 | rootInfo = lookUpFileInfo(env, traversal, syscallCache); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 566 | Verify.verify(rootInfo.type.exists(), "{%s} {%s}", traversal, rootInfo); |
| 567 | } |
| 568 | return PkgLookupResult.pkg(traversal, rootInfo); |
| 569 | } |
| 570 | } else { |
| 571 | // The traversal's root was a directory (source or generated one), no package exists under the |
| 572 | // same root-relative path. |
| 573 | return PkgLookupResult.directory(traversal, rootInfo); |
| 574 | } |
| 575 | } |
| 576 | |
| 577 | /** |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 578 | * Creates results for a file or for a symlink that points to one. |
| 579 | * |
| 580 | * <p>A symlink may be direct (points to a file) or transitive (points at a direct or transitive |
| 581 | * symlink). |
| 582 | */ |
Googler | 0d4739e | 2023-10-30 10:00:55 -0700 | [diff] [blame] | 583 | private static RecursiveFilesystemTraversalValue resultForFileRoot(RootedPath path, FileInfo info) |
| 584 | throws InconsistentFilesystemException { |
| 585 | if (!info.type.isFile() || !info.type.exists()) { |
| 586 | throw new InconsistentFilesystemException( |
| 587 | String.format( |
| 588 | "We were previously told %s was an existing file but it's actually %s", path, info)); |
| 589 | } |
| 590 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 591 | if (info.type.isSymlink()) { |
Laszlo Csomor | b4d482b | 2015-12-04 13:23:54 +0000 | [diff] [blame] | 592 | return RecursiveFilesystemTraversalValue.of( |
| 593 | ResolvedFileFactory.symlinkToFile( |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 594 | info.realPath, path, info.unresolvedSymlinkTarget, info.metadata)); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 595 | } else { |
Laszlo Csomor | b4d482b | 2015-12-04 13:23:54 +0000 | [diff] [blame] | 596 | return RecursiveFilesystemTraversalValue.of( |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 597 | ResolvedFileFactory.regularFile(path, info.metadata)); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 598 | } |
| 599 | } |
| 600 | |
emilyguo | c07e66d | 2022-03-23 18:57:24 -0700 | [diff] [blame] | 601 | private static RecursiveFilesystemTraversalValue resultForDirectory( |
| 602 | TraversalRequest traversal, |
| 603 | FileInfo rootInfo, |
emilyguo | 305fcc3 | 2022-03-29 10:10:07 -0700 | [diff] [blame] | 604 | ImmutableList<RecursiveFilesystemTraversalValue> subdirTraversals) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 605 | // Collect transitive closure of files in subdirectories. |
| 606 | NestedSetBuilder<ResolvedFile> paths = NestedSetBuilder.stableOrder(); |
| 607 | for (RecursiveFilesystemTraversalValue child : subdirTraversals) { |
| 608 | paths.addTransitive(child.getTransitiveFiles()); |
| 609 | } |
| 610 | ResolvedFile root; |
| 611 | if (rootInfo.type.isSymlink()) { |
Laszlo Csomor | 207140f | 2015-12-07 15:07:33 +0000 | [diff] [blame] | 612 | NestedSet<ResolvedFile> children = paths.build(); |
Laszlo Csomor | b4d482b | 2015-12-04 13:23:54 +0000 | [diff] [blame] | 613 | root = |
| 614 | ResolvedFileFactory.symlinkToDirectory( |
| 615 | rootInfo.realPath, |
Googler | d78fb57 | 2022-07-06 11:44:42 -0700 | [diff] [blame] | 616 | traversal.root().asRootedPath(), |
Laszlo Csomor | b4d482b | 2015-12-04 13:23:54 +0000 | [diff] [blame] | 617 | rootInfo.unresolvedSymlinkTarget, |
kush | b39c693 | 2018-07-12 21:25:23 -0700 | [diff] [blame] | 618 | hashDirectorySymlink(children, rootInfo.metadata)); |
Laszlo Csomor | 207140f | 2015-12-07 15:07:33 +0000 | [diff] [blame] | 619 | paths = NestedSetBuilder.<ResolvedFile>stableOrder().addTransitive(children).add(root); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 620 | } else { |
Laszlo Csomor | b4d482b | 2015-12-04 13:23:54 +0000 | [diff] [blame] | 621 | root = ResolvedFileFactory.directory(rootInfo.realPath); |
Fabian Meumertzheim | fbfab66 | 2022-08-19 16:08:17 -0700 | [diff] [blame] | 622 | if (traversal.emitEmptyDirectoryNodes() && paths.isEmpty()) { |
| 623 | paths.add(root); |
| 624 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 625 | } |
| 626 | return RecursiveFilesystemTraversalValue.of(root, paths.build()); |
| 627 | } |
| 628 | |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 629 | private static HasDigest hashDirectorySymlink( |
ulfjack | 61216fd | 2019-12-18 12:31:03 -0800 | [diff] [blame] | 630 | NestedSet<ResolvedFile> children, HasDigest metadata) { |
Laszlo Csomor | 207140f | 2015-12-07 15:07:33 +0000 | [diff] [blame] | 631 | // If the root is a directory symlink, the associated FileStateValue does not change when the |
| 632 | // linked directory's contents change, so we can't use the FileStateValue as metadata like we |
| 633 | // do with other ResolvedFile kinds. Instead we compute a metadata hash from the child |
| 634 | // elements and return that as the ResolvedFile's metadata hash. |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 635 | Fingerprint fp = new Fingerprint(); |
| 636 | fp.addBytes(metadata.getDigest()); |
ulfjack | 61216fd | 2019-12-18 12:31:03 -0800 | [diff] [blame] | 637 | for (ResolvedFile file : children.toList()) { |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 638 | fp.addPath(file.getNameInSymlinkTree()); |
| 639 | fp.addBytes(file.getMetadata().getDigest()); |
Laszlo Csomor | 207140f | 2015-12-07 15:07:33 +0000 | [diff] [blame] | 640 | } |
felly | f1e30f3 | 2019-07-09 12:26:10 -0700 | [diff] [blame] | 641 | byte[] result = fp.digestAndReset(); |
| 642 | return new HasDigest.ByteStringDigest(result); |
Laszlo Csomor | 207140f | 2015-12-07 15:07:33 +0000 | [diff] [blame] | 643 | } |
| 644 | |
emilyguo | c07e66d | 2022-03-23 18:57:24 -0700 | [diff] [blame] | 645 | /** Requests Skyframe to compute the dependent values and returns them. */ |
janakr | b77246c | 2021-03-05 14:41:58 -0800 | [diff] [blame] | 646 | @Nullable |
emilyguo | 305fcc3 | 2022-03-29 10:10:07 -0700 | [diff] [blame] | 647 | private ImmutableList<RecursiveFilesystemTraversalValue> traverseChildren( |
emilyguo | c07e66d | 2022-03-23 18:57:24 -0700 | [diff] [blame] | 648 | Environment env, TraversalRequest traversal) |
| 649 | throws InterruptedException, RecursiveFilesystemTraversalFunctionException, IOException { |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 650 | return traversal.isRootGenerated() |
| 651 | ? traverseGeneratedChildren(env, traversal) |
| 652 | : traverseSourceChildren(env, traversal); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 653 | } |
| 654 | |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 655 | @Nullable |
| 656 | private ImmutableList<RecursiveFilesystemTraversalValue> traverseGeneratedChildren( |
| 657 | Environment env, TraversalRequest traversal) |
| 658 | throws InterruptedException, RecursiveFilesystemTraversalFunctionException, IOException { |
| 659 | // If we're dealing with an output file, read the directory directly instead of creating |
| 660 | // filesystem nodes under the output tree. |
| 661 | Collection<Dirent> dirents = traversal.root().asPath().readdir(Symlinks.FOLLOW); |
| 662 | if (dirents.isEmpty()) { |
| 663 | return ImmutableList.of(); |
| 664 | } |
| 665 | List<Dirent> sortedDirents = new ArrayList<>(dirents); |
| 666 | Collections.sort(sortedDirents); |
| 667 | |
| 668 | ImmutableList.Builder<RecursiveFilesystemTraversalValue> childValues = |
| 669 | ImmutableList.builderWithExpectedSize(dirents.size()); |
| 670 | for (Dirent dirent : sortedDirents) { |
| 671 | RecursiveFilesystemTraversalValue childValue = |
| 672 | compute(traversal.forChildEntry(dirent.getName()), env); |
| 673 | if (childValue != null) { |
| 674 | childValues.add(childValue); |
| 675 | } |
| 676 | } |
| 677 | return env.valuesMissing() ? null : childValues.build(); |
| 678 | } |
| 679 | |
| 680 | @Nullable |
| 681 | private ImmutableList<RecursiveFilesystemTraversalValue> traverseSourceChildren( |
| 682 | Environment env, TraversalRequest traversal) throws InterruptedException, IOException { |
| 683 | DirectoryListingValue dirListingValue = |
| 684 | (DirectoryListingValue) |
| 685 | env.getValueOrThrow( |
| 686 | DirectoryListingValue.key(traversal.root().asRootedPath()), IOException.class); |
| 687 | if (dirListingValue == null) { |
| 688 | return null; |
| 689 | } |
| 690 | Dirents dirents = dirListingValue.getDirents(); |
| 691 | if (dirents.size() == 0) { |
| 692 | return ImmutableList.of(); |
| 693 | } |
| 694 | |
| 695 | List<SkyKey> childKeys = new ArrayList<>(dirents.size()); |
| 696 | for (Dirent dirent : dirents) { |
| 697 | TraversalRequest childRequest = traversal.forChildEntry(dirent.getName()); |
| 698 | // For source files, request the FileValue directly instead of another TraversalRequest. This |
| 699 | // makes the base case of the recursive traversal a directory with no subdirectories instead |
| 700 | // of a file, greatly reducing the number of skyframe nodes for directories with many files. |
| 701 | if (dirent.getType() == Dirent.Type.FILE) { |
| 702 | childKeys.add(FileValue.key(childRequest.root().asRootedPath())); |
| 703 | } else { |
| 704 | childKeys.add(childRequest); |
| 705 | } |
| 706 | } |
| 707 | |
Googler | b5052b8 | 2022-11-03 11:33:56 -0700 | [diff] [blame] | 708 | SkyframeLookupResult result = env.getValuesAndExceptions(childKeys); |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 709 | ImmutableList.Builder<RecursiveFilesystemTraversalValue> childValues = |
| 710 | ImmutableList.builderWithExpectedSize(childKeys.size()); |
| 711 | for (SkyKey key : childKeys) { |
Googler | b5052b8 | 2022-11-03 11:33:56 -0700 | [diff] [blame] | 712 | SkyValue value = result.get(key); |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 713 | if (value == null) { |
| 714 | continue; |
| 715 | } |
Googler | 6f48f1c | 2024-04-16 14:29:09 -0700 | [diff] [blame] | 716 | if (key instanceof FileValue.Key fileKey) { |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 717 | FileInfo fileInfo = |
Googler | 9fb98e3 | 2023-07-13 07:40:38 -0700 | [diff] [blame] | 718 | toFileInfo((FileValue) value, env, fileKey.argument().asPath(), syscallCache); |
Googler | e0ba796 | 2022-08-11 08:37:36 -0700 | [diff] [blame] | 719 | if (fileInfo != null) { |
| 720 | childValues.add(resultForFileRoot(fileKey.argument(), fileInfo)); |
| 721 | } |
| 722 | } else { |
| 723 | childValues.add((RecursiveFilesystemTraversalValue) value); |
| 724 | } |
| 725 | } |
| 726 | return env.valuesMissing() ? null : childValues.build(); |
| 727 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 728 | } |