Add `rctx.watch_tree()` to watch a directory tree - Added `rctx.watch_tree()` to watch a directory tree, which includes all transitive descendants' names, and if they're files, their contents. - In the future we could add glob patterns to this method. - Added a new SkyFunction DirectoryTreeDigestFunction to do the heavy lifting. - In the future, for performance, we could try to get this skyfunction to have a mode where it only digests stat(), to use as heuristics (similar to #21044) Work towards #20952. Closes #21362. PiperOrigin-RevId: 608667062 Change-Id: Ibacbb7af4cf4d7628fe8fcf06e2c4fa50e811e4e
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/BUILD index b0f34b7..bbc23ef 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/BUILD
@@ -43,6 +43,7 @@ "//src/main/java/com/google/devtools/build/lib/shell", "//src/main/java/com/google/devtools/build/lib/skyframe:action_environment_function", "//src/main/java/com/google/devtools/build/lib/skyframe:directory_listing_value", + "//src/main/java/com/google/devtools/build/lib/skyframe:directory_tree_digest_value", "//src/main/java/com/google/devtools/build/lib/skyframe:ignored_package_prefixes_value", "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value", "//src/main/java/com/google/devtools/build/lib/starlarkbuildapi/repository",
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java index feb11e9..764c35e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java
@@ -33,14 +33,18 @@ import com.google.devtools.build.lib.pkgcache.PathPackageLocator; import com.google.devtools.build.lib.repository.RepositoryFetchProgress; import com.google.devtools.build.lib.rules.repository.NeedsSkyframeRestartException; +import com.google.devtools.build.lib.rules.repository.RepoRecordedInput; +import com.google.devtools.build.lib.rules.repository.RepoRecordedInput.RepoCacheFriendlyPath; import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException; import com.google.devtools.build.lib.rules.repository.WorkspaceAttributeMapper; import com.google.devtools.build.lib.runtime.ProcessWrapper; import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutor; +import com.google.devtools.build.lib.skyframe.DirectoryTreeDigestValue; import com.google.devtools.build.lib.util.StringUtilities; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; @@ -48,6 +52,7 @@ import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.InvalidPathException; +import java.util.HashMap; import java.util.Map; import javax.annotation.Nullable; import net.starlark.java.annot.Param; @@ -78,6 +83,7 @@ private final StructImpl attrObject; private final ImmutableSet<PathFragment> ignoredPatterns; private final SyscallCache syscallCache; + private final HashMap<RepoRecordedInput.DirTree, String> recordedDirTreeInputs = new HashMap<>(); /** * Create a new context (repository_ctx) object for a Starlark repository rule ({@code rule} @@ -131,6 +137,10 @@ return RepositoryFetchProgress.repositoryFetchContextString(repoName); } + public ImmutableMap<RepoRecordedInput.DirTree, String> getRecordedDirTreeInputs() { + return ImmutableMap.copyOf(recordedDirTreeInputs); + } + @StarlarkMethod( name = "name", structField = true, @@ -566,6 +576,54 @@ env.getListener().post(new ExtractProgress(outputPath.getPath().toString())); } + @StarlarkMethod( + name = "watch_tree", + doc = + "Tells Bazel to watch for changes to any files or directories transitively under the " + + "given path. Any changes to the contents of files, the existence of files or " + + "directories, file names or directory names, will cause this repo to be " + + "refetched.<p>Note that attempting to watch paths inside the repo currently being " + + "fetched will result in an error. ", + parameters = { + @Param( + name = "path", + allowedTypes = { + @ParamType(type = String.class), + @ParamType(type = Label.class), + @ParamType(type = StarlarkPath.class) + }, + doc = "path of the directory tree to watch."), + }) + public void watchTree(Object path) + throws EvalException, InterruptedException, RepositoryFunctionException { + StarlarkPath p = getPath("watch_tree()", path); + if (!p.isDir()) { + throw Starlark.errorf("can't call watch_tree() on non-directory %s", p); + } + RepoCacheFriendlyPath repoCacheFriendlyPath = + toRepoCacheFriendlyPath(p.getPath(), ShouldWatch.YES); + if (repoCacheFriendlyPath == null) { + return; + } + RootedPath rootedPath = repoCacheFriendlyPath.getRootedPath(env, directories); + if (rootedPath == null) { + throw new NeedsSkyframeRestartException(); + } + try { + DirectoryTreeDigestValue digestValue = + (DirectoryTreeDigestValue) + env.getValueOrThrow(DirectoryTreeDigestValue.key(rootedPath), IOException.class); + if (digestValue == null) { + throw new NeedsSkyframeRestartException(); + } + + recordedDirTreeInputs.put( + new RepoRecordedInput.DirTree(repoCacheFriendlyPath), digestValue.hexDigest()); + } catch (IOException e) { + throw new RepositoryFunctionException(e, Transience.TRANSIENT); + } + } + @Override public String toString() { return "repository_ctx[" + rule.getLabel() + "]";
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryFunction.java index b711378..1d35756 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryFunction.java
@@ -312,6 +312,7 @@ // Modify marker data to include the files/dirents used by the rule's implementation function. recordedInputValues.putAll(starlarkRepositoryContext.getRecordedFileInputs()); recordedInputValues.putAll(starlarkRepositoryContext.getRecordedDirentsInputs()); + recordedInputValues.putAll(starlarkRepositoryContext.getRecordedDirTreeInputs()); // Ditto for environment variables accessed via `getenv`. for (String envKey : starlarkRepositoryContext.getAccumulatedEnvKeys()) {
diff --git a/src/main/java/com/google/devtools/build/lib/rules/BUILD b/src/main/java/com/google/devtools/build/lib/rules/BUILD index c3c100c..cfb7c41 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/BUILD +++ b/src/main/java/com/google/devtools/build/lib/rules/BUILD
@@ -374,6 +374,7 @@ "//src/main/java/com/google/devtools/build/lib/skyframe:action_environment_function", "//src/main/java/com/google/devtools/build/lib/skyframe:client_environment_value", "//src/main/java/com/google/devtools/build/lib/skyframe:directory_listing_value", + "//src/main/java/com/google/devtools/build/lib/skyframe:directory_tree_digest_value", "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value", "//src/main/java/com/google/devtools/build/lib/skyframe:repository_mapping_value", "//src/main/java/com/google/devtools/build/lib/util",
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java index 605bd65..4bef44e 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java
@@ -28,6 +28,7 @@ import com.google.devtools.build.lib.skyframe.ActionEnvironmentFunction; import com.google.devtools.build.lib.skyframe.ClientEnvironmentValue; import com.google.devtools.build.lib.skyframe.DirectoryListingValue; +import com.google.devtools.build.lib.skyframe.DirectoryTreeDigestValue; import com.google.devtools.build.lib.skyframe.PrecomputedValue; import com.google.devtools.build.lib.skyframe.RepositoryMappingValue; import com.google.devtools.build.lib.util.Fingerprint; @@ -92,7 +93,9 @@ public static RepoRecordedInput parse(String s) { List<String> parts = Splitter.on(':').limit(2).splitToList(s); for (Parser parser : - new Parser[] {File.PARSER, Dirents.PARSER, EnvVar.PARSER, RecordedRepoMapping.PARSER}) { + new Parser[] { + File.PARSER, Dirents.PARSER, DirTree.PARSER, EnvVar.PARSER, RecordedRepoMapping.PARSER + }) { if (parts.get(0).equals(parser.getPrefix())) { return parser.parse(parts.get(1)); } @@ -112,7 +115,7 @@ throws InterruptedException { env.getValuesAndExceptions( recordedInputValues.keySet().stream() - .map(rri -> rri.getSkyKey(directories)) + .map(RepoRecordedInput::getSkyKey) .filter(Objects::nonNull) .collect(toImmutableSet())); if (env.valuesMissing()) { @@ -159,13 +162,13 @@ * no SkyKey is needed. */ @Nullable - public abstract SkyKey getSkyKey(BlazeDirectories directories); + public abstract SkyKey getSkyKey(); /** * Returns whether the given {@code oldValue} is still up-to-date for this recorded input. This - * method can assume that {@link #getSkyKey(BlazeDirectories)} is already evaluated; it can - * request further Skyframe evaluations, and if any values are missing, this method can return any - * value (doesn't matter what) and will be reinvoked after a Skyframe restart. + * method can assume that {@link #getSkyKey()} is already evaluated; it can request further + * Skyframe evaluations, and if any values are missing, this method can return any value (doesn't + * matter what) and will be reinvoked after a Skyframe restart. */ public abstract boolean isUpToDate( Environment env, BlazeDirectories directories, @Nullable String oldValue) @@ -330,7 +333,7 @@ @Override @Nullable - public SkyKey getSkyKey(BlazeDirectories directories) { + public SkyKey getSkyKey() { return path.getRepoDirSkyKeyOrNull(); } @@ -405,7 +408,7 @@ @Nullable @Override - public SkyKey getSkyKey(BlazeDirectories directories) { + public SkyKey getSkyKey() { return path.getRepoDirSkyKeyOrNull(); } @@ -438,6 +441,81 @@ } } + /** + * Represents an entire directory tree accessed during the fetch. Anything under the tree changing + * (including adding/removing/renaming files or directories and changing file contents) will cause + * it to go out of date. + */ + public static final class DirTree extends RepoRecordedInput { + public static final Parser PARSER = + new Parser() { + @Override + public String getPrefix() { + return "DIRTREE"; + } + + @Override + public RepoRecordedInput parse(String s) { + return new DirTree(RepoCacheFriendlyPath.parse(s)); + } + }; + + private final RepoCacheFriendlyPath path; + + public DirTree(RepoCacheFriendlyPath path) { + this.path = path; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof DirTree)) { + return false; + } + DirTree that = (DirTree) o; + return Objects.equals(path, that.path); + } + + @Override + public int hashCode() { + return path.hashCode(); + } + + @Override + public String toStringInternal() { + return path.toString(); + } + + @Override + public Parser getParser() { + return PARSER; + } + + @Nullable + @Override + public SkyKey getSkyKey() { + return path.getRepoDirSkyKeyOrNull(); + } + + @Override + public boolean isUpToDate( + Environment env, BlazeDirectories directories, @Nullable String oldValue) + throws InterruptedException { + RootedPath rootedPath = path.getRootedPath(env, directories); + if (rootedPath == null) { + return false; + } + DirectoryTreeDigestValue value = + (DirectoryTreeDigestValue) env.getValue(DirectoryTreeDigestValue.key(rootedPath)); + if (value == null) { + return false; + } + return oldValue.equals(value.hexDigest()); + } + } + /** Represents an environment variable accessed during the repo fetch. */ public static final class EnvVar extends RepoRecordedInput { static final Parser PARSER = @@ -487,7 +565,7 @@ } @Override - public SkyKey getSkyKey(BlazeDirectories directories) { + public SkyKey getSkyKey() { return ActionEnvironmentFunction.key(name); } @@ -497,7 +575,7 @@ throws InterruptedException { String v = PrecomputedValue.REPO_ENV.get(env).get(name); if (v == null) { - v = ((ClientEnvironmentValue) env.getValue(getSkyKey(directories))).getValue(); + v = ((ClientEnvironmentValue) env.getValue(getSkyKey())).getValue(); } // Note that `oldValue` can be null if the env var was not set. return Objects.equals(oldValue, v); @@ -558,7 +636,7 @@ } @Override - public SkyKey getSkyKey(BlazeDirectories directories) { + public SkyKey getSkyKey() { // Since we only record repo mapping entries for repos defined in Bzlmod, we can request the // WORKSPACE-less version of the main repo mapping (as no repos defined in Bzlmod can see // stuff from WORKSPACE). @@ -571,8 +649,7 @@ public boolean isUpToDate( Environment env, BlazeDirectories directories, @Nullable String oldValue) throws InterruptedException { - RepositoryMappingValue repoMappingValue = - (RepositoryMappingValue) env.getValue(getSkyKey(directories)); + RepositoryMappingValue repoMappingValue = (RepositoryMappingValue) env.getValue(getSkyKey()); return repoMappingValue != RepositoryMappingValue.NOT_FOUND_VALUE && RepositoryName.createUnvalidated(oldValue) .equals(repoMappingValue.getRepositoryMapping().get(apparentName));
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/BUILD index dab78bd..8f5e460 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/BUILD +++ b/src/main/java/com/google/devtools/build/lib/skyframe/BUILD
@@ -125,6 +125,7 @@ ":diff_awareness_manager", ":directory_listing_function", ":directory_listing_state_value", + ":directory_tree_digest_function", ":ephemeral_check_if_output_consumed", ":exclusive_test_build_driver_value", ":execution_finished_event", @@ -1249,6 +1250,33 @@ ) java_library( + name = "directory_tree_digest_function", + srcs = ["DirectoryTreeDigestFunction.java"], + deps = [ + ":directory_listing_value", + ":directory_tree_digest_value", + "//src/main/java/com/google/devtools/build/lib/actions:file_metadata", + "//src/main/java/com/google/devtools/build/lib/util", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/build/skyframe", + "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", + "//third_party:guava", + "//third_party:jsr305", + ], +) + +java_library( + name = "directory_tree_digest_value", + srcs = ["DirectoryTreeDigestValue.java"], + deps = [ + ":sky_functions", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", + "//third_party:auto_value", + ], +) + +java_library( name = "dirents", srcs = ["Dirents.java"], deps = [
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryTreeDigestFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryTreeDigestFunction.java new file mode 100644 index 0000000..ae23c37 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryTreeDigestFunction.java
@@ -0,0 +1,148 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableSet.toImmutableSet; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.actions.FileValue; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.Dirent; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.build.skyframe.SkyframeLookupResult; +import java.io.IOException; +import java.util.stream.StreamSupport; +import javax.annotation.Nullable; + +/** A {@link SkyFunction} for {@link DirectoryTreeDigestValue}s. */ +public final class DirectoryTreeDigestFunction implements SkyFunction { + @Override + @Nullable + public SkyValue compute(SkyKey skyKey, Environment env) + throws InterruptedException, DirectoryTreeDigestFunctionException { + RootedPath rootedPath = (RootedPath) skyKey.argument(); + DirectoryListingValue dirListingValue = + (DirectoryListingValue) env.getValue(DirectoryListingValue.key(rootedPath)); + if (dirListingValue == null) { + return null; + } + + // Get the names of entries directly in this directory, and sort them. This sets the basis for + // subsequent digests. + ImmutableSet<String> sortedDirents = + StreamSupport.stream(dirListingValue.getDirents().spliterator(), /* parallel= */ false) + .map(Dirent::getName) + .sorted() + .collect(toImmutableSet()); + + // Turn each entry into a FileValue. + ImmutableList<FileValue> fileValues = getFileValues(env, sortedDirents, rootedPath); + if (fileValues == null) { + return null; + } + + // For each entry that is a directory (or a symlink to a directory), find its own + // DirectoryTreeDigestValue. + ImmutableList<String> subDirTreeDigests = getSubDirTreeDigests(env, fileValues); + if (subDirTreeDigests == null) { + return null; + } + + // Finally, we're ready to digest everything together! + Fingerprint fp = new Fingerprint(); + fp.addStrings(sortedDirents); + fp.addStrings(subDirTreeDigests); + try { + for (FileValue fileValue : fileValues) { + fp.addInt(fileValue.realFileStateValue().getType().ordinal()); + if (fileValue.isFile()) { + byte[] digest = fileValue.realFileStateValue().getDigest(); + if (digest == null) { + // Fast digest not available, or it would have been in the FileValue. + digest = fileValue.realRootedPath().asPath().getDigest(); + } + fp.addBytes(digest); + } + } + } catch (IOException e) { + throw new DirectoryTreeDigestFunctionException(e); + } + + return DirectoryTreeDigestValue.of(fp.hexDigestAndReset()); + } + + @Nullable + private static ImmutableList<FileValue> getFileValues( + Environment env, ImmutableSet<String> sortedDirents, RootedPath rootedPath) + throws InterruptedException { + ImmutableSet<FileValue.Key> fileValueKeys = + sortedDirents.stream() + .map( + dirent -> + FileValue.key( + RootedPath.toRootedPath( + rootedPath.getRoot(), + rootedPath.getRootRelativePath().getRelative(dirent)))) + .collect(toImmutableSet()); + SkyframeLookupResult result = env.getValuesAndExceptions(fileValueKeys); + if (env.valuesMissing()) { + return null; + } + ImmutableList<FileValue> fileValues = + fileValueKeys.stream() + .map(result::get) + .map(FileValue.class::cast) + .collect(toImmutableList()); + if (env.valuesMissing()) { + return null; + } + return fileValues; + } + + @Nullable + private static ImmutableList<String> getSubDirTreeDigests( + Environment env, ImmutableList<FileValue> fileValues) throws InterruptedException { + ImmutableSet<SkyKey> dirTreeDigestValueKeys = + fileValues.stream() + .filter(FileValue::isDirectory) + .map(fv -> DirectoryTreeDigestValue.key(fv.realRootedPath())) + .collect(toImmutableSet()); + SkyframeLookupResult result = env.getValuesAndExceptions(dirTreeDigestValueKeys); + if (env.valuesMissing()) { + return null; + } + ImmutableList<String> dirTreeDigests = + dirTreeDigestValueKeys.stream() + .map(result::get) + .map(DirectoryTreeDigestValue.class::cast) + .map(DirectoryTreeDigestValue::hexDigest) + .collect(toImmutableList()); + if (env.valuesMissing()) { + return null; + } + return dirTreeDigests; + } + + private static final class DirectoryTreeDigestFunctionException extends SkyFunctionException { + public DirectoryTreeDigestFunctionException(IOException e) { + super(e, Transience.TRANSIENT); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryTreeDigestValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryTreeDigestValue.java new file mode 100644 index 0000000..246abf0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryTreeDigestValue.java
@@ -0,0 +1,51 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.auto.value.AutoValue; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.AbstractSkyKey; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyValue; + +/** + * Contains information about the recursive digest of a directory tree, including all transitive + * descendant files and their contents. + */ +@AutoValue +public abstract class DirectoryTreeDigestValue implements SkyValue { + public abstract String hexDigest(); + + public static DirectoryTreeDigestValue of(String hexDigest) { + return new AutoValue_DirectoryTreeDigestValue(hexDigest); + } + + public static Key key(RootedPath path) { + return new Key(path); + } + + /** Key type for {@link DirectoryTreeDigestValue}. */ + public static class Key extends AbstractSkyKey<RootedPath> { + + private Key(RootedPath rootedPath) { + super(rootedPath); + } + + @Override + public SkyFunctionName functionName() { + return SkyFunctions.DIRECTORY_TREE_DIGEST; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java index 0cf31c9..ec27b8f 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java
@@ -30,6 +30,8 @@ SkyFunctionName.createNonHermetic("DIRECTORY_LISTING_STATE"); public static final SkyFunctionName DIRECTORY_LISTING = SkyFunctionName.createHermetic("DIRECTORY_LISTING"); + public static final SkyFunctionName DIRECTORY_TREE_DIGEST = + SkyFunctionName.createHermetic("DIRECTORY_TREE_DIGEST"); // Hermetic even though package lookups secretly access the set of deleted packages, because // SequencedSkyframeExecutor deletes any affected PACKAGE_LOOKUP nodes when that set changes. public static final SkyFunctionName PACKAGE_LOOKUP =
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java index d428603..ec3e8e2 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
@@ -589,6 +589,7 @@ new FileSymlinkInfiniteExpansionUniquenessFunction()); map.put(FileValue.FILE, new FileFunction(pkgLocator, directories)); map.put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction()); + map.put(SkyFunctions.DIRECTORY_TREE_DIGEST, new DirectoryTreeDigestFunction()); map.put( SkyFunctions.PACKAGE_LOOKUP, new PackageLookupFunction(
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/BUILD b/src/test/java/com/google/devtools/build/lib/skyframe/BUILD index 08cc6bb..7c19c30 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/BUILD +++ b/src/test/java/com/google/devtools/build/lib/skyframe/BUILD
@@ -224,6 +224,8 @@ "//src/main/java/com/google/devtools/build/lib/skyframe:diff_awareness_manager", "//src/main/java/com/google/devtools/build/lib/skyframe:directory_listing_function", "//src/main/java/com/google/devtools/build/lib/skyframe:directory_listing_state_value", + "//src/main/java/com/google/devtools/build/lib/skyframe:directory_tree_digest_function", + "//src/main/java/com/google/devtools/build/lib/skyframe:directory_tree_digest_value", "//src/main/java/com/google/devtools/build/lib/skyframe:file_function", "//src/main/java/com/google/devtools/build/lib/skyframe:fileset_entry_function", "//src/main/java/com/google/devtools/build/lib/skyframe:fileset_entry_key", @@ -245,6 +247,7 @@ "//src/main/java/com/google/devtools/build/lib/skyframe:package_lookup_value", "//src/main/java/com/google/devtools/build/lib/skyframe:package_progress_receiver", "//src/main/java/com/google/devtools/build/lib/skyframe:package_value", + "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_function", "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value", "//src/main/java/com/google/devtools/build/lib/skyframe:prepare_deps_of_pattern_value", "//src/main/java/com/google/devtools/build/lib/skyframe:prepare_deps_of_patterns_value",
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/DirectoryTreeDigestFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/DirectoryTreeDigestFunctionTest.java new file mode 100644 index 0000000..7dd4eed --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/skyframe/DirectoryTreeDigestFunctionTest.java
@@ -0,0 +1,226 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.FileValue; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.ServerDirectories; +import com.google.devtools.build.lib.analysis.util.AnalysisMock; +import com.google.devtools.build.lib.clock.BlazeClock; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction; +import com.google.devtools.build.lib.testutil.FoundationTestCase; +import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.FileStateKey; +import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.lib.vfs.SyscallCache; +import com.google.devtools.build.skyframe.EvaluationContext; +import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator; +import com.google.devtools.build.skyframe.MemoizingEvaluator; +import com.google.devtools.build.skyframe.RecordingDifferencer; +import com.google.devtools.build.skyframe.SequencedRecordingDifferencer; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import java.util.concurrent.atomic.AtomicReference; +import net.starlark.java.eval.StarlarkSemantics; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class DirectoryTreeDigestFunctionTest extends FoundationTestCase { + + private RecordingDifferencer differencer; + private ImmutableMap<SkyFunctionName, SkyFunction> skyFunctions; + private EvaluationContext evaluationContext; + + @Before + public void setup() throws Exception { + differencer = new SequencedRecordingDifferencer(); + evaluationContext = + EvaluationContext.newBuilder().setParallelism(8).setEventHandler(reporter).build(); + AtomicReference<PathPackageLocator> packageLocator = + new AtomicReference<>( + new PathPackageLocator( + outputBase, + ImmutableList.of(Root.fromPath(rootDirectory)), + BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY)); + BlazeDirectories directories = + new BlazeDirectories( + new ServerDirectories(rootDirectory, outputBase, rootDirectory), + rootDirectory, + /* defaultSystemJavabase= */ null, + AnalysisMock.get().getProductName()); + ExternalFilesHelper externalFilesHelper = + ExternalFilesHelper.createForTesting( + packageLocator, + ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS, + directories); + + skyFunctions = + ImmutableMap.<SkyFunctionName, SkyFunction>builder() + .put(FileValue.FILE, new FileFunction(packageLocator, directories)) + .put( + FileStateKey.FILE_STATE, + new FileStateFunction( + Suppliers.ofInstance(new TimestampGranularityMonitor(BlazeClock.instance())), + SyscallCache.NO_CACHE, + externalFilesHelper)) + .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()) + .put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction()) + .put( + SkyFunctions.DIRECTORY_LISTING_STATE, + new DirectoryListingStateFunction(externalFilesHelper, SyscallCache.NO_CACHE)) + .put(SkyFunctions.DIRECTORY_TREE_DIGEST, new DirectoryTreeDigestFunction()) + .buildOrThrow(); + + PrecomputedValue.STARLARK_SEMANTICS.set(differencer, StarlarkSemantics.DEFAULT); + PrecomputedValue.PATH_PACKAGE_LOCATOR.set(differencer, packageLocator.get()); + } + + private String getTreeDigest(String path) throws Exception { + RootedPath rootedPath = + RootedPath.toRootedPath(Root.absoluteRoot(fileSystem), scratch.resolve(path)); + SkyKey key = DirectoryTreeDigestValue.key(rootedPath); + MemoizingEvaluator evaluator = new InMemoryMemoizingEvaluator(skyFunctions, differencer); + var result = evaluator.evaluate(ImmutableList.of(key), evaluationContext); + if (result.hasError()) { + throw result.getError().getException(); + } + return ((DirectoryTreeDigestValue) result.get(key)).hexDigest(); + } + + @Test + public void basic() throws Exception { + scratch.file("a", "a"); + scratch.file("b/b", "b"); + scratch.file("c", "c"); + String oldDigest = getTreeDigest("/"); + + scratch.overwriteFile("b/b", "something else"); + assertThat(getTreeDigest("/")).isNotEqualTo(oldDigest); + } + + @Test + public void addFile() throws Exception { + scratch.file("a", "a"); + scratch.file("b/b", "b"); + scratch.file("c", "c"); + String oldDigest = getTreeDigest("/"); + + scratch.file("b/d", "something else"); + assertThat(getTreeDigest("/")).isNotEqualTo(oldDigest); + } + + @Test + public void removeFile() throws Exception { + scratch.file("a", "a"); + scratch.file("b/b", "b"); + scratch.file("c", "c"); + String oldDigest = getTreeDigest("/"); + + scratch.deleteFile("b/b"); + assertThat(getTreeDigest("/")).isNotEqualTo(oldDigest); + } + + @Test + public void renameFile() throws Exception { + scratch.file("a", "a"); + scratch.file("b/b", "b"); + scratch.file("c", "c"); + String oldDigest = getTreeDigest("/"); + + scratch.deleteFile("b/b"); + scratch.file("b/b1", "b"); + assertThat(getTreeDigest("/")).isNotEqualTo(oldDigest); + } + + @Test + public void swapDirAndFile() throws Exception { + scratch.file("a", "a"); + scratch.file("b", "b"); + scratch.file("c/inner", "inner"); + String oldDigest = getTreeDigest("/"); + + scratch.resolve("c").deleteTree(); + scratch.deleteFile("b"); + scratch.file("b/inner", "inner"); + scratch.file("c", "b"); + assertThat(getTreeDigest("/")).isNotEqualTo(oldDigest); + } + + @Test + public void changeMtime() throws Exception { + scratch.file("a", "a"); + scratch.file("b", "b"); + scratch.file("c", "c"); + String oldDigest = getTreeDigest("/"); + + // We don't digest mtimes so this shouldn't affect anything. + scratch.resolve("c").setLastModifiedTime(2024L); + assertThat(getTreeDigest("/")).isEqualTo(oldDigest); + } + + @Test + public void symlink() throws Exception { + scratch.file("dir/a", "a"); + scratch.resolve("dir/b").createSymbolicLink(scratch.resolve("otherdir")); + scratch.file("dir/c", "c"); + scratch.file("otherdir/b", "b"); + scratch.file("otherdir/sub/sub", "sub"); + String oldDigest = getTreeDigest("dir"); + + scratch.deleteFile("dir/b"); + scratch.resolve("dir/b").createSymbolicLink(scratch.resolve("yetotherdir")); + scratch.file("yetotherdir/crazy", "stuff"); + assertThat(getTreeDigest("dir")).isNotEqualTo(oldDigest); + } + + @Test + public void danglingSymlink() throws Exception { + scratch.file("dir/a", "a"); + scratch.resolve("dir/b").createSymbolicLink(scratch.resolve("otherdir")); + scratch.file("dir/c", "c"); + String oldDigest = getTreeDigest("dir"); + + scratch.file("otherdir/b", "b"); + assertThat(getTreeDigest("dir")).isNotEqualTo(oldDigest); + } + + @Test + public void symlinkPointingToSameContents() throws Exception { + scratch.file("dir/a", "a"); + scratch.file("dir/b/b", "b"); + scratch.file("dir/b/sub/sub", "sub"); + scratch.file("dir/c", "c"); + String oldDigest = getTreeDigest("dir"); + + // replace dir/b with a symlink pointing to otherdir/, which contains the same contents. + // this shouldn't affect the tree digest. + scratch.resolve("dir/b").deleteTree(); + scratch.resolve("dir/b").createSymbolicLink(scratch.resolve("otherdir")); + scratch.file("otherdir/b", "b"); + scratch.file("otherdir/sub/sub", "sub"); + assertThat(getTreeDigest("dir")).isEqualTo(oldDigest); + } +}
diff --git a/src/test/shell/bazel/starlark_repository_test.sh b/src/test/shell/bazel/starlark_repository_test.sh index aabc46e..f042329 100755 --- a/src/test/shell/bazel/starlark_repository_test.sh +++ b/src/test/shell/bazel/starlark_repository_test.sh
@@ -3087,4 +3087,54 @@ expect_not_log "I see:" } +function test_watch_tree() { + local outside_dir=$(mktemp -d "${TEST_TMPDIR}/testXXXXXX") + touch ${outside_dir}/foo + touch ${outside_dir}/bar + touch ${outside_dir}/baz + + create_new_workspace + cat > MODULE.bazel <<EOF +r = use_repo_rule("//:r.bzl", "r") +r(name = "r") +EOF + touch BUILD + cat > r.bzl <<EOF +def _r(rctx): + rctx.file("BUILD", "filegroup(name='r')") + print("I'm running!") + rctx.watch_tree("${outside_dir}") +r=repository_rule(_r) +EOF + + bazel build @r >& $TEST_log || fail "expected bazel to succeed" + expect_log "I'm running!" + + # changing the contents of a file under there should trigger a refetch. + echo haha > ${outside_dir}/foo + bazel build @r >& $TEST_log || fail "expected bazel to succeed" + expect_log "I'm running!" + + # adding a file should trigger a refetch. + touch ${outside_dir}/quux + bazel build @r >& $TEST_log || fail "expected bazel to succeed" + expect_log "I'm running!" + + # just touching an existing file shouldn't cause a refetch. + touch ${outside_dir}/bar + bazel build @r >& $TEST_log || fail "expected bazel to succeed" + expect_not_log "I'm running!" + + # changing a file to a directory should trigger a refetch. + rm ${outside_dir}/baz + mkdir ${outside_dir}/baz + bazel build @r >& $TEST_log || fail "expected bazel to succeed" + expect_log "I'm running!" + + # changing entries in subdirectories should trigger a refetch. + touch ${outside_dir}/baz/inner + bazel build @r >& $TEST_log || fail "expected bazel to succeed" + expect_log "I'm running!" +} + run_suite "local repository tests"