Introduce --strict_fileset_output which treats all output Artifacts encountered in a Fileset as a regular file.
PiperOrigin-RevId: 205152271
diff --git a/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParams.java b/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParams.java
index 703611f..7740a05 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParams.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParams.java
@@ -220,6 +220,9 @@
/** Returns the desired behavior when the traversal hits a subpackage. */
public abstract PackageBoundaryMode getPackageBoundaryMode();
+ /** Returns whether Filesets treat outputs in a strict manner, assuming regular files. */
+ public abstract boolean isStrictFilesetOutput();
+
@Memoized
@Override
public abstract int hashCode();
@@ -232,6 +235,7 @@
fp.addBoolean(isFollowingSymlinks());
fp.addBoolean(isRecursive());
fp.addBoolean(isGenerated());
+ fp.addBoolean(isStrictFilesetOutput());
getPackageBoundaryMode().fingerprint(fp);
return fp.digestAndReset();
}
@@ -242,6 +246,7 @@
boolean isPackage,
boolean followingSymlinks,
PackageBoundaryMode packageBoundaryMode,
+ boolean isStrictFilesetOutput,
boolean isRecursive,
boolean isGenerated) {
return new AutoValue_FilesetTraversalParams_DirectTraversal(
@@ -250,7 +255,8 @@
isRecursive,
isGenerated,
followingSymlinks,
- packageBoundaryMode);
+ packageBoundaryMode,
+ isStrictFilesetOutput);
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParamsFactory.java b/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParamsFactory.java
index 93941a7..d1f94f0 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParamsFactory.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParamsFactory.java
@@ -48,15 +48,16 @@
* files in subdirectories cannot be excluded
* @param symlinkBehaviorMode what to do with symlinks
* @param pkgBoundaryMode what to do when the traversal hits a subdirectory that is also a
- * subpackage (contains a BUILD file)
+ * @param strictFilesetOutput whether Fileset assumes that output Artifacts are regular files.
*/
public static FilesetTraversalParams recursiveTraversalOfPackage(Label ownerLabel,
Artifact buildFile, PathFragment destPath, @Nullable Set<String> excludes,
- SymlinkBehavior symlinkBehaviorMode, PackageBoundaryMode pkgBoundaryMode) {
+ SymlinkBehavior symlinkBehaviorMode, PackageBoundaryMode pkgBoundaryMode,
+ boolean strictFilesetOutput) {
Preconditions.checkState(buildFile.isSourceArtifact(), "%s", buildFile);
return DirectoryTraversalParams.getDirectoryTraversalParams(ownerLabel,
DirectTraversalRoot.forPackage(buildFile), true, destPath, excludes,
- symlinkBehaviorMode, pkgBoundaryMode, true, false);
+ symlinkBehaviorMode, pkgBoundaryMode, strictFilesetOutput, true, false);
}
/**
@@ -73,14 +74,15 @@
* subdirectories cannot be excluded
* @param symlinkBehaviorMode what to do with symlinks
* @param pkgBoundaryMode what to do when the traversal hits a subdirectory that is also a
- * subpackage (contains a BUILD file)
+ * @param strictFilesetOutput whether Fileset assumes that output Artifacts are regular files.
*/
public static FilesetTraversalParams recursiveTraversalOfDirectory(Label ownerLabel,
Artifact directoryToTraverse, PathFragment destPath, @Nullable Set<String> excludes,
- SymlinkBehavior symlinkBehaviorMode, PackageBoundaryMode pkgBoundaryMode) {
+ SymlinkBehavior symlinkBehaviorMode, PackageBoundaryMode pkgBoundaryMode,
+ boolean strictFilesetOutput) {
return DirectoryTraversalParams.getDirectoryTraversalParams(ownerLabel,
DirectTraversalRoot.forFileOrDirectory(directoryToTraverse), false, destPath,
- excludes, symlinkBehaviorMode, pkgBoundaryMode, true,
+ excludes, symlinkBehaviorMode, pkgBoundaryMode, strictFilesetOutput, true,
!directoryToTraverse.isSourceArtifact());
}
@@ -96,14 +98,15 @@
* respective symlink there, or the root of files found (in case this is a directory)
* @param symlinkBehaviorMode what to do with symlinks
* @param pkgBoundaryMode what to do when the traversal hits a subdirectory that is also a
- * subpackage (contains a BUILD file)
+ * @param strictFilesetOutput whether Fileset assumes that output Artifacts are regular files.
*/
public static FilesetTraversalParams fileTraversal(Label ownerLabel, Artifact fileToTraverse,
PathFragment destPath, SymlinkBehavior symlinkBehaviorMode,
- PackageBoundaryMode pkgBoundaryMode) {
+ PackageBoundaryMode pkgBoundaryMode, boolean strictFilesetOutput) {
return DirectoryTraversalParams.getDirectoryTraversalParams(ownerLabel,
DirectTraversalRoot.forFileOrDirectory(fileToTraverse), false, destPath, null,
- symlinkBehaviorMode, pkgBoundaryMode, false, !fileToTraverse.isSourceArtifact());
+ symlinkBehaviorMode, pkgBoundaryMode, strictFilesetOutput, false,
+ !fileToTraverse.isSourceArtifact());
}
/**
@@ -175,11 +178,12 @@
@Nullable Set<String> excludes,
SymlinkBehavior symlinkBehaviorMode,
PackageBoundaryMode pkgBoundaryMode,
+ boolean strictFilesetOutput,
boolean isRecursive,
boolean isGenerated) {
DirectTraversal traversal = DirectTraversal.getDirectTraversal(root, isPackage,
- symlinkBehaviorMode == SymlinkBehavior.DEREFERENCE, pkgBoundaryMode, isRecursive,
- isGenerated);
+ symlinkBehaviorMode == SymlinkBehavior.DEREFERENCE, pkgBoundaryMode, strictFilesetOutput,
+ isRecursive, isGenerated);
return create(ownerLabel, destPath, getOrderedExcludes(excludes), Optional.of(traversal));
}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
index e7443ea..38e446c 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
@@ -420,6 +420,17 @@
public boolean strictFilesets;
@Option(
+ name = "experimental_strict_fileset_output",
+ defaultValue = "false",
+ documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY,
+ effectTags = {OptionEffectTag.EXECUTION},
+ help =
+ "If this option is enabled, filesets will treat all output artifacts as regular files. "
+ + "They will not traverse directories or be sensitive to symlinks."
+ )
+ public boolean strictFilesetOutput;
+
+ @Option(
name = "stamp",
defaultValue = "false",
documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS,
@@ -1475,6 +1486,10 @@
return options.strictFilesets;
}
+ public boolean isStrictFilesetOutput() {
+ return options.strictFilesetOutput;
+ }
+
public String getMainRepositoryName() {
return mainRepositoryName.strippedName();
}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunction.java
index bc91ed7..22f5b31 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunction.java
@@ -243,6 +243,7 @@
traversal.getRoot(),
traversal.isGenerated(),
traversal.getPackageBoundaryMode(),
+ traversal.isStrictFilesetOutput(),
traversal.isPackage(),
errorInfo);
RecursiveFilesystemTraversalValue v = (RecursiveFilesystemTraversalValue) env.getValue(depKey);
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java
index 070b831..c4a23b3 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java
@@ -245,33 +245,38 @@
}
}
- // FileArtifactValue does not currently track symlinks. If it did, we could potentially remove
- // some of the filesystem operations we're doing here.
- Path path = traversal.root.asRootedPath().asPath();
- FileStatus noFollowStat = path.stat(Symlinks.NOFOLLOW);
- FileStatus followStat = path.statIfFound(Symlinks.FOLLOW);
- FileType type;
- PathFragment unresolvedLinkTarget = null;
RootedPath realPath = traversal.root.asRootedPath();
- if (followStat == null) {
- type = FileType.DANGLING_SYMLINK;
- if (!noFollowStat.isSymbolicLink()) {
- throw new IOException("Expected symlink for " + path + ", but got: " + noFollowStat);
- }
- unresolvedLinkTarget = path.readSymbolicLink();
- } else if (noFollowStat.isFile()) {
- type = FileType.FILE;
- } else if (noFollowStat.isDirectory()) {
- type = FileType.DIRECTORY;
+ if (traversal.strictOutputFiles) {
+ Preconditions.checkNotNull(fsVal, "Strict Fileset output tree has null FileArtifactValue");
+ return new FileInfo(FileType.FILE, fsVal, realPath, null);
} else {
- unresolvedLinkTarget = path.readSymbolicLink();
- realPath = RootedPath.toRootedPath(
- Root.absoluteRoot(path.getFileSystem()),
- path.resolveSymbolicLinks());
- type = followStat.isFile() ? FileType.SYMLINK_TO_FILE : FileType.SYMLINK_TO_DIRECTORY;
+ // FileArtifactValue does not currently track symlinks. If it did, we could potentially
+ // remove some of the filesystem operations we're doing here.
+ Path path = traversal.root.asRootedPath().asPath();
+ FileStatus noFollowStat = path.stat(Symlinks.NOFOLLOW);
+ FileStatus followStat = path.statIfFound(Symlinks.FOLLOW);
+ FileType type;
+ PathFragment unresolvedLinkTarget = null;
+ if (followStat == null) {
+ type = FileType.DANGLING_SYMLINK;
+ if (!noFollowStat.isSymbolicLink()) {
+ throw new IOException("Expected symlink for " + path + ", but got: " + noFollowStat);
+ }
+ unresolvedLinkTarget = path.readSymbolicLink();
+ } else if (noFollowStat.isFile()) {
+ type = FileType.FILE;
+ } else if (noFollowStat.isDirectory()) {
+ type = FileType.DIRECTORY;
+ } else {
+ unresolvedLinkTarget = path.readSymbolicLink();
+ realPath =
+ RootedPath.toRootedPath(
+ Root.absoluteRoot(path.getFileSystem()), path.resolveSymbolicLinks());
+ type = followStat.isFile() ? FileType.SYMLINK_TO_FILE : FileType.SYMLINK_TO_DIRECTORY;
+ }
+ return new FileInfo(
+ type, fsVal != null ? fsVal : noFollowStat.hashCode(), realPath, unresolvedLinkTarget);
}
- return new FileInfo(
- type, fsVal != null ? fsVal : noFollowStat.hashCode(), realPath, unresolvedLinkTarget);
} else {
// Stat the file.
FileValue fileValue =
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalValue.java
index 94041f9..c849f10 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalValue.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalValue.java
@@ -124,6 +124,9 @@
/** Whether traversal should descend into directories that are roots of subpackages. */
final PackageBoundaryMode crossPkgBoundaries;
+ /** Whether Fileset assumes that output Artifacts are regular files. */
+ final boolean strictOutputFiles;
+
/**
* Whether to skip checking if the root (if it's a directory) contains a BUILD file.
*
@@ -140,11 +143,13 @@
DirectTraversalRoot root,
boolean isRootGenerated,
PackageBoundaryMode crossPkgBoundaries,
+ boolean strictOutputFiles,
boolean skipTestingForSubpackage,
@Nullable String errorInfo) {
this.root = root;
this.isRootGenerated = isRootGenerated;
this.crossPkgBoundaries = crossPkgBoundaries;
+ this.strictOutputFiles = strictOutputFiles;
this.skipTestingForSubpackage = skipTestingForSubpackage;
this.errorInfo = errorInfo;
}
@@ -155,17 +160,20 @@
DirectTraversalRoot root,
boolean isRootGenerated,
PackageBoundaryMode crossPkgBoundaries,
+ boolean strictOutputFiles,
boolean skipTestingForSubpackage,
@Nullable String errorInfo) {
return interner.intern(
new TraversalRequest(
- root, isRootGenerated, crossPkgBoundaries, skipTestingForSubpackage, errorInfo));
+ root, isRootGenerated, crossPkgBoundaries, strictOutputFiles,
+ skipTestingForSubpackage, errorInfo));
}
private TraversalRequest duplicate(DirectTraversalRoot newRoot,
boolean newSkipTestingForSubpackage) {
return create(
- newRoot, isRootGenerated, crossPkgBoundaries, newSkipTestingForSubpackage, errorInfo);
+ newRoot, isRootGenerated, crossPkgBoundaries, strictOutputFiles,
+ newSkipTestingForSubpackage, errorInfo);
}
/** Creates a new request to traverse a child element in the current directory (the root). */
@@ -198,20 +206,23 @@
return root.equals(o.root)
&& isRootGenerated == o.isRootGenerated
&& crossPkgBoundaries == o.crossPkgBoundaries
+ && strictOutputFiles == o.strictOutputFiles
&& skipTestingForSubpackage == o.skipTestingForSubpackage;
}
@Override
public int hashCode() {
- return Objects.hashCode(root, isRootGenerated, crossPkgBoundaries, skipTestingForSubpackage);
+ return Objects.hashCode(root, isRootGenerated, crossPkgBoundaries, strictOutputFiles,
+ skipTestingForSubpackage);
}
@Override
public String toString() {
return String.format(
"TraversalParams(root=%s, is_generated=%d, skip_testing_for_subpkg=%d,"
- + " pkg_boundaries=%s)",
- root, isRootGenerated ? 1 : 0, skipTestingForSubpackage ? 1 : 0, crossPkgBoundaries);
+ + " pkg_boundaries=%s, strictOutputFiles=%d)",
+ root, isRootGenerated ? 1 : 0, skipTestingForSubpackage ? 1 : 0, crossPkgBoundaries,
+ strictOutputFiles ? 1 : 0);
}
@Override
