Optionally expand Fileset in BEP. When referenced in top-level targets, we now show the expanded Fileset in BEP when enabled.
RELNOTES: None
PiperOrigin-RevId: 258422325
diff --git a/src/main/java/com/google/devtools/build/lib/actions/CompletionContext.java b/src/main/java/com/google/devtools/build/lib/actions/CompletionContext.java
index a4f9848..e372a26 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/CompletionContext.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/CompletionContext.java
@@ -17,15 +17,23 @@
import com.google.common.collect.ImmutableList;
import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander;
import com.google.devtools.build.lib.actions.Artifact.ArtifactExpanderImpl;
+import com.google.devtools.build.lib.actions.FilesetManifest.RelativeSymlinkBehavior;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import javax.annotation.Nullable;
/**
* {@link CompletionContext} contains an {@link ArtifactExpander} and {@link ArtifactPathResolver}
* used to resolve output files during a {@link
* com.google.devtools.build.lib.skyframe.CompletionFunction} evaluation.
+ *
+ * <p>Note that output Artifacts may in fact refer to aggregations, namely tree artifacts and
+ * Filesets. We expand these aggregations when visiting artifacts.
*/
@AutoValue
public abstract class CompletionContext {
@@ -36,31 +44,59 @@
public abstract ArtifactPathResolver pathResolver();
+ public abstract boolean expandFilesets();
+
+ @Nullable
+ public abstract Path execRoot();
+
public static CompletionContext create(
Map<Artifact, Collection<Artifact>> expandedArtifacts,
Map<Artifact, ImmutableList<FilesetOutputSymlink>> expandedFilesets,
+ boolean expandFilesets,
ActionInputMap inputMap,
PathResolverFactory pathResolverFactory,
- String workspaceName) {
+ Path execRoot,
+ String workspaceName)
+ throws IOException {
ArtifactExpander expander = new ArtifactExpanderImpl(expandedArtifacts, expandedFilesets);
ArtifactPathResolver pathResolver =
pathResolverFactory.shouldCreatePathResolverForArtifactValues()
? pathResolverFactory.createPathResolverForArtifactValues(
- inputMap, expandedArtifacts, expandedFilesets.keySet(), workspaceName)
+ inputMap, expandedArtifacts, expandedFilesets, workspaceName)
: ArtifactPathResolver.IDENTITY;
- return new AutoValue_CompletionContext(expander, pathResolver);
+ return new AutoValue_CompletionContext(expander, pathResolver, expandFilesets, execRoot);
}
private static CompletionContext createNull() {
- return new AutoValue_CompletionContext((artifact, output) -> {}, ArtifactPathResolver.IDENTITY);
+ return new AutoValue_CompletionContext(
+ (artifact, output) -> {}, ArtifactPathResolver.IDENTITY, false, null);
}
public void visitArtifacts(Iterable<Artifact> artifacts, ArtifactReceiver receiver) {
for (Artifact artifact : artifacts) {
- if (artifact.isMiddlemanArtifact() || artifact.isFileset()) {
- // We never want to report middleman artifacts. They are for internal use only.
- // Filesets are not currently supported, but should be in the future.
+ if (artifact.isMiddlemanArtifact()) {
continue;
+ } else if (artifact.isFileset()) {
+ if (!expandFilesets()) {
+ continue;
+ }
+ ImmutableList<FilesetOutputSymlink> links = expander().getFileset(artifact);
+ FilesetManifest filesetManifest;
+ try {
+ filesetManifest =
+ FilesetManifest.constructFilesetManifest(
+ links, PathFragment.EMPTY_FRAGMENT, RelativeSymlinkBehavior.RESOLVE);
+ } catch (IOException e) {
+ // Unexpected: RelativeSymlinkBehavior.RESOLVE should not throw.
+ throw new IllegalStateException(e);
+ }
+
+ for (Map.Entry<PathFragment, String> mapping : filesetManifest.getEntries().entrySet()) {
+ String targetFile = mapping.getValue();
+ PathFragment locationInFileset = mapping.getKey();
+ receiver.acceptFilesetMapping(
+ artifact, locationInFileset, execRoot().getRelative(targetFile));
+ }
} else if (artifact.isTreeArtifact()) {
List<Artifact> expandedArtifacts = new ArrayList<>();
expander().expand(artifact, expandedArtifacts);
@@ -74,9 +110,10 @@
}
/** A function that accepts an {@link Artifact}. */
- @FunctionalInterface
public interface ArtifactReceiver {
- void accept(Artifact a);
+ void accept(Artifact artifact);
+
+ void acceptFilesetMapping(Artifact fileset, PathFragment relName, Path targetFile);
}
/** A factory for {@link ArtifactPathResolver}. */
@@ -84,8 +121,9 @@
ArtifactPathResolver createPathResolverForArtifactValues(
ActionInputMap actionInputMap,
Map<Artifact, Collection<Artifact>> expandedArtifacts,
- Iterable<Artifact> filesets,
- String workspaceName);
+ Map<Artifact, ImmutableList<FilesetOutputSymlink>> filesets,
+ String workspaceName)
+ throws IOException;
boolean shouldCreatePathResolverForArtifactValues();
}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/FilesetManifest.java b/src/main/java/com/google/devtools/build/lib/actions/FilesetManifest.java
new file mode 100644
index 0000000..3d457db
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/FilesetManifest.java
@@ -0,0 +1,184 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Logger;
+import javax.annotation.Nullable;
+
+/**
+ * Representation of a Fileset manifest.
+ */
+public final class FilesetManifest {
+ private static final Logger logger = Logger.getLogger(FilesetManifest.class.getName());
+
+ /**
+ * Mode that determines how to handle relative target paths.
+ */
+ public enum RelativeSymlinkBehavior {
+ /** Ignore any relative target paths. */
+ IGNORE,
+
+ /** Give an error if a relative target path is encountered. */
+ ERROR,
+
+ /** Resolve all relative target paths. */
+ RESOLVE;
+ }
+
+ public static FilesetManifest constructFilesetManifest(
+ List<FilesetOutputSymlink> outputSymlinks,
+ PathFragment targetPrefix,
+ RelativeSymlinkBehavior relSymlinkBehavior)
+ throws IOException {
+ LinkedHashMap<PathFragment, String> entries = new LinkedHashMap<>();
+ Map<PathFragment, String> relativeLinks = new HashMap<>();
+ Map<String, FileArtifactValue> artifactValues = new HashMap<>();
+ for (FilesetOutputSymlink outputSymlink : outputSymlinks) {
+ PathFragment fullLocation = targetPrefix.getRelative(outputSymlink.getName());
+ String artifact = Strings.emptyToNull(outputSymlink.getTargetPath().getPathString());
+ if (isRelativeSymlink(outputSymlink)) {
+ addRelativeSymlinkEntry(artifact, fullLocation, relSymlinkBehavior, relativeLinks);
+ } else if (!entries.containsKey(fullLocation)) { // Keep consistent behavior: no overwriting.
+ entries.put(fullLocation, artifact);
+ }
+ if (outputSymlink.getMetadata() instanceof FileArtifactValue) {
+ artifactValues.put(artifact, (FileArtifactValue) outputSymlink.getMetadata());
+ }
+ }
+ resolveRelativeSymlinks(entries, relativeLinks);
+ return new FilesetManifest(entries, artifactValues);
+ }
+
+ private static boolean isRelativeSymlink(FilesetOutputSymlink symlink) {
+ return !symlink.getTargetPath().isEmpty()
+ && !symlink.getTargetPath().isAbsolute()
+ && !symlink.isRelativeToExecRoot();
+ }
+
+ /** Potentially adds the relative symlink to the map, depending on {@code relSymlinkBehavior}. */
+ private static void addRelativeSymlinkEntry(
+ @Nullable String artifact,
+ PathFragment fullLocation,
+ RelativeSymlinkBehavior relSymlinkBehavior,
+ Map<PathFragment, String> relativeLinks)
+ throws IOException {
+ switch (relSymlinkBehavior) {
+ case ERROR:
+ throw new IOException("runfiles target is not absolute: " + artifact);
+ case RESOLVE:
+ if (!relativeLinks.containsKey(fullLocation)) { // Keep consistent behavior: no overwriting.
+ relativeLinks.put(fullLocation, artifact);
+ }
+ break;
+ case IGNORE:
+ break; // Do nothing.
+ }
+ }
+
+ private static final int MAX_SYMLINK_TRAVERSALS = 256;
+
+ /**
+ * Resolves relative symlinks and puts them in the {@code entries} map.
+ *
+ * <p>Note that {@code relativeLinks} should only contain entries in {@link
+ * RelativeSymlinkBehavior#RESOLVE} mode.
+ */
+ private static void resolveRelativeSymlinks(
+ Map<PathFragment, String> entries, Map<PathFragment, String> relativeLinks) {
+ for (Map.Entry<PathFragment, String> e : relativeLinks.entrySet()) {
+ PathFragment location = e.getKey();
+ String value = e.getValue();
+ String actual = Preconditions.checkNotNull(value, e);
+ Preconditions.checkState(!actual.startsWith("/"), e);
+ PathFragment actualLocation = location;
+
+ // Recursively resolve relative symlinks.
+ LinkedHashSet<String> seen = new LinkedHashSet<>();
+ int traversals = 0;
+ do {
+ actualLocation = actualLocation.getParentDirectory().getRelative(actual);
+ actual = relativeLinks.get(actualLocation);
+ } while (++traversals <= MAX_SYMLINK_TRAVERSALS && actual != null && seen.add(actual));
+
+ if (traversals >= MAX_SYMLINK_TRAVERSALS) {
+ logger.warning(
+ "Symlink "
+ + location
+ + " is part of a chain of length at least "
+ + traversals
+ + " which exceeds Blaze's maximum allowable symlink chain length");
+ } else if (actual != null) {
+ // TODO(b/113128395): throw here.
+ logger.warning("Symlink " + location + " forms a symlink cycle: " + seen);
+ } else if (!entries.containsKey(actualLocation)) {
+ // We've found a relative symlink that points out of the fileset. We should really always
+ // throw here, but current behavior is that we tolerate such symlinks when they occur in
+ // runfiles, which is the only time this code is hit.
+ // TODO(b/113128395): throw here.
+ logger.warning(
+ "Symlink "
+ + location
+ + " (transitively) points to "
+ + actualLocation
+ + " that is not in this fileset (or was pruned because of a cycle)");
+ } else {
+ // We have successfully resolved the symlink.
+ entries.put(location, entries.get(actualLocation));
+ }
+ }
+ }
+
+ private final Map<PathFragment, String> entries;
+ private final Map<String, FileArtifactValue> artifactValues;
+
+ private FilesetManifest(
+ Map<PathFragment, String> entries, Map<String, FileArtifactValue> artifactValues) {
+ this.entries = Collections.unmodifiableMap(entries);
+ this.artifactValues = artifactValues;
+ }
+
+ /**
+ * Returns a mapping of symlink name to its target path.
+ *
+ * <p>Values in this map can be:
+ *
+ * <ul>
+ * <li>An absolute path.
+ * <li>A relative path, which should be considered relative to the exec root.
+ * <li>{@code null}, which represents an empty file.
+ * </ul>
+ */
+ public Map<PathFragment, String> getEntries() {
+ return entries;
+ }
+
+ /**
+ * Returns a mapping of target path to {@link FileArtifactValue}.
+ *
+ * <p>The keyset of this map is a subset of the values in the map returned by {@link #getEntries}.
+ */
+ public Map<String, FileArtifactValue> getArtifactValues() {
+ return artifactValues;
+ }
+}
