ActionFS correctly tracks symlink sources.
PiperOrigin-RevId: 199820207
diff --git a/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java b/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java
index f9f80ee..0ae5592a 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java
@@ -23,6 +23,7 @@
import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec;
import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Symlinks;
import com.google.devtools.build.skyframe.SkyValue;
import java.io.ByteArrayInputStream;
@@ -426,6 +427,62 @@
}
}
+ /**
+ * Used to resolve source symlinks when diskless.
+ *
+ * <p>When {@link com.google.devtools.build.lib.skyframe.ActionFileSystem} creates symlinks, it
+ * relies on metadata ({@link FileArtifactValue}) to resolve the actual underlying data. In the
+ * case of remote or inline files, this information is self-contained. However, in the case of
+ * source files, the path is required to resolve the content.
+ */
+ public static final class SourceFileArtifactValue extends FileArtifactValue {
+ private final PathFragment execPath;
+ private final int sourceRootIndex;
+ private final byte[] digest;
+ private final long size;
+
+ public SourceFileArtifactValue(
+ PathFragment execPath, int sourceRootIndex, byte[] digest, long size) {
+ this.execPath = Preconditions.checkNotNull(execPath);
+ this.sourceRootIndex = sourceRootIndex;
+ this.digest = Preconditions.checkNotNull(digest);
+ this.size = size;
+ }
+
+ public PathFragment getExecPath() {
+ return execPath;
+ }
+
+ public int getSourceRootIndex() {
+ return sourceRootIndex;
+ }
+
+ @Override
+ public FileStateType getType() {
+ return FileStateType.REGULAR_FILE;
+ }
+
+ @Override
+ public byte[] getDigest() {
+ return digest;
+ }
+
+ @Override
+ public long getSize() {
+ return size;
+ }
+
+ @Override
+ public long getModifiedTime() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean wasModifiedSinceDigest(Path path) {
+ throw new UnsupportedOperationException();
+ }
+ }
+
private static FileContentsProxy getProxyFromFileStateValue(FileStateValue value) {
if (value instanceof FileStateValue.RegularFileStateValue) {
return ((FileStateValue.RegularFileStateValue) value).getContentsProxy();
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionFileSystem.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionFileSystem.java
index ab02229..5abb9c8 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/ActionFileSystem.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionFileSystem.java
@@ -28,7 +28,9 @@
import com.google.devtools.build.lib.actions.ActionInputMap;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.FileArtifactValue;
+import com.google.devtools.build.lib.actions.FileArtifactValue.InlineFileArtifactValue;
import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue;
+import com.google.devtools.build.lib.actions.FileArtifactValue.SourceFileArtifactValue;
import com.google.devtools.build.lib.actions.FileStateType;
import com.google.devtools.build.lib.actions.MetadataProvider;
import com.google.devtools.build.lib.profiler.Profiler;
@@ -64,6 +66,7 @@
*/
final class ActionFileSystem extends FileSystem implements MetadataProvider, InjectionListener {
private static final Logger LOGGER = Logger.getLogger(ActionFileSystem.class.getName());
+ public static final BaseEncoding LOWER_CASE_HEX = BaseEncoding.base16().lowerCase();
/** Actual underlying filesystem. */
private final FileSystem delegate;
@@ -158,7 +161,13 @@
@Override
@Nullable
public ActionInput getInput(String execPath) {
- return inputArtifactData.getInput(execPath);
+ ActionInput input = inputArtifactData.getInput(execPath);
+ if (input != null) {
+ return input;
+ }
+ OptionalInputMetadata metadata =
+ optionalInputs.get(PathFragment.createAlreadyNormalized(execPath));
+ return metadata == null ? null : metadata.getArtifact();
}
// -------------------- InjectionListener Implementation --------------------
@@ -272,6 +281,24 @@
}
@Override
+ public byte[] getxattr(Path path, String name) throws IOException {
+ FileArtifactValue metadata = getMetadataChecked(asExecPath(path));
+ if (metadata instanceof RemoteFileArtifactValue) {
+ RemoteFileArtifactValue remote = (RemoteFileArtifactValue) metadata;
+ // TODO(b/80244718): inject ActionFileSystem from elsewhere and replace with correct metadata
+ return ("/CENSORED_BY_LEAKR/"
+ + remote.getLocationIndex()
+ + "/"
+ + LOWER_CASE_HEX.encode(remote.getDigest()))
+ .getBytes(US_ASCII);
+ }
+ if (metadata instanceof SourceFileArtifactValue) {
+ return resolveSourcePath((SourceFileArtifactValue) metadata).getxattr(name);
+ }
+ return delegate.getPath(path.asFragment()).getxattr(name);
+ }
+
+ @Override
protected byte[] getFastDigest(Path path, HashFunction hash) throws IOException {
if (hash != HashFunction.MD5) {
return null;
@@ -318,7 +345,37 @@
@Override
protected void createSymbolicLink(Path linkPath, PathFragment targetFragment) throws IOException {
- PathFragment targetExecPath = asExecPath(targetFragment);
+ // TODO(shahan): this might need to be loosened, but will require more information
+ Preconditions.checkArgument(
+ targetFragment.isAbsolute(),
+ "ActionFileSystem requires symlink targets to be absolute: %s -> %s",
+ linkPath,
+ targetFragment);
+
+ // When creating symbolic links, it matters whether target is a source path or not because
+ // the metadata needs to be handled differently in that case.
+ PathFragment targetExecPath = null;
+ int sourceRootIndex = -1; // index into sourceRoots or -1 if not a source
+ if (targetFragment.startsWith(execRootFragment)) {
+ targetExecPath = targetFragment.relativeTo(execRootFragment);
+ } else {
+ for (int i = 0; i < sourceRoots.size(); ++i) {
+ if (targetFragment.startsWith(sourceRoots.get(i))) {
+ targetExecPath = targetFragment.relativeTo(sourceRoots.get(i));
+ sourceRootIndex = i;
+ break;
+ }
+ }
+ if (sourceRootIndex == -1) {
+ throw new IllegalArgumentException(
+ linkPath
+ + " was not found under any known root: "
+ + execRootFragment
+ + ", "
+ + sourceRoots);
+ }
+ }
+
FileArtifactValue inputMetadata = inputArtifactData.getMetadata(targetExecPath.getPathString());
if (inputMetadata == null) {
OptionalInputMetadata metadataHolder = optionalInputs.get(targetExecPath);
@@ -337,7 +394,14 @@
createSymbolicLinkErrorMessage(
linkPath, targetFragment, linkPath + " is not an output."));
}
- outputHolder.set(inputMetadata, /*notifyConsumer=*/ true);
+ if (sourceRootIndex >= 0) {
+ outputHolder.set(
+ new SourceFileArtifactValue(
+ targetExecPath, sourceRootIndex, inputMetadata.getDigest(), inputMetadata.getSize()),
+ true);
+ } else {
+ outputHolder.set(inputMetadata, /*notifyConsumer=*/ true);
+ }
}
@Override
@@ -380,11 +444,14 @@
@Override
protected InputStream getInputStream(Path path) throws IOException {
FileArtifactValue metadata = getMetadataChecked(asExecPath(path));
- if (metadata instanceof FileArtifactValue.InlineFileArtifactValue) {
- return ((FileArtifactValue.InlineFileArtifactValue) metadata).getInputStream();
+ if (metadata instanceof InlineFileArtifactValue) {
+ return ((InlineFileArtifactValue) metadata).getInputStream();
+ }
+ if (metadata instanceof SourceFileArtifactValue) {
+ return resolveSourcePath((SourceFileArtifactValue) metadata).getInputStream();
}
Preconditions.checkArgument(
- !(metadata instanceof FileArtifactValue.RemoteFileArtifactValue),
+ !(metadata instanceof RemoteFileArtifactValue),
"getInputStream called for remote file: %s",
path);
return delegate.getPath(path.asFragment()).getInputStream();
@@ -505,6 +572,13 @@
return ByteString.copyFrom(BaseEncoding.base16().lowerCase().encode(digest).getBytes(US_ASCII));
}
+ /** NB: resolves to the underlying filesytem instead of this one. */
+ private Path resolveSourcePath(SourceFileArtifactValue metadata) {
+ return delegate
+ .getPath(sourceRoots.get(metadata.getSourceRootIndex()))
+ .getRelative(metadata.getExecPath());
+ }
+
@FunctionalInterface
public interface MetadataConsumer {
void accept(Artifact artifact, FileArtifactValue value) throws IOException;
@@ -518,6 +592,10 @@
this.artifact = artifact;
}
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
public FileArtifactValue get() throws IOException {
if (metadata == null) {
synchronized (this) {
@@ -582,8 +660,8 @@
super.close();
byte[] data = toByteArray();
set(
- new FileArtifactValue.InlineFileArtifactValue(
- data, Hashing.md5().hashBytes(data).asBytes()), /*notifyConsumer=*/ true);
+ new InlineFileArtifactValue(data, Hashing.md5().hashBytes(data).asBytes()),
+ /*notifyConsumer=*/ true);
}
};
}
