Adds support for remote files.
PiperOrigin-RevId: 194413337
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java b/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java
index c79241b..b9a137d 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java
@@ -59,4 +59,14 @@
* and should be called.
*/
long getModifiedTime();
+
+ /**
+ * Index used to resolve remote files.
+ *
+ * <p>0 indicates that no such information is available which can mean that it's either a local
+ * file or empty.
+ */
+ default int getLocationIndex() {
+ return 0;
+ }
}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java b/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java
index fc6c229..5bda51a 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java
@@ -58,6 +58,10 @@
*/
void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest);
+ /** Injects a file that is only stored remotely. */
+ void injectRemoteFile(
+ Artifact output, byte[] digest, long size, long modifiedTime, int locationIndex);
+
/**
* Marks an artifact as intentionally omitted. Acknowledges that this Artifact could have existed,
* but was intentionally not saved, most likely as an optimization.
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java
index c809806..fe6e8af 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java
@@ -467,6 +467,50 @@
}
@Override
+ public void injectRemoteFile(
+ Artifact output, byte[] digest, long size, long modifiedTime, int locationIndex) {
+ Preconditions.checkState(
+ executionMode.get(), "Tried to inject %s outside of execution.", output);
+ Preconditions.checkArgument(
+ locationIndex != 0 || size == 0,
+ "output = %s, size = %s, locationIndex =%s",
+ output,
+ size,
+ locationIndex);
+
+ Preconditions.checkState(injectedFiles.add(output), output);
+
+ // TODO(shahan): there are a couple of things that could reduce memory usage
+ // 1. We might be able to skip creating an entry in `outputArtifactData` and only create
+ // the `FileArtifactValue`, but there are likely downstream consumers that expect it that
+ // would need to be cleaned up.
+ // 2. Instead of creating an `additionalOutputData` entry, we could add the extra
+ // `locationIndex` to `FileStateValue`.
+ FileStateValue fileStateValue =
+ new FileStateValue.RegularFileStateValue(size, digest, /*contentsProxy=*/ null);
+ RootedPath rootedPath =
+ RootedPath.toRootedPath(output.getRoot().getRoot(), output.getRootRelativePath());
+ FileValue value = FileValue.value(rootedPath, fileStateValue, rootedPath, fileStateValue);
+ FileValue oldFsValue = outputArtifactData.putIfAbsent(output, value);
+ try {
+ checkInconsistentData(output, oldFsValue, value);
+ } catch (IOException e) {
+ // Should never happen.
+ throw new IllegalStateException("Inconsistent FileValues for " + output, e);
+ }
+
+ FileArtifactValue artifactValue =
+ new FileArtifactValue.RemoteFileArtifactValue(digest, size, modifiedTime, locationIndex);
+ FileArtifactValue oldArtifactValue = additionalOutputData.putIfAbsent(output, artifactValue);
+ try {
+ checkInconsistentData(output, oldArtifactValue, artifactValue);
+ } catch (IOException e) {
+ // Should never happen.
+ throw new IllegalStateException("Inconsistent FileArtifactValues for " + output, e);
+ }
+ }
+
+ @Override
public void markOmitted(ActionInput output) {
Preconditions.checkState(executionMode.get());
if (output instanceof Artifact) {
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java
index 8d17187..fbbdca1 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java
@@ -238,6 +238,70 @@
}
}
+ static final class RemoteFileArtifactValue extends FileArtifactValue {
+ private final byte[] digest;
+ private final long size;
+ private final long modifiedTime;
+ private final int locationIndex;
+
+ RemoteFileArtifactValue(byte[] digest, long size, long modifiedTime, int locationIndex) {
+ this.digest = digest;
+ this.size = size;
+ this.modifiedTime = modifiedTime;
+ this.locationIndex = locationIndex;
+ }
+
+ @Override
+ public FileStateType getType() {
+ return FileStateType.REGULAR_FILE;
+ }
+
+ @Override
+ public byte[] getDigest() {
+ return digest;
+ }
+
+ @Override
+ public long getSize() {
+ return size;
+ }
+
+ @Override
+ public long getModifiedTime() {
+ return modifiedTime;
+ }
+
+ @Override
+ public int getLocationIndex() {
+ return locationIndex;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof RemoteFileArtifactValue)) {
+ return false;
+ }
+ RemoteFileArtifactValue r = (RemoteFileArtifactValue) o;
+ return Arrays.equals(digest, r.digest)
+ && size == r.size
+ && modifiedTime == r.modifiedTime
+ && locationIndex == r.locationIndex;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(Arrays.hashCode(digest), size, modifiedTime, locationIndex);
+ }
+
+ @Override
+ public boolean wasModifiedSinceDigest(Path path) {
+ throw new UnsupportedOperationException();
+ }
+ }
+
static FileArtifactValue create(Artifact artifact, FileValue fileValue) throws IOException {
boolean isFile = fileValue.isFile();
FileContentsProxy proxy = getProxyFromFileStateValue(fileValue.realFileStateValue());
diff --git a/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java b/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java
index 1266ab8..753a73e 100644
--- a/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java
+++ b/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java
@@ -729,6 +729,12 @@
}
@Override
+ public void injectRemoteFile(
+ Artifact output, byte[] digest, long size, long modifiedTime, int locationIndex) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public void markOmitted(ActionInput output) {
throw new UnsupportedOperationException();
}
