Add support for VirtualActionInputs to the remote cache.
RELNOTES: None
PiperOrigin-RevId: 207137932
diff --git a/src/main/java/com/google/devtools/build/lib/remote/Chunker.java b/src/main/java/com/google/devtools/build/lib/remote/Chunker.java
index fb3202c..7e960ac 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/Chunker.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/Chunker.java
@@ -22,6 +22,7 @@
import com.google.common.io.ByteStreams;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.MetadataProvider;
+import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
import com.google.devtools.build.lib.remote.util.DigestUtil;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.remoteexecution.v1test.Digest;
@@ -146,6 +147,10 @@
this(actionInput, inputCache, execRoot, getDefaultChunkSize(), digestUtil);
}
+ public Chunker(VirtualActionInput actionInput, DigestUtil digestUtil) throws IOException {
+ this(actionInput.getBytes().toByteArray(), digestUtil);
+ }
+
public Chunker(
ActionInput actionInput,
MetadataProvider inputCache,
diff --git a/src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteCache.java b/src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteCache.java
index 253ab41..c175559 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteCache.java
@@ -28,6 +28,7 @@
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.MetadataProvider;
+import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
import com.google.devtools.build.lib.remote.Retrier.RetryException;
import com.google.devtools.build.lib.remote.TreeNodeRepository.TreeNode;
@@ -169,7 +170,11 @@
if (!missingActionInputs.isEmpty()) {
MetadataProvider inputFileCache = repository.getInputFileCache();
for (ActionInput actionInput : missingActionInputs) {
- toUpload.add(new Chunker(actionInput, inputFileCache, execRoot, digestUtil));
+ if (actionInput instanceof VirtualActionInput) {
+ toUpload.add(new Chunker((VirtualActionInput) actionInput, digestUtil));
+ } else {
+ toUpload.add(new Chunker(actionInput, inputFileCache, execRoot, digestUtil));
+ }
}
}
uploader.uploadBlobs(toUpload, true);
@@ -326,22 +331,6 @@
return digest;
}
- /**
- * Put the file contents cache if it is not already in it. No-op if the file is already stored in
- * cache. The given path must be a full absolute path.
- *
- * @return The key for fetching the file contents blob from cache.
- */
- Digest uploadFileContents(ActionInput input, Path execRoot, MetadataProvider inputCache)
- throws IOException, InterruptedException {
- Digest digest = DigestUtil.getFromInputCache(input, inputCache);
- ImmutableSet<Digest> missing = getMissingDigests(ImmutableList.of(digest));
- if (!missing.isEmpty()) {
- uploader.uploadBlob(new Chunker(input, inputCache, execRoot, digestUtil), true);
- }
- return digest;
- }
-
Digest uploadBlob(byte[] blob) throws IOException, InterruptedException {
Digest digest = digestUtil.compute(blob);
ImmutableSet<Digest> missing = getMissingDigests(ImmutableList.of(digest));
diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteCacheTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteCacheTest.java
index 3c61fd1..de453b3 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteCacheTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteCacheTest.java
@@ -28,12 +28,15 @@
import com.google.bytestream.ByteStreamProto.WriteResponse;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSortedMap;
import com.google.common.util.concurrent.ListeningScheduledExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.devtools.build.lib.actions.ActionInputHelper;
+import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
import com.google.devtools.build.lib.clock.JavaClock;
+import com.google.devtools.build.lib.remote.TreeNodeRepository.TreeNode;
import com.google.devtools.build.lib.remote.util.DigestUtil;
import com.google.devtools.build.lib.remote.util.DigestUtil.ActionKey;
import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
@@ -43,11 +46,13 @@
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
import com.google.devtools.common.options.Options;
import com.google.devtools.remoteexecution.v1test.Action;
import com.google.devtools.remoteexecution.v1test.ActionCacheGrpc.ActionCacheImplBase;
import com.google.devtools.remoteexecution.v1test.ActionResult;
+import com.google.devtools.remoteexecution.v1test.Command;
import com.google.devtools.remoteexecution.v1test.ContentAddressableStorageGrpc.ContentAddressableStorageImplBase;
import com.google.devtools.remoteexecution.v1test.Digest;
import com.google.devtools.remoteexecution.v1test.Directory;
@@ -74,7 +79,10 @@
import io.grpc.util.MutableHandlerRegistry;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -201,6 +209,98 @@
uploader);
}
+ static class StringVirtualActionInput implements VirtualActionInput {
+ private final String contents;
+ private final PathFragment execPath;
+
+ StringVirtualActionInput(String contents, PathFragment execPath) {
+ this.contents = contents;
+ this.execPath = execPath;
+ }
+
+ @Override
+ public void writeTo(OutputStream out) throws IOException {
+ out.write(contents.getBytes(StandardCharsets.UTF_8));
+ }
+
+ @Override
+ public ByteString getBytes() throws IOException {
+ ByteString.Output out = ByteString.newOutput();
+ writeTo(out);
+ return out.toByteString();
+ }
+
+ @Override
+ public String getExecPathString() {
+ return execPath.getPathString();
+ }
+
+ @Override
+ public PathFragment getExecPath() {
+ return execPath;
+ }
+ }
+
+ @Test
+ public void testVirtualActionInputSupport() throws Exception {
+ GrpcRemoteCache client = newClient();
+ TreeNodeRepository treeNodeRepository =
+ new TreeNodeRepository(execRoot, fakeFileCache, DIGEST_UTIL);
+ PathFragment execPath = PathFragment.create("my/exec/path");
+ VirtualActionInput virtualActionInput = new StringVirtualActionInput("hello", execPath);
+ Digest digest = DIGEST_UTIL.compute(virtualActionInput.getBytes().toByteArray());
+ TreeNode root =
+ treeNodeRepository.buildFromActionInputs(
+ ImmutableSortedMap.of(execPath, virtualActionInput));
+
+ // Add a fake CAS that responds saying that the above virtual action input is missing
+ serviceRegistry.addService(
+ new ContentAddressableStorageImplBase() {
+ @Override
+ public void findMissingBlobs(
+ FindMissingBlobsRequest request,
+ StreamObserver<FindMissingBlobsResponse> responseObserver) {
+ responseObserver.onNext(
+ FindMissingBlobsResponse.newBuilder().addMissingBlobDigests(digest).build());
+ responseObserver.onCompleted();
+ }
+ });
+
+ // Mock a byte stream and assert that we see the virtual action input with contents 'hello'
+ AtomicBoolean writeOccurred = new AtomicBoolean();
+ serviceRegistry.addService(
+ new ByteStreamImplBase() {
+ @Override
+ public StreamObserver<WriteRequest> write(
+ final StreamObserver<WriteResponse> responseObserver) {
+ return new StreamObserver<WriteRequest>() {
+ @Override
+ public void onNext(WriteRequest request) {
+ assertThat(request.getResourceName()).contains(digest.getHash());
+ assertThat(request.getFinishWrite()).isTrue();
+ assertThat(request.getData().toStringUtf8()).isEqualTo("hello");
+ writeOccurred.set(true);
+ }
+
+ @Override
+ public void onCompleted() {
+ responseObserver.onNext(WriteResponse.newBuilder().setCommittedSize(5).build());
+ responseObserver.onCompleted();
+ }
+
+ @Override
+ public void onError(Throwable t) {
+ fail("An error occurred: " + t);
+ }
+ };
+ }
+ });
+
+ // Upload all missing inputs (that is, the virtual action input from above)
+ client.ensureInputsPresent(treeNodeRepository, execRoot, root, Command.getDefaultInstance());
+ assertThat(writeOccurred.get()).named("WriteOccurred").isTrue();
+ }
+
@Test
public void testDownloadEmptyBlob() throws Exception {
GrpcRemoteCache client = newClient();
