Pull upload(ActionResult) into super class.
Remove the custom upload(ActionResult) implementations from SimpleBlobStoreActionCache and GrpcRemoteCache.
This is a big step towards merging SimpleBlobStoreActionCache and GrpcRemoteCache.
Closes #9167.
PiperOrigin-RevId: 264563384
diff --git a/src/main/java/com/google/devtools/build/lib/remote/AbstractRemoteActionCache.java b/src/main/java/com/google/devtools/build/lib/remote/AbstractRemoteActionCache.java
index d53bb72..b5dbc4c 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/AbstractRemoteActionCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/AbstractRemoteActionCache.java
@@ -29,6 +29,7 @@
import build.bazel.remote.execution.v2.Tree;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
@@ -79,6 +80,7 @@
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nullable;
@@ -120,20 +122,8 @@
abstract ActionResult getCachedActionResult(ActionKey actionKey)
throws IOException, InterruptedException;
- /**
- * Upload the result of a locally executed action to the remote cache.
- *
- * @throws IOException if there was an error uploading to the remote cache
- * @throws ExecException if uploading any of the action outputs is not supported
- */
- abstract void upload(
- SimpleBlobStore.ActionKey actionKey,
- Action action,
- Command command,
- Path execRoot,
- Collection<Path> files,
- FileOutErr outErr)
- throws ExecException, IOException, InterruptedException;
+ protected abstract void setCachedActionResult(ActionKey actionKey, ActionResult action)
+ throws IOException, InterruptedException;
/**
* Uploads a file
@@ -157,6 +147,116 @@
*/
protected abstract ListenableFuture<Void> uploadBlob(Digest digest, ByteString data);
+ protected abstract ImmutableSet<Digest> getMissingDigests(Iterable<Digest> digests)
+ throws IOException, InterruptedException;
+
+ /**
+ * Upload the result of a locally executed action to the remote cache.
+ *
+ * @throws IOException if there was an error uploading to the remote cache
+ * @throws ExecException if uploading any of the action outputs is not supported
+ */
+ public ActionResult upload(
+ ActionKey actionKey,
+ Action action,
+ Command command,
+ Path execRoot,
+ Collection<Path> outputs,
+ FileOutErr outErr,
+ int exitCode)
+ throws ExecException, IOException, InterruptedException {
+ ActionResult.Builder resultBuilder = ActionResult.newBuilder();
+ uploadOutputs(execRoot, actionKey, action, command, outputs, outErr, resultBuilder);
+ resultBuilder.setExitCode(exitCode);
+ ActionResult result = resultBuilder.build();
+ if (exitCode == 0 && !action.getDoNotCache()) {
+ setCachedActionResult(actionKey, result);
+ }
+ return result;
+ }
+
+ public ActionResult upload(
+ ActionKey actionKey,
+ Action action,
+ Command command,
+ Path execRoot,
+ Collection<Path> outputs,
+ FileOutErr outErr)
+ throws ExecException, IOException, InterruptedException {
+ return upload(actionKey, action, command, execRoot, outputs, outErr, /* exitCode= */ 0);
+ }
+
+ private void uploadOutputs(
+ Path execRoot,
+ ActionKey actionKey,
+ Action action,
+ Command command,
+ Collection<Path> files,
+ FileOutErr outErr,
+ ActionResult.Builder result)
+ throws ExecException, IOException, InterruptedException {
+ UploadManifest manifest =
+ new UploadManifest(
+ digestUtil,
+ result,
+ execRoot,
+ options.incompatibleRemoteSymlinks,
+ options.allowSymlinkUpload);
+ manifest.addFiles(files);
+ manifest.setStdoutStderr(outErr);
+ manifest.addAction(actionKey, action, command);
+
+ Map<Digest, Path> digestToFile = manifest.getDigestToFile();
+ Map<Digest, ByteString> digestToBlobs = manifest.getDigestToBlobs();
+ Collection<Digest> digests = new ArrayList<>();
+ digests.addAll(digestToFile.keySet());
+ digests.addAll(digestToBlobs.keySet());
+
+ ImmutableSet<Digest> digestsToUpload = getMissingDigests(digests);
+ ImmutableList.Builder<ListenableFuture<Void>> uploads = ImmutableList.builder();
+ for (Digest digest : digestsToUpload) {
+ Path file = digestToFile.get(digest);
+ if (file != null) {
+ uploads.add(uploadFile(digest, file));
+ } else {
+ ByteString blob = digestToBlobs.get(digest);
+ if (blob == null) {
+ String message = "FindMissingBlobs call returned an unknown digest: " + digest;
+ throw new IOException(message);
+ }
+ uploads.add(uploadBlob(digest, blob));
+ }
+ }
+
+ waitForUploads(uploads.build());
+
+ if (manifest.getStderrDigest() != null) {
+ result.setStderrDigest(manifest.getStderrDigest());
+ }
+ if (manifest.getStdoutDigest() != null) {
+ result.setStdoutDigest(manifest.getStdoutDigest());
+ }
+ }
+
+ private static void waitForUploads(List<ListenableFuture<Void>> uploads)
+ throws IOException, InterruptedException {
+ try {
+ for (ListenableFuture<Void> upload : uploads) {
+ upload.get();
+ }
+ } catch (ExecutionException e) {
+ // TODO(buchgr): Add support for cancellation and factor this method out to be shared
+ // between ByteStreamUploader as well.
+ Throwable cause = e.getCause();
+ Throwables.throwIfInstanceOf(cause, IOException.class);
+ Throwables.throwIfInstanceOf(cause, InterruptedException.class);
+ if (cause != null) {
+ throw new IOException(cause);
+ }
+ throw new IOException(e);
+ }
+ }
+
/**
* Downloads a blob with a content hash {@code digest} to {@code out}.
*
