Suppress last-ditch download exceptions w/cleanup
Create an encapsulating DownloadException to represent the aggregate
exception set of an ActionResult download. IOExceptions will be retained
through exception suppression, and the outer exception has a property to
indicate if it only represents a sequence of CacheNotFoundExceptions.
InterruptedExceptions interception is improved to cancel pending work
and wrap, through suppression, any DownloadException that also occurred
during the download. InterruptedException being thrown on the download
control thread, it does not require suppression of further interrupts,
and can represent an outer download exception. Thread interrupt status
is suppressed for cancellations, and conveyed on throw.
These exception wrapping efforts allow non-asynchronous frame
representation in stack traces, and much clearer identification of
sources within remote strategy execution which produce failures based on
remote errors.
Any DownloadException in the last-ditch output download under
handleError in RemoteSpawnRunner is added as suppressed to the
initiating exception. Other exceptions (likely local IO) present clear
immediate traces and do not require specialized treatment.
Closes #10029.
PiperOrigin-RevId: 306619678
diff --git a/src/main/java/com/google/devtools/build/lib/remote/BulkTransferException.java b/src/main/java/com/google/devtools/build/lib/remote/BulkTransferException.java
new file mode 100644
index 0000000..977e19e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/remote/BulkTransferException.java
@@ -0,0 +1,49 @@
+// Copyright 2020 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.remote;
+
+import com.google.devtools.build.lib.remote.common.CacheNotFoundException;
+import java.io.IOException;
+
+/**
+ * Exception which represents a collection of IOExceptions for the purpose of distinguishing remote
+ * communication exceptions from those which occur on filesystems locally. This exception serves as
+ * a trace point for the actual transfer, so that the intended operation can be observed in a stack,
+ * with all constituent exceptions available for observation.
+ */
+class BulkTransferException extends IOException {
+ // true since no empty BulkTransferException is ever thrown
+ private boolean allCacheNotFoundException = true;
+
+ BulkTransferException() {}
+
+ BulkTransferException(IOException e) {
+ add(e);
+ }
+
+ /**
+ * Add an IOException to the suppressed list.
+ *
+ * <p>The Java standard addSuppressed is final and this method stands in its place to selectively
+ * filter and record whether all suppressed exceptions are CacheNotFoundExceptions
+ */
+ void add(IOException e) {
+ allCacheNotFoundException &= e instanceof CacheNotFoundException;
+ super.addSuppressed(e);
+ }
+
+ boolean onlyCausedByCacheNotFoundException() {
+ return allCacheNotFoundException;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java
index 1a1be30..caba306 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java
@@ -14,6 +14,7 @@
package com.google.devtools.build.lib.remote;
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
+import static com.google.devtools.build.lib.remote.util.Utils.getFromFuture;
import build.bazel.remote.execution.v2.Action;
import build.bazel.remote.execution.v2.ActionResult;
@@ -27,9 +28,7 @@
import build.bazel.remote.execution.v2.OutputSymlink;
import build.bazel.remote.execution.v2.SymlinkNode;
import build.bazel.remote.execution.v2.Tree;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
@@ -57,7 +56,6 @@
import com.google.devtools.build.lib.remote.common.RemoteCacheClient.ActionKey;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
import com.google.devtools.build.lib.remote.util.DigestUtil;
-import com.google.devtools.build.lib.remote.util.Utils;
import com.google.devtools.build.lib.remote.util.Utils.InMemoryOutput;
import com.google.devtools.build.lib.util.io.FileOutErr;
import com.google.devtools.build.lib.util.io.OutErr;
@@ -80,7 +78,6 @@
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nullable;
@@ -116,7 +113,7 @@
public ActionResult downloadActionResult(ActionKey actionKey, boolean inlineOutErr)
throws IOException, InterruptedException {
- return Utils.getFromFuture(cacheProtocol.downloadActionResult(actionKey, inlineOutErr));
+ return getFromFuture(cacheProtocol.downloadActionResult(actionKey, inlineOutErr));
}
/**
@@ -181,8 +178,7 @@
digests.addAll(digestToFile.keySet());
digests.addAll(digestToBlobs.keySet());
- ImmutableSet<Digest> digestsToUpload =
- Utils.getFromFuture(cacheProtocol.findMissingDigests(digests));
+ ImmutableSet<Digest> digestsToUpload = getFromFuture(cacheProtocol.findMissingDigests(digests));
ImmutableList.Builder<ListenableFuture<Void>> uploads = ImmutableList.builder();
for (Digest digest : digestsToUpload) {
Path file = digestToFile.get(digest);
@@ -198,7 +194,7 @@
}
}
- waitForUploads(uploads.build());
+ waitForBulkTransfer(uploads.build(), /* cancelRemainingOnInterrupt=*/ false);
if (manifest.getStderrDigest() != null) {
result.setStderrDigest(manifest.getStderrDigest());
@@ -208,22 +204,45 @@
}
}
- private static void waitForUploads(List<ListenableFuture<Void>> uploads)
- throws IOException, InterruptedException {
- try {
- for (ListenableFuture<Void> upload : uploads) {
- upload.get();
+ protected static <T> void waitForBulkTransfer(
+ Iterable<ListenableFuture<T>> transfers, boolean cancelRemainingOnInterrupt)
+ throws BulkTransferException, InterruptedException {
+ BulkTransferException bulkTransferException = null;
+ InterruptedException interruptedException = null;
+ boolean interrupted = Thread.currentThread().isInterrupted();
+ for (ListenableFuture<T> transfer : transfers) {
+ try {
+ if (interruptedException == null) {
+ // Wait for all downloads to finish.
+ getFromFuture(transfer);
+ } else {
+ transfer.cancel(true);
+ }
+ } catch (IOException e) {
+ if (bulkTransferException == null) {
+ bulkTransferException = new BulkTransferException();
+ }
+ bulkTransferException.add(e);
+ } catch (InterruptedException e) {
+ interrupted = Thread.interrupted() || interrupted;
+ interruptedException = e;
+ if (!cancelRemainingOnInterrupt) {
+ // leave the rest of the transfers alone
+ break;
+ }
}
- } catch (ExecutionException e) {
- // TODO(buchgr): Add support for cancellation and factor this method out to be shared
- // between ByteStreamUploader as well.
- Throwable cause = e.getCause();
- Throwables.throwIfInstanceOf(cause, IOException.class);
- Throwables.throwIfInstanceOf(cause, InterruptedException.class);
- if (cause != null) {
- throw new IOException(cause);
+ }
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ if (interruptedException != null) {
+ if (bulkTransferException != null) {
+ interruptedException.addSuppressed(bulkTransferException);
}
- throw new IOException(e);
+ throw interruptedException;
+ }
+ if (bulkTransferException != null) {
+ throw bulkTransferException;
}
}
@@ -299,40 +318,16 @@
// Subsequently we need to wait for *every* download to finish, even if we already know that
// one failed. That's so that when exiting this method we can be sure that all downloads have
// finished and don't race with the cleanup routine.
- // TODO(buchgr): Look into cancellation.
- IOException downloadException = null;
- InterruptedException interruptedException = null;
FileOutErr tmpOutErr = null;
+ if (origOutErr != null) {
+ tmpOutErr = origOutErr.childOutErr();
+ }
+ downloads.addAll(downloadOutErr(result, tmpOutErr));
+
try {
- if (origOutErr != null) {
- tmpOutErr = origOutErr.childOutErr();
- }
- downloads.addAll(downloadOutErr(result, tmpOutErr));
- } catch (IOException e) {
- downloadException = e;
- }
-
- for (ListenableFuture<FileMetadata> download : downloads) {
- try {
- // Wait for all downloads to finish.
- getFromFuture(download);
- } catch (IOException e) {
- if (downloadException == null) {
- downloadException = e;
- } else if (e != downloadException) {
- downloadException.addSuppressed(e);
- }
- } catch (InterruptedException e) {
- if (interruptedException == null) {
- interruptedException = e;
- } else if (e != interruptedException) {
- interruptedException.addSuppressed(e);
- }
- }
- }
-
- if (downloadException != null || interruptedException != null) {
+ waitForBulkTransfer(downloads, /* cancelRemainingOnInterrupt=*/ true);
+ } catch (Exception e) {
try {
// Delete any (partially) downloaded output files.
for (OutputFile file : result.getOutputFilesList()) {
@@ -347,27 +342,18 @@
tmpOutErr.clearOut();
tmpOutErr.clearErr();
}
- } catch (IOException e) {
- if (downloadException != null && e != downloadException) {
- e.addSuppressed(downloadException);
- }
- if (interruptedException != null) {
- e.addSuppressed(interruptedException);
- }
+ } catch (IOException ioEx) {
+ ioEx.addSuppressed(e);
// If deleting of output files failed, we abort the build with a decent error message as
// any subsequent local execution failure would likely be incomprehensible.
- throw new EnvironmentalExecException(
- "Failed to delete output files after incomplete download", e);
+ ExecException execEx =
+ new EnvironmentalExecException(
+ "Failed to delete output files after incomplete download", ioEx);
+ execEx.addSuppressed(e);
+ throw execEx;
}
- }
-
- if (interruptedException != null) {
- throw interruptedException;
- }
-
- if (downloadException != null) {
- throw downloadException;
+ throw e;
}
if (tmpOutErr != null) {
@@ -487,12 +473,15 @@
return outerF;
}
- private List<ListenableFuture<FileMetadata>> downloadOutErr(ActionResult result, OutErr outErr)
- throws IOException {
+ private List<ListenableFuture<FileMetadata>> downloadOutErr(ActionResult result, OutErr outErr) {
List<ListenableFuture<FileMetadata>> downloads = new ArrayList<>();
if (!result.getStdoutRaw().isEmpty()) {
- result.getStdoutRaw().writeTo(outErr.getOutputStream());
- outErr.getOutputStream().flush();
+ try {
+ result.getStdoutRaw().writeTo(outErr.getOutputStream());
+ outErr.getOutputStream().flush();
+ } catch (IOException e) {
+ downloads.add(Futures.immediateFailedFuture(e));
+ }
} else if (result.hasStdoutDigest()) {
downloads.add(
Futures.transform(
@@ -501,8 +490,12 @@
directExecutor()));
}
if (!result.getStderrRaw().isEmpty()) {
- result.getStderrRaw().writeTo(outErr.getErrorStream());
- outErr.getErrorStream().flush();
+ try {
+ result.getStderrRaw().writeTo(outErr.getErrorStream());
+ outErr.getErrorStream().flush();
+ } catch (IOException e) {
+ downloads.add(Futures.immediateFailedFuture(e));
+ }
} else if (result.hasStderrDigest()) {
downloads.add(
Futures.transform(
@@ -1115,9 +1108,4 @@
return symlinks.values();
}
}
-
- @VisibleForTesting
- protected <T> T getFromFuture(ListenableFuture<T> f) throws IOException, InterruptedException {
- return Utils.getFromFuture(f);
- }
}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionCache.java
index 4d9033b..a4aa232 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionCache.java
@@ -13,11 +13,11 @@
// limitations under the License.
package com.google.devtools.build.lib.remote;
+import static com.google.devtools.build.lib.remote.util.Utils.getFromFuture;
import static java.lang.String.format;
import build.bazel.remote.execution.v2.Digest;
import build.bazel.remote.execution.v2.Directory;
-import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.ListenableFuture;
@@ -26,7 +26,6 @@
import com.google.devtools.build.lib.remote.merkletree.MerkleTree.PathOrBytes;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
import com.google.devtools.build.lib.remote.util.DigestUtil;
-import com.google.devtools.build.lib.remote.util.Utils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.protobuf.ByteString;
import com.google.protobuf.Message;
@@ -35,7 +34,6 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ExecutionException;
/** A {@link RemoteCache} with additional functionality needed for remote execution. */
public class RemoteExecutionCache extends RemoteCache {
@@ -57,21 +55,7 @@
uploads.add(cacheProtocol.uploadBlob(entry.getKey(), entry.getValue()));
}
- try {
- for (ListenableFuture<Void> upload : uploads) {
- upload.get();
- }
- } catch (ExecutionException e) {
- // Cancel remaining uploads.
- for (ListenableFuture<Void> upload : uploads) {
- upload.cancel(/* mayInterruptIfRunning= */ true);
- }
-
- Throwable cause = e.getCause();
- Throwables.propagateIfPossible(cause, IOException.class);
- Throwables.propagateIfPossible(cause, InterruptedException.class);
- throw new IOException(cause);
- }
+ waitForBulkTransfer(uploads, /* cancelRemainingOnInterrupt=*/ false);
}
/**
@@ -91,7 +75,7 @@
Iterable<Digest> allDigests =
Iterables.concat(merkleTree.getAllDigests(), additionalInputs.keySet());
ImmutableSet<Digest> missingDigests =
- Utils.getFromFuture(cacheProtocol.findMissingDigests(allDigests));
+ getFromFuture(cacheProtocol.findMissingDigests(allDigests));
Map<Digest, Path> filesToUpload = new HashMap<>();
Map<Digest, ByteString> blobsToUpload = new HashMap<>();
for (Digest missingDigest : missingDigests) {
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
index b9abd58..a2e00a3 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
@@ -64,7 +64,6 @@
import com.google.devtools.build.lib.profiler.Profiler;
import com.google.devtools.build.lib.profiler.ProfilerTask;
import com.google.devtools.build.lib.profiler.SilentCloseable;
-import com.google.devtools.build.lib.remote.common.CacheNotFoundException;
import com.google.devtools.build.lib.remote.common.RemoteCacheClient.ActionKey;
import com.google.devtools.build.lib.remote.merkletree.MerkleTree;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
@@ -111,8 +110,9 @@
private static final String VIOLATION_TYPE_MISSING = "MISSING";
private static boolean retriableExecErrors(Exception e) {
- if (e instanceof CacheNotFoundException || e.getCause() instanceof CacheNotFoundException) {
- return true;
+ if (e instanceof BulkTransferException) {
+ BulkTransferException bulkTransferException = (BulkTransferException) e;
+ return bulkTransferException.onlyCausedByCacheNotFoundException();
}
if (!RemoteRetrierUtils.causedByStatus(e, Code.FAILED_PRECONDITION)) {
return false;
@@ -270,7 +270,10 @@
totalTime,
networkTime::getDuration,
spawnMetrics);
- } catch (CacheNotFoundException e) {
+ } catch (BulkTransferException e) {
+ if (!e.onlyCausedByCacheNotFoundException()) {
+ throw e;
+ }
// No cache hit, so we fall through to local or remote execution.
// We set acceptCachedResult to false in order to force the action re-execution.
acceptCachedResult = false;
@@ -346,10 +349,12 @@
totalTime,
networkTime::getDuration,
spawnMetrics);
- } catch (CacheNotFoundException e) {
- // No cache hit, so if we retry this execution, we must no longer accept
- // cached results, it must be reexecuted
- requestBuilder.setSkipCacheLookup(true);
+ } catch (BulkTransferException e) {
+ if (e.onlyCausedByCacheNotFoundException()) {
+ // No cache hit, so if we retry this execution, we must no longer accept
+ // cached results, it must be reexecuted
+ requestBuilder.setSkipCacheLookup(true);
+ }
throw e;
}
});
@@ -550,14 +555,23 @@
private SpawnResult handleError(
IOException exception, FileOutErr outErr, ActionKey actionKey, SpawnExecutionContext context)
throws ExecException, InterruptedException, IOException {
+ boolean remoteCacheFailed = false;
+ if (exception instanceof BulkTransferException) {
+ BulkTransferException e = (BulkTransferException) exception;
+ remoteCacheFailed = e.onlyCausedByCacheNotFoundException();
+ }
if (exception.getCause() instanceof ExecutionStatusException) {
ExecutionStatusException e = (ExecutionStatusException) exception.getCause();
if (e.getResponse() != null) {
ExecuteResponse resp = e.getResponse();
maybeDownloadServerLogs(resp, actionKey);
if (resp.hasResult()) {
- // We try to download all (partial) results even on server error, for debuggability.
- remoteCache.download(resp.getResult(), execRoot, outErr, context::lockOutputFiles);
+ try {
+ // We try to download all (partial) results even on server error, for debuggability.
+ remoteCache.download(resp.getResult(), execRoot, outErr, context::lockOutputFiles);
+ } catch (BulkTransferException bulkTransferEx) {
+ exception.addSuppressed(bulkTransferEx);
+ }
}
}
if (e.isExecutionTimeout()) {
@@ -571,7 +585,7 @@
final Status status;
if (RemoteRetrierUtils.causedByStatus(exception, Code.UNAVAILABLE)) {
status = Status.EXECUTION_FAILED_CATASTROPHICALLY;
- } else if (exception instanceof CacheNotFoundException) {
+ } else if (remoteCacheFailed) {
status = Status.REMOTE_CACHE_FAILED;
} else {
status = Status.EXECUTION_FAILED;
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheTests.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheTests.java
index 716944f..3db0bf5 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheTests.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheTests.java
@@ -641,7 +641,8 @@
OutputFile.newBuilder().setPath("outputdir/outputfile").setDigest(outputFileDigest));
result.addOutputFiles(OutputFile.newBuilder().setPath("otherfile").setDigest(otherFileDigest));
assertThrows(
- IOException.class, () -> cache.download(result.build(), execRoot, null, outputFilesLocker));
+ BulkTransferException.class,
+ () -> cache.download(result.build(), execRoot, null, outputFilesLocker));
assertThat(cache.getNumFailedDownloads()).isEqualTo(1);
assertThat(execRoot.getRelative("outputdir").exists()).isTrue();
assertThat(execRoot.getRelative("outputdir/outputfile").exists()).isFalse();
@@ -670,15 +671,17 @@
.addOutputFiles(OutputFile.newBuilder().setPath("file2").setDigest(digest2))
.addOutputFiles(OutputFile.newBuilder().setPath("file3").setDigest(digest3))
.build();
- IOException e =
+ BulkTransferException downloadException =
assertThrows(
- IOException.class,
+ BulkTransferException.class,
() ->
cache.download(
result, execRoot, new FileOutErr(stdout, stderr), outputFilesLocker));
- assertThat(e.getSuppressed()).isEmpty();
+ assertThat(downloadException.getSuppressed()).hasLength(1);
assertThat(cache.getNumSuccessfulDownloads()).isEqualTo(2);
assertThat(cache.getNumFailedDownloads()).isEqualTo(1);
+ assertThat(downloadException.getSuppressed()[0]).isInstanceOf(IOException.class);
+ IOException e = (IOException) downloadException.getSuppressed()[0];
assertThat(Throwables.getRootCause(e)).hasMessageThat().isEqualTo("download failed");
verify(outputFilesLocker, never()).lock();
}
@@ -700,17 +703,18 @@
.addOutputFiles(OutputFile.newBuilder().setPath("file2").setDigest(digest2))
.addOutputFiles(OutputFile.newBuilder().setPath("file3").setDigest(digest3))
.build();
- IOException e =
+ BulkTransferException e =
assertThrows(
- IOException.class,
+ BulkTransferException.class,
() ->
cache.download(
result, execRoot, new FileOutErr(stdout, stderr), outputFilesLocker));
- assertThat(e.getSuppressed()).hasLength(1);
+ assertThat(e.getSuppressed()).hasLength(2);
assertThat(e.getSuppressed()[0]).isInstanceOf(IOException.class);
- assertThat(e.getSuppressed()[0]).hasMessageThat().isEqualTo("file3 failed");
- assertThat(Throwables.getRootCause(e)).hasMessageThat().isEqualTo("file2 failed");
+ assertThat(e.getSuppressed()[0]).hasMessageThat().isAnyOf("file2 failed", "file3 failed");
+ assertThat(e.getSuppressed()[1]).isInstanceOf(IOException.class);
+ assertThat(e.getSuppressed()[1]).hasMessageThat().isAnyOf("file2 failed", "file3 failed");
}
@Test
@@ -731,15 +735,18 @@
.addOutputFiles(OutputFile.newBuilder().setPath("file2").setDigest(digest2))
.addOutputFiles(OutputFile.newBuilder().setPath("file3").setDigest(digest3))
.build();
- IOException e =
+ BulkTransferException downloadException =
assertThrows(
- IOException.class,
+ BulkTransferException.class,
() ->
cache.download(
result, execRoot, new FileOutErr(stdout, stderr), outputFilesLocker));
- assertThat(e.getSuppressed()).isEmpty();
- assertThat(Throwables.getRootCause(e)).hasMessageThat().isEqualTo("reused io exception");
+ for (Throwable t : downloadException.getSuppressed()) {
+ assertThat(t).isInstanceOf(IOException.class);
+ IOException e = (IOException) t;
+ assertThat(Throwables.getRootCause(e)).hasMessageThat().isEqualTo("reused io exception");
+ }
}
@Test
@@ -836,7 +843,8 @@
.setStderrDigest(digestStderr)
.build();
assertThrows(
- IOException.class, () -> cache.download(result, execRoot, spyOutErr, outputFilesLocker));
+ BulkTransferException.class,
+ () -> cache.download(result, execRoot, spyOutErr, outputFilesLocker));
verify(spyOutErr, Mockito.times(2)).childOutErr();
verify(spyChildOutErr).clearOut();
verify(spyChildOutErr).clearErr();
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java
index 0a90dea..00ecddf 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java
@@ -604,7 +604,8 @@
ActionResult cachedResult = ActionResult.newBuilder().setExitCode(0).build();
when(cache.downloadActionResult(any(ActionKey.class), /* inlineOutErr= */ eq(false)))
.thenReturn(cachedResult);
- Exception downloadFailure = new CacheNotFoundException(Digest.getDefaultInstance());
+ Exception downloadFailure =
+ new BulkTransferException(new CacheNotFoundException(Digest.getDefaultInstance()));
doThrow(downloadFailure)
.when(cache)
.download(eq(cachedResult), any(Path.class), any(FileOutErr.class), any());
@@ -641,7 +642,8 @@
when(executor.executeRemotely(any(ExecuteRequest.class)))
.thenReturn(cachedResponse)
.thenReturn(executedResponse);
- Exception downloadFailure = new CacheNotFoundException(Digest.getDefaultInstance());
+ Exception downloadFailure =
+ new BulkTransferException(new CacheNotFoundException(Digest.getDefaultInstance()));
doThrow(downloadFailure)
.when(cache)
.download(eq(cachedResult), any(Path.class), any(FileOutErr.class), any());
