remote: don't fail build if upload fails
If the upload of local build artifacts fails, the build no longer fails
but instead a warning is printed once. If --verbose_failures is
specified, a detailed warning is printed for every failure.
This helps fixing #2964, however it doesn't fully fix it due to timeouts
and retries slowing the build significantly.
Also, add some other tests related to fallback behavior.
Change-Id: Ief49941f9bc7e0123b5d93456d77428686dd5268
PiperOrigin-RevId: 165938874
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java
index 826de36..8bb5a27 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java
@@ -51,7 +51,8 @@
RemoteOptions remoteOptions = checkNotNull(env.getOptions().getOptions(RemoteOptions.class));
if (remoteOptions.experimentalRemoteSpawnCache) {
- RemoteSpawnCache spawnCache = new RemoteSpawnCache(env.getExecRoot(), remoteOptions, cache);
+ RemoteSpawnCache spawnCache = new RemoteSpawnCache(env.getExecRoot(), remoteOptions, cache,
+ executionOptions.verboseFailures, env.getReporter());
return ImmutableList.of(spawnCache);
} else {
RemoteSpawnRunner spawnRunner = new RemoteSpawnRunner(
@@ -59,6 +60,7 @@
remoteOptions,
createFallbackRunner(env),
executionOptions.verboseFailures,
+ env.getReporter(),
cache,
executor);
return ImmutableList.of(new RemoteSpawnStrategy(spawnRunner));
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
index 5d5c3e8..2b74e88 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
@@ -18,6 +18,8 @@
import com.google.devtools.build.lib.actions.ExecutionStrategy;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.exec.SpawnCache;
import com.google.devtools.build.lib.exec.SpawnResult;
import com.google.devtools.build.lib.exec.SpawnResult.Status;
@@ -34,6 +36,8 @@
import java.util.Collection;
import java.util.NoSuchElementException;
import java.util.SortedMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.annotation.Nullable;
/**
* A remote {@link SpawnCache} implementation.
@@ -50,12 +54,21 @@
private final Platform platform;
private final RemoteActionCache remoteCache;
+ private final boolean verboseFailures;
- RemoteSpawnCache(Path execRoot, RemoteOptions options, RemoteActionCache remoteCache) {
+ @Nullable private final Reporter cmdlineReporter;
+
+ // Used to ensure that a warning is reported only once.
+ private final AtomicBoolean warningReported = new AtomicBoolean();
+
+ RemoteSpawnCache(Path execRoot, RemoteOptions options, RemoteActionCache remoteCache,
+ boolean verboseFailures, @Nullable Reporter cmdlineReporter) {
this.execRoot = execRoot;
this.options = options;
this.platform = options.parseRemotePlatformOverride();
this.remoteCache = remoteCache;
+ this.verboseFailures = verboseFailures;
+ this.cmdlineReporter = cmdlineReporter;
}
@Override
@@ -118,7 +131,15 @@
if (result.status() != Status.SUCCESS || result.exitCode() != 0) {
return;
}
- remoteCache.upload(actionKey, execRoot, files, policy.getFileOutErr());
+ try {
+ remoteCache.upload(actionKey, execRoot, files, policy.getFileOutErr());
+ } catch (IOException e) {
+ if (verboseFailures) {
+ report(Event.debug("Upload to remote cache failed: " + e.getMessage()));
+ } else {
+ reportOnce(Event.warn("Some artifacts failed be uploaded to the remote cache."));
+ }
+ }
}
@Override
@@ -129,4 +150,16 @@
return SpawnCache.NO_RESULT_NO_STORE;
}
}
+
+ private void reportOnce(Event evt) {
+ if (warningReported.compareAndSet(false, true)) {
+ report(evt);
+ }
+ }
+
+ private void report(Event evt) {
+ if (cmdlineReporter != null) {
+ cmdlineReporter.handle(evt);
+ }
+ }
}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
index 72c356b..26fba9d 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
@@ -25,6 +25,8 @@
import com.google.devtools.build.lib.actions.Spawns;
import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.exec.SpawnInputExpander;
import com.google.devtools.build.lib.exec.SpawnResult;
import com.google.devtools.build.lib.exec.SpawnResult.Status;
@@ -52,6 +54,7 @@
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
/** A client for the remote execution service. */
@@ -64,14 +67,19 @@
private final SpawnRunner fallbackRunner;
private final boolean verboseFailures;
+ @Nullable private final Reporter cmdlineReporter;
@Nullable private final RemoteActionCache remoteCache;
@Nullable private final GrpcRemoteExecutor remoteExecutor;
+ // Used to ensure that a warning is reported only once.
+ private final AtomicBoolean warningReported = new AtomicBoolean();
+
RemoteSpawnRunner(
Path execRoot,
RemoteOptions options,
SpawnRunner fallbackRunner,
boolean verboseFailures,
+ @Nullable Reporter cmdlineReporter,
@Nullable RemoteActionCache remoteCache,
@Nullable GrpcRemoteExecutor remoteExecutor) {
this.execRoot = execRoot;
@@ -81,6 +89,7 @@
this.remoteCache = remoteCache;
this.remoteExecutor = remoteExecutor;
this.verboseFailures = verboseFailures;
+ this.cmdlineReporter = cmdlineReporter;
}
@Override
@@ -132,8 +141,8 @@
}
}
} catch (IOException e) {
- return execLocallyOrFail(spawn, policy, inputMap, actionKey,
- options.remoteUploadLocalResults, e);
+ return execLocallyOrFail(
+ spawn, policy, inputMap, actionKey, uploadLocalResults, e);
}
if (remoteExecutor == null) {
@@ -145,8 +154,8 @@
// Upload the command and all the inputs into the remote cache.
remoteCache.ensureInputsPresent(repository, execRoot, inputRoot, command);
} catch (IOException e) {
- return execLocallyOrFail(spawn, policy, inputMap, actionKey,
- options.remoteUploadLocalResults, e);
+ return execLocallyOrFail(
+ spawn, policy, inputMap, actionKey, uploadLocalResults, e);
}
final ActionResult result;
@@ -158,15 +167,14 @@
boolean executionFailed = result.getExitCode() != 0;
if (options.remoteLocalFallback && executionFailed) {
- return execLocally(spawn, policy, inputMap, options.remoteUploadLocalResults,
- remoteCache, actionKey);
+ return execLocally(spawn, policy, inputMap, uploadLocalResults, remoteCache, actionKey);
}
try {
return downloadRemoteResults(result, policy.getFileOutErr());
} catch (IOException e) {
- return execLocallyOrFail(spawn, policy, inputMap, actionKey,
- options.remoteUploadLocalResults, e);
+ return execLocallyOrFail(
+ spawn, policy, inputMap, actionKey, uploadLocalResults, e);
}
}
@@ -197,8 +205,7 @@
boolean uploadLocalResults, IOException cause)
throws ExecException, InterruptedException, IOException {
if (options.remoteLocalFallback) {
- return execLocally(spawn, policy, inputMap, uploadLocalResults,
- remoteCache, actionKey);
+ return execLocally(spawn, policy, inputMap, uploadLocalResults, remoteCache, actionKey);
}
throw new EnvironmentalExecException(errorMessage(cause), cause, true);
}
@@ -315,10 +322,30 @@
}
}
List<Path> outputFiles = listExistingOutputFiles(execRoot, spawn);
- remoteCache.upload(actionKey, execRoot, outputFiles, policy.getFileOutErr());
+ try {
+ remoteCache.upload(actionKey, execRoot, outputFiles, policy.getFileOutErr());
+ } catch (IOException e) {
+ if (verboseFailures) {
+ report(Event.debug("Upload to remote cache failed: " + e.getMessage()));
+ } else {
+ reportOnce(Event.warn("Some artifacts failed be uploaded to the remote cache."));
+ }
+ }
return result;
}
+ private void reportOnce(Event evt) {
+ if (warningReported.compareAndSet(false, true)) {
+ report(evt);
+ }
+ }
+
+ private void report(Event evt) {
+ if (cmdlineReporter != null) {
+ cmdlineReporter.handle(evt);
+ }
+ }
+
static List<Path> listExistingOutputFiles(Path execRoot, Spawn spawn) {
ArrayList<Path> outputFiles = new ArrayList<>();
for (ActionInput output : spawn.getOutputFiles()) {
diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD
index 033da62..cd0ea14 100644
--- a/src/test/java/com/google/devtools/build/lib/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/BUILD
@@ -1081,6 +1081,7 @@
":testutil",
"//src/main/java/com/google/devtools/build/lib:auth_and_tls_options",
"//src/main/java/com/google/devtools/build/lib:build-base",
+ "//src/main/java/com/google/devtools/build/lib:events",
"//src/main/java/com/google/devtools/build/lib:inmemoryfs",
"//src/main/java/com/google/devtools/build/lib:io",
"//src/main/java/com/google/devtools/build/lib:preconditions",
diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java
index daed01a..1db31cc 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java
@@ -228,7 +228,8 @@
GrpcUtils.newCallCredentials(Options.getDefaults(AuthAndTLSOptions.class));
GrpcRemoteCache remoteCache =
new GrpcRemoteCache(channel, creds, options, retrier);
- client = new RemoteSpawnRunner(execRoot, options, null, true, remoteCache, executor);
+ client = new RemoteSpawnRunner(execRoot, options, null, true, /*cmdlineReporter=*/null,
+ remoteCache, executor);
inputDigest = fakeFileCache.createScratchInput(simpleSpawn.getInputFiles().get(0), "xyz");
}
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java
index 930815b..43e4de4 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java
@@ -16,12 +16,14 @@
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.eventbus.EventBus;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionInputFileCache;
import com.google.devtools.build.lib.actions.ActionInputHelper;
@@ -29,6 +31,10 @@
import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander;
import com.google.devtools.build.lib.actions.ResourceSet;
import com.google.devtools.build.lib.actions.SimpleSpawn;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.events.StoredEventHandler;
import com.google.devtools.build.lib.exec.SpawnCache.CacheHandle;
import com.google.devtools.build.lib.exec.SpawnInputExpander;
import com.google.devtools.build.lib.exec.SpawnResult;
@@ -77,6 +83,8 @@
private RemoteSpawnCache cache;
private FileOutErr outErr;
+ private StoredEventHandler eventHandler = new StoredEventHandler();
+
private final SpawnExecutionPolicy simplePolicy =
new SpawnExecutionPolicy() {
@Override
@@ -155,7 +163,10 @@
FileSystemUtils.createDirectoryAndParents(stderr.getParentDirectory());
outErr = new FileOutErr(stdout, stderr);
RemoteOptions options = Options.getDefaults(RemoteOptions.class);
- cache = new RemoteSpawnCache(execRoot, options, remoteCache);
+ Reporter reporter = new Reporter(new EventBus());
+ eventHandler = new StoredEventHandler();
+ reporter.addHandler(eventHandler);
+ cache = new RemoteSpawnCache(execRoot, options, remoteCache, false, reporter);
fakeFileCache.createScratchInput(simpleSpawn.getInputFiles().get(0), "xyz");
}
@@ -200,4 +211,31 @@
eq(outputFiles),
eq(outErr));
}
+
+ @Test
+ public void printWarningIfUploadFails() throws Exception {
+ CacheHandle entry = cache.lookup(simpleSpawn, simplePolicy);
+ assertThat(entry.hasResult()).isFalse();
+ SpawnResult result = new SpawnResult.Builder().setExitCode(0).setStatus(Status.SUCCESS).build();
+ ImmutableList<Path> outputFiles = ImmutableList.of(fs.getPath("/random/file"));
+
+ doThrow(new IOException("cache down")).when(remoteCache).upload(any(ActionKey.class),
+ any(Path.class),
+ eq(outputFiles),
+ eq(outErr));
+
+ entry.store(result, outputFiles);
+ verify(remoteCache)
+ .upload(
+ any(ActionKey.class),
+ any(Path.class),
+ eq(outputFiles),
+ eq(outErr));
+
+ assertThat(eventHandler.getEvents()).hasSize(1);
+ Event evt = eventHandler.getEvents().get(0);
+ assertThat(evt.getKind()).isEqualTo(EventKind.WARNING);
+ assertThat(evt.getMessage()).contains("fail");
+ assertThat(evt.getMessage()).contains("upload");
+ }
}
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java
index 180a0df..b8e4f21 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java
@@ -17,6 +17,8 @@
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
@@ -25,6 +27,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.eventbus.EventBus;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionInputFileCache;
import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander;
@@ -33,6 +36,10 @@
import com.google.devtools.build.lib.actions.ResourceSet;
import com.google.devtools.build.lib.actions.SimpleSpawn;
import com.google.devtools.build.lib.actions.Spawn;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.events.StoredEventHandler;
import com.google.devtools.build.lib.exec.SpawnInputExpander;
import com.google.devtools.build.lib.exec.SpawnResult;
import com.google.devtools.build.lib.exec.SpawnResult.Status;
@@ -112,7 +119,8 @@
options.remoteUploadLocalResults = true;
RemoteSpawnRunner runner =
- new RemoteSpawnRunner(execRoot, options, localRunner, true, cache, executor);
+ new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null,
+ cache, executor);
ExecuteResponse succeeded = ExecuteResponse.newBuilder().setResult(
ActionResult.newBuilder().setExitCode(0).build()).build();
@@ -154,7 +162,8 @@
options.remoteUploadLocalResults = true;
RemoteSpawnRunner runner =
- new RemoteSpawnRunner(execRoot, options, localRunner, true, cache, null);
+ new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null,
+ cache, null);
// Throw an IOException to trigger the local fallback.
when(executor.executeRemotely(any(ExecuteRequest.class))).thenThrow(IOException.class);
@@ -190,7 +199,8 @@
options.remoteUploadLocalResults = true;
RemoteSpawnRunner runner =
- spy(new RemoteSpawnRunner(execRoot, options, localRunner, true, cache, null));
+ spy(new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null,
+ cache, null));
Spawn spawn = new SimpleSpawn(
new FakeOwner("foo", "bar"),
@@ -236,7 +246,8 @@
SpawnExecutionPolicy policy = new FakeSpawnExecutionPolicy(spawn);
RemoteSpawnRunner runner =
- spy(new RemoteSpawnRunner(execRoot, options, localRunner, true, cache, null));
+ spy(new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null,
+ cache, null));
try {
runner.exec(spawn, policy);
@@ -246,6 +257,132 @@
}
}
+ @Test
+ @SuppressWarnings("unchecked")
+ public void printWarningIfCacheIsDown() throws Exception {
+ // If we try to upload to a local cache, that is down a warning should be printed.
+
+ RemoteOptions options = Options.getDefaults(RemoteOptions.class);
+ options.remoteUploadLocalResults = true;
+ options.remoteLocalFallback = true;
+
+ Reporter reporter = new Reporter(new EventBus());
+ StoredEventHandler eventHandler = new StoredEventHandler();
+ reporter.addHandler(eventHandler);
+
+ RemoteSpawnRunner runner =
+ new RemoteSpawnRunner(execRoot, options, localRunner, false, reporter, cache, null);
+
+ Spawn spawn =
+ new SimpleSpawn(
+ new FakeOwner("foo", "bar"),
+ /*arguments=*/ ImmutableList.of(),
+ /*environment=*/ ImmutableMap.of(),
+ /*executionInfo=*/ ImmutableMap.of(),
+ /*inputs=*/ ImmutableList.of(),
+ /*outputs=*/ ImmutableList.<ActionInput>of(),
+ ResourceSet.ZERO);
+ SpawnExecutionPolicy policy = new FakeSpawnExecutionPolicy(spawn);
+
+ when(cache.getCachedActionResult(any(ActionKey.class)))
+ .thenThrow(new IOException("cache down"));
+
+ doThrow(new IOException("cache down")).when(cache)
+ .upload(any(ActionKey.class), any(Path.class), any(Collection.class),
+ any(FileOutErr.class));
+
+ SpawnResult res = new SpawnResult.Builder().setStatus(Status.SUCCESS).setExitCode(0).build();
+ when(localRunner.exec(eq(spawn), eq(policy))).thenReturn(res);
+
+ assertThat(runner.exec(spawn, policy)).isEqualTo(res);
+
+ verify(localRunner).exec(eq(spawn), eq(policy));
+
+ assertThat(eventHandler.getEvents()).hasSize(1);
+
+ Event evt = eventHandler.getEvents().get(0);
+ assertThat(evt.getKind()).isEqualTo(EventKind.WARNING);
+ assertThat(evt.getMessage()).contains("fail");
+ assertThat(evt.getMessage()).contains("upload");
+ }
+
+ @Test
+ public void fallbackFails() throws Exception {
+ // Errors from the fallback runner should be propogated out of the remote runner.
+
+ RemoteOptions options = Options.getDefaults(RemoteOptions.class);
+ options.remoteUploadLocalResults = true;
+ options.remoteLocalFallback = true;
+
+ RemoteSpawnRunner runner =
+ new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null,
+ cache, null);
+
+ Spawn spawn =
+ new SimpleSpawn(
+ new FakeOwner("foo", "bar"),
+ /*arguments=*/ ImmutableList.of(),
+ /*environment=*/ ImmutableMap.of(),
+ /*executionInfo=*/ ImmutableMap.of(),
+ /*inputs=*/ ImmutableList.of(),
+ /*outputs=*/ ImmutableList.<ActionInput>of(),
+ ResourceSet.ZERO);
+ SpawnExecutionPolicy policy = new FakeSpawnExecutionPolicy(spawn);
+
+ when(cache.getCachedActionResult(any(ActionKey.class))).thenReturn(null);
+
+ IOException err = new IOException("local execution error");
+ when(localRunner.exec(eq(spawn), eq(policy))).thenThrow(err);
+
+ try {
+ runner.exec(spawn, policy);
+ fail("expected IOException to be raised");
+ } catch (IOException e) {
+ assertThat(e).isSameAs(err);
+ }
+
+ verify(localRunner).exec(eq(spawn), eq(policy));
+ }
+
+ @Test
+ public void cacheDownloadFailureTriggersRemoteExecution() throws Exception {
+ // If downloading a cached action fails, remote execution should be tried.
+
+ RemoteOptions options = Options.getDefaults(RemoteOptions.class);
+
+ RemoteSpawnRunner runner =
+ new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null,
+ cache, executor);
+
+ ActionResult cachedResult = ActionResult.newBuilder().setExitCode(0).build();
+ when(cache.getCachedActionResult(any(ActionKey.class))).thenReturn(cachedResult);
+ doThrow(CacheNotFoundException.class)
+ .when(cache)
+ .download(eq(cachedResult), any(Path.class), any(FileOutErr.class));
+ ActionResult execResult = ActionResult.newBuilder().setExitCode(31).build();
+ ExecuteResponse succeeded = ExecuteResponse.newBuilder().setResult(execResult).build();
+ when(executor.executeRemotely(any(ExecuteRequest.class))).thenReturn(succeeded);
+ doNothing().when(cache).download(eq(execResult), any(Path.class), any(FileOutErr.class));
+
+ Spawn spawn =
+ new SimpleSpawn(
+ new FakeOwner("foo", "bar"),
+ /*arguments=*/ ImmutableList.of(),
+ /*environment=*/ ImmutableMap.of(),
+ /*executionInfo=*/ ImmutableMap.of(),
+ /*inputs=*/ ImmutableList.of(),
+ /*outputs=*/ ImmutableList.<ActionInput>of(),
+ ResourceSet.ZERO);
+
+ SpawnExecutionPolicy policy = new FakeSpawnExecutionPolicy(spawn);
+
+ SpawnResult res = runner.exec(spawn, policy);
+ assertThat(res.status()).isEqualTo(Status.SUCCESS);
+ assertThat(res.exitCode()).isEqualTo(31);
+
+ verify(executor).executeRemotely(any(ExecuteRequest.class));
+ }
+
// TODO(buchgr): Extract a common class to be used for testing.
class FakeSpawnExecutionPolicy implements SpawnExecutionPolicy {
