remote: don't fail build if upload fails
If the upload of local build artifacts fails, the build no longer fails
but instead a warning is printed once. If --verbose_failures is
specified, a detailed warning is printed for every failure.
This helps fixing #2964, however it doesn't fully fix it due to timeouts
and retries slowing the build significantly.
Also, add some other tests related to fallback behavior.
Change-Id: Ief49941f9bc7e0123b5d93456d77428686dd5268
PiperOrigin-RevId: 165938874
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java
index 930815b..43e4de4 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java
@@ -16,12 +16,14 @@
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.eventbus.EventBus;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionInputFileCache;
import com.google.devtools.build.lib.actions.ActionInputHelper;
@@ -29,6 +31,10 @@
import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander;
import com.google.devtools.build.lib.actions.ResourceSet;
import com.google.devtools.build.lib.actions.SimpleSpawn;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.events.StoredEventHandler;
import com.google.devtools.build.lib.exec.SpawnCache.CacheHandle;
import com.google.devtools.build.lib.exec.SpawnInputExpander;
import com.google.devtools.build.lib.exec.SpawnResult;
@@ -77,6 +83,8 @@
private RemoteSpawnCache cache;
private FileOutErr outErr;
+ private StoredEventHandler eventHandler = new StoredEventHandler();
+
private final SpawnExecutionPolicy simplePolicy =
new SpawnExecutionPolicy() {
@Override
@@ -155,7 +163,10 @@
FileSystemUtils.createDirectoryAndParents(stderr.getParentDirectory());
outErr = new FileOutErr(stdout, stderr);
RemoteOptions options = Options.getDefaults(RemoteOptions.class);
- cache = new RemoteSpawnCache(execRoot, options, remoteCache);
+ Reporter reporter = new Reporter(new EventBus());
+ eventHandler = new StoredEventHandler();
+ reporter.addHandler(eventHandler);
+ cache = new RemoteSpawnCache(execRoot, options, remoteCache, false, reporter);
fakeFileCache.createScratchInput(simpleSpawn.getInputFiles().get(0), "xyz");
}
@@ -200,4 +211,31 @@
eq(outputFiles),
eq(outErr));
}
+
+ @Test
+ public void printWarningIfUploadFails() throws Exception {
+ CacheHandle entry = cache.lookup(simpleSpawn, simplePolicy);
+ assertThat(entry.hasResult()).isFalse();
+ SpawnResult result = new SpawnResult.Builder().setExitCode(0).setStatus(Status.SUCCESS).build();
+ ImmutableList<Path> outputFiles = ImmutableList.of(fs.getPath("/random/file"));
+
+ doThrow(new IOException("cache down")).when(remoteCache).upload(any(ActionKey.class),
+ any(Path.class),
+ eq(outputFiles),
+ eq(outErr));
+
+ entry.store(result, outputFiles);
+ verify(remoteCache)
+ .upload(
+ any(ActionKey.class),
+ any(Path.class),
+ eq(outputFiles),
+ eq(outErr));
+
+ assertThat(eventHandler.getEvents()).hasSize(1);
+ Event evt = eventHandler.getEvents().get(0);
+ assertThat(evt.getKind()).isEqualTo(EventKind.WARNING);
+ assertThat(evt.getMessage()).contains("fail");
+ assertThat(evt.getMessage()).contains("upload");
+ }
}