remote: don't fail build if upload fails

If the upload of local build artifacts fails, the build no longer fails
but instead a warning is printed once. If --verbose_failures is
specified, a detailed warning is printed for every failure.

This helps fixing #2964, however it doesn't fully fix it due to timeouts
and retries slowing the build significantly.

Also, add some other tests related to fallback behavior.

Change-Id: Ief49941f9bc7e0123b5d93456d77428686dd5268
PiperOrigin-RevId: 165938874
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
index 5d5c3e8..2b74e88 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
@@ -18,6 +18,8 @@
 import com.google.devtools.build.lib.actions.ExecutionStrategy;
 import com.google.devtools.build.lib.actions.Spawn;
 import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Reporter;
 import com.google.devtools.build.lib.exec.SpawnCache;
 import com.google.devtools.build.lib.exec.SpawnResult;
 import com.google.devtools.build.lib.exec.SpawnResult.Status;
@@ -34,6 +36,8 @@
 import java.util.Collection;
 import java.util.NoSuchElementException;
 import java.util.SortedMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.annotation.Nullable;
 
 /**
  * A remote {@link SpawnCache} implementation.
@@ -50,12 +54,21 @@
   private final Platform platform;
 
   private final RemoteActionCache remoteCache;
+  private final boolean verboseFailures;
 
-  RemoteSpawnCache(Path execRoot, RemoteOptions options, RemoteActionCache remoteCache) {
+  @Nullable private final Reporter cmdlineReporter;
+
+  // Used to ensure that a warning is reported only once.
+  private final AtomicBoolean warningReported = new AtomicBoolean();
+
+  RemoteSpawnCache(Path execRoot, RemoteOptions options, RemoteActionCache remoteCache,
+      boolean verboseFailures, @Nullable Reporter cmdlineReporter) {
     this.execRoot = execRoot;
     this.options = options;
     this.platform = options.parseRemotePlatformOverride();
     this.remoteCache = remoteCache;
+    this.verboseFailures = verboseFailures;
+    this.cmdlineReporter = cmdlineReporter;
   }
 
   @Override
@@ -118,7 +131,15 @@
           if (result.status() != Status.SUCCESS || result.exitCode() != 0) {
             return;
           }
-          remoteCache.upload(actionKey, execRoot, files, policy.getFileOutErr());
+          try {
+            remoteCache.upload(actionKey, execRoot, files, policy.getFileOutErr());
+          } catch (IOException e) {
+            if (verboseFailures) {
+              report(Event.debug("Upload to remote cache failed: " + e.getMessage()));
+            } else {
+              reportOnce(Event.warn("Some artifacts failed be uploaded to the remote cache."));
+            }
+          }
         }
 
         @Override
@@ -129,4 +150,16 @@
       return SpawnCache.NO_RESULT_NO_STORE;
     }
   }
+
+  private void reportOnce(Event evt) {
+    if (warningReported.compareAndSet(false, true)) {
+      report(evt);
+    }
+  }
+
+  private void report(Event evt) {
+    if (cmdlineReporter != null) {
+      cmdlineReporter.handle(evt);
+    }
+  }
 }