commit c26ec2d1afee54f63d9539b758f42e79cd73b87f
author Benjamin Peterson <benjamin@engflow.com> Wed Jun 15 02:27:25 2022 -0700
committer Copybara-Service <copybara-worker@google.com> Wed Jun 15 02:28:26 2022 -0700
tree 7088c391c0b2d10b14e7a7ab4c79b0fe34a29a3d
parent 35db514100daa910dfc9fb0c58a262c3d0b815df

ByteStreamUploader: Open files at the last possible moment.

Open files to upload only when the network is ready to accept data. Empirically, this greatly reduces the need to set `--bep_maximum_open_remote_upload_files` because the small-to-medium-sized files that are common in real-world builds can be opened, dumped onto the network, and closed in one shot.

Closes #15670.

PiperOrigin-RevId: 455074457
Change-Id: I243c7e6c51b2d415cdc0a00e7e71d06aefadbd2c

src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java

diff --git a/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java b/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java
index de2ff4d..ca3175d 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java
@@ -30,6 +30,7 @@
 import com.google.bytestream.ByteStreamProto.WriteRequest;
 import com.google.bytestream.ByteStreamProto.WriteResponse;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.google.common.hash.HashCode;
@@ -77,7 +78,6 @@
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Supplier;
 import javax.annotation.Nullable;
 import org.junit.After;
@@ -782,23 +782,18 @@
     byte[] blob = new byte[CHUNK_SIZE];
     Chunker chunker = Mockito.mock(Chunker.class);
     Digest digest = DIGEST_UTIL.compute(blob);
-    AtomicLong committedOffset = new AtomicLong(0);
-    Mockito.doThrow(new IOException("Too many open files"))
-        .when(chunker)
-        .seek(committedOffset.get());
+    Mockito.doThrow(new IOException("Too many open files")).when(chunker).seek(0);
     Mockito.when(chunker.getSize()).thenReturn(digest.getSizeBytes());
+    serviceRegistry.addService(new MaybeFailOnceUploadService(ImmutableMap.of()));
 
-    try {
-      uploader.uploadBlob(context, digest, chunker);
-      fail("Should have thrown an exception.");
-    } catch (IOException e) {
-      String newMessage =
-          "An IOException was thrown because the process opened too many files. We recommend"
-              + " setting --bep_maximum_open_remote_upload_files flag to a number lower than your"
-              + " system default (run 'ulimit -a' for *nix-based operating systems). Original error"
-              + " message: Too many open files";
-      assertThat(newMessage).isEqualTo(e.getMessage());
-    }
+    String newMessage =
+        "An IOException was thrown because the process opened too many files. We recommend setting"
+            + " --bep_maximum_open_remote_upload_files flag to a number lower than your system"
+            + " default (run 'ulimit -a' for *nix-based operating systems). Original error message:"
+            + " Too many open files";
+    assertThat(assertThrows(IOException.class, () -> uploader.uploadBlob(context, digest, chunker)))
+        .hasMessageThat()
+        .isEqualTo(newMessage);
   }
 
   @Test