Roll-forward https://github.com/bazelbuild/bazel/commit/1f240fc2978eba6c43388e5cd6567835fb7ed050 with fix:
Add a new --bes_outerr_chunk_size flag which can be set separately from the buffer size. Even if buffering is off (buffer_size=0), we can chunk huge, individual payloads.
RELNOTES: None
PiperOrigin-RevId: 234160003
diff --git a/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceModule.java b/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceModule.java
index 9a4dd8d..b880da6 100644
--- a/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceModule.java
+++ b/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceModule.java
@@ -106,9 +106,10 @@
commandEnvironment.getReporter().addHandler(streamer);
commandEnvironment.getEventBus().register(streamer);
int bufferSize = besOptions.besOuterrBufferSize;
+ int chunkSize = besOptions.besOuterrChunkSize;
- final SynchronizedOutputStream out = new SynchronizedOutputStream(bufferSize);
- final SynchronizedOutputStream err = new SynchronizedOutputStream(bufferSize);
+ final SynchronizedOutputStream out = new SynchronizedOutputStream(bufferSize, chunkSize);
+ final SynchronizedOutputStream err = new SynchronizedOutputStream(bufferSize, chunkSize);
this.outErr = OutErr.create(out, err);
streamer.registerOutErrProvider(
new BuildEventStreamer.OutErrProvider() {
diff --git a/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceOptions.java b/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceOptions.java
index 06e0c5c..2c23ff1 100644
--- a/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceOptions.java
@@ -102,10 +102,19 @@
help =
"Specifies the maximal size of stdout or stderr to be buffered in BEP, before it is "
+ "reported as a progress event. Individual writes are still reported in a single "
- + "event, even if larger than the specified value.")
+ + "event, even if larger than the specified value up to --bes_outerr_chunk_size.")
public int besOuterrBufferSize;
@Option(
+ name = "bes_outerr_chunk_size",
+ defaultValue = "1048576", // 2^20 = 1MB
+ documentationCategory = OptionDocumentationCategory.LOGGING,
+ effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+ help =
+ "Specifies the maximal size of stdout or stderr to be sent to BEP in a single message.")
+ public int besOuterrChunkSize;
+
+ @Option(
name = "bes_results_url",
defaultValue = "",
documentationCategory = OptionDocumentationCategory.LOGGING,
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/SynchronizedOutputStream.java b/src/main/java/com/google/devtools/build/lib/runtime/SynchronizedOutputStream.java
index 9042f3b..2c4c124 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/SynchronizedOutputStream.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/SynchronizedOutputStream.java
@@ -16,6 +16,8 @@
import static java.nio.charset.StandardCharsets.UTF_8;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.io.OutputStream;
@@ -40,6 +42,8 @@
// {@link write(byte[] buffer, int offset, int count)} method.
private final int maxBufferedLength;
+ private final int maxChunkSize;
+
private byte[] buf;
private long count;
private boolean discardAll;
@@ -47,11 +51,13 @@
// The event streamer that is supposed to flush stdout/stderr.
private BuildEventStreamer streamer;
- public SynchronizedOutputStream(int maxBufferedLength) {
+ public SynchronizedOutputStream(int maxBufferedLength, int maxChunkSize) {
+ Preconditions.checkArgument(maxChunkSize > 0);
buf = new byte[64];
count = 0;
discardAll = false;
this.maxBufferedLength = maxBufferedLength;
+ this.maxChunkSize = Math.max(maxChunkSize, maxBufferedLength);
}
public void registerStreamer(BuildEventStreamer streamer) {
@@ -66,7 +72,9 @@
String content = new String(buf, 0, (int) count, UTF_8);
buf = new byte[64];
count = 0;
- return content.isEmpty() ? ImmutableList.of() : ImmutableList.of(content);
+ return content.isEmpty()
+ ? ImmutableList.of()
+ : Splitter.fixedLength(maxChunkSize).split(content);
}
@Override
