Add --build_event_upload_max_threads option
This is a follow up for https://github.com/bazelbuild/bazel/commit/7fc967c4d6435de2bb4e34aac00ca2e499f55fca, related issue https://github.com/bazelbuild/bazel/issues/6806
RELNOTES: None
PiperOrigin-RevId: 223944430
diff --git a/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploader.java b/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploader.java
index 8d2e931..3afd548 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploader.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploader.java
@@ -41,9 +41,7 @@
*/
class ByteStreamBuildEventArtifactUploader implements BuildEventArtifactUploader {
- private final ListeningExecutorService uploadExecutor =
- MoreExecutors.listeningDecorator(
- Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()));
+ private final ListeningExecutorService uploadExecutor;
private final Context ctx;
private final ByteStreamUploader uploader;
private final String remoteServerInstanceName;
@@ -51,8 +49,11 @@
private final AtomicBoolean shutdown = new AtomicBoolean();
ByteStreamBuildEventArtifactUploader(
- ByteStreamUploader uploader, String remoteServerName, Context ctx,
- @Nullable String remoteInstanceName) {
+ ByteStreamUploader uploader,
+ String remoteServerName,
+ Context ctx,
+ @Nullable String remoteInstanceName,
+ int maxUploadThreads) {
this.uploader = Preconditions.checkNotNull(uploader);
String remoteServerInstanceName = Preconditions.checkNotNull(remoteServerName);
if (!Strings.isNullOrEmpty(remoteInstanceName)) {
@@ -60,6 +61,10 @@
}
this.ctx = ctx;
this.remoteServerInstanceName = remoteServerInstanceName;
+ // Limit the maximum threads number to 1000 (chosen arbitrarily)
+ this.uploadExecutor =
+ MoreExecutors.listeningDecorator(
+ Executors.newFixedThreadPool(Math.min(maxUploadThreads, 1000)));
}
@Override
diff --git a/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderFactory.java b/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderFactory.java
index 8c57f40..53157d5 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderFactory.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderFactory.java
@@ -41,7 +41,11 @@
@Override
public BuildEventArtifactUploader create(CommandEnvironment env) {
- return new ByteStreamBuildEventArtifactUploader(uploader.retain(), remoteServerName, ctx,
- remoteInstanceName);
+ return new ByteStreamBuildEventArtifactUploader(
+ uploader.retain(),
+ remoteServerName,
+ ctx,
+ remoteInstanceName,
+ env.getOptions().getOptions(RemoteOptions.class).buildEventUploadMaxThreads);
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java
index 180e8a9..9f59acd 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java
@@ -252,6 +252,14 @@
+ "symlinks and represent them as files. See #6631 for details.")
public boolean incompatibleRemoteSymlinks;
+ @Option(
+ name = "build_event_upload_max_threads",
+ defaultValue = "100",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {OptionEffectTag.UNKNOWN},
+ help = "The number of threads used to do build event uploads. Capped at 1000.")
+ public int buildEventUploadMaxThreads;
+
@Deprecated
@Option(
name = "remote_allow_symlink_upload",
diff --git a/src/test/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderTest.java b/src/test/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderTest.java
index 69c107b..08e1c95 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderTest.java
@@ -145,7 +145,7 @@
new ByteStreamUploader("instance", refCntChannel, null, 3, retrier);
ByteStreamBuildEventArtifactUploader artifactUploader =
new ByteStreamBuildEventArtifactUploader(
- uploader, "localhost", withEmptyMetadata, "instance");
+ uploader, "localhost", withEmptyMetadata, "instance", /* maxUploadThreads= */ 100);
PathConverter pathConverter = artifactUploader.upload(filesToUpload).get();
for (Path file : filesToUpload.keySet()) {
@@ -171,7 +171,7 @@
ByteStreamUploader uploader = mock(ByteStreamUploader.class);
ByteStreamBuildEventArtifactUploader artifactUploader =
new ByteStreamBuildEventArtifactUploader(
- uploader, "localhost", withEmptyMetadata, "instance");
+ uploader, "localhost", withEmptyMetadata, "instance", /* maxUploadThreads= */ 100);
PathConverter pathConverter = artifactUploader.upload(filesToUpload).get();
assertThat(pathConverter.apply(dir)).isNull();
artifactUploader.shutdown();
@@ -234,7 +234,7 @@
new ByteStreamUploader("instance", refCntChannel, null, 3, retrier);
ByteStreamBuildEventArtifactUploader artifactUploader =
new ByteStreamBuildEventArtifactUploader(
- uploader, "localhost", withEmptyMetadata, "instance");
+ uploader, "localhost", withEmptyMetadata, "instance", /* maxUploadThreads= */ 100);
try {
artifactUploader.upload(filesToUpload).get();
