Make the number of jobs in the worker configurable via an option
PiperOrigin-RevId: 161505952
diff --git a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/BUILD b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/BUILD
index 7e85b24..60d4433 100644
--- a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/BUILD
+++ b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/BUILD
@@ -25,6 +25,7 @@
"//src/main/java/com/google/devtools/build/lib:unix",
"//src/main/java/com/google/devtools/build/lib:util",
"//src/main/java/com/google/devtools/build/lib:vfs",
+ "//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/remote",
"//src/main/java/com/google/devtools/common/options",
"//src/main/protobuf:option_filters_java_proto",
diff --git a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/ExecutionServer.java b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/ExecutionServer.java
index 034ff9d..ec8dc95 100644
--- a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/ExecutionServer.java
+++ b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/ExecutionServer.java
@@ -85,9 +85,7 @@
private final RemoteWorkerOptions workerOptions;
private final SimpleBlobStoreActionCache cache;
private final ConcurrentHashMap<String, ListenableFuture<ActionResult>> operationsCache;
- private final ListeningExecutorService executorService =
- MoreExecutors.listeningDecorator(
- new ThreadPoolExecutor(0, 8, 1000, TimeUnit.SECONDS, new LinkedBlockingQueue<>()));
+ private final ListeningExecutorService executorService;
public ExecutionServer(
Path workPath,
@@ -100,6 +98,13 @@
this.workerOptions = workerOptions;
this.cache = cache;
this.operationsCache = operationsCache;
+ this.executorService =
+ MoreExecutors.listeningDecorator(
+ new ThreadPoolExecutor(
+ 1, workerOptions.jobs, // always have one thread available, and use at most jobs
+ 1000, TimeUnit.SECONDS, // shut down idle threads after 1000 seconds
+ // TODO(ulfjack): We need to reject work eventually.
+ new LinkedBlockingQueue<>())); // no blocking, we can always take more
}
@Override
diff --git a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorkerOptions.java b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorkerOptions.java
index e8bc12a..b66e07b 100644
--- a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorkerOptions.java
+++ b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorkerOptions.java
@@ -14,9 +14,12 @@
package com.google.devtools.build.remote;
+import com.google.devtools.build.lib.actions.LocalHostCapacity;
+import com.google.devtools.common.options.Converters.RangeConverter;
import com.google.devtools.common.options.Option;
import com.google.devtools.common.options.OptionDocumentationCategory;
import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParsingException;
import com.google.devtools.common.options.proto.OptionFilters.OptionEffectTag;
import java.util.List;
@@ -116,4 +119,42 @@
help = "When using sandboxing, block network access for running actions."
)
public boolean sandboxingBlockNetwork;
+
+ @Option(
+ name = "jobs",
+ defaultValue = "auto",
+ converter = JobsConverter.class,
+ category = "build_worker",
+ documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+ effectTags = {OptionEffectTag.UNKNOWN},
+ help =
+ "The maximum number of concurrent jobs to run. \"auto\" means to use a reasonable value"
+ + " derived from the machine's hardware profile (e.g. the number of processors)."
+ + " Values above " + MAX_JOBS + " are not allowed."
+ )
+ public int jobs;
+
+ private static final int MAX_JOBS = 16384;
+
+ /** Converter for jobs: [0, MAX_JOBS] or "auto". */
+ public static class JobsConverter extends RangeConverter {
+ public JobsConverter() {
+ super(0, MAX_JOBS);
+ }
+
+ @Override
+ public Integer convert(String input) throws OptionsParsingException {
+ if (input.equals("auto")) {
+ int autoJobs = (int) Math.ceil(LocalHostCapacity.getLocalHostCapacity().getCpuUsage());
+ return Math.min(autoJobs, MAX_JOBS);
+ } else {
+ return super.convert(input);
+ }
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "\"auto\" or " + super.getTypeDescription();
+ }
+ }
}
