Support workers in CompatDexBuilder
This also fixes the `--use_workers_with_dexbuilder` flag which would pass the worker execution requirements to `CompatDexBuilder` without it knowing how to handle the work requests.
Using a worker here should help with some OOMs issues, and drastically improves build performance. We saw a 29% improvement in clean build times and a 14% improvement in incremental build times where the ABI was invalidated.
Closes #14623.
PiperOrigin-RevId: 443508457
diff --git a/src/tools/android/java/com/google/devtools/build/android/BUILD b/src/tools/android/java/com/google/devtools/build/android/BUILD
index 4774619..592451e 100644
--- a/src/tools/android/java/com/google/devtools/build/android/BUILD
+++ b/src/tools/android/java/com/google/devtools/build/android/BUILD
@@ -25,6 +25,7 @@
create_executable = 0,
runtime_deps = [
":android_builder_lib",
+ "//src/main/java/com/google/devtools/build/lib/worker:work_request_handlers",
"//src/tools/android/java/com/google/devtools/build/android/desugar",
"//src/tools/android/java/com/google/devtools/build/android/desugar/scan",
"//src/tools/android/java/com/google/devtools/build/android/dexer:dexerdeps",
diff --git a/src/tools/android/java/com/google/devtools/build/android/r8/BUILD b/src/tools/android/java/com/google/devtools/build/android/r8/BUILD
index 39343dd..400d259 100644
--- a/src/tools/android/java/com/google/devtools/build/android/r8/BUILD
+++ b/src/tools/android/java/com/google/devtools/build/android/r8/BUILD
@@ -41,6 +41,7 @@
}),
visibility = ["//src/test/java/com/google/devtools/build/android/r8:__pkg__"],
deps = [
+ "//src/main/java/com/google/devtools/build/lib/worker:work_request_handlers",
"//src/main/java/com/google/devtools/common/options",
"//src/tools/android/java/com/google/devtools/build/android:android_builder_lib",
"//third_party:asm",
diff --git a/src/tools/android/java/com/google/devtools/build/android/r8/CompatDexBuilder.java b/src/tools/android/java/com/google/devtools/build/android/r8/CompatDexBuilder.java
index 6aecd25..ef9ea90 100644
--- a/src/tools/android/java/com/google/devtools/build/android/r8/CompatDexBuilder.java
+++ b/src/tools/android/java/com/google/devtools/build/android/r8/CompatDexBuilder.java
@@ -26,14 +26,22 @@
import com.android.tools.r8.DiagnosticsHandler;
import com.android.tools.r8.origin.ArchiveEntryOrigin;
import com.android.tools.r8.origin.PathOrigin;
+import com.google.common.collect.ImmutableSet;
import com.google.common.io.ByteStreams;
+import com.google.devtools.build.lib.worker.ProtoWorkerMessageProcessor;
+import com.google.devtools.build.lib.worker.WorkRequestHandler;
import com.google.devtools.common.options.OptionsParsingException;
import java.io.BufferedOutputStream;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.PrintStream;
+import java.io.PrintWriter;
import java.nio.file.Files;
import java.nio.file.Paths;
+import java.time.Duration;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Enumeration;
import java.util.List;
import java.util.Set;
@@ -77,11 +85,60 @@
public static void main(String[] args)
throws IOException, InterruptedException, ExecutionException, OptionsParsingException {
CompatDexBuilder compatDexBuilder = new CompatDexBuilder();
- compatDexBuilder.processRequest(args);
+ if (ImmutableSet.copyOf(args).contains("--persistent_worker")) {
+ ByteArrayOutputStream buf = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(buf, true);
+ PrintStream realStdOut = System.out;
+ PrintStream realStdErr = System.err;
+
+ // Redirect all stdout and stderr output for logging.
+ System.setOut(ps);
+ System.setErr(ps);
+ try {
+ WorkRequestHandler workerHandler =
+ new WorkRequestHandler.WorkRequestHandlerBuilder(
+ new WorkRequestHandler.WorkRequestCallback(
+ (request, pw) ->
+ compatDexBuilder.processRequest(request.getArgumentsList(), pw, buf)),
+ realStdErr,
+ new ProtoWorkerMessageProcessor(System.in, realStdOut))
+ .setCpuUsageBeforeGc(Duration.ofSeconds(10))
+ .build();
+ workerHandler.processRequests();
+ } catch (IOException e) {
+ realStdErr.println(e.getMessage());
+ System.exit(1);
+ } finally {
+ System.setOut(realStdOut);
+ System.setErr(realStdErr);
+ }
+ } else {
+ compatDexBuilder.dexEntries(Arrays.asList(args));
+ }
+ }
+
+ private int processRequest(List<String> args, PrintWriter pw, ByteArrayOutputStream buf) {
+ try {
+ dexEntries(args);
+ return 0;
+ } catch (OptionsParsingException e) {
+ pw.println("CompatDexBuilder raised OptionsParsingException: " + e.getMessage());
+ return 1;
+ } catch (IOException | InterruptedException | ExecutionException e) {
+ e.printStackTrace();
+ return 1;
+ } finally {
+ // Write the captured buffer to the work response
+ synchronized (buf) {
+ String captured = buf.toString(UTF_8).trim();
+ buf.reset();
+ pw.print(captured);
+ }
+ }
}
@SuppressWarnings("JdkObsolete")
- private void processRequest(String[] args)
+ private void dexEntries(List<String> args)
throws IOException, InterruptedException, ExecutionException, OptionsParsingException {
List<String> flags = new ArrayList<>();
String input = null;
