Read the .jdeps files created from Java header compilations when finishing the action. The benefit is that reading these files is distributed more evenly across the build. A single Javac compile at the end cannot be bottlenecked upon needing to read hundreds of these files. RELNOTES: None. PiperOrigin-RevId: 253010039
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java index fcc15da..a8c6f70 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java
@@ -56,6 +56,7 @@ import com.google.devtools.build.lib.actions.SingleStringArgFormatter; import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.SpawnContinuation; +import com.google.devtools.build.lib.actions.SpawnResult; import com.google.devtools.build.lib.actions.extra.EnvironmentVariable; import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; import com.google.devtools.build.lib.actions.extra.SpawnInfo; @@ -71,6 +72,7 @@ import com.google.devtools.build.lib.syntax.SkylarkList; import com.google.devtools.build.lib.util.Fingerprint; import com.google.devtools.build.lib.util.LazyString; +import com.google.devtools.build.lib.util.Pair; import com.google.devtools.build.lib.util.ShellEscaper; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.errorprone.annotations.CompileTimeConstant; @@ -81,6 +83,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import javax.annotation.CheckReturnValue; import javax.annotation.Nullable; @@ -108,6 +111,7 @@ private final ExtraActionInfoSupplier extraActionInfoSupplier; private final Artifact primaryOutput; + private final Consumer<Pair<ActionExecutionContext, List<SpawnResult>>> resultConsumer; /** * Constructs a SpawnAction using direct initialization arguments. @@ -160,6 +164,7 @@ EmptyRunfilesSupplier.INSTANCE, mnemonic, false, + null, null); } @@ -203,7 +208,8 @@ RunfilesSupplier runfilesSupplier, String mnemonic, boolean executeUnconditionally, - ExtraActionInfoSupplier extraActionInfoSupplier) { + ExtraActionInfoSupplier extraActionInfoSupplier, + Consumer<Pair<ActionExecutionContext, List<SpawnResult>>> resultConsumer) { super(owner, tools, inputs, runfilesSupplier, outputs, env); this.primaryOutput = primaryOutput; this.resourceSet = resourceSet; @@ -215,6 +221,7 @@ this.mnemonic = mnemonic; this.executeUnconditionally = executeUnconditionally; this.extraActionInfoSupplier = extraActionInfoSupplier; + this.resultConsumer = resultConsumer; } @Override @@ -589,6 +596,8 @@ protected ExtraActionInfoSupplier extraActionInfoSupplier = null; private boolean disableSandboxing = false; + private Consumer<Pair<ActionExecutionContext, List<SpawnResult>>> resultConsumer = null; + /** * Creates a SpawnAction builder. */ @@ -752,7 +761,8 @@ runfilesSupplier, mnemonic, executeUnconditionally, - extraActionInfoSupplier); + extraActionInfoSupplier, + resultConsumer); } /** @@ -1255,6 +1265,12 @@ this.disableSandboxing = true; return this; } + + public Builder addResultConsumer( + Consumer<Pair<ActionExecutionContext, List<SpawnResult>>> resultConsumer) { + this.resultConsumer = resultConsumer; + return this; + } } /** @@ -1327,6 +1343,9 @@ try { SpawnContinuation nextContinuation = spawnContinuation.execute(); if (nextContinuation.isDone()) { + if (resultConsumer != null) { + resultConsumer.accept(Pair.of(actionExecutionContext, nextContinuation.get())); + } afterExecute(actionExecutionContext); return ActionContinuationOrResult.of(ActionResult.create(nextContinuation.get())); }
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/StarlarkAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/StarlarkAction.java index 83a9971..00173c0 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/actions/StarlarkAction.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/StarlarkAction.java
@@ -98,7 +98,8 @@ runfilesSupplier, mnemonic, /* executeUnconditionally */ false, - /* extraActionInfoSupplier */ null); + /* extraActionInfoSupplier */ null, + /* resultConsumer */ null); this.allInputs = inputs; this.unusedInputsList = unusedInputsList; }
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/extra/ExtraAction.java b/src/main/java/com/google/devtools/build/lib/analysis/extra/ExtraAction.java index 4762ba7..c4cfef3 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/extra/ExtraAction.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/extra/ExtraAction.java
@@ -91,6 +91,7 @@ CompositeRunfilesSupplier.of(shadowedAction.getRunfilesSupplier(), runfilesSupplier), mnemonic, false, + null, null); this.shadowedAction = shadowedAction; this.createDummyOutput = createDummyOutput;
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendAction.java index 05b9146..c4fc42d 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendAction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendAction.java
@@ -94,6 +94,7 @@ runfilesSupplier, mnemonic, false, + null, null); mandatoryInputs = inputs; Preconditions.checkState(
diff --git a/src/main/java/com/google/devtools/build/lib/rules/genrule/GenRuleAction.java b/src/main/java/com/google/devtools/build/lib/rules/genrule/GenRuleAction.java index ed9a977..6e69d32 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/genrule/GenRuleAction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/genrule/GenRuleAction.java
@@ -64,6 +64,7 @@ runfilesSupplier, MNEMONIC, false, + null, null); }
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaHeaderCompileActionBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaHeaderCompileActionBuilder.java index 606158e..edf4b96 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaHeaderCompileActionBuilder.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaHeaderCompileActionBuilder.java
@@ -21,10 +21,15 @@ import static java.util.stream.Collectors.joining; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionExecutionContext; import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ExecutionRequirements; import com.google.devtools.build.lib.actions.ParamFileInfo; import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; +import com.google.devtools.build.lib.actions.SpawnResult; import com.google.devtools.build.lib.analysis.FilesToRunProvider; import com.google.devtools.build.lib.analysis.RuleContext; import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; @@ -38,6 +43,9 @@ import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; import com.google.devtools.build.lib.util.LazyString; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.proto.Deps; +import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; import javax.annotation.Nullable; @@ -294,6 +302,36 @@ } } + JavaConfiguration javaConfiguration = + ruleContext.getConfiguration().getFragment(JavaConfiguration.class); + if (javaConfiguration.inmemoryJdepsFiles()) { + builder.setExecutionInfo( + ImmutableMap.of( + ExecutionRequirements.REMOTE_EXECUTION_INLINE_OUTPUTS, + outputDepsProto.getExecPathString())); + } + builder.addResultConsumer( + contextAndResults -> { + ActionExecutionContext context = contextAndResults.getFirst(); + JavaCompileActionContext javaContext = context.getContext(JavaCompileActionContext.class); + if (javaContext == null) { + return; + } + SpawnResult spawnResult = Iterables.getOnlyElement(contextAndResults.getSecond()); + try { + InputStream inMemoryOutput = spawnResult.getInMemoryOutput(outputDepsProto); + try (InputStream input = + inMemoryOutput == null + ? context.getInputPath(outputDepsProto).getInputStream() + : inMemoryOutput) { + javaContext.insertDependencies(outputDepsProto, Deps.Dependencies.parseFrom(input)); + } + } catch (IOException e) { + // Left empty. If we cannot read the .jdeps file now, we will read it later or throw + // an appropriate error then. + } + }); + // The action doesn't require annotation processing, so use the non-javac-based turbine // implementation. if (!requiresAnnotationProcessing) {