Read the .jdeps files created from Java header compilations when finishing the
action. The benefit is that reading these files is distributed more evenly
across the build. A single Javac compile at the end cannot be bottlenecked
upon needing to read hundreds of these files.
RELNOTES: None.
PiperOrigin-RevId: 253010039
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java
index fcc15da..a8c6f70 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java
@@ -56,6 +56,7 @@
import com.google.devtools.build.lib.actions.SingleStringArgFormatter;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.SpawnContinuation;
+import com.google.devtools.build.lib.actions.SpawnResult;
import com.google.devtools.build.lib.actions.extra.EnvironmentVariable;
import com.google.devtools.build.lib.actions.extra.ExtraActionInfo;
import com.google.devtools.build.lib.actions.extra.SpawnInfo;
@@ -71,6 +72,7 @@
import com.google.devtools.build.lib.syntax.SkylarkList;
import com.google.devtools.build.lib.util.Fingerprint;
import com.google.devtools.build.lib.util.LazyString;
+import com.google.devtools.build.lib.util.Pair;
import com.google.devtools.build.lib.util.ShellEscaper;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.errorprone.annotations.CompileTimeConstant;
@@ -81,6 +83,7 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.Consumer;
import javax.annotation.CheckReturnValue;
import javax.annotation.Nullable;
@@ -108,6 +111,7 @@
private final ExtraActionInfoSupplier extraActionInfoSupplier;
private final Artifact primaryOutput;
+ private final Consumer<Pair<ActionExecutionContext, List<SpawnResult>>> resultConsumer;
/**
* Constructs a SpawnAction using direct initialization arguments.
@@ -160,6 +164,7 @@
EmptyRunfilesSupplier.INSTANCE,
mnemonic,
false,
+ null,
null);
}
@@ -203,7 +208,8 @@
RunfilesSupplier runfilesSupplier,
String mnemonic,
boolean executeUnconditionally,
- ExtraActionInfoSupplier extraActionInfoSupplier) {
+ ExtraActionInfoSupplier extraActionInfoSupplier,
+ Consumer<Pair<ActionExecutionContext, List<SpawnResult>>> resultConsumer) {
super(owner, tools, inputs, runfilesSupplier, outputs, env);
this.primaryOutput = primaryOutput;
this.resourceSet = resourceSet;
@@ -215,6 +221,7 @@
this.mnemonic = mnemonic;
this.executeUnconditionally = executeUnconditionally;
this.extraActionInfoSupplier = extraActionInfoSupplier;
+ this.resultConsumer = resultConsumer;
}
@Override
@@ -589,6 +596,8 @@
protected ExtraActionInfoSupplier extraActionInfoSupplier = null;
private boolean disableSandboxing = false;
+ private Consumer<Pair<ActionExecutionContext, List<SpawnResult>>> resultConsumer = null;
+
/**
* Creates a SpawnAction builder.
*/
@@ -752,7 +761,8 @@
runfilesSupplier,
mnemonic,
executeUnconditionally,
- extraActionInfoSupplier);
+ extraActionInfoSupplier,
+ resultConsumer);
}
/**
@@ -1255,6 +1265,12 @@
this.disableSandboxing = true;
return this;
}
+
+ public Builder addResultConsumer(
+ Consumer<Pair<ActionExecutionContext, List<SpawnResult>>> resultConsumer) {
+ this.resultConsumer = resultConsumer;
+ return this;
+ }
}
/**
@@ -1327,6 +1343,9 @@
try {
SpawnContinuation nextContinuation = spawnContinuation.execute();
if (nextContinuation.isDone()) {
+ if (resultConsumer != null) {
+ resultConsumer.accept(Pair.of(actionExecutionContext, nextContinuation.get()));
+ }
afterExecute(actionExecutionContext);
return ActionContinuationOrResult.of(ActionResult.create(nextContinuation.get()));
}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/StarlarkAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/StarlarkAction.java
index 83a9971..00173c0 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/actions/StarlarkAction.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/StarlarkAction.java
@@ -98,7 +98,8 @@
runfilesSupplier,
mnemonic,
/* executeUnconditionally */ false,
- /* extraActionInfoSupplier */ null);
+ /* extraActionInfoSupplier */ null,
+ /* resultConsumer */ null);
this.allInputs = inputs;
this.unusedInputsList = unusedInputsList;
}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/extra/ExtraAction.java b/src/main/java/com/google/devtools/build/lib/analysis/extra/ExtraAction.java
index 4762ba7..c4cfef3 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/extra/ExtraAction.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/extra/ExtraAction.java
@@ -91,6 +91,7 @@
CompositeRunfilesSupplier.of(shadowedAction.getRunfilesSupplier(), runfilesSupplier),
mnemonic,
false,
+ null,
null);
this.shadowedAction = shadowedAction;
this.createDummyOutput = createDummyOutput;
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendAction.java
index 05b9146..c4fc42d 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendAction.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendAction.java
@@ -94,6 +94,7 @@
runfilesSupplier,
mnemonic,
false,
+ null,
null);
mandatoryInputs = inputs;
Preconditions.checkState(
diff --git a/src/main/java/com/google/devtools/build/lib/rules/genrule/GenRuleAction.java b/src/main/java/com/google/devtools/build/lib/rules/genrule/GenRuleAction.java
index ed9a977..6e69d32 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/genrule/GenRuleAction.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/genrule/GenRuleAction.java
@@ -64,6 +64,7 @@
runfilesSupplier,
MNEMONIC,
false,
+ null,
null);
}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaHeaderCompileActionBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaHeaderCompileActionBuilder.java
index 606158e..edf4b96 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaHeaderCompileActionBuilder.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaHeaderCompileActionBuilder.java
@@ -21,10 +21,15 @@
import static java.util.stream.Collectors.joining;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ExecutionRequirements;
import com.google.devtools.build.lib.actions.ParamFileInfo;
import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType;
+import com.google.devtools.build.lib.actions.SpawnResult;
import com.google.devtools.build.lib.analysis.FilesToRunProvider;
import com.google.devtools.build.lib.analysis.RuleContext;
import com.google.devtools.build.lib.analysis.actions.CustomCommandLine;
@@ -38,6 +43,9 @@
import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec;
import com.google.devtools.build.lib.util.LazyString;
import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.view.proto.Deps;
+import java.io.IOException;
+import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import javax.annotation.Nullable;
@@ -294,6 +302,36 @@
}
}
+ JavaConfiguration javaConfiguration =
+ ruleContext.getConfiguration().getFragment(JavaConfiguration.class);
+ if (javaConfiguration.inmemoryJdepsFiles()) {
+ builder.setExecutionInfo(
+ ImmutableMap.of(
+ ExecutionRequirements.REMOTE_EXECUTION_INLINE_OUTPUTS,
+ outputDepsProto.getExecPathString()));
+ }
+ builder.addResultConsumer(
+ contextAndResults -> {
+ ActionExecutionContext context = contextAndResults.getFirst();
+ JavaCompileActionContext javaContext = context.getContext(JavaCompileActionContext.class);
+ if (javaContext == null) {
+ return;
+ }
+ SpawnResult spawnResult = Iterables.getOnlyElement(contextAndResults.getSecond());
+ try {
+ InputStream inMemoryOutput = spawnResult.getInMemoryOutput(outputDepsProto);
+ try (InputStream input =
+ inMemoryOutput == null
+ ? context.getInputPath(outputDepsProto).getInputStream()
+ : inMemoryOutput) {
+ javaContext.insertDependencies(outputDepsProto, Deps.Dependencies.parseFrom(input));
+ }
+ } catch (IOException e) {
+ // Left empty. If we cannot read the .jdeps file now, we will read it later or throw
+ // an appropriate error then.
+ }
+ });
+
// The action doesn't require annotation processing, so use the non-javac-based turbine
// implementation.
if (!requiresAnnotationProcessing) {
