Introduce --experimental_inmemory_jdeps_files. This is equivalent to C++'s --experimental_inmemory_dotd_files and has the goal of transfering the data of the .jdeps files generated for a Java compilation in memory if possible. These .jdeps artifacts aren't currently used in anyway other than to determine whether a fallback is necessary for --experimental_java_classpath=bazel. RELNOTES: None. PiperOrigin-RevId: 234985667
diff --git a/src/main/java/com/google/devtools/build/lib/rules/android/AndroidCommon.java b/src/main/java/com/google/devtools/build/lib/rules/android/AndroidCommon.java index 7a84702..4d15e1d 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/android/AndroidCommon.java +++ b/src/main/java/com/google/devtools/build/lib/rules/android/AndroidCommon.java
@@ -58,6 +58,7 @@ import com.google.devtools.build.lib.rules.java.JavaCompilationArgsProvider.ClasspathType; import com.google.devtools.build.lib.rules.java.JavaCompilationArtifacts; import com.google.devtools.build.lib.rules.java.JavaCompilationHelper; +import com.google.devtools.build.lib.rules.java.JavaCompileAction; import com.google.devtools.build.lib.rules.java.JavaInfo; import com.google.devtools.build.lib.rules.java.JavaPluginInfoProvider; import com.google.devtools.build.lib.rules.java.JavaRuleOutputJarsProvider; @@ -623,14 +624,10 @@ nativeHeaderOutput = helper.createNativeHeaderJar(classJar); - outputDepsProto = helper.createOutputDepsProtoArtifact(classJar, javaArtifactsBuilder); - helper.createCompileActionWithInstrumentation( - classJar, - manifestProtoOutput, - genSourceJar, - outputDepsProto, - javaArtifactsBuilder, - nativeHeaderOutput); + JavaCompileAction javaCompileAction = + helper.createCompileActionWithInstrumentation( + classJar, manifestProtoOutput, genSourceJar, javaArtifactsBuilder, nativeHeaderOutput); + outputDepsProto = javaCompileAction.getOutputDepsProto(); if (generateExtensionRegistry) { generatedExtensionRegistryProvider =
diff --git a/src/main/java/com/google/devtools/build/lib/rules/android/AndroidLocalTestBase.java b/src/main/java/com/google/devtools/build/lib/rules/android/AndroidLocalTestBase.java index d48e38c..606f26d 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/android/AndroidLocalTestBase.java +++ b/src/main/java/com/google/devtools/build/lib/rules/android/AndroidLocalTestBase.java
@@ -47,6 +47,7 @@ import com.google.devtools.build.lib.rules.java.JavaCompilationArgsProvider.ClasspathType; import com.google.devtools.build.lib.rules.java.JavaCompilationArtifacts; import com.google.devtools.build.lib.rules.java.JavaCompilationHelper; +import com.google.devtools.build.lib.rules.java.JavaCompileAction; import com.google.devtools.build.lib.rules.java.JavaConfiguration; import com.google.devtools.build.lib.rules.java.JavaConfiguration.OneVersionEnforcementLevel; import com.google.devtools.build.lib.rules.java.JavaHelper; @@ -239,18 +240,16 @@ genSourceJar = helper.createGensrcJar(classJar); helper.createGenJarAction(classJar, manifestProtoOutput, genClassJar); } - Artifact outputDepsProtoArtifact = - helper.createOutputDepsProtoArtifact(classJar, javaArtifactsBuilder); - javaRuleOutputJarsProviderBuilder.setJdeps(outputDepsProtoArtifact); - helper.createCompileAction( - classJar, - manifestProtoOutput, - genSourceJar, - outputDepsProtoArtifact, - instrumentationMetadata, - /* nativeHeaderOutput= */ null); + JavaCompileAction javaCompileAction = + helper.createCompileAction( + classJar, + manifestProtoOutput, + genSourceJar, + instrumentationMetadata, + /* nativeHeaderOutput= */ null); helper.createSourceJarAction(srcJar, genSourceJar); + javaRuleOutputJarsProviderBuilder.setJdeps(javaCompileAction.getOutputDepsProto()); setUpJavaCommon(javaCommon, helper, javaArtifactsBuilder.build());
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaBinary.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaBinary.java index e94868d..98a6967 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaBinary.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaBinary.java
@@ -236,9 +236,6 @@ javaArtifactsBuilder, ruleOutputJarsProviderBuilder, javaSourceJarsProviderBuilder); - Artifact outputDepsProto = helper.createOutputDepsProtoArtifact(classJar, javaArtifactsBuilder); - ruleOutputJarsProviderBuilder.setJdeps(outputDepsProto); - JavaCompilationArtifacts javaArtifacts = javaArtifactsBuilder.build(); common.setJavaCompilationArtifacts(javaArtifacts); @@ -252,14 +249,15 @@ helper.createGenJarAction(classJar, manifestProtoOutput, genClassJar); } - helper.createCompileAction( - classJar, - manifestProtoOutput, - genSourceJar, - outputDepsProto, - instrumentationMetadata, - /* nativeHeaderOutput= */ null); + JavaCompileAction javaCompileAction = + helper.createCompileAction( + classJar, + manifestProtoOutput, + genSourceJar, + instrumentationMetadata, + /* nativeHeaderOutput= */ null); helper.createSourceJarAction(srcJar, genSourceJar); + ruleOutputJarsProviderBuilder.setJdeps(javaCompileAction.getOutputDepsProto()); common.setClassPathFragment( new ClasspathConfiguredFragment(
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationHelper.java index e2c81c4..252d1ca 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationHelper.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationHelper.java
@@ -197,11 +197,10 @@ * --experimental_java_coverage is true). * @param nativeHeaderOutput an archive of generated native header files. */ - public void createCompileAction( + public JavaCompileAction createCompileAction( Artifact outputJar, Artifact manifestProtoOutput, @Nullable Artifact gensrcOutputJar, - @Nullable Artifact outputDepsProto, @Nullable Artifact instrumentationMetadataJar, @Nullable Artifact nativeHeaderOutput) { @@ -242,7 +241,7 @@ builder.setNativeHeaderOutput(nativeHeaderOutput); builder.setManifestProtoOutput(manifestProtoOutput); builder.setGensrcOutputJar(gensrcOutputJar); - builder.setOutputDepsProto(outputDepsProto); + builder.setOutputDepsProto(getOutputDepsProtoPath(classJar)); builder.setAdditionalOutputs(attributes.getAdditionalOutputs()); builder.setMetadata(instrumentationMetadataJar); builder.setInstrumentationJars(jacocoInstrumentation); @@ -264,7 +263,7 @@ builder.setTargetLabel( attributes.getTargetLabel() == null ? ruleContext.getLabel() : attributes.getTargetLabel()); builder.setInjectingRuleKind(attributes.getInjectingRuleKind()); - builder.build(ruleContext, semantics); + return builder.build(ruleContext, semantics); } private ImmutableMap<String, String> getExecutionInfo() { @@ -327,22 +326,19 @@ * @param outputJar the class jar Artifact to create with the Action * @param manifestProtoOutput the output artifact for the manifest proto emitted from JavaBuilder * @param gensrcJar the generated sources jar Artifact to create with the Action - * @param outputDepsProto the compiler-generated jdeps file to create with the Action * @param javaArtifactsBuilder the build to store the instrumentation metadata in * @param nativeHeaderOutput an archive of generated native header files. */ - public void createCompileActionWithInstrumentation( + public JavaCompileAction createCompileActionWithInstrumentation( Artifact outputJar, Artifact manifestProtoOutput, @Nullable Artifact gensrcJar, - @Nullable Artifact outputDepsProto, JavaCompilationArtifacts.Builder javaArtifactsBuilder, @Nullable Artifact nativeHeaderOutput) { - createCompileAction( + return createCompileAction( outputJar, manifestProtoOutput, gensrcJar, - outputDepsProto, createInstrumentationMetadata(outputJar, javaArtifactsBuilder), nativeHeaderOutput); } @@ -568,26 +564,17 @@ } /** - * Creates the jdeps file artifact if needed. Returns null if the target can't emit dependency + * Creates the jdeps file path if needed. Returns null if the target can't emit dependency * information (i.e there is no compilation step, the target acts as an alias). * * @param outputJar output jar artifact used to derive the name * @return the jdeps file artifact or null if the target can't generate such a file */ - public Artifact createOutputDepsProtoArtifact( - Artifact outputJar, JavaCompilationArtifacts.Builder builder) { + public PathFragment getOutputDepsProtoPath(Artifact outputJar) { if (!generatesOutputDeps()) { return null; } - - Artifact outputDepsProtoArtifact = - getRuleContext() - .getDerivedArtifact( - FileSystemUtils.replaceExtension(outputJar.getRootRelativePath(), ".jdeps"), - outputJar.getRoot()); - - builder.setCompileTimeDependencies(outputDepsProtoArtifact); - return outputDepsProtoArtifact; + return FileSystemUtils.replaceExtension(outputJar.getExecPath(), ".jdeps"); } /**
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileAction.java index 57ed513..db44244 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileAction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileAction.java
@@ -95,11 +95,12 @@ private final ImmutableList<Artifact> sourceJars; private final JavaPluginInfo plugins; + private final ImmutableList<? extends ActionInput> outputFiles; private final NestedSet<Artifact> directJars; private final NestedSet<Artifact> mandatoryInputs; private final NestedSet<Artifact> transitiveInputs; private final NestedSet<Artifact> dependencyArtifacts; - private final Artifact outputDepsProto; + private final ActionInput outputDepsProto; private final JavaClasspathMode classpathMode; private final JavaCompileExtraActionInfoSupplier extraActionInfoSupplier; @@ -122,7 +123,7 @@ CommandLine flagLine, BuildConfiguration configuration, NestedSet<Artifact> dependencyArtifacts, - Artifact outputDepsProto, + ActionInput outputDepsProto, JavaClasspathMode classpathMode) { super( owner, @@ -147,6 +148,12 @@ this.dependencyArtifacts = dependencyArtifacts; this.outputDepsProto = outputDepsProto; this.classpathMode = classpathMode; + ImmutableList.Builder<ActionInput> outputsBuilder = ImmutableList.builder(); + outputsBuilder.addAll(outputs); + if (outputDepsProto != null) { + outputsBuilder.add(outputDepsProto); + } + outputFiles = outputsBuilder.build(); } @Override @@ -297,7 +304,12 @@ spawnActionContext.exec( getReducedSpawn(actionExecutionContext, reducedClasspath, /* fallback= */ false), actionExecutionContext); - try (InputStream input = outputDepsProto.getPath().getInputStream()) { + + SpawnResult spawnResult = Iterables.getOnlyElement(results); + try (InputStream input = + (outputDepsProto instanceof Artifact) + ? ((Artifact) outputDepsProto).getPath().getInputStream() + : spawnResult.getInMemoryOutput(outputDepsProto)) { if (!Deps.Dependencies.parseFrom(input).getRequiresReducedClasspathFallback()) { return ActionResult.create(results); } @@ -406,6 +418,11 @@ public Iterable<? extends ActionInput> getInputFiles() { return inputs; } + + @Override + public Collection<? extends ActionInput> getOutputFiles() { + return outputFiles; + } } @VisibleForTesting @@ -463,4 +480,8 @@ public Iterable<Artifact> getPossibleInputsForTesting() { return null; } + + public Artifact getOutputDepsProto() { + return (outputDepsProto instanceof Artifact) ? (Artifact) outputDepsProto : null; + } }
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileActionBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileActionBuilder.java index f5079df..e6b64cc 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileActionBuilder.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileActionBuilder.java
@@ -24,8 +24,11 @@ import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.actions.ActionAnalysisMetadata; import com.google.devtools.build.lib.actions.ActionEnvironment; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ActionInputHelper; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.EmptyRunfilesSupplier; +import com.google.devtools.build.lib.actions.ExecutionRequirements; import com.google.devtools.build.lib.actions.RunfilesSupplier; import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; import com.google.devtools.build.lib.actions.extra.JavaCompileInfo; @@ -46,6 +49,7 @@ import com.google.devtools.build.lib.rules.java.JavaPluginInfoProvider.JavaPluginInfo; import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; import com.google.devtools.build.lib.util.StringCanonicalizer; +import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.PathFragment; import java.util.ArrayList; import java.util.Collection; @@ -138,7 +142,7 @@ private Artifact nativeHeaderOutput; private Artifact gensrcOutputJar; private Artifact manifestProtoOutput; - private Artifact outputDepsProto; + private PathFragment outputDepsProto; private Collection<Artifact> additionalOutputs; private Artifact metadata; private Artifact artifactForExperimentalCoverage; @@ -169,7 +173,7 @@ private Label targetLabel; @Nullable private String injectingRuleKind; - public void build(RuleContext ruleContext, JavaSemantics javaSemantics) { + public JavaCompileAction build(RuleContext ruleContext, JavaSemantics javaSemantics) { // TODO(bazel-team): all the params should be calculated before getting here, and the various // aggregation code below should go away. ImmutableList<String> internedJcopts = @@ -197,7 +201,6 @@ metadata, gensrcOutputJar, manifestProtoOutput, - outputDepsProto, nativeHeaderOutput) .filter(x -> x != null) .forEachOrdered(outputs::add); @@ -284,6 +287,26 @@ classpathMode = JavaClasspathMode.OFF; } + ActionInput outputDepsProtoInput = null; + if (outputDepsProto != null) { + if (javaConfiguration.inmemoryJdepsFiles()) { + outputDepsProtoInput = ActionInputHelper.fromPath(outputDepsProto.getSafePathString()); + executionInfo = + ImmutableMap.<String, String>builderWithExpectedSize(this.executionInfo.size() + 1) + .putAll(this.executionInfo) + .put( + ExecutionRequirements.REMOTE_EXECUTION_INLINE_OUTPUTS, + outputDepsProtoInput.getExecPathString()) + .build(); + } else { + Artifact outputDepsProtoArtifact = + ruleContext.getDerivedArtifact( + FileSystemUtils.replaceExtension(outputJar.getRootRelativePath(), ".jdeps"), + outputJar.getRoot()); + outputDepsProtoInput = outputDepsProtoArtifact; + outputs.add(outputDepsProtoArtifact); + } + } NestedSet<Artifact> tools = toolsBuilder.build(); mandatoryInputs.addTransitive(tools); JavaCompileAction javaCompileAction = @@ -305,9 +328,10 @@ /* flagLine= */ buildParamFileContents(ruleContext.getConfiguration(), internedJcopts), /* configuration= */ ruleContext.getConfiguration(), /* dependencyArtifacts= */ compileTimeDependencyArtifacts, - /* outputDepsProto= */ outputDepsProto, + /* outputDepsProto= */ outputDepsProtoInput, /* classpathMode= */ classpathMode); ruleContext.getAnalysisEnvironment().registerAction(javaCompileAction); + return javaCompileAction; } private CustomCommandLine buildParamFileContents( @@ -327,7 +351,7 @@ if (compressJar) { result.add("--compress_jar"); } - result.addExecPath("--output_deps_proto", outputDepsProto); + result.addPath("--output_deps_proto", outputDepsProto); result.addExecPaths("--extclasspath", extdirInputs); result.addExecPaths("--bootclasspath", bootclasspathEntries); result.addExecPaths("--sourcepath", sourcePathEntries); @@ -406,7 +430,7 @@ return this; } - public JavaCompileActionBuilder setOutputDepsProto(Artifact outputDepsProto) { + public JavaCompileActionBuilder setOutputDepsProto(PathFragment outputDepsProto) { this.outputDepsProto = outputDepsProto; return this; }
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfiguration.java index 2f4e617..cc30c41 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfiguration.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfiguration.java
@@ -157,6 +157,7 @@ private final ImportDepsCheckingLevel importDepsCheckingLevel; private final boolean allowRuntimeDepsOnNeverLink; private final JavaClasspathMode javaClasspath; + private final boolean inmemoryJdepsFiles; private final ImmutableList<String> defaultJvmFlags; private final ImmutableList<String> checkedConstraints; private final StrictDepsMode strictJavaDeps; @@ -190,6 +191,7 @@ this.generateJavaDeps = javaOptions.javaDeps || javaOptions.javaClasspath != JavaClasspathMode.OFF; this.javaClasspath = javaOptions.javaClasspath; + this.inmemoryJdepsFiles = javaOptions.inmemoryJdepsFiles; this.defaultJvmFlags = ImmutableList.copyOf(javaOptions.jvmOpts); this.checkedConstraints = ImmutableList.copyOf(javaOptions.checkedConstraints); this.strictJavaDeps = javaOptions.strictJavaDeps; @@ -285,6 +287,10 @@ return javaClasspath; } + public boolean inmemoryJdepsFiles() { + return inmemoryJdepsFiles; + } + public ImmutableList<String> getDefaultJvmFlags() { return defaultJvmFlags; }
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibrary.java index 5464f3d..4856451 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibrary.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibrary.java
@@ -116,17 +116,11 @@ helper.createGenJarAction(classJar, manifestProtoOutput, genClassJar); } - Artifact outputDepsProto = helper.createOutputDepsProtoArtifact(classJar, javaArtifactsBuilder); - Artifact nativeHeaderOutput = helper.createNativeHeaderJar(classJar); - helper.createCompileActionWithInstrumentation( - classJar, - manifestProtoOutput, - genSourceJar, - outputDepsProto, - javaArtifactsBuilder, - nativeHeaderOutput); + JavaCompileAction javaCompileAction = + helper.createCompileActionWithInstrumentation( + classJar, manifestProtoOutput, genSourceJar, javaArtifactsBuilder, nativeHeaderOutput); helper.createSourceJarAction(srcJar, genSourceJar); Artifact iJar = null; @@ -136,7 +130,7 @@ JavaRuleOutputJarsProvider.Builder ruleOutputJarsProviderBuilder = JavaRuleOutputJarsProvider.builder() .addOutputJar(classJar, iJar, manifestProtoOutput, ImmutableList.of(srcJar)) - .setJdeps(outputDepsProto) + .setJdeps(javaCompileAction.getOutputDepsProto()) .setNativeHeaders(nativeHeaderOutput); GeneratedExtensionRegistryProvider generatedExtensionRegistryProvider = null;
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibraryHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibraryHelper.java index ce4fca5..e365743 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibraryHelper.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibraryHelper.java
@@ -235,8 +235,6 @@ javaToolchainProvider, hostJavabase, jacocoInstrumental); - Artifact outputDepsProto = helper.createOutputDepsProtoArtifact(output, artifactsBuilder); - Artifact manifestProtoOutput = helper.createManifestProtoOutput(output); Artifact genSourceJar = null; @@ -249,13 +247,13 @@ Artifact nativeHeaderOutput = helper.createNativeHeaderJar(output); - helper.createCompileAction( - output, - manifestProtoOutput, - genSourceJar, - outputDepsProto, - /* instrumentationMetadataJar= */ null, - nativeHeaderOutput); + JavaCompileAction javaCompileAction = + helper.createCompileAction( + output, + manifestProtoOutput, + genSourceJar, + /* instrumentationMetadataJar= */ null, + nativeHeaderOutput); Artifact iJar = null; if (!sourceJars.isEmpty() || !sourceFiles.isEmpty()) { @@ -271,7 +269,7 @@ outputSourceJar == null ? ImmutableList.of() : ImmutableList.of(outputSourceJar); outputJarsBuilder .addOutputJar(new OutputJar(output, iJar, manifestProtoOutput, outputSourceJars)) - .setJdeps(outputDepsProto) + .setJdeps(javaCompileAction.getOutputDepsProto()) .setNativeHeaders(nativeHeaderOutput); JavaCompilationArtifacts javaArtifacts = artifactsBuilder.build();
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaOptions.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaOptions.java index 70e0c06..a50f2f4 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaOptions.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaOptions.java
@@ -226,6 +226,22 @@ public JavaClasspathMode javaClasspath; @Option( + name = "experimental_inmemory_jdeps_files", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.BUILD_TIME_OPTIMIZATION, + effectTags = { + OptionEffectTag.LOADING_AND_ANALYSIS, + OptionEffectTag.EXECUTION, + OptionEffectTag.AFFECTS_OUTPUTS + }, + metadataTags = {OptionMetadataTag.EXPERIMENTAL}, + help = + "If enabled, the dependency (.jdeps) files generated from Java compilations will be " + + "passed through in memory directly from the remote build nodes instead of being " + + "written to disk.") + public boolean inmemoryJdepsFiles; + + @Option( name = "java_debug", defaultValue = "null", expansion = { @@ -679,6 +695,7 @@ host.javaDeps = javaDeps; host.javaClasspath = javaClasspath; + host.inmemoryJdepsFiles = inmemoryJdepsFiles; host.strictJavaDeps = strictJavaDeps; host.fixDepsTool = fixDepsTool;