Stream aquery proto outputs. aquery outputs proto results in a monolithic manner, which means storing the entire ActionGraphContainer in memory before printing everything to an output file. This could cause OOM error on larger queries. This CL makes aquery stream results instead. This requires a change in proto.ActionGraphContainer: instead of a wrapper object like before, it's now a collection of ActionGraphComponent. ActionGraphComponent would have a Discriminator, similar to query's build.Target. This is a breaking change for users of --incompatible_proto_output_v2. RELNOTES: None PiperOrigin-RevId: 283726774
diff --git a/src/main/java/com/google/devtools/build/lib/query2/aquery/ActionGraphProtoV2OutputFormatterCallback.java b/src/main/java/com/google/devtools/build/lib/query2/aquery/ActionGraphProtoV2OutputFormatterCallback.java index 6919478..8d7962a 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/aquery/ActionGraphProtoV2OutputFormatterCallback.java +++ b/src/main/java/com/google/devtools/build/lib/query2/aquery/ActionGraphProtoV2OutputFormatterCallback.java
@@ -13,9 +13,7 @@ // limitations under the License. package com.google.devtools.build.lib.query2.aquery; -import com.google.common.annotations.VisibleForTesting; import com.google.devtools.build.lib.actions.CommandLineExpansionException; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphContainer; import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget; import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.TargetAccessor; @@ -23,32 +21,25 @@ import com.google.devtools.build.lib.skyframe.ConfiguredTargetValue; import com.google.devtools.build.lib.skyframe.SkyframeExecutor; import com.google.devtools.build.lib.skyframe.actiongraph.v2.ActionGraphDump; -import com.google.protobuf.TextFormat; +import com.google.devtools.build.lib.skyframe.actiongraph.v2.StreamedOutputHandler; +import com.google.devtools.build.lib.skyframe.actiongraph.v2.StreamedOutputHandler.OutputType; +import com.google.protobuf.CodedOutputStream; import java.io.IOException; import java.io.OutputStream; /** Default output callback for aquery, prints proto output. */ public class ActionGraphProtoV2OutputFormatterCallback extends AqueryThreadsafeCallback { - /** Defines the types of proto output this class can handle. */ - public enum OutputType { - BINARY("proto"), - TEXT("textproto"); - - private final String formatName; - - OutputType(String formatName) { - this.formatName = formatName; - } - - public String formatName() { - return formatName; - } - } - private final OutputType outputType; private final ActionGraphDump actionGraphDump; private final AqueryActionFilter actionFilters; + private StreamedOutputHandler streamedOutputHandler; + + /** + * Pseudo-arbitrarily chosen buffer size for output. Chosen to be large enough to fit a handful of + * messages without needing to flush to the underlying output, which may not be buffered. + */ + private static final int OUTPUT_BUFFER_SIZE = 16384; ActionGraphProtoV2OutputFormatterCallback( ExtendedEventHandler eventHandler, @@ -61,12 +52,20 @@ super(eventHandler, options, out, skyframeExecutor, accessor); this.outputType = outputType; this.actionFilters = actionFilters; + if (out != null) { + this.streamedOutputHandler = + new StreamedOutputHandler( + this.outputType, + CodedOutputStream.newInstance(out, OUTPUT_BUFFER_SIZE), + this.printStream); + } this.actionGraphDump = new ActionGraphDump( options.includeCommandline, options.includeArtifacts, this.actionFilters, - options.includeParamFiles); + options.includeParamFiles, + streamedOutputHandler); } @Override @@ -98,25 +97,8 @@ @Override public void close(boolean failFast) throws IOException { - if (!failFast && printStream != null) { - ActionGraphContainer actionGraphContainer = actionGraphDump.build(); - - // Write the data. - switch (outputType) { - case BINARY: - actionGraphContainer.writeTo(printStream); - break; - case TEXT: - TextFormat.print(actionGraphContainer, printStream); - break; - default: - throw new IllegalStateException("Unknown outputType " + outputType.formatName()); - } + if (!failFast) { + streamedOutputHandler.close(); } } - - @VisibleForTesting - public ActionGraphContainer getProtoResult() { - return actionGraphDump.build(); - } }
diff --git a/src/main/java/com/google/devtools/build/lib/query2/aquery/ActionGraphQueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/aquery/ActionGraphQueryEnvironment.java index f004f83..5e0e91e 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/aquery/ActionGraphQueryEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/query2/aquery/ActionGraphQueryEnvironment.java
@@ -49,6 +49,7 @@ import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey; import com.google.devtools.build.lib.skyframe.ConfiguredTargetValue; import com.google.devtools.build.lib.skyframe.SkyframeExecutor; +import com.google.devtools.build.lib.skyframe.actiongraph.v2.StreamedOutputHandler; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.WalkableGraph; import java.io.OutputStream; @@ -167,7 +168,7 @@ out, skyframeExecutor, accessor, - ActionGraphProtoV2OutputFormatterCallback.OutputType.BINARY, + StreamedOutputHandler.OutputType.BINARY, actionFilters), new ActionGraphProtoV2OutputFormatterCallback( eventHandler, @@ -175,7 +176,7 @@ out, skyframeExecutor, accessor, - ActionGraphProtoV2OutputFormatterCallback.OutputType.TEXT, + StreamedOutputHandler.OutputType.TEXT, actionFilters), new ActionGraphTextOutputFormatterCallback( eventHandler, aqueryOptions, out, skyframeExecutor, accessor, actionFilters))
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/ActionGraphDump.java b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/ActionGraphDump.java index 2b4b1e0..eaebe6f 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/ActionGraphDump.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/ActionGraphDump.java
@@ -27,7 +27,7 @@ import com.google.devtools.build.lib.actions.CommandLineExpansionException; import com.google.devtools.build.lib.actions.ExecutionInfoSpecifier; import com.google.devtools.build.lib.analysis.AnalysisProtosV2; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphContainer; +import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphComponent; import com.google.devtools.build.lib.analysis.ConfiguredTarget; import com.google.devtools.build.lib.analysis.actions.ParameterFileWriteAction; import com.google.devtools.build.lib.analysis.actions.SpawnAction; @@ -43,6 +43,7 @@ import com.google.devtools.build.lib.rules.AliasConfiguredTarget; import com.google.devtools.build.lib.skyframe.AspectValue; import com.google.devtools.build.lib.skyframe.ConfiguredTargetValue; +import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -54,7 +55,6 @@ */ public class ActionGraphDump { - private final ActionGraphContainer.Builder actionGraphBuilder = ActionGraphContainer.newBuilder(); private final ActionKeyContext actionKeyContext = new ActionKeyContext(); private final Set<String> actionGraphTargets; @@ -68,6 +68,7 @@ private final boolean includeActionCmdLine; private final boolean includeArtifacts; private final boolean includeParamFiles; + private final StreamedOutputHandler streamedOutputHandler; private Map<String, Iterable<String>> paramFileNameToContentMap; @@ -75,23 +76,15 @@ boolean includeActionCmdLine, boolean includeArtifacts, AqueryActionFilter actionFilters, - boolean includeParamFiles) { + boolean includeParamFiles, + StreamedOutputHandler streamedOutputHandler) { this( /* actionGraphTargets= */ ImmutableList.of("..."), includeActionCmdLine, includeArtifacts, actionFilters, - includeParamFiles); - } - - public ActionGraphDump( - List<String> actionGraphTargets, boolean includeActionCmdLine, boolean includeArtifacts) { - this( - actionGraphTargets, - includeActionCmdLine, - includeArtifacts, - /* actionFilters= */ AqueryActionFilter.emptyInstance(), - /* includeParamFiles */ false); + includeParamFiles, + streamedOutputHandler); } public ActionGraphDump( @@ -99,20 +92,22 @@ boolean includeActionCmdLine, boolean includeArtifacts, AqueryActionFilter actionFilters, - boolean includeParamFiles) { + boolean includeParamFiles, + StreamedOutputHandler streamedOutputHandler) { this.actionGraphTargets = ImmutableSet.copyOf(actionGraphTargets); this.includeActionCmdLine = includeActionCmdLine; this.includeArtifacts = includeArtifacts; this.actionFilters = actionFilters; this.includeParamFiles = includeParamFiles; + this.streamedOutputHandler = streamedOutputHandler; - knownRuleClassStrings = new KnownRuleClassStrings(actionGraphBuilder); - knownArtifacts = new KnownArtifacts(actionGraphBuilder); - knownConfigurations = new KnownConfigurations(actionGraphBuilder); - knownNestedSets = new KnownNestedSets(actionGraphBuilder, knownArtifacts); - knownAspectDescriptors = new KnownAspectDescriptors(actionGraphBuilder); - knownRuleConfiguredTargets = new KnownRuleConfiguredTargets(actionGraphBuilder, - knownRuleClassStrings); + knownRuleClassStrings = new KnownRuleClassStrings(streamedOutputHandler); + knownArtifacts = new KnownArtifacts(streamedOutputHandler); + knownConfigurations = new KnownConfigurations(streamedOutputHandler); + knownNestedSets = new KnownNestedSets(streamedOutputHandler, knownArtifacts); + knownAspectDescriptors = new KnownAspectDescriptors(streamedOutputHandler); + knownRuleConfiguredTargets = + new KnownRuleConfiguredTargets(streamedOutputHandler, knownRuleClassStrings); } public ActionKeyContext getActionKeyContext() { @@ -128,7 +123,7 @@ } private void dumpSingleAction(ConfiguredTarget configuredTarget, ActionAnalysisMetadata action) - throws CommandLineExpansionException { + throws CommandLineExpansionException, IOException { // Store the content of param files. if (includeParamFiles && (action instanceof ParameterFileWriteAction)) { @@ -152,7 +147,8 @@ AnalysisProtosV2.Action.Builder actionBuilder = AnalysisProtosV2.Action.newBuilder() .setMnemonic(action.getMnemonic()) - .setTargetId(knownRuleConfiguredTargets.dataToId(ruleConfiguredTarget)); + .setTargetId( + knownRuleConfiguredTargets.dataToIdAndStreamOutputProto(ruleConfiguredTarget)); if (action instanceof ActionExecutionMetadata) { ActionExecutionMetadata actionExecutionMetadata = (ActionExecutionMetadata) action; @@ -212,7 +208,7 @@ ActionOwner actionOwner = action.getOwner(); if (actionOwner != null) { BuildEvent event = actionOwner.getConfiguration(); - actionBuilder.setConfigurationId(knownConfigurations.dataToId(event)); + actionBuilder.setConfigurationId(knownConfigurations.dataToIdAndStreamOutputProto(event)); // Store aspects. // Iterate through the aspect path and dump the aspect descriptors. @@ -220,7 +216,8 @@ // of the configured target graph. // e.g. [A, B] would imply that aspect A is applied on top of aspect B. for (AspectDescriptor aspectDescriptor : actionOwner.getAspectDescriptors().reverse()) { - actionBuilder.addAspectDescriptorIds(knownAspectDescriptors.dataToId(aspectDescriptor)); + actionBuilder.addAspectDescriptorIds( + knownAspectDescriptors.dataToIdAndStreamOutputProto(aspectDescriptor)); } } @@ -233,20 +230,26 @@ NestedSetView<Artifact> nestedSetView = new NestedSetView<>((NestedSet<Artifact>) inputs); if (nestedSetView.directs().size() > 0 || nestedSetView.transitives().size() > 0) { - actionBuilder.addInputDepSetIds(knownNestedSets.dataToId(nestedSetView)); + actionBuilder.addInputDepSetIds( + knownNestedSets.dataToIdAndStreamOutputProto(nestedSetView)); } // store outputs for (Artifact artifact : action.getOutputs()) { - actionBuilder.addOutputIds(knownArtifacts.dataToId(artifact)); + actionBuilder.addOutputIds(knownArtifacts.dataToIdAndStreamOutputProto(artifact)); } } - actionGraphBuilder.addActions(actionBuilder.build()); + printToOutput(actionBuilder.build()); + } + + private void printToOutput(AnalysisProtosV2.Action actionProto) throws IOException { + ActionGraphComponent message = ActionGraphComponent.newBuilder().setAction(actionProto).build(); + streamedOutputHandler.printActionGraphComponent(message); } public void dumpAspect(AspectValue aspectValue, ConfiguredTargetValue configuredTargetValue) - throws CommandLineExpansionException { + throws CommandLineExpansionException, IOException { ConfiguredTarget configuredTarget = configuredTargetValue.getConfiguredTarget(); if (!includeInActionGraph(configuredTarget.getLabel().toString())) { return; @@ -258,7 +261,7 @@ } public void dumpConfiguredTarget(ConfiguredTargetValue configuredTargetValue) - throws CommandLineExpansionException { + throws CommandLineExpansionException, IOException { ConfiguredTarget configuredTarget = configuredTargetValue.getConfiguredTarget(); if (!includeInActionGraph(configuredTarget.getLabel().toString())) { return; @@ -269,10 +272,6 @@ } } - public ActionGraphContainer build() { - return actionGraphBuilder.build(); - } - /** Lazy initialization of paramFileNameToContentMap. */ private Map<String, Iterable<String>> getParamFileNameToContentMap() { if (paramFileNameToContentMap == null) {
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/BaseCache.java b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/BaseCache.java index 4c40e50..2b4ebb3 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/BaseCache.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/BaseCache.java
@@ -13,7 +13,7 @@ // limitations under the License. package com.google.devtools.build.lib.skyframe.actiongraph.v2; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphContainer; +import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -22,10 +22,10 @@ */ abstract class BaseCache<K, P> { private final Map<K, Integer> cache = new HashMap<>(); - protected final ActionGraphContainer.Builder actionGraphBuilder; + protected final StreamedOutputHandler streamedOutputHandler; - BaseCache(ActionGraphContainer.Builder actionGraphBuilder) { - this.actionGraphBuilder = actionGraphBuilder; + BaseCache(StreamedOutputHandler streamedOutputHandler) { + this.streamedOutputHandler = streamedOutputHandler; } private int generateNextId() { @@ -42,8 +42,10 @@ /** * Store the data in the internal cache, if it's not yet present. Return the generated id. Ids are * positive and unique. + * + * <p>Stream the proto to output, the first time it's generated. */ - int dataToId(K data) { + int dataToIdAndStreamOutputProto(K data) throws IOException { K key = transformToKey(data); Integer id = cache.get(key); if (id == null) { @@ -53,12 +55,12 @@ id = generateNextId(); cache.put(key, id); P proto = createProto(data, id); - addToActionGraphBuilder(proto); + streamToOutput(proto); } return id; } - abstract P createProto(K key, int id); + abstract P createProto(K key, int id) throws IOException; - abstract void addToActionGraphBuilder(P proto); + abstract void streamToOutput(P proto) throws IOException; }
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownArtifacts.java b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownArtifacts.java index 4e46410..2d5f1db 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownArtifacts.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownArtifacts.java
@@ -15,32 +15,34 @@ import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.analysis.AnalysisProtosV2; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphContainer; +import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphComponent; +import java.io.IOException; /** Cache for Artifacts in the action graph. */ public class KnownArtifacts extends BaseCache<Artifact, AnalysisProtosV2.Artifact> { private final KnownPathFragments knownPathFragments; - KnownArtifacts(ActionGraphContainer.Builder actionGraphBuilder) { - super(actionGraphBuilder); - knownPathFragments = new KnownPathFragments(actionGraphBuilder); + KnownArtifacts(StreamedOutputHandler streamedOutputHandler) { + super(streamedOutputHandler); + knownPathFragments = new KnownPathFragments(streamedOutputHandler); } @Override - AnalysisProtosV2.Artifact createProto(Artifact artifact, int id) { + AnalysisProtosV2.Artifact createProto(Artifact artifact, int id) throws IOException { AnalysisProtosV2.Artifact.Builder artifactProtoBuilder = AnalysisProtosV2.Artifact.newBuilder() .setId(id) .setIsTreeArtifact(artifact.isTreeArtifact()); - int pathFragmentId = knownPathFragments.dataToId(artifact.getExecPath()); + int pathFragmentId = knownPathFragments.dataToIdAndStreamOutputProto(artifact.getExecPath()); return artifactProtoBuilder.setPathFragmentId(pathFragmentId).build(); } - @Override - void addToActionGraphBuilder(AnalysisProtosV2.Artifact artifactProto) { - actionGraphBuilder.addArtifacts(artifactProto); + void streamToOutput(AnalysisProtosV2.Artifact artifactProto) throws IOException { + ActionGraphComponent message = + ActionGraphComponent.newBuilder().setArtifact(artifactProto).build(); + streamedOutputHandler.printActionGraphComponent(message); } }
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownAspectDescriptors.java b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownAspectDescriptors.java index ee4b642..1a31729 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownAspectDescriptors.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownAspectDescriptors.java
@@ -14,21 +14,23 @@ package com.google.devtools.build.lib.skyframe.actiongraph.v2; import com.google.devtools.build.lib.analysis.AnalysisProtosV2; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphContainer; +import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphComponent; import com.google.devtools.build.lib.analysis.AnalysisProtosV2.KeyValuePair; import com.google.devtools.build.lib.packages.AspectDescriptor; +import java.io.IOException; import java.util.Map; /** Cache for AspectDescriptors in the action graph. */ public class KnownAspectDescriptors extends BaseCache<AspectDescriptor, AnalysisProtosV2.AspectDescriptor> { - KnownAspectDescriptors(ActionGraphContainer.Builder actionGraphBuilder) { - super(actionGraphBuilder); + KnownAspectDescriptors(StreamedOutputHandler streamedOutputHandler) { + super(streamedOutputHandler); } @Override - AnalysisProtosV2.AspectDescriptor createProto(AspectDescriptor aspectDescriptor, int id) { + AnalysisProtosV2.AspectDescriptor createProto(AspectDescriptor aspectDescriptor, int id) + throws IOException { AnalysisProtosV2.AspectDescriptor.Builder aspectDescriptorBuilder = AnalysisProtosV2.AspectDescriptor.newBuilder() .setId(id) @@ -43,7 +45,9 @@ } @Override - void addToActionGraphBuilder(AnalysisProtosV2.AspectDescriptor aspectDescriptorProto) { - actionGraphBuilder.addAspectDescriptors(aspectDescriptorProto); + void streamToOutput(AnalysisProtosV2.AspectDescriptor aspectDescriptorProto) throws IOException { + ActionGraphComponent message = + ActionGraphComponent.newBuilder().setAspectDescriptor(aspectDescriptorProto).build(); + streamedOutputHandler.printActionGraphComponent(message); } }
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownConfigurations.java b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownConfigurations.java index 15dab60..8fff2c7 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownConfigurations.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownConfigurations.java
@@ -13,20 +13,21 @@ // limitations under the License. package com.google.devtools.build.lib.skyframe.actiongraph.v2; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphContainer; +import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphComponent; import com.google.devtools.build.lib.analysis.AnalysisProtosV2.Configuration; import com.google.devtools.build.lib.buildeventstream.BuildEvent; import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos; +import java.io.IOException; /** Cache for BuildConfigurations in the action graph. */ public class KnownConfigurations extends BaseCache<BuildEvent, Configuration> { - KnownConfigurations(ActionGraphContainer.Builder actionGraphBuilder) { - super(actionGraphBuilder); + KnownConfigurations(StreamedOutputHandler streamedOutputHandler) { + super(streamedOutputHandler); } @Override - Configuration createProto(BuildEvent config, int id) { + Configuration createProto(BuildEvent config, int id) throws IOException { BuildEventStreamProtos.Configuration configProto = config.asStreamProto(/*context=*/ null).getConfiguration(); return Configuration.newBuilder() @@ -38,7 +39,9 @@ } @Override - void addToActionGraphBuilder(Configuration configurationProto) { - actionGraphBuilder.addConfiguration(configurationProto); + void streamToOutput(Configuration configurationProto) throws IOException { + ActionGraphComponent message = + ActionGraphComponent.newBuilder().setConfiguration(configurationProto).build(); + streamedOutputHandler.printActionGraphComponent(message); } }
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownNestedSets.java b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownNestedSets.java index 222613c..8594a62 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownNestedSets.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownNestedSets.java
@@ -14,16 +14,17 @@ package com.google.devtools.build.lib.skyframe.actiongraph.v2; import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphContainer; +import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphComponent; import com.google.devtools.build.lib.analysis.AnalysisProtosV2.DepSetOfFiles; import com.google.devtools.build.lib.collect.nestedset.NestedSetView; +import java.io.IOException; /** Cache for NestedSets in the action graph. */ public class KnownNestedSets extends BaseCache<Object, DepSetOfFiles> { private final KnownArtifacts knownArtifacts; - KnownNestedSets(ActionGraphContainer.Builder actionGraphBuilder, KnownArtifacts knownArtifacts) { - super(actionGraphBuilder); + KnownNestedSets(StreamedOutputHandler streamedOutputHandler, KnownArtifacts knownArtifacts) { + super(streamedOutputHandler); this.knownArtifacts = knownArtifacts; } @@ -36,20 +37,23 @@ } @Override - DepSetOfFiles createProto(Object nestedSetViewObject, int id) { + DepSetOfFiles createProto(Object nestedSetViewObject, int id) throws IOException { NestedSetView<?> nestedSetView = (NestedSetView) nestedSetViewObject; DepSetOfFiles.Builder depSetBuilder = DepSetOfFiles.newBuilder().setId(id); for (NestedSetView<?> transitiveNestedSet : nestedSetView.transitives()) { - depSetBuilder.addTransitiveDepSetIds(this.dataToId(transitiveNestedSet)); + depSetBuilder.addTransitiveDepSetIds(this.dataToIdAndStreamOutputProto(transitiveNestedSet)); } for (Object directArtifact : nestedSetView.directs()) { - depSetBuilder.addDirectArtifactIds(knownArtifacts.dataToId((Artifact) directArtifact)); + depSetBuilder.addDirectArtifactIds( + knownArtifacts.dataToIdAndStreamOutputProto((Artifact) directArtifact)); } return depSetBuilder.build(); } @Override - void addToActionGraphBuilder(DepSetOfFiles depSetOfFilesProto) { - actionGraphBuilder.addDepSetOfFiles(depSetOfFilesProto); + void streamToOutput(DepSetOfFiles depSetOfFilesProto) throws IOException { + ActionGraphComponent message = + ActionGraphComponent.newBuilder().setDepSetOfFiles(depSetOfFilesProto).build(); + streamedOutputHandler.printActionGraphComponent(message); } }
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownPathFragments.java b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownPathFragments.java index 3b54dc4..034e311 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownPathFragments.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownPathFragments.java
@@ -14,17 +14,18 @@ package com.google.devtools.build.lib.skyframe.actiongraph.v2; import com.google.devtools.build.lib.analysis.AnalysisProtosV2; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphContainer; +import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphComponent; import com.google.devtools.build.lib.vfs.PathFragment; +import java.io.IOException; /** Cache for {@link PathFragment} in the action graph. */ public class KnownPathFragments extends BaseCache<PathFragment, AnalysisProtosV2.PathFragment> { - KnownPathFragments(ActionGraphContainer.Builder actionGraphBuilder) { - super(actionGraphBuilder); + KnownPathFragments(StreamedOutputHandler streamedOutputHandler) { + super(streamedOutputHandler); } @Override - AnalysisProtosV2.PathFragment createProto(PathFragment pathFragment, int id) { + AnalysisProtosV2.PathFragment createProto(PathFragment pathFragment, int id) throws IOException { AnalysisProtosV2.PathFragment.Builder pathFragmentProtoBuilder = AnalysisProtosV2.PathFragment.newBuilder().setId(id).setLabel(pathFragment.getBaseName()); @@ -32,15 +33,18 @@ // If pathFragment has no parent, leave parentId blank and avoid calling dataToId // to prevent the cache from being polluted with a null entry. if (hasParent(pathFragment)) { - pathFragmentProtoBuilder.setParentId(dataToId(pathFragment.getParentDirectory())); + pathFragmentProtoBuilder.setParentId( + dataToIdAndStreamOutputProto(pathFragment.getParentDirectory())); } return pathFragmentProtoBuilder.build(); } @Override - void addToActionGraphBuilder(AnalysisProtosV2.PathFragment pathFragmentProto) { - actionGraphBuilder.addPathFragments(pathFragmentProto); + void streamToOutput(AnalysisProtosV2.PathFragment pathFragmentProto) throws IOException { + ActionGraphComponent message = + ActionGraphComponent.newBuilder().setPathFragment(pathFragmentProto).build(); + streamedOutputHandler.printActionGraphComponent(message); } private static boolean hasParent(PathFragment pathFragment) {
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownRuleClassStrings.java b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownRuleClassStrings.java index f26a698..a49ae0a 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownRuleClassStrings.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownRuleClassStrings.java
@@ -13,23 +13,26 @@ // limitations under the License. package com.google.devtools.build.lib.skyframe.actiongraph.v2; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphContainer; +import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphComponent; import com.google.devtools.build.lib.analysis.AnalysisProtosV2.RuleClass; +import java.io.IOException; /** Cache for RuleClassStrings in the action graph. */ public class KnownRuleClassStrings extends BaseCache<String, RuleClass> { - KnownRuleClassStrings(ActionGraphContainer.Builder actionGraphBuilder) { - super(actionGraphBuilder); + KnownRuleClassStrings(StreamedOutputHandler streamedOutputHandler) { + super(streamedOutputHandler); } @Override - RuleClass createProto(String ruleClassString, int id) { + RuleClass createProto(String ruleClassString, int id) throws IOException { return RuleClass.newBuilder().setId(id).setName(ruleClassString).build(); } @Override - void addToActionGraphBuilder(RuleClass ruleClassProto) { - actionGraphBuilder.addRuleClasses(ruleClassProto); + void streamToOutput(RuleClass ruleClassProto) throws IOException { + ActionGraphComponent message = + ActionGraphComponent.newBuilder().setRuleClass(ruleClassProto).build(); + streamedOutputHandler.printActionGraphComponent(message); } }
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownRuleConfiguredTargets.java b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownRuleConfiguredTargets.java index 094ef3c..fe27139 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownRuleConfiguredTargets.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownRuleConfiguredTargets.java
@@ -13,10 +13,11 @@ // limitations under the License. package com.google.devtools.build.lib.skyframe.actiongraph.v2; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphContainer; +import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphComponent; import com.google.devtools.build.lib.analysis.AnalysisProtosV2.Target; import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget; import com.google.devtools.build.lib.cmdline.Label; +import java.io.IOException; /** Cache for RuleConfiguredTargets in the action graph. */ public class KnownRuleConfiguredTargets extends BaseCache<RuleConfiguredTarget, Target> { @@ -24,25 +25,27 @@ private final KnownRuleClassStrings knownRuleClassStrings; KnownRuleConfiguredTargets( - ActionGraphContainer.Builder actionGraphBuilder, - KnownRuleClassStrings knownRuleClassStrings) { - super(actionGraphBuilder); + StreamedOutputHandler streamedOutputHandler, KnownRuleClassStrings knownRuleClassStrings) { + super(streamedOutputHandler); this.knownRuleClassStrings = knownRuleClassStrings; } @Override - Target createProto(RuleConfiguredTarget ruleConfiguredTarget, int id) { + Target createProto(RuleConfiguredTarget ruleConfiguredTarget, int id) throws IOException { Label label = ruleConfiguredTarget.getLabel(); String ruleClassString = ruleConfiguredTarget.getRuleClassString(); Target.Builder targetBuilder = Target.newBuilder().setId(id).setLabel(label.toString()); if (ruleClassString != null) { - targetBuilder.setRuleClassId(knownRuleClassStrings.dataToId(ruleClassString)); + targetBuilder.setRuleClassId( + knownRuleClassStrings.dataToIdAndStreamOutputProto(ruleClassString)); } return targetBuilder.build(); } @Override - void addToActionGraphBuilder(Target targetProto) { - actionGraphBuilder.addTargets(targetProto); + void streamToOutput(Target targetProto) throws IOException { + ActionGraphComponent message = + ActionGraphComponent.newBuilder().setConfiguredTarget(targetProto).build(); + streamedOutputHandler.printActionGraphComponent(message); } }
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/StreamedOutputHandler.java b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/StreamedOutputHandler.java new file mode 100644 index 0000000..b857b6e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/StreamedOutputHandler.java
@@ -0,0 +1,84 @@ +// Copyright 2019 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe.actiongraph.v2; + +import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphComponent; +import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphContainer; +import com.google.protobuf.CodedOutputStream; +import java.io.IOException; +import java.io.PrintStream; + +/** Manages the various streamed output channels of aquery. */ +public class StreamedOutputHandler { + /** Defines the types of proto output this class can handle. */ + public enum OutputType { + BINARY("proto"), + TEXT("textproto"); + + private final String formatName; + + OutputType(String formatName) { + this.formatName = formatName; + } + + public String formatName() { + return formatName; + } + } + + private final OutputType outputType; + private final CodedOutputStream outputStream; + private final PrintStream printStream; + + public StreamedOutputHandler( + OutputType outputType, CodedOutputStream outputStream, PrintStream printStream) { + this.outputType = outputType; + this.outputStream = outputStream; + this.printStream = printStream; + } + + /** + * Prints the ActionGraphComponent to the appropriate output channel. + * + * @param message The message to be printed. + */ + public void printActionGraphComponent(ActionGraphComponent message) throws IOException { + switch (outputType) { + case BINARY: + outputStream.writeMessage( + ActionGraphContainer.ACTION_GRAPH_COMPONENTS_FIELD_NUMBER, message); + break; + case TEXT: + printStream.print(wrapperActionGraphComponent(message)); + break; + } + } + + private static String wrapperActionGraphComponent(ActionGraphComponent message) { + return "action_graph_components {\n" + message + "}\n"; + } + + /** Called at the end of the query process. */ + public void close() throws IOException { + switch (outputType) { + case BINARY: + outputStream.flush(); + break; + case TEXT: + printStream.flush(); + printStream.close(); + break; + } + } +}
diff --git a/src/main/protobuf/analysis_v2.proto b/src/main/protobuf/analysis_v2.proto index edc4f1f..305ce74 100644 --- a/src/main/protobuf/analysis_v2.proto +++ b/src/main/protobuf/analysis_v2.proto
@@ -23,14 +23,22 @@ // Container for the action graph properties. message ActionGraphContainer { - repeated Artifact artifacts = 1; - repeated Action actions = 2; - repeated Target targets = 3; - repeated DepSetOfFiles dep_set_of_files = 4; - repeated Configuration configuration = 5; - repeated AspectDescriptor aspect_descriptors = 6; - repeated RuleClass rule_classes = 7; - repeated PathFragment path_fragments = 8; + repeated ActionGraphComponent action_graph_components = 1; +} + +// One single component of the final action graph, classified by Discriminator. +message ActionGraphComponent { + oneof component { + Artifact artifact = 1; + Action action = 2; + Target target = 3; + DepSetOfFiles dep_set_of_files = 4; + Configuration configuration = 5; + AspectDescriptor aspect_descriptor = 6; + RuleClass rule_class = 7; + PathFragment path_fragment = 8; + Target configured_target = 9; + } } // Represents a single artifact, whether it's a source file or a derived output
diff --git a/src/test/shell/integration/aquery_test.sh b/src/test/shell/integration/aquery_test.sh index ba649b8..30425c1 100755 --- a/src/test/shell/integration/aquery_test.sh +++ b/src/test/shell/integration/aquery_test.sh
@@ -1096,10 +1096,18 @@ bazel aquery --incompatible_proto_output_v2 --output=textproto "//$pkg:bar" > output 2> "$TEST_log" \ || fail "Expected success" cat output >> "$TEST_log" + # Verify than ids come in integers instead of strings. assert_contains "id: 1" output assert_not_contains "id: \"1\"" output - assert_contains "path_fragments {" output + + # Verify that it consists of action_graph_components. + assert_contains "action_graph_components {" output + + # Verify that paths are broken down to path fragments. + assert_contains "path_fragment {" output + + # Verify that the appropriate action was included. assert_contains "label: \"dummy.txt\"" output assert_contains "mnemonic: \"Genrule\"" output assert_contains "mnemonic: \".*-fastbuild\"" output