Add rule class definitions to query proto output
As rules are starlarkified and migrated out of Bazel, tools cannot rely any
longer on the (undocumented and deprecated) `info build-language` output to find
the definitions of rule classes. Moreover, the name of the rule class is no
longer unique: two different .bzl files might define different rule classes with
the same name - and both might be used in the same package.
For starlarkified rules, tools could use `starlark_doc_extract` targets to get
the rule class definition in proto form. However, this requires
(1) knowing in advance which .bzl file to introspect
(2) adding a `starlark_doc_extract` target to perform this introspection.
This is obviously not ideal when a tool needs to introspect targets without
changing the state of a repo.
The solution is to add `starlark_doc_extract`-like rule class definitions to
`query` command output. Since a RuleInfo proto (the rule class definition as
output by starlark_doc_extract and Stardoc) is large and expensive to generate,
we want to
* only output it optionally, when --proto:rule_classes flag is set
* only output each rule class definition once per output stream - in the first
rule target having a given rule class key.
* add a rule class key, unique per rule class definition (not just rule class
name!), so that later rule targets in the stream with the same rule class
definition can be connected with the rule class definition provided in the first
target.
RELNOTES: If --proto:rule_classes flag is enabled, query proto output will contain rule class definitions in Stardoc proto format.
PiperOrigin-RevId: 680642590
Change-Id: Ibdeb9c8acb7368b510f62d945c361a86cdb6f447
diff --git a/src/main/java/com/google/devtools/build/lib/query2/common/CommonQueryOptions.java b/src/main/java/com/google/devtools/build/lib/query2/common/CommonQueryOptions.java
index a6c6077..b653e2b 100644
--- a/src/main/java/com/google/devtools/build/lib/query2/common/CommonQueryOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/query2/common/CommonQueryOptions.java
@@ -306,6 +306,18 @@
+ "that the attribute came from (empty string if it did not).")
public boolean protoIncludeAttributeSourceAspects;
+ @Option(
+ name = "proto:rule_classes",
+ defaultValue = "false",
+ documentationCategory = OptionDocumentationCategory.QUERY,
+ effectTags = {OptionEffectTag.TERMINAL_OUTPUT},
+ help =
+ "Populate the rule_class_key field of each rule; and for the first rule with a given"
+ + " rule_class_key, also populate its rule_class_info proto field. The rule_class_key"
+ + " field uniquely identifies a rule class, and the rule_class_info field is a"
+ + " Stardoc-format rule class API definition.")
+ public boolean protoRuleClasses;
+
/** An enum converter for {@code AspectResolver.Mode} . Should be used internally only. */
public static class AspectResolutionModeConverter extends EnumConverter<Mode> {
public AspectResolutionModeConverter() {
diff --git a/src/main/java/com/google/devtools/build/lib/query2/query/output/BUILD b/src/main/java/com/google/devtools/build/lib/query2/query/output/BUILD
index 38f9b65..55c6fbb 100644
--- a/src/main/java/com/google/devtools/build/lib/query2/query/output/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/query2/query/output/BUILD
@@ -28,6 +28,8 @@
"//src/main/java/com/google/devtools/build/lib/query2/compat:fake-load-target",
"//src/main/java/com/google/devtools/build/lib/query2/engine",
"//src/main/java/com/google/devtools/build/lib/query2/query/aspectresolvers",
+ "//src/main/java/com/google/devtools/build/lib/starlarkdocextract:labelrenderer",
+ "//src/main/java/com/google/devtools/build/lib/starlarkdocextract:ruleinfoextractor",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
"//src/main/java/com/google/devtools/common/options",
diff --git a/src/main/java/com/google/devtools/build/lib/query2/query/output/ProtoOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/query2/query/output/ProtoOutputFormatter.java
index cf2a044..66ebbdf 100644
--- a/src/main/java/com/google/devtools/build/lib/query2/query/output/ProtoOutputFormatter.java
+++ b/src/main/java/com/google/devtools/build/lib/query2/query/output/ProtoOutputFormatter.java
@@ -61,6 +61,9 @@
import com.google.devtools.build.lib.query2.proto.proto2api.Build.QueryResult;
import com.google.devtools.build.lib.query2.proto.proto2api.Build.SourceFile;
import com.google.devtools.build.lib.query2.query.aspectresolvers.AspectResolver;
+import com.google.devtools.build.lib.starlarkdocextract.ExtractorContext;
+import com.google.devtools.build.lib.starlarkdocextract.LabelRenderer;
+import com.google.devtools.build.lib.starlarkdocextract.RuleInfoExtractor;
import com.google.protobuf.CodedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
@@ -68,6 +71,7 @@
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -114,6 +118,9 @@
protected boolean includeAttributeSourceAspects = false;
private HashFunction hashFunction = null;
+ /** Non-null if and only if --proto:rule_classes option is set. */
+ @Nullable private RuleClassInfoFormatter ruleClassInfoFormatter;
+
@Nullable private EventHandler eventHandler;
@Override
@@ -139,6 +146,7 @@
this.includeDefinitionStack = options.protoIncludeDefinitionStack;
this.includeAttributeSourceAspects = options.protoIncludeAttributeSourceAspects;
this.hashFunction = hashFunction;
+ this.ruleClassInfoFormatter = options.protoRuleClasses ? new RuleClassInfoFormatter() : null;
}
@Override
@@ -272,6 +280,10 @@
FormatUtils.getRootRelativeLocation(fr.location, rule.getPackage()) + ": " + fr.name);
}
}
+
+ if (ruleClassInfoFormatter != null) {
+ ruleClassInfoFormatter.addRuleClassKeyAndInfoIfNeeded(rulePb, rule);
+ }
targetPb.setType(RULE);
targetPb.setRule(rulePb);
} else if (target instanceof OutputFile outputFile) {
@@ -541,6 +553,33 @@
throw new AssertionError("Unknown type: " + attrType);
}
+ private static class RuleClassInfoFormatter {
+ private final HashSet<String> ruleClassKeys = new HashSet<>();
+ private final ExtractorContext extractorContext =
+ ExtractorContext.builder()
+ .labelRenderer(LabelRenderer.DEFAULT)
+ .extractNonStarlarkAttrs(true)
+ .build();
+
+ /**
+ * Sets the rule_class_key field, and if the rule class key has not been seen before, also sets
+ * the rule_class_info field.
+ */
+ public void addRuleClassKeyAndInfoIfNeeded(Build.Rule.Builder rulePb, Rule rule) {
+ String ruleClassKey = rule.getRuleClassObject().getKey();
+ rulePb.setRuleClassKey(ruleClassKey);
+ if (ruleClassKeys.add(ruleClassKey)) {
+ // TODO(b/368091415): instead of rule.getRuleClass(), we should be using the rule's public
+ // name. But to find the public name, we would need access to the globals dictionary of
+ // the compiled Starlark module in which the rule class was defined, which would have to
+ // be retrieved from skyframe.
+ rulePb.setRuleClassInfo(
+ RuleInfoExtractor.buildRuleInfo(
+ extractorContext, rule.getRuleClass(), rule.getRuleClassObject()));
+ }
+ }
+ }
+
/**
* Specialized {@link OutputFormatterCallback} implementation which produces a valid {@link
* QueryResult} in streaming fashion. Internally this class makes some reasonably sound and stable
diff --git a/src/main/protobuf/BUILD b/src/main/protobuf/BUILD
index 82e40cb..002a412 100644
--- a/src/main/protobuf/BUILD
+++ b/src/main/protobuf/BUILD
@@ -10,7 +10,6 @@
"action_cache",
"android_deploy_info",
"bazel_flags",
- "build",
"builtin",
"crash_debugging",
"crosstool_config",
@@ -42,6 +41,22 @@
) for s in FILES]
proto_library(
+ name = "build_proto",
+ srcs = ["build.proto"],
+ deps = [":stardoc_output_proto"],
+)
+
+java_proto_library(
+ name = "build_java_proto",
+ deps = [":build_proto"],
+)
+
+java_library_srcs(
+ name = "build_java_proto_srcs",
+ deps = [":build_java_proto"],
+)
+
+proto_library(
name = "analysis_v2_proto",
srcs = ["analysis_v2.proto"],
deps = [":build_proto"],
@@ -411,6 +426,7 @@
":bazel_output_service_java_grpc_srcs",
":bazel_output_service_java_proto_srcs",
":bazel_output_service_rev2_java_proto_srcs",
+ ":build_java_proto_srcs",
":cache_salt_java_proto_srcs",
":command_line_java_proto_srcs",
":command_server_java_grpc_srcs",
diff --git a/src/main/protobuf/build.proto b/src/main/protobuf/build.proto
index c4e94e9..737375d 100644
--- a/src/main/protobuf/build.proto
+++ b/src/main/protobuf/build.proto
@@ -19,6 +19,8 @@
package blaze_query;
+import "src/main/protobuf/stardoc_output.proto";
+
// option cc_api_version = 2;
// option java_api_version = 1;
option java_package = "com.google.devtools.build.lib.query2.proto.proto2api";
@@ -286,7 +288,11 @@
// The name of the rule (formatted as an absolute label, e.g. //foo/bar:baz).
required string name = 1;
- // The rule class (e.g., java_library)
+ // The rule class name (e.g., java_library).
+ //
+ // Note that the rule class name may not uniquely identify a rule class, since
+ // two different .bzl files may define different rule classes with the same
+ // name. To uniquely identify the rule class, see rule_class_key field below.
required string rule_class = 2;
// The BUILD file and line number of the location (formatted as
@@ -336,6 +342,23 @@
// enabled on the command line with the --proto:definition_stack flag or the
// rule is a native one.
repeated string definition_stack = 14;
+
+ // A key uniquely identifying the rule's rule class. Stable between repeated
+ // blaze query invocations (assuming that there are no changes to Starlark
+ // files and the same blaze binary is invoked with the same options).
+ //
+ // Requires --proto:rule_classes=true
+ optional string rule_class_key = 16;
+
+ // Stardoc-format rule class API definition for this rule. Includes both
+ // Starlark-defined and native (including inherited) attributes; does not
+ // include hidden or explicitly undocumented attributes.
+ //
+ // Populated only for the first rule in the stream with a given
+ // rule_class_key.
+ //
+ // Requires --proto:rule_classes=true
+ optional stardoc_output.RuleInfo rule_class_info = 17;
}
// Direct dependencies of a rule in <depLabel, depConfiguration> form.
diff --git a/src/main/protobuf/stardoc_output.proto b/src/main/protobuf/stardoc_output.proto
index 4b668c9..1827a2a 100644
--- a/src/main/protobuf/stardoc_output.proto
+++ b/src/main/protobuf/stardoc_output.proto
@@ -75,8 +75,12 @@
// Representation of a Starlark rule definition.
message RuleInfo {
- // The name under which the rule is made accessible to a user of this module,
- // including any structs it is nested in, for example "foo.foo_library".
+ // In Stardoc and starlark_doc_extract output, this is the name under which
+ // the rule is made accessible to a user of this module, including any structs
+ // it is nested in, for example "foo.foo_library".
+ //
+ // In query output, this is the name under which the rule was defined (which
+ // might be a private symbol prefixed with "_").
string rule_name = 1;
// The documentation string of the rule.
