Bazel DocGen (1/3): Make link expansion mechanism more powerful.
# Motivation
Linking from one part of the Blaze/Bazel (“Blazel”) documentation to another one is surprisingly difficult due to various distinctions (native code vs Starlark, Blaze vs Bazel, generated vs narrative documentation). As a result, we’ve implemented various hacks or simply tolerated broken links.
# Contributions
This CL addresses the problems by doubling down on dynamic link expansion:
- RuleLinkExpander can now be used to link to any file in the build encyclopedia (“BE”): While the `${link foo}` syntax was originally intended for linking to rule families, a previous version already contained a hack to link to some static documentation files. This CL super-charges RuleLinkExpander by allowing it to link to *any* BE file, based on an explicit link mapping that has to be defined in a JSON configuration file. As a result, it’s clearly visible which files diverge between Blaze and Bazel.
- Introduction of StarlarkDocExpander: This new class is a proxy for the improved RuleLinkExpander as well as StarlarkDocUtils.substituteVariables(). As a result, Starlark documentation can now easily link to pages in the BE.
# Why JSON?
JSON is a lightweight configuration format, and luckily both Bazel and Blaze already have a JSON parser (GSON) among their dependencies.
# Downsides
The Starlark DocGen code is a love letter to the `static` modifier, which unfortunately means that we have to pass the StarlarkDocExpander instance through many methods.
# Alternatives
We could have passed the StarlarkDocExpander instance to the Velocity Template engine, which would have allowed us to call the expand() method from within the templates. However, I think it’s easier to figure out what happens if the real logic happens inside Java.
PiperOrigin-RevId: 433039776
diff --git a/src/main/java/com/google/devtools/build/docgen/ApiExporter.java b/src/main/java/com/google/devtools/build/docgen/ApiExporter.java
index 15be4a7..1fe236f 100644
--- a/src/main/java/com/google/devtools/build/docgen/ApiExporter.java
+++ b/src/main/java/com/google/devtools/build/docgen/ApiExporter.java
@@ -22,6 +22,7 @@
import com.google.devtools.build.docgen.builtin.BuiltinProtos.Value;
import com.google.devtools.build.docgen.starlark.StarlarkBuiltinDoc;
import com.google.devtools.build.docgen.starlark.StarlarkConstructorMethodDoc;
+import com.google.devtools.build.docgen.starlark.StarlarkDocExpander;
import com.google.devtools.build.docgen.starlark.StarlarkMethodDoc;
import com.google.devtools.build.docgen.starlark.StarlarkParamDoc;
import com.google.devtools.common.options.OptionsParser;
@@ -278,10 +279,10 @@
private static void printUsage(OptionsParser parser) {
System.err.println(
- "Usage: api_exporter_bin -n product_name -p rule_class_provider (-i input_dir)+\n"
+ "Usage: api_exporter_bin -m link_map_path -p rule_class_provider (-i input_dir)+\n"
+ " -f outputFile [-b denylist] [-h]\n\n"
+ "Exports all Starlark builtins to a file including the embedded native rules.\n"
- + "The product name (-n), rule class provider (-p), output file (-f) and at least \n"
+ + "The link map path (-m), rule class provider (-p), output file (-f) and at least \n"
+ " one input_dir (-i) must be specified.\n");
System.err.println(
parser.describeOptionsWithDeprecatedCategories(
@@ -299,7 +300,7 @@
Runtime.getRuntime().exit(0);
}
- if (options.productName.isEmpty()
+ if (options.linkMapPath.isEmpty()
|| options.inputDirs.isEmpty()
|| options.provider.isEmpty()
|| options.outputFile.isEmpty()) {
@@ -308,9 +309,14 @@
}
try {
+ DocLinkMap linkMap = DocLinkMap.createFromFile(options.linkMapPath);
+ RuleLinkExpander ruleExpander = new RuleLinkExpander(true, linkMap);
SymbolFamilies symbols =
new SymbolFamilies(
- options.productName, options.provider, options.inputDirs, options.denylist);
+ new StarlarkDocExpander(ruleExpander),
+ options.provider,
+ options.inputDirs,
+ options.denylist);
Builtins.Builder builtins = Builtins.newBuilder();
appendTypes(builtins, symbols.getTypes(), symbols.getNativeRules());
diff --git a/src/main/java/com/google/devtools/build/docgen/BUILD b/src/main/java/com/google/devtools/build/docgen/BUILD
index 009be96..19186f4 100644
--- a/src/main/java/com/google/devtools/build/docgen/BUILD
+++ b/src/main/java/com/google/devtools/build/docgen/BUILD
@@ -23,7 +23,6 @@
deps = [
"//src/main/java/com/google/devtools/build/docgen/annot",
"//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster",
- "//src/main/java/com/google/devtools/build/lib/bazel:main",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/collect/nestedset",
"//src/main/java/com/google/devtools/build/lib/concurrent",
@@ -36,6 +35,7 @@
"//src/main/java/net/starlark/java/eval",
"//src/main/protobuf:builtin_java_proto",
"//third_party:apache_velocity",
+ "//third_party:gson",
"//third_party:guava",
"//third_party:jsr305",
],
@@ -73,7 +73,6 @@
"//src/main/java/com/google/devtools/common/options",
"//src/main/java/net/starlark/java/annot",
"//src/main/java/net/starlark/java/eval",
- "//src/main/java/net/starlark/java/syntax",
"//src/main/protobuf:builtin_java_proto",
"//third_party:guava",
],
@@ -86,3 +85,8 @@
"templates/**/*.vm",
]),
)
+
+filegroup(
+ name = "bazel_link_map",
+ srcs = ["bazel_link_map.json"],
+)
diff --git a/src/main/java/com/google/devtools/build/docgen/BuildDocCollector.java b/src/main/java/com/google/devtools/build/docgen/BuildDocCollector.java
index a7438ad..cb4008b 100644
--- a/src/main/java/com/google/devtools/build/docgen/BuildDocCollector.java
+++ b/src/main/java/com/google/devtools/build/docgen/BuildDocCollector.java
@@ -44,13 +44,15 @@
public class BuildDocCollector {
private static final Splitter SHARP_SPLITTER = Splitter.on('#').limit(2).trimResults();
- private final String productName;
+ private final RuleLinkExpander linkExpander;
private final ConfiguredRuleClassProvider ruleClassProvider;
private final boolean printMessages;
public BuildDocCollector(
- String productName, ConfiguredRuleClassProvider ruleClassProvider, boolean printMessages) {
- this.productName = productName;
+ RuleLinkExpander linkExpander,
+ ConfiguredRuleClassProvider ruleClassProvider,
+ boolean printMessages) {
+ this.linkExpander = linkExpander;
this.ruleClassProvider = ruleClassProvider;
this.printMessages = printMessages;
}
@@ -90,18 +92,16 @@
* <p>In the Map's values, all references pointing to other rules, rule attributes, and general
* documentation (e.g. common definitions, make variables, etc.) are expanded into hyperlinks. The
* links generated follow either the multi-page or single-page Build Encyclopedia model depending
- * on the mode set for the provided {@link RuleLinkExpander}.
+ * on the mode set for the {@link RuleLinkExpander} that was passed to the constructor.
*
* @param inputDirs list of directories to scan for documentation
* @param denyList specify an optional denylist file that list some rules that should not be
* listed in the output.
- * @param expander The RuleLinkExpander, which is used for expanding links in the rule doc.
* @throws BuildEncyclopediaDocException
* @throws IOException
* @return Map of rule class to rule documentation.
*/
- public Map<String, RuleDocumentation> collect(
- List<String> inputDirs, String denyList, RuleLinkExpander expander)
+ public Map<String, RuleDocumentation> collect(List<String> inputDirs, String denyList)
throws BuildEncyclopediaDocException, IOException {
// Read the denyList file
Set<String> denylistedRules = readDenyList(denyList);
@@ -141,38 +141,14 @@
}
processAttributeDocs(ruleDocEntries.values(), attributeDocEntries);
- expander.addIndex(buildRuleIndex(ruleDocEntries.values()));
+ linkExpander.addIndex(buildRuleIndex(ruleDocEntries.values()));
for (RuleDocumentation rule : ruleDocEntries.values()) {
- rule.setRuleLinkExpander(expander);
+ rule.setRuleLinkExpander(linkExpander);
}
return ruleDocEntries;
}
/**
- * Creates a map of rule names (keys) to rule documentation (values).
- *
- * <p>This method crawls the specified input directories for rule class definitions (as Java
- * source files) which contain the rules' and attributes' definitions as comments in a specific
- * format. The keys in the returned Map correspond to these rule classes.
- *
- * <p>In the Map's values, all references pointing to other rules, rule attributes, and general
- * documentation (e.g. common definitions, make variables, etc.) are expanded into hyperlinks. The
- * links generated follow the multi-page Build Encyclopedia model (one page per rule class.).
- *
- * @param inputDirs list of directories to scan for documentation
- * @param denyList specify an optional denylist file that list some rules that should not be
- * listed in the output.
- * @throws BuildEncyclopediaDocException
- * @throws IOException
- * @return Map of rule class to rule documentation.
- */
- public Map<String, RuleDocumentation> collect(List<String> inputDirs, String denyList)
- throws BuildEncyclopediaDocException, IOException {
- RuleLinkExpander expander = new RuleLinkExpander(productName, /* singlePage */ false);
- return collect(inputDirs, denyList, expander);
- }
-
- /**
* Generates an index mapping rule name to its normalized rule family name.
*/
private Map<String, String> buildRuleIndex(Iterable<RuleDocumentation> rules) {
diff --git a/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaGenerator.java b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaGenerator.java
index 4e5413c..c4a1dcb 100644
--- a/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaGenerator.java
+++ b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaGenerator.java
@@ -26,10 +26,10 @@
public class BuildEncyclopediaGenerator {
private static void printUsage(OptionsParser parser) {
System.err.println(
- "Usage: docgen_bin -n product_name -p rule_class_provider (-i input_dir)+\n"
+ "Usage: docgen_bin -m link_map_file -p rule_class_provider (-i input_dir)+\n"
+ " [-o outputdir] [-b denylist] [-1] [-h]\n\n"
+ "Generates the Build Encyclopedia from embedded native rule documentation.\n"
- + "The product name (-n), rule class provider (-p) and at least one input_dir\n"
+ + "The link map file (-m), rule class provider (-p) and at least one input_dir\n"
+ "(-i) must be specified.\n");
System.err.println(
parser.describeOptionsWithDeprecatedCategories(
@@ -66,7 +66,7 @@
Runtime.getRuntime().exit(0);
}
- if (options.productName.isEmpty()
+ if (options.linkMapPath.isEmpty()
|| options.inputDirs.isEmpty()
|| options.provider.isEmpty()) {
printUsage(parser);
@@ -74,15 +74,18 @@
}
try {
+ DocLinkMap linkMap = DocLinkMap.createFromFile(options.linkMapPath);
+ RuleLinkExpander linkExpander = new RuleLinkExpander(options.singlePage, linkMap);
+
BuildEncyclopediaProcessor processor = null;
if (options.singlePage) {
processor =
new SinglePageBuildEncyclopediaProcessor(
- options.productName, createRuleClassProvider(options.provider));
+ linkExpander, createRuleClassProvider(options.provider));
} else {
processor =
new MultiPageBuildEncyclopediaProcessor(
- options.productName, createRuleClassProvider(options.provider));
+ linkExpander, createRuleClassProvider(options.provider));
}
processor.generateDocumentation(options.inputDirs, options.outputDir, options.denylist);
} catch (BuildEncyclopediaDocException e) {
diff --git a/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaOptions.java b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaOptions.java
index 0cadb48..eeeea50 100644
--- a/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaOptions.java
+++ b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaOptions.java
@@ -24,14 +24,13 @@
*/
public class BuildEncyclopediaOptions extends OptionsBase {
@Option(
- name = "product_name",
- abbrev = 'n',
- defaultValue = "",
- documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
- effectTags = {OptionEffectTag.UNKNOWN},
- help = "Name of the product to put in the documentation"
- )
- public String productName;
+ name = "link_map_path",
+ abbrev = 'm',
+ defaultValue = "",
+ documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+ effectTags = {OptionEffectTag.UNKNOWN},
+ help = "Path to a JSON file that specifies link mappings (page name to URL).")
+ public String linkMapPath;
@Option(
name = "input_dir",
diff --git a/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java
index a5ef7e9..05d3751 100644
--- a/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java
+++ b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java
@@ -45,8 +45,8 @@
}
};
- /** Name of the product to insert into the documentation. */
- protected final String productName;
+ /** Class that expand links to the BE. */
+ protected final RuleLinkExpander linkExpander;
/** Rule class provider from which to extract the rule class hierarchy and attributes. */
protected final ConfiguredRuleClassProvider ruleClassProvider;
@@ -56,8 +56,8 @@
* rule class hierarchy and attribute checking.
*/
public BuildEncyclopediaProcessor(
- String productName, ConfiguredRuleClassProvider ruleClassProvider) {
- this.productName = productName;
+ RuleLinkExpander linkExpander, ConfiguredRuleClassProvider ruleClassProvider) {
+ this.linkExpander = linkExpander;
this.ruleClassProvider = Preconditions.checkNotNull(ruleClassProvider);
}
@@ -211,13 +211,12 @@
* attributes can be expanded.
*
* @param attributes The map containing the RuleDocumentationAttributes, keyed by attribute name.
- * @param expander The RuleLinkExpander to set in each of the RuleDocumentationAttributes.
* @return The provided map of attributes.
*/
- protected static Map<String, RuleDocumentationAttribute> expandCommonAttributes(
- Map<String, RuleDocumentationAttribute> attributes, RuleLinkExpander expander) {
+ protected Map<String, RuleDocumentationAttribute> expandCommonAttributes(
+ Map<String, RuleDocumentationAttribute> attributes) {
for (RuleDocumentationAttribute attribute : attributes.values()) {
- attribute.setRuleLinkExpander(expander);
+ attribute.setRuleLinkExpander(linkExpander);
}
return attributes;
}
diff --git a/src/main/java/com/google/devtools/build/docgen/DocLinkMap.java b/src/main/java/com/google/devtools/build/docgen/DocLinkMap.java
new file mode 100644
index 0000000..3311514
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/DocLinkMap.java
@@ -0,0 +1,41 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.docgen;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonSyntaxException;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+
+/** Represents a link mapping that acts as input to {@link RuleLinkExpander}. */
+public class DocLinkMap {
+ final String beRoot;
+ final Map<String, String> values;
+
+ DocLinkMap(String beRoot, Map<String, String> values) {
+ this.beRoot = beRoot;
+ this.values = new HashMap<>(values);
+ }
+
+ public static DocLinkMap createFromFile(String filePath) {
+ try {
+ return new Gson().fromJson(Files.readString(Paths.get(filePath)), DocLinkMap.class);
+ } catch (IOException | JsonSyntaxException ex) {
+ throw new IllegalArgumentException("Failed to read link map from " + filePath, ex);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/DocgenConsts.java b/src/main/java/com/google/devtools/build/docgen/DocgenConsts.java
index d6d32a2..9d967dd 100644
--- a/src/main/java/com/google/devtools/build/docgen/DocgenConsts.java
+++ b/src/main/java/com/google/devtools/build/docgen/DocgenConsts.java
@@ -176,12 +176,10 @@
// The following variables are not constants as they can be overridden from
// StarlarkDocumentationProcessor#parseOptions
+ // Their purpose is to allow generated Starlark documentation to link into the build encyclopedia.
- // Build Encyclopedia documentation root
- public static String BeDocsRoot = "/versions/main/be";
-
- // Documentation files extension
- public static String documentationExtension = "html";
+ // Root directory of *narrative* Starlark documentation files such as rules.md
+ public static String starlarkDocsRoot = "/rules";
static String toCommandLineFormat(String cmdDoc) {
// Replace html <br> tags with line breaks
diff --git a/src/main/java/com/google/devtools/build/docgen/MultiPageBuildEncyclopediaProcessor.java b/src/main/java/com/google/devtools/build/docgen/MultiPageBuildEncyclopediaProcessor.java
index d930041..2943be5 100644
--- a/src/main/java/com/google/devtools/build/docgen/MultiPageBuildEncyclopediaProcessor.java
+++ b/src/main/java/com/google/devtools/build/docgen/MultiPageBuildEncyclopediaProcessor.java
@@ -25,8 +25,8 @@
*/
public class MultiPageBuildEncyclopediaProcessor extends BuildEncyclopediaProcessor {
public MultiPageBuildEncyclopediaProcessor(
- String productName, ConfiguredRuleClassProvider ruleClassProvider) {
- super(productName, ruleClassProvider);
+ RuleLinkExpander linkExpander, ConfiguredRuleClassProvider ruleClassProvider) {
+ super(linkExpander, ruleClassProvider);
}
/**
@@ -40,52 +40,43 @@
@Override
public void generateDocumentation(List<String> inputDirs, String outputDir, String denyList)
throws BuildEncyclopediaDocException, IOException {
- BuildDocCollector collector = new BuildDocCollector(productName, ruleClassProvider, false);
- RuleLinkExpander expander = new RuleLinkExpander(productName, false);
- Map<String, RuleDocumentation> ruleDocEntries =
- collector.collect(inputDirs, denyList, expander);
+ BuildDocCollector collector = new BuildDocCollector(linkExpander, ruleClassProvider, false);
+ Map<String, RuleDocumentation> ruleDocEntries = collector.collect(inputDirs, denyList);
warnAboutUndocumentedRules(
Sets.difference(ruleClassProvider.getRuleClassMap().keySet(), ruleDocEntries.keySet()));
- writeStaticDoc(outputDir, expander, "make-variables");
- writeStaticDoc(outputDir, expander, "functions");
- writeCommonDefinitionsPage(outputDir, expander);
+ writeStaticDoc(outputDir, "make-variables");
+ writeStaticDoc(outputDir, "functions");
+ writeCommonDefinitionsPage(outputDir);
- writeRuleDocs(outputDir, expander, ruleDocEntries.values());
+ writeRuleDocs(outputDir, ruleDocEntries.values());
}
- private void writeStaticDoc(String outputDir, RuleLinkExpander expander, String name)
- throws IOException {
+ private void writeStaticDoc(String outputDir, String name) throws IOException {
// TODO(dzc): Consider splitting out the call to writePage so that this method only creates the
// Page object and adding docgen tests that test the state of Page objects constructed by
// this method, and similar methods in this class.
Page page = TemplateEngine.newPage(DocgenConsts.BE_TEMPLATE_DIR + "/" + name + ".vm");
- page.add("expander", expander);
+ page.add("expander", linkExpander);
writePage(page, outputDir, name + ".html");
}
- private void writeCommonDefinitionsPage(String outputDir, RuleLinkExpander expander)
- throws IOException {
+ private void writeCommonDefinitionsPage(String outputDir) throws IOException {
Page page = TemplateEngine.newPage(DocgenConsts.COMMON_DEFINITIONS_TEMPLATE);
- page.add("expander", expander);
- page.add(
- "typicalAttributes",
- expandCommonAttributes(PredefinedAttributes.TYPICAL_ATTRIBUTES, expander));
- page.add("commonAttributes",
- expandCommonAttributes(PredefinedAttributes.COMMON_ATTRIBUTES, expander));
- page.add("testAttributes",
- expandCommonAttributes(PredefinedAttributes.TEST_ATTRIBUTES, expander));
- page.add("binaryAttributes",
- expandCommonAttributes(PredefinedAttributes.BINARY_ATTRIBUTES, expander));
+ page.add("expander", linkExpander);
+ page.add("typicalAttributes", expandCommonAttributes(PredefinedAttributes.TYPICAL_ATTRIBUTES));
+ page.add("commonAttributes", expandCommonAttributes(PredefinedAttributes.COMMON_ATTRIBUTES));
+ page.add("testAttributes", expandCommonAttributes(PredefinedAttributes.TEST_ATTRIBUTES));
+ page.add("binaryAttributes", expandCommonAttributes(PredefinedAttributes.BINARY_ATTRIBUTES));
writePage(page, outputDir, "common-definitions.html");
}
- private void writeRuleDocs(String outputDir, RuleLinkExpander expander,
- Iterable<RuleDocumentation> docEntries) throws BuildEncyclopediaDocException, IOException {
+ private void writeRuleDocs(String outputDir, Iterable<RuleDocumentation> docEntries)
+ throws BuildEncyclopediaDocException, IOException {
RuleFamilies ruleFamilies = assembleRuleFamilies(docEntries);
// Generate documentation.
- writeOverviewPage(outputDir, expander, ruleFamilies.langSpecific, ruleFamilies.generic);
+ writeOverviewPage(outputDir, ruleFamilies.langSpecific, ruleFamilies.generic);
writeBeNav(outputDir, ruleFamilies.all);
for (RuleFamily ruleFamily : ruleFamilies.all) {
if (ruleFamily.size() > 0) {
@@ -95,12 +86,11 @@
}
private void writeOverviewPage(String outputDir,
- RuleLinkExpander expander,
List<RuleFamily> langSpecificRuleFamilies,
List<RuleFamily> genericRuleFamilies)
throws BuildEncyclopediaDocException, IOException {
Page page = TemplateEngine.newPage(DocgenConsts.OVERVIEW_TEMPLATE);
- page.add("expander", expander);
+ page.add("expander", linkExpander);
page.add("langSpecificRuleFamilies", langSpecificRuleFamilies);
page.add("genericRuleFamilies", genericRuleFamilies);
writePage(page, outputDir, "overview.html");
@@ -110,6 +100,7 @@
throws BuildEncyclopediaDocException, IOException {
Page page = TemplateEngine.newPage(DocgenConsts.RULES_TEMPLATE);
page.add("ruleFamily", ruleFamily);
+ page.add("expander", linkExpander);
writePage(page, outputDir, ruleFamily.getId() + ".html");
}
diff --git a/src/main/java/com/google/devtools/build/docgen/ProtoFileBuildEncyclopediaProcessor.java b/src/main/java/com/google/devtools/build/docgen/ProtoFileBuildEncyclopediaProcessor.java
index 03b776f..be66b95 100644
--- a/src/main/java/com/google/devtools/build/docgen/ProtoFileBuildEncyclopediaProcessor.java
+++ b/src/main/java/com/google/devtools/build/docgen/ProtoFileBuildEncyclopediaProcessor.java
@@ -25,8 +25,8 @@
private ImmutableList<RuleDocumentation> nativeRules = null;
public ProtoFileBuildEncyclopediaProcessor(
- String productName, ConfiguredRuleClassProvider ruleClassProvider) {
- super(productName, ruleClassProvider);
+ RuleLinkExpander linkExpander, ConfiguredRuleClassProvider ruleClassProvider) {
+ super(linkExpander, ruleClassProvider);
}
/*
@@ -36,10 +36,8 @@
@Override
public void generateDocumentation(List<String> inputDirs, String outputFile, String denyList)
throws BuildEncyclopediaDocException, IOException {
- BuildDocCollector collector = new BuildDocCollector(productName, ruleClassProvider, false);
- RuleLinkExpander expander = new RuleLinkExpander(productName, true);
- Map<String, RuleDocumentation> ruleDocEntries =
- collector.collect(inputDirs, denyList, expander);
+ BuildDocCollector collector = new BuildDocCollector(linkExpander, ruleClassProvider, false);
+ Map<String, RuleDocumentation> ruleDocEntries = collector.collect(inputDirs, denyList);
RuleFamilies ruleFamilies = assembleRuleFamilies(ruleDocEntries.values());
ImmutableList.Builder<RuleDocumentation> ruleDocsBuilder = new ImmutableList.Builder<>();
diff --git a/src/main/java/com/google/devtools/build/docgen/RuleDocumentationAttribute.java b/src/main/java/com/google/devtools/build/docgen/RuleDocumentationAttribute.java
index 17194dd..7828508 100644
--- a/src/main/java/com/google/devtools/build/docgen/RuleDocumentationAttribute.java
+++ b/src/main/java/com/google/devtools/build/docgen/RuleDocumentationAttribute.java
@@ -48,17 +48,17 @@
.put(Type.STRING_DICT, "Dictionary: String -> String")
.put(Type.STRING_LIST, "List of strings")
.put(BuildType.TRISTATE, "Integer")
- .put(BuildType.LABEL, "<a href=\"../build-ref.html#labels\">Label</a>")
- .put(BuildType.LABEL_LIST, "List of <a href=\"../build-ref.html#labels\">labels</a>")
+ .put(BuildType.LABEL, "<a href=\"${link build-ref#labels}\">Label</a>")
+ .put(BuildType.LABEL_LIST, "List of <a href=\"${link build-ref#labels}\">labels</a>")
.put(
BuildType.LABEL_DICT_UNARY,
- "Dictionary mapping strings to <a href=\"../build-ref.html#labels\">labels</a>")
+ "Dictionary mapping strings to <a href=\"${link build-ref#labels}\">labels</a>")
.put(BuildType.LICENSE, "Licence type")
- .put(BuildType.NODEP_LABEL, "<a href=\"../build-ref.html#name\">Name</a>")
- .put(BuildType.NODEP_LABEL_LIST, "List of <a href=\"../build-ref.html#name\">names</a>")
- .put(BuildType.OUTPUT, "<a href=\"../build-ref.html#filename\">Filename</a>")
+ .put(BuildType.NODEP_LABEL, "<a href=\"${link build-ref#name}\">Name</a>")
+ .put(BuildType.NODEP_LABEL_LIST, "List of <a href=\"${link build-ref#name}\">names</a>")
+ .put(BuildType.OUTPUT, "<a href=\"${link build-ref#filename}\">Filename</a>")
.put(
- BuildType.OUTPUT_LIST, "List of <a href=\"../build-ref.html#filename\">filenames</a>")
+ BuildType.OUTPUT_LIST, "List of <a href=\"${link build-ref#filename}\">filenames</a>")
.buildOrThrow();
private final Class<? extends RuleDefinition> definitionClass;
@@ -150,15 +150,18 @@
* Returns the html documentation of the rule attribute.
*/
public String getHtmlDocumentation() throws BuildEncyclopediaDocException {
- String expandedHtmlDoc = htmlDocumentation;
- if (linkExpander != null) {
- try {
- expandedHtmlDoc = linkExpander.expand(expandedHtmlDoc);
- } catch (IllegalArgumentException e) {
- throw new BuildEncyclopediaDocException(fileName, startLineCnt, e.getMessage());
- }
+ return tryExpand(htmlDocumentation);
+ }
+
+ public String tryExpand(String html) throws BuildEncyclopediaDocException {
+ if (linkExpander == null) {
+ return html;
}
- return expandedHtmlDoc;
+ try {
+ return linkExpander.expand(html);
+ } catch (IllegalArgumentException e) {
+ throw new BuildEncyclopediaDocException(fileName, startLineCnt, e.getMessage());
+ }
}
/** Returns whether the param is required or optional. */
@@ -182,7 +185,7 @@
} else if (value instanceof String && !((String) value).isEmpty()) {
return prefix + "\"" + value + "\"";
} else if (value instanceof TriState) {
- switch((TriState) value) {
+ switch ((TriState) value) {
case AUTO:
return prefix + "-1";
case NO:
@@ -196,16 +199,15 @@
return "";
}
- /**
- * Returns a string containing the synopsis for this attribute.
- */
- public String getSynopsis() {
+ /** Returns a string containing the synopsis for this attribute. */
+ public String getSynopsis() throws BuildEncyclopediaDocException {
if (attribute == null) {
return "";
}
+ String rawType = TYPE_DESC.get(attribute.getType());
StringBuilder sb =
new StringBuilder()
- .append(TYPE_DESC.get(attribute.getType()))
+ .append(rawType == null ? null : tryExpand(rawType))
.append("; ")
.append(attribute.isMandatory() ? "required" : "optional")
.append(
diff --git a/src/main/java/com/google/devtools/build/docgen/RuleLinkExpander.java b/src/main/java/com/google/devtools/build/docgen/RuleLinkExpander.java
index aacfe55..bce5046 100644
--- a/src/main/java/com/google/devtools/build/docgen/RuleLinkExpander.java
+++ b/src/main/java/com/google/devtools/build/docgen/RuleLinkExpander.java
@@ -14,9 +14,8 @@
package com.google.devtools.build.docgen;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
+import java.nio.file.Paths;
import java.util.HashMap;
-import java.util.Locale;
import java.util.Map;
import java.util.regex.Matcher;
@@ -26,14 +25,14 @@
* <p>See {@link com.google.devtools.build.docgen.DocgenConsts.BLAZE_RULE_LINK} for the regex used
* to match link references.
*/
+// TODO(fwe): rename to LinkExpander
+// TODO(fwe): prefix rule links with BE root?
public class RuleLinkExpander {
private static final String EXAMPLES_SUFFIX = "_examples";
private static final String ARGS_SUFFIX = "_args";
private static final String IMPLICIT_OUTPUTS_SUFFIX = "_implicit_outputs";
private static final String FUNCTIONS_PAGE = "functions";
- private static final ImmutableSet<String> STATIC_PAGES =
- ImmutableSet.<String>of("common-definitions", "make-variables");
private static final ImmutableMap<String, String> FUNCTIONS =
ImmutableMap.<String, String>builder()
.put("load", FUNCTIONS_PAGE)
@@ -47,21 +46,25 @@
.put("select", FUNCTIONS_PAGE)
.buildOrThrow();
- private final String productName;
+ private final DocLinkMap linkMap;
private final Map<String, String> ruleIndex = new HashMap<>();
private final boolean singlePage;
- RuleLinkExpander(String productName, Map<String, String> ruleIndex, boolean singlePage) {
- this.productName = productName;
+ RuleLinkExpander(Map<String, String> ruleIndex, boolean singlePage, DocLinkMap linkMap) {
this.ruleIndex.putAll(ruleIndex);
this.ruleIndex.putAll(FUNCTIONS);
this.singlePage = singlePage;
+ this.linkMap = linkMap;
}
- RuleLinkExpander(String productName, boolean singlePage) {
- this.productName = productName;
+ RuleLinkExpander(boolean singlePage, DocLinkMap linkMap) {
this.ruleIndex.putAll(FUNCTIONS);
this.singlePage = singlePage;
+ this.linkMap = linkMap;
+ }
+
+ public String beRoot() {
+ return linkMap.beRoot;
}
public void addIndex(Map<String, String> ruleIndex) {
@@ -70,9 +73,10 @@
private void appendRuleLink(Matcher matcher, StringBuffer sb, String ruleName, String ref) {
String ruleFamily = ruleIndex.get(ruleName);
- String link = singlePage
- ? "#" + ref
- : ruleFamily + ".html#" + ref;
+ String link =
+ singlePage
+ ? "#" + ref
+ : Paths.get(linkMap.beRoot, String.format("%s.html#%s", ruleFamily, ref)).toString();
matcher.appendReplacement(sb, Matcher.quoteReplacement(link));
}
@@ -119,10 +123,9 @@
// The name is not the name of a rule but is the name of a static page, such as
// common-definitions. Generate a link to that page.
- if (STATIC_PAGES.contains(name)) {
- String link = singlePage
- ? "#" + name
- : name + ".html";
+ String mapping = linkMap.values.get(name);
+ if (mapping != null) {
+ String link = singlePage ? "#" + name : mapping;
// For referencing headings on a static page, use the following syntax:
// ${link static_page_name#heading_name}, example: ${link make-variables#gendir}
String pageHeading = matcher.group(4);
@@ -151,6 +154,8 @@
Matcher matcher = DocgenConsts.BLAZE_RULE_HEADING_LINK.matcher(htmlDoc);
StringBuffer sb = new StringBuffer(htmlDoc.length());
while (matcher.find()) {
+ // The first capture group matches the entire reference, e.g. "cc_library#some_heading".
+ String ref = matcher.group(1);
// The second capture group only matches the rule name, e.g. "cc_library" in
// "cc_library#some_heading"
String name = matcher.group(2);
@@ -173,17 +178,32 @@
// The name is of a static page, such as common.definitions. Generate a link to that page, and
// append the page heading. For example, ${link common-definitions#label-expansion} expands to
// common-definitions.html#label-expansion.
- if (STATIC_PAGES.contains(name)) {
- String link = singlePage
- ? "#" + heading
- : name + ".html#" + heading;
- matcher.appendReplacement(sb, Matcher.quoteReplacement(link));
- continue;
- }
- // Links to the user manual are handled specially. Meh.
- if ("user-manual".equals(name)) {
- String link = productName.toLowerCase(Locale.US) + "-" + name + ".html#" + heading;
+ // We need to search for the entire match first since some documentation files have a 1:n
+ // relation between Blaze and Bazel. Example: build-ref#labels points to build-ref.html#labels
+ // for Blaze, but to /concepts/labels for Bazel. However, we have to consider whether a single
+ // heading or the entire page has to be redirected.
+
+ // Not-null if page#heading has a mapping (other headings on the page are unaffected):
+ String headingMapping = linkMap.values.get(ref);
+ // Not-null if the entire page has a mapping, i.e. all headings should be redirected:
+ String pageMapping = linkMap.values.get(name);
+
+ if (headingMapping != null || pageMapping != null) {
+ String link;
+ if (singlePage) {
+ // Special case: For the single-page BE we don't use the value of the mapping, we just
+ // need to know that there is one (since that means `name` is a legitimate BE page).
+ link = "#" + heading;
+ } else if (headingMapping != null) {
+ // Multi-page BE where page#heading has to be redirected.
+ link = headingMapping;
+ } else { // pageMapping != null
+ // Multi-page BE where the entire page has to be forwarded (but the new page has
+ // identical headings).
+ link = String.format("%s#%s", pageMapping, heading);
+ }
+
matcher.appendReplacement(sb, Matcher.quoteReplacement(link));
continue;
}
diff --git a/src/main/java/com/google/devtools/build/docgen/SinglePageBuildEncyclopediaProcessor.java b/src/main/java/com/google/devtools/build/docgen/SinglePageBuildEncyclopediaProcessor.java
index 346cc33..0e6bc2d 100644
--- a/src/main/java/com/google/devtools/build/docgen/SinglePageBuildEncyclopediaProcessor.java
+++ b/src/main/java/com/google/devtools/build/docgen/SinglePageBuildEncyclopediaProcessor.java
@@ -25,8 +25,8 @@
*/
public class SinglePageBuildEncyclopediaProcessor extends BuildEncyclopediaProcessor {
public SinglePageBuildEncyclopediaProcessor(
- String productName, ConfiguredRuleClassProvider ruleClassProvider) {
- super(productName, ruleClassProvider);
+ RuleLinkExpander linkExpander, ConfiguredRuleClassProvider ruleClassProvider) {
+ super(linkExpander, ruleClassProvider);
}
/**
@@ -40,10 +40,8 @@
@Override
public void generateDocumentation(List<String> inputDirs, String outputDir, String denyList)
throws BuildEncyclopediaDocException, IOException {
- BuildDocCollector collector = new BuildDocCollector(productName, ruleClassProvider, false);
- RuleLinkExpander expander = new RuleLinkExpander(productName, true);
- Map<String, RuleDocumentation> ruleDocEntries =
- collector.collect(inputDirs, denyList, expander);
+ BuildDocCollector collector = new BuildDocCollector(linkExpander, ruleClassProvider, false);
+ Map<String, RuleDocumentation> ruleDocEntries = collector.collect(inputDirs, denyList);
warnAboutUndocumentedRules(
Sets.difference(ruleClassProvider.getRuleClassMap().keySet(), ruleDocEntries.keySet()));
RuleFamilies ruleFamilies = assembleRuleFamilies(ruleDocEntries.values());
@@ -51,18 +49,13 @@
Page page = TemplateEngine.newPage(DocgenConsts.SINGLE_BE_TEMPLATE);
// Add the rule link expander.
- page.add("expander", expander);
+ page.add("expander", linkExpander);
// Populate variables for Common Definitions section.
- page.add(
- "typicalAttributes",
- expandCommonAttributes(PredefinedAttributes.TYPICAL_ATTRIBUTES, expander));
- page.add("commonAttributes",
- expandCommonAttributes(PredefinedAttributes.COMMON_ATTRIBUTES, expander));
- page.add("testAttributes",
- expandCommonAttributes(PredefinedAttributes.TEST_ATTRIBUTES, expander));
- page.add("binaryAttributes",
- expandCommonAttributes(PredefinedAttributes.BINARY_ATTRIBUTES, expander));
+ page.add("typicalAttributes", expandCommonAttributes(PredefinedAttributes.TYPICAL_ATTRIBUTES));
+ page.add("commonAttributes", expandCommonAttributes(PredefinedAttributes.COMMON_ATTRIBUTES));
+ page.add("testAttributes", expandCommonAttributes(PredefinedAttributes.TEST_ATTRIBUTES));
+ page.add("binaryAttributes", expandCommonAttributes(PredefinedAttributes.BINARY_ATTRIBUTES));
// Popualte variables for Overview section.
page.add("langSpecificRuleFamilies", ruleFamilies.langSpecific);
diff --git a/src/main/java/com/google/devtools/build/docgen/StarlarkDocumentationCollector.java b/src/main/java/com/google/devtools/build/docgen/StarlarkDocumentationCollector.java
index 36aba17..3881f05 100644
--- a/src/main/java/com/google/devtools/build/docgen/StarlarkDocumentationCollector.java
+++ b/src/main/java/com/google/devtools/build/docgen/StarlarkDocumentationCollector.java
@@ -21,6 +21,7 @@
import com.google.devtools.build.docgen.annot.StarlarkConstructor;
import com.google.devtools.build.docgen.starlark.StarlarkBuiltinDoc;
import com.google.devtools.build.docgen.starlark.StarlarkConstructorMethodDoc;
+import com.google.devtools.build.docgen.starlark.StarlarkDocExpander;
import com.google.devtools.build.docgen.starlark.StarlarkJavaMethodDoc;
import com.google.devtools.build.lib.util.Classpath;
import com.google.devtools.build.lib.util.Classpath.ClassPathException;
@@ -53,14 +54,15 @@
private static ImmutableMap<String, StarlarkBuiltinDoc> all;
/** Applies {@link #collectModules} to all Bazel and Starlark classes. */
- static synchronized ImmutableMap<String, StarlarkBuiltinDoc> getAllModules()
- throws ClassPathException {
+ static synchronized ImmutableMap<String, StarlarkBuiltinDoc> getAllModules(
+ StarlarkDocExpander expander) throws ClassPathException {
if (all == null) {
all =
collectModules(
Iterables.concat(
Classpath.findClasses("com/google/devtools/build"), // Bazel
- Classpath.findClasses("net/starlark/java"))); // Starlark
+ Classpath.findClasses("net/starlark/java")),
+ expander); // Starlark
}
return all;
}
@@ -69,20 +71,22 @@
* Collects the documentation for all Starlark modules comprised of the given classes and returns
* a map from the name of each Starlark module to its documentation.
*/
- static ImmutableMap<String, StarlarkBuiltinDoc> collectModules(Iterable<Class<?>> classes) {
+ static ImmutableMap<String, StarlarkBuiltinDoc> collectModules(
+ Iterable<Class<?>> classes, StarlarkDocExpander expander) {
Map<String, StarlarkBuiltinDoc> modules = new TreeMap<>();
// The top level module first.
// (This is a special case of {@link StarlarkBuiltinDoc} as it has no object name).
StarlarkBuiltin topLevelModule = getTopLevelModule();
modules.put(
topLevelModule.name(),
- new StarlarkBuiltinDoc(topLevelModule, /*title=*/ "Globals", TopLevelModule.class));
+ new StarlarkBuiltinDoc(
+ topLevelModule, /*title=*/ "Globals", TopLevelModule.class, expander));
// Creating module documentation is done in three passes.
// 1. Add all classes/interfaces annotated with @StarlarkBuiltin with documented = true.
for (Class<?> candidateClass : classes) {
if (candidateClass.isAnnotationPresent(StarlarkBuiltin.class)) {
- collectStarlarkModule(candidateClass, modules);
+ collectStarlarkModule(candidateClass, modules, expander);
}
}
@@ -97,7 +101,7 @@
// if (e.getValue() instanceof BuiltinFunction) {
// BuiltinFunction fn = (BuiltinFunction) e.getValue();
// topLevelModuleDoc.addMethod(
- // new StarlarkJavaMethodDoc("", fn.getJavaMethod(), fn.getAnnotation()));
+ // new StarlarkJavaMethodDoc("", fn.getJavaMethod(), fn.getAnnotation(), expander));
// }
// }
//
@@ -105,11 +109,11 @@
//
for (Class<?> candidateClass : classes) {
if (candidateClass.isAnnotationPresent(StarlarkBuiltin.class)) {
- collectModuleMethods(candidateClass, modules);
+ collectModuleMethods(candidateClass, modules, expander);
}
if (candidateClass.isAnnotationPresent(DocumentMethods.class)
|| candidateClass.getName().equals("net.starlark.java.eval.MethodLibrary")) {
- collectDocumentedMethods(candidateClass, modules);
+ collectDocumentedMethods(candidateClass, modules, expander);
}
}
@@ -117,7 +121,7 @@
for (Class<?> candidateClass : classes) {
if (candidateClass.isAnnotationPresent(StarlarkBuiltin.class)
|| candidateClass.isAnnotationPresent(DocumentMethods.class)) {
- collectConstructorMethods(candidateClass, modules);
+ collectConstructorMethods(candidateClass, modules, expander);
}
}
@@ -137,7 +141,7 @@
* moduleClass}, if it is a documented module.
*/
private static void collectStarlarkModule(
- Class<?> moduleClass, Map<String, StarlarkBuiltinDoc> modules) {
+ Class<?> moduleClass, Map<String, StarlarkBuiltinDoc> modules, StarlarkDocExpander expander) {
if (moduleClass.equals(TopLevelModule.class)) {
// The top level module doc is a special case and is handled separately.
return;
@@ -151,7 +155,8 @@
if (previousModuleDoc == null) {
modules.put(
moduleAnnotation.name(),
- new StarlarkBuiltinDoc(moduleAnnotation, moduleAnnotation.name(), moduleClass));
+ new StarlarkBuiltinDoc(
+ moduleAnnotation, moduleAnnotation.name(), moduleClass, expander));
} else {
// Handle a strange corner-case: If moduleClass has a subclass which is also
// annotated with {@link StarlarkBuiltin} with the same name, and also has the same
@@ -165,7 +170,7 @@
modules.put(
moduleAnnotation.name(),
new StarlarkBuiltinDoc(
- moduleAnnotation, /*title=*/ moduleAnnotation.name(), moduleClass));
+ moduleAnnotation, /*title=*/ moduleAnnotation.name(), moduleClass, expander));
}
}
}
@@ -194,7 +199,7 @@
}
private static void collectModuleMethods(
- Class<?> moduleClass, Map<String, StarlarkBuiltinDoc> modules) {
+ Class<?> moduleClass, Map<String, StarlarkBuiltinDoc> modules, StarlarkDocExpander expander) {
StarlarkBuiltin moduleAnnotation =
Preconditions.checkNotNull(moduleClass.getAnnotation(StarlarkBuiltin.class));
@@ -209,7 +214,8 @@
// Methods with @StarlarkConstructor are added later.
if (!entry.getKey().isAnnotationPresent(StarlarkConstructor.class)) {
moduleDoc.addMethod(
- new StarlarkJavaMethodDoc(moduleDoc.getName(), entry.getKey(), entry.getValue()));
+ new StarlarkJavaMethodDoc(
+ moduleDoc.getName(), entry.getKey(), entry.getValue(), expander));
}
}
}
@@ -230,7 +236,7 @@
* each @StarlarkMethod method defined in the given @DocumentMethods class {@code moduleClass}.
*/
private static void collectDocumentedMethods(
- Class<?> moduleClass, Map<String, StarlarkBuiltinDoc> modules) {
+ Class<?> moduleClass, Map<String, StarlarkBuiltinDoc> modules, StarlarkDocExpander expander) {
StarlarkBuiltinDoc topLevelModuleDoc = getTopLevelModuleDoc(modules);
for (Map.Entry<Method, StarlarkMethod> entry :
@@ -238,12 +244,13 @@
// Only add non-constructor global library methods. Constructors are added later.
if (!entry.getKey().isAnnotationPresent(StarlarkConstructor.class)) {
topLevelModuleDoc.addMethod(
- new StarlarkJavaMethodDoc("", entry.getKey(), entry.getValue()));
+ new StarlarkJavaMethodDoc("", entry.getKey(), entry.getValue(), expander));
}
}
}
- private static void collectConstructor(Map<String, StarlarkBuiltinDoc> modules, Method method) {
+ private static void collectConstructor(
+ Map<String, StarlarkBuiltinDoc> modules, Method method, StarlarkDocExpander expander) {
Preconditions.checkNotNull(method.getAnnotation(StarlarkConstructor.class));
StarlarkBuiltin builtinType = StarlarkAnnotations.getStarlarkBuiltin(method.getReturnType());
@@ -255,7 +262,8 @@
StarlarkMethod methodAnnot =
Preconditions.checkNotNull(method.getAnnotation(StarlarkMethod.class));
StarlarkBuiltinDoc doc = modules.get(builtinType.name());
- doc.setConstructor(new StarlarkConstructorMethodDoc(builtinType.name(), method, methodAnnot));
+ doc.setConstructor(
+ new StarlarkConstructorMethodDoc(builtinType.name(), method, methodAnnot, expander));
}
/**
@@ -269,20 +277,20 @@
* like a constructor method.)
*/
private static void collectConstructorMethods(
- Class<?> moduleClass, Map<String, StarlarkBuiltinDoc> modules) {
+ Class<?> moduleClass, Map<String, StarlarkBuiltinDoc> modules, StarlarkDocExpander expander) {
Method selfCallConstructor = getSelfCallConstructorMethod(moduleClass);
if (selfCallConstructor != null) {
- collectConstructor(modules, selfCallConstructor);
+ collectConstructor(modules, selfCallConstructor, expander);
}
for (Method method : Starlark.getMethodAnnotations(moduleClass).keySet()) {
if (method.isAnnotationPresent(StarlarkConstructor.class)) {
- collectConstructor(modules, method);
+ collectConstructor(modules, method, expander);
}
Class<?> returnClass = method.getReturnType();
Method returnClassConstructor = getSelfCallConstructorMethod(returnClass);
if (returnClassConstructor != null) {
- collectConstructor(modules, returnClassConstructor);
+ collectConstructor(modules, returnClassConstructor, expander);
}
}
}
diff --git a/src/main/java/com/google/devtools/build/docgen/StarlarkDocumentationProcessor.java b/src/main/java/com/google/devtools/build/docgen/StarlarkDocumentationProcessor.java
index b66369a..5051b34 100644
--- a/src/main/java/com/google/devtools/build/docgen/StarlarkDocumentationProcessor.java
+++ b/src/main/java/com/google/devtools/build/docgen/StarlarkDocumentationProcessor.java
@@ -16,7 +16,7 @@
import com.google.common.collect.ImmutableList;
import com.google.devtools.build.docgen.annot.DocCategory;
import com.google.devtools.build.docgen.starlark.StarlarkBuiltinDoc;
-import com.google.devtools.build.docgen.starlark.StarlarkDocUtils;
+import com.google.devtools.build.docgen.starlark.StarlarkDocExpander;
import com.google.devtools.build.docgen.starlark.StarlarkMethodDoc;
import com.google.devtools.build.lib.util.Classpath.ClassPathException;
import java.io.File;
@@ -24,6 +24,7 @@
import java.text.Collator;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
@@ -42,10 +43,24 @@
/** Generates the Starlark documentation to the given output directory. */
public static void generateDocumentation(String outputDir, String... args)
throws IOException, ClassPathException {
- parseOptions(args);
+ Map<String, String> options = parseOptions(args);
+
+ String docsRoot = options.get("--starlark_docs_root");
+ if (docsRoot != null) {
+ DocgenConsts.starlarkDocsRoot = docsRoot;
+ }
+
+ String linkMapPath = options.get("--link_map_path");
+ if (linkMapPath == null) {
+ throw new IllegalArgumentException("Required option '--link_map_path' is missing.");
+ }
+
+ DocLinkMap linkMap = DocLinkMap.createFromFile(linkMapPath);
+ StarlarkDocExpander expander =
+ new StarlarkDocExpander(new RuleLinkExpander(/*singlePage*/ false, linkMap));
Map<String, StarlarkBuiltinDoc> modules =
- new TreeMap<>(StarlarkDocumentationCollector.getAllModules());
+ new TreeMap<>(StarlarkDocumentationCollector.getAllModules(expander));
// Generate the top level module first in the doc
StarlarkBuiltinDoc topLevelModule =
@@ -71,10 +86,10 @@
for (List<StarlarkBuiltinDoc> module : modulesByCategory.values()) {
Collections.sort(module, (doc1, doc2) -> us.compare(doc1.getTitle(), doc2.getTitle()));
}
- writeCategoryPage(Category.CORE, outputDir, modulesByCategory);
- writeCategoryPage(Category.CONFIGURATION_FRAGMENT, outputDir, modulesByCategory);
- writeCategoryPage(Category.BUILTIN, outputDir, modulesByCategory);
- writeCategoryPage(Category.PROVIDER, outputDir, modulesByCategory);
+ writeCategoryPage(Category.CORE, outputDir, modulesByCategory, expander);
+ writeCategoryPage(Category.CONFIGURATION_FRAGMENT, outputDir, modulesByCategory, expander);
+ writeCategoryPage(Category.BUILTIN, outputDir, modulesByCategory, expander);
+ writeCategoryPage(Category.PROVIDER, outputDir, modulesByCategory, expander);
writeNavPage(outputDir, modulesByCategory.get(Category.TOP_LEVEL_TYPE));
// In the code, there are two StarlarkModuleCategory instances that have no heading:
@@ -138,14 +153,17 @@
}
private static void writeCategoryPage(
- Category category, String outputDir, Map<Category, List<StarlarkBuiltinDoc>> modules)
+ Category category,
+ String outputDir,
+ Map<Category, List<StarlarkBuiltinDoc>> modules,
+ StarlarkDocExpander expander)
throws IOException {
File starlarkDocPath =
new File(String.format("%s/starlark-%s.html", outputDir, category.getTemplateIdentifier()));
Page page = TemplateEngine.newPage(DocgenConsts.STARLARK_MODULE_CATEGORY_TEMPLATE);
page.add("category", category);
page.add("modules", modules.get(category));
- page.add("description", StarlarkDocUtils.substituteVariables(category.description));
+ page.add("description", expander.expand(category.description));
page.write(starlarkDocPath);
}
@@ -175,15 +193,15 @@
page.write(starlarkDocPath);
}
- private static void parseOptions(String... args) {
+ private static Map<String, String> parseOptions(String... args) {
+ Map<String, String> options = new HashMap<>();
for (String arg : args) {
- if (arg.startsWith("--be_root=")) {
- DocgenConsts.BeDocsRoot = arg.split("--be_root=", 2)[1];
- }
- if (arg.startsWith("--doc_extension=")) {
- DocgenConsts.documentationExtension = arg.split("--doc_extension=", 2)[1];
+ if (arg.startsWith("--")) {
+ String[] parts = arg.split("=", 2);
+ options.put(parts[0], parts.length > 1 ? parts[1] : null);
}
}
+ return options;
}
/**
@@ -202,8 +220,9 @@
PROVIDER(
"Providers",
- "This section lists providers available on built-in rules. See the "
- + "<a href='../rules.$DOC_EXT#providers'>Rules page</a> for more on providers."),
+ "This section lists providers available on built-in rules. See the <a"
+ + " href='$STARLARK_DOCS_ROOT/rules.html#providers'>Rules page</a> for more on"
+ + " providers."),
BUILTIN("Built-in Types", "This section lists types of Starlark objects."),
diff --git a/src/main/java/com/google/devtools/build/docgen/SymbolFamilies.java b/src/main/java/com/google/devtools/build/docgen/SymbolFamilies.java
index 2dd8863..38e9bec 100644
--- a/src/main/java/com/google/devtools/build/docgen/SymbolFamilies.java
+++ b/src/main/java/com/google/devtools/build/docgen/SymbolFamilies.java
@@ -16,6 +16,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.docgen.starlark.StarlarkBuiltinDoc;
+import com.google.devtools.build.docgen.starlark.StarlarkDocExpander;
import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
import com.google.devtools.build.lib.util.Classpath.ClassPathException;
import com.google.devtools.build.skydoc.fakebuildapi.FakeStarlarkNativeModuleApi;
@@ -39,17 +40,18 @@
private final ImmutableMap<String, Object> bzlGlobals;
public SymbolFamilies(
- String productName, String provider, List<String> inputDirs, String denyList)
+ StarlarkDocExpander expander, String provider, List<String> inputDirs, String denyList)
throws NoSuchMethodException, ClassPathException, InvocationTargetException,
IllegalAccessException, BuildEncyclopediaDocException, ClassNotFoundException,
IOException {
ConfiguredRuleClassProvider configuredRuleClassProvider = createRuleClassProvider(provider);
this.nativeRules =
ImmutableList.copyOf(
- collectNativeRules(productName, configuredRuleClassProvider, inputDirs, denyList));
+ collectNativeRules(
+ expander.ruleExpander, configuredRuleClassProvider, inputDirs, denyList));
this.globals = Starlark.UNIVERSE;
this.bzlGlobals = collectBzlGlobals(configuredRuleClassProvider);
- this.types = StarlarkDocumentationCollector.getAllModules();
+ this.types = StarlarkDocumentationCollector.getAllModules(expander);
}
/*
@@ -85,13 +87,13 @@
* and in BZL files as methods of the native package.
*/
private List<RuleDocumentation> collectNativeRules(
- String productName,
+ RuleLinkExpander linkExpander,
ConfiguredRuleClassProvider provider,
List<String> inputDirs,
String denyList)
throws BuildEncyclopediaDocException, IOException {
ProtoFileBuildEncyclopediaProcessor processor =
- new ProtoFileBuildEncyclopediaProcessor(productName, provider);
+ new ProtoFileBuildEncyclopediaProcessor(linkExpander, provider);
processor.generateDocumentation(inputDirs, "", denyList);
return processor.getNativeRules();
}
diff --git a/src/main/java/com/google/devtools/build/docgen/bazel_link_map.json b/src/main/java/com/google/devtools/build/docgen/bazel_link_map.json
new file mode 100644
index 0000000..e55dab9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/bazel_link_map.json
@@ -0,0 +1,25 @@
+{
+ "beRoot": "/reference/be",
+ "values":
+ {
+ "build-ref": "/concepts/build-ref",
+ "build-ref#labels": "/concepts/labels",
+ "build-ref#name": "/concepts/labels#target-names",
+ "build-ref#BUILD_files": "/concepts/build-files",
+ "build-ref#dependencies": "/concepts/dependencies",
+ "common-definitions": "/reference/be/common-definitions",
+ "config": "/rules/config",
+ "configurable-attributes": "/docs/configurable-attributes",
+ "exec-groups": "/reference/exec-groups",
+ "functions": "/reference/be/functions",
+ "guide": "/contribute/guide",
+ "make-variables": "/reference/be/make-variables",
+ "platforms": "/docs/platforms",
+ "protocol-buffer": "/reference/be/protocol-buffer",
+ "query": "/reference/query",
+ "test-encyclopedia": "/reference/test-encyclopedia",
+ "toolchains": "/docs/toolchains",
+ "user-manual": "/docs/user-manual",
+ "visibility": "/concepts/visibility"
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkBuiltinDoc.java b/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkBuiltinDoc.java
index e6c3f3c..0045315 100644
--- a/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkBuiltinDoc.java
+++ b/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkBuiltinDoc.java
@@ -39,7 +39,9 @@
private TreeMap<String, StarlarkMethodDoc> methodMap;
@Nullable private StarlarkConstructorMethodDoc javaConstructor;
- public StarlarkBuiltinDoc(StarlarkBuiltin module, String title, Class<?> classObject) {
+ public StarlarkBuiltinDoc(
+ StarlarkBuiltin module, String title, Class<?> classObject, StarlarkDocExpander expander) {
+ super(expander);
this.module =
Preconditions.checkNotNull(
module, "Class has to be annotated with StarlarkBuiltin: %s", classObject);
@@ -55,8 +57,8 @@
}
@Override
- public String getDocumentation() {
- return StarlarkDocUtils.substituteVariables(module.doc());
+ public String getRawDocumentation() {
+ return module.doc();
}
public String getTitle() {
diff --git a/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkConstructorMethodDoc.java b/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkConstructorMethodDoc.java
index 1865c55e..e96e3a0 100644
--- a/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkConstructorMethodDoc.java
+++ b/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkConstructorMethodDoc.java
@@ -30,7 +30,11 @@
private final ImmutableList<StarlarkParamDoc> params;
public StarlarkConstructorMethodDoc(
- String fullyQualifiedName, Method method, StarlarkMethod callable) {
+ String fullyQualifiedName,
+ Method method,
+ StarlarkMethod callable,
+ StarlarkDocExpander expander) {
+ super(expander);
this.fullyQualifiedName = fullyQualifiedName;
this.method = method;
this.callable = callable;
@@ -39,7 +43,8 @@
this,
withoutSelfParam(callable, method),
callable.extraPositionals(),
- callable.extraKeywords());
+ callable.extraKeywords(),
+ expander);
}
@Override
@@ -58,8 +63,8 @@
}
@Override
- public String getDocumentation() {
- return StarlarkDocUtils.substituteVariables(callable.doc());
+ public String getRawDocumentation() {
+ return callable.doc();
}
@Override
diff --git a/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkDoc.java b/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkDoc.java
index 6da233b..582a0da 100644
--- a/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkDoc.java
+++ b/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkDoc.java
@@ -32,15 +32,30 @@
abstract class StarlarkDoc {
protected static final String TOP_LEVEL_ID = "globals";
+ private final StarlarkDocExpander expander;
+
+ protected StarlarkDoc(StarlarkDocExpander expander) {
+ this.expander = expander;
+ }
+
/**
* Returns a string containing the name of the entity being documented.
*/
public abstract String getName();
/**
- * Returns a string containing the formatted HTML documentation of the entity being documented.
+ * Returns a string containing the formatted HTML documentation of the entity being documented
+ * (without any variables).
*/
- public abstract String getDocumentation();
+ public String getDocumentation() {
+ return expander.expand(getRawDocumentation());
+ }
+
+ /**
+ * Returns the HTML documentation of the entity being documented, which potentially contains
+ * variables and unresolved links.
+ */
+ public abstract String getRawDocumentation();
protected String getTypeAnchor(Class<?> returnType, Class<?> generic1) {
return getTypeAnchor(returnType) + " of " + getTypeAnchor(generic1) + "s";
diff --git a/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkDocExpander.java b/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkDocExpander.java
new file mode 100644
index 0000000..973f65c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkDocExpander.java
@@ -0,0 +1,32 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.docgen.starlark;
+
+import com.google.devtools.build.docgen.RuleLinkExpander;
+
+/** A utility class for replacing variables in documentation strings with their actual values. */
+public class StarlarkDocExpander {
+
+ public final RuleLinkExpander ruleExpander;
+
+ public StarlarkDocExpander(RuleLinkExpander ruleExpander) {
+ this.ruleExpander = ruleExpander;
+ }
+
+ public String expand(String docString) {
+ return ruleExpander.expand(
+ StarlarkDocUtils.substituteVariables(docString, ruleExpander.beRoot()));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkDocUtils.java b/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkDocUtils.java
index e24993c..61a55d6 100644
--- a/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkDocUtils.java
+++ b/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkDocUtils.java
@@ -27,10 +27,11 @@
*
* @return a string with substituted variables
*/
- public static String substituteVariables(String documentation) {
+ public static String substituteVariables(String documentation, String beRoot) {
+ // TODO(b/193923321): Get rid of $STARLARK_DOCS_ROOT and of this entire class, eventually.
return documentation
- .replace("$BE_ROOT", DocgenConsts.BeDocsRoot)
- .replace("$DOC_EXT", DocgenConsts.documentationExtension);
+ .replace("$STARLARK_DOCS_ROOT", DocgenConsts.starlarkDocsRoot)
+ .replace("$BE_ROOT", beRoot);
}
/**
@@ -41,18 +42,19 @@
StarlarkMethodDoc methodDoc,
Param[] userSuppliedParams,
Param extraPositionals,
- Param extraKeywords) {
+ Param extraKeywords,
+ StarlarkDocExpander expander) {
ImmutableList.Builder<StarlarkParamDoc> paramsBuilder = ImmutableList.builder();
for (Param param : userSuppliedParams) {
if (param.documented()) {
- paramsBuilder.add(new StarlarkParamDoc(methodDoc, param));
+ paramsBuilder.add(new StarlarkParamDoc(methodDoc, param, expander));
}
}
if (!extraPositionals.name().isEmpty()) {
- paramsBuilder.add(new StarlarkParamDoc(methodDoc, extraPositionals));
+ paramsBuilder.add(new StarlarkParamDoc(methodDoc, extraPositionals, expander));
}
if (!extraKeywords.name().isEmpty()) {
- paramsBuilder.add(new StarlarkParamDoc(methodDoc, extraKeywords));
+ paramsBuilder.add(new StarlarkParamDoc(methodDoc, extraKeywords, expander));
}
return paramsBuilder.build();
}
diff --git a/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkJavaMethodDoc.java b/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkJavaMethodDoc.java
index ae9eaf0..15ae698 100644
--- a/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkJavaMethodDoc.java
+++ b/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkJavaMethodDoc.java
@@ -33,7 +33,9 @@
private boolean isOverloaded;
- public StarlarkJavaMethodDoc(String moduleName, Method method, StarlarkMethod callable) {
+ public StarlarkJavaMethodDoc(
+ String moduleName, Method method, StarlarkMethod callable, StarlarkDocExpander expander) {
+ super(expander);
this.moduleName = moduleName;
this.name = callable.name();
this.method = method;
@@ -43,7 +45,8 @@
this,
withoutSelfParam(callable, method),
callable.extraPositionals(),
- callable.extraKeywords());
+ callable.extraKeywords(),
+ expander);
}
@Override
@@ -87,7 +90,7 @@
}
@Override
- public String getDocumentation() {
+ public String getRawDocumentation() {
String prefixWarning = "";
if (!callable.enableOnlyWithFlag().isEmpty()) {
prefixWarning =
@@ -105,7 +108,7 @@
+ "</code>. Use this flag "
+ "to verify your code is compatible with its imminent removal. <br>";
}
- return prefixWarning + StarlarkDocUtils.substituteVariables(callable.doc());
+ return prefixWarning + callable.doc();
}
@Override
diff --git a/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkMethodDoc.java b/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkMethodDoc.java
index 1851367..768089e 100644
--- a/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkMethodDoc.java
+++ b/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkMethodDoc.java
@@ -24,6 +24,11 @@
/** An abstract class containing documentation for a Starlark method. */
public abstract class StarlarkMethodDoc extends StarlarkDoc {
+
+ public StarlarkMethodDoc(StarlarkDocExpander expander) {
+ super(expander);
+ }
+
/** Returns whether the Starlark method is documented. */
public abstract boolean documented();
diff --git a/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkParamDoc.java b/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkParamDoc.java
index 2eafd05..8cdefba 100644
--- a/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkParamDoc.java
+++ b/src/main/java/com/google/devtools/build/docgen/starlark/StarlarkParamDoc.java
@@ -21,7 +21,8 @@
private StarlarkMethodDoc method;
private Param param;
- public StarlarkParamDoc(StarlarkMethodDoc method, Param param) {
+ public StarlarkParamDoc(StarlarkMethodDoc method, Param param, StarlarkDocExpander expander) {
+ super(expander);
this.method = method;
this.param = param;
}
@@ -65,7 +66,7 @@
}
@Override
- public String getDocumentation() {
+ public String getRawDocumentation() {
String prefixWarning = "";
if (!param.enableOnlyWithFlag().isEmpty()) {
prefixWarning =
@@ -83,6 +84,6 @@
+ "</code>. Use this flag "
+ "to verify your code is compatible with its imminent removal. <br>";
}
- return prefixWarning + StarlarkDocUtils.substituteVariables(param.doc());
+ return prefixWarning + param.doc();
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD
index eafa7dc..11ccadd 100644
--- a/src/main/java/com/google/devtools/build/lib/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/BUILD
@@ -430,12 +430,15 @@
genrule(
name = "gen_buildencyclopedia",
- srcs = [":docs_embedded_in_sources"],
+ srcs = [
+ ":docs_embedded_in_sources",
+ "//src/main/java/com/google/devtools/build/docgen:bazel_link_map",
+ ],
outs = ["build-encyclopedia.zip"],
cmd = (
"mkdir -p $(@D)/be && " +
"$(location //src/main/java/com/google/devtools/build/docgen:docgen_bin)" +
- " --product_name=bazel" +
+ " --link_map_path=$(location //src/main/java/com/google/devtools/build/docgen:bazel_link_map)" +
" --provider=com.google.devtools.build.lib.bazel.rules.BazelRuleClassProvider" +
" --input_dir=$$PWD/src/main/java/com/google/devtools/build/lib" +
" --output_dir=$(@D)/be &&" +
@@ -476,9 +479,12 @@
genrule(
name = "gen_skylarklibrary",
+ srcs = ["//src/main/java/com/google/devtools/build/docgen:bazel_link_map"],
outs = ["skylark-library.zip"],
cmd = "mkdir -p $(@D)/skylark-lib &&" +
- "$(location //src/main/java/com/google/devtools/build/docgen:skydoc_bin) $(@D)/skylark-lib" +
+ "$(location //src/main/java/com/google/devtools/build/docgen:skydoc_bin)" +
+ " $(@D)/skylark-lib" +
+ " --link_map_path=$(location //src/main/java/com/google/devtools/build/docgen:bazel_link_map)" +
" && zip -qj $@ $(@D)/skylark-lib/*",
tools = ["//src/main/java/com/google/devtools/build/docgen:skydoc_bin"],
visibility = ["//site:__pkg__"],
