The BE generator now warns if rules are undocumented. It also caches files that it already processed, since that may legally happen if there are multiple overlapping directory trees are specified as input. -- MOS_MIGRATED_REVID=87492930
diff --git a/src/main/java/BUILD b/src/main/java/BUILD index 503426f..30c1a35 100644 --- a/src/main/java/BUILD +++ b/src/main/java/BUILD
@@ -65,7 +65,6 @@ java_binary( name = "docgen_bin", srcs = glob(["com/google/devtools/build/docgen/*.java"]), - data = [":gen_be_sources"], main_class = "com.google.devtools.build.docgen.BuildEncyclopediaGenerator", resources = glob( ["com/google/devtools/build/docgen/templates/*.html"], @@ -81,7 +80,9 @@ name = "gen_buildencyclopedia", srcs = [":gen_be_sources"], outs = ["build-encyclopedia.html"], - cmd = " docgen_bin $$PWD/src/main/java/com/google/devtools/build/lib $$PWD;" + - "cp $$PWD/build-encyclopedia.html $@", + cmd = ("$(location docgen_bin)" + + " $$PWD/java/com/google/devtools/build/lib" + + " $$PWD" + + "; cp $$PWD/build-encyclopedia.html $@"), tools = [":docgen_bin"], )
diff --git a/src/main/java/com/google/devtools/build/docgen/BlazeRuleHelpPrinter.java b/src/main/java/com/google/devtools/build/docgen/BlazeRuleHelpPrinter.java index 12687db..cec6502 100644 --- a/src/main/java/com/google/devtools/build/docgen/BlazeRuleHelpPrinter.java +++ b/src/main/java/com/google/devtools/build/docgen/BlazeRuleHelpPrinter.java
@@ -19,7 +19,6 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; -import java.util.Set; /** * A helper class to load and store printable build rule documentation. The doc @@ -37,11 +36,11 @@ if (ruleDocMap == null) { try { BuildEncyclopediaProcessor processor = new BuildEncyclopediaProcessor(provider); - Set<RuleDocumentation> ruleDocs = processor.collectAndProcessRuleDocs( + Map<String, RuleDocumentation> ruleDocs = processor.collectAndProcessRuleDocs( new String[] {"java/com/google/devtools/build/lib/view", "java/com/google/devtools/build/lib/rules"}, false); ruleDocMap = new HashMap<>(); - for (RuleDocumentation ruleDoc : ruleDocs) { + for (RuleDocumentation ruleDoc : ruleDocs.values()) { ruleDocMap.put(ruleDoc.getRuleName(), ruleDoc); } } catch (BuildEncyclopediaDocException e) {
diff --git a/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java index e80b2c5..8a1a02d 100644 --- a/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java +++ b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java
@@ -13,11 +13,16 @@ // limitations under the License. package com.google.devtools.build.docgen; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.LinkedListMultimap; import com.google.common.collect.ListMultimap; +import com.google.common.collect.Ordering; +import com.google.common.collect.Sets; import com.google.devtools.build.docgen.DocgenConsts.RuleType; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.RuleDefinition; @@ -29,6 +34,7 @@ import java.io.FileWriter; import java.io.IOException; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -41,6 +47,12 @@ * generates the html format documentation. */ public class BuildEncyclopediaProcessor { + private static final Predicate<String> RULE_WORTH_DOCUMENTING = new Predicate<String>() { + @Override + public boolean apply(String name) { + return !name.contains("$"); + } + }; private ConfiguredRuleClassProvider ruleClassProvider; @@ -64,8 +76,10 @@ bw.write(DocgenConsts.HEADER_COMMENT); bw.write("\n"); // for the benefit of the block-beginning comment at the top of the template - Set<RuleDocumentation> ruleDocEntries = collectAndProcessRuleDocs(inputDirs, false); - writeRuleClassDocs(ruleDocEntries, bw); + Map<String, RuleDocumentation> ruleDocEntries = collectAndProcessRuleDocs(inputDirs, false); + warnAboutUndocumentedRules( + Sets.difference(ruleClassProvider.getRuleClassMap().keySet(), ruleDocEntries.keySet())); + writeRuleClassDocs(ruleDocEntries.values(), bw); bw.write("\n"); // for the benefit of the block-beginning comment at the top of the template bw.write(SourceFileReader.readTemplateContents(DocgenConsts.FOOTER_TEMPLATE)); @@ -76,29 +90,39 @@ * Collects all the rule and attribute documentation present in inputDirs, integrates the * attribute documentation in the rule documentation and returns the rule documentation. */ - public Set<RuleDocumentation> collectAndProcessRuleDocs(String[] inputDirs, + public Map<String, RuleDocumentation> collectAndProcessRuleDocs(String[] inputDirs, boolean printMessages) throws BuildEncyclopediaDocException, IOException { // RuleDocumentations are generated in order (based on rule type then alphabetically). // The ordering is also used to determine in which rule doc the common attribute docs are // generated (they are generated at the first appearance). - Set<RuleDocumentation> ruleDocEntries = new TreeSet<>(); + Map<String, RuleDocumentation> ruleDocEntries = new TreeMap<>(); // RuleDocumentationAttribute objects equal based on attributeName so they have to be // collected in a List instead of a Set. ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries = LinkedListMultimap.create(); + + // Map of rule class name to file that defined it. + Map<String, File> ruleClassFiles = new HashMap<>(); + + // Set of files already processed. The same file may be encountered multiple times because + // directories are processed recursively, and an input directory may be a subdirectory of + // another one. + Set<File> processedFiles = new HashSet<>(); + for (String inputDir : inputDirs) { if (printMessages) { System.out.println(" Processing input directory: " + inputDir); } int ruleNum = ruleDocEntries.size(); - collectDocs(ruleDocEntries, attributeDocEntries, new File(inputDir)); + collectDocs(processedFiles, ruleClassFiles, ruleDocEntries, attributeDocEntries, + new File(inputDir)); if (printMessages) { System.out.println( " " + (ruleDocEntries.size() - ruleNum) + " rule documentations found."); } } - processAttributeDocs(ruleDocEntries, attributeDocEntries); + processAttributeDocs(ruleDocEntries.values(), attributeDocEntries); return ruleDocEntries; } @@ -108,7 +132,7 @@ * java_library.deps documented in $rule and $java_rule then the one in $java_rule is going to * apply since it's a closer ancestor of java_library. */ - private void processAttributeDocs(Set<RuleDocumentation> ruleDocEntries, + private void processAttributeDocs(Iterable<RuleDocumentation> ruleDocEntries, ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries) throws BuildEncyclopediaDocException { for (RuleDocumentation ruleDoc : ruleDocEntries) { @@ -159,7 +183,7 @@ /** * Categorizes, checks and prints all the rule-class documentations. */ - private void writeRuleClassDocs(Set<RuleDocumentation> docEntries, BufferedWriter bw) + private void writeRuleClassDocs(Iterable<RuleDocumentation> docEntries, BufferedWriter bw) throws BuildEncyclopediaDocException, IOException { Set<RuleDocumentation> binaryDocs = new TreeSet<>(); Set<RuleDocumentation> libraryDocs = new TreeSet<>(); @@ -205,7 +229,7 @@ bw.write(SourceFileReader.readTemplateContents(DocgenConsts.BODY_TEMPLATE, sectionMapping)); } - private Map<String, String> generateBEHeaderMapping(Set<RuleDocumentation> docEntries) + private Map<String, String> generateBEHeaderMapping(Iterable<RuleDocumentation> docEntries) throws BuildEncyclopediaDocException { StringBuilder sb = new StringBuilder(); @@ -248,7 +272,7 @@ /** * Create a mapping of rules based on rule type and family. */ - private void createRuleMapping(Set<RuleDocumentation> docEntries, + private void createRuleMapping(Iterable<RuleDocumentation> docEntries, Map<String, ListMultimap<RuleType, RuleDocumentation>> ruleMapping) throws BuildEncyclopediaDocException { for (RuleDocumentation ruleDoc : docEntries) { @@ -270,7 +294,7 @@ /** * Separates all rule families in docEntries into language-specific rules and generic rules. */ - private void separateRuleFamilies(Set<RuleDocumentation> docEntries, + private void separateRuleFamilies(Iterable<RuleDocumentation> docEntries, Set<String> languageSpecificRuleFamilies, Set<String> genericRuleFamilies) throws BuildEncyclopediaDocException { for (RuleDocumentation ruleDoc : docEntries) { @@ -290,7 +314,7 @@ } } - private String generateLeftNavigationPanel(Set<RuleDocumentation> docEntries) { + private String generateLeftNavigationPanel(Iterable<RuleDocumentation> docEntries) { // Order the rules alphabetically. At this point they are ordered according to // RuleDocumentation.compareTo() which is not alphabetical. TreeMap<String, String> ruleNames = new TreeMap<>(); @@ -367,15 +391,32 @@ * Goes through all the html files and subdirs under inputPath and collects the rule * and attribute documentations using the ruleDocEntries and attributeDocEntries variable. */ - public void collectDocs(Set<RuleDocumentation> ruleDocEntries, + public void collectDocs( + Set<File> processedFiles, + Map<String, File> ruleClassFiles, + Map<String, RuleDocumentation> ruleDocEntries, ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries, File inputPath) throws BuildEncyclopediaDocException, IOException { + if (processedFiles.contains(inputPath)) { + return; + } + if (inputPath.isFile()) { if (DocgenConsts.JAVA_SOURCE_FILE_SUFFIX.apply(inputPath.getName())) { SourceFileReader sfr = new SourceFileReader( ruleClassProvider, inputPath.getAbsolutePath()); sfr.readDocsFromComments(); - ruleDocEntries.addAll(sfr.getRuleDocEntries()); + for (RuleDocumentation d : sfr.getRuleDocEntries()) { + String ruleName = d.getRuleName(); + if (ruleDocEntries.containsKey(ruleName) + && !ruleClassFiles.get(ruleName).equals(inputPath)) { + System.err.printf("WARNING: '%s' from '%s' already in map from '%s', skipping\n", + d.getRuleName(), inputPath, ruleClassFiles.get(ruleName)); + } else { + ruleClassFiles.put(ruleName, inputPath); + ruleDocEntries.put(ruleName, d); + } + } if (attributeDocEntries != null) { // Collect all attribute documentations from this file. attributeDocEntries.putAll(sfr.getAttributeDocEntries()); @@ -383,9 +424,11 @@ } } else if (inputPath.isDirectory()) { for (File childPath : inputPath.listFiles()) { - collectDocs(ruleDocEntries, attributeDocEntries, childPath); + collectDocs(processedFiles, ruleClassFiles, ruleDocEntries, attributeDocEntries, childPath); } } + + processedFiles.add(inputPath); } private File setupDirectories(String outputRootDir) { @@ -397,4 +440,11 @@ return new File(DocgenConsts.BUILD_ENCYCLOPEDIA_NAME); } } + + private static void warnAboutUndocumentedRules(Iterable<String> rulesWithoutDocumentation) { + Iterable<String> undocumentedRules = Iterables.filter(rulesWithoutDocumentation, + RULE_WORTH_DOCUMENTING); + System.err.printf("WARNING: The following rules are undocumented: [%s]\n", + Joiner.on(", ").join(Ordering.<String>natural().immutableSortedCopy(undocumentedRules))); + } }