| // Copyright 2014 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package com.google.devtools.build.docgen; |
| |
| import static java.nio.charset.StandardCharsets.UTF_8; |
| |
| import com.google.auto.value.AutoValue; |
| import com.google.common.annotations.VisibleForTesting; |
| import com.google.common.base.Ascii; |
| import com.google.common.base.Splitter; |
| import com.google.common.base.Strings; |
| import com.google.common.collect.ImmutableSet; |
| import com.google.common.collect.LinkedListMultimap; |
| import com.google.common.collect.ListMultimap; |
| import com.google.common.flogger.GoogleLogger; |
| import com.google.devtools.build.docgen.DocgenConsts.RuleType; |
| import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; |
| import com.google.devtools.build.lib.analysis.RuleDefinition; |
| import com.google.devtools.build.lib.packages.Attribute; |
| import com.google.devtools.build.lib.packages.RuleClass; |
| import com.google.devtools.build.lib.starlarkdocextract.StardocOutputProtos.AttributeInfo; |
| import com.google.devtools.build.lib.starlarkdocextract.StardocOutputProtos.ModuleInfo; |
| import com.google.devtools.build.lib.starlarkdocextract.StardocOutputProtos.RuleInfo; |
| import com.google.protobuf.ExtensionRegistry; |
| import java.io.BufferedReader; |
| import java.io.File; |
| import java.io.FileInputStream; |
| import java.io.IOException; |
| import java.nio.file.Files; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Set; |
| import java.util.TreeMap; |
| import java.util.regex.Matcher; |
| |
| /** |
| * Class that parses the documentation fragments of rule-classes and |
| * generates the html format documentation. |
| */ |
| @VisibleForTesting |
| public class BuildDocCollector { |
| private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); |
| |
| private static final Splitter SHARP_SPLITTER = Splitter.on('#').limit(2).trimResults(); |
| |
| private final RuleLinkExpander linkExpander; |
| private final SourceUrlMapper urlMapper; |
| private final ConfiguredRuleClassProvider ruleClassProvider; |
| |
| public BuildDocCollector( |
| RuleLinkExpander linkExpander, |
| SourceUrlMapper urlMapper, |
| ConfiguredRuleClassProvider ruleClassProvider) { |
| this.linkExpander = linkExpander; |
| this.urlMapper = urlMapper; |
| this.ruleClassProvider = ruleClassProvider; |
| } |
| |
| /** |
| * Parse the file containing rules blocked from documentation. The list is simply a list of rules |
| * separated by new lines. Line comments can be added to the file by starting them with #. |
| * |
| * @param denyList The name of the file containing the denylist. |
| * @return The set of denylisted rules. |
| * @throws IOException |
| */ |
| @VisibleForTesting |
| public static Set<String> readDenyList(String denyList) throws IOException { |
| Set<String> result = new HashSet<String>(); |
| if (denyList != null && !denyList.isEmpty()) { |
| File file = new File(denyList); |
| try (BufferedReader reader = Files.newBufferedReader(file.toPath(), UTF_8)) { |
| for (String line = reader.readLine(); line != null; line = reader.readLine()) { |
| String rule = SHARP_SPLITTER.split(line).iterator().next(); |
| if (!rule.isEmpty()) { |
| result.add(rule); |
| } |
| } |
| } |
| } |
| return result; |
| } |
| |
| /** |
| * Creates a map of rule names (keys) to rule documentation (values). |
| * |
| * <p>This method crawls the specified input directories for rule class definitions (as Java |
| * source files) which contain the rules' and attributes' definitions as comments in a specific |
| * format. The keys in the returned Map correspond to these rule classes. |
| * |
| * <p>In the Map's values, all references pointing to other rules, rule attributes, and general |
| * documentation (e.g. common definitions, make variables, etc.) are expanded into hyperlinks. The |
| * links generated follow either the multi-page or single-page Build Encyclopedia model depending |
| * on the mode set for the {@link RuleLinkExpander} that was passed to the constructor. |
| * |
| * @param inputJavaDirs list of directories to scan for documentation in Java source code |
| * @param inputStardocProtos list of file paths of stardoc_output.ModuleInfo binary proto files |
| * generated from Build Encyclopedia entry point .bzl files; documentation from these protos |
| * takes precedence over documentation from {@code inputJavaDirs} |
| * @param denyList specify an optional denylist file that list some rules that should not be |
| * listed in the output. |
| * @throws BuildEncyclopediaDocException |
| * @throws IOException |
| * @return Map of rule class to rule documentation. |
| */ |
| public Map<String, RuleDocumentation> collect( |
| List<String> inputJavaDirs, List<String> inputStardocProtos, String denyList) |
| throws BuildEncyclopediaDocException, IOException { |
| // Read the denyList file |
| Set<String> denylistedRules = readDenyList(denyList); |
| // RuleDocumentations are generated in order (based on rule type then alphabetically). |
| // The ordering is also used to determine in which rule doc the common attribute docs are |
| // generated (they are generated at the first appearance). |
| Map<String, RuleDocumentation> ruleDocEntries = new TreeMap<>(); |
| // RuleDocumentationAttribute objects equal based on attributeName so they have to be |
| // collected in a List instead of a Set. |
| ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries = |
| LinkedListMultimap.create(); |
| |
| // Map of rule name to the file (Java source file or Build Encyclopedia entry point .bzl file) |
| // and symbol from which its documentation was obtained. |
| Map<String, DocumentationOrigin> ruleDocOrigin = new HashMap<>(); |
| |
| // Set of files already processed. The same file may be encountered multiple times because |
| // directories are processed recursively, and an input directory may be a subdirectory of |
| // another one. |
| Set<File> processedFiles = new HashSet<>(); |
| |
| for (String inputJavaDir : inputJavaDirs) { |
| logger.atFine().log("Processing input directory: %s", inputJavaDir); |
| int ruleNum = ruleDocEntries.size(); |
| collectJavaSourceDocs( |
| processedFiles, |
| ruleDocOrigin, |
| ruleDocEntries, |
| denylistedRules, |
| attributeDocEntries, |
| new File(inputJavaDir)); |
| logger.atFine().log( |
| "%d rule documentations found in %s", ruleDocEntries.size() - ruleNum, inputJavaDir); |
| } |
| processJavaSourceRuleAttributeDocs(ruleDocEntries.values(), attributeDocEntries); |
| |
| for (String stardocProtoPath : inputStardocProtos) { |
| logger.atFine().log("Processing input file: %s", stardocProtoPath); |
| int numRulesCollected = |
| collectModuleInfoDocs( |
| ruleDocOrigin, |
| ruleDocEntries, |
| denylistedRules, |
| attributeDocEntries, |
| ModuleInfo.parseFrom( |
| new FileInputStream(stardocProtoPath), ExtensionRegistry.getEmptyRegistry()), |
| urlMapper); |
| logger.atFine().log( |
| "%d rule documentations found in %s", numRulesCollected, stardocProtoPath); |
| } |
| |
| linkExpander.addIndex(buildRuleIndex(ruleDocEntries.values())); |
| for (RuleDocumentation rule : ruleDocEntries.values()) { |
| rule.setRuleLinkExpander(linkExpander); |
| } |
| return ruleDocEntries; |
| } |
| |
| /** |
| * Generates an index mapping rule name to its normalized rule family name. |
| */ |
| private Map<String, String> buildRuleIndex(Iterable<RuleDocumentation> rules) { |
| Map<String, String> index = new HashMap<>(); |
| for (RuleDocumentation rule : rules) { |
| index.put(rule.getRuleName(), RuleFamily.normalize(rule.getRuleFamily())); |
| } |
| return index; |
| } |
| |
| /** |
| * Go through all attributes of native rules whose documentation was retrieved from Java sources, |
| * and search the best attribute documentation if exists. The best documentation is the closest |
| * documentation in the ancestor graph. E.g. if java_library.deps documented in $rule and |
| * $java_rule then the one in $java_rule is going to apply since it's a closer ancestor of |
| * java_library. |
| * |
| * <p>Note: this function should be called before any calls to collectModuleInfoDocs. |
| */ |
| private void processJavaSourceRuleAttributeDocs( |
| Iterable<RuleDocumentation> ruleDocEntries, |
| ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries) |
| throws BuildEncyclopediaDocException { |
| for (RuleDocumentation ruleDoc : ruleDocEntries) { |
| RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(ruleDoc.getRuleName()); |
| if (ruleClass != null) { |
| if (ruleClass.isDocumented()) { |
| Class<? extends RuleDefinition> ruleDefinition = |
| ruleClassProvider.getRuleClassDefinition(ruleDoc.getRuleName()).getClass(); |
| for (Attribute attribute : ruleClass.getAttributes()) { |
| if (!attribute.isDocumented()) { |
| continue; |
| } |
| String attrName = attribute.getName(); |
| List<RuleDocumentationAttribute> attributeDocList = |
| attributeDocEntries.get(attrName); |
| if (attributeDocList != null) { |
| // There are attribute docs for this attribute. |
| // Search the closest one in the ancestor graph. |
| // Note that there can be only one 'closest' attribute since we forbid multiple |
| // inheritance of the same attribute in RuleClass. |
| int minLevel = Integer.MAX_VALUE; |
| RuleDocumentationAttribute bestAttributeDoc = null; |
| for (RuleDocumentationAttribute attributeDoc : attributeDocList) { |
| int level = attributeDoc.getDefinitionClassAncestryLevel( |
| ruleDefinition, |
| ruleClassProvider); |
| if (level >= 0 && level < minLevel) { |
| bestAttributeDoc = attributeDoc; |
| minLevel = level; |
| } |
| } |
| if (bestAttributeDoc != null) { |
| // We have to copy the matching RuleDocumentationAttribute here so that we don't |
| // overwrite the reference to the actual attribute later by another attribute with |
| // the same ancestor but different default values. |
| ruleDoc.addAttribute(bestAttributeDoc.copyAndUpdateFrom(attribute)); |
| // If there is no matching attribute doc try to add the common. |
| } else if (ruleDoc.getRuleType().equals(RuleType.BINARY) |
| && PredefinedAttributes.BINARY_ATTRIBUTES.containsKey(attrName)) { |
| ruleDoc.addAttribute(PredefinedAttributes.BINARY_ATTRIBUTES.get(attrName)); |
| } else if (ruleDoc.getRuleType().equals(RuleType.TEST) |
| && PredefinedAttributes.TEST_ATTRIBUTES.containsKey(attrName)) { |
| ruleDoc.addAttribute(PredefinedAttributes.TEST_ATTRIBUTES.get(attrName)); |
| } else if (PredefinedAttributes.COMMON_ATTRIBUTES.containsKey(attrName)) { |
| ruleDoc.addAttribute(PredefinedAttributes.COMMON_ATTRIBUTES.get(attrName)); |
| } else if (PredefinedAttributes.TYPICAL_ATTRIBUTES.containsKey(attrName)) { |
| ruleDoc.addAttribute(PredefinedAttributes.TYPICAL_ATTRIBUTES.get(attrName)); |
| } |
| } |
| } |
| } |
| } else { |
| throw ruleDoc.createException("Can't find RuleClass for " + ruleDoc.getRuleName()); |
| } |
| } |
| } |
| |
| /** |
| * Crawls the specified inputPath and collects the raw rule and rule attribute documentation. |
| * |
| * <p>This method crawls the specified input directory (recursively calling itself for all |
| * subdirectories) and reads each Java source file using {@link SourceFileReader} to extract the |
| * raw rule and attribute documentation embedded in comments in a specific format. The extracted |
| * documentation is then further processed, such as by {@link |
| * BuildDocCollector#collect(List<String>, String, RuleLinkExpander), collect}, in order to |
| * associate each rule's documentation with its attribute documentation. |
| * |
| * <p>This method returns the following through its parameters: the set of Java source files |
| * processed, a map of rule name to the source file it was extracted from, a map of rule name to |
| * the documentation to the rule, and a multimap of attribute name to attribute documentation. |
| * |
| * @param processedFiles The set of Java source files files that have already been processed in |
| * order to avoid reprocessing the same file. |
| * @param ruleDocOrigin Map of rule name to the file and symbol from which its documentation was |
| * obtained. |
| * @param ruleDocEntries Map of rule name to rule documentation. |
| * @param denyList The set of denylisted rules whose documentation should not be extracted. |
| * @param attributeDocEntries Multimap of rule attribute name to attribute documentation. |
| * @param inputPath The File representing the Java source file or directory to read. |
| * @throws BuildEncyclopediaDocException |
| * @throws IOException |
| */ |
| public void collectJavaSourceDocs( |
| Set<File> processedFiles, |
| Map<String, DocumentationOrigin> ruleDocOrigin, |
| Map<String, RuleDocumentation> ruleDocEntries, |
| Set<String> denyList, |
| ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries, |
| File inputPath) |
| throws BuildEncyclopediaDocException, IOException { |
| if (processedFiles.contains(inputPath)) { |
| return; |
| } |
| |
| if (inputPath.isFile()) { |
| if (DocgenConsts.JAVA_SOURCE_FILE_SUFFIX.apply(inputPath.getName())) { |
| SourceFileReader sfr = |
| new SourceFileReader( |
| ruleClassProvider, inputPath.getAbsolutePath(), urlMapper.urlOfFile(inputPath)); |
| sfr.readDocsFromComments(); |
| for (RuleDocumentation d : sfr.getRuleDocEntries()) { |
| String ruleName = d.getRuleName(); |
| if (!denyList.contains(ruleName)) { |
| if (ruleDocEntries.containsKey(ruleName) |
| && !ruleDocOrigin.get(ruleName).file().equals(inputPath.toString())) { |
| logger.atWarning().log( |
| "Rule '%s' from '%s' overrides previously seen rule '%s' from '%s'", |
| ruleName, |
| inputPath, |
| ruleDocOrigin.get(ruleName).symbol(), |
| ruleDocOrigin.get(ruleName).file()); |
| } |
| ruleDocOrigin.put(ruleName, DocumentationOrigin.create(inputPath.toString(), ruleName)); |
| ruleDocEntries.put(ruleName, d); |
| } |
| } |
| if (attributeDocEntries != null) { |
| // Collect all attribute documentations from this file. |
| attributeDocEntries.putAll(sfr.getAttributeDocEntries()); |
| } |
| } |
| } else if (inputPath.isDirectory()) { |
| for (File childPath : inputPath.listFiles()) { |
| collectJavaSourceDocs( |
| processedFiles, |
| ruleDocOrigin, |
| ruleDocEntries, |
| denyList, |
| attributeDocEntries, |
| childPath); |
| } |
| } |
| |
| processedFiles.add(inputPath); |
| } |
| |
| /** |
| * Collects rule and rule attribute documentation from a stardoc_output.ModuleInfo message |
| * generated from a Build Encyclopedia entry point .bzl file. |
| * |
| * <p>The module doc string for the .bzl file is interpreted as the rule family name. |
| * |
| * <p>Any rule exported by the .bzl file is expected to be contained in a struct whose name is a |
| * {@link DocgenConsts.RuleType} name suffixed with "_rules" - for example, "binary_rules", |
| * "library_rules", etc. |
| * |
| * <p>This method returns the following through its parameters: a map of rule name to the file and |
| * symbol it was extracted from, a map of rule name to the documentation of the rule, and a |
| * multimap of attribute name to attribute documentation. |
| * |
| * @param ruleDocOrigin Map of rule name to the file and symbol from which its documentation was |
| * obtained. |
| * @param ruleDocEntries Map of rule name to rule documentation. |
| * @param denyList The set of denylisted rules whose documentation should not be extracted. |
| * @param attributeDocEntries Multimap of rule attribute name to attribute documentation. |
| * @param moduleInfo A stardoc_output.ModuleInfo message representing a Build Encyclopedia entry |
| * point .bzl file. |
| * @param urlMapper Mapper from source labels to source code repository URLs |
| * @return number of rules whose documentation was collected |
| */ |
| @VisibleForTesting |
| static int collectModuleInfoDocs( |
| Map<String, DocumentationOrigin> ruleDocOrigin, |
| Map<String, RuleDocumentation> ruleDocEntries, |
| Set<String> denyList, |
| ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries, |
| ModuleInfo moduleInfo, |
| SourceUrlMapper urlMapper) |
| throws BuildEncyclopediaDocException { |
| String entryPointFileLabel = moduleInfo.getFile(); |
| |
| Matcher familyMatcher = |
| DocgenConsts.STARDOC_OUTPUT_FAMILY_NAME_AND_SUMMARY.matcher( |
| moduleInfo.getModuleDocstring().strip()); |
| if (!familyMatcher.matches()) { |
| throw new BuildEncyclopediaDocException( |
| entryPointFileLabel, |
| "Module doc string is expected to be a single line representing a rule family name, " |
| + "optionally followed by a blank line and summary text; for example, " |
| + "`\"\"\"C / C++\"\"\"`"); |
| } |
| String ruleFamily = familyMatcher.group("family"); |
| String ruleFamilySummary = Strings.nullToEmpty(familyMatcher.group("summary")); |
| |
| int numRulesCollected = 0; |
| for (RuleInfo ruleInfo : moduleInfo.getRuleInfoList()) { |
| Matcher ruleNameMatcher = |
| DocgenConsts.STARDOC_OUTPUT_RULE_NAME.matcher(ruleInfo.getRuleName()); |
| if (!ruleNameMatcher.matches()) { |
| throw new BuildEncyclopediaDocException( |
| entryPointFileLabel, |
| String.format( |
| "Unexpected rule symbol: %s; rules must be exported in structs, with the struct's" |
| + " name specifying the rule type, for example, `library_rules = struct(" |
| + "java_import = _java_import, ...)`", |
| ruleInfo.getRuleName())); |
| } |
| String ruleType = Ascii.toUpperCase(ruleNameMatcher.group("type")); |
| String ruleName = ruleNameMatcher.group("name"); |
| if (!denyList.contains(ruleName)) { |
| String ruleOriginFileLabel = ruleInfo.getOriginKey().getFile(); |
| if (ruleDocEntries.containsKey(ruleName)) { |
| logger.atWarning().log( |
| "Rule '%s' from '%s' (defined in '%s') overrides previously seen rule '%s' from '%s'", |
| ruleInfo.getRuleName(), |
| entryPointFileLabel, |
| ruleOriginFileLabel, |
| ruleDocOrigin.get(ruleName).symbol(), |
| ruleDocOrigin.get(ruleName).file()); |
| } |
| ImmutableSet.Builder<String> flags = ImmutableSet.builder(); |
| if (ruleType.equals(DocgenConsts.STARLARK_GENERIC_RULE_TYPE)) { |
| // Note that if FLAG_GENERIC_RULE is set, RuleDocumentation constructor will set the rule |
| // type to OTHER. |
| flags.add(DocgenConsts.FLAG_GENERIC_RULE); |
| } |
| RuleDocumentation ruleDoc = |
| new RuleDocumentation( |
| ruleName, |
| ruleType, |
| ruleFamily, |
| ruleInfo.getDocString(), |
| ruleOriginFileLabel, |
| urlMapper.urlOfLabel(ruleOriginFileLabel), |
| flags.build(), |
| // Add family summary only to the first rule encountered, to avoid duplication in |
| // final rendered output |
| numRulesCollected == 0 ? ruleFamilySummary : ""); |
| |
| // Inject standard inherited attributes for Starlark rules (since they always inherit from |
| // one of 3 possible base rule classes; see StarlarkRuleClassFunctions#createRule). If in |
| // the future we want to document native rules via ModuleInfo protos, we will need to list |
| // inherited attributes in the proto. |
| ruleDoc.addAttributes(PredefinedAttributes.COMMON_ATTRIBUTES.values()); |
| if (ruleDoc.getRuleType().equals(RuleType.TEST)) { |
| ruleDoc.addAttributes(PredefinedAttributes.TEST_ATTRIBUTES.values()); |
| } else if (ruleDoc.getRuleType().equals(RuleType.BINARY)) { |
| ruleDoc.addAttributes(PredefinedAttributes.BINARY_ATTRIBUTES.values()); |
| } |
| |
| for (AttributeInfo attributeInfo : ruleInfo.getAttributeList()) { |
| String attributeName = attributeInfo.getName(); |
| if (attributeName.equals("name")) { |
| // We do not want the implicit "name" attribute injected into proto output by |
| // starlark_doc_extract because we inject "name" at the template level in |
| // templates/be/rules.vm |
| continue; |
| } |
| if (attributeInfo.getDocString().isEmpty() |
| && PredefinedAttributes.TYPICAL_ATTRIBUTES.containsKey(attributeName)) { |
| // We link empty-docstring attributes to the common table based purely on attribute name |
| // (same as processJavaSourceRuleAttributeDocs does for native rule attributes). |
| // TODO(arostovtsev): should we verify attribute type and default value too? That would |
| // require moving the definition of common attributes from a free-text velocity template |
| // to a structured format. |
| ruleDoc.addAttribute(PredefinedAttributes.TYPICAL_ATTRIBUTES.get(attributeName)); |
| } else { |
| boolean deprecated = |
| DocgenConsts.STARDOC_OUTPUT_DEPRECATED_DOCSTRING |
| .matcher(attributeInfo.getDocString()) |
| .find(); |
| ruleDoc.addAttribute( |
| RuleDocumentationAttribute.createFromAttributeInfo( |
| attributeInfo, |
| ruleOriginFileLabel, |
| deprecated |
| ? ImmutableSet.of(DocgenConsts.FLAG_DEPRECATED) |
| : ImmutableSet.of())); |
| } |
| } |
| |
| ruleDocOrigin.put( |
| ruleName, DocumentationOrigin.create(entryPointFileLabel, ruleInfo.getRuleName())); |
| ruleDocEntries.put(ruleName, ruleDoc); |
| numRulesCollected++; |
| } |
| } |
| return numRulesCollected; |
| } |
| |
| /** The file and symbol from which documentation was obtained. */ |
| @AutoValue |
| abstract static class DocumentationOrigin { |
| abstract String file(); |
| |
| abstract String symbol(); |
| |
| static DocumentationOrigin create(String file, String symbol) { |
| return new AutoValue_BuildDocCollector_DocumentationOrigin(file, symbol); |
| } |
| } |
| } |