src/main/java/com/google/devtools/build/docgen/BuildDocCollector.java

// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.docgen;

import static java.nio.charset.StandardCharsets.UTF_8;

import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Ascii;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.ListMultimap;
import com.google.common.flogger.GoogleLogger;
import com.google.devtools.build.docgen.DocgenConsts.RuleType;
import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
import com.google.devtools.build.lib.analysis.RuleDefinition;
import com.google.devtools.build.lib.packages.Attribute;
import com.google.devtools.build.lib.packages.RuleClass;
import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.AttributeInfo;
import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.ModuleInfo;
import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.RuleInfo;
import com.google.protobuf.ExtensionRegistry;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Matcher;

/**
 * Class that parses the documentation fragments of rule-classes and
 * generates the html format documentation.
 */
@VisibleForTesting
public class BuildDocCollector {
  private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();

  private static final Splitter SHARP_SPLITTER = Splitter.on('#').limit(2).trimResults();

  private final RuleLinkExpander linkExpander;
  private final SourceUrlMapper urlMapper;
  private final ConfiguredRuleClassProvider ruleClassProvider;

  public BuildDocCollector(
      RuleLinkExpander linkExpander,
      SourceUrlMapper urlMapper,
      ConfiguredRuleClassProvider ruleClassProvider) {
    this.linkExpander = linkExpander;
    this.urlMapper = urlMapper;
    this.ruleClassProvider = ruleClassProvider;
  }

  /**
   * Parse the file containing rules blocked from documentation. The list is simply a list of rules
   * separated by new lines. Line comments can be added to the file by starting them with #.
   *
   * @param denyList The name of the file containing the denylist.
   * @return The set of denylisted rules.
   * @throws IOException
   */
  @VisibleForTesting
  public static Set<String> readDenyList(String denyList) throws IOException {
    Set<String> result = new HashSet<String>();
    if (denyList != null && !denyList.isEmpty()) {
      File file = new File(denyList);
      try (BufferedReader reader = Files.newBufferedReader(file.toPath(), UTF_8)) {
        for (String line = reader.readLine(); line != null; line = reader.readLine()) {
          String rule = SHARP_SPLITTER.split(line).iterator().next();
          if (!rule.isEmpty()) {
            result.add(rule);
          }
        }
      }
    }
    return result;
  }

  /**
   * Creates a map of rule names (keys) to rule documentation (values).
   *
   * <p>This method crawls the specified input directories for rule class definitions (as Java
   * source files) which contain the rules' and attributes' definitions as comments in a specific
   * format. The keys in the returned Map correspond to these rule classes.
   *
   * <p>In the Map's values, all references pointing to other rules, rule attributes, and general
   * documentation (e.g. common definitions, make variables, etc.) are expanded into hyperlinks. The
   * links generated follow either the multi-page or single-page Build Encyclopedia model depending
   * on the mode set for the {@link RuleLinkExpander} that was passed to the constructor.
   *
   * @param inputJavaDirs list of directories to scan for documentation in Java source code
   * @param inputStardocProtos list of file paths of stardoc_output.ModuleInfo binary proto files
   *     generated from Build Encyclopedia entry point .bzl files; documentation from these protos
   *     takes precedence over documentation from {@code inputJavaDirs}
   * @param denyList specify an optional denylist file that list some rules that should not be
   *     listed in the output.
   * @throws BuildEncyclopediaDocException
   * @throws IOException
   * @return Map of rule class to rule documentation.
   */
  public Map<String, RuleDocumentation> collect(
      List<String> inputJavaDirs, List<String> inputStardocProtos, String denyList)
      throws BuildEncyclopediaDocException, IOException {
    // Read the denyList file
    Set<String> denylistedRules = readDenyList(denyList);
    // RuleDocumentations are generated in order (based on rule type then alphabetically).
    // The ordering is also used to determine in which rule doc the common attribute docs are
    // generated (they are generated at the first appearance).
    Map<String, RuleDocumentation> ruleDocEntries = new TreeMap<>();
    // RuleDocumentationAttribute objects equal based on attributeName so they have to be
    // collected in a List instead of a Set.
    ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries =
        LinkedListMultimap.create();

    // Map of rule name to the file (Java source file or Build Encyclopedia entry point .bzl file)
    // and symbol from which its documentation was obtained.
    Map<String, DocumentationOrigin> ruleDocOrigin = new HashMap<>();

    // Set of files already processed. The same file may be encountered multiple times because
    // directories are processed recursively, and an input directory may be a subdirectory of
    // another one.
    Set<File> processedFiles = new HashSet<>();

    for (String inputJavaDir : inputJavaDirs) {
      logger.atFine().log("Processing input directory: %s", inputJavaDir);
      int ruleNum = ruleDocEntries.size();
      collectJavaSourceDocs(
          processedFiles,
          ruleDocOrigin,
          ruleDocEntries,
          denylistedRules,
          attributeDocEntries,
          new File(inputJavaDir));
      logger.atFine().log(
          "%d rule documentations found in %s", ruleDocEntries.size() - ruleNum, inputJavaDir);
    }
    processJavaSourceRuleAttributeDocs(ruleDocEntries.values(), attributeDocEntries);

    for (String stardocProtoPath : inputStardocProtos) {
      logger.atFine().log("Processing input file: %s", stardocProtoPath);
      int numRulesCollected =
          collectModuleInfoDocs(
              ruleDocOrigin,
              ruleDocEntries,
              denylistedRules,
              attributeDocEntries,
              ModuleInfo.parseFrom(
                  new FileInputStream(stardocProtoPath), ExtensionRegistry.getEmptyRegistry()),
              urlMapper);
      logger.atFine().log(
          "%d rule documentations found in %s", numRulesCollected, stardocProtoPath);
    }

    linkExpander.addIndex(buildRuleIndex(ruleDocEntries.values()));
    for (RuleDocumentation rule : ruleDocEntries.values()) {
      rule.setRuleLinkExpander(linkExpander);
    }
    return ruleDocEntries;
  }

  /**
   * Generates an index mapping rule name to its normalized rule family name.
   */
  private Map<String, String> buildRuleIndex(Iterable<RuleDocumentation> rules) {
    Map<String, String> index = new HashMap<>();
    for (RuleDocumentation rule : rules) {
      index.put(rule.getRuleName(), RuleFamily.normalize(rule.getRuleFamily()));
    }
    return index;
  }

  /**
   * Go through all attributes of native rules whose documentation was retrieved from Java sources,
   * and search the best attribute documentation if exists. The best documentation is the closest
   * documentation in the ancestor graph. E.g. if java_library.deps documented in $rule and
   * $java_rule then the one in $java_rule is going to apply since it's a closer ancestor of
   * java_library.
   *
   * <p>Note: this function should be called before any calls to collectModuleInfoDocs.
   */
  private void processJavaSourceRuleAttributeDocs(
      Iterable<RuleDocumentation> ruleDocEntries,
      ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries)
      throws BuildEncyclopediaDocException {
    for (RuleDocumentation ruleDoc : ruleDocEntries) {
      RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(ruleDoc.getRuleName());
      if (ruleClass != null) {
        if (ruleClass.isDocumented()) {
          Class<? extends RuleDefinition> ruleDefinition =
              ruleClassProvider.getRuleClassDefinition(ruleDoc.getRuleName()).getClass();
          for (Attribute attribute : ruleClass.getAttributes()) {
            if (!attribute.isDocumented()) {
              continue;
            }
            String attrName = attribute.getName();
            List<RuleDocumentationAttribute> attributeDocList =
                attributeDocEntries.get(attrName);
            if (attributeDocList != null) {
              // There are attribute docs for this attribute.
              // Search the closest one in the ancestor graph.
              // Note that there can be only one 'closest' attribute since we forbid multiple
              // inheritance of the same attribute in RuleClass.
              int minLevel = Integer.MAX_VALUE;
              RuleDocumentationAttribute bestAttributeDoc = null;
              for (RuleDocumentationAttribute attributeDoc : attributeDocList) {
                int level = attributeDoc.getDefinitionClassAncestryLevel(
                    ruleDefinition,
                    ruleClassProvider);
                if (level >= 0 && level < minLevel) {
                  bestAttributeDoc = attributeDoc;
                  minLevel = level;
                }
              }
              if (bestAttributeDoc != null) {
                // We have to copy the matching RuleDocumentationAttribute here so that we don't
                // overwrite the reference to the actual attribute later by another attribute with
                // the same ancestor but different default values.
                ruleDoc.addAttribute(bestAttributeDoc.copyAndUpdateFrom(attribute));
              // If there is no matching attribute doc try to add the common.
              } else if (ruleDoc.getRuleType().equals(RuleType.BINARY)
                  && PredefinedAttributes.BINARY_ATTRIBUTES.containsKey(attrName)) {
                ruleDoc.addAttribute(PredefinedAttributes.BINARY_ATTRIBUTES.get(attrName));
              } else if (ruleDoc.getRuleType().equals(RuleType.TEST)
                  && PredefinedAttributes.TEST_ATTRIBUTES.containsKey(attrName)) {
                ruleDoc.addAttribute(PredefinedAttributes.TEST_ATTRIBUTES.get(attrName));
              } else if (PredefinedAttributes.COMMON_ATTRIBUTES.containsKey(attrName)) {
                ruleDoc.addAttribute(PredefinedAttributes.COMMON_ATTRIBUTES.get(attrName));
              } else if (PredefinedAttributes.TYPICAL_ATTRIBUTES.containsKey(attrName)) {
                ruleDoc.addAttribute(PredefinedAttributes.TYPICAL_ATTRIBUTES.get(attrName));
              }
            }
          }
        }
      } else {
        throw ruleDoc.createException("Can't find RuleClass for " + ruleDoc.getRuleName());
      }
    }
  }

  /**
   * Crawls the specified inputPath and collects the raw rule and rule attribute documentation.
   *
   * <p>This method crawls the specified input directory (recursively calling itself for all
   * subdirectories) and reads each Java source file using {@link SourceFileReader} to extract the
   * raw rule and attribute documentation embedded in comments in a specific format. The extracted
   * documentation is then further processed, such as by {@link
   * BuildDocCollector#collect(List<String>, String, RuleLinkExpander), collect}, in order to
   * associate each rule's documentation with its attribute documentation.
   *
   * <p>This method returns the following through its parameters: the set of Java source files
   * processed, a map of rule name to the source file it was extracted from, a map of rule name to
   * the documentation to the rule, and a multimap of attribute name to attribute documentation.
   *
   * @param processedFiles The set of Java source files files that have already been processed in
   *     order to avoid reprocessing the same file.
   * @param ruleDocOrigin Map of rule name to the file and symbol from which its documentation was
   *     obtained.
   * @param ruleDocEntries Map of rule name to rule documentation.
   * @param denyList The set of denylisted rules whose documentation should not be extracted.
   * @param attributeDocEntries Multimap of rule attribute name to attribute documentation.
   * @param inputPath The File representing the Java source file or directory to read.
   * @throws BuildEncyclopediaDocException
   * @throws IOException
   */
  public void collectJavaSourceDocs(
      Set<File> processedFiles,
      Map<String, DocumentationOrigin> ruleDocOrigin,
      Map<String, RuleDocumentation> ruleDocEntries,
      Set<String> denyList,
      ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries,
      File inputPath)
      throws BuildEncyclopediaDocException, IOException {
    if (processedFiles.contains(inputPath)) {
      return;
    }

    if (inputPath.isFile()) {
      if (DocgenConsts.JAVA_SOURCE_FILE_SUFFIX.apply(inputPath.getName())) {
        SourceFileReader sfr =
            new SourceFileReader(
                ruleClassProvider, inputPath.getAbsolutePath(), urlMapper.urlOfFile(inputPath));
        sfr.readDocsFromComments();
        for (RuleDocumentation d : sfr.getRuleDocEntries()) {
          String ruleName = d.getRuleName();
          if (!denyList.contains(ruleName)) {
            if (ruleDocEntries.containsKey(ruleName)
                && !ruleDocOrigin.get(ruleName).file().equals(inputPath.toString())) {
              logger.atWarning().log(
                  "Rule '%s' from '%s' overrides previously seen rule '%s' from '%s'",
                  ruleName,
                  inputPath,
                  ruleDocOrigin.get(ruleName).symbol(),
                  ruleDocOrigin.get(ruleName).file());
            }
            ruleDocOrigin.put(ruleName, DocumentationOrigin.create(inputPath.toString(), ruleName));
            ruleDocEntries.put(ruleName, d);
          }
        }
        if (attributeDocEntries != null) {
          // Collect all attribute documentations from this file.
          attributeDocEntries.putAll(sfr.getAttributeDocEntries());
        }
      }
    } else if (inputPath.isDirectory()) {
      for (File childPath : inputPath.listFiles()) {
        collectJavaSourceDocs(
            processedFiles,
            ruleDocOrigin,
            ruleDocEntries,
            denyList,
            attributeDocEntries,
            childPath);
      }
    }

    processedFiles.add(inputPath);
  }

  /**
   * Collects rule and rule attribute documentation from a stardoc_output.ModuleInfo message
   * generated from a Build Encyclopedia entry point .bzl file.
   *
   * <p>The module doc string for the .bzl file is interpreted as the rule family name.
   *
   * <p>Any rule exported by the .bzl file is expected to be contained in a struct whose name is a
   * {@link DocgenConsts.RuleType} name suffixed with "_rules" - for example, "binary_rules",
   * "library_rules", etc.
   *
   * <p>This method returns the following through its parameters: a map of rule name to the file and
   * symbol it was extracted from, a map of rule name to the documentation of the rule, and a
   * multimap of attribute name to attribute documentation.
   *
   * @param ruleDocOrigin Map of rule name to the file and symbol from which its documentation was
   *     obtained.
   * @param ruleDocEntries Map of rule name to rule documentation.
   * @param denyList The set of denylisted rules whose documentation should not be extracted.
   * @param attributeDocEntries Multimap of rule attribute name to attribute documentation.
   * @param moduleInfo A stardoc_output.ModuleInfo message representing a Build Encyclopedia entry
   *     point .bzl file.
   * @param urlMapper Mapper from source labels to source code repository URLs
   * @return number of rules whose documentation was collected
   */
  @VisibleForTesting
  static int collectModuleInfoDocs(
      Map<String, DocumentationOrigin> ruleDocOrigin,
      Map<String, RuleDocumentation> ruleDocEntries,
      Set<String> denyList,
      ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries,
      ModuleInfo moduleInfo,
      SourceUrlMapper urlMapper)
      throws BuildEncyclopediaDocException {
    String entryPointFileLabel = moduleInfo.getFile();

    Matcher familyMatcher =
        DocgenConsts.STARDOC_OUTPUT_FAMILY_NAME_AND_SUMMARY.matcher(
            moduleInfo.getModuleDocstring().strip());
    if (!familyMatcher.matches()) {
      throw new BuildEncyclopediaDocException(
          entryPointFileLabel,
          "Module doc string is expected to be a single line representing a rule family name, "
              + "optionally followed by a blank line and summary text; for example, "
              + "`\"\"\"C / C++\"\"\"`");
    }
    String ruleFamily = familyMatcher.group("family");
    String ruleFamilySummary = Strings.nullToEmpty(familyMatcher.group("summary"));

    int numRulesCollected = 0;
    for (RuleInfo ruleInfo : moduleInfo.getRuleInfoList()) {
      Matcher ruleNameMatcher =
          DocgenConsts.STARDOC_OUTPUT_RULE_NAME.matcher(ruleInfo.getRuleName());
      if (!ruleNameMatcher.matches()) {
        throw new BuildEncyclopediaDocException(
            entryPointFileLabel,
            String.format(
                "Unexpected rule symbol: %s; rules must be exported in structs, with the struct's"
                    + " name specifying the rule type, for example, `library_rules = struct("
                    + "java_import = _java_import, ...)`",
                ruleInfo.getRuleName()));
      }
      String ruleType = Ascii.toUpperCase(ruleNameMatcher.group("type"));
      String ruleName = ruleNameMatcher.group("name");
      if (!denyList.contains(ruleName)) {
        String ruleOriginFileLabel = ruleInfo.getOriginKey().getFile();
        if (ruleDocEntries.containsKey(ruleName)) {
          logger.atWarning().log(
              "Rule '%s' from '%s' (defined in '%s') overrides previously seen rule '%s' from '%s'",
              ruleInfo.getRuleName(),
              entryPointFileLabel,
              ruleOriginFileLabel,
              ruleDocOrigin.get(ruleName).symbol(),
              ruleDocOrigin.get(ruleName).file());
        }
        ImmutableSet.Builder<String> flags = ImmutableSet.builder();
        if (ruleType.equals(DocgenConsts.STARLARK_GENERIC_RULE_TYPE)) {
          // Note that if FLAG_GENERIC_RULE is set, RuleDocumentation constructor will set the rule
          // type to OTHER.
          flags.add(DocgenConsts.FLAG_GENERIC_RULE);
        }
        RuleDocumentation ruleDoc =
            new RuleDocumentation(
                ruleName,
                ruleType,
                ruleFamily,
                ruleInfo.getDocString(),
                ruleOriginFileLabel,
                urlMapper.urlOfLabel(ruleOriginFileLabel),
                flags.build(),
                // Add family summary only to the first rule encountered, to avoid duplication in
                // final rendered output
                numRulesCollected == 0 ? ruleFamilySummary : "");

        // Inject standard inherited attributes for Starlark rules (since they always inherit from
        // one of 3 possible base rule classes; see StarlarkRuleClassFunctions#createRule). If in
        // the future we want to document native rules via ModuleInfo protos, we will need to list
        // inherited attributes in the proto.
        ruleDoc.addAttributes(PredefinedAttributes.COMMON_ATTRIBUTES.values());
        if (ruleDoc.getRuleType().equals(RuleType.TEST)) {
          ruleDoc.addAttributes(PredefinedAttributes.TEST_ATTRIBUTES.values());
        } else if (ruleDoc.getRuleType().equals(RuleType.BINARY)) {
          ruleDoc.addAttributes(PredefinedAttributes.BINARY_ATTRIBUTES.values());
        }

        for (AttributeInfo attributeInfo : ruleInfo.getAttributeList()) {
          String attributeName = attributeInfo.getName();
          if (attributeName.equals("name")) {
            // We do not want the implicit "name" attribute injected into proto output by
            // starlark_doc_extract because we inject "name" at the template level in
            // templates/be/rules.vm
            continue;
          }
          if (attributeInfo.getDocString().isEmpty()
              && PredefinedAttributes.TYPICAL_ATTRIBUTES.containsKey(attributeName)) {
            // We link empty-docstring attributes to the common table based purely on attribute name
            // (same as processJavaSourceRuleAttributeDocs does for native rule attributes).
            // TODO(arostovtsev): should we verify attribute type and default value too? That would
            // require moving the definition of common attributes from a free-text velocity template
            // to a structured format.
            ruleDoc.addAttribute(PredefinedAttributes.TYPICAL_ATTRIBUTES.get(attributeName));
          } else {
            boolean deprecated =
                DocgenConsts.STARDOC_OUTPUT_DEPRECATED_DOCSTRING
                    .matcher(attributeInfo.getDocString())
                    .find();
            ruleDoc.addAttribute(
                RuleDocumentationAttribute.createFromAttributeInfo(
                    attributeInfo,
                    ruleOriginFileLabel,
                    deprecated
                        ? ImmutableSet.of(DocgenConsts.FLAG_DEPRECATED)
                        : ImmutableSet.of()));
          }
        }

        ruleDocOrigin.put(
            ruleName, DocumentationOrigin.create(entryPointFileLabel, ruleInfo.getRuleName()));
        ruleDocEntries.put(ruleName, ruleDoc);
        numRulesCollected++;
      }
    }
    return numRulesCollected;
  }

  /** The file and symbol from which documentation was obtained. */
  @AutoValue
  abstract static class DocumentationOrigin {
    abstract String file();

    abstract String symbol();

    static DocumentationOrigin create(String file, String symbol) {
      return new AutoValue_BuildDocCollector_DocumentationOrigin(file, symbol);
    }
  }
}