src/main/java/com/google/devtools/build/docgen/SourceFileReader.java

// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.docgen;

import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.ListMultimap;
import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import javax.annotation.Nullable;

/**
 * A helper class to read and process documentations for rule classes and attributes
 * from exactly one java source file.
 */
public class SourceFileReader {

  private Collection<RuleDocumentation> ruleDocEntries;
  private ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries;
  private final ConfiguredRuleClassProvider ruleClassProvider;
  private final String javaSourceFilePath;
  private final String sourceUrl;

  public SourceFileReader(
      ConfiguredRuleClassProvider ruleClassProvider, String javaSourceFilePath, String sourceUrl) {
    this.ruleClassProvider = ruleClassProvider;
    this.javaSourceFilePath = javaSourceFilePath;
    this.sourceUrl = sourceUrl;
  }

  /**
   * The handler class of the line read from the text file.
   */
  public abstract static class ReadAction {

    // Text file line indexing starts from 1
    private int lineCnt = 1;

    protected abstract void readLineImpl(String line)
        throws BuildEncyclopediaDocException, IOException;

    protected int getLineCnt() {
      return lineCnt;
    }

    public void readLine(String line)
        throws BuildEncyclopediaDocException, IOException {
      readLineImpl(line);
      lineCnt++;
    }
  }

  private static final String LS = DocgenConsts.LS;

  /**
   * Reads the attribute and rule documentation present in the file represented by
   * SourceFileReader.javaSourceFilePath. The rule doc variables are added to the rule
   * documentation (which therefore must be defined in the same file). The attribute docs are
   * stored in a different class member, so they need to be handled outside this method.
   */
  public void readDocsFromComments() throws BuildEncyclopediaDocException, IOException {
    final Map<String, RuleDocumentation> docMap = new HashMap<>();
    final List<RuleDocumentationVariable> docVariables = new LinkedList<>();
    final ListMultimap<String, RuleDocumentationAttribute> docAttributes =
        LinkedListMultimap.create();
    readTextFile(
        javaSourceFilePath,
        new ReadAction() {

          private boolean inBlazeRuleDocs = false;
          private boolean inBlazeRuleVarDocs = false;
          private boolean inBlazeAttributeDocs = false;
          private boolean inFamilySummary = false;
          private StringBuilder sb = new StringBuilder();
          private String ruleName;
          private String familySummary = "";
          private String ruleType;
          private String ruleFamily;
          private String variableName;
          private String attributeName;
          private ImmutableSet<String> flags;
          private int startLineCnt;

          @Override
          public void readLineImpl(String line) throws BuildEncyclopediaDocException {
            // TODO(bazel-team): check if copy paste code can be reduced using inner classes
            if (inBlazeRuleDocs) {
              if (DocgenConsts.BLAZE_RULE_END.matcher(line).matches()) {
                endBlazeRuleDoc(docMap);
              } else {
                appendLine(line);
              }
            } else if (inBlazeRuleVarDocs) {
              if (DocgenConsts.BLAZE_RULE_VAR_END.matcher(line).matches()) {
                endBlazeRuleVarDoc(docVariables);
              } else {
                appendLine(line);
              }
            } else if (inBlazeAttributeDocs) {
              if (DocgenConsts.BLAZE_RULE_ATTR_END.matcher(line).matches()) {
                endBlazeAttributeDoc(docAttributes);
              } else {
                appendLine(line);
              }
            } else if (inFamilySummary) {
              if (DocgenConsts.FAMILY_SUMMARY_END.matcher(line).matches()) {
                endFamilySummary();
              } else {
                appendLine(line);
              }
            }
            Matcher familySummaryStartMatcher = DocgenConsts.FAMILY_SUMMARY_START.matcher(line);
            Matcher ruleStartMatcher = DocgenConsts.BLAZE_RULE_START.matcher(line);
            Matcher ruleVarStartMatcher = DocgenConsts.BLAZE_RULE_VAR_START.matcher(line);
            Matcher ruleAttrStartMatcher = DocgenConsts.BLAZE_RULE_ATTR_START.matcher(line);
            if (familySummaryStartMatcher.find()) {
              startFamilySummary();
            } else if (ruleStartMatcher.find()) {
              startBlazeRuleDoc(line, ruleStartMatcher);
            } else if (ruleVarStartMatcher.find()) {
              startBlazeRuleVarDoc(ruleVarStartMatcher);
            } else if (ruleAttrStartMatcher.find()) {
              startBlazeAttributeDoc(line, ruleAttrStartMatcher);
            }
          }

          private void appendLine(String line) {
            // Add another line of html code to the building rule documentation
            // Removing whitespace and java comment asterisk from the beginning of the line
            sb.append(line.replaceAll("^[\\s]*\\*", "") + LS);
          }

          private void startBlazeRuleDoc(String line, Matcher matcher)
              throws BuildEncyclopediaDocException {
            sb = new StringBuilder();
            checkDocValidity();
            // Start of a new rule.
            // e.g.: matcher.group(1) = "NAME = cc_binary, TYPE = BINARY, FAMILY = C / C++"
            for (String group : Splitter.on(",").split(matcher.group(1))) {
              List<String> parts = Splitter.on("=").limit(2).splitToList(group);
              boolean good = false;
              if (parts.size() == 2) {
                String key = parts.get(0).trim();
                String value = parts.get(1).trim();
                good = true;
                if (DocgenConsts.META_KEY_NAME.equals(key)) {
                  ruleName = value;
                } else if (DocgenConsts.META_KEY_TYPE.equals(key)) {
                  ruleType = value;
                } else if (DocgenConsts.META_KEY_FAMILY.equals(key)) {
                  ruleFamily = value;
                } else {
                  good = false;
                }
              }
              if (!good) {
                System.err.printf(
                    "WARNING: bad rule definition in line %d: '%s'", getLineCnt(), line);
              }
            }

            startLineCnt = getLineCnt();
            addFlags(line);
            inBlazeRuleDocs = true;
          }

          private void startFamilySummary() {
            sb = new StringBuilder();
            inFamilySummary = true;
          }

          private void endFamilySummary() {
            familySummary = sb.toString();
          }

          private void endBlazeRuleDoc(final Map<String, RuleDocumentation> documentations)
              throws BuildEncyclopediaDocException {
            // End of a rule, create RuleDocumentation object
            documentations.put(
                ruleName,
                new RuleDocumentation(
                    ruleName,
                    ruleType,
                    ruleFamily,
                    sb.toString(),
                    javaSourceFilePath,
                    getLineCnt(),
                    sourceUrl,
                    flags,
                    familySummary));
            sb = new StringBuilder();
            inBlazeRuleDocs = false;
          }

          private void startBlazeRuleVarDoc(Matcher matcher) throws BuildEncyclopediaDocException {
            checkDocValidity();
            // Start of a new rule variable
            ruleName = matcher.group(1).replaceAll("[\\s]", "");
            variableName = matcher.group(2).replaceAll("[\\s]", "");
            startLineCnt = getLineCnt();
            inBlazeRuleVarDocs = true;
          }

          private void endBlazeRuleVarDoc(final List<RuleDocumentationVariable> docVariables) {
            // End of a rule, create RuleDocumentationVariable object
            docVariables.add(
                new RuleDocumentationVariable(ruleName, variableName, sb.toString(), startLineCnt));
            sb = new StringBuilder();
            inBlazeRuleVarDocs = false;
          }

          private void startBlazeAttributeDoc(String line, Matcher matcher)
              throws BuildEncyclopediaDocException {
            checkDocValidity();
            // Start of a new attribute
            ruleName = matcher.group(1).replaceAll("[\\s]", "");
            attributeName = matcher.group(2).replaceAll("[\\s]", "");
            startLineCnt = getLineCnt();
            addFlags(line);
            inBlazeAttributeDocs = true;
          }

          private void endBlazeAttributeDoc(
              final ListMultimap<String, RuleDocumentationAttribute> docAttributes) {
            // End of a attribute, create RuleDocumentationAttribute object
            docAttributes.put(
                attributeName,
                RuleDocumentationAttribute.create(
                    ruleClassProvider.getRuleClassDefinition(ruleName).getClass(),
                    attributeName,
                    sb.toString(),
                    javaSourceFilePath,
                    startLineCnt,
                    flags));
            sb = new StringBuilder();
            inBlazeAttributeDocs = false;
          }

          private void addFlags(String line) {
            // Add flags if there's any
            Matcher matcher = DocgenConsts.BLAZE_RULE_FLAGS.matcher(line);
            if (matcher.find()) {
              flags = ImmutableSet.<String>copyOf(matcher.group(1).split(","));
            } else {
              flags = ImmutableSet.<String>of();
            }
          }

          private void checkDocValidity() throws BuildEncyclopediaDocException {
            if (inBlazeRuleDocs || inBlazeRuleVarDocs || inBlazeAttributeDocs) {
              throw new BuildEncyclopediaDocException(
                  javaSourceFilePath,
                  getLineCnt(),
                  "Malformed documentation, #BLAZE_RULE started after another #BLAZE_RULE.");
            }
          }
        });

    // Adding rule doc variables to the corresponding rules
    for (RuleDocumentationVariable docVariable : docVariables) {
      if (docMap.containsKey(docVariable.getRuleName())) {
        docMap.get(docVariable.getRuleName()).addDocVariable(
          docVariable.getVariableName(), docVariable.getValue());
      } else {
        throw new BuildEncyclopediaDocException(javaSourceFilePath, docVariable.getStartLineCnt(),
            String.format("Malformed rule variable #BLAZE_RULE(%s).%s, rule %s not found in file.",
                docVariable.getRuleName(), docVariable.getVariableName(),
                docVariable.getRuleName()));
      }
    }
    ruleDocEntries = docMap.values();
    attributeDocEntries = docAttributes;
  }

  public Collection<RuleDocumentation> getRuleDocEntries() {
    return ruleDocEntries;
  }

  public ListMultimap<String, RuleDocumentationAttribute> getAttributeDocEntries() {
    return attributeDocEntries;
  }

  /**
   * Reads the template file without variable substitution.
   */
  public static String readTemplateContents(String templateFilePath)
      throws BuildEncyclopediaDocException, IOException {
    return readTemplateContents(templateFilePath, null);
  }

  /**
   * Reads a template file and substitutes variables of the format ${FOO}.
   *
   * @param variables keys are the possible variable names, e.g. "FOO", values are the substitutions
   *     (can be null)
   */
  public static String readTemplateContents(String templateFilePath,
      final Map<String, String> variables) throws BuildEncyclopediaDocException, IOException {
    final StringBuilder sb = new StringBuilder();
    readTextFile(templateFilePath, new ReadAction() {
      @Override
      public void readLineImpl(String line) {
        sb.append(expandVariables(line, variables)).append(LS);
      }
    });
    return sb.toString();
  }

  private static String expandVariables(String line, Map<String, String> variables) {
    if (variables == null || line.indexOf("${") == -1) {
      return line;
    }

    for (Map.Entry<String, String> variable : variables.entrySet()) {
      line = line.replace("${" + variable.getKey() + "}", variable.getValue());
    }
    return line;
  }

  @Nullable
  private static BufferedReader createReader(String filePath) throws IOException {
    File file = new File(filePath);
    if (file.exists()) {
      return Files.newBufferedReader(file.toPath(), StandardCharsets.UTF_8);
    } else {
      InputStream is = SourceFileReader.class.getResourceAsStream(filePath);
      if (is != null) {
        return new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
      } else {
        return null;
      }
    }
  }

  public static void readTextFile(String filePath, ReadAction action)
      throws BuildEncyclopediaDocException, IOException {
    try (BufferedReader br = createReader(filePath)) {
      if (br != null) {
        String line = null;
        while ((line = br.readLine()) != null) {
          action.readLine(line);
        }
      } else {
        System.out.println("Couldn't find file or resource: " + filePath);
      }
    }
  }
}