src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java - bazel - Git at Google

 // Copyright 2014 Google Inc. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 package com.google.devtools.build.docgen;

 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Predicate;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.LinkedListMultimap;
 import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Ordering;
 import com.google.common.collect.Sets;
 import com.google.devtools.build.docgen.DocgenConsts.RuleType;
 import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
 import com.google.devtools.build.lib.analysis.RuleDefinition;
 import com.google.devtools.build.lib.packages.Attribute;
 import com.google.devtools.build.lib.packages.RuleClass;

 import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;

 /**
  * A class to assemble documentation for the Build Encyclopedia. The
  * program parses the documentation fragments of rule-classes and
  * generates the html format documentation.
  */
 public class BuildEncyclopediaProcessor {
   private static final Predicate<String> RULE_WORTH_DOCUMENTING = new Predicate<String>() {
     @Override
     public boolean apply(String name) {
       return !name.contains("$");
     }
   };

   private ConfiguredRuleClassProvider ruleClassProvider;

   /**
    * Creates the BuildEncyclopediaProcessor instance. The ruleClassProvider parameter
    * is used for rule class hierarchy and attribute checking.
    *
    */
   public BuildEncyclopediaProcessor(ConfiguredRuleClassProvider ruleClassProvider) {
     this.ruleClassProvider = Preconditions.checkNotNull(ruleClassProvider);
   }

   /**
    * Collects and processes all the rule and attribute documentation in inputDirs and
    * generates the Build Encyclopedia into the outputRootDir.
    */
   public void generateDocumentation(String[] inputDirs, String outputRootDir)
       throws BuildEncyclopediaDocException, IOException {
     File buildEncyclopediaPath = setupDirectories(outputRootDir);
     try (BufferedWriter bw = new BufferedWriter(new FileWriter(buildEncyclopediaPath))) {
       bw.write(DocgenConsts.HEADER_COMMENT);
       bw.write("\n");  // for the benefit of the block-beginning comment at the top of the template

       Map<String, RuleDocumentation> ruleDocEntries = collectAndProcessRuleDocs(inputDirs, false);
       warnAboutUndocumentedRules(
           Sets.difference(ruleClassProvider.getRuleClassMap().keySet(), ruleDocEntries.keySet()));
       writeRuleClassDocs(ruleDocEntries.values(), bw);

       bw.write("\n");  // for the benefit of the block-beginning comment at the top of the template
       bw.write(SourceFileReader.readTemplateContents(DocgenConsts.FOOTER_TEMPLATE));
     }
   }

   /**
    * Collects all the rule and attribute documentation present in inputDirs, integrates the
    * attribute documentation in the rule documentation and returns the rule documentation.
    */
   public Map<String, RuleDocumentation> collectAndProcessRuleDocs(String[] inputDirs,
       boolean printMessages) throws BuildEncyclopediaDocException, IOException {
     // RuleDocumentations are generated in order (based on rule type then alphabetically).
     // The ordering is also used to determine in which rule doc the common attribute docs are
     // generated (they are generated at the first appearance).
     Map<String, RuleDocumentation> ruleDocEntries = new TreeMap<>();
     // RuleDocumentationAttribute objects equal based on attributeName so they have to be
     // collected in a List instead of a Set.
     ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries =
         LinkedListMultimap.create();

     // Map of rule class name to file that defined it.
     Map<String, File> ruleClassFiles = new HashMap<>();

     // Set of files already processed. The same file may be encountered multiple times because
     // directories are processed recursively, and an input directory may be a subdirectory of
     // another one.
     Set<File> processedFiles = new HashSet<>();

     for (String inputDir : inputDirs) {
       if (printMessages) {
         System.out.println(" Processing input directory: " + inputDir);
       }
       int ruleNum = ruleDocEntries.size();
       collectDocs(processedFiles, ruleClassFiles, ruleDocEntries, attributeDocEntries,
           new File(inputDir));
       if (printMessages) {
         System.out.println(
           " " + (ruleDocEntries.size() - ruleNum) + " rule documentations found.");
       }
     }

     processAttributeDocs(ruleDocEntries.values(), attributeDocEntries);
     return ruleDocEntries;
   }

   /**
    * Go through all attributes of all documented rules and search the best attribute documentation
    * if exists. The best documentation is the closest documentation in the ancestor graph. E.g. if
    * java_library.deps documented in $rule and $java_rule then the one in $java_rule is going to
    * apply since it's a closer ancestor of java_library.
    */
   private void processAttributeDocs(Iterable<RuleDocumentation> ruleDocEntries,
       ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries)
           throws BuildEncyclopediaDocException {
     for (RuleDocumentation ruleDoc : ruleDocEntries) {
       RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(ruleDoc.getRuleName());
       if (ruleClass != null) {
         if (ruleClass.isDocumented()) {
           Class<? extends RuleDefinition> ruleDefinition =
               ruleClassProvider.getRuleClassDefinition(ruleDoc.getRuleName());
           for (Attribute attribute : ruleClass.getAttributes()) {
             String attrName = attribute.getName();
             List<RuleDocumentationAttribute> attributeDocList =
                 attributeDocEntries.get(attrName);
             if (attributeDocList != null) {
               // There are attribute docs for this attribute.
               // Search the closest one in the ancestor graph.
               // Note that there can be only one 'closest' attribute since we forbid multiple
               // inheritance of the same attribute in RuleClass.
               int minLevel = Integer.MAX_VALUE;
               RuleDocumentationAttribute bestAttributeDoc = null;
               for (RuleDocumentationAttribute attributeDoc : attributeDocList) {
                 int level = attributeDoc.getDefinitionClassAncestryLevel(ruleDefinition);
                 if (level >= 0 && level < minLevel) {
                   bestAttributeDoc = attributeDoc;
                   minLevel = level;
                 }
               }
               if (bestAttributeDoc != null) {
                 ruleDoc.addAttribute(bestAttributeDoc);
               // If there is no matching attribute doc try to add the common.
               } else if (ruleDoc.getRuleType().equals(RuleType.BINARY)
                   && PredefinedAttributes.BINARY_ATTRIBUTES.containsKey(attrName)) {
                 ruleDoc.addAttribute(PredefinedAttributes.BINARY_ATTRIBUTES.get(attrName));
               } else if (ruleDoc.getRuleType().equals(RuleType.TEST)
                   && PredefinedAttributes.TEST_ATTRIBUTES.containsKey(attrName)) {
                 ruleDoc.addAttribute(PredefinedAttributes.TEST_ATTRIBUTES.get(attrName));
               } else if (PredefinedAttributes.COMMON_ATTRIBUTES.containsKey(attrName)) {
                 ruleDoc.addAttribute(PredefinedAttributes.COMMON_ATTRIBUTES.get(attrName));
               }
             }
           }
         }
       } else {
         throw ruleDoc.createException("Can't find RuleClass for " + ruleDoc.getRuleName());
       }
     }
   }

   /**
    * Categorizes, checks and prints all the rule-class documentations.
    */
   private void writeRuleClassDocs(Iterable<RuleDocumentation> docEntries, BufferedWriter bw)
       throws BuildEncyclopediaDocException, IOException {
     Set<RuleDocumentation> binaryDocs = new TreeSet<>();
     Set<RuleDocumentation> libraryDocs = new TreeSet<>();
     Set<RuleDocumentation> testDocs = new TreeSet<>();
     Set<RuleDocumentation> otherDocs = new TreeSet<>();

     for (RuleDocumentation doc : docEntries) {
       RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(doc.getRuleName());
       if (!ruleClass.isDocumented()) {
         continue;
       }

       if (doc.isLanguageSpecific()) {
         switch(doc.getRuleType()) {
           case BINARY:
             binaryDocs.add(doc);
             break;
           case LIBRARY:
             libraryDocs.add(doc);
             break;
           case TEST:
             testDocs.add(doc);
             break;
           case OTHER:
             otherDocs.add(doc);
             break;
         }
       } else {
         otherDocs.add(doc);
       }
     }

     bw.write("\n");  // for the benefit of the block-beginning comment at the top of the template
     bw.write(SourceFileReader.readTemplateContents(DocgenConsts.HEADER_TEMPLATE,
         generateBEHeaderMapping(docEntries)));

     Map<String, String> sectionMapping = ImmutableMap.of(
         DocgenConsts.VAR_SECTION_BINARY,   getRuleDocs(binaryDocs),
         DocgenConsts.VAR_SECTION_LIBRARY,  getRuleDocs(libraryDocs),
         DocgenConsts.VAR_SECTION_TEST,     getRuleDocs(testDocs),
         DocgenConsts.VAR_SECTION_OTHER,    getRuleDocs(otherDocs));
     bw.write("\n");  // for the benefit of the block-beginning comment at the top of the template
     bw.write(SourceFileReader.readTemplateContents(DocgenConsts.BODY_TEMPLATE, sectionMapping));
   }

   private Map<String, String> generateBEHeaderMapping(Iterable<RuleDocumentation> docEntries)
       throws BuildEncyclopediaDocException {
     StringBuilder sb = new StringBuilder();

     sb.append("<table id=\"rules\" summary=\"Table of rules sorted by language\">\n")
       .append("<colgroup span=\"5\" width=\"20%\"></colgroup>\n")
       .append("<tr><th>Language</th><th>Binary rules</th><th>Library rules</th>"
         + "<th>Test rules</th><th>Other rules</th><th></th></tr>\n");

     // Separate rule families into language-specific and generic ones.
     Set<String> languageSpecificRuleFamilies = new TreeSet<>();
     Set<String> genericRuleFamilies = new TreeSet<>();
     separateRuleFamilies(docEntries, languageSpecificRuleFamilies, genericRuleFamilies);

     // Create a mapping of rules based on rule type and family.
     Map<String, ListMultimap<RuleType, RuleDocumentation>> ruleMapping = new HashMap<>();
     createRuleMapping(docEntries, ruleMapping);

     // Generate the table.
     for (String ruleFamily : languageSpecificRuleFamilies) {
       generateHeaderTableRuleFamily(sb, ruleMapping.get(ruleFamily), ruleFamily);
     }

     sb.append("<tr><th>&nbsp;</th></tr>");
     sb.append("<tr><th colspan=\"5\">Rules that do not apply to a "
             + "specific programming language</th></tr>");
     for (String ruleFamily : genericRuleFamilies) {
       generateHeaderTableRuleFamily(sb, ruleMapping.get(ruleFamily), ruleFamily);
     }
     sb.append("</table>\n");
     return ImmutableMap.<String, String>of(DocgenConsts.VAR_HEADER_TABLE, sb.toString(),
         DocgenConsts.VAR_COMMON_ATTRIBUTE_DEFINITION, generateCommonAttributeDocs(
             PredefinedAttributes.COMMON_ATTRIBUTES, DocgenConsts.COMMON_ATTRIBUTES),
         DocgenConsts.VAR_TEST_ATTRIBUTE_DEFINITION, generateCommonAttributeDocs(
             PredefinedAttributes.TEST_ATTRIBUTES, DocgenConsts.TEST_ATTRIBUTES),
         DocgenConsts.VAR_BINARY_ATTRIBUTE_DEFINITION, generateCommonAttributeDocs(
             PredefinedAttributes.BINARY_ATTRIBUTES, DocgenConsts.BINARY_ATTRIBUTES),
         DocgenConsts.VAR_LEFT_PANEL, generateLeftNavigationPanel(docEntries));
   }

   /**
    * Create a mapping of rules based on rule type and family.
    */
   private void createRuleMapping(Iterable<RuleDocumentation> docEntries,
       Map<String, ListMultimap<RuleType, RuleDocumentation>> ruleMapping)
       throws BuildEncyclopediaDocException {
     for (RuleDocumentation ruleDoc : docEntries) {
       RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(ruleDoc.getRuleName());
       if (ruleClass != null) {
         String ruleFamily = ruleDoc.getRuleFamily();
         if (!ruleMapping.containsKey(ruleFamily)) {
           ruleMapping.put(ruleFamily, LinkedListMultimap.<RuleType, RuleDocumentation>create());
         }
         if (ruleClass.isDocumented()) {
           ruleMapping.get(ruleFamily).put(ruleDoc.getRuleType(), ruleDoc);
         }
       } else {
         throw ruleDoc.createException("Can't find RuleClass for " + ruleDoc.getRuleName());
       }
     }
   }

   /**
    * Separates all rule families in docEntries into language-specific rules and generic rules.
    */
   private void separateRuleFamilies(Iterable<RuleDocumentation> docEntries,
       Set<String> languageSpecificRuleFamilies, Set<String> genericRuleFamilies)
       throws BuildEncyclopediaDocException {
     for (RuleDocumentation ruleDoc : docEntries) {
       if (ruleDoc.isLanguageSpecific()) {
         if (genericRuleFamilies.contains(ruleDoc.getRuleFamily())) {
           throw ruleDoc.createException("The rule is marked as being language-specific, but other "
               + "rules of the same family have already been marked as being not.");
         }
         languageSpecificRuleFamilies.add(ruleDoc.getRuleFamily());
       } else {
         if (languageSpecificRuleFamilies.contains(ruleDoc.getRuleFamily())) {
           throw ruleDoc.createException("The rule is marked as being generic, but other rules of "
               + "the same family have already been marked as being language-specific.");
         }
         genericRuleFamilies.add(ruleDoc.getRuleFamily());
       }
     }
   }

   private String generateLeftNavigationPanel(Iterable<RuleDocumentation> docEntries) {
     // Order the rules alphabetically. At this point they are ordered according to
     // RuleDocumentation.compareTo() which is not alphabetical.
     TreeMap<String, String> ruleNames = new TreeMap<>();
     for (RuleDocumentation ruleDoc : docEntries) {
       String ruleName = ruleDoc.getRuleName();
       ruleNames.put(ruleName.toLowerCase(), ruleName);
     }
     StringBuilder sb = new StringBuilder();
     for (String ruleName : ruleNames.values()) {
       RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(ruleName);
       Preconditions.checkNotNull(ruleClass);
       if (ruleClass.isDocumented()) {
         sb.append(String.format("<a href=\"#%s\">%s</a><br/>\n", ruleName, ruleName));
       }
     }
     return sb.toString();
   }

   private String generateCommonAttributeDocs(Map<String, RuleDocumentationAttribute> attributes,
       String attributeGroupName) throws BuildEncyclopediaDocException {
     RuleDocumentation ruleDoc = new RuleDocumentation(
         attributeGroupName, "OTHER", null, null, 0, null, ImmutableSet.<String>of(),
         ruleClassProvider);
     for (RuleDocumentationAttribute attribute : attributes.values()) {
       ruleDoc.addAttribute(attribute);
     }
     return ruleDoc.generateAttributeDefinitions();
   }

   private void generateHeaderTableRuleFamily(StringBuilder sb,
       ListMultimap<RuleType, RuleDocumentation> ruleTypeMap, String ruleFamily) {
     sb.append("<tr>\n")
       .append(String.format("<td class=\"lang\">%s</td>\n", ruleFamily));
     boolean otherRulesSplitted = false;
     for (RuleType ruleType : DocgenConsts.RuleType.values()) {
       sb.append("<td>");
       int i = 0;
       List<RuleDocumentation> ruleDocList = ruleTypeMap.get(ruleType);
       for (RuleDocumentation ruleDoc : ruleDocList) {
         if (i > 0) {
           if (ruleType.equals(RuleType.OTHER)
               && ruleDocList.size() >= 4 && i == (ruleDocList.size() + 1) / 2) {
             // Split 'other rules' into two columns if there are too many of them.
             sb.append("</td>\n<td>");
             otherRulesSplitted = true;
           } else {
             sb.append("<br/>");
           }
         }
         String ruleName = ruleDoc.getRuleName();
         String deprecatedString = ruleDoc.hasFlag(DocgenConsts.FLAG_DEPRECATED)
             ? " class=\"deprecated\"" : "";
         sb.append(String.format("<a href=\"#%s\"%s>%s</a>", ruleName, deprecatedString, ruleName));
         i++;
       }
       sb.append("</td>\n");
     }
     // There should be 6 columns.
     if (!otherRulesSplitted) {
       sb.append("<td></td>\n");
     }
     sb.append("</tr>\n");
   }

   private String getRuleDocs(Iterable<RuleDocumentation> docEntries) {
     StringBuilder sb = new StringBuilder();
     for (RuleDocumentation doc : docEntries) {
       sb.append(doc.getHtmlDocumentation());
     }
     return sb.toString();
   }

   /**
    * Goes through all the html files and subdirs under inputPath and collects the rule
    * and attribute documentations using the ruleDocEntries and attributeDocEntries variable.
    */
   public void collectDocs(
       Set<File> processedFiles,
       Map<String, File> ruleClassFiles,
       Map<String, RuleDocumentation> ruleDocEntries,
       ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries,
       File inputPath) throws BuildEncyclopediaDocException, IOException {
     if (processedFiles.contains(inputPath)) {
       return;
     }

     if (inputPath.isFile()) {
       if (DocgenConsts.JAVA_SOURCE_FILE_SUFFIX.apply(inputPath.getName())) {
         SourceFileReader sfr = new SourceFileReader(
             ruleClassProvider, inputPath.getAbsolutePath());
         sfr.readDocsFromComments();
         for (RuleDocumentation d : sfr.getRuleDocEntries()) {
           String ruleName = d.getRuleName();
           if (ruleDocEntries.containsKey(ruleName)
               && !ruleClassFiles.get(ruleName).equals(inputPath)) {
             System.err.printf("WARNING: '%s' from '%s' overrides value already in map from '%s'\n",
                 d.getRuleName(), inputPath, ruleClassFiles.get(ruleName));
           }
           ruleClassFiles.put(ruleName, inputPath);
           ruleDocEntries.put(ruleName, d);
         }
         if (attributeDocEntries != null) {
           // Collect all attribute documentations from this file.
           attributeDocEntries.putAll(sfr.getAttributeDocEntries());
         }
       }
     } else if (inputPath.isDirectory()) {
       for (File childPath : inputPath.listFiles()) {
         collectDocs(processedFiles, ruleClassFiles, ruleDocEntries, attributeDocEntries, childPath);
       }
     }

     processedFiles.add(inputPath);
   }

   private File setupDirectories(String outputRootDir) {
     if (outputRootDir != null) {
       File outputRootPath = new File(outputRootDir);
       outputRootPath.mkdirs();
       return new File(outputRootDir + File.separator + DocgenConsts.BUILD_ENCYCLOPEDIA_NAME);
     } else {
       return new File(DocgenConsts.BUILD_ENCYCLOPEDIA_NAME);
     }
   }

   private static void warnAboutUndocumentedRules(Iterable<String> rulesWithoutDocumentation) {
       Iterable<String> undocumentedRules = Iterables.filter(rulesWithoutDocumentation,
           RULE_WORTH_DOCUMENTING);
       System.err.printf("WARNING: The following rules are undocumented: [%s]\n",
           Joiner.on(", ").join(Ordering.<String>natural().immutableSortedCopy(undocumentedRules)));
   }
 }
	// Copyright 2014 Google Inc. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.
	package com.google.devtools.build.docgen;

	import com.google.common.base.Joiner;
	import com.google.common.base.Preconditions;
	import com.google.common.base.Predicate;
	import com.google.common.collect.ImmutableMap;
	import com.google.common.collect.ImmutableSet;
	import com.google.common.collect.Iterables;
	import com.google.common.collect.LinkedListMultimap;
	import com.google.common.collect.ListMultimap;
	import com.google.common.collect.Ordering;
	import com.google.common.collect.Sets;
	import com.google.devtools.build.docgen.DocgenConsts.RuleType;
	import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
	import com.google.devtools.build.lib.analysis.RuleDefinition;
	import com.google.devtools.build.lib.packages.Attribute;
	import com.google.devtools.build.lib.packages.RuleClass;

	import java.io.BufferedWriter;
	import java.io.File;
	import java.io.FileWriter;
	import java.io.IOException;
	import java.util.HashMap;
	import java.util.HashSet;
	import java.util.List;
	import java.util.Map;
	import java.util.Set;
	import java.util.TreeMap;
	import java.util.TreeSet;

	/**
	* A class to assemble documentation for the Build Encyclopedia. The
	* program parses the documentation fragments of rule-classes and
	* generates the html format documentation.
	*/
	public class BuildEncyclopediaProcessor {
	private static final Predicate<String> RULE_WORTH_DOCUMENTING = new Predicate<String>() {
	@Override
	public boolean apply(String name) {
	return !name.contains("$");
	}
	};

	private ConfiguredRuleClassProvider ruleClassProvider;

	/**
	* Creates the BuildEncyclopediaProcessor instance. The ruleClassProvider parameter
	* is used for rule class hierarchy and attribute checking.
	*
	*/
	public BuildEncyclopediaProcessor(ConfiguredRuleClassProvider ruleClassProvider) {
	this.ruleClassProvider = Preconditions.checkNotNull(ruleClassProvider);
	}

	/**
	* Collects and processes all the rule and attribute documentation in inputDirs and
	* generates the Build Encyclopedia into the outputRootDir.
	*/
	public void generateDocumentation(String[] inputDirs, String outputRootDir)
	throws BuildEncyclopediaDocException, IOException {
	File buildEncyclopediaPath = setupDirectories(outputRootDir);
	try (BufferedWriter bw = new BufferedWriter(new FileWriter(buildEncyclopediaPath))) {
	bw.write(DocgenConsts.HEADER_COMMENT);
	bw.write("\n"); // for the benefit of the block-beginning comment at the top of the template

	Map<String, RuleDocumentation> ruleDocEntries = collectAndProcessRuleDocs(inputDirs, false);
	warnAboutUndocumentedRules(
	Sets.difference(ruleClassProvider.getRuleClassMap().keySet(), ruleDocEntries.keySet()));
	writeRuleClassDocs(ruleDocEntries.values(), bw);

	bw.write("\n"); // for the benefit of the block-beginning comment at the top of the template
	bw.write(SourceFileReader.readTemplateContents(DocgenConsts.FOOTER_TEMPLATE));
	}
	}

	/**
	* Collects all the rule and attribute documentation present in inputDirs, integrates the
	* attribute documentation in the rule documentation and returns the rule documentation.
	*/
	public Map<String, RuleDocumentation> collectAndProcessRuleDocs(String[] inputDirs,
	boolean printMessages) throws BuildEncyclopediaDocException, IOException {
	// RuleDocumentations are generated in order (based on rule type then alphabetically).
	// The ordering is also used to determine in which rule doc the common attribute docs are
	// generated (they are generated at the first appearance).
	Map<String, RuleDocumentation> ruleDocEntries = new TreeMap<>();
	// RuleDocumentationAttribute objects equal based on attributeName so they have to be
	// collected in a List instead of a Set.
	ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries =
	LinkedListMultimap.create();

	// Map of rule class name to file that defined it.
	Map<String, File> ruleClassFiles = new HashMap<>();

	// Set of files already processed. The same file may be encountered multiple times because
	// directories are processed recursively, and an input directory may be a subdirectory of
	// another one.
	Set<File> processedFiles = new HashSet<>();

	for (String inputDir : inputDirs) {
	if (printMessages) {
	System.out.println(" Processing input directory: " + inputDir);
	}
	int ruleNum = ruleDocEntries.size();
	collectDocs(processedFiles, ruleClassFiles, ruleDocEntries, attributeDocEntries,
	new File(inputDir));
	if (printMessages) {
	System.out.println(
	" " + (ruleDocEntries.size() - ruleNum) + " rule documentations found.");
	}
	}

	processAttributeDocs(ruleDocEntries.values(), attributeDocEntries);
	return ruleDocEntries;
	}

	/**
	* Go through all attributes of all documented rules and search the best attribute documentation
	* if exists. The best documentation is the closest documentation in the ancestor graph. E.g. if
	* java_library.deps documented in $rule and $java_rule then the one in $java_rule is going to
	* apply since it's a closer ancestor of java_library.
	*/
	private void processAttributeDocs(Iterable<RuleDocumentation> ruleDocEntries,
	ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries)
	throws BuildEncyclopediaDocException {
	for (RuleDocumentation ruleDoc : ruleDocEntries) {
	RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(ruleDoc.getRuleName());
	if (ruleClass != null) {
	if (ruleClass.isDocumented()) {
	Class<? extends RuleDefinition> ruleDefinition =
	ruleClassProvider.getRuleClassDefinition(ruleDoc.getRuleName());
	for (Attribute attribute : ruleClass.getAttributes()) {
	String attrName = attribute.getName();
	List<RuleDocumentationAttribute> attributeDocList =
	attributeDocEntries.get(attrName);
	if (attributeDocList != null) {
	// There are attribute docs for this attribute.
	// Search the closest one in the ancestor graph.
	// Note that there can be only one 'closest' attribute since we forbid multiple
	// inheritance of the same attribute in RuleClass.
	int minLevel = Integer.MAX_VALUE;
	RuleDocumentationAttribute bestAttributeDoc = null;
	for (RuleDocumentationAttribute attributeDoc : attributeDocList) {
	int level = attributeDoc.getDefinitionClassAncestryLevel(ruleDefinition);
	if (level >= 0 && level < minLevel) {
	bestAttributeDoc = attributeDoc;
	minLevel = level;
	}
	}
	if (bestAttributeDoc != null) {
	ruleDoc.addAttribute(bestAttributeDoc);
	// If there is no matching attribute doc try to add the common.
	} else if (ruleDoc.getRuleType().equals(RuleType.BINARY)
	&& PredefinedAttributes.BINARY_ATTRIBUTES.containsKey(attrName)) {
	ruleDoc.addAttribute(PredefinedAttributes.BINARY_ATTRIBUTES.get(attrName));
	} else if (ruleDoc.getRuleType().equals(RuleType.TEST)
	&& PredefinedAttributes.TEST_ATTRIBUTES.containsKey(attrName)) {
	ruleDoc.addAttribute(PredefinedAttributes.TEST_ATTRIBUTES.get(attrName));
	} else if (PredefinedAttributes.COMMON_ATTRIBUTES.containsKey(attrName)) {
	ruleDoc.addAttribute(PredefinedAttributes.COMMON_ATTRIBUTES.get(attrName));
	}
	}
	}
	}
	} else {
	throw ruleDoc.createException("Can't find RuleClass for " + ruleDoc.getRuleName());
	}
	}
	}

	/**
	* Categorizes, checks and prints all the rule-class documentations.
	*/
	private void writeRuleClassDocs(Iterable<RuleDocumentation> docEntries, BufferedWriter bw)
	throws BuildEncyclopediaDocException, IOException {
	Set<RuleDocumentation> binaryDocs = new TreeSet<>();
	Set<RuleDocumentation> libraryDocs = new TreeSet<>();
	Set<RuleDocumentation> testDocs = new TreeSet<>();
	Set<RuleDocumentation> otherDocs = new TreeSet<>();

	for (RuleDocumentation doc : docEntries) {
	RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(doc.getRuleName());
	if (!ruleClass.isDocumented()) {
	continue;
	}

	if (doc.isLanguageSpecific()) {
	switch(doc.getRuleType()) {
	case BINARY:
	binaryDocs.add(doc);
	break;
	case LIBRARY:
	libraryDocs.add(doc);
	break;
	case TEST:
	testDocs.add(doc);
	break;
	case OTHER:
	otherDocs.add(doc);
	break;
	}
	} else {
	otherDocs.add(doc);
	}
	}

	bw.write("\n"); // for the benefit of the block-beginning comment at the top of the template
	bw.write(SourceFileReader.readTemplateContents(DocgenConsts.HEADER_TEMPLATE,
	generateBEHeaderMapping(docEntries)));

	Map<String, String> sectionMapping = ImmutableMap.of(
	DocgenConsts.VAR_SECTION_BINARY, getRuleDocs(binaryDocs),
	DocgenConsts.VAR_SECTION_LIBRARY, getRuleDocs(libraryDocs),
	DocgenConsts.VAR_SECTION_TEST, getRuleDocs(testDocs),
	DocgenConsts.VAR_SECTION_OTHER, getRuleDocs(otherDocs));
	bw.write("\n"); // for the benefit of the block-beginning comment at the top of the template
	bw.write(SourceFileReader.readTemplateContents(DocgenConsts.BODY_TEMPLATE, sectionMapping));
	}

	private Map<String, String> generateBEHeaderMapping(Iterable<RuleDocumentation> docEntries)
	throws BuildEncyclopediaDocException {
	StringBuilder sb = new StringBuilder();

	sb.append("<table id=\"rules\" summary=\"Table of rules sorted by language\">\n")
	.append("<colgroup span=\"5\" width=\"20%\"></colgroup>\n")
	.append("<tr><th>Language</th><th>Binary rules</th><th>Library rules</th>"
	+ "<th>Test rules</th><th>Other rules</th><th></th></tr>\n");

	// Separate rule families into language-specific and generic ones.
	Set<String> languageSpecificRuleFamilies = new TreeSet<>();
	Set<String> genericRuleFamilies = new TreeSet<>();
	separateRuleFamilies(docEntries, languageSpecificRuleFamilies, genericRuleFamilies);

	// Create a mapping of rules based on rule type and family.
	Map<String, ListMultimap<RuleType, RuleDocumentation>> ruleMapping = new HashMap<>();
	createRuleMapping(docEntries, ruleMapping);

	// Generate the table.
	for (String ruleFamily : languageSpecificRuleFamilies) {
	generateHeaderTableRuleFamily(sb, ruleMapping.get(ruleFamily), ruleFamily);
	}

	sb.append("<tr><th> </th></tr>");
	sb.append("<tr><th colspan=\"5\">Rules that do not apply to a "
	+ "specific programming language</th></tr>");
	for (String ruleFamily : genericRuleFamilies) {
	generateHeaderTableRuleFamily(sb, ruleMapping.get(ruleFamily), ruleFamily);
	}
	sb.append("</table>\n");
	return ImmutableMap.<String, String>of(DocgenConsts.VAR_HEADER_TABLE, sb.toString(),
	DocgenConsts.VAR_COMMON_ATTRIBUTE_DEFINITION, generateCommonAttributeDocs(
	PredefinedAttributes.COMMON_ATTRIBUTES, DocgenConsts.COMMON_ATTRIBUTES),
	DocgenConsts.VAR_TEST_ATTRIBUTE_DEFINITION, generateCommonAttributeDocs(
	PredefinedAttributes.TEST_ATTRIBUTES, DocgenConsts.TEST_ATTRIBUTES),
	DocgenConsts.VAR_BINARY_ATTRIBUTE_DEFINITION, generateCommonAttributeDocs(
	PredefinedAttributes.BINARY_ATTRIBUTES, DocgenConsts.BINARY_ATTRIBUTES),
	DocgenConsts.VAR_LEFT_PANEL, generateLeftNavigationPanel(docEntries));
	}

	/**
	* Create a mapping of rules based on rule type and family.
	*/
	private void createRuleMapping(Iterable<RuleDocumentation> docEntries,
	Map<String, ListMultimap<RuleType, RuleDocumentation>> ruleMapping)
	throws BuildEncyclopediaDocException {
	for (RuleDocumentation ruleDoc : docEntries) {
	RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(ruleDoc.getRuleName());
	if (ruleClass != null) {
	String ruleFamily = ruleDoc.getRuleFamily();
	if (!ruleMapping.containsKey(ruleFamily)) {
	ruleMapping.put(ruleFamily, LinkedListMultimap.<RuleType, RuleDocumentation>create());
	}
	if (ruleClass.isDocumented()) {
	ruleMapping.get(ruleFamily).put(ruleDoc.getRuleType(), ruleDoc);
	}
	} else {
	throw ruleDoc.createException("Can't find RuleClass for " + ruleDoc.getRuleName());
	}
	}
	}

	/**
	* Separates all rule families in docEntries into language-specific rules and generic rules.
	*/
	private void separateRuleFamilies(Iterable<RuleDocumentation> docEntries,
	Set<String> languageSpecificRuleFamilies, Set<String> genericRuleFamilies)
	throws BuildEncyclopediaDocException {
	for (RuleDocumentation ruleDoc : docEntries) {
	if (ruleDoc.isLanguageSpecific()) {
	if (genericRuleFamilies.contains(ruleDoc.getRuleFamily())) {
	throw ruleDoc.createException("The rule is marked as being language-specific, but other "
	+ "rules of the same family have already been marked as being not.");
	}
	languageSpecificRuleFamilies.add(ruleDoc.getRuleFamily());
	} else {
	if (languageSpecificRuleFamilies.contains(ruleDoc.getRuleFamily())) {
	throw ruleDoc.createException("The rule is marked as being generic, but other rules of "
	+ "the same family have already been marked as being language-specific.");
	}
	genericRuleFamilies.add(ruleDoc.getRuleFamily());
	}
	}
	}

	private String generateLeftNavigationPanel(Iterable<RuleDocumentation> docEntries) {
	// Order the rules alphabetically. At this point they are ordered according to
	// RuleDocumentation.compareTo() which is not alphabetical.
	TreeMap<String, String> ruleNames = new TreeMap<>();
	for (RuleDocumentation ruleDoc : docEntries) {
	String ruleName = ruleDoc.getRuleName();
	ruleNames.put(ruleName.toLowerCase(), ruleName);
	}
	StringBuilder sb = new StringBuilder();
	for (String ruleName : ruleNames.values()) {
	RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(ruleName);
	Preconditions.checkNotNull(ruleClass);
	if (ruleClass.isDocumented()) {
	sb.append(String.format("<a href=\"#%s\">%s</a><br/>\n", ruleName, ruleName));
	}
	}
	return sb.toString();
	}

	private String generateCommonAttributeDocs(Map<String, RuleDocumentationAttribute> attributes,
	String attributeGroupName) throws BuildEncyclopediaDocException {
	RuleDocumentation ruleDoc = new RuleDocumentation(
	attributeGroupName, "OTHER", null, null, 0, null, ImmutableSet.<String>of(),
	ruleClassProvider);
	for (RuleDocumentationAttribute attribute : attributes.values()) {
	ruleDoc.addAttribute(attribute);
	}
	return ruleDoc.generateAttributeDefinitions();
	}

	private void generateHeaderTableRuleFamily(StringBuilder sb,
	ListMultimap<RuleType, RuleDocumentation> ruleTypeMap, String ruleFamily) {
	sb.append("<tr>\n")
	.append(String.format("<td class=\"lang\">%s</td>\n", ruleFamily));
	boolean otherRulesSplitted = false;
	for (RuleType ruleType : DocgenConsts.RuleType.values()) {
	sb.append("<td>");
	int i = 0;
	List<RuleDocumentation> ruleDocList = ruleTypeMap.get(ruleType);
	for (RuleDocumentation ruleDoc : ruleDocList) {
	if (i > 0) {
	if (ruleType.equals(RuleType.OTHER)
	&& ruleDocList.size() >= 4 && i == (ruleDocList.size() + 1) / 2) {
	// Split 'other rules' into two columns if there are too many of them.
	sb.append("</td>\n<td>");
	otherRulesSplitted = true;
	} else {
	sb.append("<br/>");
	}
	}
	String ruleName = ruleDoc.getRuleName();
	String deprecatedString = ruleDoc.hasFlag(DocgenConsts.FLAG_DEPRECATED)
	? " class=\"deprecated\"" : "";
	sb.append(String.format("<a href=\"#%s\"%s>%s</a>", ruleName, deprecatedString, ruleName));
	i++;
	}
	sb.append("</td>\n");
	}
	// There should be 6 columns.
	if (!otherRulesSplitted) {
	sb.append("<td></td>\n");
	}
	sb.append("</tr>\n");
	}

	private String getRuleDocs(Iterable<RuleDocumentation> docEntries) {
	StringBuilder sb = new StringBuilder();
	for (RuleDocumentation doc : docEntries) {
	sb.append(doc.getHtmlDocumentation());
	}
	return sb.toString();
	}

	/**
	* Goes through all the html files and subdirs under inputPath and collects the rule
	* and attribute documentations using the ruleDocEntries and attributeDocEntries variable.
	*/
	public void collectDocs(
	Set<File> processedFiles,
	Map<String, File> ruleClassFiles,
	Map<String, RuleDocumentation> ruleDocEntries,
	ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries,
	File inputPath) throws BuildEncyclopediaDocException, IOException {
	if (processedFiles.contains(inputPath)) {
	return;
	}

	if (inputPath.isFile()) {
	if (DocgenConsts.JAVA_SOURCE_FILE_SUFFIX.apply(inputPath.getName())) {
	SourceFileReader sfr = new SourceFileReader(
	ruleClassProvider, inputPath.getAbsolutePath());
	sfr.readDocsFromComments();
	for (RuleDocumentation d : sfr.getRuleDocEntries()) {
	String ruleName = d.getRuleName();
	if (ruleDocEntries.containsKey(ruleName)
	&& !ruleClassFiles.get(ruleName).equals(inputPath)) {
	System.err.printf("WARNING: '%s' from '%s' overrides value already in map from '%s'\n",
	d.getRuleName(), inputPath, ruleClassFiles.get(ruleName));
	}
	ruleClassFiles.put(ruleName, inputPath);
	ruleDocEntries.put(ruleName, d);
	}
	if (attributeDocEntries != null) {
	// Collect all attribute documentations from this file.
	attributeDocEntries.putAll(sfr.getAttributeDocEntries());
	}
	}
	} else if (inputPath.isDirectory()) {
	for (File childPath : inputPath.listFiles()) {
	collectDocs(processedFiles, ruleClassFiles, ruleDocEntries, attributeDocEntries, childPath);
	}
	}

	processedFiles.add(inputPath);
	}

	private File setupDirectories(String outputRootDir) {
	if (outputRootDir != null) {
	File outputRootPath = new File(outputRootDir);
	outputRootPath.mkdirs();
	return new File(outputRootDir + File.separator + DocgenConsts.BUILD_ENCYCLOPEDIA_NAME);
	} else {
	return new File(DocgenConsts.BUILD_ENCYCLOPEDIA_NAME);
	}
	}

	private static void warnAboutUndocumentedRules(Iterable<String> rulesWithoutDocumentation) {
	Iterable<String> undocumentedRules = Iterables.filter(rulesWithoutDocumentation,
	RULE_WORTH_DOCUMENTING);
	System.err.printf("WARNING: The following rules are undocumented: [%s]\n",
	Joiner.on(", ").join(Ordering.<String>natural().immutableSortedCopy(undocumentedRules)));
	}
	}