src/main/java/com/google/devtools/build/docgen/SkylarkDocumentationProcessor.java - bazel - Git at Google

 // Copyright 2014 Google Inc. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 package com.google.devtools.build.docgen;

 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Function;
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.io.Files;
 import com.google.devtools.build.docgen.SkylarkJavaInterfaceExplorer.SkylarkBuiltinMethod;
 import com.google.devtools.build.docgen.SkylarkJavaInterfaceExplorer.SkylarkJavaMethod;
 import com.google.devtools.build.docgen.SkylarkJavaInterfaceExplorer.SkylarkModuleDoc;
 import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
 import com.google.devtools.build.lib.packages.MethodLibrary;
 import com.google.devtools.build.lib.rules.SkylarkModules;
 import com.google.devtools.build.lib.rules.SkylarkRuleContext;
 import com.google.devtools.build.lib.syntax.BaseFunction;
 import com.google.devtools.build.lib.syntax.Environment;
 import com.google.devtools.build.lib.syntax.Environment.NoneType;
 import com.google.devtools.build.lib.syntax.EvalUtils;
 import com.google.devtools.build.lib.syntax.FuncallExpression;
 import com.google.devtools.build.lib.syntax.SkylarkCallable;
 import com.google.devtools.build.lib.syntax.SkylarkList;
 import com.google.devtools.build.lib.syntax.SkylarkModule;
 import com.google.devtools.build.lib.syntax.SkylarkSignature;
 import com.google.devtools.build.lib.syntax.SkylarkSignature.Param;
 import com.google.devtools.build.lib.syntax.SkylarkSignatureProcessor.HackHackEitherList;

 import java.io.BufferedWriter;
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.TreeMap;

 /**
  * A class to assemble documentation for Skylark.
  */
 public class SkylarkDocumentationProcessor {

   private static final String TOP_LEVEL_ID = "_top_level";

   private static final boolean USE_TEMPLATE = false;

   @SkylarkModule(name = "Global objects, functions and modules",
       doc = "Objects, functions and modules registered in the global environment.")
   private static final class TopLevelModule {}

   static SkylarkModule getTopLevelModule() {
     return TopLevelModule.class.getAnnotation(SkylarkModule.class);
   }

   /**
    * Generates the Skylark documentation to the given output directory.
    */
   public void generateDocumentation(String outputPath) throws IOException,
       BuildEncyclopediaDocException {
     File skylarkDocPath = new File(outputPath);
     try (BufferedWriter bw = new BufferedWriter(
         Files.newWriter(skylarkDocPath, StandardCharsets.UTF_8))) {
       if (USE_TEMPLATE) {
         bw.write(SourceFileReader.readTemplateContents(DocgenConsts.SKYLARK_BODY_TEMPLATE,
             ImmutableMap.<String, String>of(
                 DocgenConsts.VAR_SECTION_SKYLARK_BUILTIN, generateAllBuiltinDoc())));
       } else {
         bw.write(generateAllBuiltinDoc());
       }
       System.out.println("Skylark documentation generated: " + skylarkDocPath.getAbsolutePath());
     }
   }

   @VisibleForTesting
   Map<String, SkylarkModuleDoc> collectModules() {
     Map<String, SkylarkModuleDoc> modules = new TreeMap<>();
     Map<String, SkylarkModuleDoc> builtinModules = collectBuiltinModules();
     Map<SkylarkModule, Class<?>> builtinJavaObjects = collectBuiltinJavaObjects();

     modules.putAll(builtinModules);
     SkylarkJavaInterfaceExplorer explorer = new SkylarkJavaInterfaceExplorer();
     for (SkylarkModuleDoc builtinObject : builtinModules.values()) {
       // Check the return type for built-in functions, it can be a module previously not added.
       for (SkylarkBuiltinMethod builtinMethod : builtinObject.getBuiltinMethods().values()) {
         Class<?> type = builtinMethod.annotation.returnType();
         if (type.isAnnotationPresent(SkylarkModule.class)) {
           explorer.collect(type.getAnnotation(SkylarkModule.class), type, modules);
         }
       }
       explorer.collect(builtinObject.getAnnotation(), builtinObject.getClassObject(), modules);
     }
     for (Entry<SkylarkModule, Class<?>> builtinModule : builtinJavaObjects.entrySet()) {
       explorer.collect(builtinModule.getKey(), builtinModule.getValue(), modules);
     }
     return modules;
   }

   private String generateAllBuiltinDoc() {
     Map<String, SkylarkModuleDoc> modules = collectModules();

     StringBuilder sb = new StringBuilder();
     // Generate the top level module first in the doc
     SkylarkModuleDoc topLevelModule = modules.remove(getTopLevelModule().name());
     generateModuleDoc(topLevelModule, sb);
     for (SkylarkModuleDoc module : modules.values()) {
       if (module.getAnnotation().documented()) {
         sb.append("<hr>");
         generateModuleDoc(module, sb);
       }
     }
     return sb.toString();
   }

   private void generateModuleDoc(SkylarkModuleDoc module, StringBuilder sb) {
     SkylarkModule annotation = module.getAnnotation();
     sb.append(String.format("<h2 id=\"modules.%s\">%s</h2>\n",
           getModuleId(annotation),
           annotation.name()))
       .append(annotation.doc())
       .append("\n");
     sb.append("<ul>");
     // Sort Java and Skylark builtin methods together. The map key is only used for sorting.
     TreeMap<String, Object> methodMap = new TreeMap<>();
     for (SkylarkJavaMethod method : module.getJavaMethods()) {
       methodMap.put(method.name + method.method.getParameterTypes().length, method);
     }
     for (SkylarkBuiltinMethod builtin : module.getBuiltinMethods().values()) {
       methodMap.put(builtin.annotation.name(), builtin);
     }
     for (Object object : methodMap.values()) {
       if (object instanceof SkylarkJavaMethod) {
         SkylarkJavaMethod method = (SkylarkJavaMethod) object;
         generateDirectJavaMethodDoc(annotation.name(), method.name, method.method,
             method.callable, sb);
       }
       if (object instanceof SkylarkBuiltinMethod) {
         generateBuiltinItemDoc(getModuleId(annotation), (SkylarkBuiltinMethod) object, sb);
       }
     }
     sb.append("</ul>");
   }

   private String getModuleId(SkylarkModule annotation) {
     if (annotation == getTopLevelModule()) {
       return TOP_LEVEL_ID;
     } else {
       return annotation.name();
     }
   }

   private void generateBuiltinItemDoc(
       String moduleId, SkylarkBuiltinMethod method, StringBuilder sb) {
     SkylarkSignature annotation = method.annotation;
     if (!annotation.documented()) {
       return;
     }
     sb.append(String.format("<li><h3 id=\"modules.%s.%s\">%s</h3>\n",
           moduleId,
           annotation.name(),
           annotation.name()));

     if (BaseFunction.class.isAssignableFrom(method.fieldClass)) {
       sb.append(getSignature(moduleId, annotation));
     } else {
       if (!annotation.returnType().equals(Object.class)) {
         sb.append("<code>" + getTypeAnchor(annotation.returnType()) + "</code><br>");
       }
     }

     sb.append(annotation.doc() + "\n");
     printParams(moduleId, annotation, sb);
   }

   // Elide self parameter from mandatoryPositionals in class methods.
   private static Param[] adjustedMandatoryPositionals(SkylarkSignature annotation) {
     Param[] mandatoryPos = annotation.mandatoryPositionals();
     if (mandatoryPos.length > 0
         && annotation.objectType() != Object.class
         && !FuncallExpression.isNamespace(annotation.objectType())) {
       // Skip the self parameter, which is the first mandatory positional parameter.
       return Arrays.copyOfRange(mandatoryPos, 1, mandatoryPos.length);
     } else {
       return mandatoryPos;
     }
   }

   private void printParams(String moduleId, SkylarkSignature annotation, StringBuilder sb) {
     Param[] mandatoryPos = adjustedMandatoryPositionals(annotation);
     Param[] optionalPos = annotation.optionalPositionals();
     Param[] optionalKey = annotation.optionalNamedOnly();
     Param[] mandatoryKey = annotation.mandatoryNamedOnly();
     Param[] star = annotation.extraPositionals();
     Param[] starStar = annotation.extraKeywords();

     if (mandatoryPos.length + optionalPos.length + optionalKey.length + mandatoryKey.length
         + star.length + starStar.length > 0) {
       sb.append("<h4>Parameters</h4>\n");
       printParams(moduleId, annotation.name(), mandatoryPos, sb);
       printParams(moduleId, annotation.name(), optionalPos, sb);
       printParams(moduleId, annotation.name(), star, sb);
       printParams(moduleId, annotation.name(), mandatoryKey, sb);
       printParams(moduleId, annotation.name(), optionalKey, sb);
       printParams(moduleId, annotation.name(), starStar, sb);
     } else {
       sb.append("<br/>\n");
     }
   }

   private void generateDirectJavaMethodDoc(String objectName, String methodName,
       Method method, SkylarkCallable annotation, StringBuilder sb) {
     if (!annotation.documented()) {
       return;
     }
     if (annotation.doc().isEmpty()) {
       throw new RuntimeException(String.format(
           "empty SkylarkCallable.doc() for object %s, method %s", objectName, methodName));
     }

     sb.append(String.format("<li><h3 id=\"modules.%s.%s\">%s</h3>\n%s\n",
             objectName,
             methodName,
             methodName,
             getSignature(objectName, methodName, method)))
         .append(annotation.doc())
         .append(getReturnTypeExtraMessage(annotation))
         .append("\n");
   }

   private String getReturnTypeExtraMessage(SkylarkCallable annotation) {
     if (annotation.allowReturnNones()) {
       return " May return <code>None</code>.\n";
     }
     return "";
   }

   private String getSignature(String objectName, String methodName, Method method) {
     String args = method.getAnnotation(SkylarkCallable.class).structField()
         ? "" : "(" + getParameterString(method) + ")";

     return String.format("<code>%s %s.%s%s</code><br>",
         getTypeAnchor(method.getReturnType()), objectName, methodName, args);
   }

   private String getSignature(String objectName, SkylarkSignature method) {
     List<String> argList = new ArrayList<>();
     for (Param param : adjustedMandatoryPositionals(method)) {
       argList.add(param.name());
     }
     for (Param param : method.optionalPositionals()) {
       argList.add(param.name() + "?"); // or should we use pythonic " = &#ellipsis;" instead?
     }
     for (Param param : method.extraPositionals()) {
       argList.add("*" + param.name());
     }
     if (method.extraPositionals().length == 0
         && (method.optionalNamedOnly().length > 0 || method.mandatoryNamedOnly().length > 0)) {
       argList.add("*");
     }
     for (Param param : method.mandatoryNamedOnly()) {
       argList.add(param.name());
     }
     for (Param param : method.optionalNamedOnly()) {
       argList.add(param.name() + "?"); // or should we be more pythonic with this? " = ..."
     }
     for (Param param : method.extraKeywords()) {
       argList.add("**" + param.name());
     }
     String args = "(" + Joiner.on(", ").join(argList) + ")";
     if (!objectName.equals(TOP_LEVEL_ID)) {
       return String.format("<code>%s %s.%s%s</code><br>\n",
           getTypeAnchor(method.returnType()), objectName, method.name(), args);
     } else {
       return String.format("<code>%s %s%s</code><br>\n",
           getTypeAnchor(method.returnType()), method.name(), args);
     }
   }

   private String getTypeAnchor(Class<?> returnType, Class<?> generic1) {
     return getTypeAnchor(returnType) + " of " + getTypeAnchor(generic1) + "s";
   }

   private String getTypeAnchor(Class<?> type) {
     if (type.equals(Boolean.class) || type.equals(boolean.class)) {
       return "<a class=\"anchor\" href=\"#modules._top_level.bool\">bool</a>";
     } else if (type.equals(String.class)) {
       return "<a class=\"anchor\" href=\"#modules.string\">string</a>";
     } else if (Map.class.isAssignableFrom(type)) {
       return "<a class=\"anchor\" href=\"#modules.dict\">dict</a>";
     } else if (List.class.isAssignableFrom(type) || SkylarkList.class.isAssignableFrom(type)
         || type == HackHackEitherList.class) {
       // Annotated Java methods can return simple java.util.Lists (which get auto-converted).
       return "<a class=\"anchor\" href=\"#modules.list\">list</a>";
     } else if (type.equals(Void.TYPE) || type.equals(NoneType.class)) {
       return "<a class=\"anchor\" href=\"#modules." + TOP_LEVEL_ID + ".None\">None</a>";
     } else if (type.isAnnotationPresent(SkylarkModule.class)) {
       // TODO(bazel-team): this can produce dead links for types don't show up in the doc.
       // The correct fix is to generate those types (e.g. SkylarkFileType) too.
       String module = type.getAnnotation(SkylarkModule.class).name();
       return "<a class=\"anchor\" href=\"#modules." + module + "\">" + module + "</a>";
     } else {
       return EvalUtils.getDataTypeNameFromClass(type);
     }
   }

   private String getParameterString(Method method) {
     return Joiner.on(", ").join(Iterables.transform(
         ImmutableList.copyOf(method.getParameterTypes()), new Function<Class<?>, String>() {
           @Override
           public String apply(Class<?> input) {
             return getTypeAnchor(input);
           }
         }));
   }

   private void printParams(String moduleId, String methodName,
       Param[] params, StringBuilder sb) {
     if (params.length > 0) {
       sb.append("<ul>\n");
       for (Param param : params) {
         String paramType = param.type().equals(Object.class) ? ""
             : (param.generic1().equals(Object.class)
                 ? " (" + getTypeAnchor(param.type()) + ")"
                 : " (" + getTypeAnchor(param.type(), param.generic1()) + ")");
         sb.append(String.format("\t<li id=\"modules.%s.%s.%s\"><code>%s%s</code>: ",
                 moduleId,
                 methodName,
                 param.name(),
                 param.name(),
                 paramType))
             .append(param.doc())
             .append("\n\t</li>\n");
       }
       sb.append("</ul>\n");
     }
   }

   private Map<String, SkylarkModuleDoc> collectBuiltinModules() {
     Map<String, SkylarkModuleDoc> modules = new HashMap<>();
     collectBuiltinDoc(modules, Environment.class.getDeclaredFields());
     collectBuiltinDoc(modules, MethodLibrary.class.getDeclaredFields());
     for (Class<?> moduleClass : SkylarkModules.MODULES) {
       collectBuiltinDoc(modules, moduleClass.getDeclaredFields());
     }
     return modules;
   }

   private Map<SkylarkModule, Class<?>> collectBuiltinJavaObjects() {
     Map<SkylarkModule, Class<?>> modules = new HashMap<>();
     collectBuiltinModule(modules, SkylarkRuleContext.class);
     collectBuiltinModule(modules, TransitiveInfoCollection.class);
     return modules;
   }

   /**
    * Returns the top level modules and functions with their documentation in a command-line
    * printable format.
    */
   public Map<String, String> collectTopLevelModules() {
     Map<String, String> modules = new TreeMap<>();
     for (SkylarkModuleDoc doc : collectBuiltinModules().values()) {
       if (doc.getAnnotation() == getTopLevelModule()) {
         for (Map.Entry<String, SkylarkBuiltinMethod> entry : doc.getBuiltinMethods().entrySet()) {
           if (entry.getValue().annotation.documented()) {
             modules.put(entry.getKey(),
                 DocgenConsts.toCommandLineFormat(entry.getValue().annotation.doc()));
           }
         }
       } else {
         modules.put(doc.getAnnotation().name(),
             DocgenConsts.toCommandLineFormat(doc.getAnnotation().doc()));
       }
     }
     return modules;
   }

   /**
    * Returns the API doc for the specified Skylark object in a command line printable format,
    * params[0] identifies either a module or a top-level object, the optional params[1] identifies a
    * method in the module.<br>
    * Returns null if no Skylark object is found.
    */
   public String getCommandLineAPIDoc(String[] params) {
     Map<String, SkylarkModuleDoc> modules = collectModules();
     SkylarkModuleDoc toplevelModuleDoc = modules.get(getTopLevelModule().name());
     if (modules.containsKey(params[0])) {
       // Top level module
       SkylarkModuleDoc module = modules.get(params[0]);
       if (params.length == 1) {
         String moduleName = module.getAnnotation().name();
         StringBuilder sb = new StringBuilder();
         sb.append(moduleName).append("\n\t").append(module.getAnnotation().doc()).append("\n");
         // Print the signature of all built-in methods
         for (SkylarkBuiltinMethod method : module.getBuiltinMethods().values()) {
           printBuiltinFunctionDoc(moduleName, method.annotation, sb);
         }
         // Print all Java methods
         for (SkylarkJavaMethod method : module.getJavaMethods()) {
           printJavaFunctionDoc(moduleName, method, sb);
         }
         return DocgenConsts.toCommandLineFormat(sb.toString());
       } else {
         return getFunctionDoc(module.getAnnotation().name(), params[1], module);
       }
     } else if (toplevelModuleDoc.getBuiltinMethods().containsKey(params[0])){
       // Top level object / function
       return getFunctionDoc(null, params[0], toplevelModuleDoc);
     }
     return null;
   }

   private String getFunctionDoc(String moduleName, String methodName, SkylarkModuleDoc module) {
     if (module.getBuiltinMethods().containsKey(methodName)) {
       // Create the doc for the built-in function
       SkylarkBuiltinMethod method = module.getBuiltinMethods().get(methodName);
       StringBuilder sb = new StringBuilder();
       printBuiltinFunctionDoc(moduleName, method.annotation, sb);
       printParams(moduleName, method.annotation, sb);
       return DocgenConsts.removeDuplicatedNewLines(DocgenConsts.toCommandLineFormat(sb.toString()));
     } else {
       // Search if there are matching Java functions
       StringBuilder sb = new StringBuilder();
       boolean foundMatchingMethod = false;
       for (SkylarkJavaMethod method : module.getJavaMethods()) {
         if (method.name.equals(methodName)) {
           printJavaFunctionDoc(moduleName, method, sb);
           foundMatchingMethod = true;
         }
       }
       if (foundMatchingMethod) {
         return DocgenConsts.toCommandLineFormat(sb.toString());
       }
     }
     return null;
   }

   private void printBuiltinFunctionDoc(
       String moduleName, SkylarkSignature annotation, StringBuilder sb) {
     if (moduleName != null) {
       sb.append(moduleName).append(".");
     }
     sb.append(annotation.name()).append("\n\t").append(annotation.doc()).append("\n");
   }

   private void printJavaFunctionDoc(String moduleName, SkylarkJavaMethod method, StringBuilder sb) {
     sb.append(getSignature(moduleName, method.name, method.method))
       .append("\t").append(method.callable.doc()).append("\n");
   }

   private void collectBuiltinModule(
       Map<SkylarkModule, Class<?>> modules, Class<?> moduleClass) {
     if (moduleClass.isAnnotationPresent(SkylarkModule.class)) {
       SkylarkModule skylarkModule = moduleClass.getAnnotation(SkylarkModule.class);
       modules.put(skylarkModule, moduleClass);
     }
   }

   private void collectBuiltinDoc(Map<String, SkylarkModuleDoc> modules, Field[] fields) {
     for (Field field : fields) {
       if (field.isAnnotationPresent(SkylarkSignature.class)) {
         SkylarkSignature skylarkSignature = field.getAnnotation(SkylarkSignature.class);
         Class<?> moduleClass = skylarkSignature.objectType();
         SkylarkModule skylarkModule = moduleClass.equals(Object.class)
             ? getTopLevelModule()
             : moduleClass.getAnnotation(SkylarkModule.class);
         if (!modules.containsKey(skylarkModule.name())) {
           modules.put(skylarkModule.name(), new SkylarkModuleDoc(skylarkModule, moduleClass));
         }
         modules.get(skylarkModule.name()).getBuiltinMethods()
             .put(skylarkSignature.name(),
                 new SkylarkBuiltinMethod(skylarkSignature, field.getType()));
       }
     }
   }
 }
	// Copyright 2014 Google Inc. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.
	package com.google.devtools.build.docgen;

	import com.google.common.annotations.VisibleForTesting;
	import com.google.common.base.Function;
	import com.google.common.base.Joiner;
	import com.google.common.collect.ImmutableList;
	import com.google.common.collect.ImmutableMap;
	import com.google.common.collect.Iterables;
	import com.google.common.io.Files;
	import com.google.devtools.build.docgen.SkylarkJavaInterfaceExplorer.SkylarkBuiltinMethod;
	import com.google.devtools.build.docgen.SkylarkJavaInterfaceExplorer.SkylarkJavaMethod;
	import com.google.devtools.build.docgen.SkylarkJavaInterfaceExplorer.SkylarkModuleDoc;
	import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
	import com.google.devtools.build.lib.packages.MethodLibrary;
	import com.google.devtools.build.lib.rules.SkylarkModules;
	import com.google.devtools.build.lib.rules.SkylarkRuleContext;
	import com.google.devtools.build.lib.syntax.BaseFunction;
	import com.google.devtools.build.lib.syntax.Environment;
	import com.google.devtools.build.lib.syntax.Environment.NoneType;
	import com.google.devtools.build.lib.syntax.EvalUtils;
	import com.google.devtools.build.lib.syntax.FuncallExpression;
	import com.google.devtools.build.lib.syntax.SkylarkCallable;
	import com.google.devtools.build.lib.syntax.SkylarkList;
	import com.google.devtools.build.lib.syntax.SkylarkModule;
	import com.google.devtools.build.lib.syntax.SkylarkSignature;
	import com.google.devtools.build.lib.syntax.SkylarkSignature.Param;
	import com.google.devtools.build.lib.syntax.SkylarkSignatureProcessor.HackHackEitherList;

	import java.io.BufferedWriter;
	import java.io.File;
	import java.io.IOException;
	import java.lang.reflect.Field;
	import java.lang.reflect.Method;
	import java.nio.charset.StandardCharsets;
	import java.util.ArrayList;
	import java.util.Arrays;
	import java.util.HashMap;
	import java.util.List;
	import java.util.Map;
	import java.util.Map.Entry;
	import java.util.TreeMap;

	/**
	* A class to assemble documentation for Skylark.
	*/
	public class SkylarkDocumentationProcessor {

	private static final String TOP_LEVEL_ID = "_top_level";

	private static final boolean USE_TEMPLATE = false;

	@SkylarkModule(name = "Global objects, functions and modules",
	doc = "Objects, functions and modules registered in the global environment.")
	private static final class TopLevelModule {}

	static SkylarkModule getTopLevelModule() {
	return TopLevelModule.class.getAnnotation(SkylarkModule.class);
	}

	/**
	* Generates the Skylark documentation to the given output directory.
	*/
	public void generateDocumentation(String outputPath) throws IOException,
	BuildEncyclopediaDocException {
	File skylarkDocPath = new File(outputPath);
	try (BufferedWriter bw = new BufferedWriter(
	Files.newWriter(skylarkDocPath, StandardCharsets.UTF_8))) {
	if (USE_TEMPLATE) {
	bw.write(SourceFileReader.readTemplateContents(DocgenConsts.SKYLARK_BODY_TEMPLATE,
	ImmutableMap.<String, String>of(
	DocgenConsts.VAR_SECTION_SKYLARK_BUILTIN, generateAllBuiltinDoc())));
	} else {
	bw.write(generateAllBuiltinDoc());
	}
	System.out.println("Skylark documentation generated: " + skylarkDocPath.getAbsolutePath());
	}
	}

	@VisibleForTesting
	Map<String, SkylarkModuleDoc> collectModules() {
	Map<String, SkylarkModuleDoc> modules = new TreeMap<>();
	Map<String, SkylarkModuleDoc> builtinModules = collectBuiltinModules();
	Map<SkylarkModule, Class<?>> builtinJavaObjects = collectBuiltinJavaObjects();

	modules.putAll(builtinModules);
	SkylarkJavaInterfaceExplorer explorer = new SkylarkJavaInterfaceExplorer();
	for (SkylarkModuleDoc builtinObject : builtinModules.values()) {
	// Check the return type for built-in functions, it can be a module previously not added.
	for (SkylarkBuiltinMethod builtinMethod : builtinObject.getBuiltinMethods().values()) {
	Class<?> type = builtinMethod.annotation.returnType();
	if (type.isAnnotationPresent(SkylarkModule.class)) {
	explorer.collect(type.getAnnotation(SkylarkModule.class), type, modules);
	}
	}
	explorer.collect(builtinObject.getAnnotation(), builtinObject.getClassObject(), modules);
	}
	for (Entry<SkylarkModule, Class<?>> builtinModule : builtinJavaObjects.entrySet()) {
	explorer.collect(builtinModule.getKey(), builtinModule.getValue(), modules);
	}
	return modules;
	}

	private String generateAllBuiltinDoc() {
	Map<String, SkylarkModuleDoc> modules = collectModules();

	StringBuilder sb = new StringBuilder();
	// Generate the top level module first in the doc
	SkylarkModuleDoc topLevelModule = modules.remove(getTopLevelModule().name());
	generateModuleDoc(topLevelModule, sb);
	for (SkylarkModuleDoc module : modules.values()) {
	if (module.getAnnotation().documented()) {
	sb.append("<hr>");
	generateModuleDoc(module, sb);
	}
	}
	return sb.toString();
	}

	private void generateModuleDoc(SkylarkModuleDoc module, StringBuilder sb) {
	SkylarkModule annotation = module.getAnnotation();
	sb.append(String.format("<h2 id=\"modules.%s\">%s</h2>\n",
	getModuleId(annotation),
	annotation.name()))
	.append(annotation.doc())
	.append("\n");
	sb.append("<ul>");
	// Sort Java and Skylark builtin methods together. The map key is only used for sorting.
	TreeMap<String, Object> methodMap = new TreeMap<>();
	for (SkylarkJavaMethod method : module.getJavaMethods()) {
	methodMap.put(method.name + method.method.getParameterTypes().length, method);
	}
	for (SkylarkBuiltinMethod builtin : module.getBuiltinMethods().values()) {
	methodMap.put(builtin.annotation.name(), builtin);
	}
	for (Object object : methodMap.values()) {
	if (object instanceof SkylarkJavaMethod) {
	SkylarkJavaMethod method = (SkylarkJavaMethod) object;
	generateDirectJavaMethodDoc(annotation.name(), method.name, method.method,
	method.callable, sb);
	}
	if (object instanceof SkylarkBuiltinMethod) {
	generateBuiltinItemDoc(getModuleId(annotation), (SkylarkBuiltinMethod) object, sb);
	}
	}
	sb.append("</ul>");
	}

	private String getModuleId(SkylarkModule annotation) {
	if (annotation == getTopLevelModule()) {
	return TOP_LEVEL_ID;
	} else {
	return annotation.name();
	}
	}

	private void generateBuiltinItemDoc(
	String moduleId, SkylarkBuiltinMethod method, StringBuilder sb) {
	SkylarkSignature annotation = method.annotation;
	if (!annotation.documented()) {
	return;
	}
	sb.append(String.format("<li><h3 id=\"modules.%s.%s\">%s</h3>\n",
	moduleId,
	annotation.name(),
	annotation.name()));

	if (BaseFunction.class.isAssignableFrom(method.fieldClass)) {
	sb.append(getSignature(moduleId, annotation));
	} else {
	if (!annotation.returnType().equals(Object.class)) {
	sb.append("<code>" + getTypeAnchor(annotation.returnType()) + "</code><br>");
	}
	}

	sb.append(annotation.doc() + "\n");
	printParams(moduleId, annotation, sb);
	}

	// Elide self parameter from mandatoryPositionals in class methods.
	private static Param[] adjustedMandatoryPositionals(SkylarkSignature annotation) {
	Param[] mandatoryPos = annotation.mandatoryPositionals();
	if (mandatoryPos.length > 0
	&& annotation.objectType() != Object.class
	&& !FuncallExpression.isNamespace(annotation.objectType())) {
	// Skip the self parameter, which is the first mandatory positional parameter.
	return Arrays.copyOfRange(mandatoryPos, 1, mandatoryPos.length);
	} else {
	return mandatoryPos;
	}
	}

	private void printParams(String moduleId, SkylarkSignature annotation, StringBuilder sb) {
	Param[] mandatoryPos = adjustedMandatoryPositionals(annotation);
	Param[] optionalPos = annotation.optionalPositionals();
	Param[] optionalKey = annotation.optionalNamedOnly();
	Param[] mandatoryKey = annotation.mandatoryNamedOnly();
	Param[] star = annotation.extraPositionals();
	Param[] starStar = annotation.extraKeywords();

	if (mandatoryPos.length + optionalPos.length + optionalKey.length + mandatoryKey.length
	+ star.length + starStar.length > 0) {
	sb.append("<h4>Parameters</h4>\n");
	printParams(moduleId, annotation.name(), mandatoryPos, sb);
	printParams(moduleId, annotation.name(), optionalPos, sb);
	printParams(moduleId, annotation.name(), star, sb);
	printParams(moduleId, annotation.name(), mandatoryKey, sb);
	printParams(moduleId, annotation.name(), optionalKey, sb);
	printParams(moduleId, annotation.name(), starStar, sb);
	} else {
	sb.append("<br/>\n");
	}
	}

	private void generateDirectJavaMethodDoc(String objectName, String methodName,
	Method method, SkylarkCallable annotation, StringBuilder sb) {
	if (!annotation.documented()) {
	return;
	}
	if (annotation.doc().isEmpty()) {
	throw new RuntimeException(String.format(
	"empty SkylarkCallable.doc() for object %s, method %s", objectName, methodName));
	}

	sb.append(String.format("<li><h3 id=\"modules.%s.%s\">%s</h3>\n%s\n",
	objectName,
	methodName,
	methodName,
	getSignature(objectName, methodName, method)))
	.append(annotation.doc())
	.append(getReturnTypeExtraMessage(annotation))
	.append("\n");
	}

	private String getReturnTypeExtraMessage(SkylarkCallable annotation) {
	if (annotation.allowReturnNones()) {
	return " May return <code>None</code>.\n";
	}
	return "";
	}

	private String getSignature(String objectName, String methodName, Method method) {
	String args = method.getAnnotation(SkylarkCallable.class).structField()
	? "" : "(" + getParameterString(method) + ")";

	return String.format("<code>%s %s.%s%s</code><br>",
	getTypeAnchor(method.getReturnType()), objectName, methodName, args);
	}

	private String getSignature(String objectName, SkylarkSignature method) {
	List<String> argList = new ArrayList<>();
	for (Param param : adjustedMandatoryPositionals(method)) {
	argList.add(param.name());
	}
	for (Param param : method.optionalPositionals()) {
	argList.add(param.name() + "?"); // or should we use pythonic " = &#ellipsis;" instead?
	}
	for (Param param : method.extraPositionals()) {
	argList.add("*" + param.name());
	}
	if (method.extraPositionals().length == 0
	&& (method.optionalNamedOnly().length > 0 \|\| method.mandatoryNamedOnly().length > 0)) {
	argList.add("*");
	}
	for (Param param : method.mandatoryNamedOnly()) {
	argList.add(param.name());
	}
	for (Param param : method.optionalNamedOnly()) {
	argList.add(param.name() + "?"); // or should we be more pythonic with this? " = ..."
	}
	for (Param param : method.extraKeywords()) {
	argList.add("**" + param.name());
	}
	String args = "(" + Joiner.on(", ").join(argList) + ")";
	if (!objectName.equals(TOP_LEVEL_ID)) {
	return String.format("<code>%s %s.%s%s</code><br>\n",
	getTypeAnchor(method.returnType()), objectName, method.name(), args);
	} else {
	return String.format("<code>%s %s%s</code><br>\n",
	getTypeAnchor(method.returnType()), method.name(), args);
	}
	}

	private String getTypeAnchor(Class<?> returnType, Class<?> generic1) {
	return getTypeAnchor(returnType) + " of " + getTypeAnchor(generic1) + "s";
	}

	private String getTypeAnchor(Class<?> type) {
	if (type.equals(Boolean.class) \|\| type.equals(boolean.class)) {
	return "<a class=\"anchor\" href=\"#modules._top_level.bool\">bool</a>";
	} else if (type.equals(String.class)) {
	return "<a class=\"anchor\" href=\"#modules.string\">string</a>";
	} else if (Map.class.isAssignableFrom(type)) {
	return "<a class=\"anchor\" href=\"#modules.dict\">dict</a>";
	} else if (List.class.isAssignableFrom(type) \|\| SkylarkList.class.isAssignableFrom(type)
	\|\| type == HackHackEitherList.class) {
	// Annotated Java methods can return simple java.util.Lists (which get auto-converted).
	return "<a class=\"anchor\" href=\"#modules.list\">list</a>";
	} else if (type.equals(Void.TYPE) \|\| type.equals(NoneType.class)) {
	return "<a class=\"anchor\" href=\"#modules." + TOP_LEVEL_ID + ".None\">None</a>";
	} else if (type.isAnnotationPresent(SkylarkModule.class)) {
	// TODO(bazel-team): this can produce dead links for types don't show up in the doc.
	// The correct fix is to generate those types (e.g. SkylarkFileType) too.
	String module = type.getAnnotation(SkylarkModule.class).name();
	return "<a class=\"anchor\" href=\"#modules." + module + "\">" + module + "</a>";
	} else {
	return EvalUtils.getDataTypeNameFromClass(type);
	}
	}

	private String getParameterString(Method method) {
	return Joiner.on(", ").join(Iterables.transform(
	ImmutableList.copyOf(method.getParameterTypes()), new Function<Class<?>, String>() {
	@Override
	public String apply(Class<?> input) {
	return getTypeAnchor(input);
	}
	}));
	}

	private void printParams(String moduleId, String methodName,
	Param[] params, StringBuilder sb) {
	if (params.length > 0) {
	sb.append("<ul>\n");
	for (Param param : params) {
	String paramType = param.type().equals(Object.class) ? ""
	: (param.generic1().equals(Object.class)
	? " (" + getTypeAnchor(param.type()) + ")"
	: " (" + getTypeAnchor(param.type(), param.generic1()) + ")");
	sb.append(String.format("\t<li id=\"modules.%s.%s.%s\"><code>%s%s</code>: ",
	moduleId,
	methodName,
	param.name(),
	param.name(),
	paramType))
	.append(param.doc())
	.append("\n\t</li>\n");
	}
	sb.append("</ul>\n");
	}
	}

	private Map<String, SkylarkModuleDoc> collectBuiltinModules() {
	Map<String, SkylarkModuleDoc> modules = new HashMap<>();
	collectBuiltinDoc(modules, Environment.class.getDeclaredFields());
	collectBuiltinDoc(modules, MethodLibrary.class.getDeclaredFields());
	for (Class<?> moduleClass : SkylarkModules.MODULES) {
	collectBuiltinDoc(modules, moduleClass.getDeclaredFields());
	}
	return modules;
	}

	private Map<SkylarkModule, Class<?>> collectBuiltinJavaObjects() {
	Map<SkylarkModule, Class<?>> modules = new HashMap<>();
	collectBuiltinModule(modules, SkylarkRuleContext.class);
	collectBuiltinModule(modules, TransitiveInfoCollection.class);
	return modules;
	}

	/**
	* Returns the top level modules and functions with their documentation in a command-line
	* printable format.
	*/
	public Map<String, String> collectTopLevelModules() {
	Map<String, String> modules = new TreeMap<>();
	for (SkylarkModuleDoc doc : collectBuiltinModules().values()) {
	if (doc.getAnnotation() == getTopLevelModule()) {
	for (Map.Entry<String, SkylarkBuiltinMethod> entry : doc.getBuiltinMethods().entrySet()) {
	if (entry.getValue().annotation.documented()) {
	modules.put(entry.getKey(),
	DocgenConsts.toCommandLineFormat(entry.getValue().annotation.doc()));
	}
	}
	} else {
	modules.put(doc.getAnnotation().name(),
	DocgenConsts.toCommandLineFormat(doc.getAnnotation().doc()));
	}
	}
	return modules;
	}

	/**
	* Returns the API doc for the specified Skylark object in a command line printable format,
	* params[0] identifies either a module or a top-level object, the optional params[1] identifies a
	* method in the module.<br>
	* Returns null if no Skylark object is found.
	*/
	public String getCommandLineAPIDoc(String[] params) {
	Map<String, SkylarkModuleDoc> modules = collectModules();
	SkylarkModuleDoc toplevelModuleDoc = modules.get(getTopLevelModule().name());
	if (modules.containsKey(params[0])) {
	// Top level module
	SkylarkModuleDoc module = modules.get(params[0]);
	if (params.length == 1) {
	String moduleName = module.getAnnotation().name();
	StringBuilder sb = new StringBuilder();
	sb.append(moduleName).append("\n\t").append(module.getAnnotation().doc()).append("\n");
	// Print the signature of all built-in methods
	for (SkylarkBuiltinMethod method : module.getBuiltinMethods().values()) {
	printBuiltinFunctionDoc(moduleName, method.annotation, sb);
	}
	// Print all Java methods
	for (SkylarkJavaMethod method : module.getJavaMethods()) {
	printJavaFunctionDoc(moduleName, method, sb);
	}
	return DocgenConsts.toCommandLineFormat(sb.toString());
	} else {
	return getFunctionDoc(module.getAnnotation().name(), params[1], module);
	}
	} else if (toplevelModuleDoc.getBuiltinMethods().containsKey(params[0])){
	// Top level object / function
	return getFunctionDoc(null, params[0], toplevelModuleDoc);
	}
	return null;
	}

	private String getFunctionDoc(String moduleName, String methodName, SkylarkModuleDoc module) {
	if (module.getBuiltinMethods().containsKey(methodName)) {
	// Create the doc for the built-in function
	SkylarkBuiltinMethod method = module.getBuiltinMethods().get(methodName);
	StringBuilder sb = new StringBuilder();
	printBuiltinFunctionDoc(moduleName, method.annotation, sb);
	printParams(moduleName, method.annotation, sb);
	return DocgenConsts.removeDuplicatedNewLines(DocgenConsts.toCommandLineFormat(sb.toString()));
	} else {
	// Search if there are matching Java functions
	StringBuilder sb = new StringBuilder();
	boolean foundMatchingMethod = false;
	for (SkylarkJavaMethod method : module.getJavaMethods()) {
	if (method.name.equals(methodName)) {
	printJavaFunctionDoc(moduleName, method, sb);
	foundMatchingMethod = true;
	}
	}
	if (foundMatchingMethod) {
	return DocgenConsts.toCommandLineFormat(sb.toString());
	}
	}
	return null;
	}

	private void printBuiltinFunctionDoc(
	String moduleName, SkylarkSignature annotation, StringBuilder sb) {
	if (moduleName != null) {
	sb.append(moduleName).append(".");
	}
	sb.append(annotation.name()).append("\n\t").append(annotation.doc()).append("\n");
	}

	private void printJavaFunctionDoc(String moduleName, SkylarkJavaMethod method, StringBuilder sb) {
	sb.append(getSignature(moduleName, method.name, method.method))
	.append("\t").append(method.callable.doc()).append("\n");
	}

	private void collectBuiltinModule(
	Map<SkylarkModule, Class<?>> modules, Class<?> moduleClass) {
	if (moduleClass.isAnnotationPresent(SkylarkModule.class)) {
	SkylarkModule skylarkModule = moduleClass.getAnnotation(SkylarkModule.class);
	modules.put(skylarkModule, moduleClass);
	}
	}

	private void collectBuiltinDoc(Map<String, SkylarkModuleDoc> modules, Field[] fields) {
	for (Field field : fields) {
	if (field.isAnnotationPresent(SkylarkSignature.class)) {
	SkylarkSignature skylarkSignature = field.getAnnotation(SkylarkSignature.class);
	Class<?> moduleClass = skylarkSignature.objectType();
	SkylarkModule skylarkModule = moduleClass.equals(Object.class)
	? getTopLevelModule()
	: moduleClass.getAnnotation(SkylarkModule.class);
	if (!modules.containsKey(skylarkModule.name())) {
	modules.put(skylarkModule.name(), new SkylarkModuleDoc(skylarkModule, moduleClass));
	}
	modules.get(skylarkModule.name()).getBuiltinMethods()
	.put(skylarkSignature.name(),
	new SkylarkBuiltinMethod(skylarkSignature, field.getType()));
	}
	}
	}
	}