src/main/java/com/google/devtools/build/docgen/StarlarkDocumentationProcessor.java

// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.docgen;

import com.google.common.collect.ImmutableList;
import com.google.devtools.build.docgen.annot.DocCategory;
import com.google.devtools.build.docgen.starlark.StarlarkBuiltinDoc;
import com.google.devtools.build.docgen.starlark.StarlarkDocExpander;
import com.google.devtools.build.docgen.starlark.StarlarkMethodDoc;
import com.google.devtools.build.lib.util.Classpath.ClassPathException;
import java.io.File;
import java.io.IOException;
import java.text.Collator;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.TreeMap;
import net.starlark.java.annot.StarlarkBuiltin;

/** A class to assemble documentation for Starlark. */
public final class StarlarkDocumentationProcessor {

  private static final ImmutableList<Category> GLOBAL_CATEGORIES =
      ImmutableList.<Category>of(Category.NONE, Category.TOP_LEVEL_TYPE);

  private StarlarkDocumentationProcessor() {}

  /** Generates the Starlark documentation to the given output directory. */
  public static void generateDocumentation(String outputDir, String... args)
      throws IOException, ClassPathException {
    Map<String, String> options = parseOptions(args);

    String docsRoot = options.get("--starlark_docs_root");
    if (docsRoot != null) {
      DocgenConsts.starlarkDocsRoot = docsRoot;
    }

    String linkMapPath = options.get("--link_map_path");
    if (linkMapPath == null) {
      throw new IllegalArgumentException("Required option '--link_map_path' is missing.");
    }

    DocLinkMap linkMap = DocLinkMap.createFromFile(linkMapPath);
    StarlarkDocExpander expander =
        new StarlarkDocExpander(new RuleLinkExpander(/* singlePage= */ false, linkMap));

    Map<String, StarlarkBuiltinDoc> modules =
        new TreeMap<>(StarlarkDocumentationCollector.getAllModules(expander));

    // Generate the top level module first in the doc
    StarlarkBuiltinDoc topLevelModule =
        modules.remove(StarlarkDocumentationCollector.getTopLevelModule().name());
    writePage(outputDir, topLevelModule);

    // Use a LinkedHashMap to preserve ordering of categories, as the output iterates over
    // this map's entry set to determine category ordering.
    Map<Category, List<StarlarkBuiltinDoc>> modulesByCategory = new LinkedHashMap<>();
    for (Category c : Category.values()) {
      modulesByCategory.put(c, new ArrayList<StarlarkBuiltinDoc>());
    }

    modulesByCategory.get(Category.of(topLevelModule.getAnnotation())).add(topLevelModule);

    for (StarlarkBuiltinDoc module : modules.values()) {
      if (module.getAnnotation().documented()) {
        writePage(outputDir, module);
        modulesByCategory.get(Category.of(module.getAnnotation())).add(module);
      }
    }
    Collator us = Collator.getInstance(Locale.US);
    for (List<StarlarkBuiltinDoc> module : modulesByCategory.values()) {
      Collections.sort(module, (doc1, doc2) -> us.compare(doc1.getTitle(), doc2.getTitle()));
    }
    writeCategoryPage(Category.CORE, outputDir, modulesByCategory, expander);
    writeCategoryPage(Category.CONFIGURATION_FRAGMENT, outputDir, modulesByCategory, expander);
    writeCategoryPage(Category.BUILTIN, outputDir, modulesByCategory, expander);
    writeCategoryPage(Category.PROVIDER, outputDir, modulesByCategory, expander);
    writeNavPage(outputDir, modulesByCategory.get(Category.TOP_LEVEL_TYPE));

    // In the code, there are two StarlarkModuleCategory instances that have no heading:
    // TOP_LEVEL_TYPE and NONE.

    // TOP_LEVEL_TYPE also contains the "global" module.
    // We remove both categories and the "global" module from the map and display them manually:
    // - Methods in the "global" module are displayed under "Global Methods and Constants".
    // - Modules in both categories are displayed under "Global Modules" (except for the global
    // module itself).
    List<String> globalFunctions = new ArrayList<>();
    List<String> globalConstants = new ArrayList<>();
    StarlarkBuiltinDoc globalModule = findGlobalModule(modulesByCategory);
    for (StarlarkMethodDoc method : globalModule.getMethods()) {
      if (method.documented()) {
        if (method.isCallable()) {
          globalFunctions.add(method.getName());
        } else {
          globalConstants.add(method.getName());
        }
      }
    }

    List<StarlarkBuiltinDoc> globalModules = new ArrayList<>();
    for (Category globalCategory : GLOBAL_CATEGORIES) {
      for (StarlarkBuiltinDoc module : modulesByCategory.remove(globalCategory)) {
        if (!module.getName().equals(globalModule.getName())) {
          globalModules.add(module);
        }
      }
    }

    Collections.sort(globalModules, (doc1, doc2) -> us.compare(doc1.getName(), doc2.getName()));
    writeOverviewPage(
        outputDir,
        globalModule.getName(),
        globalFunctions,
        globalConstants,
        globalModules,
        modulesByCategory);
    if (shouldCreateToc(options)) {
      writeTableOfContents(outputDir, globalModules, modulesByCategory);
    }
  }

  private static StarlarkBuiltinDoc findGlobalModule(
      Map<Category, List<StarlarkBuiltinDoc>> modulesByCategory) {
    List<StarlarkBuiltinDoc> topLevelModules = modulesByCategory.get(Category.TOP_LEVEL_TYPE);
    String globalModuleName = StarlarkDocumentationCollector.getTopLevelModule().name();
    for (StarlarkBuiltinDoc module : topLevelModules) {
      if (module.getName().equals(globalModuleName)) {
        return module;
      }
    }

    throw new IllegalStateException("No globals module in the top level category.");
  }

  private static void writePage(String outputDir, StarlarkBuiltinDoc module) throws IOException {
    File starlarkDocPath = new File(outputDir + "/" + module.getName() + ".html");
    Page page = TemplateEngine.newPage(DocgenConsts.STARLARK_LIBRARY_TEMPLATE);
    page.add("module", module);
    page.write(starlarkDocPath);
  }

  private static void writeCategoryPage(
      Category category,
      String outputDir,
      Map<Category, List<StarlarkBuiltinDoc>> modules,
      StarlarkDocExpander expander)
      throws IOException {
    File starlarkDocPath =
        new File(String.format("%s/starlark-%s.html", outputDir, category.getTemplateIdentifier()));
    Page page = TemplateEngine.newPage(DocgenConsts.STARLARK_MODULE_CATEGORY_TEMPLATE);
    page.add("category", category);
    page.add("modules", modules.get(category));
    page.add("description", expander.expand(category.description));
    page.write(starlarkDocPath);
  }

  private static void writeNavPage(String outputDir, List<StarlarkBuiltinDoc> navModules)
      throws IOException {
    File navFile = new File(outputDir + "/starlark-nav.html");
    Page page = TemplateEngine.newPage(DocgenConsts.STARLARK_NAV_TEMPLATE);
    page.add("modules", navModules);
    page.write(navFile);
  }

  private static void writeOverviewPage(
      String outputDir,
      String globalModuleName,
      List<String> globalFunctions,
      List<String> globalConstants,
      List<StarlarkBuiltinDoc> globalModules,
      Map<Category, List<StarlarkBuiltinDoc>> modulesPerCategory)
      throws IOException {
    File starlarkDocPath = new File(outputDir + "/starlark-overview.html");
    Page page = TemplateEngine.newPage(DocgenConsts.STARLARK_OVERVIEW_TEMPLATE);
    page.add("global_name", globalModuleName);
    page.add("global_functions", globalFunctions);
    page.add("global_constants", globalConstants);
    page.add("global_modules", globalModules);
    page.add("modules", modulesPerCategory);
    page.write(starlarkDocPath);
  }

  private static void writeTableOfContents(
      String outputDir,
      List<StarlarkBuiltinDoc> globalModules,
      Map<Category, List<StarlarkBuiltinDoc>> modulesPerCategory)
      throws IOException {
    File starlarkDocPath = new File(outputDir + "/_toc.yaml");
    Page page = TemplateEngine.newPage(DocgenConsts.STARLARK_TOC_TEMPLATE);
    page.add("global_modules", globalModules);
    page.add("modules", modulesPerCategory);
    page.write(starlarkDocPath);
  }

  private static Map<String, String> parseOptions(String... args) {
    Map<String, String> options = new HashMap<>();
    for (String arg : args) {
      if (arg.startsWith("--")) {
        String[] parts = arg.split("=", 2);
        options.put(parts[0], parts.length > 1 ? parts[1] : null);
      }
    }
    return options;
  }

  private static boolean shouldCreateToc(Map<String, String> options) {
    String arg = options.get("--create_toc");
    if (arg == null) {
      return false;
    }
    return Boolean.parseBoolean(arg) || arg.equals("1");
  }

  /**
   * An enumeration of categories used to organize the API index. Instances of this class are
   * accessed by templates, using reflection.
   */
  public enum Category {
    CONFIGURATION_FRAGMENT(
        "Configuration Fragments",
        "Configuration fragments give rules access to "
            + "language-specific parts of <a href=\"configuration.html\">"
            + "configuration</a>. "
            + "<p>Rule implementations can get them using "
            + "<code><a href=\"ctx.html#fragments\">ctx."
            + "fragments</a>.<i>[fragment name]</i></code>"),

    PROVIDER(
        "Providers",
        "This section lists providers available on built-in rules. See the <a"
            + " href='https://bazel.build/extending/rules#providers'>Rules page</a> for more on"
            + " providers."),

    BUILTIN("Built-in Types", "This section lists types of Starlark objects."),

    // Used for top-level modules of functions in the global namespace. Such modules will always
    // be usable solely by accessing their members, via modulename.funcname() or
    // modulename.constantname.
    // Examples: attr, cc_common, config, java_common
    TOP_LEVEL_TYPE(null, null),

    CORE(
        "Core Starlark data types",
        "This section lists the data types of the <a"
            + " href='https://github.com/bazelbuild/starlark/blob/master/spec.md#built-in-constants-and-functions'>Starlark"
            + " core language</a>."),

    // Legacy uncategorized type; these are treated like TOP_LEVEL_TYPE in documentation.
    NONE(null, null);

    // Maps (essentially free-form) strings in annotations to permitted categories.
    static Category of(StarlarkBuiltin annot) {
      switch (annot.category()) {
        case DocCategory.CONFIGURATION_FRAGMENT:
          return CONFIGURATION_FRAGMENT;
        case DocCategory.PROVIDER:
          return PROVIDER;
        case DocCategory.BUILTIN:
          return BUILTIN;
        case DocCategory.TOP_LEVEL_TYPE:
          return TOP_LEVEL_TYPE;
        case DocCategory.NONE:
          return NONE;
        case "core": // interpreter built-ins (e.g. int)
        case "core.lib": // Starlark standard modules (e.g. json)
          return CORE;
        case "": // no annotation
          return TOP_LEVEL_TYPE;
        default:
          throw new IllegalStateException(
              String.format(
                  "docgen does not recognize DocCategory '%s' for StarlarkBuiltin '%s'",
                  annot.category(), annot.name()));
      }
    }

    private Category(String title, String description) {
      this.title = title;
      this.description = description;
    }

    private final String title;
    private final String description;

    public String getTitle() {
      return title;
    }

    public String getDescription() {
      return description;
    }

    public String getTemplateIdentifier() {
      return name().toLowerCase().replace("_", "-");
    }
  }
}