blob: 11b904321a029d1f4a2f4fdf2f1887289e71caab [file] [log] [blame]
// Copyright 2018 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.skydoc;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Functions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.Label.PackageContext;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
import com.google.devtools.build.lib.cmdline.RepositoryMapping;
import com.google.devtools.build.lib.collect.nestedset.Depset;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.collect.nestedset.Order;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
import com.google.devtools.build.runfiles.Runfiles;
import com.google.devtools.build.runfiles.RunfilesForStardoc;
import com.google.devtools.build.skydoc.fakebuildapi.FakeApi;
import com.google.devtools.build.skydoc.fakebuildapi.FakeDeepStructure;
import com.google.devtools.build.skydoc.fakebuildapi.FakeProviderApi;
import com.google.devtools.build.skydoc.fakebuildapi.FakeStructApi;
import com.google.devtools.build.skydoc.rendering.AspectInfoWrapper;
import com.google.devtools.build.skydoc.rendering.DocstringParseException;
import com.google.devtools.build.skydoc.rendering.ProtoRenderer;
import com.google.devtools.build.skydoc.rendering.ProviderInfoWrapper;
import com.google.devtools.build.skydoc.rendering.RuleInfoWrapper;
import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.AspectInfo;
import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.ProviderInfo;
import com.google.devtools.build.skydoc.rendering.proto.StardocOutputProtos.RuleInfo;
import com.google.devtools.common.options.OptionsParser;
import java.io.BufferedOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.stream.Collectors;
import net.starlark.java.annot.Param;
import net.starlark.java.annot.StarlarkBuiltin;
import net.starlark.java.annot.StarlarkMethod;
import net.starlark.java.eval.Dict;
import net.starlark.java.eval.EvalException;
import net.starlark.java.eval.Module;
import net.starlark.java.eval.Mutability;
import net.starlark.java.eval.Starlark;
import net.starlark.java.eval.StarlarkCallable;
import net.starlark.java.eval.StarlarkFunction;
import net.starlark.java.eval.StarlarkSemantics;
import net.starlark.java.eval.StarlarkThread;
import net.starlark.java.eval.StarlarkValue;
import net.starlark.java.lib.json.Json;
import net.starlark.java.syntax.FileOptions;
import net.starlark.java.syntax.ParserInput;
import net.starlark.java.syntax.Program;
import net.starlark.java.syntax.Resolver;
import net.starlark.java.syntax.Resolver.Scope;
import net.starlark.java.syntax.StarlarkFile;
import net.starlark.java.syntax.SyntaxError;
/**
* Main entry point for the Skydoc binary.
*
* <p>Skydoc generates human-readable documentation for relevant details of Starlark files by
* running a Starlark interpreter with a fake implementation of the build API.
*
* <p>Currently, Skydoc generates documentation for Starlark rule definitions (discovered by
* invocations of the build API function {@code rule()}.
*
* <p>Usage:
*
* <pre>
* skydoc {target_starlark_file_label} {output_file} [symbol_name]...
* </pre>
*
* <p>Generates documentation for all exported symbols of the target Starlark file that are
* specified in the list of symbol names. If no symbol names are supplied, outputs documentation for
* all exported symbols in the target Starlark file.
*/
public class SkydocMain {
private final EventHandler eventHandler = new SystemOutEventHandler();
private final LinkedHashSet<Path> pending = new LinkedHashSet<>();
private final Map<Path, Module> loaded = new HashMap<>();
private final String workspaceName;
private final Runfiles.Preloaded runfiles;
public SkydocMain(String workspaceName, Runfiles.Preloaded runfiles) {
this.workspaceName = workspaceName;
this.runfiles = runfiles;
}
public static void main(String[] args)
throws IOException, InterruptedException, LabelSyntaxException, DocstringParseException {
OptionsParser parser =
OptionsParser.builder()
.optionsClasses(BuildLanguageOptions.class, SkydocOptions.class)
.build();
parser.parseAndExitUponError(args);
BuildLanguageOptions semanticsOptions = parser.getOptions(BuildLanguageOptions.class);
SkydocOptions skydocOptions = parser.getOptions(SkydocOptions.class);
String targetFileLabelString;
String outputPath;
if (Strings.isNullOrEmpty(skydocOptions.targetFileLabel)
|| Strings.isNullOrEmpty(skydocOptions.outputFilePath)) {
throw new IllegalArgumentException("Expected a target file label and an output file path.");
}
targetFileLabelString = skydocOptions.targetFileLabel;
outputPath = skydocOptions.outputFilePath;
Label targetFileLabel = Label.parseCanonical(targetFileLabelString);
ImmutableMap.Builder<String, RuleInfo> ruleInfoMap = ImmutableMap.builder();
ImmutableMap.Builder<String, ProviderInfo> providerInfoMap = ImmutableMap.builder();
ImmutableMap.Builder<String, StarlarkFunction> userDefinedFunctions = ImmutableMap.builder();
ImmutableMap.Builder<String, AspectInfo> aspectInfoMap = ImmutableMap.builder();
Module module = null;
try {
module =
new SkydocMain(skydocOptions.workspaceName, Runfiles.preload())
.eval(
semanticsOptions.toStarlarkSemantics(),
// The label passed on the command line is assumed to be canonical.
targetFileLabel,
ruleInfoMap,
providerInfoMap,
userDefinedFunctions,
aspectInfoMap);
} catch (StarlarkEvaluationException | EvalException exception) {
exception.printStackTrace();
System.err.println("Stardoc documentation generation failed: " + exception.getMessage());
System.exit(1);
}
ProtoRenderer renderer =
render(
module,
ImmutableSet.copyOf(skydocOptions.symbolNames),
ruleInfoMap.buildOrThrow(),
providerInfoMap.buildOrThrow(),
userDefinedFunctions.buildOrThrow(),
aspectInfoMap.buildOrThrow());
try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outputPath))) {
renderer.writeModuleInfo(out);
}
}
private static boolean validSymbolName(ImmutableSet<String> symbolNames, String symbolName) {
if (symbolNames.isEmpty()) {
// Symbols prefixed with an underscore are private, and thus, by default, documentation
// should not be generated for them.
return !symbolName.startsWith("_");
} else if (symbolNames.contains(symbolName)) {
return true;
} else if (symbolName.contains(".")) {
return symbolNames.contains(symbolName.substring(0, symbolName.indexOf('.')));
}
return false;
}
/**
* Renders a Starlark module to proto form.
*
* @param symbolNames symbols to render; if empty, all non-private symbols (i.e. those whose names
* do not start with '_') will be rendered.
* @param ruleInfoMap a map of rule definition information for named rules. Keys are exported
* names of rules, and values are their {@link RuleInfo} rule descriptions. For example,
* 'my_rule = rule(...)' has key 'my_rule'
* @param providerInfoMap a map of provider definition information for named providers. Keys are
* exported names of providers, and values are their {@link ProviderInfo} descriptions. For
* example, 'my_provider = provider(...)' has key 'my_provider'
* @param userDefinedFunctions a map of user-defined functions. Keys are exported names of
* functions, and values are the {@link StarlarkFunction} objects. For example, 'def
* my_function(foo):' is a function with key 'my_function'.
* @param aspectInfoMap a map of aspect definition information for named aspects. Keys are
* exported names of aspects, and values are the {@link AspectInfo} asepct descriptions. For
* example, 'my_aspect = aspect(...)' has key 'my_aspect'
*/
@VisibleForTesting
public static ProtoRenderer render(
Module module,
ImmutableSet<String> symbolNames,
ImmutableMap<String, RuleInfo> ruleInfoMap,
ImmutableMap<String, ProviderInfo> providerInfoMap,
ImmutableMap<String, StarlarkFunction> userDefinedFunctions,
ImmutableMap<String, AspectInfo> aspectInfoMap)
throws DocstringParseException {
ImmutableMap<String, RuleInfo> filteredRuleInfos =
ruleInfoMap.entrySet().stream()
.filter(entry -> validSymbolName(symbolNames, entry.getKey()))
.collect(toImmutableMap(Entry::getKey, Entry::getValue));
ImmutableMap<String, ProviderInfo> filteredProviderInfos =
providerInfoMap.entrySet().stream()
.filter(entry -> validSymbolName(symbolNames, entry.getKey()))
.collect(toImmutableMap(Entry::getKey, Entry::getValue));
ImmutableMap<String, StarlarkFunction> filteredStarlarkFunctions =
userDefinedFunctions.entrySet().stream()
.filter(entry -> validSymbolName(symbolNames, entry.getKey()))
.collect(toImmutableMap(Entry::getKey, Entry::getValue));
ImmutableMap<String, AspectInfo> filteredAspectInfos =
aspectInfoMap.entrySet().stream()
.filter(entry -> validSymbolName(symbolNames, entry.getKey()))
.collect(toImmutableMap(Entry::getKey, Entry::getValue));
String moduleDocstring = module.getDocumentation();
if (moduleDocstring == null) {
moduleDocstring = "";
}
return new ProtoRenderer()
.appendRuleInfos(filteredRuleInfos.values())
.appendProviderInfos(filteredProviderInfos.values())
.appendStarlarkFunctionInfos(filteredStarlarkFunctions)
.appendAspectInfos(filteredAspectInfos.values())
.setModuleDocstring(moduleDocstring);
}
/**
* Evaluates/interprets the Starlark file at a given path and its transitive Starlark dependencies
* using a fake build API and collects information about all rule definitions made in the root
* Starlark file.
*
* @param canonicalLabel the canonical label of the Starlark file to evaluate
* @param ruleInfoMap a map builder to be populated with rule definition information for named
* rules. Keys are exported names of rules, and values are their {@link RuleInfo} rule
* descriptions. For example, 'my_rule = rule(...)' has key 'my_rule'
* @param providerInfoMap a map builder to be populated with provider definition information for
* named providers. Keys are exported names of providers, and values are their {@link
* ProviderInfo} descriptions. For example, 'my_provider = provider(...)' has key
* 'my_provider'
* @param userDefinedFunctionMap a map builder to be populated with user-defined functions. Keys
* are exported names of functions, and values are the {@link StarlarkFunction} objects. For
* example, 'def my_function(foo):' is a function with key 'my_function'.
* @param aspectInfoMap a map builder to be populated with aspect definition information for named
* aspects. Keys are exported names of aspects, and values are the {@link AspectInfo} asepct
* descriptions. For example, 'my_aspect = aspect(...)' has key 'my_aspect'
* @throws InterruptedException if evaluation is interrupted
*/
@VisibleForTesting
public Module eval(
StarlarkSemantics semantics,
Label canonicalLabel,
ImmutableMap.Builder<String, RuleInfo> ruleInfoMap,
ImmutableMap.Builder<String, ProviderInfo> providerInfoMap,
ImmutableMap.Builder<String, StarlarkFunction> userDefinedFunctionMap,
ImmutableMap.Builder<String, AspectInfo> aspectInfoMap)
throws InterruptedException,
IOException,
LabelSyntaxException,
EvalException,
StarlarkEvaluationException {
List<RuleInfoWrapper> ruleInfoList = new ArrayList<>();
List<ProviderInfoWrapper> providerInfoList = new ArrayList<>();
List<AspectInfoWrapper> aspectInfoList = new ArrayList<>();
Module module =
recursiveEval(semantics, canonicalLabel, ruleInfoList, providerInfoList, aspectInfoList);
Map<StarlarkCallable, RuleInfoWrapper> ruleFunctions =
ruleInfoList.stream()
.collect(
Collectors.toMap(RuleInfoWrapper::getIdentifierFunction, Functions.identity()));
Map<StarlarkCallable, ProviderInfoWrapper> providerInfos =
providerInfoList.stream()
.collect(Collectors.toMap(ProviderInfoWrapper::getIdentifier, Functions.identity()));
Map<StarlarkCallable, AspectInfoWrapper> aspectFunctions =
aspectInfoList.stream()
.collect(
Collectors.toMap(AspectInfoWrapper::getIdentifierFunction, Functions.identity()));
// Sort the globals bindings by name.
TreeMap<String, Object> sortedBindings = new TreeMap<>(module.getGlobals());
for (Entry<String, Object> envEntry : sortedBindings.entrySet()) {
if (ruleFunctions.containsKey(envEntry.getValue())) {
RuleInfo.Builder ruleInfoBuild = ruleFunctions.get(envEntry.getValue()).getRuleInfo();
RuleInfo ruleInfo = ruleInfoBuild.setRuleName(envEntry.getKey()).build();
ruleInfoMap.put(envEntry.getKey(), ruleInfo);
}
if (providerInfos.containsKey(envEntry.getValue())) {
ProviderInfo.Builder providerInfoBuild =
providerInfos.get(envEntry.getValue()).getProviderInfo();
ProviderInfo providerInfo = providerInfoBuild.setProviderName(envEntry.getKey()).build();
providerInfoMap.put(envEntry.getKey(), providerInfo);
}
if (envEntry.getValue() instanceof StarlarkFunction) {
StarlarkFunction userDefinedFunction = (StarlarkFunction) envEntry.getValue();
userDefinedFunctionMap.put(envEntry.getKey(), userDefinedFunction);
}
if (envEntry.getValue() instanceof FakeStructApi) {
String namespaceName = envEntry.getKey();
FakeStructApi namespace = (FakeStructApi) envEntry.getValue();
putStructFields(
namespaceName, namespace, ruleFunctions, ruleInfoMap, userDefinedFunctionMap);
}
if (aspectFunctions.containsKey(envEntry.getValue())) {
AspectInfo.Builder aspectInfoBuild =
aspectFunctions.get(envEntry.getValue()).getAspectInfo();
AspectInfo aspectInfo = aspectInfoBuild.setAspectName(envEntry.getKey()).build();
aspectInfoMap.put(envEntry.getKey(), aspectInfo);
}
}
return module;
}
/**
* Recursively adds functions defined in {@code namespace}, and in its nested namespaces, to
* {@code userDefinedFunctionMap}.
*
* <p>Each entry's key is the fully qualified function name, e.g. {@code
* "outernamespace.innernamespace.func"}. {@code namespaceName} is the fully qualified name of
* {@code namespace} itself.
*/
private static void putStructFields(
String namespaceName,
FakeStructApi namespace,
Map<StarlarkCallable, RuleInfoWrapper> ruleFunctions,
ImmutableMap.Builder<String, RuleInfo> ruleInfoMap,
ImmutableMap.Builder<String, StarlarkFunction> userDefinedFunctionMap)
throws EvalException {
for (String field : namespace.getFieldNames()) {
String qualifiedFieldName = namespaceName + "." + field;
if (ruleFunctions.containsKey(namespace.getValue(field))) {
ruleInfoMap.put(
qualifiedFieldName, ruleFunctions.get(namespace.getValue(field)).getRuleInfo().build());
} else if (namespace.getValue(field) instanceof StarlarkFunction) {
StarlarkFunction userDefinedFunction = (StarlarkFunction) namespace.getValue(field);
userDefinedFunctionMap.put(qualifiedFieldName, userDefinedFunction);
} else if (namespace.getValue(field) instanceof FakeStructApi) {
FakeStructApi innerNamespace = (FakeStructApi) namespace.getValue(field);
putStructFields(
qualifiedFieldName, innerNamespace, ruleFunctions, ruleInfoMap, userDefinedFunctionMap);
}
}
}
/**
* Recursively evaluates/interprets the Starlark file at a given path and its transitive Starlark
* dependencies using a fake build API and collects information about all rule definitions made in
* those files.
*
* @param canonicalLabel the canonical label of the Starlark file to evaluate
* @param ruleInfoList a collection of all rule definitions made so far (using rule()); this
* method will add to this list as it evaluates additional files
* @throws InterruptedException if evaluation is interrupted
*/
private Module recursiveEval(
StarlarkSemantics semantics,
Label canonicalLabel,
List<RuleInfoWrapper> ruleInfoList,
List<ProviderInfoWrapper> providerInfoList,
List<AspectInfoWrapper> aspectInfoList)
throws InterruptedException, IOException, LabelSyntaxException, StarlarkEvaluationException {
Path path = pathOfCanonicalLabel(canonicalLabel);
String sourceRepository = canonicalLabel.getRepository().getName();
if (pending.contains(path)) {
throw new StarlarkEvaluationException("cycle with " + path);
} else if (loaded.containsKey(path)) {
return loaded.get(path);
}
pending.add(path);
// Create an initial environment with a fake build API. Then use Starlark's name resolution
// step to further populate the environment with all additional symbols not in the fake build
// API but used by the program; these become FakeDeepStructures.
ImmutableMap.Builder<String, Object> initialEnvBuilder = ImmutableMap.builder();
FakeApi.addPredeclared(initialEnvBuilder, ruleInfoList, providerInfoList, aspectInfoList);
addMorePredeclared(initialEnvBuilder);
ImmutableMap<String, Object> initialEnv = initialEnvBuilder.build();
Map<String, Object> predeclaredSymbols = new HashMap<>();
predeclaredSymbols.putAll(initialEnv);
Resolver.Module predeclaredResolver =
(name) -> {
if (predeclaredSymbols.containsKey(name)) {
return Scope.PREDECLARED;
}
if (!Starlark.UNIVERSE.containsKey(name)) {
predeclaredSymbols.put(name, FakeDeepStructure.create(name));
return Scope.PREDECLARED;
}
return Resolver.Scope.UNIVERSAL;
};
// parse & compile (and get doc)
ParserInput input = ParserInput.fromLatin1(Files.readAllBytes(path), path.toString());
Program prog;
try {
StarlarkFile file = StarlarkFile.parse(input, FileOptions.DEFAULT);
prog = Program.compileFile(file, predeclaredResolver);
} catch (SyntaxError.Exception ex) {
Event.replayEventsOn(eventHandler, ex.errors());
throw new StarlarkEvaluationException(ex.getMessage());
}
// process loads
Map<String, Module> imports = new HashMap<>();
for (String load : prog.getLoads()) {
Label apparentLoad =
Label.parseWithPackageContext(
load,
PackageContext.of(
canonicalLabel.getPackageIdentifier(), RepositoryMapping.ALWAYS_FALLBACK));
Label canonicalLoad = toCanonicalLabel(apparentLoad, sourceRepository);
try {
Module loadedModule =
recursiveEval(semantics, canonicalLoad, ruleInfoList, providerInfoList, aspectInfoList);
imports.put(load, loadedModule);
} catch (NoSuchFileException noSuchFileException) {
throw new StarlarkEvaluationException(
String.format(
"File %s imported '%s', yet %s was not found.",
path, load, pathOfCanonicalLabel(canonicalLoad)),
noSuchFileException);
}
}
// execute
Module module = Module.withPredeclared(semantics, predeclaredSymbols);
try (Mutability mu = Mutability.create("Skydoc")) {
StarlarkThread thread = new StarlarkThread(mu, semantics);
// We use the default print handler, which writes to stderr.
thread.setLoader(imports::get);
// Fake Bazel's "export" hack, by which provider symbols
// bound to global variables take on the name of the global variable.
thread.setPostAssignHook(
(name, value) -> {
if (value instanceof FakeProviderApi) {
((FakeProviderApi) value).setName(name);
}
});
Starlark.execFileProgram(prog, module, thread);
} catch (EvalException ex) {
throw new StarlarkEvaluationException(ex.getMessageWithStack());
}
pending.remove(path);
loaded.put(path, module);
return module;
}
private Label toCanonicalLabel(Label apparentLabel, String sourceRepository) {
String canonicalRepositoryName =
RunfilesForStardoc.getCanonicalRepositoryName(
runfiles.withSourceRepository(sourceRepository),
apparentLabel.getRepository().getName());
return Label.parseCanonicalUnchecked(
String.format(
"@%s//%s:%s",
canonicalRepositoryName,
apparentLabel.getPackageIdentifier().getPackageFragment().getPathString(),
apparentLabel.getName()));
}
private Path pathOfCanonicalLabel(Label label) {
String runfilesDirName =
label.getRepository().isMain() ? workspaceName : label.getRepository().getName();
String rlocationPath = runfilesDirName + "/" + label.toPathFragment();
return Paths.get(runfiles.unmapped().rlocation(rlocationPath));
}
private static void addMorePredeclared(ImmutableMap.Builder<String, Object> env) {
// Add dummy declarations that would come from packages.StarlarkGlobals#getUtilToplevels()
// were Skydoc allowed to depend on it. See hack for select below.
env.put("json", Json.INSTANCE);
env.put("proto", new ProtoModule());
env.put(
"depset",
new StarlarkCallable() {
@Override
public Object fastcall(StarlarkThread thread, Object[] positional, Object[] named) {
// Accept any arguments, return empty Depset.
return Depset.of(Object.class, NestedSetBuilder.emptySet(Order.STABLE_ORDER));
}
@Override
public String getName() {
return "depset";
}
});
// Declare a fake implementation of select that just returns the first
// value in the dict. (This program is forbidden from depending on the real
// implementation of 'select' in lib.packages, and so the hacks multiply.)
env.put(
"select",
new StarlarkCallable() {
@Override
public Object fastcall(StarlarkThread thread, Object[] positional, Object[] named)
throws EvalException {
for (Map.Entry<?, ?> e : ((Dict<?, ?>) positional[0]).entrySet()) {
return e.getValue();
}
throw Starlark.errorf("select: empty dict");
}
@Override
public String getName() {
return "select";
}
});
}
@StarlarkBuiltin(name = "ProtoModule", documented = false)
private static final class ProtoModule implements StarlarkValue {
@StarlarkMethod(
name = "encode_text",
doc = ".",
parameters = {@Param(name = "x")})
public String encodeText(Object x) {
return "";
}
}
/** Exception thrown when Starlark evaluation fails (due to malformed Starlark). */
@VisibleForTesting
static class StarlarkEvaluationException extends Exception {
public StarlarkEvaluationException(String message) {
super(message);
}
public StarlarkEvaluationException(String message, Throwable cause) {
super(message, cause);
}
}
}