| // Copyright 2020 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package com.google.devtools.build.lib.skyframe; |
| |
| import com.google.common.base.Preconditions; |
| import com.google.common.collect.ImmutableMap; |
| import com.google.devtools.build.lib.actions.InconsistentFilesystemException; |
| import com.google.devtools.build.lib.cmdline.Label; |
| import com.google.devtools.build.lib.packages.PackageFactory; |
| import com.google.devtools.build.lib.packages.RuleClassProvider; |
| import com.google.devtools.build.lib.packages.StructProvider; |
| import com.google.devtools.build.lib.skyframe.BzlLoadFunction.BzlLoadFailedException; |
| import com.google.devtools.build.lib.syntax.Dict; |
| import com.google.devtools.build.lib.syntax.EvalException; |
| import com.google.devtools.build.lib.syntax.Location; |
| import com.google.devtools.build.lib.syntax.Module; |
| import com.google.devtools.build.skyframe.RecordingSkyFunctionEnvironment; |
| import com.google.devtools.build.skyframe.SkyFunction; |
| import com.google.devtools.build.skyframe.SkyFunctionException; |
| import com.google.devtools.build.skyframe.SkyFunctionException.Transience; |
| import com.google.devtools.build.skyframe.SkyKey; |
| import com.google.devtools.build.skyframe.SkyValue; |
| import java.util.LinkedHashMap; |
| import java.util.Map; |
| import javax.annotation.Nullable; |
| |
| // TODO(#11437): Teach ASTFileLookup about builtins keys, then have them modify their env the same |
| // way as BzlLoadFunction. This will allow us to define the _internal symbol for @builtins. |
| |
| // TODO(#11437): Determine places where we need to teach Skyframe about this Skyfunction. Look for |
| // special treatment of BzlLoadFunction or ASTFileLookupFunction in existing code. |
| |
| // TODO(#11437): Add support to StarlarkModuleCycleReporter to pretty-print cycles involving |
| // @builtins. Blocked on us actually loading files from @builtins. |
| |
| // TODO(#11437): Add tombstone feature: If a native symbol is a tombstone object, this signals to |
| // StarlarkBuiltinsFunction that the corresponding symbol *must* be defined by @builtins. |
| // Furthermore, if exports.bzl also says the symbol is a tombstone, any attempt to use it results |
| // in failure, as if the symbol doesn't exist at all (or with a user-friendly error message saying |
| // to migrate by adding a load()). Combine tombstones with reading the current incompatible flags |
| // within @builtins for awesomeness. |
| |
| // TODO(#11437): Currently, BUILD-loaded .bzls and WORKSPACE-loaded .bzls have the same initial |
| // static environment. Therefore, we should apply injection of top-level symbols to both |
| // environments, not just BUILD .bzls. Likewise for builtins that are shared by BUILD and WORKSPACE |
| // files, and for when the dynamic `native` values of the two .bzl dialects are unified. This can be |
| // facilitated by 1) making PackageFactory better understand the similarities between BUILD and |
| // WORKSPACE, e.g. by refactoring things like WorkspaceFactory#createWorkspaceFunctions into |
| // PackageFactory; and 2) making PackageFactory responsible for performing the actual injection |
| // (given the override mappings from exports.bzl) and returning the modified environments. Then any |
| // refactoring to unify BUILD with WORKPSACE and BUILD-bzl with WORKSPACE-bzl can proceed in |
| // PackageFactory without regard to this file. |
| |
| /** |
| * A Skyframe function that evaluates the {@code @builtins} pseudo-repository and reports the values |
| * exported by {@code @builtins//:exports.bzl}. |
| * |
| * <p>The process of "builtins injection" refers to evaluating this Skyfunction and applying its |
| * result to {@link BzlLoadFunction}'s computation. See also the <a |
| * href="https://docs.google.com/document/d/1GW7UVo1s9X0cti9OMgT3ga5ozKYUWLPk9k8c4-34rC4">design |
| * doc</a>: |
| * |
| * <p>This function has a trivial key, so there can only be one value in the build at a time. It has |
| * a single dependency, on the result of evaluating the exports.bzl file to a {@link BzlLoadValue}. |
| */ |
| public class StarlarkBuiltinsFunction implements SkyFunction { |
| |
| /** |
| * The label where {@code @builtins} symbols are exported from. (This is never conflated with any |
| * actual repository named "{@code @builtins}" because it is only accessed through a special |
| * SkyKey. |
| */ |
| private static final Label EXPORTS_ENTRYPOINT = |
| Label.parseAbsoluteUnchecked("@builtins//:exports.bzl"); |
| |
| /** Same as above, as a {@link Location} for errors. */ |
| private static final Location EXPORTS_ENTRYPOINT_LOC = |
| new Location(EXPORTS_ENTRYPOINT.getCanonicalForm(), /*line=*/ 0, /*column=*/ 0); |
| |
| /** |
| * Key for loading exports.bzl. {@code keyForBuiltins} (as opposed to {@code keyForBuild} ensures |
| * that 1) we can resolve the {@code @builtins} name appropriately, and 2) loading it does not |
| * trigger a cyclic call back into {@code StarlarkBuiltinsFunction}. |
| */ |
| private static final BzlLoadValue.Key EXPORTS_ENTRYPOINT_KEY = |
| BzlLoadValue.keyForBuiltins( |
| // TODO(#11437): Replace by EXPORTS_ENTRYPOINT once BzlLoadFunction can resolve the |
| // @builtins namespace. |
| Label.parseAbsoluteUnchecked("//tools/builtins_staging:exports.bzl")); |
| |
| // Used to obtain the default .bzl top-level environment, sans "native". |
| private final RuleClassProvider ruleClassProvider; |
| // Used to obtain the default contents of the "native" object. |
| private final PackageFactory packageFactory; |
| |
| public StarlarkBuiltinsFunction( |
| RuleClassProvider ruleClassProvider, PackageFactory packageFactory) { |
| this.ruleClassProvider = ruleClassProvider; |
| this.packageFactory = packageFactory; |
| } |
| |
| @Override |
| @Nullable |
| public SkyValue compute(SkyKey skyKey, Environment env) |
| throws StarlarkBuiltinsFunctionException, InterruptedException { |
| // skyKey is a singleton, unused. |
| |
| BzlLoadValue exportsValue; |
| try { |
| exportsValue = |
| (BzlLoadValue) env.getValueOrThrow(EXPORTS_ENTRYPOINT_KEY, BzlLoadFailedException.class); |
| } catch (BzlLoadFailedException ex) { |
| throw new StarlarkBuiltinsFunctionException( |
| BuiltinsFailedException.errorEvaluatingBuiltinsBzls(ex)); |
| } |
| if (exportsValue == null) { |
| return null; |
| } |
| |
| try { |
| return computeWithExports(exportsValue, ruleClassProvider, packageFactory); |
| } catch (BuiltinsFailedException e) { |
| throw new StarlarkBuiltinsFunctionException(e); |
| } |
| } |
| |
| /** |
| * Computes this Skyfunction under inlining of {@link BzlLoadFunction}, forwarding the given |
| * inlining state. |
| * |
| * <p>The given Skyframe environment must be a {@link RecordingSkyFunctionEnvironment}. It is |
| * unwrapped before calling {@link BzlLoadFunction}'s inlining code path. |
| * |
| * <p>Returns null on Skyframe restart or error. |
| */ |
| @Nullable |
| public static StarlarkBuiltinsValue computeInline( |
| StarlarkBuiltinsValue.Key key, // singleton value, unused |
| Environment env, |
| BzlLoadFunction.InliningState inliningState, |
| BzlLoadFunction bzlLoadFunction, |
| RuleClassProvider ruleClassProvider, |
| PackageFactory packageFactory) |
| throws BuiltinsFailedException, InconsistentFilesystemException, InterruptedException { |
| Preconditions.checkState( |
| env instanceof RecordingSkyFunctionEnvironment, |
| "Expected to be recording dep requests when inlining StarlarkBuiltinsFunction"); |
| // We don't need any direct Skyframe calls because we only depend on the exports.bzl file. If we |
| // did make Skyframe calls, they'd use the original recording environment (env), so that they're |
| // properly registered in the CachedBzlLoadData object of the .bzl that is requesting the |
| // builtins. |
| // |
| // We unwrap the environment before calling computeInline(). Any Skyframe deps needed to |
| // evaluate exports.bzl and its transitive deps will be reported by their CachedBzlLoadData |
| // objects. |
| // |
| // TODO(#11437): Update these comments for when we can also inline builtins computations for |
| // BUILD files. |
| Environment strippedEnv = ((RecordingSkyFunctionEnvironment) env).getDelegate(); |
| BzlLoadValue exportsValue; |
| try { |
| exportsValue = |
| bzlLoadFunction.computeInline(EXPORTS_ENTRYPOINT_KEY, strippedEnv, inliningState); |
| } catch (BzlLoadFailedException e) { |
| throw BuiltinsFailedException.errorEvaluatingBuiltinsBzls(e); |
| } |
| if (exportsValue == null) { |
| return null; |
| } |
| |
| return computeWithExports(exportsValue, ruleClassProvider, packageFactory); |
| } |
| |
| /** |
| * Applies the declarations of exports.bzl to the native predeclared symbols to obtain the final |
| * {@link StarlarkBuiltinsValue}. |
| */ |
| private static StarlarkBuiltinsValue computeWithExports( |
| BzlLoadValue exportsValue, RuleClassProvider ruleClassProvider, PackageFactory packageFactory) |
| throws BuiltinsFailedException { |
| byte[] transitiveDigest = exportsValue.getTransitiveDigest(); |
| Module module = exportsValue.getModule(); |
| |
| try { |
| ImmutableMap<String, Object> exportedToplevels = getDict(module, "exported_toplevels"); |
| ImmutableMap<String, Object> exportedRules = getDict(module, "exported_rules"); |
| ImmutableMap<String, Object> exportedToJava = getDict(module, "exported_to_java"); |
| ImmutableMap<String, Object> predeclared = |
| createPredeclaredForBuildBzlUsingInjection( |
| ruleClassProvider, packageFactory, exportedToplevels, exportedRules); |
| return new StarlarkBuiltinsValue(predeclared, exportedToJava, transitiveDigest); |
| } catch (EvalException ex) { |
| ex.ensureLocation(EXPORTS_ENTRYPOINT_LOC); |
| throw BuiltinsFailedException.errorApplyingExports(ex); |
| } |
| } |
| |
| /** |
| * Returns the set of predeclared symbols to initialize a Starlark {@link Module} with, for |
| * evaluating .bzls loaded from a BUILD file. |
| */ |
| private static ImmutableMap<String, Object> createPredeclaredForBuildBzlUsingInjection( |
| RuleClassProvider ruleClassProvider, |
| PackageFactory packageFactory, |
| ImmutableMap<String, Object> exportedToplevels, |
| ImmutableMap<String, Object> exportedRules) { |
| // TODO(#11437): Validate that all symbols supplied to exportedToplevels and exportedRules are |
| // existing rule-logic symbols. |
| |
| // It's probably not necessary to preserve order, but let's do it just in case. |
| Map<String, Object> predeclared = new LinkedHashMap<>(); |
| |
| // Determine the top-level bindings. |
| predeclared.putAll(ruleClassProvider.getEnvironment()); |
| predeclared.putAll(exportedToplevels); |
| // TODO(#11437): We *should* be able to uncomment the following line, but the native module is |
| // added prematurely (without its rule-logic fields) and overridden unconditionally. Fix this |
| // once ASTFileLookupFunction takes in the set of predeclared bindings (currently it directly |
| // // checks getEnvironment()). |
| // Preconditions.checkState(!predeclared.containsKey("native")); |
| |
| // Determine the "native" module. |
| // TODO(bazel-team): Use the same "native" object for both BUILD- and WORKSPACE-loaded .bzls, |
| // and just have it be a dynamic error to call the wrong thing at the wrong time. This is a |
| // breaking change. |
| Map<String, Object> nativeEntries = new LinkedHashMap<>(); |
| for (Map.Entry<String, Object> entry : |
| packageFactory.getNativeModuleBindingsForBuild().entrySet()) { |
| String symbolName = entry.getKey(); |
| Object replacementSymbol = exportedRules.get(symbolName); |
| if (replacementSymbol != null) { |
| nativeEntries.put(symbolName, replacementSymbol); |
| } else { |
| nativeEntries.put(symbolName, entry.getValue()); |
| } |
| } |
| predeclared.put("native", createNativeModule(nativeEntries)); |
| |
| return ImmutableMap.copyOf(predeclared); |
| } |
| |
| /** Returns a {@link StarlarkBuiltinsValue} that completely ignores injected builtins. */ |
| // TODO(#11437): Delete once injection cannot be disabled. |
| static StarlarkBuiltinsValue createStarlarkBuiltinsValueWithoutInjection( |
| RuleClassProvider ruleClassProvider, PackageFactory packageFactory) { |
| ImmutableMap<String, Object> predeclared = |
| createPredeclaredForBuildBzlUsingInjection( |
| ruleClassProvider, |
| packageFactory, |
| /*exportedToplevels=*/ ImmutableMap.of(), |
| /*exportedRules=*/ ImmutableMap.of()); |
| return new StarlarkBuiltinsValue( |
| /*predeclaredForBuildBzl=*/ predeclared, |
| /*exportedToJava=*/ ImmutableMap.of(), |
| /*transitiveDigest=*/ new byte[] {}); |
| } |
| |
| /** |
| * Returns the set of predeclared symbols to initialize a Starlark {@link Module} with, for |
| * evaluating .bzls loaded from a WORKSPACE file. |
| */ |
| static ImmutableMap<String, Object> createPredeclaredForWorkspaceBzl( |
| RuleClassProvider ruleClassProvider, PackageFactory packageFactory) { |
| // Preserve order, just in case. |
| Map<String, Object> predeclared = new LinkedHashMap<>(); |
| predeclared.putAll(ruleClassProvider.getEnvironment()); |
| Object nativeModule = createNativeModule(packageFactory.getNativeModuleBindingsForWorkspace()); |
| // TODO(#11437): Assert not already present; see createPreclaredsForBuildBzl. |
| predeclared.put("native", nativeModule); |
| return ImmutableMap.copyOf(predeclared); |
| } |
| |
| /** |
| * Returns the set of predeclared symbols to initialize a Starlark {@link Module} with, for |
| * evaluating .bzls loaded from the {@code @builtins} pseudo-repository. |
| */ |
| // TODO(#11437): create the _internal name, prohibit other rule logic names. Take in a |
| // PackageFactory. |
| static ImmutableMap<String, Object> createPredeclaredForBuiltinsBzl( |
| RuleClassProvider ruleClassProvider) { |
| return ruleClassProvider.getEnvironment(); |
| } |
| |
| private static Object createNativeModule(Map<String, Object> bindings) { |
| return StructProvider.STRUCT.create(bindings, "no native function or rule '%s'"); |
| } |
| |
| /** |
| * Attempts to retrieve the string-keyed dict named {@code dictName} from the given {@code |
| * module}. |
| * |
| * @return a copy of the dict mappings on success |
| * @throws EvalException if the symbol isn't present or is not a dict whose keys are all strings |
| */ |
| @Nullable |
| private static ImmutableMap<String, Object> getDict(Module module, String dictName) |
| throws EvalException { |
| Object value = module.get(dictName); |
| if (value == null) { |
| throw new EvalException( |
| /*location=*/ null, String.format("expected a '%s' dictionary to be defined", dictName)); |
| } |
| return ImmutableMap.copyOf(Dict.cast(value, String.class, Object.class, dictName + " dict")); |
| } |
| |
| @Override |
| public String extractTag(SkyKey skyKey) { |
| return null; |
| } |
| |
| /** |
| * An exception that occurs while trying to determine the injected builtins. |
| * |
| * <p>This exception type typically wraps a {@link BzlLoadFailedException} and is wrapped by a |
| * {@link BzlLoadFailedException} in turn. |
| */ |
| static final class BuiltinsFailedException extends Exception { |
| |
| private final Transience transience; |
| |
| private BuiltinsFailedException(String errorMessage, Exception cause, Transience transience) { |
| super(errorMessage, cause); |
| this.transience = transience; |
| } |
| |
| Transience getTransience() { |
| return transience; |
| } |
| |
| static BuiltinsFailedException errorEvaluatingBuiltinsBzls(BzlLoadFailedException cause) { |
| return new BuiltinsFailedException( |
| String.format("Failed to load builtins sources: %s", cause.getMessage()), |
| cause, |
| cause.getTransience()); |
| } |
| |
| static BuiltinsFailedException errorApplyingExports(EvalException cause) { |
| return new BuiltinsFailedException( |
| String.format("Failed to apply declared builtins: %s", cause.getMessage()), |
| cause, |
| Transience.PERSISTENT); |
| } |
| } |
| |
| /** The exception type thrown by {@link StarlarkBuiltinsFunction}. */ |
| static final class StarlarkBuiltinsFunctionException extends SkyFunctionException { |
| |
| private StarlarkBuiltinsFunctionException(BuiltinsFailedException cause) { |
| super(cause, cause.transience); |
| } |
| } |
| } |