src/main/java/com/google/devtools/build/lib/skyframe/StarlarkBuiltinsFunction.java

// Copyright 2020 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.skyframe;

import static com.google.common.base.Preconditions.checkNotNull;

import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.packages.AutoloadSymbols;
import com.google.devtools.build.lib.packages.AutoloadSymbols.AutoloadException;
import com.google.devtools.build.lib.packages.BazelStarlarkEnvironment;
import com.google.devtools.build.lib.packages.BazelStarlarkEnvironment.InjectionException;
import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
import com.google.devtools.build.lib.skyframe.BzlLoadFunction.InlineCacheManager;
import com.google.devtools.build.lib.util.Fingerprint;
import com.google.devtools.build.skyframe.RecordingSkyFunctionEnvironment;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import com.google.devtools.build.skyframe.SkyframeLookupResult;
import javax.annotation.Nullable;
import net.starlark.java.eval.Dict;
import net.starlark.java.eval.EvalException;
import net.starlark.java.eval.Module;
import net.starlark.java.eval.Starlark;
import net.starlark.java.eval.StarlarkSemantics;

// TODO(#11437): Update the design doc to change `@builtins` -> `@_builtins`.

// TODO(#11437): Add support to BzlLoadCycleReporter to pretty-print cycles involving
// @_builtins.

// TODO(#11437): Add tombstone feature: If a native symbol is a tombstone object, this signals to
// StarlarkBuiltinsFunction that the corresponding symbol *must* be defined by @_builtins.
// Furthermore, if exports.bzl also says the symbol is a tombstone, any attempt to use it results
// in failure, as if the symbol doesn't exist at all (or with a user-friendly error message saying
// to migrate by adding a load()). Combine tombstones with reading the current incompatible flags
// within @_builtins for awesomeness.

// TODO(#11437, #11954, #11983): To the extent that BUILD-loaded .bzls and WORKSPACE-loaded .bzls
// have the same environment, builtins injection should apply to both of them, not just to
// BUILD-loaded .bzls.

/**
 * A Skyframe function that evaluates the {@code @_builtins} pseudo-repository and reports the
 * values exported by {@link #EXPORTS_ENTRYPOINT}. The {@code @_builtins} pseudo-repository shares a
 * repo mapping with the {@code @bazel_tools} repository.
 *
 * <p>The process of "builtins injection" refers to evaluating this Skyfunction and applying its
 * result to {@link BzlLoadFunction}'s computation. See also the <a
 * href="https://docs.google.com/document/d/1GW7UVo1s9X0cti9OMgT3ga5ozKYUWLPk9k8c4-34rC4">design
 * doc</a>:
 *
 * <p>This function has two trivial keys, so there can only be two values in the build at a time.
 * First key/value (without autoloads) has a single dependency, on the result of evaluating the
 * exports.bzl file to a {@link BzlLoadValue}. Second key/value has dependencies on the results of
 * extenally loaded symbols and rules. For more information on the second value see {@link
 * AutoloadSymbols}.
 *
 * <p>This function supports a special "inlining" mode, similar to {@link BzlLoadFunction} (see that
 * class's javadoc and code comments). Whenever we inline {@link BzlLoadFunction} we also inline
 * {@link StarlarkBuiltinsFunction} (and {@link StarlarkBuiltinsFunction}'s calls to {@link
 * BzlLoadFunction} are then themselves inlined!). Similar to {@link BzlLoadFunction}'s inlining, we
 * cache the result of this computation, and this caching is managed by {@link
 * BzlLoadFunction.InlineCacheManager}. But since there's only a single {@link
 * StarlarkBuiltinsValue} node and we don't need to worry about that node's value changing at future
 * invocations or subsequent versions (see {@link InlineCacheManager#reset} for why), our caching
 * strategy is much simpler and we don't need to bother inlining deps of the Skyframe subgraph.
 */
public class StarlarkBuiltinsFunction implements SkyFunction {

  /**
   * The label where {@code @_builtins} symbols are exported from. (Note that this is never
   * conflated with an actual repository named "{@code @_builtins}" because 1) it is only ever
   * accessed through a special SkyKey, and 2) we disallow the user from defining a repo named
   * {@code @_builtins} to avoid confusion.)
   */
  static final Label EXPORTS_ENTRYPOINT =
      Label.parseCanonicalUnchecked("@_builtins//:exports.bzl"); // unused

  /**
   * Key for loading exports.bzl. Note that {@code keyForBuiltins} (as opposed to {@code
   * keyForBuild}) ensures we can resolve {@code @_builtins}, which is otherwise inaccessible. It
   * also prevents us from cyclically requesting StarlarkBuiltinsFunction again to evaluate
   * exports.bzl.
   */
  static final BzlLoadValue.Key EXPORTS_ENTRYPOINT_KEY =
      BzlLoadValue.keyForBuiltins(EXPORTS_ENTRYPOINT);

  // Used to obtain the injected environment.
  private final BazelStarlarkEnvironment bazelStarlarkEnvironment;

  public StarlarkBuiltinsFunction(BazelStarlarkEnvironment bazelStarlarkEnvironment) {
    this.bazelStarlarkEnvironment = bazelStarlarkEnvironment;
  }

  @Override
  @Nullable
  public SkyValue compute(SkyKey skyKey, Environment env)
      throws StarlarkBuiltinsFunctionException, InterruptedException {
    try {
      return computeInternal(
          env,
          bazelStarlarkEnvironment,
          ((StarlarkBuiltinsValue.Key) skyKey.argument()).isWithAutoloads(),
          /* inliningState= */ null,
          /* bzlLoadFunction= */ null);
    } catch (BuiltinsFailedException e) {
      throw new StarlarkBuiltinsFunctionException(e);
    }
  }

  /**
   * Computes this Skyfunction under inlining of {@link BzlLoadFunction}, forwarding the given
   * inlining state.
   *
   * <p>The given Skyframe environment must be a {@link RecordingSkyFunctionEnvironment}. It is
   * unwrapped before calling {@link BzlLoadFunction}'s inlining code path.
   *
   * <p>Returns null on Skyframe restart or error.
   */
  @Nullable
  public static StarlarkBuiltinsValue computeInline(
      StarlarkBuiltinsValue.Key key,
      BzlLoadFunction.InliningState inliningState,
      BazelStarlarkEnvironment bazelStarlarkEnvironment,
      BzlLoadFunction bzlLoadFunction)
      throws BuiltinsFailedException, InterruptedException {
    checkNotNull(bzlLoadFunction.inlineCacheManager);
    StarlarkBuiltinsValue cachedBuiltins = bzlLoadFunction.inlineCacheManager.builtinsRef.get();
    if (cachedBuiltins != null) {
      // See the comment in InlineCacheManager#reset for why it's sound to not inline deps of the
      // entire subgraph here.
      return cachedBuiltins;
    }

    // See BzlLoadFunction#computeInline and BzlLoadFunction.InliningState for an explanation of the
    // inlining mechanism and its invariants. For our purposes, the Skyframe environment to use
    // comes from inliningState.
    StarlarkBuiltinsValue computedBuiltins =
        computeInternal(
            inliningState.getEnvironment(),
            bazelStarlarkEnvironment,
            key.isWithAutoloads(),
            inliningState,
            bzlLoadFunction);
    if (computedBuiltins == null) {
      return null;
    }
    // There's a benign race where multiple threads may try to compute-and-cache the single builtins
    // value. Ensure the value computed by winner of that race gets used by everyone.
    bzlLoadFunction.inlineCacheManager.builtinsRef.compareAndSet(null, computedBuiltins);
    return bzlLoadFunction.inlineCacheManager.builtinsRef.get();
  }

  // bzlLoadFunction and inliningState are non-null iff using inlining code path.
  @Nullable
  private static StarlarkBuiltinsValue computeInternal(
      Environment env,
      BazelStarlarkEnvironment bazelStarlarkEnvironment,
      boolean isWithAutoloads,
      @Nullable BzlLoadFunction.InliningState inliningState,
      @Nullable BzlLoadFunction bzlLoadFunction)
      throws BuiltinsFailedException, InterruptedException {
    StarlarkSemantics starlarkSemantics = PrecomputedValue.STARLARK_SEMANTICS.get(env);
    if (starlarkSemantics == null) {
      return null;
    }
    // Return the empty value if builtins injection is disabled.
    if (starlarkSemantics.get(BuildLanguageOptions.EXPERIMENTAL_BUILTINS_BZL_PATH).isEmpty()) {
      return StarlarkBuiltinsValue.createEmpty(starlarkSemantics);
    }
    AutoloadSymbols autoloadSymbols = AutoloadSymbols.AUTOLOAD_SYMBOLS.get(env);
    if (autoloadSymbols == null) {
      return null;
    }

    if (autoloadSymbols.isEnabled()
        && starlarkSemantics.getBool(BuildLanguageOptions.ENABLE_BZLMOD)) {
      // We can't do autoloads where the rules are implemented (disabling them when running in
      // main repository named rules_python)
      ModuleFileValue mainModule =
          (ModuleFileValue) env.getValue(ModuleFileValue.KEY_FOR_ROOT_MODULE);
      if (mainModule == null) {
        return null;
      }
      if (autoloadSymbols.autoloadsDisabledForRepo(mainModule.getModule().getName())) {
        isWithAutoloads = false;
      }
    }

    // Load exports.bzl. If we were requested using inlining, make sure to inline the call back into
    // BzlLoadFunction.
    // If requested also loads "autoloads": rules and providers from external repositories.
    BzlLoadValue exportsValue;
    try {
      if (inliningState == null) {
        exportsValue =
            (BzlLoadValue)
                env.getValueOrThrow(EXPORTS_ENTRYPOINT_KEY, BzlLoadFailedException.class);
      } else {
        exportsValue = bzlLoadFunction.computeInline(EXPORTS_ENTRYPOINT_KEY, inliningState);
      }
    } catch (BzlLoadFailedException ex) {
      throw BuiltinsFailedException.errorEvaluatingBuiltinsBzls(ex);
    }

    ImmutableMap<String, BzlLoadValue> autoBzlLoadValues;
    try {
      ImmutableMap<String, BzlLoadValue.Key> autoBzlLoadKeys =
          isWithAutoloads ? autoloadSymbols.getLoadKeys(env) : ImmutableMap.of();
      if (autoBzlLoadKeys == null) {
        return null;
      }
      ImmutableMap.Builder<String, BzlLoadValue> autoBzlLoadValuesBuilder =
          ImmutableMap.builderWithExpectedSize(autoBzlLoadKeys.size());
      if (inliningState == null) {
        SkyframeLookupResult values = env.getValuesAndExceptions(autoBzlLoadKeys.values());
        for (var symbolKeyEntry : autoBzlLoadKeys.entrySet()) {
          String symbol = symbolKeyEntry.getKey();
          BzlLoadValue value =
              (BzlLoadValue)
                  values.getOrThrow(symbolKeyEntry.getValue(), BzlLoadFailedException.class);
          if (value != null) {
            autoBzlLoadValuesBuilder.put(symbol, value);
          }
        }
      } else {
        for (var symbolKeyEntry : autoBzlLoadKeys.entrySet()) {
          String symbol = symbolKeyEntry.getKey();
          BzlLoadValue value =
              bzlLoadFunction.computeInline(symbolKeyEntry.getValue(), inliningState);
          if (value != null) {
            autoBzlLoadValuesBuilder.put(symbol, value);
          }
        }
      }
      autoBzlLoadValues = autoBzlLoadValuesBuilder.buildOrThrow();
    } catch (BzlLoadFailedException ex) {
      throw BuiltinsFailedException.errorEvaluatingAutoloadedBzls(
          ex, starlarkSemantics.getBool(BuildLanguageOptions.ENABLE_BZLMOD));
    }
    if (env.valuesMissing()) {
      return null;
    }

    // Compute digest of exports.bzl and possibly externally loaded symbols (Bazel)
    byte[] transitiveDigest = exportsValue.getTransitiveDigest();
    if (!autoBzlLoadValues.isEmpty()) {
      Fingerprint fp = new Fingerprint();
      fp.addBytes(transitiveDigest);
      for (BzlLoadValue value : autoBzlLoadValues.values()) {
        fp.addBytes(value.getTransitiveDigest());
      }
      transitiveDigest = fp.digestAndReset();
    }

    // Apply declarations of exports.bzl to the native predeclared symbols.
    Module module = exportsValue.getModule();
    try {
      ImmutableMap<String, Object> exportedToplevels = getDict(module, "exported_toplevels");
      ImmutableMap<String, Object> exportedRules = getDict(module, "exported_rules");
      ImmutableMap<String, Object> exportedToJava = getDict(module, "exported_to_java");
      ImmutableMap<String, Object> predeclaredForBuildBzl =
          bazelStarlarkEnvironment.createBuildBzlEnvUsingInjection(
              exportedToplevels,
              exportedRules,
              starlarkSemantics.get(BuildLanguageOptions.EXPERIMENTAL_BUILTINS_INJECTION_OVERRIDE));
      ImmutableMap<String, Object> predeclaredForWorkspaceBzl =
          bazelStarlarkEnvironment.createWorkspaceBzlEnvUsingInjection(
              exportedToplevels,
              exportedRules,
              starlarkSemantics.get(BuildLanguageOptions.EXPERIMENTAL_BUILTINS_INJECTION_OVERRIDE));
      ImmutableMap<String, Object> predeclaredForBuild =
          bazelStarlarkEnvironment.createBuildEnvUsingInjection(
              exportedRules,
              starlarkSemantics.get(BuildLanguageOptions.EXPERIMENTAL_BUILTINS_INJECTION_OVERRIDE));

      // Apply declarations of externally loaded symbols to the native predeclared symbols.
      ImmutableMap<String, Object> newSymbols = autoloadSymbols.processLoads(autoBzlLoadValues);
      predeclaredForBuild =
          autoloadSymbols.modifyBuildEnv(isWithAutoloads, predeclaredForBuild, newSymbols);
      predeclaredForBuildBzl =
          autoloadSymbols.modifyBuildBzlEnv(isWithAutoloads, predeclaredForBuildBzl, newSymbols);

      return StarlarkBuiltinsValue.create(
          predeclaredForBuildBzl,
          predeclaredForWorkspaceBzl,
          predeclaredForBuild,
          exportedToJava,
          transitiveDigest,
          starlarkSemantics);
    } catch (EvalException | InjectionException ex) {
      throw BuiltinsFailedException.errorApplyingExports(ex);
    } catch (AutoloadException ex) {
      throw new BuiltinsFailedException(
          String.format("Failed to apply symbols loaded externally: %s", ex.getMessage()),
          ex,
          Transience.PERSISTENT);
    }
  }

  /**
   * Attempts to retrieve the string-keyed dict named {@code dictName} from the given {@code
   * module}.
   *
   * @return a copy of the dict mappings on success
   * @throws EvalException if the symbol isn't present or is not a dict whose keys are all strings
   */
  @Nullable
  private static ImmutableMap<String, Object> getDict(Module module, String dictName)
      throws EvalException {
    Object value = module.getGlobal(dictName);
    if (value == null) {
      throw Starlark.errorf("expected a '%s' dictionary to be defined", dictName);
    }
    return ImmutableMap.copyOf(Dict.cast(value, String.class, Object.class, dictName + " dict"));
  }

  /**
   * An exception that occurs while trying to determine the injected builtins.
   *
   * <p>This exception type typically wraps a {@link BzlLoadFailedException} and is wrapped by a
   * {@link BzlLoadFailedException} in turn.
   */
  static final class BuiltinsFailedException extends Exception {

    private final Transience transience;

    private BuiltinsFailedException(String errorMessage, Exception cause, Transience transience) {
      super(errorMessage, cause);
      this.transience = transience;
    }

    Transience getTransience() {
      return transience;
    }

    static BuiltinsFailedException errorEvaluatingBuiltinsBzls(BzlLoadFailedException cause) {
      return errorEvaluatingBuiltinsBzls(cause, cause.getTransience());
    }

    static BuiltinsFailedException errorEvaluatingBuiltinsBzls(
        Exception cause, Transience transience) {
      return new BuiltinsFailedException(
          String.format("Failed to load builtins sources: %s", cause.getMessage()),
          cause,
          transience);
    }

    static BuiltinsFailedException errorEvaluatingAutoloadedBzls(
        BzlLoadFailedException cause, boolean bzlmodEnabled) {
      String additionalMessage =
          bzlmodEnabled
              ? ""
              : " Most likely you need to upgrade the version of rules repository in the"
                  + " WORKSPACE file.";
      return new BuiltinsFailedException(
          String.format(
              "Failed to autoload external symbols: %s%s", cause.getMessage(), additionalMessage),
          cause,
          cause.getTransience());
    }

    static BuiltinsFailedException errorApplyingExports(Exception cause) {
      return new BuiltinsFailedException(
          String.format("Failed to apply declared builtins: %s", cause.getMessage()),
          cause,
          Transience.PERSISTENT);
    }
  }

  /** The exception type thrown by {@link StarlarkBuiltinsFunction}. */
  static final class StarlarkBuiltinsFunctionException extends SkyFunctionException {

    private StarlarkBuiltinsFunctionException(BuiltinsFailedException cause) {
      super(cause, cause.transience);
    }
  }
}