| // Copyright 2024 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package com.google.devtools.build.lib.packages; |
| |
| import static com.google.common.collect.ImmutableList.toImmutableList; |
| import static com.google.devtools.build.lib.packages.BuildType.NODEP_LABEL_LIST; |
| |
| import com.google.common.base.Preconditions; |
| import com.google.common.collect.ImmutableList; |
| import com.google.common.collect.ImmutableMap; |
| import com.google.common.collect.ImmutableSet; |
| import com.google.common.collect.Lists; |
| import com.google.devtools.build.lib.cmdline.Label; |
| import com.google.devtools.build.lib.cmdline.PackageIdentifier; |
| import com.google.devtools.build.lib.events.Event; |
| import com.google.devtools.build.lib.packages.TargetRecorder.MacroFrame; |
| import com.google.devtools.build.lib.packages.TargetRecorder.NameConflictException; |
| import com.google.devtools.build.lib.server.FailureDetails.PackageLoading.Code; |
| import com.google.errorprone.annotations.CanIgnoreReturnValue; |
| import java.util.ArrayList; |
| import java.util.LinkedHashMap; |
| import java.util.List; |
| import java.util.Map; |
| import javax.annotation.Nullable; |
| import net.starlark.java.eval.EvalException; |
| import net.starlark.java.eval.Mutability; |
| import net.starlark.java.eval.Starlark; |
| import net.starlark.java.eval.StarlarkFunction; |
| import net.starlark.java.eval.StarlarkSemantics; |
| import net.starlark.java.eval.StarlarkThread; |
| import net.starlark.java.eval.SymbolGenerator; |
| import net.starlark.java.spelling.SpellChecker; |
| |
| /** |
| * Represents a symbolic macro, defined in a .bzl file, that may be instantiated during Package |
| * evaluation. |
| * |
| * <p>This is analogous to {@link RuleClass}. In essence, a {@code MacroClass} consists of the |
| * macro's schema and its implementation function. |
| */ |
| public final class MacroClass { |
| |
| /** |
| * Names that users may not pass as keys of the {@code attrs} dict when calling {@code macro()}. |
| * |
| * <p>Of these, {@code name} is special cased as an actual attribute, and the rest do not exist. |
| */ |
| // Keep in sync with `macro()`'s `attrs` user documentation in StarlarkRuleFunctionsApi. |
| // But we should avoid adding new entries here, since it's a backwards-incompatible change. |
| public static final ImmutableSet<String> RESERVED_MACRO_ATTR_NAMES = |
| ImmutableSet.of("name", "visibility"); |
| |
| /** |
| * "visibility" attribute present on all symbolic macros. |
| * |
| * <p>This is similar to the visibility attribute for rules, but lacks the exec transitions. |
| */ |
| public static final Attribute VISIBILITY_ATTRIBUTE = |
| Attribute.attr("visibility", NODEP_LABEL_LIST) |
| .orderIndependent() |
| .nonconfigurable("special attribute integrated more deeply into Bazel's core logic") |
| .build(); |
| |
| private final String name; |
| private final Label definingBzlLabel; |
| private final StarlarkFunction implementation; |
| // Implicit attributes are stored under their given name ("_foo"), not a mangled name ("$foo"). |
| private final ImmutableMap<String, Attribute> attributes; |
| private final boolean isFinalizer; |
| |
| private MacroClass( |
| String name, |
| Label definingBzlLabel, |
| StarlarkFunction implementation, |
| ImmutableMap<String, Attribute> attributes, |
| boolean isFinalizer) { |
| this.name = name; |
| this.definingBzlLabel = definingBzlLabel; |
| this.implementation = implementation; |
| this.attributes = attributes; |
| this.isFinalizer = isFinalizer; |
| } |
| |
| /** Returns the macro's exported name. */ |
| public String getName() { |
| return name; |
| } |
| |
| /** Returns the label of the .bzl file where the macro was exported. */ |
| public Label getDefiningBzlLabel() { |
| return definingBzlLabel; |
| } |
| |
| public StarlarkFunction getImplementation() { |
| return implementation; |
| } |
| |
| // NB: Order is preserved from what was passed to the constructor. |
| public ImmutableMap<String, Attribute> getAttributes() { |
| return attributes; |
| } |
| |
| /** |
| * Returns whether this symbolic macro is a finalizer. All finalizers are run deferred to the end |
| * of the BUILD file's evaluation, rather than synchronously with their instantiation. |
| */ |
| public boolean isFinalizer() { |
| return isFinalizer; |
| } |
| |
| /** Builder for {@link MacroClass}. */ |
| public static final class Builder { |
| @Nullable private String name = null; |
| @Nullable private Label definingBzlLabel = null; |
| private final StarlarkFunction implementation; |
| private final ImmutableMap.Builder<String, Attribute> attributes = ImmutableMap.builder(); |
| private boolean isFinalizer = false; |
| |
| public Builder(StarlarkFunction implementation) { |
| this.implementation = implementation; |
| |
| addAttribute(RuleClass.NAME_ATTRIBUTE); |
| addAttribute(VISIBILITY_ATTRIBUTE); |
| } |
| |
| @CanIgnoreReturnValue |
| public Builder setName(String name) { |
| this.name = name; |
| return this; |
| } |
| |
| @CanIgnoreReturnValue |
| public Builder setDefiningBzlLabel(Label label) { |
| this.definingBzlLabel = label; |
| return this; |
| } |
| |
| @CanIgnoreReturnValue |
| public Builder addAttribute(Attribute attribute) { |
| attributes.put(attribute.getName(), attribute); |
| return this; |
| } |
| |
| @CanIgnoreReturnValue |
| public Builder setIsFinalizer() { |
| this.isFinalizer = true; |
| return this; |
| } |
| |
| public MacroClass build() { |
| Preconditions.checkNotNull(name); |
| Preconditions.checkNotNull(definingBzlLabel); |
| return new MacroClass( |
| name, |
| definingBzlLabel, |
| implementation, |
| attributes.buildOrThrow(), |
| /* isFinalizer= */ isFinalizer); |
| } |
| } |
| |
| /** |
| * Constructs and returns a new {@link MacroInstance} associated with this {@code MacroClass}. |
| * |
| * <p>See {@link #instantiateAndAddMacro}. |
| */ |
| // TODO(#19922): Consider reporting multiple events instead of failing on the first one. See |
| // analogous implementation in RuleClass#populateDefinedRuleAttributeValues. |
| private MacroInstance instantiateMacro(Package.Builder pkgBuilder, Map<String, Object> kwargs) |
| throws EvalException { |
| // A word on edge cases: |
| // - If an attr is implicit but does not have a default specified, its value is just the |
| // default value for its attr type (e.g. `[]` for `attr.label_list()`). |
| // - If an attr is implicit but also mandatory, it's impossible to instantiate it without |
| // error. |
| // - If an attr is mandatory but also has a default, the default is meaningless. |
| // These behaviors align with rule attributes. |
| |
| LinkedHashMap<String, Object> attrValues = new LinkedHashMap<>(); |
| |
| // For each given attr value, validate that the attr exists and can be set. |
| for (Map.Entry<String, Object> entry : kwargs.entrySet()) { |
| String attrName = entry.getKey(); |
| Object value = entry.getValue(); |
| Attribute attr = attributes.get(attrName); |
| |
| // Check for unknown attr. |
| if (attr == null) { |
| throw Starlark.errorf( |
| "no such attribute '%s' in '%s' macro%s", |
| attrName, |
| name, |
| SpellChecker.didYouMean( |
| attrName, |
| attributes.values().stream() |
| .filter(Attribute::isDocumented) |
| .map(Attribute::getName) |
| .collect(toImmutableList()))); |
| } |
| |
| // Setting an attr to None is the same as omitting it (except that it's still an error to set |
| // an unknown attr to None). If the attr is optional, skip adding it to the map now but put it |
| // in below when we realize it's missing. |
| if (value == Starlark.NONE) { |
| continue; |
| } |
| |
| // Can't set implicit default. |
| // (We don't check Attribute#isImplicit() because that assumes "_" -> "$" prefix mangling.) |
| // TODO: #19922 - The lack of "_" -> "$" mangling may impact the future feature of inheriting |
| // attributes from rules. We could consider just doing the mangling for macros too so they're |
| // consistent. |
| if (attr.getName().startsWith("_")) { |
| throw Starlark.errorf("cannot set value of implicit attribute '%s'", attr.getName()); |
| } |
| |
| attrValues.put(attrName, value); |
| } |
| |
| // Special processing of the "visibility" attribute. |
| // TODO(brandjon): When we add introspection of attributes of symbolic macros, we'll want to |
| // distinguish between the different types of visibility a la Target#getRawVisibility / |
| // #getVisibility / #getActualVisibility. |
| @Nullable MacroFrame parentMacroFrame = pkgBuilder.getCurrentMacroFrame(); |
| @Nullable Object rawVisibility = attrValues.get("visibility"); |
| RuleVisibility parsedVisibility; |
| if (rawVisibility == null) { |
| // Visibility wasn't explicitly supplied. If we're not in another symbolic macro, use the |
| // package's default visibility, otherwise use private visibility. |
| if (parentMacroFrame == null) { |
| parsedVisibility = pkgBuilder.getPartialPackageArgs().defaultVisibility(); |
| } else { |
| parsedVisibility = RuleVisibility.PRIVATE; |
| } |
| } else { |
| @SuppressWarnings("unchecked") |
| List<Label> liftedVisibility = |
| (List<Label>) |
| BuildType.copyAndLiftStarlarkValue( |
| name, VISIBILITY_ATTRIBUTE, rawVisibility, pkgBuilder.getLabelConverter()); |
| parsedVisibility = RuleVisibility.parse(liftedVisibility); |
| } |
| // Concatenate the visibility (as previously populated) with the instantiation site's location. |
| PackageIdentifier instantiatingLoc = |
| parentMacroFrame == null |
| ? pkgBuilder.getPackageIdentifier() |
| : parentMacroFrame.macroInstance.getDefinitionPackage(); |
| RuleVisibility actualVisibility = parsedVisibility.concatWithPackage(instantiatingLoc); |
| attrValues.put("visibility", actualVisibility.getDeclaredLabels()); |
| |
| // Populate defaults for the rest, and validate that no mandatory attr was missed. |
| for (Attribute attr : attributes.values()) { |
| if (attrValues.containsKey(attr.getName())) { |
| continue; |
| } |
| if (attr.isMandatory()) { |
| throw Starlark.errorf( |
| "missing value for mandatory attribute '%s' in '%s' macro", attr.getName(), name); |
| } else { |
| Object defaultValue = attr.getDefaultValueUnchecked(); |
| // For attributes defined directly in this macro's `attrs` param, we've already validated |
| // that the default is not a computed default, late-bound default, etc. But these may still |
| // appear in inherited attributes. Those should be replaced by None as the default. |
| // |
| // We may also see a null default for LabelType, which also should be replaced by None. |
| // |
| // TODO(arostovtsev): All inherited attributes should be forced to have None defaults. |
| if (defaultValue == null || shouldForceDefaultToNone(attr)) { |
| // Set the default value as None if: |
| defaultValue = Starlark.NONE; |
| } |
| attrValues.put(attr.getName(), defaultValue); |
| } |
| } |
| |
| // Normalize and validate all attr values. (E.g., convert strings to labels, promote |
| // configurable attribute values to select()s, fail if bool was passed instead of label, ensure |
| // values are immutable.) Note that this applies even to default values, although as a special |
| // case None is never promoted to select(). |
| for (Map.Entry<String, Object> entry : ImmutableMap.copyOf(attrValues).entrySet()) { |
| String attrName = entry.getKey(); |
| Object value = entry.getValue(); |
| Attribute attribute = attributes.get(attrName); |
| if (value.equals(Starlark.NONE)) { |
| // Don't promote None to select({"//conditions:default": None}). |
| continue; |
| } |
| Object normalizedValue = |
| // copyAndLiftStarlarkValue ensures immutability. |
| BuildType.copyAndLiftStarlarkValue( |
| name, attribute, value, pkgBuilder.getLabelConverter()); |
| // TODO(#19922): Validate that LABEL_LIST type attributes don't contain duplicates, to match |
| // the behavior of rules. This probably requires factoring out logic from |
| // AggregatingAttributeMapper. |
| if (attribute.isConfigurable() && !(normalizedValue instanceof SelectorList)) { |
| normalizedValue = SelectorList.wrapSingleValue(normalizedValue); |
| } |
| attrValues.put(attrName, normalizedValue); |
| } |
| |
| // Type and existence enforced by RuleClass.NAME_ATTRIBUTE. |
| String name = (String) Preconditions.checkNotNull(attrValues.get("name")); |
| // Determine the id for this macro. If we're in another macro by the same name, increment the |
| // number, otherwise use 1 for the number. |
| int sameNameDepth = |
| parentMacroFrame == null || !name.equals(parentMacroFrame.macroInstance.getName()) |
| ? 1 |
| : parentMacroFrame.macroInstance.getSameNameDepth() + 1; |
| |
| return pkgBuilder.createMacro(this, attrValues, sameNameDepth); |
| } |
| |
| /** |
| * Returns true if the given attribute's default value should be considered {@code None}. |
| * |
| * <p>This is the case for non-direct defaults and legacy licenses and distribs attributes, |
| * because None may (depending on attribute type) violate type checking - and that is ok, since |
| * the macro implementation will pass the None to the rule function, which will then set the |
| * default as expected. |
| */ |
| private static boolean shouldForceDefaultToNone(Attribute attr) { |
| return attr.hasComputedDefault() |
| || attr.isLateBound() |
| || attr.isMaterializing() |
| || attr.getType() == BuildType.LICENSE |
| || attr.getType() == BuildType.DISTRIBUTIONS; |
| } |
| |
| /** |
| * Constructs a new {@link MacroInstance} associated with this {@code MacroClass}, adds it to the |
| * package, and returns it. |
| * |
| * @param pkgBuilder The builder corresponding to the package in which this instance will live. |
| * @param kwargs A map from attribute name to its given Starlark value, such as passed in a BUILD |
| * file (i.e., prior to attribute type conversion, {@code select()} promotion, default value |
| * substitution, or even validation that the attribute exists). |
| */ |
| public MacroInstance instantiateAndAddMacro( |
| Package.Builder pkgBuilder, Map<String, Object> kwargs) throws EvalException { |
| MacroInstance macroInstance = instantiateMacro(pkgBuilder, kwargs); |
| try { |
| pkgBuilder.addMacro(macroInstance); |
| } catch (NameConflictException e) { |
| throw new EvalException(e); |
| } |
| return macroInstance; |
| } |
| |
| /** |
| * Executes a symbolic macro's implementation function, in a new Starlark thread, mutating the |
| * given package under construction. |
| */ |
| // TODO: #19922 - Take a new type, PackagePiece.Builder, in place of Package.Builder. PackagePiece |
| // would represent the collection of targets/macros instantiated by expanding a single symbolic |
| // macro. |
| public static void executeMacroImplementation( |
| MacroInstance macro, Package.Builder builder, StarlarkSemantics semantics) |
| throws InterruptedException { |
| // Ensure we're not expanding a (possibly indirect) recursive macro. This is morally analogous |
| // to StarlarkThread#isRecursiveCall, except in this context, recursion is through the chain of |
| // macro instantiations, which may or may not actually be concurrently executing on the stack |
| // depending on whether the evaluation is eager or deferred. |
| @Nullable String recursionMsg = getRecursionErrorMessage(macro); |
| if (recursionMsg != null) { |
| builder |
| .getLocalEventHandler() |
| .handle(Package.error(/* location= */ null, recursionMsg, Code.STARLARK_EVAL_ERROR)); |
| builder.setContainsErrors(); |
| // Don't try to evaluate this macro again. |
| builder.markMacroComplete(macro); |
| return; |
| } |
| |
| try (Mutability mu = |
| Mutability.create("macro", builder.getPackageIdentifier(), macro.getName())) { |
| StarlarkThread thread = |
| StarlarkThread.create( |
| mu, |
| semantics, |
| /* contextDescription= */ "", |
| SymbolGenerator.create( |
| MacroInstance.UniqueId.create( |
| macro.getPackage().getPackageIdentifier(), macro.getId()))); |
| thread.setPrintHandler(Event.makeDebugPrintHandler(builder.getLocalEventHandler())); |
| |
| // TODO: #19922 - Technically the embedded SymbolGenerator field should use a different key |
| // than the one in the main BUILD thread, but that'll be fixed when we change the type to |
| // PackagePiece.Builder. |
| builder.storeInThread(thread); |
| |
| // TODO: #19922 - If we want to support creating analysis_test rules inside symbolic macros, |
| // we'd need to call `thread.setThreadLocal(RuleDefinitionEnvironment.class, |
| // ruleClassProvider)`. In that case we'll need to consider how to get access to the |
| // ConfiguredRuleClassProvider. For instance, we could put it in the builder. |
| |
| MacroFrame childMacroFrame = new MacroFrame(macro); |
| @Nullable MacroFrame parentMacroFrame = builder.setCurrentMacroFrame(childMacroFrame); |
| try { |
| Object returnValue = |
| Starlark.call( |
| thread, |
| macro.getMacroClass().getImplementation(), |
| /* args= */ ImmutableList.of(), |
| /* kwargs= */ macro.getAttrValues()); |
| if (returnValue != Starlark.NONE) { |
| throw Starlark.errorf( |
| "macro '%s' may not return a non-None value (got %s)", |
| macro.getName(), Starlark.repr(returnValue)); |
| } |
| } catch (EvalException ex) { // from either call() or non-None return |
| builder |
| .getLocalEventHandler() |
| .handle( |
| Package.error( |
| /* location= */ null, ex.getMessageWithStack(), Code.STARLARK_EVAL_ERROR)); |
| builder.setContainsErrors(); |
| } finally { |
| // Restore the previously running symbolic macro's state (if any). |
| @Nullable MacroFrame top = builder.setCurrentMacroFrame(parentMacroFrame); |
| Preconditions.checkState(top == childMacroFrame, "inconsistent macro stack state"); |
| // Mark the macro as having completed, even if it was in error (or interrupted?). |
| builder.markMacroComplete(macro); |
| } |
| } |
| } |
| |
| /** |
| * If the instantiation of {@code macro} was recursive, i.e. if it was transitively declared by |
| * another macro instance having the same macro class, then returns an error string identifying |
| * this macro's name and a "traceback" of the instantiating macros. Otherwise, returns null. |
| */ |
| @Nullable |
| private static String getRecursionErrorMessage(MacroInstance macro) { |
| MacroInstance ancestor = macro.getParent(); |
| boolean foundRecursion = false; |
| boolean onImmediateParent = true; |
| while (ancestor != null) { |
| // TODO: #19922 - We're checking based on object identity here. If we need to worry about |
| // macro classes being serialized and deserialized in a context that also does macro |
| // evaluation, then we should use the more durable identifier of its definition label + name. |
| if (ancestor.getMacroClass() == macro.getMacroClass()) { |
| foundRecursion = true; |
| break; |
| } |
| ancestor = ancestor.getParent(); |
| onImmediateParent = false; |
| } |
| if (!foundRecursion) { |
| return null; |
| } |
| |
| StringBuilder msg = new StringBuilder(); |
| msg.append( |
| String.format( |
| "macro '%s' is %s recursive call of '%s'. Macro instantiation traceback (most" |
| + " recent call last):", |
| macro.getName(), onImmediateParent ? "a direct" : "an indirect", ancestor.getName())); |
| |
| // Materialize the stack as an ArrayList, since we want to output it in reverse order (outermost |
| // first). |
| ArrayList<MacroInstance> allAncestors = new ArrayList<>(); |
| ancestor = macro; |
| while (ancestor != null) { |
| allAncestors.add(ancestor); |
| ancestor = ancestor.getParent(); |
| } |
| for (MacroInstance item : Lists.reverse(allAncestors)) { |
| String pkg = item.getPackage().getPackageIdentifier().getCanonicalForm(); |
| String type = |
| item.getMacroClass().getDefiningBzlLabel().getCanonicalForm() |
| + "%" |
| + item.getMacroClass().getName(); |
| msg.append(String.format("\n\tPackage %s, macro '%s' of type %s", pkg, item.getName(), type)); |
| } |
| return msg.toString(); |
| } |
| } |