| // Copyright 2019 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package com.google.devtools.build.lib.analysis.config; |
| |
| import static com.google.common.base.Preconditions.checkNotNull; |
| import static com.google.common.collect.ImmutableList.toImmutableList; |
| import static com.google.devtools.build.lib.packages.ExecGroup.DEFAULT_EXEC_GROUP_NAME; |
| |
| import com.github.benmanes.caffeine.cache.Cache; |
| import com.github.benmanes.caffeine.cache.Caffeine; |
| import com.google.common.base.VerifyException; |
| import com.google.common.collect.ImmutableList; |
| import com.google.common.collect.ImmutableSet; |
| import com.google.common.collect.Iterables; |
| import com.google.devtools.build.lib.analysis.PlatformOptions; |
| import com.google.devtools.build.lib.analysis.config.transitions.ComparingTransition; |
| import com.google.devtools.build.lib.analysis.config.transitions.ConfigurationTransition; |
| import com.google.devtools.build.lib.analysis.config.transitions.PatchTransition; |
| import com.google.devtools.build.lib.analysis.config.transitions.TransitionFactory; |
| import com.google.devtools.build.lib.analysis.starlark.FunctionTransitionUtil; |
| import com.google.devtools.build.lib.cmdline.Label; |
| import com.google.devtools.build.lib.events.EventHandler; |
| import com.google.devtools.build.lib.packages.AttributeTransitionData; |
| import com.google.devtools.build.lib.rules.config.FeatureFlagValue; |
| import com.google.devtools.build.lib.starlarkbuildapi.StarlarkConfigApi.ExecTransitionFactoryApi; |
| import com.google.devtools.build.lib.util.Pair; |
| import java.util.Map; |
| import javax.annotation.Nullable; |
| |
| /** |
| * {@link TransitionFactory} implementation which creates a {@link PatchTransition} which will |
| * transition to a configuration suitable for building dependencies for the execution platform of |
| * the depending target. |
| * |
| * <p>Note that execGroup is not directly consumed by the involved transition but instead stored |
| * here. Instead, the rule definition stores it in this factory. Then, toolchain resolution extracts |
| * and consumes it to store an execution platform in attrs. Finally, the execution platform is read |
| * by the factory to create the transition. |
| */ |
| public class ExecutionTransitionFactory |
| implements TransitionFactory<AttributeTransitionData>, ExecTransitionFactoryApi { |
| |
| /** |
| * Returns a new {@link ExecutionTransitionFactory} for the default {@link |
| * com.google.devtools.build.lib.packages.ExecGroup}. |
| */ |
| public static ExecutionTransitionFactory createFactory() { |
| return new ExecutionTransitionFactory(DEFAULT_EXEC_GROUP_NAME); |
| } |
| |
| /** |
| * Returns a new {@link ExecutionTransitionFactory} for the given {@link |
| * com.google.devtools.build.lib.packages.ExecGroup}. |
| */ |
| public static ExecutionTransitionFactory createFactory(String execGroup) { |
| return new ExecutionTransitionFactory(execGroup); |
| } |
| |
| /** Returns a new {@link NativeExecTransition} immediately. */ |
| public static PatchTransition createTransition(@Nullable Label executionPlatform) { |
| // TODO(b/288258583): support Starlark transitions. |
| return new ExecTransitionFinalizer(executionPlatform, NativeExecTransition.INSTANCE); |
| } |
| |
| /** |
| * Guarantees we don't duplicate instances of the same transition. |
| * |
| * <p>Bazel already does a lot of the work for us: there's one global native exec transition |
| * instance in the code base: {@link NativeExecTransition#INSTANCE}. Bazel's Starlark logic also |
| * maintains a distinct instance for each Starlark transition. |
| * |
| * <p>While those make this cache seem unnecessary, it still serves two purposes: |
| * |
| * <ol> |
| * <li>This file creates its own transitions that wrap the original transitions. We have to make |
| * sure those transitions don't duplicate. TODO(b/292619013): once we remove the native |
| * transition this probably isn't necessary: remove this rationale |
| * <li>The exec transition uniquely takes an extra parameter: the execution platform label. This |
| * is provided by toolchain resolution - the transition can't read it from input build |
| * options. So we need to cache on {@code label, originalTransition} pairs. |
| * </ol> |
| */ |
| private static final Cache<Pair<Label, Integer>, PatchTransition> transitionInstanceCache = |
| Caffeine.newBuilder().weakValues().build(); |
| |
| @Override |
| public PatchTransition create(AttributeTransitionData dataWithTargetAttributes) { |
| // Delete AttributeTransitionData.attributes() so the exec transition doesn't try to read the |
| // attributes of the target it's attached to. This is for two reasons: |
| // |
| // 1) While per-target exec transitions may be interesting, we're not ready to expose that |
| // level of API flexibility |
| // 2) No need for StarlarkTransitionCache misses due to different StarlarkTransition instances |
| // bound to different attributes that shouldn't affect output. |
| AttributeTransitionData data = |
| AttributeTransitionData.builder() |
| .analysisData(dataWithTargetAttributes.analysisData()) |
| .executionPlatform(dataWithTargetAttributes.executionPlatform()) |
| .build(); |
| |
| // Always get the native transition. |
| PatchTransition nativeTransition = |
| transitionInstanceCache.get( |
| Pair.of( |
| data.executionPlatform(), System.identityHashCode(NativeExecTransition.INSTANCE)), |
| (p) -> |
| new ExecTransitionFinalizer( |
| data.executionPlatform(), NativeExecTransition.INSTANCE)); |
| |
| @SuppressWarnings("unchecked") |
| TransitionFactory<AttributeTransitionData> starlarkExecTransitionProvider = |
| (TransitionFactory<AttributeTransitionData>) data.analysisData(); |
| if (starlarkExecTransitionProvider == null) { |
| return nativeTransition; |
| } |
| |
| return transitionInstanceCache.get( |
| // A Starlark transition keeps the same instance unless we modify its .bzl file. |
| Pair.of(data.executionPlatform(), starlarkExecTransitionProvider.hashCode()), |
| (p) -> { |
| PatchTransition starlarkTransition = |
| new ExecTransitionFinalizer( |
| data.executionPlatform(), starlarkExecTransitionProvider.create(data)); |
| |
| // We don't yet know if --experimental_exec_config_diff is set because this method |
| // doesn't have access to BuildOptions. Universally create a ComparingTransition and |
| // let that figure out if it should run both transitions or just the Starlark one. |
| return new ComparingTransition( |
| /* activeTransition= */ starlarkTransition, |
| /* activeTransitionDesc= */ "starlark", |
| /* altTransition= */ nativeTransition, |
| /* altTransitionDesc= */ "native", |
| /* runBoth= */ b -> b.get(CoreOptions.class).execConfigDiff); |
| }); |
| } |
| |
| private final String execGroup; |
| |
| private ExecutionTransitionFactory(String execGroup) { |
| this.execGroup = execGroup; |
| } |
| |
| @Override |
| public TransitionType transitionType() { |
| return TransitionType.ATTRIBUTE; |
| } |
| |
| public String getExecGroup() { |
| return execGroup; |
| } |
| |
| @Override |
| public boolean isTool() { |
| return true; |
| } |
| |
| /** |
| * Complete exec transition. |
| * |
| * <p>Takes as input the execution platform and the "main" transition used for this build: either |
| * a native or Starlark transition. Calls the main transitino, the runs finalizer logic that's |
| * common to both transition modes. |
| */ |
| private static class ExecTransitionFinalizer implements PatchTransition { |
| private static final ImmutableSet<Class<? extends FragmentOptions>> FRAGMENTS = |
| ImmutableSet.of(CoreOptions.class, PlatformOptions.class); |
| |
| // We added this cache after observing an O(100,000)-node build graph that applied multiple exec |
| // transitions on every node via an aspect. Before this cache, this produced O(500,000) |
| // BuildOptions instances that consumed over 3 gigabytes of memory. |
| private static final BuildOptionsCache<Pair<Label, ConfigurationTransition>> |
| nativeApplicationCache = new BuildOptionsCache<>(ExecTransitionFinalizer::transitionImpl); |
| |
| @Nullable private final Label executionPlatform; |
| |
| private final ConfigurationTransition mainTransition; |
| |
| ExecTransitionFinalizer( |
| @Nullable Label executionPlatform, ConfigurationTransition mainTransition) { |
| this.executionPlatform = executionPlatform; |
| this.mainTransition = mainTransition; |
| } |
| |
| @Override |
| public String getName() { |
| return "exec"; |
| } |
| |
| /** |
| * Implement {@link ConfigurationTransition#visit}} so {@link |
| * com.google.devtools.build.lib.analysis.config.StarlarkTransitionCache} caches application if |
| * this is a Starlark transition. |
| */ |
| @Override |
| public <E extends Exception> void visit(Visitor<E> visitor) throws E { |
| this.mainTransition.visit(visitor); |
| } |
| |
| @Override |
| public ImmutableSet<Class<? extends FragmentOptions>> requiresOptionFragments() { |
| // This is technically a lie since the call to underlying().createExecOptions is transitively |
| // reading and potentially modifying all fragments. There is currently no way for the |
| // transition to actually list all fragments like this and thus only lists the ones that are |
| // directly being read here. Note that this transition is exceptional in its implementation. |
| return FRAGMENTS; |
| } |
| |
| @Override |
| public BuildOptions patch(BuildOptionsView options, EventHandler eventHandler) { |
| if (executionPlatform == null) { |
| // No execution platform is known, so don't change anything. |
| return options.underlying(); |
| } |
| |
| // If this is the Starlark exec transition, StarlarkTransitionCache caches application. If |
| // this is the native exec transition, we need to directly cache application here. |
| // |
| // That means we technically don't need to call this cache if this is a Starlark transition |
| // (we could instead call transitionImpl() directly, trusting StarlarkTransitionCache to |
| // control when that's called). But it's simpler to universally call it here and causes no |
| // harm. And once we remove the native transition we can eliminate this cache outright. |
| // TODO(b/292619013): remove this cache when we remove the native exec transition. |
| return nativeApplicationCache.applyTransition( |
| options, |
| // The execution platform impacts the output's --platform_suffix and --platforms flags. |
| Pair.of(executionPlatform, mainTransition), |
| eventHandler); |
| } |
| |
| private static BuildOptions transitionImpl( |
| BuildOptionsView options, |
| Pair<Label, ConfigurationTransition> data, |
| @Nullable EventHandler eventHandler) { |
| Label executionPlatform = data.first; |
| ConfigurationTransition mainTransition = data.second; |
| |
| BuildOptions execOptions; |
| try { |
| Map.Entry<String, BuildOptions> splitOptions = |
| Iterables.getOnlyElement(mainTransition.apply(options, eventHandler).entrySet()); |
| execOptions = splitOptions.getValue(); |
| } catch (InterruptedException e) { |
| throw new VerifyException(e); |
| } |
| |
| // Set the target to the saved execution platform if there is one. |
| PlatformOptions platformOptions = execOptions.get(PlatformOptions.class); |
| if (platformOptions != null) { |
| platformOptions.platforms = ImmutableList.of(executionPlatform); |
| } |
| |
| // Remove any FeatureFlags that were set. |
| ImmutableList<Label> featureFlags = |
| execOptions.getStarlarkOptions().entrySet().stream() |
| .filter(entry -> entry.getValue() instanceof FeatureFlagValue) |
| .map(Map.Entry::getKey) |
| .collect(toImmutableList()); |
| |
| BuildOptions result = execOptions; |
| if (!featureFlags.isEmpty()) { |
| BuildOptions.Builder resultBuilder = result.toBuilder(); |
| featureFlags.forEach(resultBuilder::removeStarlarkOption); |
| result = resultBuilder.build(); |
| } |
| |
| // The conditional use of a Builder above may have replaced result and underlying options |
| // with a clone so must refresh it. |
| CoreOptions coreOptions = result.get(CoreOptions.class); |
| // TODO(blaze-configurability-team): These updates probably requires a bit too much knowledge |
| // of exactly how the immutable state and mutable state of BuildOptions is interacting. |
| // Might be good to have an option to wipeout that state rather than cloning so much. |
| switch (coreOptions.execConfigurationDistinguisherScheme) { |
| case LEGACY: |
| coreOptions.platformSuffix = |
| String.format("exec-%X", executionPlatform.getCanonicalForm().hashCode()); |
| break; |
| case FULL_HASH: |
| coreOptions.platformSuffix = ""; |
| // execOptions creation above made a clone, which will have a fresh hashCode |
| int fullHash = result.hashCode(); |
| coreOptions.platformSuffix = String.format("exec-%X", fullHash); |
| // Previous call to hashCode irreparably locked in state so must clone to refresh since |
| // options mutated after that |
| result = result.clone(); |
| break; |
| case DIFF_TO_AFFECTED: |
| // Setting platform_suffix here should not be necessary for correctness but |
| // done for user clarity. |
| coreOptions.platformSuffix = "exec"; |
| ImmutableSet<String> diff = |
| FunctionTransitionUtil.getAffectedByStarlarkTransitionViaDiff( |
| result, options.underlying()); |
| FunctionTransitionUtil.updateAffectedByStarlarkTransition(coreOptions, diff); |
| // Previous call to diff irreparably locked in state so must clone to refresh. |
| result = result.clone(); |
| break; |
| default: |
| // else if OFF just mark that we are now in an exec transition |
| coreOptions.platformSuffix = "exec"; |
| } |
| coreOptions.affectedByStarlarkTransition = |
| options.underlying().get(CoreOptions.class).affectedByStarlarkTransition; |
| coreOptions.executionInfoModifier = |
| options.underlying().get(CoreOptions.class).executionInfoModifier; |
| return result; |
| } |
| } |
| |
| /** Logic unique to the native exec transition. */ |
| private static class NativeExecTransition implements PatchTransition { |
| private static final NativeExecTransition INSTANCE = new NativeExecTransition(); |
| |
| private static final ImmutableSet<Class<? extends FragmentOptions>> FRAGMENTS = |
| ImmutableSet.of(CoreOptions.class, PlatformOptions.class); |
| |
| @Override |
| public BuildOptions patch(BuildOptionsView options, EventHandler eventHandler) { |
| // Start by converting to exec options. |
| BuildOptionsView execOptions = |
| new BuildOptionsView(options.underlying().createExecOptions(), FRAGMENTS); |
| |
| CoreOptions coreOptions = checkNotNull(execOptions.get(CoreOptions.class)); |
| coreOptions.isExec = true; |
| // Disable extra actions |
| coreOptions.actionListeners = ImmutableList.of(); |
| |
| return execOptions.underlying(); |
| } |
| } |
| } |