blob: 4d5ad2ea8cbdff381fd9be1262e538f32917c6a7 [file] [log] [blame]
Damien Martin-Guillerezf88f4d82015-09-25 13:56:55 +00001// Copyright 2014 The Bazel Authors. All rights reserved.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +01002//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package com.google.devtools.build.lib.skyframe;
15
adgarebc0f2c2019-08-15 15:36:56 -070016import com.google.common.base.MoreObjects;
tomlua155b532017-11-08 20:12:47 +010017import com.google.common.base.Preconditions;
Janak Ramakrishnandf0531f2015-09-23 17:30:04 +000018import com.google.common.base.Predicates;
adgarebc0f2c2019-08-15 15:36:56 -070019import com.google.common.base.Throwables;
shreyax755a0a12018-03-19 20:47:37 -070020import com.google.common.cache.Cache;
21import com.google.common.cache.CacheBuilder;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010022import com.google.common.collect.ImmutableList;
John Fielda97e17f2015-11-13 02:19:52 +000023import com.google.common.collect.ImmutableMap;
John Fielda97e17f2015-11-13 02:19:52 +000024import com.google.common.collect.Lists;
Janak Ramakrishnandfd34972015-09-22 02:53:05 +000025import com.google.common.collect.Maps;
janakrc3bcb982020-04-14 06:50:08 -070026import com.google.common.flogger.GoogleLogger;
shahan602cc852018-06-06 20:09:57 -070027import com.google.devtools.build.lib.actions.InconsistentFilesystemException;
Lukacs Berki6e91eb92015-09-21 09:12:37 +000028import com.google.devtools.build.lib.cmdline.Label;
Googlerb2e1fe32019-09-20 14:00:39 -070029import com.google.devtools.build.lib.cmdline.LabelConstants;
30import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
Kristina Chodorow73fa2032015-08-28 17:57:46 +000031import com.google.devtools.build.lib.cmdline.PackageIdentifier;
dannark4e42c322018-11-08 19:32:04 -080032import com.google.devtools.build.lib.cmdline.RepositoryName;
shreyax755a0a12018-03-19 20:47:37 -070033import com.google.devtools.build.lib.concurrent.BlazeInterners;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010034import com.google.devtools.build.lib.events.Event;
Dmitry Lomov950310f2017-03-01 17:45:12 +000035import com.google.devtools.build.lib.events.EventHandler;
brandjona0c6f812020-06-06 14:13:19 -070036import com.google.devtools.build.lib.events.ExtendedEventHandler;
Klaus Aehligda0a7012017-06-14 13:40:23 +020037import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010038import com.google.devtools.build.lib.events.StoredEventHandler;
ajurkowski9f2cab52020-05-12 12:00:24 -070039import com.google.devtools.build.lib.packages.BazelModuleContext;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010040import com.google.devtools.build.lib.packages.BuildFileNotFoundException;
41import com.google.devtools.build.lib.packages.PackageFactory;
gregced281df72020-05-11 12:27:06 -070042import com.google.devtools.build.lib.packages.StarlarkExportable;
janakr15e15c22019-01-30 11:24:49 -080043import com.google.devtools.build.lib.packages.WorkspaceFileValue;
brandjon844fef52020-06-08 10:30:36 -070044import com.google.devtools.build.lib.skyframe.StarlarkBuiltinsFunction.BuiltinsFailedException;
Dmitry Lomov7b599452015-11-26 10:07:32 +000045import com.google.devtools.build.lib.syntax.EvalException;
Googlerf0890f02019-10-01 07:28:48 -070046import com.google.devtools.build.lib.syntax.EvalUtils;
John Fielda97e17f2015-11-13 02:19:52 +000047import com.google.devtools.build.lib.syntax.LoadStatement;
adonovan1d93d262020-05-07 14:50:42 -070048import com.google.devtools.build.lib.syntax.Module;
Francois-Rene Rideau89312fb2015-09-10 18:53:03 +000049import com.google.devtools.build.lib.syntax.Mutability;
Googler66d099e2019-09-26 08:07:06 -070050import com.google.devtools.build.lib.syntax.StarlarkFile;
laurentlb6659b4c2019-02-18 07:23:36 -080051import com.google.devtools.build.lib.syntax.StarlarkSemantics;
Googlera3421e22019-09-26 06:48:32 -070052import com.google.devtools.build.lib.syntax.StarlarkThread;
Dmitry Lomov950310f2017-03-01 17:45:12 +000053import com.google.devtools.build.lib.syntax.Statement;
adonovan1d93d262020-05-07 14:50:42 -070054import com.google.devtools.build.lib.util.Fingerprint;
adonovan699a90a2020-05-20 19:48:53 -070055import com.google.devtools.build.lib.util.Pair;
nharmata34879302020-04-22 11:30:15 -070056import com.google.devtools.build.lib.vfs.DigestHashFunction;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010057import com.google.devtools.build.lib.vfs.PathFragment;
shreyax755a0a12018-03-19 20:47:37 -070058import com.google.devtools.build.skyframe.RecordingSkyFunctionEnvironment;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010059import com.google.devtools.build.skyframe.SkyFunction;
60import com.google.devtools.build.skyframe.SkyFunctionException;
61import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
62import com.google.devtools.build.skyframe.SkyKey;
63import com.google.devtools.build.skyframe.SkyValue;
laurentlbdd612a82018-10-16 19:42:48 -070064import com.google.devtools.build.skyframe.ValueOrException;
brandjon1d3f3092020-06-08 10:58:25 -070065import java.util.HashMap;
nharmata636fb5e2020-07-24 15:30:08 -070066import java.util.HashSet;
shreyax39ffd972019-02-07 15:41:31 -080067import java.util.LinkedHashSet;
John Fielda97e17f2015-11-13 02:19:52 +000068import java.util.List;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010069import java.util.Map;
brandjon91988192020-05-28 14:57:45 -070070import java.util.function.Consumer;
Janak Ramakrishnandf0531f2015-09-23 17:30:04 +000071import javax.annotation.Nullable;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010072
73/**
adonovan1d93d262020-05-07 14:50:42 -070074 * A Skyframe function to look up and load a single .bzl module.
John Fielda97e17f2015-11-13 02:19:52 +000075 *
adonovan1d93d262020-05-07 14:50:42 -070076 * <p>Given a {@link Label} referencing a .bzl file, attempts to locate the file and load it. The
77 * Label must be absolute, and must not reference the special {@code external} package. If loading
brandjon771a0292020-05-26 12:04:16 -070078 * is successful, returns a {@link BzlLoadValue} that encapsulates the loaded {@link Module} and its
adonovan39e42d12020-07-09 09:16:58 -070079 * transitive digest information. If loading is unsuccessful, throws a {@link
80 * BzlLoadFunctionException} that encapsulates the cause of the failure.
brandjon91988192020-05-28 14:57:45 -070081 *
brandjonb4ebbfc2020-08-13 15:50:23 -070082 * <p>This Skyframe function supports a special bzl "inlining" mode in which all (indirectly)
83 * recursive calls to {@code BzlLoadFunction} are made in the same thread rather than through
84 * Skyframe. This inlining mode's entry point is {@link #computeInline}; see that method for more
85 * details. Note that it may only be called on an instance of this Skyfunction created by {@link
86 * #createForInlining}. Bzl inlining is not to be confused with the separate inlining of {@code
87 * ASTFileLookupFunction}
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010088 */
brandjon771a0292020-05-26 12:04:16 -070089public class BzlLoadFunction implements SkyFunction {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010090
brandjondb4d1ea2020-08-14 11:31:08 -070091 // Used for: 1) obtaining a RuleClassProvider to create the BazelStarlarkContext for Starlark
92 // evaluation; 2) providing predeclared environments to other Skyfunctions
93 // (StarlarkBuiltinsFunction, ASTFileLookupFunction) when they are inlined and called via a static
94 // computeInline() entry point.
95 private final PackageFactory packageFactory;
brandjon28632522020-06-08 13:14:07 -070096
brandjon1af65cf2020-06-06 18:17:53 -070097 // Used for BUILD .bzls if injection is disabled.
98 // TODO(#11437): Remove once injection is on unconditionally.
99 private final StarlarkBuiltinsValue uninjectedStarlarkBuiltins;
brandjona0c6f812020-06-06 14:13:19 -0700100
brandjonc61bc042020-08-11 11:52:16 -0700101 // Predeclareds for workspace .bzls and builtins .bzls are not subject to builtins injection, so
102 // these environments are stored globally.
brandjona0c6f812020-06-06 14:13:19 -0700103 private final ImmutableMap<String, Object> predeclaredForWorkspaceBzl;
brandjona0c6f812020-06-06 14:13:19 -0700104 private final ImmutableMap<String, Object> predeclaredForBuiltinsBzl;
nharmatadc1d9dc2020-04-18 16:53:28 -0700105
brandjon8d669292020-05-28 15:27:38 -0700106 // Handles retrieving ASTFileLookupValues, either by calling Skyframe or by inlining
107 // ASTFileLookupFunction; the latter is not to be confused with inlining of BzlLoadFunction. See
108 // comment in create() for rationale.
109 private final ASTManager astManager;
110
111 // Handles inlining of BzlLoadFunction calls.
112 @Nullable private final CachedBzlLoadDataManager cachedBzlLoadDataManager;
shreyax755a0a12018-03-19 20:47:37 -0700113
janakrc3bcb982020-04-14 06:50:08 -0700114 private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100115
brandjon771a0292020-05-26 12:04:16 -0700116 private BzlLoadFunction(
nharmatadc1d9dc2020-04-18 16:53:28 -0700117 PackageFactory packageFactory,
brandjon8d669292020-05-28 15:27:38 -0700118 ASTManager astManager,
119 @Nullable CachedBzlLoadDataManager cachedBzlLoadDataManager) {
brandjon28632522020-06-08 13:14:07 -0700120 this.packageFactory = packageFactory;
brandjon1af65cf2020-06-06 18:17:53 -0700121 this.uninjectedStarlarkBuiltins =
brandjondb4d1ea2020-08-14 11:31:08 -0700122 StarlarkBuiltinsFunction.createStarlarkBuiltinsValueWithoutInjection(packageFactory);
brandjona0c6f812020-06-06 14:13:19 -0700123 this.predeclaredForWorkspaceBzl =
brandjondb4d1ea2020-08-14 11:31:08 -0700124 StarlarkBuiltinsFunction.createPredeclaredForWorkspaceBzl(packageFactory);
brandjona0c6f812020-06-06 14:13:19 -0700125 this.predeclaredForBuiltinsBzl =
brandjondb4d1ea2020-08-14 11:31:08 -0700126 StarlarkBuiltinsFunction.createPredeclaredForBuiltinsBzl(packageFactory);
brandjon8d669292020-05-28 15:27:38 -0700127 this.astManager = astManager;
128 this.cachedBzlLoadDataManager = cachedBzlLoadDataManager;
shreyax31262052019-08-13 13:40:06 -0700129 }
130
brandjon771a0292020-05-26 12:04:16 -0700131 public static BzlLoadFunction create(
nharmatadc1d9dc2020-04-18 16:53:28 -0700132 PackageFactory packageFactory,
nharmata34879302020-04-22 11:30:15 -0700133 DigestHashFunction digestHashFunction,
brandjonc782bbc2020-08-05 13:47:13 -0700134 Cache<ASTFileLookupValue.Key, ASTFileLookupValue> astFileLookupValueCache) {
brandjon771a0292020-05-26 12:04:16 -0700135 return new BzlLoadFunction(
nharmatadc1d9dc2020-04-18 16:53:28 -0700136 packageFactory,
brandjon771a0292020-05-26 12:04:16 -0700137 // When we are not inlining BzlLoadValue nodes, there is no need to have separate
brandjon8d669292020-05-28 15:27:38 -0700138 // ASTFileLookupValue nodes for bzl files. Instead we inline ASTFileLookupFunction for a
139 // strict memory win, at a small code complexity cost.
nharmatadc1d9dc2020-04-18 16:53:28 -0700140 //
141 // Detailed explanation:
142 // (1) The ASTFileLookupValue node for a bzl file is used only for the computation of
brandjon771a0292020-05-26 12:04:16 -0700143 // that file's BzlLoadValue node. So there's no concern about duplicate work that would
144 // otherwise get deduped by Skyframe.
nharmatadc1d9dc2020-04-18 16:53:28 -0700145 // (2) ASTFileLookupValue doesn't have an interesting equality relation, so we have no
146 // hope of getting any interesting change-pruning of ASTFileLookupValue nodes. If we
147 // had an interesting equality relation that was e.g. able to ignore benign
148 // whitespace, then there would be a hypothetical benefit to having separate
149 // ASTFileLookupValue nodes (e.g. on incremental builds we'd be able to not re-execute
150 // top-level code in bzl files if the file were reparsed to an equivalent AST).
brandjon771a0292020-05-26 12:04:16 -0700151 // (3) A ASTFileLookupValue node lets us avoid redoing work on a BzlLoadFunction Skyframe
152 // restart, but we can also achieve that result ourselves with a cache that persists between
153 // Skyframe restarts.
nharmatadc1d9dc2020-04-18 16:53:28 -0700154 //
155 // Therefore, ASTFileLookupValue nodes are wasteful from two perspectives:
156 // (a) ASTFileLookupValue contains a StarlarkFile, and that business object is really
157 // just a temporary thing for bzl execution. Retaining it forever is pure waste.
158 // (b) The memory overhead of the extra Skyframe node and edge per bzl file is pure
159 // waste.
brandjon8d669292020-05-28 15:27:38 -0700160 new InliningAndCachingASTManager(
brandjondb4d1ea2020-08-14 11:31:08 -0700161 packageFactory, digestHashFunction, astFileLookupValueCache),
brandjon8d669292020-05-28 15:27:38 -0700162 /*cachedBzlLoadDataManager=*/ null);
nharmatadc1d9dc2020-04-18 16:53:28 -0700163 }
164
brandjon24912642020-05-27 11:28:45 -0700165 public static BzlLoadFunction createForInlining(
brandjondb4d1ea2020-08-14 11:31:08 -0700166 PackageFactory packageFactory, int bzlLoadValueCacheSize) {
brandjon771a0292020-05-26 12:04:16 -0700167 return new BzlLoadFunction(
nharmatadc1d9dc2020-04-18 16:53:28 -0700168 packageFactory,
brandjon771a0292020-05-26 12:04:16 -0700169 // When we are inlining BzlLoadValue nodes, then we want to have explicit ASTFileLookupValue
170 // nodes, since now (1) in the comment above doesn't hold. This way we read and parse each
171 // needed bzl file at most once total globally, rather than once per need (in the worst-case
172 // of a BzlLoadValue inlining cache miss). This is important in the situation where a bzl
173 // file is loaded by a lot of other bzl files or BUILD files.
brandjon8d669292020-05-28 15:27:38 -0700174 RegularSkyframeASTManager.INSTANCE,
175 new CachedBzlLoadDataManager(bzlLoadValueCacheSize));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100176 }
177
178 @Override
shreyax755a0a12018-03-19 20:47:37 -0700179 @Nullable
180 public SkyValue compute(SkyKey skyKey, Environment env)
181 throws SkyFunctionException, InterruptedException {
brandjon771a0292020-05-26 12:04:16 -0700182 BzlLoadValue.Key key = (BzlLoadValue.Key) skyKey.argument();
John Fielda97e17f2015-11-13 02:19:52 +0000183 try {
brandjon7154be02020-05-22 09:47:32 -0700184 return computeInternal(key, env, /*inliningState=*/ null);
John Fielda97e17f2015-11-13 02:19:52 +0000185 } catch (InconsistentFilesystemException e) {
brandjon771a0292020-05-26 12:04:16 -0700186 throw new BzlLoadFunctionException(e, Transience.PERSISTENT);
187 } catch (BzlLoadFailedException e) {
188 throw new BzlLoadFunctionException(e);
John Fielda97e17f2015-11-13 02:19:52 +0000189 }
Janak Ramakrishnandf0531f2015-09-23 17:30:04 +0000190 }
191
brandjon24912642020-05-27 11:28:45 -0700192 /**
193 * Entry point for computing "inline", without any direct or indirect Skyframe calls back into
194 * {@link BzlLoadFunction}. (Other Skyframe calls are permitted.)
195 *
nharmata60d04602020-08-11 12:28:57 -0700196 * <p><b>USAGE NOTES:</b>
197 *
198 * <ul>
199 * <li>This method is intended to be called from {@link PackageFunction} and {@link
200 * StarlarkBuiltinsFunction} and probably shouldn't be used anywhere else. If you think you
201 * need inline Starlark computation, consult with the Core subteam and check out
202 * cl/305127325 for an example of correcting a misuse.
203 * <li>If this method is used with --keep_going and if Skyframe evaluation will never be
brandjonb4ebbfc2020-08-13 15:50:23 -0700204 * interrupted, then this function ensures that the evaluation graph and any error reported
205 * are deterministic.
nharmata60d04602020-08-11 12:28:57 -0700206 * </ul>
brandjon8d669292020-05-28 15:27:38 -0700207 *
208 * <p>Under bzl inlining, there is some calling context that wants to obtain a set of {@link
brandjon91988192020-05-28 14:57:45 -0700209 * BzlLoadValue}s without Skyframe evaluation. For example, a calling context can be a BUILD file
210 * trying to resolve its top-level {@code load()} statements. Although this work proceeds in a
211 * single thread, multiple calling contexts may evaluate .bzls in parallel. To avoid redundant
212 * work, they share a single (global to this Skyfunction instance) cache in lieu of the regular
nharmata636fb5e2020-07-24 15:30:08 -0700213 * Skyframe cache. Unlike the regular Skyframe cache, this cache stores only successes.
brandjon91988192020-05-28 14:57:45 -0700214 *
215 * <p>If two calling contexts race to compute the same .bzl, each one will see a different copy of
216 * it, and only one will end up in the shared cache. This presents a hazard: Suppose A and B both
217 * need foo.bzl, and A needs it twice due to a diamond dependency. If A and B race to compute
218 * foo.bzl, but B's computation populates the cache, then when A comes back to resolve it the
219 * second time it will observe a different {@code BzlLoadValue}. This leads to incorrect Starlark
220 * evaluation since Starlark values may rely on Java object identity (see b/138598337). Even if we
221 * weren't concerned about racing, A may also reevaluate previously computed items due to cache
222 * evictions.
223 *
nharmata636fb5e2020-07-24 15:30:08 -0700224 * <p>To solve this, we keep a second cache, {@link InliningState#successfulLoads}, that is local
225 * to the current calling context, and which never evicts entries. Like the global cache discussed
226 * above, this cache stores only successes. This cache is always checked in preference to the
227 * shared one; it may deviate from the shared one in some of its entries, but the calling context
228 * won't know the difference. (Since bzl inlining is only used for the loading phase, we don't
229 * need to worry about Starlark values from different packages interacting.) The cache is stored
230 * as part of the {@code inliningState} passed in by the caller; the caller can obtain this object
231 * using {@link InliningState#create}.
brandjon91988192020-05-28 14:57:45 -0700232 *
nharmata636fb5e2020-07-24 15:30:08 -0700233 * <p>As an aside, note that we can't avoid having {@link InliningState#successfulLoads} by simply
234 * naively blocking evaluation of .bzls on retrievals from the shared cache. This is because two
235 * contexts could deadlock while trying to evaluate an illegal {@code load()} cycle from opposite
236 * ends. It would be possible to construct a waits-for graph and perform cycle detection, or to
237 * monitor slow threads and do detection lazily, but these do not address the cache eviction
238 * issue. Alternatively, we could make Starlark tolerant of reloading, but that would be
239 * tantamount to implementing full Starlark serialization.
brandjon91988192020-05-28 14:57:45 -0700240 *
nharmata636fb5e2020-07-24 15:30:08 -0700241 * <p>Since our local {@link InliningState#successfulLoads} stores only successes, a separate
242 * concern is that we don't want to unsuccessfully visit the same .bzl more than once in the same
243 * context. (A visitation is unsuccessful if it fails due to an error or if it cannot complete
244 * because of a missing Skyframe dep.) To address this concern we maintain a separate {@link
245 * InliningState#unsuccessfulLoads} set, and use this set to return null instead of duplicating an
246 * unsuccessful visitation.
247 *
248 * @return the requested {@code BzlLoadValue}, or null if there was a missing Skyframe dep, an
249 * unspecified exception in a Skyframe dep request, or if this was a duplicate unsuccessful
250 * visitation
brandjon24912642020-05-27 11:28:45 -0700251 */
nharmata636fb5e2020-07-24 15:30:08 -0700252 // TODO(brandjon): Pick one of the nouns "load" and "bzl" and use that term consistently.
shreyax755a0a12018-03-19 20:47:37 -0700253 @Nullable
brandjon1d3f3092020-06-08 10:58:25 -0700254 BzlLoadValue computeInline(BzlLoadValue.Key key, Environment env, InliningState inliningState)
brandjon771a0292020-05-26 12:04:16 -0700255 throws InconsistentFilesystemException, BzlLoadFailedException, InterruptedException {
brandjon91988192020-05-28 14:57:45 -0700256 // Note to refactorors: No Skyframe calls may be made before the RecordingSkyFunctionEnvironment
257 // is set up below in computeInlineForCacheMiss.
brandjon8d669292020-05-28 15:27:38 -0700258 Preconditions.checkNotNull(cachedBzlLoadDataManager);
brandjon1d3f3092020-06-08 10:58:25 -0700259 CachedBzlLoadData cachedData = computeInlineCachedData(key, env, inliningState);
brandjon24912642020-05-27 11:28:45 -0700260 return cachedData != null ? cachedData.getValue() : null;
Janak Ramakrishnandf0531f2015-09-23 17:30:04 +0000261 }
262
brandjon24912642020-05-27 11:28:45 -0700263 /**
nharmata636fb5e2020-07-24 15:30:08 -0700264 * Retrieves or creates the requested {@link CachedBzlLoadData} object for the given bzl, entering
265 * it into the local and shared caches. This is the entry point for recursive calls to the inline
266 * code path.
brandjon24912642020-05-27 11:28:45 -0700267 *
brandjon91988192020-05-28 14:57:45 -0700268 * <p>Skyframe calls made underneath this function will be logged in the resulting {@code
brandjon1d3f3092020-06-08 10:58:25 -0700269 * CachedBzlLoadData) (or its transitive dependencies). The given Skyframe environment must not
270 * be a {@link RecordingSkyFunctionEnvironment}, since that would imply that calls are being
271 * logged in both the returned value and the parent value.
brandjon91988192020-05-28 14:57:45 -0700272 *
nharmata636fb5e2020-07-24 15:30:08 -0700273 * @return null if there was a missing Skyframe dep, an unspecified exception in a Skyframe dep
274 * request, or if this was a duplicate unsuccessful visitation
brandjon24912642020-05-27 11:28:45 -0700275 */
shreyax755a0a12018-03-19 20:47:37 -0700276 @Nullable
brandjon1d3f3092020-06-08 10:58:25 -0700277 private CachedBzlLoadData computeInlineCachedData(
brandjon91988192020-05-28 14:57:45 -0700278 BzlLoadValue.Key key, Environment env, InliningState inliningState)
brandjon771a0292020-05-26 12:04:16 -0700279 throws InconsistentFilesystemException, BzlLoadFailedException, InterruptedException {
brandjon91988192020-05-28 14:57:45 -0700280 // Note to refactorors: No Skyframe calls may be made before the RecordingSkyFunctionEnvironment
281 // is set up below in computeInlineForCacheMiss.
282
nharmata636fb5e2020-07-24 15:30:08 -0700283 // Try the caches of successful loads. We must try the thread-local cache before the shared, for
284 // consistency purposes (see the javadoc of #computeInline).
285 CachedBzlLoadData cachedData = inliningState.successfulLoads.get(key);
brandjon24912642020-05-27 11:28:45 -0700286 if (cachedData == null) {
brandjon8d669292020-05-28 15:27:38 -0700287 cachedData = cachedBzlLoadDataManager.cache.getIfPresent(key);
brandjon24912642020-05-27 11:28:45 -0700288 if (cachedData != null) {
289 // Found a cache hit from another thread's computation; register the recorded deps as if our
brandjonc782bbc2020-08-05 13:47:13 -0700290 // thread required them for the current key. Incorporate into successfulLoads any transitive
brandjon24912642020-05-27 11:28:45 -0700291 // cache hits it does not already contain.
nharmata636fb5e2020-07-24 15:30:08 -0700292 cachedData.traverse(env::registerDependencies, inliningState.successfulLoads);
shreyax31262052019-08-13 13:40:06 -0700293 }
294 }
brandjon91988192020-05-28 14:57:45 -0700295
nharmata636fb5e2020-07-24 15:30:08 -0700296 // See if we've already unsuccessfully visited the bzl.
297 if (inliningState.unsuccessfulLoads.contains(key)) {
298 return null;
299 }
300
301 // If we're here, the bzl must have never been visited before in this calling context. Compute
302 // it ourselves, updating the other data structures as appropriate.
brandjon91988192020-05-28 14:57:45 -0700303 if (cachedData == null) {
nharmata636fb5e2020-07-24 15:30:08 -0700304 try {
305 cachedData = computeInlineForCacheMiss(key, env, inliningState);
306 } finally {
307 if (cachedData != null) {
308 inliningState.successfulLoads.put(key, cachedData);
309 cachedBzlLoadDataManager.cache.put(key, cachedData);
310 } else {
311 inliningState.unsuccessfulLoads.add(key);
312 // Either propagate an exception or fall through for null return.
313 }
brandjon91988192020-05-28 14:57:45 -0700314 }
315 }
316
nharmata636fb5e2020-07-24 15:30:08 -0700317 // On success (from cache hit or from scratch), notify the parent CachedBzlLoadData of its new
318 // child.
brandjon24912642020-05-27 11:28:45 -0700319 if (cachedData != null) {
brandjon91988192020-05-28 14:57:45 -0700320 inliningState.childCachedDataHandler.accept(cachedData);
shreyax755a0a12018-03-19 20:47:37 -0700321 }
322
brandjon91988192020-05-28 14:57:45 -0700323 return cachedData;
324 }
shreyaxc163c912019-02-08 12:45:25 -0800325
brandjon91988192020-05-28 14:57:45 -0700326 @Nullable
327 private CachedBzlLoadData computeInlineForCacheMiss(
328 BzlLoadValue.Key key, Environment env, InliningState inliningState)
329 throws InconsistentFilesystemException, BzlLoadFailedException, InterruptedException {
330 // We use an instrumented Skyframe env to capture Skyframe deps in the CachedBzlLoadData. This
331 // generally includes transitive Skyframe deps, but specifically excludes deps underneath
332 // recursively loaded .bzls. We unwrap the instrumented env right before recursively calling
brandjon1d3f3092020-06-08 10:58:25 -0700333 // back into computeInlineCachedData.
brandjon8d669292020-05-28 15:27:38 -0700334 CachedBzlLoadData.Builder cachedDataBuilder = cachedBzlLoadDataManager.cachedDataBuilder();
shreyax755a0a12018-03-19 20:47:37 -0700335 Preconditions.checkState(
336 !(env instanceof RecordingSkyFunctionEnvironment),
337 "Found nested RecordingSkyFunctionEnvironment but it should have been stripped: %s",
338 env);
339 RecordingSkyFunctionEnvironment recordingEnv =
340 new RecordingSkyFunctionEnvironment(
341 env,
brandjon24912642020-05-27 11:28:45 -0700342 cachedDataBuilder::addDep,
343 cachedDataBuilder::addDeps,
344 cachedDataBuilder::noteException);
shreyax755a0a12018-03-19 20:47:37 -0700345
brandjon91988192020-05-28 14:57:45 -0700346 inliningState.beginLoad(key); // track for cyclic load() detection
347 BzlLoadValue value;
348 try {
349 value =
350 computeInternal(
351 key,
352 recordingEnv,
353 inliningState.createChildState(
354 /*childCachedDataHandler=*/ cachedDataBuilder::addTransitiveDeps));
355 } finally {
356 inliningState.finishLoad(key);
shreyax755a0a12018-03-19 20:47:37 -0700357 }
brandjon91988192020-05-28 14:57:45 -0700358 if (value == null) {
359 return null;
360 }
361
362 cachedDataBuilder.setValue(value);
363 cachedDataBuilder.setKey(key);
364 return cachedDataBuilder.build();
Michajlo Matijkiw10c363d2015-10-09 22:11:34 +0000365 }
366
brandjon24912642020-05-27 11:28:45 -0700367 public void resetInliningCache() {
brandjon8d669292020-05-28 15:27:38 -0700368 cachedBzlLoadDataManager.reset();
shreyax755a0a12018-03-19 20:47:37 -0700369 }
370
brandjon1af65cf2020-06-06 18:17:53 -0700371 /**
brandjonb4ebbfc2020-08-13 15:50:23 -0700372 * An opaque object that holds state for the bzl inlining computation initiated by {@link
brandjon1d3f3092020-06-08 10:58:25 -0700373 * #computeInline}.
374 *
375 * <p>An original caller of {@code computeInline} (e.g., {@link PackageFunction}) should obtain
376 * one of these objects using {@link InliningState#create}. When the same caller makes several
377 * calls to {@code computeInline} (e.g., for multiple top-level loads in the same BUILD file), the
378 * same object must be passed to each call.
379 *
380 * <p>When a Skyfunction that is called by {@code BzlLoadFunction}'s inlining code path in turn
381 * calls back into {@code computeInline}, it should forward along the same {@code InliningState}
382 * that it received. In particular, {@link StarlarkBuiltinsFunction} forwards the inlining state
383 * to ensure that 1) the .bzls that get loaded from the {@code @builtins} pseudo-repository are
384 * properly recorded as dependencies of all .bzl files that use builtins injection, and 2) the
385 * {@code @builtins} .bzls are not reevaluated.
brandjon24912642020-05-27 11:28:45 -0700386 */
brandjon1d3f3092020-06-08 10:58:25 -0700387 static class InliningState {
brandjon91988192020-05-28 14:57:45 -0700388 /**
nharmata636fb5e2020-07-24 15:30:08 -0700389 * The set of bzls we're currently in the process of loading but haven't fully visited yet. This
390 * is used for cycle detection since we don't have the benefit of Skyframe's internal cycle
391 * detection. The set must use insertion order for correct error reporting.
392 *
393 * <p>This is disjoint with {@link #successfulLoads} and {@link #unsuccessfulLoads}.
394 *
395 * <p>This is local to current calling context. See {@link #computeInline}.
brandjon91988192020-05-28 14:57:45 -0700396 */
397 // Keyed on the SkyKey, not the label, since label could theoretically be ambiguous, even though
398 // in practice keys from BUILD / WORKSPACE / @builtins don't call each other. (Not sure if
399 // WORKSPACE chunking can cause duplicate labels to appear, but we're robust regardless.)
400 private final LinkedHashSet<BzlLoadValue.Key> loadStack;
401
nharmata636fb5e2020-07-24 15:30:08 -0700402 /**
403 * Cache of bzls that have been fully visited and successfully loaded to a value.
404 *
405 * <p>This and {@link #unsuccessfulLoads} partition the set of fully visited bzls.
406 *
407 * <p>This is local to current calling context. See {@link #computeInline}.
408 */
409 private final Map<BzlLoadValue.Key, CachedBzlLoadData> successfulLoads;
410
411 /**
412 * Set of bzls that have been fully visited, but were not successfully loaded to a value.
413 *
414 * <p>This and {@link #successfulLoads} partition the set of fully visited bzls, and is disjoint
415 * with {@link #loadStack}.
416 *
417 * <p>This is local to current calling context. See {@link #computeInline}.
418 */
419 private final HashSet<BzlLoadValue.Key> unsuccessfulLoads;
420
brandjon91988192020-05-28 14:57:45 -0700421 /** Called when a transitive {@code CachedBzlLoadData} is produced. */
422 private final Consumer<CachedBzlLoadData> childCachedDataHandler;
423
nharmatadc1d9dc2020-04-18 16:53:28 -0700424 private InliningState(
brandjon91988192020-05-28 14:57:45 -0700425 LinkedHashSet<BzlLoadValue.Key> loadStack,
nharmata636fb5e2020-07-24 15:30:08 -0700426 Map<BzlLoadValue.Key, CachedBzlLoadData> successfulLoads,
427 HashSet<BzlLoadValue.Key> unsuccessfulLoads,
428 Consumer<CachedBzlLoadData> childCachedDataHandler) {
brandjon24912642020-05-27 11:28:45 -0700429 this.loadStack = loadStack;
nharmata636fb5e2020-07-24 15:30:08 -0700430 this.successfulLoads = successfulLoads;
431 this.unsuccessfulLoads = unsuccessfulLoads;
brandjon91988192020-05-28 14:57:45 -0700432 this.childCachedDataHandler = childCachedDataHandler;
nharmatadc1d9dc2020-04-18 16:53:28 -0700433 }
brandjon91988192020-05-28 14:57:45 -0700434
brandjon1d3f3092020-06-08 10:58:25 -0700435 /**
436 * Creates an initial {@code InliningState} with no information about previously loaded files
437 * (except the shared cache stored in {@link BzlLoadFunction}).
438 */
439 static InliningState create() {
brandjon91988192020-05-28 14:57:45 -0700440 return new InliningState(
brandjon1d3f3092020-06-08 10:58:25 -0700441 /*loadStack=*/ new LinkedHashSet<>(),
nharmata636fb5e2020-07-24 15:30:08 -0700442 /*successfulLoads=*/ new HashMap<>(),
443 /*unsuccessfulLoads=*/ new HashSet<>(),
444 // No parent value to mutate
445 /*childCachedDataHandler=*/ x -> {});
brandjon91988192020-05-28 14:57:45 -0700446 }
447
brandjon1d3f3092020-06-08 10:58:25 -0700448 private InliningState createChildState(Consumer<CachedBzlLoadData> childCachedDataHandler) {
nharmata636fb5e2020-07-24 15:30:08 -0700449 return new InliningState(
450 loadStack, successfulLoads, unsuccessfulLoads, childCachedDataHandler);
brandjon91988192020-05-28 14:57:45 -0700451 }
452
453 /** Records entry to a {@code load()}, throwing an exception if a cycle is detected. */
brandjon1d3f3092020-06-08 10:58:25 -0700454 private void beginLoad(BzlLoadValue.Key key) throws BzlLoadFailedException {
brandjon91988192020-05-28 14:57:45 -0700455 if (!loadStack.add(key)) {
456 ImmutableList<BzlLoadValue.Key> cycle =
457 CycleUtils.splitIntoPathAndChain(Predicates.equalTo(key), loadStack).second;
458 throw new BzlLoadFailedException("Starlark load cycle: " + cycle);
459 }
460 }
461
462 /** Records exit from a {@code load()}. */
brandjon1d3f3092020-06-08 10:58:25 -0700463 private void finishLoad(BzlLoadValue.Key key) throws BzlLoadFailedException {
brandjon91988192020-05-28 14:57:45 -0700464 Preconditions.checkState(loadStack.remove(key), key);
465 }
nharmatadc1d9dc2020-04-18 16:53:28 -0700466 }
467
brandjonb4ebbfc2020-08-13 15:50:23 -0700468 /**
469 * Entry point for compute logic that's common to both (bzl) inlining and non-inlining code paths.
470 */
shreyax755a0a12018-03-19 20:47:37 -0700471 // It is vital that we don't return any value if any call to env#getValue(s)OrThrow throws an
472 // exception. We are allowed to wrap the thrown exception and rethrow it for any calling functions
473 // to handle though.
474 @Nullable
brandjon771a0292020-05-26 12:04:16 -0700475 private BzlLoadValue computeInternal(
476 BzlLoadValue.Key key, Environment env, @Nullable InliningState inliningState)
477 throws InconsistentFilesystemException, BzlLoadFailedException, InterruptedException {
brandjonbc884e12020-05-22 10:24:37 -0700478 Label label = key.getLabel();
brandjon7154be02020-05-22 09:47:32 -0700479 PathFragment filePath = label.toPathFragment();
John Field110b0652015-11-13 21:56:42 +0000480
brandjon4d15f2b2020-08-12 09:15:52 -0700481 ASTFileLookupValue.Key astKey = validatePackageAndGetASTKey(key, env);
482 if (astKey == null) {
483 return null;
484 }
485 ASTFileLookupValue astLookup;
486 try {
487 astLookup = astManager.getASTFileLookupValue(astKey, env);
488 } catch (ErrorReadingStarlarkExtensionException e) {
489 throw BzlLoadFailedException.errorReadingFile(filePath, e);
490 }
491 if (astLookup == null) {
492 return null;
493 }
494
495 BzlLoadValue result = null;
496 // Release the AST iff the value gets completely evaluated (to either error or non-null result).
497 boolean completed = true;
498 try {
499 result = computeInternalWithAST(key, astLookup, env, inliningState);
500 completed = result != null;
501 } finally {
502 if (completed) { // only false on unexceptional null result
503 astManager.doneWithASTFileLookupValue(astKey);
504 }
505 }
506 return result;
507 }
508
509 /**
510 * Returns the AST key for a bzl, or null for a missing Skyframe dep or unspecified exception.
511 *
512 * <p>Except for builtins bzls, a bzl is not considered loadable unless its load label matches its
513 * file target label.
514 */
515 @Nullable
516 private static ASTFileLookupValue.Key validatePackageAndGetASTKey(
517 BzlLoadValue.Key key, Environment env)
518 throws BzlLoadFailedException, InconsistentFilesystemException, InterruptedException {
519 Label label = key.getLabel();
520
521 // Do package lookup.
522 PathFragment dir = Label.getContainingDirectory(label);
523 PackageIdentifier dirId =
524 PackageIdentifier.create(label.getPackageIdentifier().getRepository(), dir);
525 ContainingPackageLookupValue packageLookup;
526 try {
527 packageLookup =
528 (ContainingPackageLookupValue)
529 env.getValueOrThrow(
530 ContainingPackageLookupValue.key(dirId),
531 BuildFileNotFoundException.class,
532 InconsistentFilesystemException.class);
533 } catch (BuildFileNotFoundException e) {
534 throw BzlLoadFailedException.errorReadingFile(
535 label.toPathFragment(), new ErrorReadingStarlarkExtensionException(e));
536 }
537 if (packageLookup == null) {
538 return null;
539 }
540
541 // Resolve to AST key or error.
542 ASTFileLookupValue.Key astKey;
543 boolean packageOk =
544 packageLookup.hasContainingPackage()
545 && packageLookup.getContainingPackageName().equals(label.getPackageIdentifier());
546 if (key.isBuildPrelude() && !packageOk) {
547 // Ignore the prelude, its package doesn't exist.
548 astKey = ASTFileLookupValue.EMPTY_PRELUDE_KEY;
549 } else {
550 if (packageOk) {
551 astKey = key.getASTKey(packageLookup.getContainingPackageRoot());
552 } else {
553 if (!packageLookup.hasContainingPackage()) {
554 throw BzlLoadFailedException.noBuildFile(
555 label, packageLookup.getReasonForNoContainingPackage());
556 } else {
557 throw BzlLoadFailedException.labelCrossesPackageBoundary(label, packageLookup);
558 }
559 }
560 }
561 return astKey;
562 }
563
564 /**
565 * Compute logic for once the AST has been fetched and confirmed to exist (though it may have
566 * Starlark errors).
567 */
568 @Nullable
569 private BzlLoadValue computeInternalWithAST(
570 BzlLoadValue.Key key,
571 ASTFileLookupValue astLookup,
572 Environment env,
573 @Nullable InliningState inliningState)
574 throws InconsistentFilesystemException, BzlLoadFailedException, InterruptedException {
575 Label label = key.getLabel();
576 PathFragment filePath = label.toPathFragment();
577
578 if (!astLookup.lookupSuccessful()) {
579 // Starlark code must exist. (A missing prelude file still returns a valid but empty
580 // ASTFileLookupValue.)
581 throw new BzlLoadFailedException(astLookup.getError());
582 }
583 StarlarkFile file = astLookup.getAST();
584 if (!file.ok()) {
585 throw BzlLoadFailedException.starlarkErrors(filePath);
586 }
587
laurentlb8c02aff2019-02-18 10:53:34 -0800588 StarlarkSemantics starlarkSemantics = PrecomputedValue.STARLARK_SEMANTICS.get(env);
laurentlb6659b4c2019-02-18 07:23:36 -0800589 if (starlarkSemantics == null) {
brandjonb712f332017-04-29 16:03:32 +0200590 return null;
591 }
brandjonbc9bd062020-08-12 09:41:12 -0700592 ImmutableMap<String, Object> predeclared =
593 getPredeclaredEnvironment(key, env, starlarkSemantics, inliningState);
594 if (predeclared == null) {
brandjon1af65cf2020-06-06 18:17:53 -0700595 return null;
596 }
597
adonovan87b46082020-07-08 15:58:04 -0700598 // Process load statements in .bzl file (recursive .bzl -> .bzl loads),
Googlerb2e1fe32019-09-20 14:00:39 -0700599 // resolving labels relative to the current repo mapping.
brandjon7154be02020-05-22 09:47:32 -0700600 ImmutableMap<RepositoryName, RepositoryName> repoMapping = getRepositoryMapping(key, env);
Googlerb2e1fe32019-09-20 14:00:39 -0700601 if (repoMapping == null) {
dannark4e42c322018-11-08 19:32:04 -0800602 return null;
603 }
adonovan699a90a2020-05-20 19:48:53 -0700604 List<Pair<String, Label>> loads =
brandjon7154be02020-05-22 09:47:32 -0700605 getLoadLabels(env.getListener(), file, label.getPackageIdentifier(), repoMapping);
adonovan699a90a2020-05-20 19:48:53 -0700606 if (loads == null) {
Googlerb2e1fe32019-09-20 14:00:39 -0700607 // malformed load statements
gregce0503fee2020-06-11 09:22:27 -0700608 throw BzlLoadFailedException.starlarkErrors(filePath);
Googlerb2e1fe32019-09-20 14:00:39 -0700609 }
Janak Ramakrishnandf0531f2015-09-23 17:30:04 +0000610
adonovan699a90a2020-05-20 19:48:53 -0700611 // Compute Skyframe key for each label in 'loads'.
brandjon771a0292020-05-26 12:04:16 -0700612 List<BzlLoadValue.Key> loadKeys = Lists.newArrayListWithExpectedSize(loads.size());
adonovan699a90a2020-05-20 19:48:53 -0700613 for (Pair<String, Label> load : loads) {
brandjon7154be02020-05-22 09:47:32 -0700614 loadKeys.add(key.getKeyForLoad(load.second));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100615 }
adonovan699a90a2020-05-20 19:48:53 -0700616
617 // Load .bzl modules in parallel.
brandjon844fef52020-06-08 10:30:36 -0700618 // TODO(bazel-team): In case of a failed load(), we should report the location of the load()
619 // statement in the requesting file, e.g. using
620 // file.getLoadStatements().get(...).getStartLocation(). We should also probably catch and
brandjonb4ebbfc2020-08-13 15:50:23 -0700621 // rethrow InconsistentFilesystemException with location info in the non-bzl-inlining code path
622 // so the error message is the same in both code paths.
brandjon771a0292020-05-26 12:04:16 -0700623 List<BzlLoadValue> bzlLoads =
adonovan699a90a2020-05-20 19:48:53 -0700624 inliningState == null
brandjon844fef52020-06-08 10:30:36 -0700625 ? computeBzlLoadsWithSkyframe(env, loadKeys, file)
626 : computeBzlLoadsWithInlining(env, loadKeys, file, inliningState);
brandjon771a0292020-05-26 12:04:16 -0700627 if (bzlLoads == null) {
adonovan699a90a2020-05-20 19:48:53 -0700628 return null; // Skyframe deps unavailable
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100629 }
Janak Ramakrishnandf0531f2015-09-23 17:30:04 +0000630
brandjon771a0292020-05-26 12:04:16 -0700631 // Process the loaded modules.
adonovan1d93d262020-05-07 14:50:42 -0700632 //
633 // Compute a digest of the file itself plus the transitive hashes of the modules it directly
634 // loads. Loop iteration order matches the source order of load statements.
635 Fingerprint fp = new Fingerprint();
brandjon4d15f2b2020-08-12 09:15:52 -0700636 fp.addBytes(astLookup.getDigest());
adonovan87b46082020-07-08 15:58:04 -0700637 Map<String, Module> loadedModules = Maps.newLinkedHashMapWithExpectedSize(loads.size());
adonovan699a90a2020-05-20 19:48:53 -0700638 for (int i = 0; i < loads.size(); i++) {
639 String loadString = loads.get(i).first;
brandjon771a0292020-05-26 12:04:16 -0700640 BzlLoadValue v = bzlLoads.get(i);
adonovan87b46082020-07-08 15:58:04 -0700641 loadedModules.put(loadString, v.getModule()); // dups ok
adonovan1d93d262020-05-07 14:50:42 -0700642 fp.addBytes(v.getTransitiveDigest());
Janak Ramakrishnandfd34972015-09-22 02:53:05 +0000643 }
adonovan1d93d262020-05-07 14:50:42 -0700644 byte[] transitiveDigest = fp.digestAndReset();
John Field1ea7fc32015-12-22 19:37:19 +0000645
brandjonbc9bd062020-08-12 09:41:12 -0700646 Module module = Module.withPredeclared(starlarkSemantics, predeclared);
adonovan87b46082020-07-08 15:58:04 -0700647
648 // Record the module's filename, label, digest, and the set of modules it loads,
649 // forming a complete representation of the load DAG.
650 module.setClientData(
651 BazelModuleContext.create(
652 label,
653 file.getStartLocation().file(),
654 ImmutableMap.copyOf(loadedModules),
655 transitiveDigest));
656
brandjona0c6f812020-06-06 14:13:19 -0700657 // executeBzlFile may post events to the Environment's handler, but events do not matter when
658 // caching BzlLoadValues. Note that executing the module mutates it.
659 executeBzlFile(
660 file,
661 key.getLabel(),
662 module,
663 loadedModules,
664 starlarkSemantics,
665 env.getListener(),
666 repoMapping);
adonovan39e42d12020-07-09 09:16:58 -0700667 return new BzlLoadValue(module, transitiveDigest);
John Fielda97e17f2015-11-13 02:19:52 +0000668 }
669
laurentlb4efb6c22019-07-03 15:33:46 -0700670 private static ImmutableMap<RepositoryName, RepositoryName> getRepositoryMapping(
brandjon771a0292020-05-26 12:04:16 -0700671 BzlLoadValue.Key key, Environment env) throws InterruptedException {
brandjon7154be02020-05-22 09:47:32 -0700672 Label enclosingFileLabel = key.getLabel();
pcloudy260c3aa2019-01-04 02:33:50 -0800673
dannarkd102a392019-01-09 13:12:01 -0800674 ImmutableMap<RepositoryName, RepositoryName> repositoryMapping;
brandjonb28a76b2020-05-26 12:53:22 -0700675 if (key instanceof BzlLoadValue.KeyForWorkspace) {
brandjon7154be02020-05-22 09:47:32 -0700676 // Still during workspace file evaluation
brandjonb28a76b2020-05-26 12:53:22 -0700677 BzlLoadValue.KeyForWorkspace keyForWorkspace = (BzlLoadValue.KeyForWorkspace) key;
678 if (keyForWorkspace.getWorkspaceChunk() == 0) {
brandjon7154be02020-05-22 09:47:32 -0700679 // There is no previous workspace chunk
680 repositoryMapping = ImmutableMap.of();
681 } else {
682 SkyKey workspaceFileKey =
683 WorkspaceFileValue.key(
brandjonb28a76b2020-05-26 12:53:22 -0700684 keyForWorkspace.getWorkspacePath(), keyForWorkspace.getWorkspaceChunk() - 1);
brandjon7154be02020-05-22 09:47:32 -0700685 WorkspaceFileValue workspaceFileValue = (WorkspaceFileValue) env.getValue(workspaceFileKey);
686 // Note: we know for sure that the requested WorkspaceFileValue is fully computed so we do
687 // not need to check if it is null
688 repositoryMapping =
689 workspaceFileValue
690 .getRepositoryMapping()
691 .getOrDefault(
692 enclosingFileLabel.getPackageIdentifier().getRepository(), ImmutableMap.of());
693 }
694 } else {
695 // We are fully done with workspace evaluation so we should get the mappings from the
696 // final RepositoryMappingValue
dannark4e42c322018-11-08 19:32:04 -0800697 PackageIdentifier packageIdentifier = enclosingFileLabel.getPackageIdentifier();
698 RepositoryMappingValue repositoryMappingValue =
699 (RepositoryMappingValue)
700 env.getValue(RepositoryMappingValue.key(packageIdentifier.getRepository()));
701 if (repositoryMappingValue == null) {
702 return null;
703 }
704 repositoryMapping = repositoryMappingValue.getRepositoryMapping();
dannark4e42c322018-11-08 19:32:04 -0800705 }
dannarkd102a392019-01-09 13:12:01 -0800706 return repositoryMapping;
707 }
708
709 /**
adonovan699a90a2020-05-20 19:48:53 -0700710 * Returns a list of pairs mapping each load string in the BUILD or .bzl file to the Label it
711 * resolves to. Labels are resolved relative to {@code base}, the file's package. If any load
712 * statement is malformed, the function reports one or more errors to the handler and returns
713 * null. Order matches the source.
John Fielda97e17f2015-11-13 02:19:52 +0000714 */
715 @Nullable
adonovan699a90a2020-05-20 19:48:53 -0700716 static List<Pair<String, Label>> getLoadLabels(
Googlerb2e1fe32019-09-20 14:00:39 -0700717 EventHandler handler,
Googler66d099e2019-09-26 08:07:06 -0700718 StarlarkFile file,
Googlerb2e1fe32019-09-20 14:00:39 -0700719 PackageIdentifier base,
720 ImmutableMap<RepositoryName, RepositoryName> repoMapping) {
721 Preconditions.checkArgument(!base.getRepository().isDefault());
722
723 // It's redundant that getRelativeWithRemapping needs a Label;
724 // a PackageIdentifier should suffice. Make one here.
725 Label buildLabel = getBUILDLabel(base);
726
727 boolean ok = true;
adonovan699a90a2020-05-20 19:48:53 -0700728 List<Pair<String, Label>> loads = Lists.newArrayList();
Googlerb2e1fe32019-09-20 14:00:39 -0700729 for (Statement stmt : file.getStatements()) {
730 if (stmt instanceof LoadStatement) {
731 LoadStatement load = (LoadStatement) stmt;
732 String module = load.getImport().getValue();
733
734 // Parse the load statement's module string as a label.
735 // It must end in .bzl and not be in package "//external".
Googlerb2e1fe32019-09-20 14:00:39 -0700736 try {
737 Label label = buildLabel.getRelativeWithRemapping(module, repoMapping);
738 if (!label.getName().endsWith(".bzl")) {
739 throw new LabelSyntaxException("The label must reference a file with extension '.bzl'");
740 }
741 if (label.getPackageIdentifier().equals(LabelConstants.EXTERNAL_PACKAGE_IDENTIFIER)) {
742 throw new LabelSyntaxException(
743 "Starlark files may not be loaded from the //external package");
744 }
adonovan699a90a2020-05-20 19:48:53 -0700745 loads.add(Pair.of(module, label));
Googlerb2e1fe32019-09-20 14:00:39 -0700746 } catch (LabelSyntaxException ex) {
Googler83439e62019-09-24 12:11:30 -0700747 handler.handle(
adonovan22096af2020-01-08 07:09:52 -0800748 Event.error(
749 load.getImport().getStartLocation(), "in load statement: " + ex.getMessage()));
Googlerb2e1fe32019-09-20 14:00:39 -0700750 ok = false;
751 }
752 }
753 }
adonovan699a90a2020-05-20 19:48:53 -0700754 return ok ? loads : null;
Googlerb2e1fe32019-09-20 14:00:39 -0700755 }
756
757 private static Label getBUILDLabel(PackageIdentifier pkgid) {
758 try {
759 return Label.create(pkgid, "BUILD");
760 } catch (LabelSyntaxException e) {
761 // Shouldn't happen; the Label is well-formed by construction.
762 throw new IllegalStateException(e);
763 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100764 }
765
adgarebc0f2c2019-08-15 15:36:56 -0700766 /**
brandjon24912642020-05-27 11:28:45 -0700767 * Computes the BzlLoadValue for all given keys using vanilla Skyframe evaluation, returning
768 * {@code null} if Skyframe deps were missing and have been requested.
adgarebc0f2c2019-08-15 15:36:56 -0700769 */
770 @Nullable
brandjon8d669292020-05-28 15:27:38 -0700771 private static List<BzlLoadValue> computeBzlLoadsWithSkyframe(
brandjon844fef52020-06-08 10:30:36 -0700772 Environment env, List<BzlLoadValue.Key> keys, StarlarkFile requestingFile)
brandjon771a0292020-05-26 12:04:16 -0700773 throws BzlLoadFailedException, InterruptedException {
774 List<BzlLoadValue> bzlLoads = Lists.newArrayListWithExpectedSize(keys.size());
775 Map<SkyKey, ValueOrException<BzlLoadFailedException>> values =
776 env.getValuesOrThrow(keys, BzlLoadFailedException.class);
brandjonf4f91932020-05-11 10:20:48 -0700777 // Uses same order as load()s in the file. Order matters since we report the first error.
brandjon771a0292020-05-26 12:04:16 -0700778 for (BzlLoadValue.Key key : keys) {
adgarebc0f2c2019-08-15 15:36:56 -0700779 try {
brandjon771a0292020-05-26 12:04:16 -0700780 bzlLoads.add((BzlLoadValue) values.get(key).get());
781 } catch (BzlLoadFailedException exn) {
brandjon844fef52020-06-08 10:30:36 -0700782 throw BzlLoadFailedException.whileLoadingDep(requestingFile.getStartLocation().file(), exn);
adgarebc0f2c2019-08-15 15:36:56 -0700783 }
784 }
brandjon771a0292020-05-26 12:04:16 -0700785 return env.valuesMissing() ? null : bzlLoads;
adgarebc0f2c2019-08-15 15:36:56 -0700786 }
787
788 /**
brandjon24912642020-05-27 11:28:45 -0700789 * Computes the BzlLoadValue for all given keys by reusing this instance of the BzlLoadFunction,
nharmata636fb5e2020-07-24 15:30:08 -0700790 * bypassing traditional Skyframe evaluation.
791 *
792 * @return null if there was a missing Skyframe dep, an unspecified exception in a Skyframe dep
793 * request, or if this was a duplicate unsuccessful visitation
adgarebc0f2c2019-08-15 15:36:56 -0700794 */
795 @Nullable
brandjon24912642020-05-27 11:28:45 -0700796 private List<BzlLoadValue> computeBzlLoadsWithInlining(
brandjon844fef52020-06-08 10:30:36 -0700797 Environment env,
798 List<BzlLoadValue.Key> keys,
799 StarlarkFile requestingFile,
800 InliningState inliningState)
brandjon771a0292020-05-26 12:04:16 -0700801 throws InterruptedException, BzlLoadFailedException, InconsistentFilesystemException {
brandjon844fef52020-06-08 10:30:36 -0700802 String filePathForErrors = requestingFile.getStartLocation().file();
adgarebc0f2c2019-08-15 15:36:56 -0700803 Preconditions.checkState(
804 env instanceof RecordingSkyFunctionEnvironment,
brandjon771a0292020-05-26 12:04:16 -0700805 "Expected to be recording dep requests when inlining BzlLoadFunction: %s",
brandjon844fef52020-06-08 10:30:36 -0700806 filePathForErrors);
adgarebc0f2c2019-08-15 15:36:56 -0700807 Environment strippedEnv = ((RecordingSkyFunctionEnvironment) env).getDelegate();
brandjon91988192020-05-28 14:57:45 -0700808
brandjon771a0292020-05-26 12:04:16 -0700809 List<BzlLoadValue> bzlLoads = Lists.newArrayListWithExpectedSize(keys.size());
nharmata60d04602020-08-11 12:28:57 -0700810 // For the sake of ensuring the graph structure is deterministic, we need to request all of our
brandjonb4ebbfc2020-08-13 15:50:23 -0700811 // deps, even if some of them yield errors. The first exception that is seen gets deferred, to
812 // be raised after the loop. All other exceptions are swallowed.
nharmata60d04602020-08-11 12:28:57 -0700813 //
brandjonb4ebbfc2020-08-13 15:50:23 -0700814 // To see how immediately returning the first error leads to non-determinism, consider the case
815 // of two dependencies A and B, where A is in error and appears in a load statement above B.
816 // If A has completed at the time we request it, and if we were to immediately propagate that
817 // error, we never request B. On the other hand, if A is missing (null return), we do request B
818 // in the meantime for the sake of parallelism.
819 //
820 // This approach assumes --keep_going; determinism is not guaranteed otherwise. It also assumes
821 // InterruptedException does not occur, since we don't catch and defer it.
adgarebc0f2c2019-08-15 15:36:56 -0700822 Exception deferredException = null;
823 boolean valuesMissing = false;
brandjon771a0292020-05-26 12:04:16 -0700824 // NOTE: Iterating over loads in the order listed in the file.
825 for (BzlLoadValue.Key key : keys) {
brandjon24912642020-05-27 11:28:45 -0700826 CachedBzlLoadData cachedData;
adgarebc0f2c2019-08-15 15:36:56 -0700827 try {
brandjon1d3f3092020-06-08 10:58:25 -0700828 cachedData = computeInlineCachedData(key, strippedEnv, inliningState);
brandjon844fef52020-06-08 10:30:36 -0700829 } catch (BzlLoadFailedException e) {
830 e = BzlLoadFailedException.whileLoadingDep(filePathForErrors, e);
831 deferredException = MoreObjects.firstNonNull(deferredException, e);
832 continue;
833 } catch (InconsistentFilesystemException e) {
adgarebc0f2c2019-08-15 15:36:56 -0700834 deferredException = MoreObjects.firstNonNull(deferredException, e);
835 continue;
836 }
brandjon24912642020-05-27 11:28:45 -0700837 if (cachedData == null) {
nharmata636fb5e2020-07-24 15:30:08 -0700838 // A null value for `cachedData` can occur when it (or its transitive loads) has a Skyframe
839 // dep that is missing or in error. It can also occur if there's a transitive load on a bzl
840 // that was already seen by inliningState and which returned null (note: in this case, it's
841 // not necessarily true that there are missing Skyframe deps because this bzl could have
842 // already been visited unsuccessfully). In both these cases, we want to continue making our
843 // inline calls, so as to maximize the number of dependent (non-inlined) SkyFunctions that
844 // are requested and avoid a quadratic number of restarts.
adgarebc0f2c2019-08-15 15:36:56 -0700845 valuesMissing = true;
846 } else {
brandjon24912642020-05-27 11:28:45 -0700847 bzlLoads.add(cachedData.getValue());
adgarebc0f2c2019-08-15 15:36:56 -0700848 }
849 }
850 if (deferredException != null) {
brandjon771a0292020-05-26 12:04:16 -0700851 Throwables.throwIfInstanceOf(deferredException, BzlLoadFailedException.class);
adgarebc0f2c2019-08-15 15:36:56 -0700852 Throwables.throwIfInstanceOf(deferredException, InconsistentFilesystemException.class);
853 throw new IllegalStateException(
854 "caught a checked exception of unexpected type", deferredException);
855 }
brandjon771a0292020-05-26 12:04:16 -0700856 return valuesMissing ? null : bzlLoads;
adgarebc0f2c2019-08-15 15:36:56 -0700857 }
858
brandjona0c6f812020-06-06 14:13:19 -0700859 /**
brandjonbc9bd062020-08-12 09:41:12 -0700860 * Obtains the predeclared environment for a .bzl file, based on the type of .bzl and (if
brandjona0c6f812020-06-06 14:13:19 -0700861 * applicable) the injected builtins.
brandjonbc9bd062020-08-12 09:41:12 -0700862 *
863 * <p>Returns null if there was a missing Skyframe dep or unspecified exception.
brandjona0c6f812020-06-06 14:13:19 -0700864 */
brandjonbc9bd062020-08-12 09:41:12 -0700865 @Nullable
brandjon1af65cf2020-06-06 18:17:53 -0700866 private ImmutableMap<String, Object> getPredeclaredEnvironment(
brandjonbc9bd062020-08-12 09:41:12 -0700867 BzlLoadValue.Key key,
868 Environment env,
869 StarlarkSemantics starlarkSemantics,
870 InliningState inliningState)
871 throws BzlLoadFailedException, InterruptedException {
872 Label label = key.getLabel();
brandjona0c6f812020-06-06 14:13:19 -0700873 if (key instanceof BzlLoadValue.KeyForBuild) {
brandjonbc9bd062020-08-12 09:41:12 -0700874 StarlarkBuiltinsValue starlarkBuiltinsValue;
875 try {
876 // TODO(#11437): Remove ability to disable injection by setting flag to empty string.
877 if (starlarkSemantics.experimentalBuiltinsBzlPath().isEmpty()) {
878 starlarkBuiltinsValue = uninjectedStarlarkBuiltins;
879 } else {
880 if (inliningState == null) {
881 starlarkBuiltinsValue =
882 (StarlarkBuiltinsValue)
883 env.getValueOrThrow(StarlarkBuiltinsValue.key(), BuiltinsFailedException.class);
884 } else {
885 starlarkBuiltinsValue =
886 StarlarkBuiltinsFunction.computeInline(
brandjondb4d1ea2020-08-14 11:31:08 -0700887 StarlarkBuiltinsValue.key(),
888 env,
889 inliningState,
890 packageFactory,
891 /*bzlLoadFunction=*/ this);
brandjonbc9bd062020-08-12 09:41:12 -0700892 }
893 }
894 } catch (BuiltinsFailedException e) {
895 throw BzlLoadFailedException.builtinsFailed(label, e);
896 }
897 if (starlarkBuiltinsValue == null) {
898 return null;
899 }
brandjon1af65cf2020-06-06 18:17:53 -0700900 return starlarkBuiltinsValue.predeclaredForBuildBzl;
brandjona0c6f812020-06-06 14:13:19 -0700901 } else if (key instanceof BzlLoadValue.KeyForWorkspace) {
902 return predeclaredForWorkspaceBzl;
903 } else if (key instanceof BzlLoadValue.KeyForBuiltins) {
904 return predeclaredForBuiltinsBzl;
905 } else {
906 throw new AssertionError("Unknown key type: " + key.getClass());
907 }
908 }
909
brandjon771a0292020-05-26 12:04:16 -0700910 /** Executes the .bzl file defining the module to be loaded. */
brandjona0c6f812020-06-06 14:13:19 -0700911 private void executeBzlFile(
Googlerd54a4992019-10-18 12:35:06 -0700912 StarlarkFile file,
brandjon7154be02020-05-22 09:47:32 -0700913 Label label,
brandjona0c6f812020-06-06 14:13:19 -0700914 Module module,
adonovan1d93d262020-05-07 14:50:42 -0700915 Map<String, Module> loadedModules,
laurentlb6659b4c2019-02-18 07:23:36 -0800916 StarlarkSemantics starlarkSemantics,
brandjona0c6f812020-06-06 14:13:19 -0700917 ExtendedEventHandler skyframeEventHandler,
dannarkd102a392019-01-09 13:12:01 -0800918 ImmutableMap<RepositoryName, RepositoryName> repositoryMapping)
brandjon771a0292020-05-26 12:04:16 -0700919 throws BzlLoadFailedException, InterruptedException {
brandjon771a0292020-05-26 12:04:16 -0700920 try (Mutability mu = Mutability.create("loading", label)) {
adonovan09d13702020-05-19 08:26:55 -0700921 StarlarkThread thread = new StarlarkThread(mu, starlarkSemantics);
adonovandfcda9e2020-05-08 13:10:45 -0700922 thread.setLoader(loadedModules::get);
brandjona0c6f812020-06-06 14:13:19 -0700923 StoredEventHandler starlarkEventHandler = new StoredEventHandler();
924 thread.setPrintHandler(Event.makeDebugPrintHandler(starlarkEventHandler));
brandjondb4d1ea2020-08-14 11:31:08 -0700925 packageFactory
926 .getRuleClassProvider()
927 .setStarlarkThreadContext(thread, label, repositoryMapping);
brandjona0c6f812020-06-06 14:13:19 -0700928 execAndExport(file, label, starlarkEventHandler, module, thread);
Googler2234d382015-03-24 17:18:02 +0000929
brandjona0c6f812020-06-06 14:13:19 -0700930 Event.replayEventsOn(skyframeEventHandler, starlarkEventHandler.getEvents());
931 for (Postable post : starlarkEventHandler.getPosts()) {
932 skyframeEventHandler.post(post);
Klaus Aehligda0a7012017-06-14 13:40:23 +0200933 }
brandjona0c6f812020-06-06 14:13:19 -0700934 if (starlarkEventHandler.hasErrors()) {
brandjon771a0292020-05-26 12:04:16 -0700935 throw BzlLoadFailedException.errors(label.toPathFragment());
Francois-Rene Rideau89312fb2015-09-10 18:53:03 +0000936 }
Googler2234d382015-03-24 17:18:02 +0000937 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100938 }
939
Googlerf0890f02019-10-01 07:28:48 -0700940 // Precondition: file is validated and error-free.
adonovan1d93d262020-05-07 14:50:42 -0700941 // Precondition: thread has a valid transitiveDigest.
brandjona0c6f812020-06-06 14:13:19 -0700942 // TODO(adonovan): executeBzlFile would make a better public API than this function.
Googlera3421e22019-09-26 06:48:32 -0700943 public static void execAndExport(
brandjon7154be02020-05-22 09:47:32 -0700944 StarlarkFile file, Label label, EventHandler handler, Module module, StarlarkThread thread)
Googlera3421e22019-09-26 06:48:32 -0700945 throws InterruptedException {
Dmitry Lomov950310f2017-03-01 17:45:12 +0000946
Googlerba868542019-10-09 07:26:27 -0700947 // Intercept execution after every assignment at top level
948 // and "export" any newly assigned exportable globals.
949 // TODO(adonovan): change the semantics; see b/65374671.
950 thread.setPostAssignHook(
951 (name, value) -> {
gregced281df72020-05-11 12:27:06 -0700952 if (value instanceof StarlarkExportable) {
953 StarlarkExportable exp = (StarlarkExportable) value;
Googlerba868542019-10-09 07:26:27 -0700954 if (!exp.isExported()) {
955 try {
brandjon7154be02020-05-22 09:47:32 -0700956 exp.export(label, name);
Googlerba868542019-10-09 07:26:27 -0700957 } catch (EvalException ex) {
adonovan3391e172020-08-05 14:21:14 -0700958 handler.handle(Event.error(null, ex.getMessageWithStack()));
Googlerba868542019-10-09 07:26:27 -0700959 }
960 }
Dmitry Lomov950310f2017-03-01 17:45:12 +0000961 }
Googlerba868542019-10-09 07:26:27 -0700962 });
963
964 try {
adonovan09d13702020-05-19 08:26:55 -0700965 EvalUtils.exec(file, module, thread);
Googlerba868542019-10-09 07:26:27 -0700966 } catch (EvalException ex) {
adonovan3391e172020-08-05 14:21:14 -0700967 handler.handle(Event.error(null, ex.getMessageWithStack()));
Dmitry Lomov950310f2017-03-01 17:45:12 +0000968 }
969 }
970
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100971 @Override
972 public String extractTag(SkyKey skyKey) {
973 return null;
974 }
975
brandjon771a0292020-05-26 12:04:16 -0700976 static final class BzlLoadFailedException extends Exception implements SaneAnalysisException {
nharmatadc1d9dc2020-04-18 16:53:28 -0700977 private final Transience transience;
978
brandjon771a0292020-05-26 12:04:16 -0700979 private BzlLoadFailedException(String errorMessage) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100980 super(errorMessage);
nharmatadc1d9dc2020-04-18 16:53:28 -0700981 this.transience = Transience.PERSISTENT;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100982 }
983
brandjon771a0292020-05-26 12:04:16 -0700984 private BzlLoadFailedException(String errorMessage, Exception cause, Transience transience) {
nharmatabea67e92017-06-16 00:26:27 +0200985 super(errorMessage, cause);
nharmatadc1d9dc2020-04-18 16:53:28 -0700986 this.transience = transience;
nharmatabea67e92017-06-16 00:26:27 +0200987 }
988
brandjon844fef52020-06-08 10:30:36 -0700989 Transience getTransience() {
990 return transience;
991 }
992
993 // TODO(bazel-team): This exception should hold a Location of the requesting file's load
994 // statement, and code that catches it should use the location in the Event they create.
995 static BzlLoadFailedException whileLoadingDep(
996 String requestingFile, BzlLoadFailedException cause) {
997 // Don't chain exception cause, just incorporate the message with a prefix.
998 return new BzlLoadFailedException("in " + requestingFile + ": " + cause.getMessage());
999 }
1000
brandjon771a0292020-05-26 12:04:16 -07001001 static BzlLoadFailedException errors(PathFragment file) {
1002 return new BzlLoadFailedException(String.format("Extension file '%s' has errors", file));
Googler2234d382015-03-24 17:18:02 +00001003 }
1004
brandjon771a0292020-05-26 12:04:16 -07001005 static BzlLoadFailedException errorReadingFile(
gregce5c8a5f52020-05-13 10:35:36 -07001006 PathFragment file, ErrorReadingStarlarkExtensionException cause) {
brandjon771a0292020-05-26 12:04:16 -07001007 return new BzlLoadFailedException(
nharmatabea67e92017-06-16 00:26:27 +02001008 String.format(
gregceebd616f2020-04-10 16:47:41 -07001009 "Encountered error while reading extension file '%s': %s", file, cause.getMessage()),
nharmatadc1d9dc2020-04-18 16:53:28 -07001010 cause,
1011 cause.getTransience());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +01001012 }
1013
brandjon771a0292020-05-26 12:04:16 -07001014 static BzlLoadFailedException noBuildFile(Label file, @Nullable String reason) {
Googler06eb1bb2019-02-26 15:33:15 -08001015 if (reason != null) {
brandjon771a0292020-05-26 12:04:16 -07001016 return new BzlLoadFailedException(
Googler06eb1bb2019-02-26 15:33:15 -08001017 String.format("Unable to find package for %s: %s.", file, reason));
1018 }
brandjon771a0292020-05-26 12:04:16 -07001019 return new BzlLoadFailedException(
gregceebd616f2020-04-10 16:47:41 -07001020 String.format(
1021 "Every .bzl file must have a corresponding package, but '%s' does not have one."
1022 + " Please create a BUILD file in the same or any parent directory. Note that"
1023 + " this BUILD file does not need to do anything except exist.",
1024 file));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +01001025 }
1026
brandjon771a0292020-05-26 12:04:16 -07001027 static BzlLoadFailedException labelCrossesPackageBoundary(
brandjon7154be02020-05-22 09:47:32 -07001028 Label label, ContainingPackageLookupValue containingPackageLookupValue) {
brandjon771a0292020-05-26 12:04:16 -07001029 return new BzlLoadFailedException(
nharmatad86b5092018-10-16 15:50:21 -07001030 ContainingPackageLookupValue.getErrorMessageForLabelCrossingPackageBoundary(
1031 // We don't actually know the proper Root to pass in here (since we don't e.g. know
brandjon7154be02020-05-22 09:47:32 -07001032 // the root of the bzl/BUILD file that is trying to load 'label'). Therefore we just
1033 // pass in the Root of the containing package in order to still get a useful error
1034 // message for the user.
nharmatad86b5092018-10-16 15:50:21 -07001035 containingPackageLookupValue.getContainingPackageRoot(),
brandjon7154be02020-05-22 09:47:32 -07001036 label,
nharmatad86b5092018-10-16 15:50:21 -07001037 containingPackageLookupValue));
1038 }
1039
gregce0503fee2020-06-11 09:22:27 -07001040 static BzlLoadFailedException starlarkErrors(PathFragment file) {
brandjon771a0292020-05-26 12:04:16 -07001041 return new BzlLoadFailedException(String.format("Extension '%s' has errors", file));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +01001042 }
brandjon1af65cf2020-06-06 18:17:53 -07001043
brandjon844fef52020-06-08 10:30:36 -07001044 static BzlLoadFailedException builtinsFailed(Label file, BuiltinsFailedException cause) {
1045 return new BzlLoadFailedException(
1046 String.format(
1047 "Internal error while loading Starlark builtins for %s: %s",
1048 file, cause.getMessage()),
1049 cause,
1050 cause.getTransience());
1051 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +01001052 }
1053
brandjon8d669292020-05-28 15:27:38 -07001054 /**
1055 * A manager abstracting over the method for obtaining {@code ASTFileLookupValue}s. See comment in
1056 * {@link #create}.
1057 */
1058 private interface ASTManager {
nharmatadc1d9dc2020-04-18 16:53:28 -07001059 @Nullable
brandjonc782bbc2020-08-05 13:47:13 -07001060 ASTFileLookupValue getASTFileLookupValue(ASTFileLookupValue.Key key, Environment env)
nharmatadc1d9dc2020-04-18 16:53:28 -07001061 throws InconsistentFilesystemException, InterruptedException,
gregce5c8a5f52020-05-13 10:35:36 -07001062 ErrorReadingStarlarkExtensionException;
nharmatadc1d9dc2020-04-18 16:53:28 -07001063
brandjonc782bbc2020-08-05 13:47:13 -07001064 void doneWithASTFileLookupValue(ASTFileLookupValue.Key key);
nharmatadc1d9dc2020-04-18 16:53:28 -07001065 }
1066
brandjon8d669292020-05-28 15:27:38 -07001067 /** A manager that obtains ASTs from Skyframe calls. */
1068 private static class RegularSkyframeASTManager implements ASTManager {
1069 private static final RegularSkyframeASTManager INSTANCE = new RegularSkyframeASTManager();
nharmatadc1d9dc2020-04-18 16:53:28 -07001070
1071 @Nullable
1072 @Override
brandjonc782bbc2020-08-05 13:47:13 -07001073 public ASTFileLookupValue getASTFileLookupValue(ASTFileLookupValue.Key key, Environment env)
nharmatadc1d9dc2020-04-18 16:53:28 -07001074 throws InconsistentFilesystemException, InterruptedException,
gregce5c8a5f52020-05-13 10:35:36 -07001075 ErrorReadingStarlarkExtensionException {
nharmatadc1d9dc2020-04-18 16:53:28 -07001076 return (ASTFileLookupValue)
1077 env.getValueOrThrow(
brandjonc782bbc2020-08-05 13:47:13 -07001078 key,
gregce5c8a5f52020-05-13 10:35:36 -07001079 ErrorReadingStarlarkExtensionException.class,
nharmatadc1d9dc2020-04-18 16:53:28 -07001080 InconsistentFilesystemException.class);
1081 }
1082
1083 @Override
brandjonc782bbc2020-08-05 13:47:13 -07001084 public void doneWithASTFileLookupValue(ASTFileLookupValue.Key key) {}
nharmatadc1d9dc2020-04-18 16:53:28 -07001085 }
1086
brandjon8d669292020-05-28 15:27:38 -07001087 /**
1088 * A manager that obtains ASTs by inlining {@link ASTFileLookupFunction} (not to be confused with
1089 * inlining of {@code BzlLoadFunction}). Values are cached within the manager and released
1090 * explicitly by calling {@link #doneWithASTFileLookupValue}.
1091 */
1092 private static class InliningAndCachingASTManager implements ASTManager {
brandjondb4d1ea2020-08-14 11:31:08 -07001093 private final PackageFactory packageFactory;
nharmata34879302020-04-22 11:30:15 -07001094 private final DigestHashFunction digestHashFunction;
brandjonf4f91932020-05-11 10:20:48 -07001095 // We keep a cache of ASTFileLookupValues that have been computed but whose corresponding
brandjon771a0292020-05-26 12:04:16 -07001096 // BzlLoadValue has not yet completed. This avoids repeating the ASTFileLookupValue work in case
1097 // of Skyframe restarts. (If we weren't inlining, Skyframe would cache this for us.)
brandjonc782bbc2020-08-05 13:47:13 -07001098 private final Cache<ASTFileLookupValue.Key, ASTFileLookupValue> astFileLookupValueCache;
nharmatadc1d9dc2020-04-18 16:53:28 -07001099
brandjon8d669292020-05-28 15:27:38 -07001100 private InliningAndCachingASTManager(
brandjondb4d1ea2020-08-14 11:31:08 -07001101 PackageFactory packageFactory,
nharmata34879302020-04-22 11:30:15 -07001102 DigestHashFunction digestHashFunction,
brandjonc782bbc2020-08-05 13:47:13 -07001103 Cache<ASTFileLookupValue.Key, ASTFileLookupValue> astFileLookupValueCache) {
brandjondb4d1ea2020-08-14 11:31:08 -07001104 this.packageFactory = packageFactory;
nharmata34879302020-04-22 11:30:15 -07001105 this.digestHashFunction = digestHashFunction;
nharmatadc1d9dc2020-04-18 16:53:28 -07001106 this.astFileLookupValueCache = astFileLookupValueCache;
1107 }
1108
1109 @Nullable
1110 @Override
brandjonc782bbc2020-08-05 13:47:13 -07001111 public ASTFileLookupValue getASTFileLookupValue(ASTFileLookupValue.Key key, Environment env)
nharmatadc1d9dc2020-04-18 16:53:28 -07001112 throws InconsistentFilesystemException, InterruptedException,
gregce5c8a5f52020-05-13 10:35:36 -07001113 ErrorReadingStarlarkExtensionException {
brandjonc782bbc2020-08-05 13:47:13 -07001114 ASTFileLookupValue value = astFileLookupValueCache.getIfPresent(key);
nharmatadc1d9dc2020-04-18 16:53:28 -07001115 if (value == null) {
brandjondb4d1ea2020-08-14 11:31:08 -07001116 value = ASTFileLookupFunction.computeInline(key, env, packageFactory, digestHashFunction);
nharmatadc1d9dc2020-04-18 16:53:28 -07001117 if (value != null) {
brandjonc782bbc2020-08-05 13:47:13 -07001118 astFileLookupValueCache.put(key, value);
nharmatadc1d9dc2020-04-18 16:53:28 -07001119 }
1120 }
1121 return value;
1122 }
1123
1124 @Override
brandjonc782bbc2020-08-05 13:47:13 -07001125 public void doneWithASTFileLookupValue(ASTFileLookupValue.Key key) {
1126 astFileLookupValueCache.invalidate(key);
nharmatadc1d9dc2020-04-18 16:53:28 -07001127 }
1128 }
1129
brandjon8d669292020-05-28 15:27:38 -07001130 /**
1131 * Per-instance manager for {@link CachedBzlLoadData}, used when {@code BzlLoadFunction} calls are
1132 * inlined.
1133 */
1134 private static class CachedBzlLoadDataManager {
1135 private final int cacheSize;
1136 private Cache<BzlLoadValue.Key, CachedBzlLoadData> cache;
1137 private CachedBzlLoadDataBuilderFactory cachedDataBuilderFactory =
brandjon24912642020-05-27 11:28:45 -07001138 new CachedBzlLoadDataBuilderFactory();
nharmatadc1d9dc2020-04-18 16:53:28 -07001139
brandjon8d669292020-05-28 15:27:38 -07001140 private CachedBzlLoadDataManager(int cacheSize) {
1141 this.cacheSize = cacheSize;
nharmatadc1d9dc2020-04-18 16:53:28 -07001142 }
1143
brandjon24912642020-05-27 11:28:45 -07001144 private CachedBzlLoadData.Builder cachedDataBuilder() {
brandjon8d669292020-05-28 15:27:38 -07001145 return cachedDataBuilderFactory.newCachedBzlLoadDataBuilder();
brandjon24912642020-05-27 11:28:45 -07001146 }
1147
nharmatadc1d9dc2020-04-18 16:53:28 -07001148 private void reset() {
brandjon8d669292020-05-28 15:27:38 -07001149 if (cache != null) {
1150 logger.atInfo().log("Starlark inlining cache stats from earlier build: " + cache.stats());
nharmatadc1d9dc2020-04-18 16:53:28 -07001151 }
brandjon8d669292020-05-28 15:27:38 -07001152 cachedDataBuilderFactory = new CachedBzlLoadDataBuilderFactory();
nharmatadc1d9dc2020-04-18 16:53:28 -07001153 Preconditions.checkState(
brandjon8d669292020-05-28 15:27:38 -07001154 cacheSize >= 0, "Expected positive Starlark cache size if caching. %s", cacheSize);
1155 cache =
nharmatadc1d9dc2020-04-18 16:53:28 -07001156 CacheBuilder.newBuilder()
1157 .concurrencyLevel(BlazeInterners.concurrencyLevel())
brandjon8d669292020-05-28 15:27:38 -07001158 .maximumSize(cacheSize)
nharmatadc1d9dc2020-04-18 16:53:28 -07001159 .recordStats()
1160 .build();
1161 }
1162 }
1163
brandjon771a0292020-05-26 12:04:16 -07001164 private static final class BzlLoadFunctionException extends SkyFunctionException {
1165 private BzlLoadFunctionException(BzlLoadFailedException cause) {
nharmatadc1d9dc2020-04-18 16:53:28 -07001166 super(cause, cause.transience);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +01001167 }
1168
brandjon771a0292020-05-26 12:04:16 -07001169 private BzlLoadFunctionException(InconsistentFilesystemException e, Transience transience) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +01001170 super(e, transience);
1171 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +01001172 }
1173}