Damien Martin-Guillerez | f88f4d8 | 2015-09-25 13:56:55 +0000 | [diff] [blame] | 1 | // Copyright 2014 The Bazel Authors. All rights reserved. |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | package com.google.devtools.build.lib.skyframe; |
| 15 | |
adgar | ebc0f2c | 2019-08-15 15:36:56 -0700 | [diff] [blame] | 16 | import com.google.common.base.MoreObjects; |
tomlu | a155b53 | 2017-11-08 20:12:47 +0100 | [diff] [blame] | 17 | import com.google.common.base.Preconditions; |
Janak Ramakrishnan | df0531f | 2015-09-23 17:30:04 +0000 | [diff] [blame] | 18 | import com.google.common.base.Predicates; |
adgar | ebc0f2c | 2019-08-15 15:36:56 -0700 | [diff] [blame] | 19 | import com.google.common.base.Throwables; |
shreyax | 755a0a1 | 2018-03-19 20:47:37 -0700 | [diff] [blame] | 20 | import com.google.common.cache.Cache; |
| 21 | import com.google.common.cache.CacheBuilder; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 22 | import com.google.common.collect.ImmutableList; |
John Field | a97e17f | 2015-11-13 02:19:52 +0000 | [diff] [blame] | 23 | import com.google.common.collect.ImmutableMap; |
John Field | a97e17f | 2015-11-13 02:19:52 +0000 | [diff] [blame] | 24 | import com.google.common.collect.Lists; |
Janak Ramakrishnan | dfd3497 | 2015-09-22 02:53:05 +0000 | [diff] [blame] | 25 | import com.google.common.collect.Maps; |
janakr | c3bcb98 | 2020-04-14 06:50:08 -0700 | [diff] [blame] | 26 | import com.google.common.flogger.GoogleLogger; |
shahan | 602cc85 | 2018-06-06 20:09:57 -0700 | [diff] [blame] | 27 | import com.google.devtools.build.lib.actions.InconsistentFilesystemException; |
Lukacs Berki | 6e91eb9 | 2015-09-21 09:12:37 +0000 | [diff] [blame] | 28 | import com.google.devtools.build.lib.cmdline.Label; |
Googler | b2e1fe3 | 2019-09-20 14:00:39 -0700 | [diff] [blame] | 29 | import com.google.devtools.build.lib.cmdline.LabelConstants; |
| 30 | import com.google.devtools.build.lib.cmdline.LabelSyntaxException; |
Kristina Chodorow | 73fa203 | 2015-08-28 17:57:46 +0000 | [diff] [blame] | 31 | import com.google.devtools.build.lib.cmdline.PackageIdentifier; |
dannark | 4e42c32 | 2018-11-08 19:32:04 -0800 | [diff] [blame] | 32 | import com.google.devtools.build.lib.cmdline.RepositoryName; |
shreyax | 755a0a1 | 2018-03-19 20:47:37 -0700 | [diff] [blame] | 33 | import com.google.devtools.build.lib.concurrent.BlazeInterners; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 34 | import com.google.devtools.build.lib.events.Event; |
Dmitry Lomov | 950310f | 2017-03-01 17:45:12 +0000 | [diff] [blame] | 35 | import com.google.devtools.build.lib.events.EventHandler; |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 36 | import com.google.devtools.build.lib.events.ExtendedEventHandler; |
Klaus Aehlig | da0a701 | 2017-06-14 13:40:23 +0200 | [diff] [blame] | 37 | import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 38 | import com.google.devtools.build.lib.events.StoredEventHandler; |
ajurkowski | 9f2cab5 | 2020-05-12 12:00:24 -0700 | [diff] [blame] | 39 | import com.google.devtools.build.lib.packages.BazelModuleContext; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 40 | import com.google.devtools.build.lib.packages.BuildFileNotFoundException; |
| 41 | import com.google.devtools.build.lib.packages.PackageFactory; |
gregce | d281df7 | 2020-05-11 12:27:06 -0700 | [diff] [blame] | 42 | import com.google.devtools.build.lib.packages.StarlarkExportable; |
janakr | 15e15c2 | 2019-01-30 11:24:49 -0800 | [diff] [blame] | 43 | import com.google.devtools.build.lib.packages.WorkspaceFileValue; |
brandjon | 844fef5 | 2020-06-08 10:30:36 -0700 | [diff] [blame] | 44 | import com.google.devtools.build.lib.skyframe.StarlarkBuiltinsFunction.BuiltinsFailedException; |
Dmitry Lomov | 7b59945 | 2015-11-26 10:07:32 +0000 | [diff] [blame] | 45 | import com.google.devtools.build.lib.syntax.EvalException; |
Googler | f0890f0 | 2019-10-01 07:28:48 -0700 | [diff] [blame] | 46 | import com.google.devtools.build.lib.syntax.EvalUtils; |
John Field | a97e17f | 2015-11-13 02:19:52 +0000 | [diff] [blame] | 47 | import com.google.devtools.build.lib.syntax.LoadStatement; |
adonovan | 1d93d26 | 2020-05-07 14:50:42 -0700 | [diff] [blame] | 48 | import com.google.devtools.build.lib.syntax.Module; |
Francois-Rene Rideau | 89312fb | 2015-09-10 18:53:03 +0000 | [diff] [blame] | 49 | import com.google.devtools.build.lib.syntax.Mutability; |
Googler | 66d099e | 2019-09-26 08:07:06 -0700 | [diff] [blame] | 50 | import com.google.devtools.build.lib.syntax.StarlarkFile; |
laurentlb | 6659b4c | 2019-02-18 07:23:36 -0800 | [diff] [blame] | 51 | import com.google.devtools.build.lib.syntax.StarlarkSemantics; |
Googler | a3421e2 | 2019-09-26 06:48:32 -0700 | [diff] [blame] | 52 | import com.google.devtools.build.lib.syntax.StarlarkThread; |
Dmitry Lomov | 950310f | 2017-03-01 17:45:12 +0000 | [diff] [blame] | 53 | import com.google.devtools.build.lib.syntax.Statement; |
adonovan | 1d93d26 | 2020-05-07 14:50:42 -0700 | [diff] [blame] | 54 | import com.google.devtools.build.lib.util.Fingerprint; |
adonovan | 699a90a | 2020-05-20 19:48:53 -0700 | [diff] [blame] | 55 | import com.google.devtools.build.lib.util.Pair; |
nharmata | 3487930 | 2020-04-22 11:30:15 -0700 | [diff] [blame] | 56 | import com.google.devtools.build.lib.vfs.DigestHashFunction; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 57 | import com.google.devtools.build.lib.vfs.PathFragment; |
shreyax | 755a0a1 | 2018-03-19 20:47:37 -0700 | [diff] [blame] | 58 | import com.google.devtools.build.skyframe.RecordingSkyFunctionEnvironment; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 59 | import com.google.devtools.build.skyframe.SkyFunction; |
| 60 | import com.google.devtools.build.skyframe.SkyFunctionException; |
| 61 | import com.google.devtools.build.skyframe.SkyFunctionException.Transience; |
| 62 | import com.google.devtools.build.skyframe.SkyKey; |
| 63 | import com.google.devtools.build.skyframe.SkyValue; |
laurentlb | dd612a8 | 2018-10-16 19:42:48 -0700 | [diff] [blame] | 64 | import com.google.devtools.build.skyframe.ValueOrException; |
brandjon | 1d3f309 | 2020-06-08 10:58:25 -0700 | [diff] [blame] | 65 | import java.util.HashMap; |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 66 | import java.util.HashSet; |
shreyax | 39ffd97 | 2019-02-07 15:41:31 -0800 | [diff] [blame] | 67 | import java.util.LinkedHashSet; |
John Field | a97e17f | 2015-11-13 02:19:52 +0000 | [diff] [blame] | 68 | import java.util.List; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 69 | import java.util.Map; |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 70 | import java.util.function.Consumer; |
Janak Ramakrishnan | df0531f | 2015-09-23 17:30:04 +0000 | [diff] [blame] | 71 | import javax.annotation.Nullable; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 72 | |
| 73 | /** |
adonovan | 1d93d26 | 2020-05-07 14:50:42 -0700 | [diff] [blame] | 74 | * A Skyframe function to look up and load a single .bzl module. |
John Field | a97e17f | 2015-11-13 02:19:52 +0000 | [diff] [blame] | 75 | * |
adonovan | 1d93d26 | 2020-05-07 14:50:42 -0700 | [diff] [blame] | 76 | * <p>Given a {@link Label} referencing a .bzl file, attempts to locate the file and load it. The |
| 77 | * Label must be absolute, and must not reference the special {@code external} package. If loading |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 78 | * is successful, returns a {@link BzlLoadValue} that encapsulates the loaded {@link Module} and its |
adonovan | 39e42d1 | 2020-07-09 09:16:58 -0700 | [diff] [blame] | 79 | * transitive digest information. If loading is unsuccessful, throws a {@link |
| 80 | * BzlLoadFunctionException} that encapsulates the cause of the failure. |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 81 | * |
brandjon | b4ebbfc | 2020-08-13 15:50:23 -0700 | [diff] [blame] | 82 | * <p>This Skyframe function supports a special bzl "inlining" mode in which all (indirectly) |
| 83 | * recursive calls to {@code BzlLoadFunction} are made in the same thread rather than through |
| 84 | * Skyframe. This inlining mode's entry point is {@link #computeInline}; see that method for more |
| 85 | * details. Note that it may only be called on an instance of this Skyfunction created by {@link |
| 86 | * #createForInlining}. Bzl inlining is not to be confused with the separate inlining of {@code |
| 87 | * ASTFileLookupFunction} |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 88 | */ |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 89 | public class BzlLoadFunction implements SkyFunction { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 90 | |
brandjon | db4d1ea | 2020-08-14 11:31:08 -0700 | [diff] [blame^] | 91 | // Used for: 1) obtaining a RuleClassProvider to create the BazelStarlarkContext for Starlark |
| 92 | // evaluation; 2) providing predeclared environments to other Skyfunctions |
| 93 | // (StarlarkBuiltinsFunction, ASTFileLookupFunction) when they are inlined and called via a static |
| 94 | // computeInline() entry point. |
| 95 | private final PackageFactory packageFactory; |
brandjon | 2863252 | 2020-06-08 13:14:07 -0700 | [diff] [blame] | 96 | |
brandjon | 1af65cf | 2020-06-06 18:17:53 -0700 | [diff] [blame] | 97 | // Used for BUILD .bzls if injection is disabled. |
| 98 | // TODO(#11437): Remove once injection is on unconditionally. |
| 99 | private final StarlarkBuiltinsValue uninjectedStarlarkBuiltins; |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 100 | |
brandjon | c61bc04 | 2020-08-11 11:52:16 -0700 | [diff] [blame] | 101 | // Predeclareds for workspace .bzls and builtins .bzls are not subject to builtins injection, so |
| 102 | // these environments are stored globally. |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 103 | private final ImmutableMap<String, Object> predeclaredForWorkspaceBzl; |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 104 | private final ImmutableMap<String, Object> predeclaredForBuiltinsBzl; |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 105 | |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 106 | // Handles retrieving ASTFileLookupValues, either by calling Skyframe or by inlining |
| 107 | // ASTFileLookupFunction; the latter is not to be confused with inlining of BzlLoadFunction. See |
| 108 | // comment in create() for rationale. |
| 109 | private final ASTManager astManager; |
| 110 | |
| 111 | // Handles inlining of BzlLoadFunction calls. |
| 112 | @Nullable private final CachedBzlLoadDataManager cachedBzlLoadDataManager; |
shreyax | 755a0a1 | 2018-03-19 20:47:37 -0700 | [diff] [blame] | 113 | |
janakr | c3bcb98 | 2020-04-14 06:50:08 -0700 | [diff] [blame] | 114 | private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 115 | |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 116 | private BzlLoadFunction( |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 117 | PackageFactory packageFactory, |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 118 | ASTManager astManager, |
| 119 | @Nullable CachedBzlLoadDataManager cachedBzlLoadDataManager) { |
brandjon | 2863252 | 2020-06-08 13:14:07 -0700 | [diff] [blame] | 120 | this.packageFactory = packageFactory; |
brandjon | 1af65cf | 2020-06-06 18:17:53 -0700 | [diff] [blame] | 121 | this.uninjectedStarlarkBuiltins = |
brandjon | db4d1ea | 2020-08-14 11:31:08 -0700 | [diff] [blame^] | 122 | StarlarkBuiltinsFunction.createStarlarkBuiltinsValueWithoutInjection(packageFactory); |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 123 | this.predeclaredForWorkspaceBzl = |
brandjon | db4d1ea | 2020-08-14 11:31:08 -0700 | [diff] [blame^] | 124 | StarlarkBuiltinsFunction.createPredeclaredForWorkspaceBzl(packageFactory); |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 125 | this.predeclaredForBuiltinsBzl = |
brandjon | db4d1ea | 2020-08-14 11:31:08 -0700 | [diff] [blame^] | 126 | StarlarkBuiltinsFunction.createPredeclaredForBuiltinsBzl(packageFactory); |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 127 | this.astManager = astManager; |
| 128 | this.cachedBzlLoadDataManager = cachedBzlLoadDataManager; |
shreyax | 3126205 | 2019-08-13 13:40:06 -0700 | [diff] [blame] | 129 | } |
| 130 | |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 131 | public static BzlLoadFunction create( |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 132 | PackageFactory packageFactory, |
nharmata | 3487930 | 2020-04-22 11:30:15 -0700 | [diff] [blame] | 133 | DigestHashFunction digestHashFunction, |
brandjon | c782bbc | 2020-08-05 13:47:13 -0700 | [diff] [blame] | 134 | Cache<ASTFileLookupValue.Key, ASTFileLookupValue> astFileLookupValueCache) { |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 135 | return new BzlLoadFunction( |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 136 | packageFactory, |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 137 | // When we are not inlining BzlLoadValue nodes, there is no need to have separate |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 138 | // ASTFileLookupValue nodes for bzl files. Instead we inline ASTFileLookupFunction for a |
| 139 | // strict memory win, at a small code complexity cost. |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 140 | // |
| 141 | // Detailed explanation: |
| 142 | // (1) The ASTFileLookupValue node for a bzl file is used only for the computation of |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 143 | // that file's BzlLoadValue node. So there's no concern about duplicate work that would |
| 144 | // otherwise get deduped by Skyframe. |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 145 | // (2) ASTFileLookupValue doesn't have an interesting equality relation, so we have no |
| 146 | // hope of getting any interesting change-pruning of ASTFileLookupValue nodes. If we |
| 147 | // had an interesting equality relation that was e.g. able to ignore benign |
| 148 | // whitespace, then there would be a hypothetical benefit to having separate |
| 149 | // ASTFileLookupValue nodes (e.g. on incremental builds we'd be able to not re-execute |
| 150 | // top-level code in bzl files if the file were reparsed to an equivalent AST). |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 151 | // (3) A ASTFileLookupValue node lets us avoid redoing work on a BzlLoadFunction Skyframe |
| 152 | // restart, but we can also achieve that result ourselves with a cache that persists between |
| 153 | // Skyframe restarts. |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 154 | // |
| 155 | // Therefore, ASTFileLookupValue nodes are wasteful from two perspectives: |
| 156 | // (a) ASTFileLookupValue contains a StarlarkFile, and that business object is really |
| 157 | // just a temporary thing for bzl execution. Retaining it forever is pure waste. |
| 158 | // (b) The memory overhead of the extra Skyframe node and edge per bzl file is pure |
| 159 | // waste. |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 160 | new InliningAndCachingASTManager( |
brandjon | db4d1ea | 2020-08-14 11:31:08 -0700 | [diff] [blame^] | 161 | packageFactory, digestHashFunction, astFileLookupValueCache), |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 162 | /*cachedBzlLoadDataManager=*/ null); |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 163 | } |
| 164 | |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 165 | public static BzlLoadFunction createForInlining( |
brandjon | db4d1ea | 2020-08-14 11:31:08 -0700 | [diff] [blame^] | 166 | PackageFactory packageFactory, int bzlLoadValueCacheSize) { |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 167 | return new BzlLoadFunction( |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 168 | packageFactory, |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 169 | // When we are inlining BzlLoadValue nodes, then we want to have explicit ASTFileLookupValue |
| 170 | // nodes, since now (1) in the comment above doesn't hold. This way we read and parse each |
| 171 | // needed bzl file at most once total globally, rather than once per need (in the worst-case |
| 172 | // of a BzlLoadValue inlining cache miss). This is important in the situation where a bzl |
| 173 | // file is loaded by a lot of other bzl files or BUILD files. |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 174 | RegularSkyframeASTManager.INSTANCE, |
| 175 | new CachedBzlLoadDataManager(bzlLoadValueCacheSize)); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 176 | } |
| 177 | |
| 178 | @Override |
shreyax | 755a0a1 | 2018-03-19 20:47:37 -0700 | [diff] [blame] | 179 | @Nullable |
| 180 | public SkyValue compute(SkyKey skyKey, Environment env) |
| 181 | throws SkyFunctionException, InterruptedException { |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 182 | BzlLoadValue.Key key = (BzlLoadValue.Key) skyKey.argument(); |
John Field | a97e17f | 2015-11-13 02:19:52 +0000 | [diff] [blame] | 183 | try { |
brandjon | 7154be0 | 2020-05-22 09:47:32 -0700 | [diff] [blame] | 184 | return computeInternal(key, env, /*inliningState=*/ null); |
John Field | a97e17f | 2015-11-13 02:19:52 +0000 | [diff] [blame] | 185 | } catch (InconsistentFilesystemException e) { |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 186 | throw new BzlLoadFunctionException(e, Transience.PERSISTENT); |
| 187 | } catch (BzlLoadFailedException e) { |
| 188 | throw new BzlLoadFunctionException(e); |
John Field | a97e17f | 2015-11-13 02:19:52 +0000 | [diff] [blame] | 189 | } |
Janak Ramakrishnan | df0531f | 2015-09-23 17:30:04 +0000 | [diff] [blame] | 190 | } |
| 191 | |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 192 | /** |
| 193 | * Entry point for computing "inline", without any direct or indirect Skyframe calls back into |
| 194 | * {@link BzlLoadFunction}. (Other Skyframe calls are permitted.) |
| 195 | * |
nharmata | 60d0460 | 2020-08-11 12:28:57 -0700 | [diff] [blame] | 196 | * <p><b>USAGE NOTES:</b> |
| 197 | * |
| 198 | * <ul> |
| 199 | * <li>This method is intended to be called from {@link PackageFunction} and {@link |
| 200 | * StarlarkBuiltinsFunction} and probably shouldn't be used anywhere else. If you think you |
| 201 | * need inline Starlark computation, consult with the Core subteam and check out |
| 202 | * cl/305127325 for an example of correcting a misuse. |
| 203 | * <li>If this method is used with --keep_going and if Skyframe evaluation will never be |
brandjon | b4ebbfc | 2020-08-13 15:50:23 -0700 | [diff] [blame] | 204 | * interrupted, then this function ensures that the evaluation graph and any error reported |
| 205 | * are deterministic. |
nharmata | 60d0460 | 2020-08-11 12:28:57 -0700 | [diff] [blame] | 206 | * </ul> |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 207 | * |
| 208 | * <p>Under bzl inlining, there is some calling context that wants to obtain a set of {@link |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 209 | * BzlLoadValue}s without Skyframe evaluation. For example, a calling context can be a BUILD file |
| 210 | * trying to resolve its top-level {@code load()} statements. Although this work proceeds in a |
| 211 | * single thread, multiple calling contexts may evaluate .bzls in parallel. To avoid redundant |
| 212 | * work, they share a single (global to this Skyfunction instance) cache in lieu of the regular |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 213 | * Skyframe cache. Unlike the regular Skyframe cache, this cache stores only successes. |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 214 | * |
| 215 | * <p>If two calling contexts race to compute the same .bzl, each one will see a different copy of |
| 216 | * it, and only one will end up in the shared cache. This presents a hazard: Suppose A and B both |
| 217 | * need foo.bzl, and A needs it twice due to a diamond dependency. If A and B race to compute |
| 218 | * foo.bzl, but B's computation populates the cache, then when A comes back to resolve it the |
| 219 | * second time it will observe a different {@code BzlLoadValue}. This leads to incorrect Starlark |
| 220 | * evaluation since Starlark values may rely on Java object identity (see b/138598337). Even if we |
| 221 | * weren't concerned about racing, A may also reevaluate previously computed items due to cache |
| 222 | * evictions. |
| 223 | * |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 224 | * <p>To solve this, we keep a second cache, {@link InliningState#successfulLoads}, that is local |
| 225 | * to the current calling context, and which never evicts entries. Like the global cache discussed |
| 226 | * above, this cache stores only successes. This cache is always checked in preference to the |
| 227 | * shared one; it may deviate from the shared one in some of its entries, but the calling context |
| 228 | * won't know the difference. (Since bzl inlining is only used for the loading phase, we don't |
| 229 | * need to worry about Starlark values from different packages interacting.) The cache is stored |
| 230 | * as part of the {@code inliningState} passed in by the caller; the caller can obtain this object |
| 231 | * using {@link InliningState#create}. |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 232 | * |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 233 | * <p>As an aside, note that we can't avoid having {@link InliningState#successfulLoads} by simply |
| 234 | * naively blocking evaluation of .bzls on retrievals from the shared cache. This is because two |
| 235 | * contexts could deadlock while trying to evaluate an illegal {@code load()} cycle from opposite |
| 236 | * ends. It would be possible to construct a waits-for graph and perform cycle detection, or to |
| 237 | * monitor slow threads and do detection lazily, but these do not address the cache eviction |
| 238 | * issue. Alternatively, we could make Starlark tolerant of reloading, but that would be |
| 239 | * tantamount to implementing full Starlark serialization. |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 240 | * |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 241 | * <p>Since our local {@link InliningState#successfulLoads} stores only successes, a separate |
| 242 | * concern is that we don't want to unsuccessfully visit the same .bzl more than once in the same |
| 243 | * context. (A visitation is unsuccessful if it fails due to an error or if it cannot complete |
| 244 | * because of a missing Skyframe dep.) To address this concern we maintain a separate {@link |
| 245 | * InliningState#unsuccessfulLoads} set, and use this set to return null instead of duplicating an |
| 246 | * unsuccessful visitation. |
| 247 | * |
| 248 | * @return the requested {@code BzlLoadValue}, or null if there was a missing Skyframe dep, an |
| 249 | * unspecified exception in a Skyframe dep request, or if this was a duplicate unsuccessful |
| 250 | * visitation |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 251 | */ |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 252 | // TODO(brandjon): Pick one of the nouns "load" and "bzl" and use that term consistently. |
shreyax | 755a0a1 | 2018-03-19 20:47:37 -0700 | [diff] [blame] | 253 | @Nullable |
brandjon | 1d3f309 | 2020-06-08 10:58:25 -0700 | [diff] [blame] | 254 | BzlLoadValue computeInline(BzlLoadValue.Key key, Environment env, InliningState inliningState) |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 255 | throws InconsistentFilesystemException, BzlLoadFailedException, InterruptedException { |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 256 | // Note to refactorors: No Skyframe calls may be made before the RecordingSkyFunctionEnvironment |
| 257 | // is set up below in computeInlineForCacheMiss. |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 258 | Preconditions.checkNotNull(cachedBzlLoadDataManager); |
brandjon | 1d3f309 | 2020-06-08 10:58:25 -0700 | [diff] [blame] | 259 | CachedBzlLoadData cachedData = computeInlineCachedData(key, env, inliningState); |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 260 | return cachedData != null ? cachedData.getValue() : null; |
Janak Ramakrishnan | df0531f | 2015-09-23 17:30:04 +0000 | [diff] [blame] | 261 | } |
| 262 | |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 263 | /** |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 264 | * Retrieves or creates the requested {@link CachedBzlLoadData} object for the given bzl, entering |
| 265 | * it into the local and shared caches. This is the entry point for recursive calls to the inline |
| 266 | * code path. |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 267 | * |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 268 | * <p>Skyframe calls made underneath this function will be logged in the resulting {@code |
brandjon | 1d3f309 | 2020-06-08 10:58:25 -0700 | [diff] [blame] | 269 | * CachedBzlLoadData) (or its transitive dependencies). The given Skyframe environment must not |
| 270 | * be a {@link RecordingSkyFunctionEnvironment}, since that would imply that calls are being |
| 271 | * logged in both the returned value and the parent value. |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 272 | * |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 273 | * @return null if there was a missing Skyframe dep, an unspecified exception in a Skyframe dep |
| 274 | * request, or if this was a duplicate unsuccessful visitation |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 275 | */ |
shreyax | 755a0a1 | 2018-03-19 20:47:37 -0700 | [diff] [blame] | 276 | @Nullable |
brandjon | 1d3f309 | 2020-06-08 10:58:25 -0700 | [diff] [blame] | 277 | private CachedBzlLoadData computeInlineCachedData( |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 278 | BzlLoadValue.Key key, Environment env, InliningState inliningState) |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 279 | throws InconsistentFilesystemException, BzlLoadFailedException, InterruptedException { |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 280 | // Note to refactorors: No Skyframe calls may be made before the RecordingSkyFunctionEnvironment |
| 281 | // is set up below in computeInlineForCacheMiss. |
| 282 | |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 283 | // Try the caches of successful loads. We must try the thread-local cache before the shared, for |
| 284 | // consistency purposes (see the javadoc of #computeInline). |
| 285 | CachedBzlLoadData cachedData = inliningState.successfulLoads.get(key); |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 286 | if (cachedData == null) { |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 287 | cachedData = cachedBzlLoadDataManager.cache.getIfPresent(key); |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 288 | if (cachedData != null) { |
| 289 | // Found a cache hit from another thread's computation; register the recorded deps as if our |
brandjon | c782bbc | 2020-08-05 13:47:13 -0700 | [diff] [blame] | 290 | // thread required them for the current key. Incorporate into successfulLoads any transitive |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 291 | // cache hits it does not already contain. |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 292 | cachedData.traverse(env::registerDependencies, inliningState.successfulLoads); |
shreyax | 3126205 | 2019-08-13 13:40:06 -0700 | [diff] [blame] | 293 | } |
| 294 | } |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 295 | |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 296 | // See if we've already unsuccessfully visited the bzl. |
| 297 | if (inliningState.unsuccessfulLoads.contains(key)) { |
| 298 | return null; |
| 299 | } |
| 300 | |
| 301 | // If we're here, the bzl must have never been visited before in this calling context. Compute |
| 302 | // it ourselves, updating the other data structures as appropriate. |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 303 | if (cachedData == null) { |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 304 | try { |
| 305 | cachedData = computeInlineForCacheMiss(key, env, inliningState); |
| 306 | } finally { |
| 307 | if (cachedData != null) { |
| 308 | inliningState.successfulLoads.put(key, cachedData); |
| 309 | cachedBzlLoadDataManager.cache.put(key, cachedData); |
| 310 | } else { |
| 311 | inliningState.unsuccessfulLoads.add(key); |
| 312 | // Either propagate an exception or fall through for null return. |
| 313 | } |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 314 | } |
| 315 | } |
| 316 | |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 317 | // On success (from cache hit or from scratch), notify the parent CachedBzlLoadData of its new |
| 318 | // child. |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 319 | if (cachedData != null) { |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 320 | inliningState.childCachedDataHandler.accept(cachedData); |
shreyax | 755a0a1 | 2018-03-19 20:47:37 -0700 | [diff] [blame] | 321 | } |
| 322 | |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 323 | return cachedData; |
| 324 | } |
shreyax | c163c91 | 2019-02-08 12:45:25 -0800 | [diff] [blame] | 325 | |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 326 | @Nullable |
| 327 | private CachedBzlLoadData computeInlineForCacheMiss( |
| 328 | BzlLoadValue.Key key, Environment env, InliningState inliningState) |
| 329 | throws InconsistentFilesystemException, BzlLoadFailedException, InterruptedException { |
| 330 | // We use an instrumented Skyframe env to capture Skyframe deps in the CachedBzlLoadData. This |
| 331 | // generally includes transitive Skyframe deps, but specifically excludes deps underneath |
| 332 | // recursively loaded .bzls. We unwrap the instrumented env right before recursively calling |
brandjon | 1d3f309 | 2020-06-08 10:58:25 -0700 | [diff] [blame] | 333 | // back into computeInlineCachedData. |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 334 | CachedBzlLoadData.Builder cachedDataBuilder = cachedBzlLoadDataManager.cachedDataBuilder(); |
shreyax | 755a0a1 | 2018-03-19 20:47:37 -0700 | [diff] [blame] | 335 | Preconditions.checkState( |
| 336 | !(env instanceof RecordingSkyFunctionEnvironment), |
| 337 | "Found nested RecordingSkyFunctionEnvironment but it should have been stripped: %s", |
| 338 | env); |
| 339 | RecordingSkyFunctionEnvironment recordingEnv = |
| 340 | new RecordingSkyFunctionEnvironment( |
| 341 | env, |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 342 | cachedDataBuilder::addDep, |
| 343 | cachedDataBuilder::addDeps, |
| 344 | cachedDataBuilder::noteException); |
shreyax | 755a0a1 | 2018-03-19 20:47:37 -0700 | [diff] [blame] | 345 | |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 346 | inliningState.beginLoad(key); // track for cyclic load() detection |
| 347 | BzlLoadValue value; |
| 348 | try { |
| 349 | value = |
| 350 | computeInternal( |
| 351 | key, |
| 352 | recordingEnv, |
| 353 | inliningState.createChildState( |
| 354 | /*childCachedDataHandler=*/ cachedDataBuilder::addTransitiveDeps)); |
| 355 | } finally { |
| 356 | inliningState.finishLoad(key); |
shreyax | 755a0a1 | 2018-03-19 20:47:37 -0700 | [diff] [blame] | 357 | } |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 358 | if (value == null) { |
| 359 | return null; |
| 360 | } |
| 361 | |
| 362 | cachedDataBuilder.setValue(value); |
| 363 | cachedDataBuilder.setKey(key); |
| 364 | return cachedDataBuilder.build(); |
Michajlo Matijkiw | 10c363d | 2015-10-09 22:11:34 +0000 | [diff] [blame] | 365 | } |
| 366 | |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 367 | public void resetInliningCache() { |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 368 | cachedBzlLoadDataManager.reset(); |
shreyax | 755a0a1 | 2018-03-19 20:47:37 -0700 | [diff] [blame] | 369 | } |
| 370 | |
brandjon | 1af65cf | 2020-06-06 18:17:53 -0700 | [diff] [blame] | 371 | /** |
brandjon | b4ebbfc | 2020-08-13 15:50:23 -0700 | [diff] [blame] | 372 | * An opaque object that holds state for the bzl inlining computation initiated by {@link |
brandjon | 1d3f309 | 2020-06-08 10:58:25 -0700 | [diff] [blame] | 373 | * #computeInline}. |
| 374 | * |
| 375 | * <p>An original caller of {@code computeInline} (e.g., {@link PackageFunction}) should obtain |
| 376 | * one of these objects using {@link InliningState#create}. When the same caller makes several |
| 377 | * calls to {@code computeInline} (e.g., for multiple top-level loads in the same BUILD file), the |
| 378 | * same object must be passed to each call. |
| 379 | * |
| 380 | * <p>When a Skyfunction that is called by {@code BzlLoadFunction}'s inlining code path in turn |
| 381 | * calls back into {@code computeInline}, it should forward along the same {@code InliningState} |
| 382 | * that it received. In particular, {@link StarlarkBuiltinsFunction} forwards the inlining state |
| 383 | * to ensure that 1) the .bzls that get loaded from the {@code @builtins} pseudo-repository are |
| 384 | * properly recorded as dependencies of all .bzl files that use builtins injection, and 2) the |
| 385 | * {@code @builtins} .bzls are not reevaluated. |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 386 | */ |
brandjon | 1d3f309 | 2020-06-08 10:58:25 -0700 | [diff] [blame] | 387 | static class InliningState { |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 388 | /** |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 389 | * The set of bzls we're currently in the process of loading but haven't fully visited yet. This |
| 390 | * is used for cycle detection since we don't have the benefit of Skyframe's internal cycle |
| 391 | * detection. The set must use insertion order for correct error reporting. |
| 392 | * |
| 393 | * <p>This is disjoint with {@link #successfulLoads} and {@link #unsuccessfulLoads}. |
| 394 | * |
| 395 | * <p>This is local to current calling context. See {@link #computeInline}. |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 396 | */ |
| 397 | // Keyed on the SkyKey, not the label, since label could theoretically be ambiguous, even though |
| 398 | // in practice keys from BUILD / WORKSPACE / @builtins don't call each other. (Not sure if |
| 399 | // WORKSPACE chunking can cause duplicate labels to appear, but we're robust regardless.) |
| 400 | private final LinkedHashSet<BzlLoadValue.Key> loadStack; |
| 401 | |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 402 | /** |
| 403 | * Cache of bzls that have been fully visited and successfully loaded to a value. |
| 404 | * |
| 405 | * <p>This and {@link #unsuccessfulLoads} partition the set of fully visited bzls. |
| 406 | * |
| 407 | * <p>This is local to current calling context. See {@link #computeInline}. |
| 408 | */ |
| 409 | private final Map<BzlLoadValue.Key, CachedBzlLoadData> successfulLoads; |
| 410 | |
| 411 | /** |
| 412 | * Set of bzls that have been fully visited, but were not successfully loaded to a value. |
| 413 | * |
| 414 | * <p>This and {@link #successfulLoads} partition the set of fully visited bzls, and is disjoint |
| 415 | * with {@link #loadStack}. |
| 416 | * |
| 417 | * <p>This is local to current calling context. See {@link #computeInline}. |
| 418 | */ |
| 419 | private final HashSet<BzlLoadValue.Key> unsuccessfulLoads; |
| 420 | |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 421 | /** Called when a transitive {@code CachedBzlLoadData} is produced. */ |
| 422 | private final Consumer<CachedBzlLoadData> childCachedDataHandler; |
| 423 | |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 424 | private InliningState( |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 425 | LinkedHashSet<BzlLoadValue.Key> loadStack, |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 426 | Map<BzlLoadValue.Key, CachedBzlLoadData> successfulLoads, |
| 427 | HashSet<BzlLoadValue.Key> unsuccessfulLoads, |
| 428 | Consumer<CachedBzlLoadData> childCachedDataHandler) { |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 429 | this.loadStack = loadStack; |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 430 | this.successfulLoads = successfulLoads; |
| 431 | this.unsuccessfulLoads = unsuccessfulLoads; |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 432 | this.childCachedDataHandler = childCachedDataHandler; |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 433 | } |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 434 | |
brandjon | 1d3f309 | 2020-06-08 10:58:25 -0700 | [diff] [blame] | 435 | /** |
| 436 | * Creates an initial {@code InliningState} with no information about previously loaded files |
| 437 | * (except the shared cache stored in {@link BzlLoadFunction}). |
| 438 | */ |
| 439 | static InliningState create() { |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 440 | return new InliningState( |
brandjon | 1d3f309 | 2020-06-08 10:58:25 -0700 | [diff] [blame] | 441 | /*loadStack=*/ new LinkedHashSet<>(), |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 442 | /*successfulLoads=*/ new HashMap<>(), |
| 443 | /*unsuccessfulLoads=*/ new HashSet<>(), |
| 444 | // No parent value to mutate |
| 445 | /*childCachedDataHandler=*/ x -> {}); |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 446 | } |
| 447 | |
brandjon | 1d3f309 | 2020-06-08 10:58:25 -0700 | [diff] [blame] | 448 | private InliningState createChildState(Consumer<CachedBzlLoadData> childCachedDataHandler) { |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 449 | return new InliningState( |
| 450 | loadStack, successfulLoads, unsuccessfulLoads, childCachedDataHandler); |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 451 | } |
| 452 | |
| 453 | /** Records entry to a {@code load()}, throwing an exception if a cycle is detected. */ |
brandjon | 1d3f309 | 2020-06-08 10:58:25 -0700 | [diff] [blame] | 454 | private void beginLoad(BzlLoadValue.Key key) throws BzlLoadFailedException { |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 455 | if (!loadStack.add(key)) { |
| 456 | ImmutableList<BzlLoadValue.Key> cycle = |
| 457 | CycleUtils.splitIntoPathAndChain(Predicates.equalTo(key), loadStack).second; |
| 458 | throw new BzlLoadFailedException("Starlark load cycle: " + cycle); |
| 459 | } |
| 460 | } |
| 461 | |
| 462 | /** Records exit from a {@code load()}. */ |
brandjon | 1d3f309 | 2020-06-08 10:58:25 -0700 | [diff] [blame] | 463 | private void finishLoad(BzlLoadValue.Key key) throws BzlLoadFailedException { |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 464 | Preconditions.checkState(loadStack.remove(key), key); |
| 465 | } |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 466 | } |
| 467 | |
brandjon | b4ebbfc | 2020-08-13 15:50:23 -0700 | [diff] [blame] | 468 | /** |
| 469 | * Entry point for compute logic that's common to both (bzl) inlining and non-inlining code paths. |
| 470 | */ |
shreyax | 755a0a1 | 2018-03-19 20:47:37 -0700 | [diff] [blame] | 471 | // It is vital that we don't return any value if any call to env#getValue(s)OrThrow throws an |
| 472 | // exception. We are allowed to wrap the thrown exception and rethrow it for any calling functions |
| 473 | // to handle though. |
| 474 | @Nullable |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 475 | private BzlLoadValue computeInternal( |
| 476 | BzlLoadValue.Key key, Environment env, @Nullable InliningState inliningState) |
| 477 | throws InconsistentFilesystemException, BzlLoadFailedException, InterruptedException { |
brandjon | bc884e1 | 2020-05-22 10:24:37 -0700 | [diff] [blame] | 478 | Label label = key.getLabel(); |
brandjon | 7154be0 | 2020-05-22 09:47:32 -0700 | [diff] [blame] | 479 | PathFragment filePath = label.toPathFragment(); |
John Field | 110b065 | 2015-11-13 21:56:42 +0000 | [diff] [blame] | 480 | |
brandjon | 4d15f2b | 2020-08-12 09:15:52 -0700 | [diff] [blame] | 481 | ASTFileLookupValue.Key astKey = validatePackageAndGetASTKey(key, env); |
| 482 | if (astKey == null) { |
| 483 | return null; |
| 484 | } |
| 485 | ASTFileLookupValue astLookup; |
| 486 | try { |
| 487 | astLookup = astManager.getASTFileLookupValue(astKey, env); |
| 488 | } catch (ErrorReadingStarlarkExtensionException e) { |
| 489 | throw BzlLoadFailedException.errorReadingFile(filePath, e); |
| 490 | } |
| 491 | if (astLookup == null) { |
| 492 | return null; |
| 493 | } |
| 494 | |
| 495 | BzlLoadValue result = null; |
| 496 | // Release the AST iff the value gets completely evaluated (to either error or non-null result). |
| 497 | boolean completed = true; |
| 498 | try { |
| 499 | result = computeInternalWithAST(key, astLookup, env, inliningState); |
| 500 | completed = result != null; |
| 501 | } finally { |
| 502 | if (completed) { // only false on unexceptional null result |
| 503 | astManager.doneWithASTFileLookupValue(astKey); |
| 504 | } |
| 505 | } |
| 506 | return result; |
| 507 | } |
| 508 | |
| 509 | /** |
| 510 | * Returns the AST key for a bzl, or null for a missing Skyframe dep or unspecified exception. |
| 511 | * |
| 512 | * <p>Except for builtins bzls, a bzl is not considered loadable unless its load label matches its |
| 513 | * file target label. |
| 514 | */ |
| 515 | @Nullable |
| 516 | private static ASTFileLookupValue.Key validatePackageAndGetASTKey( |
| 517 | BzlLoadValue.Key key, Environment env) |
| 518 | throws BzlLoadFailedException, InconsistentFilesystemException, InterruptedException { |
| 519 | Label label = key.getLabel(); |
| 520 | |
| 521 | // Do package lookup. |
| 522 | PathFragment dir = Label.getContainingDirectory(label); |
| 523 | PackageIdentifier dirId = |
| 524 | PackageIdentifier.create(label.getPackageIdentifier().getRepository(), dir); |
| 525 | ContainingPackageLookupValue packageLookup; |
| 526 | try { |
| 527 | packageLookup = |
| 528 | (ContainingPackageLookupValue) |
| 529 | env.getValueOrThrow( |
| 530 | ContainingPackageLookupValue.key(dirId), |
| 531 | BuildFileNotFoundException.class, |
| 532 | InconsistentFilesystemException.class); |
| 533 | } catch (BuildFileNotFoundException e) { |
| 534 | throw BzlLoadFailedException.errorReadingFile( |
| 535 | label.toPathFragment(), new ErrorReadingStarlarkExtensionException(e)); |
| 536 | } |
| 537 | if (packageLookup == null) { |
| 538 | return null; |
| 539 | } |
| 540 | |
| 541 | // Resolve to AST key or error. |
| 542 | ASTFileLookupValue.Key astKey; |
| 543 | boolean packageOk = |
| 544 | packageLookup.hasContainingPackage() |
| 545 | && packageLookup.getContainingPackageName().equals(label.getPackageIdentifier()); |
| 546 | if (key.isBuildPrelude() && !packageOk) { |
| 547 | // Ignore the prelude, its package doesn't exist. |
| 548 | astKey = ASTFileLookupValue.EMPTY_PRELUDE_KEY; |
| 549 | } else { |
| 550 | if (packageOk) { |
| 551 | astKey = key.getASTKey(packageLookup.getContainingPackageRoot()); |
| 552 | } else { |
| 553 | if (!packageLookup.hasContainingPackage()) { |
| 554 | throw BzlLoadFailedException.noBuildFile( |
| 555 | label, packageLookup.getReasonForNoContainingPackage()); |
| 556 | } else { |
| 557 | throw BzlLoadFailedException.labelCrossesPackageBoundary(label, packageLookup); |
| 558 | } |
| 559 | } |
| 560 | } |
| 561 | return astKey; |
| 562 | } |
| 563 | |
| 564 | /** |
| 565 | * Compute logic for once the AST has been fetched and confirmed to exist (though it may have |
| 566 | * Starlark errors). |
| 567 | */ |
| 568 | @Nullable |
| 569 | private BzlLoadValue computeInternalWithAST( |
| 570 | BzlLoadValue.Key key, |
| 571 | ASTFileLookupValue astLookup, |
| 572 | Environment env, |
| 573 | @Nullable InliningState inliningState) |
| 574 | throws InconsistentFilesystemException, BzlLoadFailedException, InterruptedException { |
| 575 | Label label = key.getLabel(); |
| 576 | PathFragment filePath = label.toPathFragment(); |
| 577 | |
| 578 | if (!astLookup.lookupSuccessful()) { |
| 579 | // Starlark code must exist. (A missing prelude file still returns a valid but empty |
| 580 | // ASTFileLookupValue.) |
| 581 | throw new BzlLoadFailedException(astLookup.getError()); |
| 582 | } |
| 583 | StarlarkFile file = astLookup.getAST(); |
| 584 | if (!file.ok()) { |
| 585 | throw BzlLoadFailedException.starlarkErrors(filePath); |
| 586 | } |
| 587 | |
laurentlb | 8c02aff | 2019-02-18 10:53:34 -0800 | [diff] [blame] | 588 | StarlarkSemantics starlarkSemantics = PrecomputedValue.STARLARK_SEMANTICS.get(env); |
laurentlb | 6659b4c | 2019-02-18 07:23:36 -0800 | [diff] [blame] | 589 | if (starlarkSemantics == null) { |
brandjon | b712f33 | 2017-04-29 16:03:32 +0200 | [diff] [blame] | 590 | return null; |
| 591 | } |
brandjon | bc9bd06 | 2020-08-12 09:41:12 -0700 | [diff] [blame] | 592 | ImmutableMap<String, Object> predeclared = |
| 593 | getPredeclaredEnvironment(key, env, starlarkSemantics, inliningState); |
| 594 | if (predeclared == null) { |
brandjon | 1af65cf | 2020-06-06 18:17:53 -0700 | [diff] [blame] | 595 | return null; |
| 596 | } |
| 597 | |
adonovan | 87b4608 | 2020-07-08 15:58:04 -0700 | [diff] [blame] | 598 | // Process load statements in .bzl file (recursive .bzl -> .bzl loads), |
Googler | b2e1fe3 | 2019-09-20 14:00:39 -0700 | [diff] [blame] | 599 | // resolving labels relative to the current repo mapping. |
brandjon | 7154be0 | 2020-05-22 09:47:32 -0700 | [diff] [blame] | 600 | ImmutableMap<RepositoryName, RepositoryName> repoMapping = getRepositoryMapping(key, env); |
Googler | b2e1fe3 | 2019-09-20 14:00:39 -0700 | [diff] [blame] | 601 | if (repoMapping == null) { |
dannark | 4e42c32 | 2018-11-08 19:32:04 -0800 | [diff] [blame] | 602 | return null; |
| 603 | } |
adonovan | 699a90a | 2020-05-20 19:48:53 -0700 | [diff] [blame] | 604 | List<Pair<String, Label>> loads = |
brandjon | 7154be0 | 2020-05-22 09:47:32 -0700 | [diff] [blame] | 605 | getLoadLabels(env.getListener(), file, label.getPackageIdentifier(), repoMapping); |
adonovan | 699a90a | 2020-05-20 19:48:53 -0700 | [diff] [blame] | 606 | if (loads == null) { |
Googler | b2e1fe3 | 2019-09-20 14:00:39 -0700 | [diff] [blame] | 607 | // malformed load statements |
gregce | 0503fee | 2020-06-11 09:22:27 -0700 | [diff] [blame] | 608 | throw BzlLoadFailedException.starlarkErrors(filePath); |
Googler | b2e1fe3 | 2019-09-20 14:00:39 -0700 | [diff] [blame] | 609 | } |
Janak Ramakrishnan | df0531f | 2015-09-23 17:30:04 +0000 | [diff] [blame] | 610 | |
adonovan | 699a90a | 2020-05-20 19:48:53 -0700 | [diff] [blame] | 611 | // Compute Skyframe key for each label in 'loads'. |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 612 | List<BzlLoadValue.Key> loadKeys = Lists.newArrayListWithExpectedSize(loads.size()); |
adonovan | 699a90a | 2020-05-20 19:48:53 -0700 | [diff] [blame] | 613 | for (Pair<String, Label> load : loads) { |
brandjon | 7154be0 | 2020-05-22 09:47:32 -0700 | [diff] [blame] | 614 | loadKeys.add(key.getKeyForLoad(load.second)); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 615 | } |
adonovan | 699a90a | 2020-05-20 19:48:53 -0700 | [diff] [blame] | 616 | |
| 617 | // Load .bzl modules in parallel. |
brandjon | 844fef5 | 2020-06-08 10:30:36 -0700 | [diff] [blame] | 618 | // TODO(bazel-team): In case of a failed load(), we should report the location of the load() |
| 619 | // statement in the requesting file, e.g. using |
| 620 | // file.getLoadStatements().get(...).getStartLocation(). We should also probably catch and |
brandjon | b4ebbfc | 2020-08-13 15:50:23 -0700 | [diff] [blame] | 621 | // rethrow InconsistentFilesystemException with location info in the non-bzl-inlining code path |
| 622 | // so the error message is the same in both code paths. |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 623 | List<BzlLoadValue> bzlLoads = |
adonovan | 699a90a | 2020-05-20 19:48:53 -0700 | [diff] [blame] | 624 | inliningState == null |
brandjon | 844fef5 | 2020-06-08 10:30:36 -0700 | [diff] [blame] | 625 | ? computeBzlLoadsWithSkyframe(env, loadKeys, file) |
| 626 | : computeBzlLoadsWithInlining(env, loadKeys, file, inliningState); |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 627 | if (bzlLoads == null) { |
adonovan | 699a90a | 2020-05-20 19:48:53 -0700 | [diff] [blame] | 628 | return null; // Skyframe deps unavailable |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 629 | } |
Janak Ramakrishnan | df0531f | 2015-09-23 17:30:04 +0000 | [diff] [blame] | 630 | |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 631 | // Process the loaded modules. |
adonovan | 1d93d26 | 2020-05-07 14:50:42 -0700 | [diff] [blame] | 632 | // |
| 633 | // Compute a digest of the file itself plus the transitive hashes of the modules it directly |
| 634 | // loads. Loop iteration order matches the source order of load statements. |
| 635 | Fingerprint fp = new Fingerprint(); |
brandjon | 4d15f2b | 2020-08-12 09:15:52 -0700 | [diff] [blame] | 636 | fp.addBytes(astLookup.getDigest()); |
adonovan | 87b4608 | 2020-07-08 15:58:04 -0700 | [diff] [blame] | 637 | Map<String, Module> loadedModules = Maps.newLinkedHashMapWithExpectedSize(loads.size()); |
adonovan | 699a90a | 2020-05-20 19:48:53 -0700 | [diff] [blame] | 638 | for (int i = 0; i < loads.size(); i++) { |
| 639 | String loadString = loads.get(i).first; |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 640 | BzlLoadValue v = bzlLoads.get(i); |
adonovan | 87b4608 | 2020-07-08 15:58:04 -0700 | [diff] [blame] | 641 | loadedModules.put(loadString, v.getModule()); // dups ok |
adonovan | 1d93d26 | 2020-05-07 14:50:42 -0700 | [diff] [blame] | 642 | fp.addBytes(v.getTransitiveDigest()); |
Janak Ramakrishnan | dfd3497 | 2015-09-22 02:53:05 +0000 | [diff] [blame] | 643 | } |
adonovan | 1d93d26 | 2020-05-07 14:50:42 -0700 | [diff] [blame] | 644 | byte[] transitiveDigest = fp.digestAndReset(); |
John Field | 1ea7fc3 | 2015-12-22 19:37:19 +0000 | [diff] [blame] | 645 | |
brandjon | bc9bd06 | 2020-08-12 09:41:12 -0700 | [diff] [blame] | 646 | Module module = Module.withPredeclared(starlarkSemantics, predeclared); |
adonovan | 87b4608 | 2020-07-08 15:58:04 -0700 | [diff] [blame] | 647 | |
| 648 | // Record the module's filename, label, digest, and the set of modules it loads, |
| 649 | // forming a complete representation of the load DAG. |
| 650 | module.setClientData( |
| 651 | BazelModuleContext.create( |
| 652 | label, |
| 653 | file.getStartLocation().file(), |
| 654 | ImmutableMap.copyOf(loadedModules), |
| 655 | transitiveDigest)); |
| 656 | |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 657 | // executeBzlFile may post events to the Environment's handler, but events do not matter when |
| 658 | // caching BzlLoadValues. Note that executing the module mutates it. |
| 659 | executeBzlFile( |
| 660 | file, |
| 661 | key.getLabel(), |
| 662 | module, |
| 663 | loadedModules, |
| 664 | starlarkSemantics, |
| 665 | env.getListener(), |
| 666 | repoMapping); |
adonovan | 39e42d1 | 2020-07-09 09:16:58 -0700 | [diff] [blame] | 667 | return new BzlLoadValue(module, transitiveDigest); |
John Field | a97e17f | 2015-11-13 02:19:52 +0000 | [diff] [blame] | 668 | } |
| 669 | |
laurentlb | 4efb6c2 | 2019-07-03 15:33:46 -0700 | [diff] [blame] | 670 | private static ImmutableMap<RepositoryName, RepositoryName> getRepositoryMapping( |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 671 | BzlLoadValue.Key key, Environment env) throws InterruptedException { |
brandjon | 7154be0 | 2020-05-22 09:47:32 -0700 | [diff] [blame] | 672 | Label enclosingFileLabel = key.getLabel(); |
pcloudy | 260c3aa | 2019-01-04 02:33:50 -0800 | [diff] [blame] | 673 | |
dannark | d102a39 | 2019-01-09 13:12:01 -0800 | [diff] [blame] | 674 | ImmutableMap<RepositoryName, RepositoryName> repositoryMapping; |
brandjon | b28a76b | 2020-05-26 12:53:22 -0700 | [diff] [blame] | 675 | if (key instanceof BzlLoadValue.KeyForWorkspace) { |
brandjon | 7154be0 | 2020-05-22 09:47:32 -0700 | [diff] [blame] | 676 | // Still during workspace file evaluation |
brandjon | b28a76b | 2020-05-26 12:53:22 -0700 | [diff] [blame] | 677 | BzlLoadValue.KeyForWorkspace keyForWorkspace = (BzlLoadValue.KeyForWorkspace) key; |
| 678 | if (keyForWorkspace.getWorkspaceChunk() == 0) { |
brandjon | 7154be0 | 2020-05-22 09:47:32 -0700 | [diff] [blame] | 679 | // There is no previous workspace chunk |
| 680 | repositoryMapping = ImmutableMap.of(); |
| 681 | } else { |
| 682 | SkyKey workspaceFileKey = |
| 683 | WorkspaceFileValue.key( |
brandjon | b28a76b | 2020-05-26 12:53:22 -0700 | [diff] [blame] | 684 | keyForWorkspace.getWorkspacePath(), keyForWorkspace.getWorkspaceChunk() - 1); |
brandjon | 7154be0 | 2020-05-22 09:47:32 -0700 | [diff] [blame] | 685 | WorkspaceFileValue workspaceFileValue = (WorkspaceFileValue) env.getValue(workspaceFileKey); |
| 686 | // Note: we know for sure that the requested WorkspaceFileValue is fully computed so we do |
| 687 | // not need to check if it is null |
| 688 | repositoryMapping = |
| 689 | workspaceFileValue |
| 690 | .getRepositoryMapping() |
| 691 | .getOrDefault( |
| 692 | enclosingFileLabel.getPackageIdentifier().getRepository(), ImmutableMap.of()); |
| 693 | } |
| 694 | } else { |
| 695 | // We are fully done with workspace evaluation so we should get the mappings from the |
| 696 | // final RepositoryMappingValue |
dannark | 4e42c32 | 2018-11-08 19:32:04 -0800 | [diff] [blame] | 697 | PackageIdentifier packageIdentifier = enclosingFileLabel.getPackageIdentifier(); |
| 698 | RepositoryMappingValue repositoryMappingValue = |
| 699 | (RepositoryMappingValue) |
| 700 | env.getValue(RepositoryMappingValue.key(packageIdentifier.getRepository())); |
| 701 | if (repositoryMappingValue == null) { |
| 702 | return null; |
| 703 | } |
| 704 | repositoryMapping = repositoryMappingValue.getRepositoryMapping(); |
dannark | 4e42c32 | 2018-11-08 19:32:04 -0800 | [diff] [blame] | 705 | } |
dannark | d102a39 | 2019-01-09 13:12:01 -0800 | [diff] [blame] | 706 | return repositoryMapping; |
| 707 | } |
| 708 | |
| 709 | /** |
adonovan | 699a90a | 2020-05-20 19:48:53 -0700 | [diff] [blame] | 710 | * Returns a list of pairs mapping each load string in the BUILD or .bzl file to the Label it |
| 711 | * resolves to. Labels are resolved relative to {@code base}, the file's package. If any load |
| 712 | * statement is malformed, the function reports one or more errors to the handler and returns |
| 713 | * null. Order matches the source. |
John Field | a97e17f | 2015-11-13 02:19:52 +0000 | [diff] [blame] | 714 | */ |
| 715 | @Nullable |
adonovan | 699a90a | 2020-05-20 19:48:53 -0700 | [diff] [blame] | 716 | static List<Pair<String, Label>> getLoadLabels( |
Googler | b2e1fe3 | 2019-09-20 14:00:39 -0700 | [diff] [blame] | 717 | EventHandler handler, |
Googler | 66d099e | 2019-09-26 08:07:06 -0700 | [diff] [blame] | 718 | StarlarkFile file, |
Googler | b2e1fe3 | 2019-09-20 14:00:39 -0700 | [diff] [blame] | 719 | PackageIdentifier base, |
| 720 | ImmutableMap<RepositoryName, RepositoryName> repoMapping) { |
| 721 | Preconditions.checkArgument(!base.getRepository().isDefault()); |
| 722 | |
| 723 | // It's redundant that getRelativeWithRemapping needs a Label; |
| 724 | // a PackageIdentifier should suffice. Make one here. |
| 725 | Label buildLabel = getBUILDLabel(base); |
| 726 | |
| 727 | boolean ok = true; |
adonovan | 699a90a | 2020-05-20 19:48:53 -0700 | [diff] [blame] | 728 | List<Pair<String, Label>> loads = Lists.newArrayList(); |
Googler | b2e1fe3 | 2019-09-20 14:00:39 -0700 | [diff] [blame] | 729 | for (Statement stmt : file.getStatements()) { |
| 730 | if (stmt instanceof LoadStatement) { |
| 731 | LoadStatement load = (LoadStatement) stmt; |
| 732 | String module = load.getImport().getValue(); |
| 733 | |
| 734 | // Parse the load statement's module string as a label. |
| 735 | // It must end in .bzl and not be in package "//external". |
Googler | b2e1fe3 | 2019-09-20 14:00:39 -0700 | [diff] [blame] | 736 | try { |
| 737 | Label label = buildLabel.getRelativeWithRemapping(module, repoMapping); |
| 738 | if (!label.getName().endsWith(".bzl")) { |
| 739 | throw new LabelSyntaxException("The label must reference a file with extension '.bzl'"); |
| 740 | } |
| 741 | if (label.getPackageIdentifier().equals(LabelConstants.EXTERNAL_PACKAGE_IDENTIFIER)) { |
| 742 | throw new LabelSyntaxException( |
| 743 | "Starlark files may not be loaded from the //external package"); |
| 744 | } |
adonovan | 699a90a | 2020-05-20 19:48:53 -0700 | [diff] [blame] | 745 | loads.add(Pair.of(module, label)); |
Googler | b2e1fe3 | 2019-09-20 14:00:39 -0700 | [diff] [blame] | 746 | } catch (LabelSyntaxException ex) { |
Googler | 83439e6 | 2019-09-24 12:11:30 -0700 | [diff] [blame] | 747 | handler.handle( |
adonovan | 22096af | 2020-01-08 07:09:52 -0800 | [diff] [blame] | 748 | Event.error( |
| 749 | load.getImport().getStartLocation(), "in load statement: " + ex.getMessage())); |
Googler | b2e1fe3 | 2019-09-20 14:00:39 -0700 | [diff] [blame] | 750 | ok = false; |
| 751 | } |
| 752 | } |
| 753 | } |
adonovan | 699a90a | 2020-05-20 19:48:53 -0700 | [diff] [blame] | 754 | return ok ? loads : null; |
Googler | b2e1fe3 | 2019-09-20 14:00:39 -0700 | [diff] [blame] | 755 | } |
| 756 | |
| 757 | private static Label getBUILDLabel(PackageIdentifier pkgid) { |
| 758 | try { |
| 759 | return Label.create(pkgid, "BUILD"); |
| 760 | } catch (LabelSyntaxException e) { |
| 761 | // Shouldn't happen; the Label is well-formed by construction. |
| 762 | throw new IllegalStateException(e); |
| 763 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 764 | } |
| 765 | |
adgar | ebc0f2c | 2019-08-15 15:36:56 -0700 | [diff] [blame] | 766 | /** |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 767 | * Computes the BzlLoadValue for all given keys using vanilla Skyframe evaluation, returning |
| 768 | * {@code null} if Skyframe deps were missing and have been requested. |
adgar | ebc0f2c | 2019-08-15 15:36:56 -0700 | [diff] [blame] | 769 | */ |
| 770 | @Nullable |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 771 | private static List<BzlLoadValue> computeBzlLoadsWithSkyframe( |
brandjon | 844fef5 | 2020-06-08 10:30:36 -0700 | [diff] [blame] | 772 | Environment env, List<BzlLoadValue.Key> keys, StarlarkFile requestingFile) |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 773 | throws BzlLoadFailedException, InterruptedException { |
| 774 | List<BzlLoadValue> bzlLoads = Lists.newArrayListWithExpectedSize(keys.size()); |
| 775 | Map<SkyKey, ValueOrException<BzlLoadFailedException>> values = |
| 776 | env.getValuesOrThrow(keys, BzlLoadFailedException.class); |
brandjon | f4f9193 | 2020-05-11 10:20:48 -0700 | [diff] [blame] | 777 | // Uses same order as load()s in the file. Order matters since we report the first error. |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 778 | for (BzlLoadValue.Key key : keys) { |
adgar | ebc0f2c | 2019-08-15 15:36:56 -0700 | [diff] [blame] | 779 | try { |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 780 | bzlLoads.add((BzlLoadValue) values.get(key).get()); |
| 781 | } catch (BzlLoadFailedException exn) { |
brandjon | 844fef5 | 2020-06-08 10:30:36 -0700 | [diff] [blame] | 782 | throw BzlLoadFailedException.whileLoadingDep(requestingFile.getStartLocation().file(), exn); |
adgar | ebc0f2c | 2019-08-15 15:36:56 -0700 | [diff] [blame] | 783 | } |
| 784 | } |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 785 | return env.valuesMissing() ? null : bzlLoads; |
adgar | ebc0f2c | 2019-08-15 15:36:56 -0700 | [diff] [blame] | 786 | } |
| 787 | |
| 788 | /** |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 789 | * Computes the BzlLoadValue for all given keys by reusing this instance of the BzlLoadFunction, |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 790 | * bypassing traditional Skyframe evaluation. |
| 791 | * |
| 792 | * @return null if there was a missing Skyframe dep, an unspecified exception in a Skyframe dep |
| 793 | * request, or if this was a duplicate unsuccessful visitation |
adgar | ebc0f2c | 2019-08-15 15:36:56 -0700 | [diff] [blame] | 794 | */ |
| 795 | @Nullable |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 796 | private List<BzlLoadValue> computeBzlLoadsWithInlining( |
brandjon | 844fef5 | 2020-06-08 10:30:36 -0700 | [diff] [blame] | 797 | Environment env, |
| 798 | List<BzlLoadValue.Key> keys, |
| 799 | StarlarkFile requestingFile, |
| 800 | InliningState inliningState) |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 801 | throws InterruptedException, BzlLoadFailedException, InconsistentFilesystemException { |
brandjon | 844fef5 | 2020-06-08 10:30:36 -0700 | [diff] [blame] | 802 | String filePathForErrors = requestingFile.getStartLocation().file(); |
adgar | ebc0f2c | 2019-08-15 15:36:56 -0700 | [diff] [blame] | 803 | Preconditions.checkState( |
| 804 | env instanceof RecordingSkyFunctionEnvironment, |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 805 | "Expected to be recording dep requests when inlining BzlLoadFunction: %s", |
brandjon | 844fef5 | 2020-06-08 10:30:36 -0700 | [diff] [blame] | 806 | filePathForErrors); |
adgar | ebc0f2c | 2019-08-15 15:36:56 -0700 | [diff] [blame] | 807 | Environment strippedEnv = ((RecordingSkyFunctionEnvironment) env).getDelegate(); |
brandjon | 9198819 | 2020-05-28 14:57:45 -0700 | [diff] [blame] | 808 | |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 809 | List<BzlLoadValue> bzlLoads = Lists.newArrayListWithExpectedSize(keys.size()); |
nharmata | 60d0460 | 2020-08-11 12:28:57 -0700 | [diff] [blame] | 810 | // For the sake of ensuring the graph structure is deterministic, we need to request all of our |
brandjon | b4ebbfc | 2020-08-13 15:50:23 -0700 | [diff] [blame] | 811 | // deps, even if some of them yield errors. The first exception that is seen gets deferred, to |
| 812 | // be raised after the loop. All other exceptions are swallowed. |
nharmata | 60d0460 | 2020-08-11 12:28:57 -0700 | [diff] [blame] | 813 | // |
brandjon | b4ebbfc | 2020-08-13 15:50:23 -0700 | [diff] [blame] | 814 | // To see how immediately returning the first error leads to non-determinism, consider the case |
| 815 | // of two dependencies A and B, where A is in error and appears in a load statement above B. |
| 816 | // If A has completed at the time we request it, and if we were to immediately propagate that |
| 817 | // error, we never request B. On the other hand, if A is missing (null return), we do request B |
| 818 | // in the meantime for the sake of parallelism. |
| 819 | // |
| 820 | // This approach assumes --keep_going; determinism is not guaranteed otherwise. It also assumes |
| 821 | // InterruptedException does not occur, since we don't catch and defer it. |
adgar | ebc0f2c | 2019-08-15 15:36:56 -0700 | [diff] [blame] | 822 | Exception deferredException = null; |
| 823 | boolean valuesMissing = false; |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 824 | // NOTE: Iterating over loads in the order listed in the file. |
| 825 | for (BzlLoadValue.Key key : keys) { |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 826 | CachedBzlLoadData cachedData; |
adgar | ebc0f2c | 2019-08-15 15:36:56 -0700 | [diff] [blame] | 827 | try { |
brandjon | 1d3f309 | 2020-06-08 10:58:25 -0700 | [diff] [blame] | 828 | cachedData = computeInlineCachedData(key, strippedEnv, inliningState); |
brandjon | 844fef5 | 2020-06-08 10:30:36 -0700 | [diff] [blame] | 829 | } catch (BzlLoadFailedException e) { |
| 830 | e = BzlLoadFailedException.whileLoadingDep(filePathForErrors, e); |
| 831 | deferredException = MoreObjects.firstNonNull(deferredException, e); |
| 832 | continue; |
| 833 | } catch (InconsistentFilesystemException e) { |
adgar | ebc0f2c | 2019-08-15 15:36:56 -0700 | [diff] [blame] | 834 | deferredException = MoreObjects.firstNonNull(deferredException, e); |
| 835 | continue; |
| 836 | } |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 837 | if (cachedData == null) { |
nharmata | 636fb5e | 2020-07-24 15:30:08 -0700 | [diff] [blame] | 838 | // A null value for `cachedData` can occur when it (or its transitive loads) has a Skyframe |
| 839 | // dep that is missing or in error. It can also occur if there's a transitive load on a bzl |
| 840 | // that was already seen by inliningState and which returned null (note: in this case, it's |
| 841 | // not necessarily true that there are missing Skyframe deps because this bzl could have |
| 842 | // already been visited unsuccessfully). In both these cases, we want to continue making our |
| 843 | // inline calls, so as to maximize the number of dependent (non-inlined) SkyFunctions that |
| 844 | // are requested and avoid a quadratic number of restarts. |
adgar | ebc0f2c | 2019-08-15 15:36:56 -0700 | [diff] [blame] | 845 | valuesMissing = true; |
| 846 | } else { |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 847 | bzlLoads.add(cachedData.getValue()); |
adgar | ebc0f2c | 2019-08-15 15:36:56 -0700 | [diff] [blame] | 848 | } |
| 849 | } |
| 850 | if (deferredException != null) { |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 851 | Throwables.throwIfInstanceOf(deferredException, BzlLoadFailedException.class); |
adgar | ebc0f2c | 2019-08-15 15:36:56 -0700 | [diff] [blame] | 852 | Throwables.throwIfInstanceOf(deferredException, InconsistentFilesystemException.class); |
| 853 | throw new IllegalStateException( |
| 854 | "caught a checked exception of unexpected type", deferredException); |
| 855 | } |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 856 | return valuesMissing ? null : bzlLoads; |
adgar | ebc0f2c | 2019-08-15 15:36:56 -0700 | [diff] [blame] | 857 | } |
| 858 | |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 859 | /** |
brandjon | bc9bd06 | 2020-08-12 09:41:12 -0700 | [diff] [blame] | 860 | * Obtains the predeclared environment for a .bzl file, based on the type of .bzl and (if |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 861 | * applicable) the injected builtins. |
brandjon | bc9bd06 | 2020-08-12 09:41:12 -0700 | [diff] [blame] | 862 | * |
| 863 | * <p>Returns null if there was a missing Skyframe dep or unspecified exception. |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 864 | */ |
brandjon | bc9bd06 | 2020-08-12 09:41:12 -0700 | [diff] [blame] | 865 | @Nullable |
brandjon | 1af65cf | 2020-06-06 18:17:53 -0700 | [diff] [blame] | 866 | private ImmutableMap<String, Object> getPredeclaredEnvironment( |
brandjon | bc9bd06 | 2020-08-12 09:41:12 -0700 | [diff] [blame] | 867 | BzlLoadValue.Key key, |
| 868 | Environment env, |
| 869 | StarlarkSemantics starlarkSemantics, |
| 870 | InliningState inliningState) |
| 871 | throws BzlLoadFailedException, InterruptedException { |
| 872 | Label label = key.getLabel(); |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 873 | if (key instanceof BzlLoadValue.KeyForBuild) { |
brandjon | bc9bd06 | 2020-08-12 09:41:12 -0700 | [diff] [blame] | 874 | StarlarkBuiltinsValue starlarkBuiltinsValue; |
| 875 | try { |
| 876 | // TODO(#11437): Remove ability to disable injection by setting flag to empty string. |
| 877 | if (starlarkSemantics.experimentalBuiltinsBzlPath().isEmpty()) { |
| 878 | starlarkBuiltinsValue = uninjectedStarlarkBuiltins; |
| 879 | } else { |
| 880 | if (inliningState == null) { |
| 881 | starlarkBuiltinsValue = |
| 882 | (StarlarkBuiltinsValue) |
| 883 | env.getValueOrThrow(StarlarkBuiltinsValue.key(), BuiltinsFailedException.class); |
| 884 | } else { |
| 885 | starlarkBuiltinsValue = |
| 886 | StarlarkBuiltinsFunction.computeInline( |
brandjon | db4d1ea | 2020-08-14 11:31:08 -0700 | [diff] [blame^] | 887 | StarlarkBuiltinsValue.key(), |
| 888 | env, |
| 889 | inliningState, |
| 890 | packageFactory, |
| 891 | /*bzlLoadFunction=*/ this); |
brandjon | bc9bd06 | 2020-08-12 09:41:12 -0700 | [diff] [blame] | 892 | } |
| 893 | } |
| 894 | } catch (BuiltinsFailedException e) { |
| 895 | throw BzlLoadFailedException.builtinsFailed(label, e); |
| 896 | } |
| 897 | if (starlarkBuiltinsValue == null) { |
| 898 | return null; |
| 899 | } |
brandjon | 1af65cf | 2020-06-06 18:17:53 -0700 | [diff] [blame] | 900 | return starlarkBuiltinsValue.predeclaredForBuildBzl; |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 901 | } else if (key instanceof BzlLoadValue.KeyForWorkspace) { |
| 902 | return predeclaredForWorkspaceBzl; |
| 903 | } else if (key instanceof BzlLoadValue.KeyForBuiltins) { |
| 904 | return predeclaredForBuiltinsBzl; |
| 905 | } else { |
| 906 | throw new AssertionError("Unknown key type: " + key.getClass()); |
| 907 | } |
| 908 | } |
| 909 | |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 910 | /** Executes the .bzl file defining the module to be loaded. */ |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 911 | private void executeBzlFile( |
Googler | d54a499 | 2019-10-18 12:35:06 -0700 | [diff] [blame] | 912 | StarlarkFile file, |
brandjon | 7154be0 | 2020-05-22 09:47:32 -0700 | [diff] [blame] | 913 | Label label, |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 914 | Module module, |
adonovan | 1d93d26 | 2020-05-07 14:50:42 -0700 | [diff] [blame] | 915 | Map<String, Module> loadedModules, |
laurentlb | 6659b4c | 2019-02-18 07:23:36 -0800 | [diff] [blame] | 916 | StarlarkSemantics starlarkSemantics, |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 917 | ExtendedEventHandler skyframeEventHandler, |
dannark | d102a39 | 2019-01-09 13:12:01 -0800 | [diff] [blame] | 918 | ImmutableMap<RepositoryName, RepositoryName> repositoryMapping) |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 919 | throws BzlLoadFailedException, InterruptedException { |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 920 | try (Mutability mu = Mutability.create("loading", label)) { |
adonovan | 09d1370 | 2020-05-19 08:26:55 -0700 | [diff] [blame] | 921 | StarlarkThread thread = new StarlarkThread(mu, starlarkSemantics); |
adonovan | dfcda9e | 2020-05-08 13:10:45 -0700 | [diff] [blame] | 922 | thread.setLoader(loadedModules::get); |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 923 | StoredEventHandler starlarkEventHandler = new StoredEventHandler(); |
| 924 | thread.setPrintHandler(Event.makeDebugPrintHandler(starlarkEventHandler)); |
brandjon | db4d1ea | 2020-08-14 11:31:08 -0700 | [diff] [blame^] | 925 | packageFactory |
| 926 | .getRuleClassProvider() |
| 927 | .setStarlarkThreadContext(thread, label, repositoryMapping); |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 928 | execAndExport(file, label, starlarkEventHandler, module, thread); |
Googler | 2234d38 | 2015-03-24 17:18:02 +0000 | [diff] [blame] | 929 | |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 930 | Event.replayEventsOn(skyframeEventHandler, starlarkEventHandler.getEvents()); |
| 931 | for (Postable post : starlarkEventHandler.getPosts()) { |
| 932 | skyframeEventHandler.post(post); |
Klaus Aehlig | da0a701 | 2017-06-14 13:40:23 +0200 | [diff] [blame] | 933 | } |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 934 | if (starlarkEventHandler.hasErrors()) { |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 935 | throw BzlLoadFailedException.errors(label.toPathFragment()); |
Francois-Rene Rideau | 89312fb | 2015-09-10 18:53:03 +0000 | [diff] [blame] | 936 | } |
Googler | 2234d38 | 2015-03-24 17:18:02 +0000 | [diff] [blame] | 937 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 938 | } |
| 939 | |
Googler | f0890f0 | 2019-10-01 07:28:48 -0700 | [diff] [blame] | 940 | // Precondition: file is validated and error-free. |
adonovan | 1d93d26 | 2020-05-07 14:50:42 -0700 | [diff] [blame] | 941 | // Precondition: thread has a valid transitiveDigest. |
brandjon | a0c6f81 | 2020-06-06 14:13:19 -0700 | [diff] [blame] | 942 | // TODO(adonovan): executeBzlFile would make a better public API than this function. |
Googler | a3421e2 | 2019-09-26 06:48:32 -0700 | [diff] [blame] | 943 | public static void execAndExport( |
brandjon | 7154be0 | 2020-05-22 09:47:32 -0700 | [diff] [blame] | 944 | StarlarkFile file, Label label, EventHandler handler, Module module, StarlarkThread thread) |
Googler | a3421e2 | 2019-09-26 06:48:32 -0700 | [diff] [blame] | 945 | throws InterruptedException { |
Dmitry Lomov | 950310f | 2017-03-01 17:45:12 +0000 | [diff] [blame] | 946 | |
Googler | ba86854 | 2019-10-09 07:26:27 -0700 | [diff] [blame] | 947 | // Intercept execution after every assignment at top level |
| 948 | // and "export" any newly assigned exportable globals. |
| 949 | // TODO(adonovan): change the semantics; see b/65374671. |
| 950 | thread.setPostAssignHook( |
| 951 | (name, value) -> { |
gregce | d281df7 | 2020-05-11 12:27:06 -0700 | [diff] [blame] | 952 | if (value instanceof StarlarkExportable) { |
| 953 | StarlarkExportable exp = (StarlarkExportable) value; |
Googler | ba86854 | 2019-10-09 07:26:27 -0700 | [diff] [blame] | 954 | if (!exp.isExported()) { |
| 955 | try { |
brandjon | 7154be0 | 2020-05-22 09:47:32 -0700 | [diff] [blame] | 956 | exp.export(label, name); |
Googler | ba86854 | 2019-10-09 07:26:27 -0700 | [diff] [blame] | 957 | } catch (EvalException ex) { |
adonovan | 3391e17 | 2020-08-05 14:21:14 -0700 | [diff] [blame] | 958 | handler.handle(Event.error(null, ex.getMessageWithStack())); |
Googler | ba86854 | 2019-10-09 07:26:27 -0700 | [diff] [blame] | 959 | } |
| 960 | } |
Dmitry Lomov | 950310f | 2017-03-01 17:45:12 +0000 | [diff] [blame] | 961 | } |
Googler | ba86854 | 2019-10-09 07:26:27 -0700 | [diff] [blame] | 962 | }); |
| 963 | |
| 964 | try { |
adonovan | 09d1370 | 2020-05-19 08:26:55 -0700 | [diff] [blame] | 965 | EvalUtils.exec(file, module, thread); |
Googler | ba86854 | 2019-10-09 07:26:27 -0700 | [diff] [blame] | 966 | } catch (EvalException ex) { |
adonovan | 3391e17 | 2020-08-05 14:21:14 -0700 | [diff] [blame] | 967 | handler.handle(Event.error(null, ex.getMessageWithStack())); |
Dmitry Lomov | 950310f | 2017-03-01 17:45:12 +0000 | [diff] [blame] | 968 | } |
| 969 | } |
| 970 | |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 971 | @Override |
| 972 | public String extractTag(SkyKey skyKey) { |
| 973 | return null; |
| 974 | } |
| 975 | |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 976 | static final class BzlLoadFailedException extends Exception implements SaneAnalysisException { |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 977 | private final Transience transience; |
| 978 | |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 979 | private BzlLoadFailedException(String errorMessage) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 980 | super(errorMessage); |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 981 | this.transience = Transience.PERSISTENT; |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 982 | } |
| 983 | |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 984 | private BzlLoadFailedException(String errorMessage, Exception cause, Transience transience) { |
nharmata | bea67e9 | 2017-06-16 00:26:27 +0200 | [diff] [blame] | 985 | super(errorMessage, cause); |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 986 | this.transience = transience; |
nharmata | bea67e9 | 2017-06-16 00:26:27 +0200 | [diff] [blame] | 987 | } |
| 988 | |
brandjon | 844fef5 | 2020-06-08 10:30:36 -0700 | [diff] [blame] | 989 | Transience getTransience() { |
| 990 | return transience; |
| 991 | } |
| 992 | |
| 993 | // TODO(bazel-team): This exception should hold a Location of the requesting file's load |
| 994 | // statement, and code that catches it should use the location in the Event they create. |
| 995 | static BzlLoadFailedException whileLoadingDep( |
| 996 | String requestingFile, BzlLoadFailedException cause) { |
| 997 | // Don't chain exception cause, just incorporate the message with a prefix. |
| 998 | return new BzlLoadFailedException("in " + requestingFile + ": " + cause.getMessage()); |
| 999 | } |
| 1000 | |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 1001 | static BzlLoadFailedException errors(PathFragment file) { |
| 1002 | return new BzlLoadFailedException(String.format("Extension file '%s' has errors", file)); |
Googler | 2234d38 | 2015-03-24 17:18:02 +0000 | [diff] [blame] | 1003 | } |
| 1004 | |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 1005 | static BzlLoadFailedException errorReadingFile( |
gregce | 5c8a5f5 | 2020-05-13 10:35:36 -0700 | [diff] [blame] | 1006 | PathFragment file, ErrorReadingStarlarkExtensionException cause) { |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 1007 | return new BzlLoadFailedException( |
nharmata | bea67e9 | 2017-06-16 00:26:27 +0200 | [diff] [blame] | 1008 | String.format( |
gregce | ebd616f | 2020-04-10 16:47:41 -0700 | [diff] [blame] | 1009 | "Encountered error while reading extension file '%s': %s", file, cause.getMessage()), |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1010 | cause, |
| 1011 | cause.getTransience()); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 1012 | } |
| 1013 | |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 1014 | static BzlLoadFailedException noBuildFile(Label file, @Nullable String reason) { |
Googler | 06eb1bb | 2019-02-26 15:33:15 -0800 | [diff] [blame] | 1015 | if (reason != null) { |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 1016 | return new BzlLoadFailedException( |
Googler | 06eb1bb | 2019-02-26 15:33:15 -0800 | [diff] [blame] | 1017 | String.format("Unable to find package for %s: %s.", file, reason)); |
| 1018 | } |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 1019 | return new BzlLoadFailedException( |
gregce | ebd616f | 2020-04-10 16:47:41 -0700 | [diff] [blame] | 1020 | String.format( |
| 1021 | "Every .bzl file must have a corresponding package, but '%s' does not have one." |
| 1022 | + " Please create a BUILD file in the same or any parent directory. Note that" |
| 1023 | + " this BUILD file does not need to do anything except exist.", |
| 1024 | file)); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 1025 | } |
| 1026 | |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 1027 | static BzlLoadFailedException labelCrossesPackageBoundary( |
brandjon | 7154be0 | 2020-05-22 09:47:32 -0700 | [diff] [blame] | 1028 | Label label, ContainingPackageLookupValue containingPackageLookupValue) { |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 1029 | return new BzlLoadFailedException( |
nharmata | d86b509 | 2018-10-16 15:50:21 -0700 | [diff] [blame] | 1030 | ContainingPackageLookupValue.getErrorMessageForLabelCrossingPackageBoundary( |
| 1031 | // We don't actually know the proper Root to pass in here (since we don't e.g. know |
brandjon | 7154be0 | 2020-05-22 09:47:32 -0700 | [diff] [blame] | 1032 | // the root of the bzl/BUILD file that is trying to load 'label'). Therefore we just |
| 1033 | // pass in the Root of the containing package in order to still get a useful error |
| 1034 | // message for the user. |
nharmata | d86b509 | 2018-10-16 15:50:21 -0700 | [diff] [blame] | 1035 | containingPackageLookupValue.getContainingPackageRoot(), |
brandjon | 7154be0 | 2020-05-22 09:47:32 -0700 | [diff] [blame] | 1036 | label, |
nharmata | d86b509 | 2018-10-16 15:50:21 -0700 | [diff] [blame] | 1037 | containingPackageLookupValue)); |
| 1038 | } |
| 1039 | |
gregce | 0503fee | 2020-06-11 09:22:27 -0700 | [diff] [blame] | 1040 | static BzlLoadFailedException starlarkErrors(PathFragment file) { |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 1041 | return new BzlLoadFailedException(String.format("Extension '%s' has errors", file)); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 1042 | } |
brandjon | 1af65cf | 2020-06-06 18:17:53 -0700 | [diff] [blame] | 1043 | |
brandjon | 844fef5 | 2020-06-08 10:30:36 -0700 | [diff] [blame] | 1044 | static BzlLoadFailedException builtinsFailed(Label file, BuiltinsFailedException cause) { |
| 1045 | return new BzlLoadFailedException( |
| 1046 | String.format( |
| 1047 | "Internal error while loading Starlark builtins for %s: %s", |
| 1048 | file, cause.getMessage()), |
| 1049 | cause, |
| 1050 | cause.getTransience()); |
| 1051 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 1052 | } |
| 1053 | |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 1054 | /** |
| 1055 | * A manager abstracting over the method for obtaining {@code ASTFileLookupValue}s. See comment in |
| 1056 | * {@link #create}. |
| 1057 | */ |
| 1058 | private interface ASTManager { |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1059 | @Nullable |
brandjon | c782bbc | 2020-08-05 13:47:13 -0700 | [diff] [blame] | 1060 | ASTFileLookupValue getASTFileLookupValue(ASTFileLookupValue.Key key, Environment env) |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1061 | throws InconsistentFilesystemException, InterruptedException, |
gregce | 5c8a5f5 | 2020-05-13 10:35:36 -0700 | [diff] [blame] | 1062 | ErrorReadingStarlarkExtensionException; |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1063 | |
brandjon | c782bbc | 2020-08-05 13:47:13 -0700 | [diff] [blame] | 1064 | void doneWithASTFileLookupValue(ASTFileLookupValue.Key key); |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1065 | } |
| 1066 | |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 1067 | /** A manager that obtains ASTs from Skyframe calls. */ |
| 1068 | private static class RegularSkyframeASTManager implements ASTManager { |
| 1069 | private static final RegularSkyframeASTManager INSTANCE = new RegularSkyframeASTManager(); |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1070 | |
| 1071 | @Nullable |
| 1072 | @Override |
brandjon | c782bbc | 2020-08-05 13:47:13 -0700 | [diff] [blame] | 1073 | public ASTFileLookupValue getASTFileLookupValue(ASTFileLookupValue.Key key, Environment env) |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1074 | throws InconsistentFilesystemException, InterruptedException, |
gregce | 5c8a5f5 | 2020-05-13 10:35:36 -0700 | [diff] [blame] | 1075 | ErrorReadingStarlarkExtensionException { |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1076 | return (ASTFileLookupValue) |
| 1077 | env.getValueOrThrow( |
brandjon | c782bbc | 2020-08-05 13:47:13 -0700 | [diff] [blame] | 1078 | key, |
gregce | 5c8a5f5 | 2020-05-13 10:35:36 -0700 | [diff] [blame] | 1079 | ErrorReadingStarlarkExtensionException.class, |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1080 | InconsistentFilesystemException.class); |
| 1081 | } |
| 1082 | |
| 1083 | @Override |
brandjon | c782bbc | 2020-08-05 13:47:13 -0700 | [diff] [blame] | 1084 | public void doneWithASTFileLookupValue(ASTFileLookupValue.Key key) {} |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1085 | } |
| 1086 | |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 1087 | /** |
| 1088 | * A manager that obtains ASTs by inlining {@link ASTFileLookupFunction} (not to be confused with |
| 1089 | * inlining of {@code BzlLoadFunction}). Values are cached within the manager and released |
| 1090 | * explicitly by calling {@link #doneWithASTFileLookupValue}. |
| 1091 | */ |
| 1092 | private static class InliningAndCachingASTManager implements ASTManager { |
brandjon | db4d1ea | 2020-08-14 11:31:08 -0700 | [diff] [blame^] | 1093 | private final PackageFactory packageFactory; |
nharmata | 3487930 | 2020-04-22 11:30:15 -0700 | [diff] [blame] | 1094 | private final DigestHashFunction digestHashFunction; |
brandjon | f4f9193 | 2020-05-11 10:20:48 -0700 | [diff] [blame] | 1095 | // We keep a cache of ASTFileLookupValues that have been computed but whose corresponding |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 1096 | // BzlLoadValue has not yet completed. This avoids repeating the ASTFileLookupValue work in case |
| 1097 | // of Skyframe restarts. (If we weren't inlining, Skyframe would cache this for us.) |
brandjon | c782bbc | 2020-08-05 13:47:13 -0700 | [diff] [blame] | 1098 | private final Cache<ASTFileLookupValue.Key, ASTFileLookupValue> astFileLookupValueCache; |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1099 | |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 1100 | private InliningAndCachingASTManager( |
brandjon | db4d1ea | 2020-08-14 11:31:08 -0700 | [diff] [blame^] | 1101 | PackageFactory packageFactory, |
nharmata | 3487930 | 2020-04-22 11:30:15 -0700 | [diff] [blame] | 1102 | DigestHashFunction digestHashFunction, |
brandjon | c782bbc | 2020-08-05 13:47:13 -0700 | [diff] [blame] | 1103 | Cache<ASTFileLookupValue.Key, ASTFileLookupValue> astFileLookupValueCache) { |
brandjon | db4d1ea | 2020-08-14 11:31:08 -0700 | [diff] [blame^] | 1104 | this.packageFactory = packageFactory; |
nharmata | 3487930 | 2020-04-22 11:30:15 -0700 | [diff] [blame] | 1105 | this.digestHashFunction = digestHashFunction; |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1106 | this.astFileLookupValueCache = astFileLookupValueCache; |
| 1107 | } |
| 1108 | |
| 1109 | @Nullable |
| 1110 | @Override |
brandjon | c782bbc | 2020-08-05 13:47:13 -0700 | [diff] [blame] | 1111 | public ASTFileLookupValue getASTFileLookupValue(ASTFileLookupValue.Key key, Environment env) |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1112 | throws InconsistentFilesystemException, InterruptedException, |
gregce | 5c8a5f5 | 2020-05-13 10:35:36 -0700 | [diff] [blame] | 1113 | ErrorReadingStarlarkExtensionException { |
brandjon | c782bbc | 2020-08-05 13:47:13 -0700 | [diff] [blame] | 1114 | ASTFileLookupValue value = astFileLookupValueCache.getIfPresent(key); |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1115 | if (value == null) { |
brandjon | db4d1ea | 2020-08-14 11:31:08 -0700 | [diff] [blame^] | 1116 | value = ASTFileLookupFunction.computeInline(key, env, packageFactory, digestHashFunction); |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1117 | if (value != null) { |
brandjon | c782bbc | 2020-08-05 13:47:13 -0700 | [diff] [blame] | 1118 | astFileLookupValueCache.put(key, value); |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1119 | } |
| 1120 | } |
| 1121 | return value; |
| 1122 | } |
| 1123 | |
| 1124 | @Override |
brandjon | c782bbc | 2020-08-05 13:47:13 -0700 | [diff] [blame] | 1125 | public void doneWithASTFileLookupValue(ASTFileLookupValue.Key key) { |
| 1126 | astFileLookupValueCache.invalidate(key); |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1127 | } |
| 1128 | } |
| 1129 | |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 1130 | /** |
| 1131 | * Per-instance manager for {@link CachedBzlLoadData}, used when {@code BzlLoadFunction} calls are |
| 1132 | * inlined. |
| 1133 | */ |
| 1134 | private static class CachedBzlLoadDataManager { |
| 1135 | private final int cacheSize; |
| 1136 | private Cache<BzlLoadValue.Key, CachedBzlLoadData> cache; |
| 1137 | private CachedBzlLoadDataBuilderFactory cachedDataBuilderFactory = |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 1138 | new CachedBzlLoadDataBuilderFactory(); |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1139 | |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 1140 | private CachedBzlLoadDataManager(int cacheSize) { |
| 1141 | this.cacheSize = cacheSize; |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1142 | } |
| 1143 | |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 1144 | private CachedBzlLoadData.Builder cachedDataBuilder() { |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 1145 | return cachedDataBuilderFactory.newCachedBzlLoadDataBuilder(); |
brandjon | 2491264 | 2020-05-27 11:28:45 -0700 | [diff] [blame] | 1146 | } |
| 1147 | |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1148 | private void reset() { |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 1149 | if (cache != null) { |
| 1150 | logger.atInfo().log("Starlark inlining cache stats from earlier build: " + cache.stats()); |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1151 | } |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 1152 | cachedDataBuilderFactory = new CachedBzlLoadDataBuilderFactory(); |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1153 | Preconditions.checkState( |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 1154 | cacheSize >= 0, "Expected positive Starlark cache size if caching. %s", cacheSize); |
| 1155 | cache = |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1156 | CacheBuilder.newBuilder() |
| 1157 | .concurrencyLevel(BlazeInterners.concurrencyLevel()) |
brandjon | 8d66929 | 2020-05-28 15:27:38 -0700 | [diff] [blame] | 1158 | .maximumSize(cacheSize) |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1159 | .recordStats() |
| 1160 | .build(); |
| 1161 | } |
| 1162 | } |
| 1163 | |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 1164 | private static final class BzlLoadFunctionException extends SkyFunctionException { |
| 1165 | private BzlLoadFunctionException(BzlLoadFailedException cause) { |
nharmata | dc1d9dc | 2020-04-18 16:53:28 -0700 | [diff] [blame] | 1166 | super(cause, cause.transience); |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 1167 | } |
| 1168 | |
brandjon | 771a029 | 2020-05-26 12:04:16 -0700 | [diff] [blame] | 1169 | private BzlLoadFunctionException(InconsistentFilesystemException e, Transience transience) { |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 1170 | super(e, transience); |
| 1171 | } |
Han-Wen Nienhuys | d08b27f | 2015-02-25 16:45:20 +0100 | [diff] [blame] | 1172 | } |
| 1173 | } |