| // Copyright 2014 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package com.google.devtools.build.lib.skyframe; |
| |
| import com.google.common.cache.Cache; |
| import com.google.common.cache.CacheBuilder; |
| import com.google.common.collect.ImmutableMap; |
| import com.google.common.collect.ImmutableSet; |
| import com.google.common.hash.HashFunction; |
| import com.google.devtools.build.lib.actions.FileValue; |
| import com.google.devtools.build.lib.cmdline.BazelCompileContext; |
| import com.google.devtools.build.lib.cmdline.Label; |
| import com.google.devtools.build.lib.cmdline.PackageIdentifier; |
| import com.google.devtools.build.lib.events.Event; |
| import com.google.devtools.build.lib.events.EventHandler; |
| import com.google.devtools.build.lib.io.InconsistentFilesystemException; |
| import com.google.devtools.build.lib.packages.BazelStarlarkEnvironment; |
| import com.google.devtools.build.lib.packages.BuildFileNotFoundException; |
| import com.google.devtools.build.lib.packages.PackageLoadingListener; |
| import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; |
| import com.google.devtools.build.lib.skyframe.BzlCompileValue.TypeOptions; |
| import com.google.devtools.build.lib.vfs.FileSystemUtils; |
| import com.google.devtools.build.lib.vfs.Path; |
| import com.google.devtools.build.lib.vfs.PathFragment; |
| import com.google.devtools.build.lib.vfs.Root; |
| import com.google.devtools.build.lib.vfs.RootedPath; |
| import com.google.devtools.build.skyframe.SkyFunction; |
| import com.google.devtools.build.skyframe.SkyFunctionException; |
| import com.google.devtools.build.skyframe.SkyFunctionException.Transience; |
| import com.google.devtools.build.skyframe.SkyKey; |
| import com.google.devtools.build.skyframe.SkyValue; |
| import java.io.IOException; |
| import java.util.List; |
| import java.util.concurrent.ExecutionException; |
| import javax.annotation.Nullable; |
| import net.starlark.java.eval.EvalException; |
| import net.starlark.java.eval.Module; |
| import net.starlark.java.eval.Mutability; |
| import net.starlark.java.eval.Sequence; |
| import net.starlark.java.eval.Starlark; |
| import net.starlark.java.eval.StarlarkSemantics; |
| import net.starlark.java.eval.StarlarkThread; |
| import net.starlark.java.syntax.AssignmentStatement; |
| import net.starlark.java.syntax.FileOptions; |
| import net.starlark.java.syntax.Identifier; |
| import net.starlark.java.syntax.Location; |
| import net.starlark.java.syntax.ParserInput; |
| import net.starlark.java.syntax.Program; |
| import net.starlark.java.syntax.StarlarkFile; |
| import net.starlark.java.syntax.Statement; |
| import net.starlark.java.syntax.StringLiteral; |
| import net.starlark.java.syntax.SyntaxError; |
| |
| /** |
| * A Skyframe function that compiles the .bzl file denoted by a Label. |
| * |
| * <p>Given a {@link Label} referencing a Starlark file, BzlCompileFunction loads, parses, resolves, |
| * and compiles it. The Label must be absolute, and must not reference the special {@code external} |
| * package. If the file (or the package containing it) doesn't exist, the function doesn't fail, but |
| * instead returns a specific {@code NO_FILE} {@link BzlCompileValue}. |
| */ |
| // TODO(adonovan): actually compile. The name is a step ahead of the implementation. |
| public class BzlCompileFunction implements SkyFunction { |
| |
| private final BazelStarlarkEnvironment bazelStarlarkEnvironment; |
| private final HashFunction hashFunction; |
| private final PackageLoadingListener packageLoadingListener; |
| |
| public BzlCompileFunction( |
| BazelStarlarkEnvironment bazelStarlarkEnvironment, |
| HashFunction hashFunction, |
| PackageLoadingListener packageLoadingListener) { |
| this.bazelStarlarkEnvironment = bazelStarlarkEnvironment; |
| this.hashFunction = hashFunction; |
| this.packageLoadingListener = packageLoadingListener; |
| } |
| |
| @Override |
| public SkyValue compute(SkyKey skyKey, Environment env) |
| throws SkyFunctionException, InterruptedException { |
| try { |
| return computeInline( |
| (BzlCompileValue.Key) skyKey.argument(), |
| env, |
| bazelStarlarkEnvironment, |
| hashFunction, |
| packageLoadingListener); |
| } catch (FailedIOException e) { |
| throw new FunctionException(e); |
| } |
| } |
| |
| @Nullable |
| static BzlCompileValue computeInline( |
| BzlCompileValue.Key key, |
| Environment env, |
| BazelStarlarkEnvironment bazelStarlarkEnvironment, |
| HashFunction hashFunction, |
| PackageLoadingListener packageLoadingListener) |
| throws FailedIOException, InterruptedException { |
| byte[] bytes; |
| byte[] digest; |
| String inputName; |
| RootedPath rootedPath = null; |
| |
| StarlarkSemantics semantics = PrecomputedValue.STARLARK_SEMANTICS.get(env); |
| if (semantics == null) { |
| return null; |
| } |
| |
| if (key.kind == BzlCompileValue.Kind.EMPTY_PRELUDE) { |
| // Default prelude is empty. |
| bytes = new byte[] {}; |
| digest = null; |
| inputName = "<default prelude>"; |
| } else { |
| // Obtain the file. |
| rootedPath = RootedPath.toRootedPath(key.root, key.label.toPathFragment()); |
| SkyKey fileSkyKey = FileValue.key(rootedPath); |
| FileValue fileValue = null; |
| try { |
| fileValue = (FileValue) env.getValueOrThrow(fileSkyKey, IOException.class); |
| } catch (IOException e) { |
| throw new FailedIOException(e, Transience.PERSISTENT); |
| } |
| if (fileValue == null) { |
| return null; |
| } |
| |
| if (fileValue.exists()) { |
| if (!fileValue.isFile()) { |
| return fileValue.isDirectory() |
| ? BzlCompileValue.noFile("cannot load '%s': is a directory", key.label) |
| : BzlCompileValue.noFile( |
| "cannot load '%s': not a regular file (dangling link?)", key.label); |
| } |
| |
| Path path = rootedPath.asPath(); |
| Location location = Location.fromFile(path.toString()); |
| |
| long fileSize = fileValue.getSize(); |
| // Special files (such as named pipes or sockets used to coordinate timing in tests) do not |
| // report a meaningful size in stat(), so their size is checked after reading below. |
| if (key.kind == BzlCompileValue.Kind.NORMAL && !fileValue.isSpecialFile()) { |
| BzlCompileValue limitFailure = |
| checkHardFileSizeLimit( |
| key, fileSize, location, semantics, bazelStarlarkEnvironment, env); |
| if (env.valuesMissing()) { |
| return null; |
| } |
| if (limitFailure != null) { |
| return limitFailure; |
| } |
| } |
| |
| // Read the file. |
| try { |
| bytes = |
| fileValue.isSpecialFile() |
| ? FileSystemUtils.readContent(path) |
| : FileSystemUtils.readWithKnownFileSize(path, fileSize); |
| } catch (IOException e) { |
| throw new FailedIOException(e, Transience.TRANSIENT); |
| } |
| |
| if (key.kind == BzlCompileValue.Kind.NORMAL && fileValue.isSpecialFile()) { |
| BzlCompileValue limitFailure = |
| checkHardFileSizeLimit( |
| key, bytes.length, location, semantics, bazelStarlarkEnvironment, env); |
| if (env.valuesMissing()) { |
| return null; |
| } |
| if (limitFailure != null) { |
| return limitFailure; |
| } |
| } |
| |
| digest = fileValue.getDigest(); // may be null |
| inputName = path.toString(); |
| } else { |
| if (key.kind == BzlCompileValue.Kind.PRELUDE) { |
| // A non-existent prelude is fine. |
| bytes = new byte[] {}; |
| digest = null; |
| inputName = "<default prelude>"; |
| } else { |
| return BzlCompileValue.noFile("cannot load '%s': no such file", key.label); |
| } |
| } |
| } |
| |
| // Compute digest if we didn't already get it from a fileValue. |
| if (digest == null) { |
| digest = hashFunction.hashBytes(bytes).asBytes(); |
| } |
| |
| TypeOptions typeOptions = getTypeOptions(semantics, key); |
| boolean resolveTypeSyntax = |
| typeOptions.wantStaticTypeChecking() || typeOptions.wantDynamicTypeChecking(); |
| |
| ImmutableMap<String, Object> predeclared; |
| if (key.isSclDialect()) { |
| predeclared = bazelStarlarkEnvironment.getStarlarkGlobals().getSclToplevels(); |
| } else if (key.kind == BzlCompileValue.Kind.BUILTINS) { |
| predeclared = |
| resolveTypeSyntax |
| ? bazelStarlarkEnvironment.getBuiltinsBzlEnvWithExtraTypeConstructors() |
| : bazelStarlarkEnvironment.getBuiltinsBzlEnv(); |
| } else { |
| // Use the predeclared environment for BUILD-loaded bzl files, ignoring injection. It is not |
| // the right env for the actual evaluation of BUILD-loaded bzl files because it doesn't |
| // map to the injected symbols. But the names of the symbols are the same, and the names are |
| // all we need to do symbol resolution. |
| // |
| // For WORKSPACE-loaded bzl files, the env isn't quite right not because of injection but |
| // because the "native" object is different. But A) that will be fixed with #11954, and B) we |
| // don't care for the same reason as above. |
| |
| predeclared = |
| resolveTypeSyntax |
| ? bazelStarlarkEnvironment.getUninjectedBuildBzlEnvWithExtraTypeConstructors() |
| : bazelStarlarkEnvironment.getUninjectedBuildBzlEnv(); |
| } |
| |
| // We have all deps. Parse, resolve, and return. |
| ParserInput input; |
| try { |
| input = |
| StarlarkUtil.createParserInput( |
| bytes, |
| inputName, |
| semantics.get(BuildLanguageOptions.INCOMPATIBLE_ENFORCE_STARLARK_UTF8), |
| env.getListener()); |
| } catch ( |
| @SuppressWarnings("UnusedException") // createParserInput() reports its own error message |
| StarlarkUtil.InvalidUtf8Exception e) { |
| return BzlCompileValue.noFile("compilation of '%s' failed", inputName); |
| } |
| |
| FileOptions.Builder optionsBuilder = |
| FileOptions.builder() |
| // By default, Starlark load statements create file-local bindings. |
| // However, the BUILD prelude typically contains nothing but load |
| // statements whose bindings are intended to be visible in all BUILD |
| // files. The loadBindsGlobally flag allows us to retrieve them. |
| .loadBindsGlobally(key.isBuildPrelude()) |
| // .scl files should be ASCII-only in string literals. |
| // TODO(bazel-team): It'd be nice if we could intercept non-ASCII errors from the lexer, |
| // and modify the displayed message to clarify to the user that the string would be |
| // permitted in a .bzl file. But there's no easy way to do that short of either string |
| // matching the error message or reworking the interpreter API to put more structured |
| // detail in errors (i.e. new fields or error subclasses). |
| .stringLiteralsAreAsciiOnly(key.isSclDialect()); |
| updateFileOptions(optionsBuilder, typeOptions); |
| StarlarkFile file = StarlarkFile.parse(input, optionsBuilder.build()); |
| |
| if (key.kind == BzlCompileValue.Kind.NORMAL) { |
| BzlCompileValue softLimitFailure = |
| checkSoftFileSizeLimit( |
| key, |
| bytes.length, |
| file, |
| Location.fromFile(inputName), |
| semantics, |
| bazelStarlarkEnvironment, |
| env); |
| if (env.valuesMissing()) { |
| return null; |
| } |
| if (softLimitFailure != null) { |
| return softLimitFailure; |
| } |
| } |
| |
| // compile |
| final Module module; |
| |
| if (key.kind == BzlCompileValue.Kind.EMPTY_PRELUDE) { |
| // The empty prelude has no label, so we can't use it to filter the predeclareds. |
| // This doesn't matter since the empty prelude doesn't attempt to access any predeclareds |
| // anyway. |
| module = Module.withPredeclared(semantics, predeclared); |
| } else { |
| // The BazelCompileContext holds additional contextual info to be associated with the Module |
| // The information is used to filter predeclareds |
| BazelCompileContext bazelCompileContext = |
| BazelCompileContext.create(key.label, file.getName()); |
| module = Module.withPredeclaredAndData(semantics, predeclared, bazelCompileContext); |
| } |
| try { |
| Program prog = Program.compileFile(file, module); |
| if (key.kind == BzlCompileValue.Kind.NORMAL) { |
| packageLoadingListener.onBzlCompileCompleteAndSuccessful(rootedPath, bytes.length); |
| } |
| return BzlCompileValue.withProgram(prog, digest, typeOptions); |
| } catch (SyntaxError.Exception ex) { |
| addSyntaxErrorsToListener(env.getListener(), ex.errors(), key); |
| return BzlCompileValue.noFile( |
| "compilation of module '%s'%s failed", |
| key.label.toPathFragment(), |
| StarlarkBuiltinsValue.isBuiltinsRepo(key.label.getRepository()) ? " (internal)" : ""); |
| } |
| } |
| |
| /** |
| * Whether the file should permit type syntax (annotations, etc.) based on flags and the type of |
| * file. |
| */ |
| private static TypeOptions getTypeOptions(StarlarkSemantics semantics, BzlCompileValue.Key key) { |
| boolean typeSyntaxFlag = semantics.getBool(StarlarkSemantics.EXPERIMENTAL_STARLARK_TYPE_SYNTAX); |
| List<String> allowlist = |
| semantics.get(BuildLanguageOptions.EXPERIMENTAL_STARLARK_TYPES_ALLOWED_PATHS); |
| |
| boolean okFiletype = |
| // annotations in prelude not allowed (it has null key.label) |
| !key.isBuildPrelude() |
| // annotations in SCL not allowed (not yet compatible with Go-Starlark interpreter) |
| && !key.isSclDialect(); |
| |
| boolean useTypeSyntax = false; |
| if (okFiletype) { |
| if (typeSyntaxFlag) { |
| if (allowlist.isEmpty() |
| || allowlist.stream().anyMatch(s -> key.label.getCanonicalForm().startsWith(s))) { |
| useTypeSyntax = true; |
| } |
| } |
| if (key.isBuiltins()) { |
| // Always enable type syntax for @_builtins |
| useTypeSyntax = true; |
| } |
| } |
| boolean doStaticTypeChecking = |
| useTypeSyntax |
| && (semantics.getBool(StarlarkSemantics.EXPERIMENTAL_STARLARK_STATIC_TYPE_CHECKING) |
| // Always enable static type checking for @_builtins |
| || key.isBuiltins()); |
| boolean doDynamicTypeChecking = |
| useTypeSyntax |
| && semantics.getBool(StarlarkSemantics.EXPERIMENTAL_STARLARK_DYNAMIC_TYPE_CHECKING); |
| |
| return new TypeOptions(useTypeSyntax, doStaticTypeChecking, doDynamicTypeChecking); |
| } |
| |
| private static void updateFileOptions(FileOptions.Builder builder, TypeOptions typeOptions) { |
| boolean needsTypeInfo = |
| typeOptions.wantStaticTypeChecking() || typeOptions.wantDynamicTypeChecking(); |
| builder |
| .allowTypeSyntax(typeOptions.useTypeSyntax()) |
| .resolveTypeSyntax(needsTypeInfo) |
| .tolerateInvalidTypeExpressions(!needsTypeInfo); |
| } |
| |
| /** |
| * Replays the syntax errors from a file onto an event handler, adding more context if necessary. |
| */ |
| private static void addSyntaxErrorsToListener( |
| EventHandler handler, List<SyntaxError> errors, BzlCompileValue.Key key) { |
| Event.replayEventsOn(handler, errors); |
| // If type annotations are disallowed, it could either be because the required flags aren't |
| // enabled or because the filetype disallows it. |
| for (var err : errors) { |
| if (err.message().contains(": type annotations are disallowed")) { |
| Location fileLoc = Location.fromFile(err.location().file()); |
| String explanation = |
| key.isSclDialect() |
| ? "Type annotations are not permitted in .scl files." |
| : """ |
| Type annotations syntax can be enabled with --experimental_starlark_type_syntax \ |
| and/or --experimental_starlark_types_allowed_paths.\ |
| """; |
| handler.handle(Event.error(fileLoc, explanation)); |
| } |
| } |
| } |
| |
| static final class FailedIOException extends Exception { |
| private final Transience transience; |
| |
| FailedIOException(IOException cause, Transience transience) { |
| super(cause.getMessage(), cause); |
| this.transience = transience; |
| } |
| |
| Transience getTransience() { |
| return transience; |
| } |
| } |
| |
| private static final class FunctionException extends SkyFunctionException { |
| private FunctionException(FailedIOException cause) { |
| super(cause, cause.transience); |
| } |
| } |
| |
| private static final String KNOWN_OVERSIZED_BZL_FILE_SYMBOL = "_KNOWN_OVERSIZED_BZL_FILE"; |
| |
| @Nullable |
| private static Label parseAllowlistLabel(String raw) { |
| if (raw.isEmpty()) { |
| return null; |
| } |
| return Label.parseCanonicalUnchecked(raw); |
| } |
| |
| private record AllowlistCheckResult(boolean allowlisted, @Nullable String errorMessage) { |
| static final AllowlistCheckResult ALLOWLISTED = new AllowlistCheckResult(true, null); |
| static final AllowlistCheckResult NOT_ALLOWLISTED = new AllowlistCheckResult(false, null); |
| |
| static AllowlistCheckResult error(String errorMessage) { |
| return new AllowlistCheckResult(false, errorMessage); |
| } |
| } |
| |
| private static final Cache<BzlCompileValue, ParsedAllowlist> allowlistCache = |
| CacheBuilder.newBuilder().weakKeys().build(); |
| |
| private record ParsedAllowlist( |
| @Nullable ImmutableSet<String> allowedEntries, @Nullable String errorMessage) { |
| static ParsedAllowlist success(ImmutableSet<String> allowedEntries) { |
| return new ParsedAllowlist(allowedEntries, null); |
| } |
| |
| static ParsedAllowlist error(String errorMessage) { |
| return new ParsedAllowlist(null, errorMessage); |
| } |
| } |
| |
| private static ParsedAllowlist evaluateAllowlist( |
| Label allowlistLabel, |
| BzlCompileValue allowlistCompileValue, |
| StarlarkSemantics semantics, |
| BazelStarlarkEnvironment bazelStarlarkEnvironment) |
| throws InterruptedException { |
| if (!allowlistCompileValue.lookupSuccessful()) { |
| return ParsedAllowlist.error( |
| String.format( |
| "Failed to compile allowlist file '%s': %s", |
| allowlistLabel.getCanonicalForm(), allowlistCompileValue.getError())); |
| } |
| |
| ImmutableMap<String, Object> predeclared = |
| bazelStarlarkEnvironment.getStarlarkGlobals().getSclToplevels(); |
| Module module = Module.withPredeclared(semantics, predeclared); |
| try (Mutability mu = Mutability.create()) { |
| StarlarkThread thread = StarlarkThread.createTransient(mu, semantics); |
| Starlark.execFileProgram(allowlistCompileValue.getProgram(), module, thread); |
| } catch (EvalException e) { |
| return ParsedAllowlist.error( |
| String.format( |
| "Failed to evaluate allowlist file '%s': %s", |
| allowlistLabel.getCanonicalForm(), e.getMessage())); |
| } |
| |
| Object allowedObj = module.getGlobal("ALLOWED"); |
| if (allowedObj == null) { |
| return ParsedAllowlist.error( |
| String.format( |
| "Allowlist file '%s' does not define an 'ALLOWED' symbol.", |
| allowlistLabel.getCanonicalForm())); |
| } |
| if (!(allowedObj instanceof Sequence<?> allowedSeq)) { |
| return ParsedAllowlist.error( |
| String.format( |
| "Allowlist file '%s': 'ALLOWED' must be a list of strings, but got %s.", |
| allowlistLabel.getCanonicalForm(), Starlark.type(allowedObj))); |
| } |
| ImmutableSet.Builder<String> allowedBuilder = ImmutableSet.builder(); |
| for (Object item : allowedSeq) { |
| if (item instanceof String s) { |
| allowedBuilder.add(s); |
| } |
| } |
| return ParsedAllowlist.success(allowedBuilder.build()); |
| } |
| |
| @Nullable |
| private static BzlCompileValue checkHardFileSizeLimit( |
| BzlCompileValue.Key key, |
| long fileSize, |
| Location location, |
| StarlarkSemantics semantics, |
| BazelStarlarkEnvironment bazelStarlarkEnvironment, |
| Environment env) |
| throws InterruptedException { |
| long maxLimit = semantics.get(BuildLanguageOptions.MAX_BZL_FILE_SIZE); |
| if (maxLimit <= 0 || fileSize <= maxLimit) { |
| return null; |
| } |
| AllowlistCheckResult allowlistResult = |
| isFileAllowlisted(key, fileSize, maxLimit, semantics, bazelStarlarkEnvironment, env); |
| if (allowlistResult == null || allowlistResult.allowlisted()) { |
| return null; |
| } |
| String msg = |
| allowlistResult.errorMessage() != null |
| ? allowlistResult.errorMessage() |
| : String.format( |
| "File '%s' size (%d bytes) exceeds the maximum allowed size (%d bytes). Loading" |
| + " oversized Starlark files is blocked to protect Blaze from JVM memory" |
| + " exhaustion.", |
| key.label, fileSize, maxLimit); |
| env.getListener().handle(Event.error(location, msg)); |
| return BzlCompileValue.noFile("%s", msg); |
| } |
| |
| @Nullable |
| private static BzlCompileValue checkSoftFileSizeLimit( |
| BzlCompileValue.Key key, |
| long fileSize, |
| StarlarkFile file, |
| Location location, |
| StarlarkSemantics semantics, |
| BazelStarlarkEnvironment bazelStarlarkEnvironment, |
| Environment env) |
| throws InterruptedException { |
| long softLimit = semantics.get(BuildLanguageOptions.SOFT_MAX_BZL_FILE_SIZE); |
| if (softLimit <= 0 || fileSize <= softLimit || hasKnownOversizedOptOut(file)) { |
| return null; |
| } |
| AllowlistCheckResult allowlistResult = |
| isFileAllowlisted(key, fileSize, softLimit, semantics, bazelStarlarkEnvironment, env); |
| if (allowlistResult == null || allowlistResult.allowlisted()) { |
| return null; |
| } |
| String msg = |
| allowlistResult.errorMessage() != null |
| ? allowlistResult.errorMessage() |
| : String.format( |
| "File '%s' size (%d bytes) exceeds the soft maximum size (%d bytes). Split this" |
| + " file or move data out of Starlark. To opt out up to the hard limit, set" |
| + " `%s = \"%s\"` at the top level of the file.", |
| key.label, |
| fileSize, |
| softLimit, |
| KNOWN_OVERSIZED_BZL_FILE_SYMBOL, |
| BuildLanguageOptions.KNOWN_OVERSIZED_BZL_FILE_VALUE_EXAMPLE); |
| env.getListener().handle(Event.error(location, msg)); |
| return BzlCompileValue.noFile("%s", msg); |
| } |
| |
| static boolean registerAllowlistDepIfOversized( |
| BzlCompileValue.Key key, SkyValue bzlFileSkyValue, Environment env) |
| throws InterruptedException { |
| if (!(bzlFileSkyValue instanceof FileValue bzlFileValue) |
| || key.kind != BzlCompileValue.Kind.NORMAL |
| || !bzlFileValue.exists() |
| || !bzlFileValue.isFile()) { |
| return true; |
| } |
| StarlarkSemantics semantics = PrecomputedValue.STARLARK_SEMANTICS.get(env); |
| if (semantics == null) { |
| return false; |
| } |
| Label allowlistLabel = |
| parseAllowlistLabel(semantics.get(BuildLanguageOptions.BZL_FILE_SIZE_LIMIT_ALLOWLIST)); |
| if (allowlistLabel == null || key.label == null || key.label.equals(allowlistLabel)) { |
| return true; |
| } |
| long maxLimit = semantics.get(BuildLanguageOptions.MAX_BZL_FILE_SIZE); |
| long softLimit = semantics.get(BuildLanguageOptions.SOFT_MAX_BZL_FILE_SIZE); |
| boolean exceedsLimit = |
| bzlFileValue.isSpecialFile() |
| || (maxLimit > 0 && bzlFileValue.getSize() > maxLimit) |
| || (softLimit > 0 && bzlFileValue.getSize() > softLimit); |
| if (!exceedsLimit) { |
| return true; |
| } |
| |
| PathFragment dir = Label.getContainingDirectory(allowlistLabel); |
| PackageIdentifier dirId = PackageIdentifier.create(allowlistLabel.getRepository(), dir); |
| ContainingPackageLookupValue packageLookup; |
| try { |
| packageLookup = |
| (ContainingPackageLookupValue) |
| env.getValueOrThrow( |
| ContainingPackageLookupValue.key(dirId), |
| BuildFileNotFoundException.class, |
| InconsistentFilesystemException.class); |
| } catch (BuildFileNotFoundException | InconsistentFilesystemException e) { |
| return true; |
| } |
| if (packageLookup == null || !packageLookup.hasContainingPackage()) { |
| return !env.valuesMissing(); |
| } |
| Root root = packageLookup.getContainingPackageRoot(); |
| |
| SkyKey allowlistFileKey = |
| FileValue.key(RootedPath.toRootedPath(root, allowlistLabel.toPathFragment())); |
| try { |
| var unused = env.getValueOrThrow(allowlistFileKey, IOException.class); |
| } catch (IOException e) { |
| return true; |
| } |
| if (env.valuesMissing()) { |
| return false; |
| } |
| |
| BzlCompileValue.Key allowlistCompileKey = BzlCompileValue.key(root, allowlistLabel); |
| try { |
| var unused = env.getValueOrThrow(allowlistCompileKey, FailedIOException.class); |
| } catch (FailedIOException e) { |
| return true; |
| } |
| return !env.valuesMissing(); |
| } |
| |
| private static boolean hasKnownOversizedOptOut(StarlarkFile file) { |
| for (Statement stmt : file.getStatements()) { |
| if (stmt instanceof AssignmentStatement assignment |
| && assignment.getOperator() == null |
| && assignment.getLHS() instanceof Identifier id |
| && id.getName().equals(KNOWN_OVERSIZED_BZL_FILE_SYMBOL) |
| && assignment.getRHS() instanceof StringLiteral str |
| && BuildLanguageOptions.KNOWN_OVERSIZED_BZL_FILE_VALUE_PATTERN |
| .matcher(str.getValue()) |
| .matches()) { |
| return true; |
| } |
| } |
| return false; |
| } |
| |
| @Nullable |
| private static AllowlistCheckResult isFileAllowlisted( |
| BzlCompileValue.Key key, |
| long fileSize, |
| long limit, |
| StarlarkSemantics semantics, |
| BazelStarlarkEnvironment bazelStarlarkEnvironment, |
| Environment env) |
| throws InterruptedException { |
| Label allowlistLabel = |
| parseAllowlistLabel(semantics.get(BuildLanguageOptions.BZL_FILE_SIZE_LIMIT_ALLOWLIST)); |
| if (allowlistLabel == null || key.label == null) { |
| return AllowlistCheckResult.NOT_ALLOWLISTED; |
| } |
| if (allowlistLabel.equals(key.label)) { |
| return AllowlistCheckResult.error( |
| String.format( |
| "Allowlist file '%s' size (%d bytes) exceeds the maximum allowed size (%d bytes)." |
| + " The allowlist file itself cannot be oversized to avoid dependency cycles.", |
| key.label, fileSize, limit)); |
| } |
| |
| PathFragment dir = Label.getContainingDirectory(allowlistLabel); |
| PackageIdentifier dirId = PackageIdentifier.create(allowlistLabel.getRepository(), dir); |
| ContainingPackageLookupValue packageLookup; |
| try { |
| packageLookup = |
| (ContainingPackageLookupValue) |
| env.getValueOrThrow( |
| ContainingPackageLookupValue.key(dirId), |
| BuildFileNotFoundException.class, |
| InconsistentFilesystemException.class); |
| } catch (BuildFileNotFoundException | InconsistentFilesystemException e) { |
| return AllowlistCheckResult.error( |
| String.format( |
| "Failed to find containing package for allowlist '%s': %s", |
| allowlistLabel.getCanonicalForm(), e.getMessage())); |
| } |
| if (packageLookup == null) { |
| return null; |
| } |
| if (!packageLookup.hasContainingPackage()) { |
| return AllowlistCheckResult.error( |
| String.format( |
| "Failed to load allowlist file '%s': no containing package", |
| allowlistLabel.getCanonicalForm())); |
| } |
| Root root = packageLookup.getContainingPackageRoot(); |
| |
| SkyKey allowlistFileKey = |
| FileValue.key(RootedPath.toRootedPath(root, allowlistLabel.toPathFragment())); |
| FileValue allowlistFileValue; |
| try { |
| allowlistFileValue = (FileValue) env.getValueOrThrow(allowlistFileKey, IOException.class); |
| } catch (IOException e) { |
| return AllowlistCheckResult.error( |
| String.format( |
| "Failed to read allowlist file '%s': %s", |
| allowlistLabel.getCanonicalForm(), e.getMessage())); |
| } |
| if (allowlistFileValue == null) { |
| return null; |
| } |
| if (!allowlistFileValue.exists()) { |
| return AllowlistCheckResult.error( |
| String.format("Allowlist file '%s' does not exist.", allowlistLabel.getCanonicalForm())); |
| } |
| |
| BzlCompileValue.Key allowlistCompileKey = BzlCompileValue.key(root, allowlistLabel); |
| BzlCompileValue allowlistCompileValue; |
| try { |
| allowlistCompileValue = |
| (BzlCompileValue) env.getValueOrThrow(allowlistCompileKey, FailedIOException.class); |
| } catch (FailedIOException e) { |
| return AllowlistCheckResult.error( |
| String.format( |
| "Failed to compile allowlist file '%s': %s", |
| allowlistLabel.getCanonicalForm(), e.getMessage())); |
| } |
| if (allowlistCompileValue == null) { |
| return null; |
| } |
| |
| ParsedAllowlist parsedAllowlist; |
| try { |
| parsedAllowlist = |
| allowlistCache.get( |
| allowlistCompileValue, |
| () -> |
| evaluateAllowlist( |
| allowlistLabel, allowlistCompileValue, semantics, bazelStarlarkEnvironment)); |
| } catch (ExecutionException e) { |
| Throwable cause = e.getCause(); |
| if (cause instanceof InterruptedException interruptedException) { |
| throw interruptedException; |
| } |
| return AllowlistCheckResult.error( |
| String.format( |
| "Failed to evaluate allowlist file '%s': %s", |
| allowlistLabel.getCanonicalForm(), |
| cause != null ? cause.getMessage() : e.getMessage())); |
| } |
| |
| if (parsedAllowlist.errorMessage() != null) { |
| return AllowlistCheckResult.error(parsedAllowlist.errorMessage()); |
| } |
| ImmutableSet<String> allowed = parsedAllowlist.allowedEntries(); |
| String pathString = key.label.toPathFragment().getPathString(); |
| String canonicalLabel = key.label.getCanonicalForm(); |
| String repoRelativeLabel = "//" + key.label.getPackageName() + ":" + key.label.getName(); |
| if (allowed != null |
| && (allowed.contains(pathString) |
| || allowed.contains(canonicalLabel) |
| || allowed.contains(repoRelativeLabel))) { |
| return AllowlistCheckResult.ALLOWLISTED; |
| } |
| return AllowlistCheckResult.NOT_ALLOWLISTED; |
| } |
| } |
