bazel syntax: add FileOptions
FileOptions is a set of per-file options that control the front-end ("static")
processing of a single Starlark file: scanning, parsing, validation,
and, eventually, compilation.
FileOptions determine language dialect and "code generation",
like the options to a compiler. Different files may have different options,
as in the case of BUILD and .bzl files, for example.
By contrast, a StarlarkSemantics affects only dynamic behavior,
and is carried down the thread, possibly through many files that
vary in their FileOptions.
FileOptions are specified at the construction of a StarlarkFile (parsing)
and are retained by it thereafter. This reduces the burden of ensuring
that consistent options are provided to the operations of parsing and validation
when these steps are widely separated in the source, as is often the case
in Bazel.
This change allows Copybara do validation on all its files.
It was previously unable because it could not select validation
options "a la carte". This will soon allow us to assume that all
files are resolved before execution.
The API for creating a Starlark thread is getting more verbose.
This is temporary; to "refactor" a term in an equation one must
first expand it out.
Details:
- Add FileOptions parameters to scanner, parser, and their wrappers.
- Remove the StarlarkSemantics parameter from getUndeclaredNameError.
It was only used for an assertion.
- Move StarlakSemantics into the evaluator package.
ValidationTest no longer depends on semantics or lib.packages.
- Decompose the validator's previous isBuildFile parameter into
separate features: top-level rebinding, whether to call setScope, etc.
- Simplify the "restrict string escapes" feature by pushing it
from the validator to the scanner.
- Simplify the "load disregards privacy" feature by pushing it
from the syntax tree into the validator. Delete parseVirtualBuildFile.
- Add a placeholder for a "load binds globally" feature,
for load in the REPL and for upcoming changes to the implementation
of the prelude. Its implementation will come later.
- Move the "load of private symbol" check from Eval to Validator.
This is a breaking API change for Copybara.
PiperOrigin-RevId: 302691302
diff --git a/src/main/java/com/google/devtools/build/lib/packages/BUILD b/src/main/java/com/google/devtools/build/lib/packages/BUILD
index b2a68b8..144e776 100644
--- a/src/main/java/com/google/devtools/build/lib/packages/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/packages/BUILD
@@ -73,7 +73,7 @@
srcs = ["StarlarkSemanticsOptions.java"],
deps = [
"//src/main/java/com/google/devtools/build/lib/concurrent",
- "//src/main/java/com/google/devtools/build/lib/syntax:frontend",
+ "//src/main/java/com/google/devtools/build/lib/syntax:evaluator",
"//src/main/java/com/google/devtools/common/options",
"//third_party:guava",
],
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java b/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java
index 88bc70a..6c3e6b3 100644
--- a/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java
+++ b/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java
@@ -44,6 +44,7 @@
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.EvalUtils;
import com.google.devtools.build.lib.syntax.Expression;
+import com.google.devtools.build.lib.syntax.FileOptions;
import com.google.devtools.build.lib.syntax.ForStatement;
import com.google.devtools.build.lib.syntax.FunctionSignature;
import com.google.devtools.build.lib.syntax.Identifier;
@@ -396,12 +397,24 @@
// Exposed to skyframe.PackageFunction.
public static StarlarkFile parseBuildFile(
- PackageIdentifier packageId, ParserInput input, List<Statement> preludeStatements) {
+ PackageIdentifier packageId,
+ ParserInput input,
+ List<Statement> preludeStatements,
+ StarlarkSemantics semantics) {
+ // Options for processing BUILD files.
+ FileOptions options =
+ FileOptions.builder()
+ .recordScope(false) // don't mutate BUILD syntax trees due to shared prelude
+ .requireLoadStatementsFirst(false)
+ .allowToplevelRebinding(true)
+ .restrictStringEscapes(semantics.incompatibleRestrictStringEscapes())
+ .build();
+
// Log messages are expected as signs of progress by a single very old test:
// testCreatePackageIsolatedFromOuterErrors, see CL 6198296.
// Removing them will cause it to time out. TODO(adonovan): clean this up.
logger.fine("Starting to parse " + packageId);
- StarlarkFile file = StarlarkFile.parseWithPrelude(input, preludeStatements);
+ StarlarkFile file = StarlarkFile.parseWithPrelude(input, preludeStatements, options);
logger.fine("Finished parsing of " + packageId);
return file;
}
@@ -511,7 +524,8 @@
ParserInput.create(
FileSystemUtils.convertFromLatin1(buildFileBytes), buildFile.asPath().toString());
StarlarkFile file =
- parseBuildFile(packageId, input, /*preludeStatements=*/ ImmutableList.<Statement>of());
+ parseBuildFile(
+ packageId, input, /*preludeStatements=*/ ImmutableList.<Statement>of(), semantics);
Package result =
createPackageFromAst(
externalPkg.getWorkspaceName(),
@@ -773,7 +787,7 @@
Module module = thread.getGlobals();
// Validate.
- ValidationEnvironment.validateFile(file, module, semantics, /*isBuildFile=*/ true);
+ ValidationEnvironment.validateFile(file, module);
if (!file.ok()) {
Event.replayEventsOn(pkgContext.eventHandler, file.errors());
return false;
diff --git a/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFactory.java b/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFactory.java
index 132a15b..eb8c776 100644
--- a/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFactory.java
+++ b/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFactory.java
@@ -118,7 +118,8 @@
if (localReporter == null) {
localReporter = new StoredEventHandler();
}
- StarlarkFile file = StarlarkFile.parse(source);
+
+ StarlarkFile file = StarlarkFile.parse(source); // use default options in tests
if (!file.ok()) {
Event.replayEventsOn(localReporter, file.errors());
throw new BuildFileContainsErrorsException(
@@ -185,9 +186,7 @@
/* analysisRuleLabel= */ null)
.storeInThread(thread);
- // Validate the file, apply BUILD dialect checks, then execute.
- ValidationEnvironment.validateFile(
- file, thread.getGlobals(), this.starlarkSemantics, /*isBuildFile=*/ true);
+ ValidationEnvironment.validateFile(file, thread.getGlobals());
List<String> globs = new ArrayList<>(); // unused
if (!file.ok()) {
Event.replayEventsOn(localReporter, file.errors());
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupFunction.java
index 8505c58..d29edb6 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupFunction.java
@@ -14,13 +14,13 @@
package com.google.devtools.build.lib.skyframe;
-import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.actions.FileValue;
import com.google.devtools.build.lib.actions.InconsistentFilesystemException;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.packages.BuildFileNotFoundException;
import com.google.devtools.build.lib.packages.RuleClassProvider;
+import com.google.devtools.build.lib.syntax.FileOptions;
import com.google.devtools.build.lib.syntax.Module;
import com.google.devtools.build.lib.syntax.ParserInput;
import com.google.devtools.build.lib.syntax.StarlarkFile;
@@ -39,7 +39,7 @@
import javax.annotation.Nullable;
/**
- * A SkyFunction for {@link ASTFileLookupValue}s.
+ * A Skyframe function that reads, parses and validates the .bzl file denoted by a Label.
*
* <p>Given a {@link Label} referencing a Skylark file, loads it as a syntax tree ({@link
* StarlarkFile}). The Label must be absolute, and must not reference the special {@code external}
@@ -103,27 +103,32 @@
if (!fileValue.isFile()) {
return ASTFileLookupValue.forBadFile(fileLabel);
}
- StarlarkSemantics starlarkSemantics = PrecomputedValue.STARLARK_SEMANTICS.get(env);
- if (starlarkSemantics == null) {
+ StarlarkSemantics semantics = PrecomputedValue.STARLARK_SEMANTICS.get(env);
+ if (semantics == null) {
return null;
}
+ // Options for scanning, parsing, and validating a .bzl file (including the prelude).
+ FileOptions options =
+ FileOptions.builder()
+ .restrictStringEscapes(semantics.incompatibleRestrictStringEscapes())
+ .build();
+
// Both the package and the file exist; load and parse the file.
Path path = rootedPath.asPath();
StarlarkFile file = null;
try {
byte[] bytes = FileSystemUtils.readWithKnownFileSize(path, fileValue.getSize());
ParserInput input = ParserInput.create(bytes, path.toString());
- file = StarlarkFile.parseWithDigest(input, path.getDigest());
+ file = StarlarkFile.parseWithDigest(input, path.getDigest(), options);
} catch (IOException e) {
throw new ASTLookupFunctionException(new ErrorReadingSkylarkExtensionException(e),
Transience.TRANSIENT);
}
// validate (and soon, compile)
- ImmutableMap<String, Object> predeclared = ruleClassProvider.getEnvironment();
ValidationEnvironment.validateFile(
- file, Module.createForBuiltins(predeclared), starlarkSemantics, /*isBuildFile=*/ false);
+ file, Module.createForBuiltins(ruleClassProvider.getEnvironment()));
Event.replayEventsOn(env.getListener(), file.errors()); // TODO(adonovan): fail if !ok()?
return ASTFileLookupValue.withFile(file);
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PackageFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/PackageFunction.java
index 96b802c..9d91d8a 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/PackageFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PackageFunction.java
@@ -1207,7 +1207,8 @@
// See the javadoc for ActionOnIOExceptionReadingBuildFile.
}
input = ParserInput.create(buildFileBytes, inputFile.toString());
- file = PackageFactory.parseBuildFile(packageId, input, preludeStatements);
+ file =
+ PackageFactory.parseBuildFile(packageId, input, preludeStatements, starlarkSemantics);
fileSyntaxCache.put(packageId, file);
}
SkylarkImportResult importResult;
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceASTFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceASTFunction.java
index 24aec92..e759b4b 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceASTFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceASTFunction.java
@@ -28,6 +28,7 @@
import com.google.devtools.build.lib.packages.RuleClassProvider;
import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction;
import com.google.devtools.build.lib.rules.repository.ResolvedFileValue;
+import com.google.devtools.build.lib.syntax.FileOptions;
import com.google.devtools.build.lib.syntax.LoadStatement;
import com.google.devtools.build.lib.syntax.ParserInput;
import com.google.devtools.build.lib.syntax.Printer;
@@ -45,6 +46,7 @@
import java.io.IOException;
import java.util.List;
import java.util.Map;
+import javax.annotation.Nullable;
/** A SkyFunction to parse WORKSPACE files into a StarlarkFile. */
public class WorkspaceASTFunction implements SkyFunction {
@@ -77,48 +79,61 @@
return null;
}
}
+ @Nullable StarlarkSemantics semantics = PrecomputedValue.STARLARK_SEMANTICS.get(env);
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ FileOptions options =
+ FileOptions.builder()
+ // These three options follow BUILD norms, but should probably be flipped.
+ .allowToplevelRebinding(true)
+ .requireLoadStatementsFirst(false)
+ .recordScope(false)
+ .restrictStringEscapes(
+ semantics != null && semantics.incompatibleRestrictStringEscapes())
+ .build();
Path repoWorkspace = workspaceRoot.getRoot().getRelative(workspaceRoot.getRootRelativePath());
try {
StarlarkFile file =
StarlarkFile.parse(
ParserInput.create(
- ruleClassProvider.getDefaultWorkspacePrefix(), "/DEFAULT.WORKSPACE"));
+ ruleClassProvider.getDefaultWorkspacePrefix(), "/DEFAULT.WORKSPACE"),
+ options);
if (!file.ok()) {
Event.replayEventsOn(env.getListener(), file.errors());
throw resolvedValueError("Failed to parse default WORKSPACE file");
}
if (newWorkspaceFileContents != null) {
file =
- StarlarkFile.parseVirtualBuildFile(
+ StarlarkFile.parseWithPrelude(
ParserInput.create(
newWorkspaceFileContents, resolvedFile.get().asPath().toString()),
- file.getStatements());
+ file.getStatements(),
+ // The WORKSPACE.resolved file breaks through the usual privacy mechanism.
+ options.toBuilder().allowLoadPrivateSymbols(true).build());
} else if (workspaceFileValue.exists()) {
byte[] bytes =
FileSystemUtils.readWithKnownFileSize(repoWorkspace, repoWorkspace.getFileSize());
file =
StarlarkFile.parseWithPrelude(
- ParserInput.create(bytes, repoWorkspace.toString()), file.getStatements());
+ ParserInput.create(bytes, repoWorkspace.toString()), file.getStatements(), options);
if (!file.ok()) {
Event.replayEventsOn(env.getListener(), file.errors());
throw resolvedValueError("Failed to parse WORKSPACE file");
}
}
- StarlarkSemantics starlarkSemantics = PrecomputedValue.STARLARK_SEMANTICS.get(env);
- if (env.valuesMissing()) {
- return null;
- }
String suffix;
if (resolvedFile.isPresent()) {
suffix = "";
- } else if (starlarkSemantics == null) {
+ } else if (semantics == null) {
// Starlark semantics was not found, but Skyframe is happy. That means we're in the test
// that didn't provide complete Skyframe environment. Just move along.
suffix = ruleClassProvider.getDefaultWorkspaceSuffix();
// TODO(hlopko): Uncomment once Bazel tests pass with --all_incompatible_changes
- // } else if (starlarkSemantics.incompatibleUseCcConfigureFromRulesCc()) {
+ // } else if (semantics.incompatibleUseCcConfigureFromRulesCc()) {
// suffix = ruleClassProvider.getDefaultWorkspaceSuffix();
} else if (!ruleClassProvider.getDefaultWorkspaceSuffix().contains("sh_configure")) {
// It might look fragile to check for sh_configure in the WORKSPACE file, but it turns
@@ -135,7 +150,10 @@
file =
StarlarkFile.parseWithPrelude(
- ParserInput.create(suffix, "/DEFAULT.WORKSPACE.SUFFIX"), file.getStatements());
+ ParserInput.create(suffix, "/DEFAULT.WORKSPACE.SUFFIX"),
+ file.getStatements(),
+ // The DEFAULT.WORKSPACE.SUFFIX file breaks through the usual privacy mechanism.
+ options.toBuilder().allowLoadPrivateSymbols(true).build());
if (!file.ok()) {
Event.replayEventsOn(env.getListener(), file.errors());
throw resolvedValueError("Failed to parse default WORKSPACE file suffix");
diff --git a/src/main/java/com/google/devtools/build/lib/skylarkdebug/server/ThreadHandler.java b/src/main/java/com/google/devtools/build/lib/skylarkdebug/server/ThreadHandler.java
index 3848eba..b2f3f58 100644
--- a/src/main/java/com/google/devtools/build/lib/skylarkdebug/server/ThreadHandler.java
+++ b/src/main/java/com/google/devtools/build/lib/skylarkdebug/server/ThreadHandler.java
@@ -27,6 +27,7 @@
import com.google.devtools.build.lib.syntax.Debug;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.EvalUtils;
+import com.google.devtools.build.lib.syntax.FileOptions;
import com.google.devtools.build.lib.syntax.ParserInput;
import com.google.devtools.build.lib.syntax.Starlark;
import com.google.devtools.build.lib.syntax.StarlarkThread;
@@ -298,7 +299,9 @@
servicingEvalRequest.set(true);
ParserInput input = ParserInput.create(content, "<debug eval>");
- Object x = EvalUtils.execAndEvalOptionalFinalExpression(input, thread.getGlobals(), thread);
+ Object x =
+ EvalUtils.execAndEvalOptionalFinalExpression(
+ input, FileOptions.DEFAULT, thread.getGlobals(), thread);
return x != null ? x : Starlark.NONE;
} finally {
servicingEvalRequest.set(false);
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/BUILD b/src/main/java/com/google/devtools/build/lib/syntax/BUILD
index 3c33cee..bab8365 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/syntax/BUILD
@@ -41,6 +41,7 @@
"DotExpression.java",
"Expression.java",
"ExpressionStatement.java",
+ "FileOptions.java",
"FlowStatement.java",
"ForStatement.java",
"FunctionSignature.java",
@@ -61,7 +62,6 @@
"ReturnStatement.java",
"SliceExpression.java",
"StarlarkFile.java",
- "StarlarkSemantics.java",
"Statement.java",
"StringLiteral.java",
"SyntaxError.java",
@@ -124,6 +124,7 @@
"StarlarkFunction.java",
"StarlarkIterable.java",
"StarlarkList.java",
+ "StarlarkSemantics.java",
"StarlarkThread.java",
"StarlarkValue.java",
"StringModule.java",
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Eval.java b/src/main/java/com/google/devtools/build/lib/syntax/Eval.java
index c9e09b5..4a367c0 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/Eval.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Eval.java
@@ -168,14 +168,6 @@
private static void execLoad(StarlarkThread.Frame fr, LoadStatement node) throws EvalException {
for (LoadStatement.Binding binding : node.getBindings()) {
- Identifier orig = binding.getOriginalName();
-
- // TODO(adonovan): make this a static check.
- if (orig.isPrivate() && !node.mayLoadInternalSymbols()) {
- throw new EvalException(
- orig.getStartLocation(),
- "symbol '" + orig.getName() + "' is private and cannot be imported.");
- }
// Load module.
String moduleName = node.getImport().getValue();
@@ -190,6 +182,7 @@
}
// Extract symbol.
+ Identifier orig = binding.getOriginalName();
Object value = module.getBindings().get(orig.getName());
if (value == null) {
throw new EvalException(
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/EvalUtils.java b/src/main/java/com/google/devtools/build/lib/syntax/EvalUtils.java
index 2fa19df..d9ee4ae 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/EvalUtils.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/EvalUtils.java
@@ -745,28 +745,25 @@
}
/**
- * Parses the input as a file, validates it in the module environment using options defined by
- * {@code thread.getSemantics}, and returns the syntax tree. It uses Starlark (not BUILD)
- * validation semantics.
- *
- * <p>The thread is primarily used for its Module. Scan/parse/validate errors are recorded in the
- * StarlarkFile. It is the caller's responsibility to inspect them.
+ * Parses the input as a file, validates it in the module environment using the specified options
+ * and returns the syntax tree. Scan/parse/validate errors are recorded in the StarlarkFile. It is
+ * the caller's responsibility to inspect them.
*/
public static StarlarkFile parseAndValidate(
- ParserInput input, Module module, StarlarkSemantics semantics) {
- StarlarkFile file = StarlarkFile.parse(input);
- ValidationEnvironment.validateFile(file, module, semantics, /*isBuildFile=*/ false);
+ ParserInput input, FileOptions options, Module module) {
+ StarlarkFile file = StarlarkFile.parse(input, options);
+ ValidationEnvironment.validateFile(file, module);
return file;
}
/**
- * Parses the input as a file, validates it in the module environment using options defined by
- * {@code thread.getSemantics}, and executes it. It uses Starlark (not BUILD) validation
- * semantics.
+ * Parses the input as a file, validates it in the module environment using the specified options
+ * and executes it.
*/
- public static void exec(ParserInput input, Module module, StarlarkThread thread)
+ public static void exec(
+ ParserInput input, FileOptions options, Module module, StarlarkThread thread)
throws SyntaxError, EvalException, InterruptedException {
- StarlarkFile file = parseAndValidate(input, module, thread.getSemantics());
+ StarlarkFile file = parseAndValidate(input, options, module);
if (!file.ok()) {
throw new SyntaxError(file.errors());
}
@@ -791,14 +788,14 @@
}
/**
- * Parses the input as an expression, validates it in the module environment using options defined
- * by {@code thread.getSemantics}, and evaluates it. It uses Starlark (not BUILD) validation
- * semantics.
+ * Parses the input as an expression, validates it in the module environment using the specified
+ * options, and evaluates it.
*/
- public static Object eval(ParserInput input, Module module, StarlarkThread thread)
+ public static Object eval(
+ ParserInput input, FileOptions options, Module module, StarlarkThread thread)
throws SyntaxError, EvalException, InterruptedException {
- Expression expr = Expression.parse(input);
- ValidationEnvironment.validateExpr(expr, module, thread.getSemantics());
+ Expression expr = Expression.parse(input, options);
+ ValidationEnvironment.validateExpr(expr, module, options);
// Turn expression into a no-arg StarlarkFunction and call it.
StarlarkFunction fn =
@@ -825,10 +822,10 @@
*/
@Nullable
public static Object execAndEvalOptionalFinalExpression(
- ParserInput input, Module module, StarlarkThread thread)
+ ParserInput input, FileOptions options, Module module, StarlarkThread thread)
throws SyntaxError, EvalException, InterruptedException {
- StarlarkFile file = StarlarkFile.parse(input);
- ValidationEnvironment.validateFile(file, module, thread.getSemantics(), /*isBuildFile=*/ false);
+ StarlarkFile file = StarlarkFile.parse(input, options);
+ ValidationEnvironment.validateFile(file, module);
if (!file.ok()) {
throw new SyntaxError(file.errors());
}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Expression.java b/src/main/java/com/google/devtools/build/lib/syntax/Expression.java
index 8bd5bf6..d730763 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/Expression.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Expression.java
@@ -51,9 +51,14 @@
*/
public abstract Kind kind();
- /** Parses an expression. */
+ /** Parses an expression with the default options. */
public static Expression parse(ParserInput input) throws SyntaxError {
- return Parser.parseExpression(input);
+ return parse(input, FileOptions.DEFAULT);
+ }
+
+ /** Parses an expression. */
+ public static Expression parse(ParserInput input, FileOptions options) throws SyntaxError {
+ return Parser.parseExpression(input, options);
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/FileOptions.java b/src/main/java/com/google/devtools/build/lib/syntax/FileOptions.java
new file mode 100644
index 0000000..99be932
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/FileOptions.java
@@ -0,0 +1,116 @@
+// Copyright 2020 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.auto.value.AutoValue;
+
+/**
+ * FileOptions is a set of options that affect the static processing---scanning, parsing, validation
+ * (identifier resolution), and compilation---of a single Starlark file. These options affect the
+ * language accepted by the frontend (in effect, the dialect), and "code generation", analogous to
+ * the command-line options of a typical compiler.
+ *
+ * <p>Different files within the same application and even executed within the same thread may be
+ * subject to different file options. For example, in Bazel, load statements in WORKSPACE files may
+ * need to be interleaved with other statements, whereas in .bzl files, load statements must appear
+ * all at the top. A single thread may execute a WORKSPACE file and call functions defined in .bzl
+ * files.
+ *
+ * <p>The {@link #DEFAULT} options represent the desired behavior for new uses of Starlark. It is a
+ * goal to keep this set of options small and closed. Each represents a language feature, perhaps a
+ * deprecated, obscure, or regrettable one. By contrast, {@link StarlarkSemantics} defines a
+ * (soon-to-be) open-ended set of options that affect the dynamic behavior of Starlark threads and
+ * (mostly application-defined) built-in functions, and particularly attribute selection operations
+ * {@code x.f}.
+ */
+@AutoValue
+public abstract class FileOptions {
+
+ /** The default options for Starlark static processing. New clients should use these defaults. */
+ public static final FileOptions DEFAULT = builder().build();
+
+ // Options are presented in phase order: scanner, parser, validator, compiler.
+
+ // --- scanner options ---
+
+ /** Disallow ineffective escape sequences such as {@code \a} when scanning string literals. */
+ public abstract boolean restrictStringEscapes();
+
+ // --- validator options ---
+
+ /**
+ * During resolution, permit load statements to access private names such as {@code _x}. <br>
+ * (Required for continued support of Bazel "WORKSPACE.resolved" files.)
+ */
+ public abstract boolean allowLoadPrivateSymbols();
+
+ /**
+ * During resolution, permit multiple bindings of top-level variables. <br>
+ * (Required for continued support of Bazel BUILD files and Copybara files.)
+ */
+ public abstract boolean allowToplevelRebinding();
+
+ // TODO(adonovan): implement this option to support the REPL and prelude.
+ //
+ // /**
+ // * During resolution, make load statements bind global variables of the module, not file-local
+ // * variables. (Intended for use in REPLs, and the prelude.)
+ // */
+ // public abstract boolean loadBindsGlobally();
+
+ /**
+ * During resolution, require load statements to appear before other kinds of statements. <br>
+ * (Required for continued support of Bazel BUILD and especially WORKSPACE files.)
+ */
+ public abstract boolean requireLoadStatementsFirst();
+
+ /**
+ * Record the results of name resolution in the syntax tree by setting {@code Identifer.scope}.
+ * (Disabled for Bazel BUILD files, as its prelude's syntax trees are shared.)
+ */
+ public abstract boolean recordScope();
+
+ public static Builder builder() {
+ // These are the DEFAULT values.
+ return new AutoValue_FileOptions.Builder()
+ .restrictStringEscapes(true)
+ .allowLoadPrivateSymbols(false)
+ .allowToplevelRebinding(false)
+ // .loadBindsGlobally(false)
+ .requireLoadStatementsFirst(true)
+ .recordScope(true);
+ }
+
+ public abstract Builder toBuilder();
+
+ /** This javadoc comment states that FileOptions.Builder is a builder for FileOptions. */
+ @AutoValue.Builder
+ public abstract static class Builder {
+ // AutoValue why u make me say it 3 times?
+ public abstract Builder restrictStringEscapes(boolean value);
+
+ public abstract Builder allowLoadPrivateSymbols(boolean value);
+
+ public abstract Builder allowToplevelRebinding(boolean value);
+
+ // public abstract Builder loadBindsGlobally(boolean value);
+
+ public abstract Builder requireLoadStatementsFirst(boolean value);
+
+ public abstract Builder recordScope(boolean value);
+
+ public abstract FileOptions build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/FlagGuardedValue.java b/src/main/java/com/google/devtools/build/lib/syntax/FlagGuardedValue.java
index 59d70dd..2c7bfd2 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/FlagGuardedValue.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/FlagGuardedValue.java
@@ -14,8 +14,6 @@
package com.google.devtools.build.lib.syntax;
-import com.google.common.base.Preconditions;
-
/**
* Wrapper on a value that controls its accessibility in Starlark based on the value of a
* semantic flag.
@@ -62,15 +60,9 @@
/**
* Returns an error describing an attempt to access this guard's protected object when it should
- * be inaccessible in the given semantics.
- *
- * @throws IllegalArgumentException if {@link #isObjectAccessibleUsingSemantics} is true given the
- * semantics
+ * be inaccessible in the (contextually implied) semantics.
*/
- String getErrorFromAttemptingAccess(StarlarkSemantics semantics, String name) {
- Preconditions.checkArgument(!isObjectAccessibleUsingSemantics(semantics),
- "getEvalExceptionFromAttemptingAccess should only be called if the underlying "
- + "object is inaccessible given the semantics");
+ String getErrorFromAttemptingAccess(String name) {
return flagType == FlagType.EXPERIMENTAL
? name
+ " is experimental and thus unavailable with the current flags. It may be enabled by"
@@ -85,20 +77,14 @@
/**
* Returns this guard's underlying object. This should be called when appropriate validation has
- * occurred to ensure that the object is accessible with the given semantics.
- *
- * @throws IllegalArgumentException if {@link #isObjectAccessibleUsingSemantics} is false given
- * the semantics
+ * occurred to ensure that the object is accessible with the (implied) semantics.
*/
- public Object getObject(StarlarkSemantics semantics) {
- Preconditions.checkArgument(isObjectAccessibleUsingSemantics(semantics),
- "getObject should only be called if the underlying object is accessible given the "
- + "semantics");
+ public Object getObject() {
return obj;
}
/** Returns true if this guard's underlying object is accessible under the given semantics. */
- public boolean isObjectAccessibleUsingSemantics(StarlarkSemantics semantics) {
+ boolean isObjectAccessibleUsingSemantics(StarlarkSemantics semantics) {
if (flagType == FlagType.EXPERIMENTAL) {
return semantics.isFeatureEnabledBasedOnTogglingFlags(flag, "");
} else {
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
index 34d19f1..f7e7a86 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
@@ -51,6 +51,8 @@
private final char[] buffer;
private int pos;
+ private final FileOptions options;
+
private final LineNumberTable lnt; // maps offsets to Locations
// The stack of enclosing indentation levels; always contains '0' at the
@@ -81,17 +83,10 @@
private int dents; // number of saved INDENT (>0) or OUTDENT (<0) tokens to return
- /**
- * StringEscapeEvents contains the errors related to invalid escape sequences like "\a". This is
- * not handled by the normal eventHandler. Instead, it is passed to the parser and then the AST.
- * During the evaluation, we can decide to show the events based on a flag in StarlarkSemantics.
- * This code is temporary, during the migration.
- */
- private final List<Event> stringEscapeEvents = new ArrayList<>();
-
/** Constructs a lexer which tokenizes the parser input. Errors are appended to {@code errors}. */
- Lexer(ParserInput input, List<Event> errors) {
+ Lexer(ParserInput input, FileOptions options, List<Event> errors) {
this.lnt = LineNumberTable.create(input.getContent(), input.getFile());
+ this.options = options;
this.buffer = input.getContent();
this.pos = 0;
this.errors = errors;
@@ -107,10 +102,6 @@
return comments;
}
- List<Event> getStringEscapeEvents() {
- return stringEscapeEvents;
- }
-
/** Returns the apparent name of the lexer's input file. */
String getFile() {
return lnt.getFile();
@@ -424,14 +415,15 @@
break;
default:
// unknown char escape => "\literal"
- stringEscapeEvents.add(
- Event.error(
- createLocation(pos - 1, pos),
- "invalid escape sequence: \\"
- + c
- + ". You can enable unknown escape sequences by passing the flag "
- + "--incompatible_restrict_string_escapes=false"));
-
+ if (options.restrictStringEscapes()) {
+ errors.add(
+ Event.error(
+ createLocation(pos - 1, pos),
+ "invalid escape sequence: \\"
+ + c
+ + ". You can enable unknown escape sequences by passing the flag "
+ + "--incompatible_restrict_string_escapes=false"));
+ }
literal.append('\\');
literal.append(c);
break;
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/LoadStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/LoadStatement.java
index 7d0e7e2..a73af95 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/LoadStatement.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/LoadStatement.java
@@ -42,8 +42,7 @@
}
private final ImmutableList<Binding> bindings;
- private final StringLiteral imp;
- private final boolean mayLoadInternalSymbols;
+ private final StringLiteral module;
/**
* Constructs an import statement.
@@ -51,29 +50,10 @@
* <p>{@code bindings} maps a symbol to the original name under which it was defined in the bzl
* file that should be loaded. If aliasing is used, the value differs from its key's {@code
* symbol.getName()}. Otherwise, both values are identical.
- *
- * <p>Import statements generated this way are bound to the usual restriction that private symbols
- * cannot be loaded.
*/
- LoadStatement(StringLiteral imp, ImmutableList<Binding> bindings) {
- this.imp = imp;
+ LoadStatement(StringLiteral module, ImmutableList<Binding> bindings) {
+ this.module = module;
this.bindings = bindings;
- this.mayLoadInternalSymbols = false;
- }
-
- private LoadStatement(
- StringLiteral imp, ImmutableList<Binding> bindings, boolean mayLoadInternalSymbols) {
- this.imp = imp;
- this.bindings = bindings;
- this.mayLoadInternalSymbols = mayLoadInternalSymbols;
- }
-
- /**
- * Out of a {@code LoadStatement} construct a new one loading the same symbols, but free from the
- * usual visibility restriction of not being able to load private symbols.
- */
- public static LoadStatement allowLoadingOfInternalSymbols(LoadStatement load) {
- return new LoadStatement(load.getImport(), load.getBindings(), true);
}
public ImmutableList<Binding> getBindings() {
@@ -81,15 +61,7 @@
}
public StringLiteral getImport() {
- return imp;
- }
-
- /**
- * Indicate whether this import statement is exempt from the restriction that private symbols may
- * not be loaded.
- */
- public boolean mayLoadInternalSymbols() {
- return mayLoadInternalSymbols;
+ return module;
}
@Override
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Module.java b/src/main/java/com/google/devtools/build/lib/syntax/Module.java
index 96e5be8..976e07e 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/Module.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Module.java
@@ -175,7 +175,7 @@
if (binding.getValue() instanceof FlagGuardedValue) {
FlagGuardedValue val = (FlagGuardedValue) binding.getValue();
if (val.isObjectAccessibleUsingSemantics(semantics)) {
- filteredBindings.put(binding.getKey(), val.getObject(semantics));
+ filteredBindings.put(binding.getKey(), val.getObject());
} else {
restrictedBindings.put(binding.getKey(), val);
}
@@ -187,7 +187,7 @@
restrictedBindings.putAll(parent.restrictedBindings);
return new Module(
- mutability, null /*parent */, parent.label, filteredBindings, restrictedBindings);
+ mutability, /*universe=*/ null, parent.label, filteredBindings, restrictedBindings);
}
private void checkInitialized() {
@@ -223,7 +223,8 @@
*/
public Module withLabel(Object label) {
checkInitialized();
- return new Module(mutability, /*universe*/ null, label, bindings, /*restrictedBindings*/ null);
+ return new Module(
+ mutability, /*universe=*/ null, label, bindings, /*restrictedBindings=*/ null);
}
/** Returns the {@link Mutability} of this {@link Module}. */
@@ -290,9 +291,9 @@
}
@Override
- public String getUndeclaredNameError(StarlarkSemantics semantics, String name) {
+ public String getUndeclaredNameError(String name) {
FlagGuardedValue v = restrictedBindings.get(name);
- return v == null ? null : v.getErrorFromAttemptingAccess(semantics, name);
+ return v == null ? null : v.getErrorFromAttemptingAccess(name);
}
/** Returns an environment containing both module and predeclared bindings. */
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ParamDescriptor.java b/src/main/java/com/google/devtools/build/lib/syntax/ParamDescriptor.java
index b05c709..a6be014 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/ParamDescriptor.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ParamDescriptor.java
@@ -215,7 +215,7 @@
// See https://docs.oracle.com/javase/specs/jls/se12/html/jls-12.html#jls-12.4
thread.ignoreThreadInterrupts();
- x = EvalUtils.eval(ParserInput.fromLines(expr), module, thread);
+ x = EvalUtils.eval(ParserInput.fromLines(expr), FileOptions.DEFAULT, module, thread);
} catch (InterruptedException ex) {
throw new IllegalStateException(ex); // can't happen
} catch (SyntaxError | EvalException ex) {
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Parser.java b/src/main/java/com/google/devtools/build/lib/syntax/Parser.java
index e657695..01fe5d2 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/Parser.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Parser.java
@@ -53,21 +53,18 @@
// Errors encountered during scanning or parsing.
// These lists are ultimately owned by StarlarkFile.
final List<Event> errors;
- final List<Event> stringEscapeEvents;
ParseResult(
List<Statement> statements,
List<Comment> comments,
Lexer.LexerLocation location,
- List<Event> errors,
- List<Event> stringEscapeEvents) {
+ List<Event> errors) {
// No need to copy here; when the object is created, the parser instance is just about to go
// out of scope and be garbage collected.
this.statements = Preconditions.checkNotNull(statements);
this.comments = Preconditions.checkNotNull(comments);
this.location = location;
this.errors = errors;
- this.stringEscapeEvents = stringEscapeEvents;
}
}
@@ -181,9 +178,9 @@
}
// Main entry point for parsing a file.
- static ParseResult parseFile(ParserInput input) {
+ static ParseResult parseFile(ParserInput input, FileOptions options) {
List<Event> errors = new ArrayList<>();
- Lexer lexer = new Lexer(input, errors);
+ Lexer lexer = new Lexer(input, options, errors);
Parser parser = new Parser(lexer, errors);
List<Statement> statements;
try (SilentCloseable c =
@@ -191,11 +188,7 @@
statements = parser.parseFileInput();
}
return new ParseResult(
- statements,
- lexer.getComments(),
- locationFromStatements(lexer, statements),
- errors,
- lexer.getStringEscapeEvents());
+ statements, lexer.getComments(), locationFromStatements(lexer, statements), errors);
}
// stmt ::= simple_stmt
@@ -215,9 +208,9 @@
}
/** Parses an expression, possibly followed by newline tokens. */
- static Expression parseExpression(ParserInput input) throws SyntaxError {
+ static Expression parseExpression(ParserInput input, FileOptions options) throws SyntaxError {
List<Event> errors = new ArrayList<>();
- Lexer lexer = new Lexer(input, errors);
+ Lexer lexer = new Lexer(input, options, errors);
Parser parser = new Parser(lexer, errors);
Expression result = parser.parseExpression();
while (parser.token.kind == TokenKind.NEWLINE) {
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/StarlarkFile.java b/src/main/java/com/google/devtools/build/lib/syntax/StarlarkFile.java
index bd6c16d..c009711 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/StarlarkFile.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/StarlarkFile.java
@@ -30,68 +30,51 @@
public final class StarlarkFile extends Node {
private final ImmutableList<Statement> statements;
+ private final FileOptions options;
private final ImmutableList<Comment> comments;
final List<Event> errors; // appended to by ValidationEnvironment
- private final List<Event> stringEscapeEvents;
@Nullable private final String contentHashCode;
private StarlarkFile(
ImmutableList<Statement> statements,
+ FileOptions options,
+ ImmutableList<Comment> comments,
List<Event> errors,
String contentHashCode,
- Lexer.LexerLocation location,
- ImmutableList<Comment> comments,
- List<Event> stringEscapeEvents) {
+ Lexer.LexerLocation location) {
this.statements = statements;
+ this.options = options;
this.comments = comments;
this.errors = errors;
- this.stringEscapeEvents = stringEscapeEvents;
this.contentHashCode = contentHashCode;
this.setLocation(location);
}
+ // Creates a StarlarkFile from the given effective list of statements,
+ // which may include the prelude.
private static StarlarkFile create(
- List<Statement> preludeStatements,
+ ImmutableList<Statement> statements,
+ FileOptions options,
Parser.ParseResult result,
- String contentHashCode,
- boolean allowImportInternal) {
- ImmutableList.Builder<Statement> statementsbuilder =
- ImmutableList.<Statement>builder().addAll(preludeStatements);
-
- if (allowImportInternal) {
- for (Statement stmt : result.statements) {
- if (stmt instanceof LoadStatement) {
- statementsbuilder.add(LoadStatement.allowLoadingOfInternalSymbols((LoadStatement) stmt));
- } else {
- statementsbuilder.add(stmt);
- }
- }
- } else {
- statementsbuilder.addAll(result.statements);
- }
- ImmutableList<Statement> statements = statementsbuilder.build();
+ String contentHashCode) {
return new StarlarkFile(
statements,
+ options,
+ ImmutableList.copyOf(result.comments),
result.errors,
contentHashCode,
- result.location,
- ImmutableList.copyOf(result.comments),
- result.stringEscapeEvents);
+ result.location);
}
- /**
- * Extract a subtree containing only statements from {@code firstStatement} (included) up to
- * {@code lastStatement} excluded.
- */
- public StarlarkFile subTree(int firstStatement, int lastStatement) {
- ImmutableList<Statement> statements = this.statements.subList(firstStatement, lastStatement);
+ /** Extract a subtree containing only statements from i (included) to j (excluded). */
+ public StarlarkFile subTree(int i, int j) {
return new StarlarkFile(
- statements,
+ this.statements.subList(i, j),
+ this.options,
+ /*comments=*/ ImmutableList.of(),
errors,
- null,
- (Lexer.LexerLocation) this.statements.get(firstStatement).getStartLocation(),
- ImmutableList.of(),
- stringEscapeEvents);
+ /*contentHashCode=*/ null,
+ (Lexer.LexerLocation) this.statements.get(i).getStartLocation());
}
/**
@@ -107,18 +90,6 @@
return errors.isEmpty();
}
- /**
- * Appends string escaping errors to {@code errors}. The Lexer diverts such errors into a separate
- * bucket as they should be selectively reported depending on a StarlarkSemantics, to which the
- * lexer/parser does not have access. This function is called by ValidationEnvironment, which has
- * access to a StarlarkSemantics and can thus decide whether to respect or ignore these events.
- *
- * <p>Naturally this function should be called at most once.
- */
- void addStringEscapeEvents() {
- errors.addAll(stringEscapeEvents);
- }
-
/** Returns an (immutable, ordered) list of statements in this BUILD file. */
public ImmutableList<Statement> getStatements() {
return statements;
@@ -140,36 +111,29 @@
}
/**
- * Parse the specified file, returning its syntax tree with the preludeStatements inserted at the
+ * Parse the specified file, returning its syntax tree with the prelude statements inserted at the
* front of its statement list.
*/
public static StarlarkFile parseWithPrelude(
- ParserInput input, List<Statement> preludeStatements) {
- Parser.ParseResult result = Parser.parseFile(input);
- return create(
- preludeStatements, result, /* contentHashCode= */ null, /*allowImportInternal=*/ false);
- }
+ ParserInput input, List<Statement> prelude, FileOptions options) {
+ Parser.ParseResult result = Parser.parseFile(input, options);
- /**
- * Parse the specified build file, returning its AST. All load statements parsed that way will be
- * exempt from visibility restrictions.
- */
- // TODO(adonovan): make LoadStatement.allowInternal publicly settable, and delete this.
- public static StarlarkFile parseVirtualBuildFile(
- ParserInput input, List<Statement> preludeStatements) {
- Parser.ParseResult result = Parser.parseFile(input);
- return create(
- preludeStatements, result, /* contentHashCode= */ null, /*allowImportInternal=*/ true);
+ ImmutableList.Builder<Statement> stmts = ImmutableList.builder();
+ stmts.addAll(prelude);
+ stmts.addAll(result.statements);
+
+ return create(stmts.build(), options, result, /*contentHashCode=*/ null);
}
// TODO(adonovan): make the digest publicly settable, and delete this.
- public static StarlarkFile parseWithDigest(ParserInput input, byte[] digest) throws IOException {
- Parser.ParseResult result = Parser.parseFile(input);
+ public static StarlarkFile parseWithDigest(ParserInput input, byte[] digest, FileOptions options)
+ throws IOException {
+ Parser.ParseResult result = Parser.parseFile(input, options);
return create(
- /* preludeStatements= */ ImmutableList.of(),
+ ImmutableList.copyOf(result.statements),
+ options,
result,
- HashCode.fromBytes(digest).toString(),
- /* allowImportInternal= */ false);
+ HashCode.fromBytes(digest).toString());
}
/**
@@ -179,20 +143,27 @@
* Example usage:
*
* <pre>
- * StarlarkFile file = StarlarkFile.parse(input);
+ * StarlarkFile file = StarlarkFile.parse(input, options);
* if (!file.ok()) {
* Event.replayEventsOn(handler, file.errors());
* ...
* }
* </pre>
*/
- public static StarlarkFile parse(ParserInput input) {
- Parser.ParseResult result = Parser.parseFile(input);
+ public static StarlarkFile parse(ParserInput input, FileOptions options) {
+ Parser.ParseResult result = Parser.parseFile(input, options);
return create(
- /* preludeStatements= */ ImmutableList.of(),
- result,
- /* contentHashCode= */ null,
- /* allowImportInternal=*/ false);
+ ImmutableList.copyOf(result.statements), options, result, /*contentHashCode=*/ null);
+ }
+
+ /** Parse a Starlark file with default options. */
+ public static StarlarkFile parse(ParserInput input) {
+ return parse(input, FileOptions.DEFAULT);
+ }
+
+ /** Returns the options specified when parsing this file. */
+ public FileOptions getOptions() {
+ return options;
}
/**
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/StarlarkSemantics.java b/src/main/java/com/google/devtools/build/lib/syntax/StarlarkSemantics.java
index d1b04ad..a69bf73 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/StarlarkSemantics.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/StarlarkSemantics.java
@@ -22,15 +22,37 @@
import java.util.List;
/**
- * Options that affect Starlark semantics.
+ * Options that affect the dynamic behavior of Starlark execution and operators.
*
- * <p>For descriptions of what these options do, see {@link StarlarkSemanticsOptions}.
+ * <p>For descriptions of what these options do, see {@link packages.StarlarkSemanticsOptions}.
+ *
+ * <p>For options that affect the static behavior of the Starlark frontend (lexer, parser,
+ * validator, compiler), see FileOptions.
*/
// TODO(brandjon): User error messages that reference options should maybe be substituted with the
// option name outside of the core Starlark interpreter?
// TODO(brandjon): Eventually these should be documented in full here, and StarlarkSemanticsOptions
// should refer to this class for documentation. But this doesn't play nice with the options
// parser's annotation mechanism.
+//
+// TODO(adonovan): nearly all of these options are Bazel-isms.
+// The only ones that affect the Starlark interpreter directly are are:
+// - incompatibleRestrictNamedParams, which affects calls to many built-ins,
+// but is used only by copybara and will be deleted soon (CL 298871155);
+// - incompatibleRestrictStringEscapes, which affects the lexer and is thus
+// properly one of the FileOptions, but piggybacks on the command-line flag
+// plumbing of StarlarkSemantics; and
+// - internalSkylarkFlagTestCanary, which is used to test propagation of Bazel
+// command-line flags to the 'print' built-in, but this could easily be
+// achieved using some other Bazel-specific built-in.
+// Most of the rest are used generically to disable parameters to built-ins,
+// or to disable fields of modules, based on flags. In both of those cases,
+// a generic set-of-feature-strings representation would do.
+// A few could be expressed as Bazel-specific thread state,
+// though several are inspected by the implementations of operations
+// such as SkylarkIndexable, SkylarkQueryable, and SkylarkClassObject.
+// TODO(adonovan): move to lib.packages.BuildLanguageSemantics.
+//
@AutoValue
public abstract class StarlarkSemantics {
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ValidationEnvironment.java b/src/main/java/com/google/devtools/build/lib/syntax/ValidationEnvironment.java
index 15afd2a..3de6a0f 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/ValidationEnvironment.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ValidationEnvironment.java
@@ -79,18 +79,17 @@
//
// A single method will then suffice:
// Scope resolve(String name) throws Undeclared
- // This requires that the Module retain its semantics.
/** Returns the set of names defined by this module. The caller must not modify the set. */
Set<String> getNames();
/**
* Returns (optionally) a more specific error for an undeclared name than the generic message.
- * This hook allows the module to implement "semantics-restricted" names without any knowledge
- * in this file.
+ * This hook allows the module to implement flag-enabled names without any knowledge in this
+ * file.
*/
@Nullable
- String getUndeclaredNameError(StarlarkSemantics semantics, String name);
+ String getUndeclaredNameError(String name);
}
private static final Identifier PREDECLARED = new Identifier("");
@@ -107,25 +106,15 @@
}
private final List<Event> errors;
- private final StarlarkSemantics semantics;
+ private final FileOptions options;
private final Module module;
private Block block;
private int loopCount;
- // In BUILD files, we have a slightly different behavior for legacy reasons.
- // TODO(adonovan): eliminate isBuildFile. It is necessary because the prelude is implemented
- // by inserting shared Statements, which must not be mutated, into each StarlarkFile.
- // Instead, we should implement the prelude by executing it like a .bzl module
- // and putting its members in the initial environment of the StarlarkFile.
- // In the meantime, let's move this flag into Module (GlobalFrame).
- private final boolean isBuildFile;
-
- private ValidationEnvironment(
- List<Event> errors, Module module, StarlarkSemantics semantics, boolean isBuildFile) {
+ private ValidationEnvironment(List<Event> errors, Module module, FileOptions options) {
this.errors = errors;
this.module = module;
- this.semantics = semantics;
- this.isBuildFile = isBuildFile;
+ this.options = options;
block = new Block(Scope.Universe, null);
for (String name : module.getNames()) {
block.variables.put(name, PREDECLARED);
@@ -170,13 +159,19 @@
break;
case LOAD:
LoadStatement load = (LoadStatement) stmt;
-
- // The global reassignment check is not yet enabled for BUILD files,
- // but we apply it to load statements as a special case.
- // Because (for now) its error message is better than the general
- // message emitted by 'declare', we'll apply it to non-BUILD files too.
Set<String> names = new HashSet<>();
for (LoadStatement.Binding b : load.getBindings()) {
+ // Reject load('...', '_private').
+ Identifier orig = b.getOriginalName();
+ if (orig.isPrivate() && !options.allowLoadPrivateSymbols()) {
+ addError(
+ orig.getStartLocation(),
+ "symbol '" + orig.getName() + "' is private and cannot be imported.");
+ }
+
+ // The allowToplevelRebinding check is not applied to all files
+ // but we apply it to each load statement as a special case,
+ // and emit a better error message than the generic check.
if (!names.add(b.getLocalName().getName())) {
addError(
b.getLocalName().getStartLocation(),
@@ -185,6 +180,11 @@
}
}
+ // TODO(adonovan): support options.loadBindsGlobally().
+ // Requires that we open a Local block for each file,
+ // as well as its Module block, and select which block
+ // to declare it in. See go.starlark.net implementation.
+
for (LoadStatement.Binding b : load.getBindings()) {
declare(b.getLocalName());
}
@@ -204,7 +204,7 @@
private void assign(Expression lhs) {
if (lhs instanceof Identifier) {
- if (!isBuildFile) {
+ if (options.recordScope()) {
((Identifier) lhs).setScope(block.scope);
}
// no-op
@@ -224,9 +224,9 @@
String name = node.getName();
@Nullable Block b = blockThatDefines(name);
if (b == null) {
- // The identifier might not exist because it was restricted (hidden) by the current semantics.
+ // The identifier might not exist because it was restricted (hidden) by flags.
// If this is the case, output a more helpful error message than 'not found'.
- String error = module.getUndeclaredNameError(semantics, name);
+ String error = module.getUndeclaredNameError(name);
if (error == null) {
// generic error
error = createInvalidIdentifierException(node.getName(), getAllSymbols());
@@ -234,9 +234,7 @@
addError(node.getStartLocation(), error);
return;
}
- // TODO(laurentlb): In BUILD files, calling setScope will throw an exception. This happens
- // because some AST nodes are shared across multipe ASTs (due to the prelude file).
- if (!isBuildFile) {
+ if (options.recordScope()) {
node.setScope(b.scope);
}
}
@@ -393,8 +391,7 @@
Identifier prev = block.variables.putIfAbsent(id.getName(), id);
// Symbols defined in the module scope cannot be reassigned.
- // TODO(laurentlb): Forbid reassignment in BUILD files too.
- if (prev != null && block.scope == Scope.Module && !isBuildFile) {
+ if (prev != null && block.scope == Scope.Module && !options.allowToplevelRebinding()) {
addError(
id.getStartLocation(),
String.format(
@@ -457,7 +454,7 @@
private void validateToplevelStatements(List<Statement> statements) {
// Check that load() statements are on top.
- if (!isBuildFile) {
+ if (options.requireLoadStatementsFirst()) {
checkLoadAfterStatement(statements);
}
@@ -475,16 +472,10 @@
/**
* Performs static checks, including resolution of identifiers in {@code file} in the environment
* defined by {@code module}. The StarlarkFile is mutated. Errors are appended to {@link
- * StarlarkFile#errors}. {@code isBuildFile} enables Bazel's legacy mode for BUILD files in which
- * reassignment at top-level is permitted.
+ * StarlarkFile#errors}.
*/
- public static void validateFile(
- StarlarkFile file, Module module, StarlarkSemantics semantics, boolean isBuildFile) {
- ValidationEnvironment venv =
- new ValidationEnvironment(file.errors, module, semantics, isBuildFile);
- if (semantics.incompatibleRestrictStringEscapes()) {
- file.addStringEscapeEvents();
- }
+ public static void validateFile(StarlarkFile file, Module module) {
+ ValidationEnvironment venv = new ValidationEnvironment(file.errors, module, file.getOptions());
venv.validateToplevelStatements(file.getStatements());
// Check that no closeBlock was forgotten.
Preconditions.checkState(venv.block.parent == null);
@@ -494,11 +485,10 @@
* Performs static checks, including resolution of identifiers in {@code expr} in the environment
* defined by {@code module}. This operation mutates the Expression.
*/
- public static void validateExpr(Expression expr, Module module, StarlarkSemantics semantics)
+ public static void validateExpr(Expression expr, Module module, FileOptions options)
throws SyntaxError {
List<Event> errors = new ArrayList<>();
- ValidationEnvironment venv =
- new ValidationEnvironment(errors, module, semantics, /*isBuildFile=*/ false);
+ ValidationEnvironment venv = new ValidationEnvironment(errors, module, options);
venv.visit(expr);
diff --git a/src/main/java/com/google/devtools/starlark/cmd/Starlark.java b/src/main/java/com/google/devtools/starlark/cmd/Starlark.java
index 91ea8c6..5cc353d 100644
--- a/src/main/java/com/google/devtools/starlark/cmd/Starlark.java
+++ b/src/main/java/com/google/devtools/starlark/cmd/Starlark.java
@@ -16,6 +16,7 @@
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.EvalUtils;
+import com.google.devtools.build.lib.syntax.FileOptions;
import com.google.devtools.build.lib.syntax.Module;
import com.google.devtools.build.lib.syntax.Mutability;
import com.google.devtools.build.lib.syntax.ParserInput;
@@ -46,6 +47,10 @@
private final StarlarkThread thread;
private final Module module;
+ // TODO(adonovan): set load-binds-globally option when we support load,
+ // so that loads bound in one REPL chunk are visible in the next.
+ private final FileOptions options = FileOptions.DEFAULT;
+
{
thread =
StarlarkThread.builder(Mutability.create("interpreter"))
@@ -94,7 +99,8 @@
while ((line = prompt()) != null) {
ParserInput input = ParserInput.create(line, "<stdin>");
try {
- Object result = EvalUtils.execAndEvalOptionalFinalExpression(input, module, thread);
+ Object result =
+ EvalUtils.execAndEvalOptionalFinalExpression(input, options, module, thread);
if (result != null) {
System.out.println(com.google.devtools.build.lib.syntax.Starlark.repr(result));
}
@@ -125,7 +131,7 @@
/** Execute a Starlark file. */
private int execute(String filename, String content) {
try {
- EvalUtils.exec(ParserInput.create(content, filename), module, thread);
+ EvalUtils.exec(ParserInput.create(content, filename), options, module, thread);
return 0;
} catch (SyntaxError ex) {
for (Event ev : ex.errors()) {
diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD
index 6796acd..7a5ada8 100644
--- a/src/test/java/com/google/devtools/build/lib/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/BUILD
@@ -92,7 +92,7 @@
":AllTests",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/cmdline:LabelValidator",
- "//src/main/java/com/google/devtools/build/lib/syntax:frontend",
+ "//src/main/java/com/google/devtools/build/lib/syntax:evaluator",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
"//src/test/java/com/google/devtools/build/lib/testutil",
"//src/test/java/com/google/devtools/build/lib/testutil:JunitUtils",
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContextTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContextTest.java
index 331a0e0..86f6555 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContextTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContextTest.java
@@ -42,6 +42,7 @@
import com.google.devtools.build.lib.syntax.Dict;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.EvalUtils;
+import com.google.devtools.build.lib.syntax.FileOptions;
import com.google.devtools.build.lib.syntax.Module;
import com.google.devtools.build.lib.syntax.Mutability;
import com.google.devtools.build.lib.syntax.ParserInput;
@@ -107,7 +108,7 @@
StarlarkThread thread = StarlarkThread.builder(mu).useDefaultSemantics().build();
Module module = thread.getGlobals();
return EvalUtils.execAndEvalOptionalFinalExpression(
- ParserInput.fromLines(lines), module, thread);
+ ParserInput.fromLines(lines), FileOptions.DEFAULT, module, thread);
} catch (Exception ex) { // SyntaxError | EvalException | InterruptedException
throw new AssertionError("exec failed", ex);
}
diff --git a/src/test/java/com/google/devtools/build/lib/packages/SelectTest.java b/src/test/java/com/google/devtools/build/lib/packages/SelectTest.java
index acd0be1..7c632d1 100644
--- a/src/test/java/com/google/devtools/build/lib/packages/SelectTest.java
+++ b/src/test/java/com/google/devtools/build/lib/packages/SelectTest.java
@@ -19,6 +19,7 @@
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.EvalUtils;
+import com.google.devtools.build.lib.syntax.FileOptions;
import com.google.devtools.build.lib.syntax.Module;
import com.google.devtools.build.lib.syntax.Mutability;
import com.google.devtools.build.lib.syntax.ParserInput;
@@ -41,7 +42,7 @@
.useDefaultSemantics()
.build();
Module module = thread.getGlobals();
- return EvalUtils.eval(input, module, thread);
+ return EvalUtils.eval(input, FileOptions.DEFAULT, module, thread);
}
private static void assertFails(String expr, String wantError) {
diff --git a/src/test/java/com/google/devtools/build/lib/profiler/memory/AllocationTrackerTest.java b/src/test/java/com/google/devtools/build/lib/profiler/memory/AllocationTrackerTest.java
index 41fc7c6..b1f30bc 100644
--- a/src/test/java/com/google/devtools/build/lib/profiler/memory/AllocationTrackerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/profiler/memory/AllocationTrackerTest.java
@@ -25,6 +25,7 @@
import com.google.devtools.build.lib.syntax.Debug;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.EvalUtils;
+import com.google.devtools.build.lib.syntax.FileOptions;
import com.google.devtools.build.lib.syntax.HasBinary;
import com.google.devtools.build.lib.syntax.Module;
import com.google.devtools.build.lib.syntax.Mutability;
@@ -188,6 +189,7 @@
}
private void exec(String... lines) throws SyntaxError, EvalException, InterruptedException {
+ ParserInput input = ParserInput.create(Joiner.on("\n").join(lines), "a.star");
Mutability mu = Mutability.create("test");
StarlarkThread thread =
StarlarkThread.builder(mu)
@@ -198,9 +200,8 @@
"sample", new SamplerValue(),
"myrule", new MyRuleFunction())))
.build();
- ParserInput input = ParserInput.create(Joiner.on("\n").join(lines), "a.star");
Module module = thread.getGlobals();
- EvalUtils.exec(input, module, thread);
+ EvalUtils.exec(input, FileOptions.DEFAULT, module, thread);
}
// A fake Bazel rule. The allocation tracker reports retained memory broken down by rule class.
diff --git a/src/test/java/com/google/devtools/build/lib/skylark/SkylarkRuleClassFunctionsTest.java b/src/test/java/com/google/devtools/build/lib/skylark/SkylarkRuleClassFunctionsTest.java
index 7728813..b1ec447 100644
--- a/src/test/java/com/google/devtools/build/lib/skylark/SkylarkRuleClassFunctionsTest.java
+++ b/src/test/java/com/google/devtools/build/lib/skylark/SkylarkRuleClassFunctionsTest.java
@@ -53,6 +53,8 @@
import com.google.devtools.build.lib.syntax.Dict;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.EvalUtils;
+import com.google.devtools.build.lib.syntax.FileOptions;
+import com.google.devtools.build.lib.syntax.Module;
import com.google.devtools.build.lib.syntax.Mutability;
import com.google.devtools.build.lib.syntax.ParserInput;
import com.google.devtools.build.lib.syntax.StarlarkFile;
@@ -729,12 +731,11 @@
assertThat(c.hasAttr("a1", Type.STRING)).isTrue();
}
- // TODO(adonovan): rename execAndExport
private void evalAndExport(String... lines) throws Exception {
ParserInput input = ParserInput.fromLines(lines);
StarlarkThread thread = ev.getStarlarkThread();
- StarlarkFile file =
- EvalUtils.parseAndValidate(input, thread.getGlobals(), thread.getSemantics());
+ Module module = thread.getGlobals();
+ StarlarkFile file = EvalUtils.parseAndValidate(input, FileOptions.DEFAULT, module);
if (!file.ok()) {
throw new SyntaxError(file.errors());
}
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/CpuProfilerTest.java b/src/test/java/com/google/devtools/build/lib/syntax/CpuProfilerTest.java
index 8b79dc9..599fd40 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/CpuProfilerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/CpuProfilerTest.java
@@ -67,7 +67,7 @@
.setGlobals(Module.createForBuiltins(Starlark.UNIVERSE))
.useDefaultSemantics()
.build();
- EvalUtils.exec(input, thread.getGlobals(), thread);
+ EvalUtils.exec(input, FileOptions.DEFAULT, thread.getGlobals(), thread);
Starlark.stopCpuProfile();
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/EvaluationTest.java b/src/test/java/com/google/devtools/build/lib/syntax/EvaluationTest.java
index c2aa692..367d614 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/EvaluationTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/EvaluationTest.java
@@ -35,9 +35,10 @@
EventCollector printEvents = new EventCollector();
StarlarkThread thread = createStarlarkThread(StarlarkThread.makeDebugPrintHandler(printEvents));
ParserInput input = ParserInput.fromLines("print('hello'); x = 1//0; print('goodbye')");
- Module module = thread.getGlobals();
- assertThrows(EvalException.class, () -> EvalUtils.exec(input, module, thread));
+ Module module = thread.getGlobals();
+ assertThrows(
+ EvalException.class, () -> EvalUtils.exec(input, FileOptions.DEFAULT, module, thread));
// Only expect hello, should have been an error before goodbye.
assertThat(printEvents).hasSize(1);
@@ -53,7 +54,9 @@
try {
Thread.currentThread().interrupt();
- assertThrows(InterruptedException.class, () -> EvalUtils.exec(input, module, thread));
+ assertThrows(
+ InterruptedException.class,
+ () -> EvalUtils.exec(input, FileOptions.DEFAULT, module, thread));
} finally {
// Reset interrupt bit in case the test failed to do so.
Thread.interrupted();
@@ -77,7 +80,9 @@
"foo()");
try {
- assertThrows(InterruptedException.class, () -> EvalUtils.exec(input, module, thread));
+ assertThrows(
+ InterruptedException.class,
+ () -> EvalUtils.exec(input, FileOptions.DEFAULT, module, thread));
} finally {
// Reset interrupt bit in case the test failed to do so.
Thread.interrupted();
@@ -96,7 +101,9 @@
ParserInput input = ParserInput.fromLines("[interrupt(i == 5) for i in range(100)]");
try {
- assertThrows(InterruptedException.class, () -> EvalUtils.exec(input, module, thread));
+ assertThrows(
+ InterruptedException.class,
+ () -> EvalUtils.exec(input, FileOptions.DEFAULT, module, thread));
} finally {
// Reset interrupt bit in case the test failed to do so.
Thread.interrupted();
@@ -116,7 +123,9 @@
ParserInput.fromLines("interrupt(False); interrupt(True); interrupt(False);");
try {
- assertThrows(InterruptedException.class, () -> EvalUtils.exec(input, module, thread));
+ assertThrows(
+ InterruptedException.class,
+ () -> EvalUtils.exec(input, FileOptions.DEFAULT, module, thread));
} finally {
// Reset interrupt bit in case the test failed to do so.
Thread.interrupted();
@@ -557,16 +566,11 @@
private static void execBUILD(String... lines)
throws SyntaxError, EvalException, InterruptedException {
ParserInput input = ParserInput.fromLines(lines);
- StarlarkFile file = StarlarkFile.parse(input);
- StarlarkSemantics semantics = StarlarkSemantics.DEFAULT_SEMANTICS;
StarlarkThread thread =
- StarlarkThread.builder(Mutability.create("test")).setSemantics(semantics).build();
+ StarlarkThread.builder(Mutability.create("test")).useDefaultSemantics().build();
Module module = thread.getGlobals();
- ValidationEnvironment.validateFile(file, module, semantics, /*isBuildFile=*/ true);
- if (!file.ok()) {
- throw new SyntaxError(file.errors());
- }
- EvalUtils.exec(file, module, thread);
+ FileOptions options = FileOptions.builder().recordScope(false).build();
+ EvalUtils.exec(input, options, module, thread);
}
@Test
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java b/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java
index 9bf7d4e..3b727eb 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java
@@ -42,7 +42,7 @@
ParserInput inputSource = ParserInput.create(input, "/some/path.txt");
errors.clear();
lastError = null;
- return new Lexer(inputSource, errors);
+ return new Lexer(inputSource, FileOptions.DEFAULT, errors);
}
private ArrayList<Token> allTokens(Lexer lexer) {
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadDebuggingTest.java b/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadDebuggingTest.java
index a510a75..97ff0d3 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadDebuggingTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadDebuggingTest.java
@@ -40,7 +40,7 @@
// and returns the function value.
private static StarlarkFunction defineFunc(StarlarkThread thread) throws Exception {
Module module = thread.getGlobals();
- EvalUtils.exec(ParserInput.fromLines("def f(): pass"), module, thread);
+ EvalUtils.exec(ParserInput.fromLines("def f(): pass"), FileOptions.DEFAULT, module, thread);
return (StarlarkFunction) thread.getGlobals().lookup("f");
}
@@ -94,7 +94,7 @@
+ " f()\n"
+ "g(4, 5, 6)",
"main.star");
- EvalUtils.exec(input, module, thread);
+ EvalUtils.exec(input, FileOptions.DEFAULT, module, thread);
@SuppressWarnings("unchecked")
ImmutableList<Debug.Frame> stack = (ImmutableList<Debug.Frame>) result[0];
@@ -216,7 +216,8 @@
module.put("a", 1);
Object a =
- EvalUtils.execAndEvalOptionalFinalExpression(ParserInput.fromLines("a"), module, thread);
+ EvalUtils.execAndEvalOptionalFinalExpression(
+ ParserInput.fromLines("a"), FileOptions.DEFAULT, module, thread);
assertThat(a).isEqualTo(1);
}
@@ -231,7 +232,8 @@
SyntaxError.class,
() ->
EvalUtils.execAndEvalOptionalFinalExpression(
- ParserInput.fromLines("b"), module, thread));
+ ParserInput.fromLines("b"), FileOptions.DEFAULT, module, thread));
+
assertThat(e).hasMessageThat().isEqualTo("name 'b' is not defined");
}
@@ -243,15 +245,12 @@
assertThat(
EvalUtils.execAndEvalOptionalFinalExpression(
- ParserInput.fromLines("a.startswith('str')"), module, thread))
+ ParserInput.fromLines("a.startswith('str')"), FileOptions.DEFAULT, module, thread))
.isEqualTo(true);
- EvalUtils.exec(
- EvalUtils.parseAndValidate(ParserInput.fromLines("a = 1"), module, thread.getSemantics()),
- module,
- thread);
+ EvalUtils.exec(ParserInput.fromLines("a = 1"), FileOptions.DEFAULT, module, thread);
assertThat(
EvalUtils.execAndEvalOptionalFinalExpression(
- ParserInput.fromLines("a"), module, thread))
+ ParserInput.fromLines("a"), FileOptions.DEFAULT, module, thread))
.isEqualTo(1);
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadTest.java b/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadTest.java
index a6ec3d9..3d1db27 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadTest.java
@@ -173,7 +173,8 @@
@Test
public void testBuiltinsCanBeShadowed() throws Exception {
StarlarkThread thread = newStarlarkThread();
- EvalUtils.exec(ParserInput.fromLines("True = 123"), thread.getGlobals(), thread);
+ EvalUtils.exec(
+ ParserInput.fromLines("True = 123"), FileOptions.DEFAULT, thread.getGlobals(), thread);
assertThat(thread.getGlobals().lookup("True")).isEqualTo(123);
}
@@ -187,6 +188,7 @@
ParserInput.fromLines(
"def foo(x): x += global_var; global_var = 36; return x", //
"foo(1)"),
+ FileOptions.DEFAULT,
module,
thread);
throw new AssertionError("failed to fail");
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/ValidationTest.java b/src/test/java/com/google/devtools/build/lib/syntax/ValidationTest.java
index efadf48..f3776e6 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/ValidationTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/ValidationTest.java
@@ -18,9 +18,6 @@
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.EventCollector;
-import com.google.devtools.build.lib.packages.StarlarkSemanticsOptions; // TODO(adonovan): break!
-import com.google.devtools.common.options.Options;
-import com.google.devtools.common.options.OptionsParsingException;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -29,18 +26,13 @@
@RunWith(JUnit4.class)
public class ValidationTest {
- private StarlarkSemantics semantics = StarlarkSemantics.DEFAULT_SEMANTICS;
+ private final FileOptions.Builder options = FileOptions.builder();
- private void setSemantics(String... options) throws OptionsParsingException {
- this.semantics =
- Options.parse(StarlarkSemanticsOptions.class, options).getOptions().toSkylarkSemantics();
- }
-
- // Validates a file using the current semantics.
+ // Validates a file using the current options.
private StarlarkFile validateFile(String... lines) throws SyntaxError {
ParserInput input = ParserInput.fromLines(lines);
Module module = Module.createForBuiltins(Starlark.UNIVERSE);
- return EvalUtils.parseAndValidate(input, module, semantics);
+ return EvalUtils.parseAndValidate(input, options.build(), module);
}
// Assertions that parsing and validation succeeds.
@@ -90,6 +82,7 @@
@Test
public void testLoadAfterStatement() throws Exception {
+ options.requireLoadStatementsFirst(true);
assertInvalid(
"load() statements must be called before any other statement", //
"a = 5",
@@ -97,6 +90,14 @@
}
@Test
+ public void testAllowLoadAfterStatement() throws Exception {
+ options.requireLoadStatementsFirst(false);
+ assertValid(
+ "a = 5", //
+ "load(':b.bzl', 'c')");
+ }
+
+ @Test
public void testLoadDuplicateSymbols() throws Exception {
assertInvalid(
"load statement defines 'x' more than once", //
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/util/EvaluationTestCase.java b/src/test/java/com/google/devtools/build/lib/syntax/util/EvaluationTestCase.java
index 27d0b04..f3314b6 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/util/EvaluationTestCase.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/util/EvaluationTestCase.java
@@ -27,6 +27,7 @@
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.EvalUtils;
import com.google.devtools.build.lib.syntax.Expression;
+import com.google.devtools.build.lib.syntax.FileOptions;
import com.google.devtools.build.lib.syntax.Module;
import com.google.devtools.build.lib.syntax.Mutability;
import com.google.devtools.build.lib.syntax.ParserInput;
@@ -127,13 +128,13 @@
/** Joins the lines, parses them as an expression, and evaluates it. */
public final Object eval(String... lines) throws Exception {
ParserInput input = ParserInput.fromLines(lines);
- return EvalUtils.eval(input, thread.getGlobals(), thread);
+ return EvalUtils.eval(input, FileOptions.DEFAULT, thread.getGlobals(), thread);
}
/** Joins the lines, parses them as a file, and executes it. */
public final void exec(String... lines) throws SyntaxError, EvalException, InterruptedException {
ParserInput input = ParserInput.fromLines(lines);
- EvalUtils.exec(input, thread.getGlobals(), thread);
+ EvalUtils.exec(input, FileOptions.DEFAULT, thread.getGlobals(), thread);
}
public void checkEvalError(String msg, String... input) throws Exception {
diff --git a/src/test/starlark/testdata/string_split.sky b/src/test/starlark/testdata/string_split.sky
index 9990226..9113996 100644
--- a/src/test/starlark/testdata/string_split.sky
+++ b/src/test/starlark/testdata/string_split.sky
@@ -17,7 +17,7 @@
assert_eq('foo/bar.lisp'.rsplit('.'), ['foo/bar', 'lisp'])
assert_eq('foo/bar.?lisp'.rsplit('.?'), ['foo/bar', 'lisp'])
assert_eq('fwe$foo'.rsplit('$'), ['fwe', 'foo'])
-assert_eq('windows'.rsplit('\w'), ['windows'])
+assert_eq('windows'.rsplit('\\w'), ['windows'])
# rsplit no match
assert_eq(''.rsplit('o'), [''])
