bazel syntax: add FileOptions
FileOptions is a set of per-file options that control the front-end ("static")
processing of a single Starlark file: scanning, parsing, validation,
and, eventually, compilation.
FileOptions determine language dialect and "code generation",
like the options to a compiler. Different files may have different options,
as in the case of BUILD and .bzl files, for example.
By contrast, a StarlarkSemantics affects only dynamic behavior,
and is carried down the thread, possibly through many files that
vary in their FileOptions.
FileOptions are specified at the construction of a StarlarkFile (parsing)
and are retained by it thereafter. This reduces the burden of ensuring
that consistent options are provided to the operations of parsing and validation
when these steps are widely separated in the source, as is often the case
in Bazel.
This change allows Copybara do validation on all its files.
It was previously unable because it could not select validation
options "a la carte". This will soon allow us to assume that all
files are resolved before execution.
The API for creating a Starlark thread is getting more verbose.
This is temporary; to "refactor" a term in an equation one must
first expand it out.
Details:
- Add FileOptions parameters to scanner, parser, and their wrappers.
- Remove the StarlarkSemantics parameter from getUndeclaredNameError.
It was only used for an assertion.
- Move StarlakSemantics into the evaluator package.
ValidationTest no longer depends on semantics or lib.packages.
- Decompose the validator's previous isBuildFile parameter into
separate features: top-level rebinding, whether to call setScope, etc.
- Simplify the "restrict string escapes" feature by pushing it
from the validator to the scanner.
- Simplify the "load disregards privacy" feature by pushing it
from the syntax tree into the validator. Delete parseVirtualBuildFile.
- Add a placeholder for a "load binds globally" feature,
for load in the REPL and for upcoming changes to the implementation
of the prelude. Its implementation will come later.
- Move the "load of private symbol" check from Eval to Validator.
This is a breaking API change for Copybara.
PiperOrigin-RevId: 302691302
diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD
index 6796acd..7a5ada8 100644
--- a/src/test/java/com/google/devtools/build/lib/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/BUILD
@@ -92,7 +92,7 @@
":AllTests",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/cmdline:LabelValidator",
- "//src/main/java/com/google/devtools/build/lib/syntax:frontend",
+ "//src/main/java/com/google/devtools/build/lib/syntax:evaluator",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
"//src/test/java/com/google/devtools/build/lib/testutil",
"//src/test/java/com/google/devtools/build/lib/testutil:JunitUtils",
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContextTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContextTest.java
index 331a0e0..86f6555 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContextTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContextTest.java
@@ -42,6 +42,7 @@
import com.google.devtools.build.lib.syntax.Dict;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.EvalUtils;
+import com.google.devtools.build.lib.syntax.FileOptions;
import com.google.devtools.build.lib.syntax.Module;
import com.google.devtools.build.lib.syntax.Mutability;
import com.google.devtools.build.lib.syntax.ParserInput;
@@ -107,7 +108,7 @@
StarlarkThread thread = StarlarkThread.builder(mu).useDefaultSemantics().build();
Module module = thread.getGlobals();
return EvalUtils.execAndEvalOptionalFinalExpression(
- ParserInput.fromLines(lines), module, thread);
+ ParserInput.fromLines(lines), FileOptions.DEFAULT, module, thread);
} catch (Exception ex) { // SyntaxError | EvalException | InterruptedException
throw new AssertionError("exec failed", ex);
}
diff --git a/src/test/java/com/google/devtools/build/lib/packages/SelectTest.java b/src/test/java/com/google/devtools/build/lib/packages/SelectTest.java
index acd0be1..7c632d1 100644
--- a/src/test/java/com/google/devtools/build/lib/packages/SelectTest.java
+++ b/src/test/java/com/google/devtools/build/lib/packages/SelectTest.java
@@ -19,6 +19,7 @@
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.EvalUtils;
+import com.google.devtools.build.lib.syntax.FileOptions;
import com.google.devtools.build.lib.syntax.Module;
import com.google.devtools.build.lib.syntax.Mutability;
import com.google.devtools.build.lib.syntax.ParserInput;
@@ -41,7 +42,7 @@
.useDefaultSemantics()
.build();
Module module = thread.getGlobals();
- return EvalUtils.eval(input, module, thread);
+ return EvalUtils.eval(input, FileOptions.DEFAULT, module, thread);
}
private static void assertFails(String expr, String wantError) {
diff --git a/src/test/java/com/google/devtools/build/lib/profiler/memory/AllocationTrackerTest.java b/src/test/java/com/google/devtools/build/lib/profiler/memory/AllocationTrackerTest.java
index 41fc7c6..b1f30bc 100644
--- a/src/test/java/com/google/devtools/build/lib/profiler/memory/AllocationTrackerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/profiler/memory/AllocationTrackerTest.java
@@ -25,6 +25,7 @@
import com.google.devtools.build.lib.syntax.Debug;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.EvalUtils;
+import com.google.devtools.build.lib.syntax.FileOptions;
import com.google.devtools.build.lib.syntax.HasBinary;
import com.google.devtools.build.lib.syntax.Module;
import com.google.devtools.build.lib.syntax.Mutability;
@@ -188,6 +189,7 @@
}
private void exec(String... lines) throws SyntaxError, EvalException, InterruptedException {
+ ParserInput input = ParserInput.create(Joiner.on("\n").join(lines), "a.star");
Mutability mu = Mutability.create("test");
StarlarkThread thread =
StarlarkThread.builder(mu)
@@ -198,9 +200,8 @@
"sample", new SamplerValue(),
"myrule", new MyRuleFunction())))
.build();
- ParserInput input = ParserInput.create(Joiner.on("\n").join(lines), "a.star");
Module module = thread.getGlobals();
- EvalUtils.exec(input, module, thread);
+ EvalUtils.exec(input, FileOptions.DEFAULT, module, thread);
}
// A fake Bazel rule. The allocation tracker reports retained memory broken down by rule class.
diff --git a/src/test/java/com/google/devtools/build/lib/skylark/SkylarkRuleClassFunctionsTest.java b/src/test/java/com/google/devtools/build/lib/skylark/SkylarkRuleClassFunctionsTest.java
index 7728813..b1ec447 100644
--- a/src/test/java/com/google/devtools/build/lib/skylark/SkylarkRuleClassFunctionsTest.java
+++ b/src/test/java/com/google/devtools/build/lib/skylark/SkylarkRuleClassFunctionsTest.java
@@ -53,6 +53,8 @@
import com.google.devtools.build.lib.syntax.Dict;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.EvalUtils;
+import com.google.devtools.build.lib.syntax.FileOptions;
+import com.google.devtools.build.lib.syntax.Module;
import com.google.devtools.build.lib.syntax.Mutability;
import com.google.devtools.build.lib.syntax.ParserInput;
import com.google.devtools.build.lib.syntax.StarlarkFile;
@@ -729,12 +731,11 @@
assertThat(c.hasAttr("a1", Type.STRING)).isTrue();
}
- // TODO(adonovan): rename execAndExport
private void evalAndExport(String... lines) throws Exception {
ParserInput input = ParserInput.fromLines(lines);
StarlarkThread thread = ev.getStarlarkThread();
- StarlarkFile file =
- EvalUtils.parseAndValidate(input, thread.getGlobals(), thread.getSemantics());
+ Module module = thread.getGlobals();
+ StarlarkFile file = EvalUtils.parseAndValidate(input, FileOptions.DEFAULT, module);
if (!file.ok()) {
throw new SyntaxError(file.errors());
}
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/CpuProfilerTest.java b/src/test/java/com/google/devtools/build/lib/syntax/CpuProfilerTest.java
index 8b79dc9..599fd40 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/CpuProfilerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/CpuProfilerTest.java
@@ -67,7 +67,7 @@
.setGlobals(Module.createForBuiltins(Starlark.UNIVERSE))
.useDefaultSemantics()
.build();
- EvalUtils.exec(input, thread.getGlobals(), thread);
+ EvalUtils.exec(input, FileOptions.DEFAULT, thread.getGlobals(), thread);
Starlark.stopCpuProfile();
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/EvaluationTest.java b/src/test/java/com/google/devtools/build/lib/syntax/EvaluationTest.java
index c2aa692..367d614 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/EvaluationTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/EvaluationTest.java
@@ -35,9 +35,10 @@
EventCollector printEvents = new EventCollector();
StarlarkThread thread = createStarlarkThread(StarlarkThread.makeDebugPrintHandler(printEvents));
ParserInput input = ParserInput.fromLines("print('hello'); x = 1//0; print('goodbye')");
- Module module = thread.getGlobals();
- assertThrows(EvalException.class, () -> EvalUtils.exec(input, module, thread));
+ Module module = thread.getGlobals();
+ assertThrows(
+ EvalException.class, () -> EvalUtils.exec(input, FileOptions.DEFAULT, module, thread));
// Only expect hello, should have been an error before goodbye.
assertThat(printEvents).hasSize(1);
@@ -53,7 +54,9 @@
try {
Thread.currentThread().interrupt();
- assertThrows(InterruptedException.class, () -> EvalUtils.exec(input, module, thread));
+ assertThrows(
+ InterruptedException.class,
+ () -> EvalUtils.exec(input, FileOptions.DEFAULT, module, thread));
} finally {
// Reset interrupt bit in case the test failed to do so.
Thread.interrupted();
@@ -77,7 +80,9 @@
"foo()");
try {
- assertThrows(InterruptedException.class, () -> EvalUtils.exec(input, module, thread));
+ assertThrows(
+ InterruptedException.class,
+ () -> EvalUtils.exec(input, FileOptions.DEFAULT, module, thread));
} finally {
// Reset interrupt bit in case the test failed to do so.
Thread.interrupted();
@@ -96,7 +101,9 @@
ParserInput input = ParserInput.fromLines("[interrupt(i == 5) for i in range(100)]");
try {
- assertThrows(InterruptedException.class, () -> EvalUtils.exec(input, module, thread));
+ assertThrows(
+ InterruptedException.class,
+ () -> EvalUtils.exec(input, FileOptions.DEFAULT, module, thread));
} finally {
// Reset interrupt bit in case the test failed to do so.
Thread.interrupted();
@@ -116,7 +123,9 @@
ParserInput.fromLines("interrupt(False); interrupt(True); interrupt(False);");
try {
- assertThrows(InterruptedException.class, () -> EvalUtils.exec(input, module, thread));
+ assertThrows(
+ InterruptedException.class,
+ () -> EvalUtils.exec(input, FileOptions.DEFAULT, module, thread));
} finally {
// Reset interrupt bit in case the test failed to do so.
Thread.interrupted();
@@ -557,16 +566,11 @@
private static void execBUILD(String... lines)
throws SyntaxError, EvalException, InterruptedException {
ParserInput input = ParserInput.fromLines(lines);
- StarlarkFile file = StarlarkFile.parse(input);
- StarlarkSemantics semantics = StarlarkSemantics.DEFAULT_SEMANTICS;
StarlarkThread thread =
- StarlarkThread.builder(Mutability.create("test")).setSemantics(semantics).build();
+ StarlarkThread.builder(Mutability.create("test")).useDefaultSemantics().build();
Module module = thread.getGlobals();
- ValidationEnvironment.validateFile(file, module, semantics, /*isBuildFile=*/ true);
- if (!file.ok()) {
- throw new SyntaxError(file.errors());
- }
- EvalUtils.exec(file, module, thread);
+ FileOptions options = FileOptions.builder().recordScope(false).build();
+ EvalUtils.exec(input, options, module, thread);
}
@Test
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java b/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java
index 9bf7d4e..3b727eb 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java
@@ -42,7 +42,7 @@
ParserInput inputSource = ParserInput.create(input, "/some/path.txt");
errors.clear();
lastError = null;
- return new Lexer(inputSource, errors);
+ return new Lexer(inputSource, FileOptions.DEFAULT, errors);
}
private ArrayList<Token> allTokens(Lexer lexer) {
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadDebuggingTest.java b/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadDebuggingTest.java
index a510a75..97ff0d3 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadDebuggingTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadDebuggingTest.java
@@ -40,7 +40,7 @@
// and returns the function value.
private static StarlarkFunction defineFunc(StarlarkThread thread) throws Exception {
Module module = thread.getGlobals();
- EvalUtils.exec(ParserInput.fromLines("def f(): pass"), module, thread);
+ EvalUtils.exec(ParserInput.fromLines("def f(): pass"), FileOptions.DEFAULT, module, thread);
return (StarlarkFunction) thread.getGlobals().lookup("f");
}
@@ -94,7 +94,7 @@
+ " f()\n"
+ "g(4, 5, 6)",
"main.star");
- EvalUtils.exec(input, module, thread);
+ EvalUtils.exec(input, FileOptions.DEFAULT, module, thread);
@SuppressWarnings("unchecked")
ImmutableList<Debug.Frame> stack = (ImmutableList<Debug.Frame>) result[0];
@@ -216,7 +216,8 @@
module.put("a", 1);
Object a =
- EvalUtils.execAndEvalOptionalFinalExpression(ParserInput.fromLines("a"), module, thread);
+ EvalUtils.execAndEvalOptionalFinalExpression(
+ ParserInput.fromLines("a"), FileOptions.DEFAULT, module, thread);
assertThat(a).isEqualTo(1);
}
@@ -231,7 +232,8 @@
SyntaxError.class,
() ->
EvalUtils.execAndEvalOptionalFinalExpression(
- ParserInput.fromLines("b"), module, thread));
+ ParserInput.fromLines("b"), FileOptions.DEFAULT, module, thread));
+
assertThat(e).hasMessageThat().isEqualTo("name 'b' is not defined");
}
@@ -243,15 +245,12 @@
assertThat(
EvalUtils.execAndEvalOptionalFinalExpression(
- ParserInput.fromLines("a.startswith('str')"), module, thread))
+ ParserInput.fromLines("a.startswith('str')"), FileOptions.DEFAULT, module, thread))
.isEqualTo(true);
- EvalUtils.exec(
- EvalUtils.parseAndValidate(ParserInput.fromLines("a = 1"), module, thread.getSemantics()),
- module,
- thread);
+ EvalUtils.exec(ParserInput.fromLines("a = 1"), FileOptions.DEFAULT, module, thread);
assertThat(
EvalUtils.execAndEvalOptionalFinalExpression(
- ParserInput.fromLines("a"), module, thread))
+ ParserInput.fromLines("a"), FileOptions.DEFAULT, module, thread))
.isEqualTo(1);
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadTest.java b/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadTest.java
index a6ec3d9..3d1db27 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadTest.java
@@ -173,7 +173,8 @@
@Test
public void testBuiltinsCanBeShadowed() throws Exception {
StarlarkThread thread = newStarlarkThread();
- EvalUtils.exec(ParserInput.fromLines("True = 123"), thread.getGlobals(), thread);
+ EvalUtils.exec(
+ ParserInput.fromLines("True = 123"), FileOptions.DEFAULT, thread.getGlobals(), thread);
assertThat(thread.getGlobals().lookup("True")).isEqualTo(123);
}
@@ -187,6 +188,7 @@
ParserInput.fromLines(
"def foo(x): x += global_var; global_var = 36; return x", //
"foo(1)"),
+ FileOptions.DEFAULT,
module,
thread);
throw new AssertionError("failed to fail");
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/ValidationTest.java b/src/test/java/com/google/devtools/build/lib/syntax/ValidationTest.java
index efadf48..f3776e6 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/ValidationTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/ValidationTest.java
@@ -18,9 +18,6 @@
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.EventCollector;
-import com.google.devtools.build.lib.packages.StarlarkSemanticsOptions; // TODO(adonovan): break!
-import com.google.devtools.common.options.Options;
-import com.google.devtools.common.options.OptionsParsingException;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -29,18 +26,13 @@
@RunWith(JUnit4.class)
public class ValidationTest {
- private StarlarkSemantics semantics = StarlarkSemantics.DEFAULT_SEMANTICS;
+ private final FileOptions.Builder options = FileOptions.builder();
- private void setSemantics(String... options) throws OptionsParsingException {
- this.semantics =
- Options.parse(StarlarkSemanticsOptions.class, options).getOptions().toSkylarkSemantics();
- }
-
- // Validates a file using the current semantics.
+ // Validates a file using the current options.
private StarlarkFile validateFile(String... lines) throws SyntaxError {
ParserInput input = ParserInput.fromLines(lines);
Module module = Module.createForBuiltins(Starlark.UNIVERSE);
- return EvalUtils.parseAndValidate(input, module, semantics);
+ return EvalUtils.parseAndValidate(input, options.build(), module);
}
// Assertions that parsing and validation succeeds.
@@ -90,6 +82,7 @@
@Test
public void testLoadAfterStatement() throws Exception {
+ options.requireLoadStatementsFirst(true);
assertInvalid(
"load() statements must be called before any other statement", //
"a = 5",
@@ -97,6 +90,14 @@
}
@Test
+ public void testAllowLoadAfterStatement() throws Exception {
+ options.requireLoadStatementsFirst(false);
+ assertValid(
+ "a = 5", //
+ "load(':b.bzl', 'c')");
+ }
+
+ @Test
public void testLoadDuplicateSymbols() throws Exception {
assertInvalid(
"load statement defines 'x' more than once", //
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/util/EvaluationTestCase.java b/src/test/java/com/google/devtools/build/lib/syntax/util/EvaluationTestCase.java
index 27d0b04..f3314b6 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/util/EvaluationTestCase.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/util/EvaluationTestCase.java
@@ -27,6 +27,7 @@
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.EvalUtils;
import com.google.devtools.build.lib.syntax.Expression;
+import com.google.devtools.build.lib.syntax.FileOptions;
import com.google.devtools.build.lib.syntax.Module;
import com.google.devtools.build.lib.syntax.Mutability;
import com.google.devtools.build.lib.syntax.ParserInput;
@@ -127,13 +128,13 @@
/** Joins the lines, parses them as an expression, and evaluates it. */
public final Object eval(String... lines) throws Exception {
ParserInput input = ParserInput.fromLines(lines);
- return EvalUtils.eval(input, thread.getGlobals(), thread);
+ return EvalUtils.eval(input, FileOptions.DEFAULT, thread.getGlobals(), thread);
}
/** Joins the lines, parses them as a file, and executes it. */
public final void exec(String... lines) throws SyntaxError, EvalException, InterruptedException {
ParserInput input = ParserInput.fromLines(lines);
- EvalUtils.exec(input, thread.getGlobals(), thread);
+ EvalUtils.exec(input, FileOptions.DEFAULT, thread.getGlobals(), thread);
}
public void checkEvalError(String msg, String... input) throws Exception {
