bazel skyframe: use CompiledBuildFile for results of static BUILD processing
This change introduces a new type, PackageFunction.CompiledBuildFile,
to hold all the results of static processing of the syntax tree of a BUILD
file, so that the syntax tree can be discarded. The results include:
- the compiled Program,
- any errors from parsing, name resolution, or BUILD dialect checks,
- the set of literal strings passed to glob(), and
- the mapping of top-level function calls to generator_name strings.
The goal of this change is to cleanly separate the static processing
of BUILD files from the dynamic processing (execution), to unblock work
on various Starlark optimizations such as "flat frames" and a compiler.
These optimizations require that the syntax tree can be decorated with
information during name resolution, which is impossible in the current
code because the cache holds unresolved syntax trees, and trees are
resolved as a side effect of execution.
However, this change also opens up opportunities for Skyframe
optimizations, such as separating BUILD compilation and execution
so that we needn't re-read/parse/resolve/compile a BUILD file just
because one of its .bzl loads has changed; and prefetching loads
as soon as the BUILD file has been parsed.
Errors from checkBuildSyntax are now reported before Starlark name
resolution, and resolution is now performed even if a BUILD file
is not executed due to others (hence test changes), but there
should otherwise be no observable behavior change.
Once this lands, we can remove the resolveScope(false) hack from Starlark
and (at long last) get to work on major optimizations.
Sorry for the messy diff. A number of things have crossed the boundary
between lib.packages and the loading-phase parts of lib.skyframe,
which should one day become part of lib.packages.
PiperOrigin-RevId: 343386400
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/PackageFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/PackageFunctionTest.java
index 72a486d..be98af2 100644
--- a/src/test/java/com/google/devtools/build/lib/skyframe/PackageFunctionTest.java
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/PackageFunctionTest.java
@@ -607,12 +607,7 @@
@Test
public void testNonExistingStarlarkExtension() throws Exception {
reporter.removeHandler(failFastHandler);
- scratch.file(
- "test/starlark/BUILD",
- "load('//test/starlark:bad_extension.bzl', 'some_symbol')",
- "genrule(name = gr,",
- " outs = ['out.txt'],",
- " cmd = 'echo hello >@')");
+ scratch.file("test/starlark/BUILD", "load('//test/starlark:bad_extension.bzl', 'some_symbol')");
invalidatePackages();
SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//test/starlark"));
@@ -636,12 +631,7 @@
"test/starlark/extension.bzl",
"load('//test/starlark:bad_extension.bzl', 'some_symbol')",
"a = 'a'");
- scratch.file(
- "test/starlark/BUILD",
- "load('//test/starlark:extension.bzl', 'a')",
- "genrule(name = gr,",
- " outs = ['out.txt'],",
- " cmd = 'echo hello >@')");
+ scratch.file("test/starlark/BUILD", "load('//test/starlark:extension.bzl', 'a')");
invalidatePackages();
SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//test/starlark"));
@@ -665,12 +655,7 @@
reporter.removeHandler(failFastHandler);
Path extensionFilePath = scratch.resolve("/workspace/test/starlark/extension.bzl");
FileSystemUtils.ensureSymbolicLink(extensionFilePath, PathFragment.create("extension.bzl"));
- scratch.file(
- "test/starlark/BUILD",
- "load('//test/starlark:extension.bzl', 'a')",
- "genrule(name = gr,",
- " outs = ['out.txt'],",
- " cmd = 'echo hello >@')");
+ scratch.file("test/starlark/BUILD", "load('//test/starlark:extension.bzl', 'a')");
invalidatePackages();
SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//test/starlark"));
@@ -1261,11 +1246,13 @@
public void veryBrokenPackagePostsDoneToProgressReceiver() throws Exception {
reporter.removeHandler(failFastHandler);
+ // Note: syntax error (recovered), non-existent .bzl file.
scratch.file("pkg/BUILD", "load('//does_not:exist.bzl', 'broken'");
SkyKey key = PackageValue.key(PackageIdentifier.parse("@//pkg"));
EvaluationResult<PackageValue> result =
SkyframeExecutorTestUtils.evaluate(getSkyframeExecutor(), key, false, reporter);
- assertThatEvaluationResult(result).hasError();
+ assertThatEvaluationResult(result).hasErrorEntryForKeyThat(key);
+ assertContainsEvent("syntax error at 'newline': expected ,");
assertThat(getSkyframeExecutor().getPackageProgressReceiver().progressState())
.isEqualTo(new Pair<String, String>("1 packages loaded", ""));
}
