bazel packages: record load graph (DAG over Modules) in Module and Package
This is a preparatory step for a new, rich query output format that will provide
all the information needed to implement stardoc as a client of blaze, thus allowing
us to delete the fakebuildapi implementations and the starlarkbuildapi abstractions.
In some situations this could increase live heap usage by causing a package
to keep its loaded modules live for longer. Specifically, if packages P and Q
both load module M, then M is changed and package Q is reloaded, the stale
package P remains live, and it holds a reference to the stale M, whereas before
it did not. The next time P is loaded, the stale P and M are evicted.
Note that today, P already keeps alive any modules required by rules instantiated
by P, so an additional cost will be incurred only if P uses functions ("macros")
or data defined in M, but no rules. I expect the effect to be quite marginal.
A number of opportunities for simplification (and memory saving) have been
identified and will be acted on in a follow-up.
Also:
- define BazelModuleContext.of helper function.
RELNOTES: N/A
PiperOrigin-RevId: 320283510
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BzlLoadFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/BzlLoadFunction.java
index 9767fc6..926ea91 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/BzlLoadFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/BzlLoadFunction.java
@@ -566,7 +566,7 @@
throw BzlLoadFailedException.starlarkErrors(filePath);
}
- // Process the load statements in the file,
+ // Process load statements in .bzl file (recursive .bzl -> .bzl loads),
// resolving labels relative to the current repo mapping.
ImmutableMap<RepositoryName, RepositoryName> repoMapping = getRepositoryMapping(key, env);
if (repoMapping == null) {
@@ -605,13 +605,13 @@
// loads. Loop iteration order matches the source order of load statements.
Fingerprint fp = new Fingerprint();
fp.addBytes(astLookupValue.getDigest());
- Map<String, Module> loadedModules = Maps.newHashMapWithExpectedSize(loads.size());
+ Map<String, Module> loadedModules = Maps.newLinkedHashMapWithExpectedSize(loads.size());
ImmutableList.Builder<StarlarkFileDependency> fileDependencies =
ImmutableList.builderWithExpectedSize(loads.size());
for (int i = 0; i < loads.size(); i++) {
String loadString = loads.get(i).first;
BzlLoadValue v = bzlLoads.get(i);
- loadedModules.put(loadString, v.getModule());
+ loadedModules.put(loadString, v.getModule()); // dups ok
fileDependencies.add(v.getDependency());
fp.addBytes(v.getTransitiveDigest());
}
@@ -620,7 +620,16 @@
Module module =
Module.withPredeclared(
starlarkSemantics, getPredeclaredEnvironment(key, starlarkBuiltinsValue));
- module.setClientData(BazelModuleContext.create(label, transitiveDigest));
+
+ // Record the module's filename, label, digest, and the set of modules it loads,
+ // forming a complete representation of the load DAG.
+ module.setClientData(
+ BazelModuleContext.create(
+ label,
+ file.getStartLocation().file(),
+ ImmutableMap.copyOf(loadedModules),
+ transitiveDigest));
+
// executeBzlFile may post events to the Environment's handler, but events do not matter when
// caching BzlLoadValues. Note that executing the module mutates it.
executeBzlFile(
@@ -631,10 +640,8 @@
starlarkSemantics,
env.getListener(),
repoMapping);
- BzlLoadValue result =
- new BzlLoadValue(
- module, transitiveDigest, new StarlarkFileDependency(label, fileDependencies.build()));
- return result;
+ return new BzlLoadValue(
+ module, transitiveDigest, new StarlarkFileDependency(label, fileDependencies.build()));
}
private static ImmutableMap<RepositoryName, RepositoryName> getRepositoryMapping(