Add a new `include()` directive to MODULE.bazel files
This new directive allows the root module to divide its `MODULE.bazel` into multiple segments. This directive can only be used by root modules; only files in the main repo may be included; variable bindings are only visible in the file they occur in, not in any included or including files. See the docs for `include()` (in `ModuleFileGlobals.java`) for more details.
In follow-ups, we'll need to address:
1. Enforcing the loaded files to have some sort of naming format (tentatively `foo.MODULE.bazel` where `foo` is anything)
2. Making `bazel mod tidy` work with included files
RELNOTES: Added a new `include()` directive to `MODULE.bazel` files.
Fixes https://github.com/bazelbuild/bazel/issues/17880.
Closes #21855.
PiperOrigin-RevId: 627034184
Change-Id: Ifc2f616cf0791445daeeac9ca5ec4478e83382aa
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
index 32146c8..750742a 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
@@ -123,6 +123,7 @@
"BazelModuleResolutionEvent.java",
"BazelModuleResolutionValue.java",
"BzlmodFlagsAndEnvVars.java",
+ "CompiledModuleFile.java",
"GitOverride.java",
"InterimModule.java",
"LocalPathOverride.java",
@@ -149,6 +150,7 @@
],
deps = [
":common",
+ ":exception",
":inspection",
":module_extension",
":module_extension_metadata",
@@ -169,6 +171,7 @@
"//src/main/java/net/starlark/java/annot",
"//src/main/java/net/starlark/java/eval",
"//src/main/java/net/starlark/java/syntax",
+ "//src/main/protobuf:failure_details_java_proto",
"//third_party:auto_value",
"//third_party:gson",
"//third_party:guava",
@@ -232,6 +235,8 @@
"//src/main/java/com/google/devtools/build/lib/skyframe:bzl_load_value",
"//src/main/java/com/google/devtools/build/lib/skyframe:client_environment_function",
"//src/main/java/com/google/devtools/build/lib/skyframe:client_environment_value",
+ "//src/main/java/com/google/devtools/build/lib/skyframe:package_lookup_function",
+ "//src/main/java/com/google/devtools/build/lib/skyframe:package_lookup_value",
"//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value",
"//src/main/java/com/google/devtools/build/lib/skyframe:repository_mapping_value",
"//src/main/java/com/google/devtools/build/lib/skyframe:skyframe_cluster",
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModTidyFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModTidyFunction.java
index 6cdf50a..7f9e184 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModTidyFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModTidyFunction.java
@@ -23,6 +23,7 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
import com.google.devtools.build.lib.cmdline.RepositoryName;
@@ -50,6 +51,11 @@
@Nullable
public SkyValue compute(SkyKey skyKey, Environment env)
throws InterruptedException, SkyFunctionException {
+ RootModuleFileValue rootModuleFileValue =
+ (RootModuleFileValue) env.getValue(ModuleFileValue.KEY_FOR_ROOT_MODULE);
+ if (rootModuleFileValue == null) {
+ return null;
+ }
BazelDepGraphValue depGraphValue = (BazelDepGraphValue) env.getValue(BazelDepGraphValue.KEY);
if (depGraphValue == null) {
return null;
@@ -112,6 +118,7 @@
return BazelModTidyValue.create(
buildozer.asPath(),
+ rootModuleFileValue.getIncludeLabelToCompiledModuleFile(),
MODULE_OVERRIDES.get(env),
IGNORE_DEV_DEPS.get(env),
LOCKFILE_MODE.get(env),
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModTidyValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModTidyValue.java
index 281d45c..cfd34f1 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModTidyValue.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModTidyValue.java
@@ -35,6 +35,8 @@
/** The path of the buildozer binary provided by the "buildozer" module. */
public abstract Path buildozer();
+ public abstract ImmutableMap<String, CompiledModuleFile> includeLabelToCompiledModuleFile();
+
/** The value of {@link ModuleFileFunction#MODULE_OVERRIDES}. */
public abstract ImmutableMap<String, ModuleOverride> moduleOverrides();
@@ -52,12 +54,14 @@
static BazelModTidyValue create(
Path buildozer,
+ ImmutableMap<String, CompiledModuleFile> includeLabelToCompiledModuleFile,
Map<String, ModuleOverride> moduleOverrides,
boolean ignoreDevDeps,
LockfileMode lockfileMode,
StarlarkSemantics starlarkSemantics) {
return new AutoValue_BazelModTidyValue(
buildozer,
+ includeLabelToCompiledModuleFile,
ImmutableMap.copyOf(moduleOverrides),
ignoreDevDeps,
lockfileMode,
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/CompiledModuleFile.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/CompiledModuleFile.java
new file mode 100644
index 0000000..8d5b57e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/CompiledModuleFile.java
@@ -0,0 +1,170 @@
+// Copyright 2024 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+package com.google.devtools.build.lib.bazel.bzlmod;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.ExtendedEventHandler;
+import com.google.devtools.build.lib.packages.BazelStarlarkEnvironment;
+import com.google.devtools.build.lib.packages.DotBazelFileSyntaxChecker;
+import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code;
+import net.starlark.java.eval.EvalException;
+import net.starlark.java.eval.Module;
+import net.starlark.java.eval.Starlark;
+import net.starlark.java.eval.StarlarkSemantics;
+import net.starlark.java.eval.StarlarkThread;
+import net.starlark.java.syntax.Argument;
+import net.starlark.java.syntax.AssignmentStatement;
+import net.starlark.java.syntax.CallExpression;
+import net.starlark.java.syntax.DotExpression;
+import net.starlark.java.syntax.ExpressionStatement;
+import net.starlark.java.syntax.Identifier;
+import net.starlark.java.syntax.Location;
+import net.starlark.java.syntax.ParserInput;
+import net.starlark.java.syntax.Program;
+import net.starlark.java.syntax.StarlarkFile;
+import net.starlark.java.syntax.StringLiteral;
+import net.starlark.java.syntax.SyntaxError;
+
+/**
+ * Represents a compiled MODULE.bazel file, ready to be executed on a {@link StarlarkThread}. It's
+ * been successfully checked for syntax errors.
+ *
+ * <p>Use the {@link #parseAndCompile} factory method instead of directly instantiating this record.
+ */
+public record CompiledModuleFile(
+ ModuleFile moduleFile,
+ Program program,
+ Module predeclaredEnv,
+ ImmutableList<IncludeStatement> includeStatements) {
+ public static final String INCLUDE_IDENTIFIER = "include";
+
+ record IncludeStatement(String includeLabel, Location location) {}
+
+ /** Parses and compiles a given module file, checking it for syntax errors. */
+ public static CompiledModuleFile parseAndCompile(
+ ModuleFile moduleFile,
+ ModuleKey moduleKey,
+ StarlarkSemantics starlarkSemantics,
+ BazelStarlarkEnvironment starlarkEnv,
+ ExtendedEventHandler eventHandler)
+ throws ExternalDepsException {
+ StarlarkFile starlarkFile =
+ StarlarkFile.parse(ParserInput.fromUTF8(moduleFile.getContent(), moduleFile.getLocation()));
+ if (!starlarkFile.ok()) {
+ Event.replayEventsOn(eventHandler, starlarkFile.errors());
+ throw ExternalDepsException.withMessage(
+ Code.BAD_MODULE, "error parsing MODULE.bazel file for %s", moduleKey);
+ }
+ try {
+ ImmutableList<IncludeStatement> includeStatements = checkModuleFileSyntax(starlarkFile);
+ Module predeclaredEnv =
+ Module.withPredeclared(starlarkSemantics, starlarkEnv.getModuleBazelEnv());
+ Program program = Program.compileFile(starlarkFile, predeclaredEnv);
+ return new CompiledModuleFile(moduleFile, program, predeclaredEnv, includeStatements);
+ } catch (SyntaxError.Exception e) {
+ Event.replayEventsOn(eventHandler, e.errors());
+ throw ExternalDepsException.withMessage(
+ Code.BAD_MODULE, "syntax error in MODULE.bazel file for %s", moduleKey);
+ }
+ }
+
+ /**
+ * Checks the given `starlarkFile` for module file syntax, and returns the list of `include`
+ * statements it contains. This is a somewhat crude sweep over the AST; we loudly complain about
+ * any usage of `include` that is not in a top-level function call statement with one single
+ * string literal positional argument, *except* that we don't do this check once `include` is
+ * assigned to, due to backwards compatibility concerns.
+ */
+ @VisibleForTesting
+ static ImmutableList<IncludeStatement> checkModuleFileSyntax(StarlarkFile starlarkFile)
+ throws SyntaxError.Exception {
+ var includeStatements = ImmutableList.<IncludeStatement>builder();
+ new DotBazelFileSyntaxChecker("MODULE.bazel files", /* canLoadBzl= */ false) {
+ // Once `include` the identifier is assigned to, we no longer care about its appearance
+ // anywhere. This allows `include` to be used as a module extension proxy (and technically
+ // any other variable binding).
+ private boolean includeWasAssigned = false;
+
+ @Override
+ public void visit(ExpressionStatement node) {
+ // We can assume this statement isn't nested in any block, since we don't allow
+ // `if`/`def`/`for` in MODULE.bazel.
+ if (!includeWasAssigned
+ && node.getExpression() instanceof CallExpression call
+ && call.getFunction() instanceof Identifier id
+ && id.getName().equals(INCLUDE_IDENTIFIER)) {
+ // Found a top-level call to `include`!
+ if (call.getArguments().size() == 1
+ && call.getArguments().getFirst() instanceof Argument.Positional pos
+ && pos.getValue() instanceof StringLiteral str) {
+ includeStatements.add(new IncludeStatement(str.getValue(), call.getStartLocation()));
+ // Nothing else to check, we can stop visiting sub-nodes now.
+ return;
+ }
+ error(
+ node.getStartLocation(),
+ "the `include` directive MUST be called with exactly one positional argument that "
+ + "is a string literal");
+ return;
+ }
+ super.visit(node);
+ }
+
+ @Override
+ public void visit(AssignmentStatement node) {
+ visit(node.getRHS());
+ if (!includeWasAssigned
+ && node.getLHS() instanceof Identifier id
+ && id.getName().equals(INCLUDE_IDENTIFIER)) {
+ includeWasAssigned = true;
+ // Technically someone could do something like
+ // (include, myvar) = (print, 3)
+ // and work around our check, but at that point IDGAF.
+ } else {
+ visit(node.getLHS());
+ }
+ }
+
+ @Override
+ public void visit(DotExpression node) {
+ visit(node.getObject());
+ if (includeWasAssigned || !node.getField().getName().equals(INCLUDE_IDENTIFIER)) {
+ // This is fine: `whatever.include`
+ // (so `include` can be used as a tag class name)
+ visit(node.getField());
+ }
+ }
+
+ @Override
+ public void visit(Identifier node) {
+ if (!includeWasAssigned && node.getName().equals(INCLUDE_IDENTIFIER)) {
+ // If we somehow reach the `include` identifier but NOT as the other allowed cases above,
+ // cry foul.
+ error(
+ node.getStartLocation(),
+ "the `include` directive MUST be called directly at the top-level");
+ }
+ super.visit(node);
+ }
+ }.check(starlarkFile);
+ return includeStatements.build();
+ }
+
+ public void runOnThread(StarlarkThread thread) throws EvalException, InterruptedException {
+ Starlark.execFileProgram(program, predeclaredEnv, thread);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java
index 85b210e..6b97bb5 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java
@@ -17,6 +17,7 @@
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.common.collect.ImmutableList;
@@ -24,16 +25,17 @@
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.devtools.build.lib.actions.FileValue;
+import com.google.devtools.build.lib.bazel.bzlmod.CompiledModuleFile.IncludeStatement;
import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.NonRootModuleFileValue;
import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.LabelConstants;
+import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.packages.BazelStarlarkEnvironment;
-import com.google.devtools.build.lib.packages.DotBazelFileSyntaxChecker;
import com.google.devtools.build.lib.packages.StarlarkExportable;
import com.google.devtools.build.lib.profiler.Profiler;
import com.google.devtools.build.lib.profiler.ProfilerTask;
@@ -42,6 +44,8 @@
import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code;
import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction;
import com.google.devtools.build.lib.skyframe.ClientEnvironmentValue;
+import com.google.devtools.build.lib.skyframe.PackageLookupFunction;
+import com.google.devtools.build.lib.skyframe.PackageLookupValue;
import com.google.devtools.build.lib.skyframe.PrecomputedValue;
import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed;
import com.google.devtools.build.lib.util.Fingerprint;
@@ -52,10 +56,14 @@
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
+import com.google.devtools.build.skyframe.SkyframeLookupResult;
import com.google.errorprone.annotations.FormatMethod;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -63,14 +71,9 @@
import javax.annotation.Nullable;
import net.starlark.java.eval.EvalException;
import net.starlark.java.eval.Mutability;
-import net.starlark.java.eval.Starlark;
import net.starlark.java.eval.StarlarkSemantics;
import net.starlark.java.eval.StarlarkThread;
import net.starlark.java.eval.SymbolGenerator;
-import net.starlark.java.syntax.ParserInput;
-import net.starlark.java.syntax.Program;
-import net.starlark.java.syntax.StarlarkFile;
-import net.starlark.java.syntax.SyntaxError;
/**
* Takes a {@link ModuleKey} and its override (if any), retrieves the module file from a registry or
@@ -90,13 +93,14 @@
private final ImmutableMap<String, NonRegistryOverride> builtinModules;
private static final String BZLMOD_REMINDER =
- "###############################################################################\n"
- + "# Bazel now uses Bzlmod by default to manage external dependencies.\n"
- + "# Please consider migrating your external dependencies from WORKSPACE to"
- + " MODULE.bazel.\n"
- + "#\n"
- + "# For more details, please check https://github.com/bazelbuild/bazel/issues/18958\n"
- + "###############################################################################\n";
+ """
+ ###############################################################################
+ # Bazel now uses Bzlmod by default to manage external dependencies.
+ # Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel.
+ #
+ # For more details, please check https://github.com/bazelbuild/bazel/issues/18958
+ ###############################################################################
+ """;
/**
* @param builtinModules A list of "built-in" modules that are treated as implicit dependencies of
@@ -113,7 +117,17 @@
}
private static class State implements Environment.SkyKeyComputeState {
+ // The following field is used during non-root module file evaluation. We store the module file
+ // here so that a later attempt to retrieve yanked versions wouldn't be overly expensive.
GetModuleFileResult getModuleFileResult;
+
+ // The following fields are used during root module file evaluation. We try to compile the root
+ // module file itself first, and then read, parse, and compile any included module files layer
+ // by layer, in a BFS fashion (hence the `horizon` field). Finally, everything is collected into
+ // the `includeLabelToCompiledModuleFile` map for use during actual Starlark execution.
+ CompiledModuleFile compiledRootModuleFile;
+ ImmutableList<IncludeStatement> horizon;
+ HashMap<String, CompiledModuleFile> includeLabelToCompiledModuleFile = new HashMap<>();
}
@Nullable
@@ -181,10 +195,28 @@
.addBytes(state.getModuleFileResult.moduleFile.getContent())
.hexDigestAndReset();
+ CompiledModuleFile compiledModuleFile;
+ try {
+ compiledModuleFile =
+ CompiledModuleFile.parseAndCompile(
+ state.getModuleFileResult.moduleFile,
+ moduleKey,
+ starlarkSemantics,
+ starlarkEnv,
+ env.getListener());
+ } catch (ExternalDepsException e) {
+ throw new ModuleFileFunctionException(e, Transience.PERSISTENT);
+ }
+ if (!compiledModuleFile.includeStatements().isEmpty()) {
+ throw errorf(
+ Code.BAD_MODULE,
+ "include() directive found at %s, but it can only be used in the root module",
+ compiledModuleFile.includeStatements().getFirst().location());
+ }
ModuleThreadContext moduleThreadContext =
execModuleFile(
- state.getModuleFileResult.moduleFile,
- state.getModuleFileResult.registry,
+ compiledModuleFile,
+ /* includeLabelToParsedModuleFile= */ null,
moduleKey,
// Dev dependencies should always be ignored if the current module isn't the root module
/* ignoreDevDeps= */ true,
@@ -192,14 +224,13 @@
// We try to prevent most side effects of yanked modules, in particular print().
/* printIsNoop= */ yankedInfo.isPresent(),
starlarkSemantics,
- starlarkEnv,
env.getListener(),
SymbolGenerator.create(skyKey));
// Perform some sanity checks.
InterimModule module;
try {
- module = moduleThreadContext.buildModule();
+ module = moduleThreadContext.buildModule(state.getModuleFileResult.registry);
} catch (EvalException e) {
env.getListener().handle(Event.error(e.getMessageWithStack()));
throw errorf(Code.BAD_MODULE, "error executing MODULE.bazel file for %s", moduleKey);
@@ -243,69 +274,206 @@
private SkyValue computeForRootModule(
StarlarkSemantics starlarkSemantics, Environment env, SymbolGenerator<?> symbolGenerator)
throws ModuleFileFunctionException, InterruptedException {
- RootedPath moduleFilePath = getModuleFilePath(workspaceRoot);
- if (env.getValue(FileValue.key(moduleFilePath)) == null) {
- return null;
+ State state = env.getState(State::new);
+ if (state.compiledRootModuleFile == null) {
+ RootedPath moduleFilePath = getModuleFilePath(workspaceRoot);
+ if (env.getValue(FileValue.key(moduleFilePath)) == null) {
+ return null;
+ }
+ byte[] moduleFileContents;
+ if (moduleFilePath.asPath().exists()) {
+ moduleFileContents = readModuleFile(moduleFilePath.asPath());
+ } else {
+ moduleFileContents = BZLMOD_REMINDER.getBytes(UTF_8);
+ createModuleFile(moduleFilePath.asPath(), moduleFileContents);
+ env.getListener()
+ .handle(
+ Event.warn(
+ "--enable_bzlmod is set, but no MODULE.bazel file was found at the workspace"
+ + " root. Bazel will create an empty MODULE.bazel file. Please consider"
+ + " migrating your external dependencies from WORKSPACE to MODULE.bazel."
+ + " For more details, please refer to"
+ + " https://github.com/bazelbuild/bazel/issues/18958."));
+ }
+ try {
+ state.compiledRootModuleFile =
+ CompiledModuleFile.parseAndCompile(
+ ModuleFile.create(moduleFileContents, moduleFilePath.asPath().toString()),
+ ModuleKey.ROOT,
+ starlarkSemantics,
+ starlarkEnv,
+ env.getListener());
+ } catch (ExternalDepsException e) {
+ throw new ModuleFileFunctionException(e, Transience.PERSISTENT);
+ }
+ state.horizon = state.compiledRootModuleFile.includeStatements();
}
- byte[] moduleFileContents;
- if (moduleFilePath.asPath().exists()) {
- moduleFileContents = readModuleFile(moduleFilePath.asPath());
- } else {
- moduleFileContents = BZLMOD_REMINDER.getBytes(UTF_8);
- createModuleFile(moduleFilePath.asPath(), moduleFileContents);
- env.getListener()
- .handle(
- Event.warn(
- "--enable_bzlmod is set, but no MODULE.bazel file was found at the workspace"
- + " root. Bazel will create an empty MODULE.bazel file. Please consider"
- + " migrating your external dependencies from WORKSPACE to MODULE.bazel. For"
- + " more details, please refer to"
- + " https://github.com/bazelbuild/bazel/issues/18958."));
+ while (!state.horizon.isEmpty()) {
+ var newHorizon =
+ advanceHorizon(
+ state.includeLabelToCompiledModuleFile,
+ state.horizon,
+ env,
+ starlarkSemantics,
+ starlarkEnv);
+ if (newHorizon == null) {
+ return null;
+ }
+ state.horizon = newHorizon;
}
return evaluateRootModuleFile(
- moduleFileContents,
- moduleFilePath,
+ state.compiledRootModuleFile,
+ ImmutableMap.copyOf(state.includeLabelToCompiledModuleFile),
builtinModules,
MODULE_OVERRIDES.get(env),
IGNORE_DEV_DEPS.get(env),
starlarkSemantics,
- starlarkEnv,
env.getListener(),
symbolGenerator);
}
+ /**
+ * Reads, parses, and compiles all included module files named by {@code horizon}, stores the
+ * result in {@code includeLabelToCompiledModuleFile}, and finally returns the include statements
+ * of these newly compiled module files as a new "horizon".
+ */
+ @Nullable
+ private static ImmutableList<IncludeStatement> advanceHorizon(
+ HashMap<String, CompiledModuleFile> includeLabelToCompiledModuleFile,
+ ImmutableList<IncludeStatement> horizon,
+ Environment env,
+ StarlarkSemantics starlarkSemantics,
+ BazelStarlarkEnvironment starlarkEnv)
+ throws ModuleFileFunctionException, InterruptedException {
+ var includeLabels = new ArrayList<Label>(horizon.size());
+ for (var includeStatement : horizon) {
+ if (!includeStatement.includeLabel().startsWith("//")) {
+ throw errorf(
+ Code.BAD_MODULE,
+ "bad include label '%s' at %s: include() must be called with main repo labels "
+ + "(starting with double slashes)",
+ includeStatement.includeLabel(),
+ includeStatement.location());
+ }
+ try {
+ includeLabels.add(Label.parseCanonical(includeStatement.includeLabel()));
+ } catch (LabelSyntaxException e) {
+ throw errorf(
+ Code.BAD_MODULE,
+ "bad include label '%s' at %s: %s",
+ includeStatement.includeLabel(),
+ includeStatement.location(),
+ e.getMessage());
+ }
+ }
+ SkyframeLookupResult result =
+ env.getValuesAndExceptions(
+ includeLabels.stream()
+ .map(l -> (SkyKey) PackageLookupValue.key(l.getPackageIdentifier()))
+ .collect(toImmutableSet()));
+ var rootedPaths = new ArrayList<RootedPath>(horizon.size());
+ for (int i = 0; i < horizon.size(); i++) {
+ Label includeLabel = includeLabels.get(i);
+ PackageLookupValue pkgLookupValue =
+ (PackageLookupValue)
+ result.get(PackageLookupValue.key(includeLabel.getPackageIdentifier()));
+ if (pkgLookupValue == null) {
+ return null;
+ }
+ if (!pkgLookupValue.packageExists()) {
+ String message = pkgLookupValue.getErrorMsg();
+ if (pkgLookupValue == PackageLookupValue.NO_BUILD_FILE_VALUE) {
+ message =
+ PackageLookupFunction.explainNoBuildFileValue(
+ includeLabel.getPackageIdentifier(), env);
+ }
+ throw errorf(
+ Code.BAD_MODULE,
+ "unable to load package for '%s' included at %s: %s",
+ horizon.get(i).includeLabel(),
+ horizon.get(i).location(),
+ message);
+ }
+ rootedPaths.add(
+ RootedPath.toRootedPath(pkgLookupValue.getRoot(), includeLabel.toPathFragment()));
+ }
+ result =
+ env.getValuesAndExceptions(
+ rootedPaths.stream().map(FileValue::key).collect(toImmutableSet()));
+ var newHorizon = ImmutableList.<IncludeStatement>builder();
+ for (int i = 0; i < horizon.size(); i++) {
+ FileValue fileValue = (FileValue) result.get(FileValue.key(rootedPaths.get(i)));
+ if (fileValue == null) {
+ return null;
+ }
+ if (!fileValue.isFile()) {
+ throw errorf(
+ Code.BAD_MODULE,
+ "error reading '%s' included at %s: not a regular file",
+ horizon.get(i).includeLabel(),
+ horizon.get(i).location());
+ }
+ byte[] bytes;
+ try {
+ bytes = FileSystemUtils.readContent(rootedPaths.get(i).asPath());
+ } catch (IOException e) {
+ throw errorf(
+ Code.BAD_MODULE,
+ "error reading '%s' included at %s: %s",
+ horizon.get(i).includeLabel(),
+ horizon.get(i).location(),
+ e.getMessage());
+ }
+ try {
+ var compiledModuleFile =
+ CompiledModuleFile.parseAndCompile(
+ ModuleFile.create(bytes, rootedPaths.get(i).asPath().toString()),
+ ModuleKey.ROOT,
+ starlarkSemantics,
+ starlarkEnv,
+ env.getListener());
+ includeLabelToCompiledModuleFile.put(horizon.get(i).includeLabel(), compiledModuleFile);
+ newHorizon.addAll(compiledModuleFile.includeStatements());
+ } catch (ExternalDepsException e) {
+ throw new ModuleFileFunctionException(e, Transience.PERSISTENT);
+ }
+ }
+ return newHorizon.build();
+ }
+
public static RootedPath getModuleFilePath(Path workspaceRoot) {
return RootedPath.toRootedPath(
Root.fromPath(workspaceRoot), LabelConstants.MODULE_DOT_BAZEL_FILE_NAME);
}
public static RootModuleFileValue evaluateRootModuleFile(
- byte[] moduleFileContents,
- RootedPath moduleFilePath,
+ CompiledModuleFile compiledRootModuleFile,
+ ImmutableMap<String, CompiledModuleFile> includeLabelToCompiledModuleFile,
ImmutableMap<String, NonRegistryOverride> builtinModules,
Map<String, ModuleOverride> commandOverrides,
boolean ignoreDevDeps,
StarlarkSemantics starlarkSemantics,
- BazelStarlarkEnvironment starlarkEnv,
ExtendedEventHandler eventHandler,
SymbolGenerator<?> symbolGenerator)
throws ModuleFileFunctionException, InterruptedException {
- String moduleFileHash = new Fingerprint().addBytes(moduleFileContents).hexDigestAndReset();
+ String moduleFileHash =
+ new Fingerprint()
+ .addBytes(compiledRootModuleFile.moduleFile().getContent())
+ .hexDigestAndReset();
ModuleThreadContext moduleThreadContext =
execModuleFile(
- ModuleFile.create(moduleFileContents, moduleFilePath.asPath().toString()),
- /* registry= */ null,
+ compiledRootModuleFile,
+ includeLabelToCompiledModuleFile,
ModuleKey.ROOT,
ignoreDevDeps,
builtinModules,
/* printIsNoop= */ false,
starlarkSemantics,
- starlarkEnv,
eventHandler,
symbolGenerator);
InterimModule module;
try {
- module = moduleThreadContext.buildModule();
+ module = moduleThreadContext.buildModule(/* registry= */ null);
} catch (EvalException e) {
eventHandler.handle(Event.error(e.getMessageWithStack()));
throw errorf(Code.BAD_MODULE, "error executing MODULE.bazel file for the root module");
@@ -347,40 +515,31 @@
ModuleKey.create(name, Version.EMPTY).getCanonicalRepoNameWithoutVersion(),
name -> name));
return RootModuleFileValue.create(
- module, moduleFileHash, overrides, nonRegistryOverrideCanonicalRepoNameLookup);
+ module,
+ moduleFileHash,
+ overrides,
+ nonRegistryOverrideCanonicalRepoNameLookup,
+ includeLabelToCompiledModuleFile);
}
private static ModuleThreadContext execModuleFile(
- ModuleFile moduleFile,
- @Nullable Registry registry,
+ CompiledModuleFile compiledRootModuleFile,
+ @Nullable ImmutableMap<String, CompiledModuleFile> includeLabelToParsedModuleFile,
ModuleKey moduleKey,
boolean ignoreDevDeps,
ImmutableMap<String, NonRegistryOverride> builtinModules,
boolean printIsNoop,
StarlarkSemantics starlarkSemantics,
- BazelStarlarkEnvironment starlarkEnv,
ExtendedEventHandler eventHandler,
SymbolGenerator<?> symbolGenerator)
throws ModuleFileFunctionException, InterruptedException {
- StarlarkFile starlarkFile =
- StarlarkFile.parse(ParserInput.fromUTF8(moduleFile.getContent(), moduleFile.getLocation()));
- if (!starlarkFile.ok()) {
- Event.replayEventsOn(eventHandler, starlarkFile.errors());
- throw errorf(Code.BAD_MODULE, "error parsing MODULE.bazel file for %s", moduleKey);
- }
-
ModuleThreadContext context =
- new ModuleThreadContext(builtinModules, moduleKey, registry, ignoreDevDeps);
+ new ModuleThreadContext(
+ builtinModules, moduleKey, ignoreDevDeps, includeLabelToParsedModuleFile);
try (SilentCloseable c =
Profiler.instance()
.profile(ProfilerTask.BZLMOD, () -> "evaluate module file: " + moduleKey);
Mutability mu = Mutability.create("module file", moduleKey)) {
- new DotBazelFileSyntaxChecker("MODULE.bazel files", /* canLoadBzl= */ false)
- .check(starlarkFile);
- net.starlark.java.eval.Module predeclaredEnv =
- net.starlark.java.eval.Module.withPredeclared(
- starlarkSemantics, starlarkEnv.getModuleBazelEnv());
- Program program = Program.compileFile(starlarkFile, predeclaredEnv);
StarlarkThread thread =
StarlarkThread.create(
mu, starlarkSemantics, /* contextDescription= */ "", symbolGenerator);
@@ -398,10 +557,7 @@
}
}
});
- Starlark.execFileProgram(program, predeclaredEnv, thread);
- } catch (SyntaxError.Exception e) {
- Event.replayEventsOn(eventHandler, e.errors());
- throw errorf(Code.BAD_MODULE, "error executing MODULE.bazel file for %s", moduleKey);
+ compiledRootModuleFile.runOnThread(thread);
} catch (EvalException e) {
eventHandler.handle(Event.error(e.getMessageWithStack()));
throw errorf(Code.BAD_MODULE, "error executing MODULE.bazel file for %s", moduleKey);
@@ -409,11 +565,12 @@
return context;
}
- private static class GetModuleFileResult {
- ModuleFile moduleFile;
- // `registry` can be null if this module has a non-registry override.
- @Nullable Registry registry;
- }
+ /**
+ * Result of a {@link #getModuleFile} call.
+ *
+ * @param registry can be null if this module has a non-registry override.
+ */
+ private record GetModuleFileResult(ModuleFile moduleFile, @Nullable Registry registry) {}
@Nullable
private GetModuleFileResult getModuleFile(
@@ -436,16 +593,15 @@
if (env.getValue(FileValue.key(moduleFilePath)) == null) {
return null;
}
- GetModuleFileResult result = new GetModuleFileResult();
Label moduleFileLabel =
Label.createUnvalidated(
PackageIdentifier.create(canonicalRepoName, PathFragment.EMPTY_FRAGMENT),
LabelConstants.MODULE_DOT_BAZEL_FILE_NAME.getBaseName());
- result.moduleFile =
+ return new GetModuleFileResult(
ModuleFile.create(
readModuleFile(moduleFilePath.asPath()),
- moduleFileLabel.getUnambiguousCanonicalForm());
- return result;
+ moduleFileLabel.getUnambiguousCanonicalForm()),
+ /* registry= */ null);
}
// Otherwise, we should get the module file from a registry.
@@ -489,16 +645,13 @@
// Now go through the list of registries and use the first one that contains the requested
// module.
- GetModuleFileResult result = new GetModuleFileResult();
for (Registry registry : registryObjects) {
try {
Optional<ModuleFile> moduleFile = registry.getModuleFile(key, env.getListener());
if (moduleFile.isEmpty()) {
continue;
}
- result.moduleFile = moduleFile.get();
- result.registry = registry;
- return result;
+ return new GetModuleFileResult(moduleFile.get(), registry);
} catch (IOException e) {
throw errorf(
Code.ERROR_ACCESSING_REGISTRY, e, "Error accessing registry %s", registry.getUrl());
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java
index 2d74793..1f367f2 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java
@@ -79,7 +79,8 @@
"Declares certain properties of the Bazel module represented by the current Bazel repo."
+ " These properties are either essential metadata of the module (such as the name"
+ " and version), or affect behavior of the current module and its dependents. <p>It"
- + " should be called at most once. It can be omitted only if this module is the root"
+ + " should be called at most once, and if called, it must be the very first directive"
+ + " in the MODULE.bazel file. It can be omitted only if this module is the root"
+ " module (as in, if it's not going to be depended on by another module).",
parameters = {
@Param(
@@ -664,7 +665,7 @@
}
@StarlarkBuiltin(name = "repo_rule_proxy", documented = false)
- class RepoRuleProxy implements StarlarkValue {
+ static class RepoRuleProxy implements StarlarkValue {
private final ModuleExtensionUsageBuilder usageBuilder;
private final String tagName;
@@ -699,6 +700,36 @@
}
@StarlarkMethod(
+ name = CompiledModuleFile.INCLUDE_IDENTIFIER,
+ doc =
+ "Includes the contents of another MODULE.bazel-like file. Effectively,"
+ + " <code>include()</code> behaves as if the included file is textually placed at the"
+ + " location of the <code>include()</code> call, except that variable bindings (such"
+ + " as those used for <code>use_extension</code>) are only ever visible in the file"
+ + " they occur in, not in any included or including files.<p>Only the root module may"
+ + " use <code>include()</code>; it is an error if a <code>bazel_dep</code>'s MODULE"
+ + " file uses <code>include()</code>.<p>Only files in the main repo may be"
+ + " included.<p><code>include()</code> allows you to segment the root module file"
+ + " into multiple parts, to avoid having an enormous MODULE.bazel file or to better"
+ + " manage access control for individual semantic segments.",
+ parameters = {
+ @Param(
+ name = "label",
+ doc =
+ "The label pointing to the file to include. The label must point to a file in the"
+ + " main repo; in other words, it <strong>must<strong> start with double"
+ + " slashes (<code>//</code>)."),
+ },
+ useStarlarkThread = true)
+ public void include(String label, StarlarkThread thread)
+ throws InterruptedException, EvalException {
+ ModuleThreadContext context =
+ ModuleThreadContext.fromOrFail(thread, CompiledModuleFile.INCLUDE_IDENTIFIER + "()");
+ context.setNonModuleCalled();
+ context.include(label, thread);
+ }
+
+ @StarlarkMethod(
name = "single_version_override",
doc =
"Specifies that a dependency should still come from a registry, but its version should"
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java
index 4fe216a..7c7e38f 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java
@@ -69,13 +69,26 @@
public abstract ImmutableMap<RepositoryName, String>
getNonRegistryOverrideCanonicalRepoNameLookup();
+ /**
+ * TODO: This field is a hack. It's not needed by anything other than {@code ModCommand}, during
+ * the {@code bazel mod tidy} command. Doing it this way assumes that {@code bazel mod tidy}
+ * cannot touch any included segments. This is unsatisfactory; we should do it properly at some
+ * point, although that seems quite difficult.
+ */
+ public abstract ImmutableMap<String, CompiledModuleFile> getIncludeLabelToCompiledModuleFile();
+
public static RootModuleFileValue create(
InterimModule module,
String moduleFileHash,
ImmutableMap<String, ModuleOverride> overrides,
- ImmutableMap<RepositoryName, String> nonRegistryOverrideCanonicalRepoNameLookup) {
+ ImmutableMap<RepositoryName, String> nonRegistryOverrideCanonicalRepoNameLookup,
+ ImmutableMap<String, CompiledModuleFile> includeLabelToCompiledModuleFile) {
return new AutoValue_ModuleFileValue_RootModuleFileValue(
- module, moduleFileHash, overrides, nonRegistryOverrideCanonicalRepoNameLookup);
+ module,
+ moduleFileHash,
+ overrides,
+ nonRegistryOverrideCanonicalRepoNameLookup,
+ includeLabelToCompiledModuleFile);
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleThreadContext.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleThreadContext.java
index a4de08f..8126ffc 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleThreadContext.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleThreadContext.java
@@ -34,13 +34,14 @@
import net.starlark.java.eval.StarlarkThread;
import net.starlark.java.syntax.Location;
-/** Context object for a Starlark thread evaluating the MODULE.bazel file and its imports. */
+/** Context object for a Starlark thread evaluating the MODULE.bazel file and files it includes. */
public class ModuleThreadContext {
private boolean moduleCalled = false;
private boolean hadNonModuleCall = false;
private final boolean ignoreDevDeps;
private final InterimModule.Builder module;
private final ImmutableMap<String, NonRegistryOverride> builtinModules;
+ @Nullable private final ImmutableMap<String, CompiledModuleFile> includeLabelToCompiledModuleFile;
private final Map<String, DepSpec> deps = new LinkedHashMap<>();
private final List<ModuleExtensionUsageBuilder> extensionUsageBuilders = new ArrayList<>();
private final Map<String, ModuleOverride> overrides = new HashMap<>();
@@ -50,7 +51,7 @@
throws EvalException {
ModuleThreadContext context = thread.getThreadLocal(ModuleThreadContext.class);
if (context == null) {
- throw Starlark.errorf("%s can only be called from MODULE.bazel and its imports", what);
+ throw Starlark.errorf("%s can only be called from MODULE.bazel and files it includes", what);
}
return context;
}
@@ -62,11 +63,12 @@
public ModuleThreadContext(
ImmutableMap<String, NonRegistryOverride> builtinModules,
ModuleKey key,
- @Nullable Registry registry,
- boolean ignoreDevDeps) {
- module = InterimModule.builder().setKey(key).setRegistry(registry);
+ boolean ignoreDevDeps,
+ @Nullable ImmutableMap<String, CompiledModuleFile> includeLabelToCompiledModuleFile) {
+ module = InterimModule.builder().setKey(key);
this.ignoreDevDeps = ignoreDevDeps;
this.builtinModules = builtinModules;
+ this.includeLabelToCompiledModuleFile = includeLabelToCompiledModuleFile;
}
record RepoNameUsage(String how, Location where) {}
@@ -224,6 +226,22 @@
}
}
+ public void include(String includeLabel, StarlarkThread thread)
+ throws InterruptedException, EvalException {
+ if (includeLabelToCompiledModuleFile == null) {
+ // This should never happen because compiling the non-root module file should have failed, way
+ // before evaluation started.
+ throw Starlark.errorf("trying to call `include()` from a non-root module");
+ }
+ var compiledModuleFile = includeLabelToCompiledModuleFile.get(includeLabel);
+ if (compiledModuleFile == null) {
+ // This should never happen because the file we're trying to include should have already been
+ // compiled before evaluation started.
+ throw Starlark.errorf("internal error; included file %s not compiled", includeLabel);
+ }
+ compiledModuleFile.runOnThread(thread);
+ }
+
public void addOverride(String moduleName, ModuleOverride override) throws EvalException {
ModuleOverride existingOverride = overrides.putIfAbsent(moduleName, override);
if (existingOverride != null) {
@@ -231,7 +249,7 @@
}
}
- public InterimModule buildModule() throws EvalException {
+ public InterimModule buildModule(@Nullable Registry registry) throws EvalException {
// Add builtin modules as default deps of the current module.
for (String builtinModule : builtinModules.keySet()) {
if (module.getKey().getName().equals(builtinModule)) {
@@ -257,6 +275,7 @@
extensionUsages.add(extensionUsageBuilder.buildUsage());
}
return module
+ .setRegistry(registry)
.setDeps(ImmutableMap.copyOf(deps))
.setOriginalDeps(ImmutableMap.copyOf(deps))
.setExtensionUsages(extensionUsages.build())
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD
index 5c074c8..09bb1dc 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD
@@ -31,6 +31,7 @@
"//src/main/java/com/google/devtools/build/lib/analysis:no_build_request_finished_event",
"//src/main/java/com/google/devtools/build/lib/bazel:resolved_event",
"//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common",
+ "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:exception",
"//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection",
"//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:module_extension",
"//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:module_file_fixup_event",
@@ -47,9 +48,6 @@
"//src/main/java/com/google/devtools/build/lib/packages",
"//src/main/java/com/google/devtools/build/lib/packages/semantics",
"//src/main/java/com/google/devtools/build/lib/pkgcache",
- "//src/main/java/com/google/devtools/build/lib/query2",
- "//src/main/java/com/google/devtools/build/lib/query2/common:cquery-node",
- "//src/main/java/com/google/devtools/build/lib/query2/engine",
"//src/main/java/com/google/devtools/build/lib/rules:repository/repository_directory_value",
"//src/main/java/com/google/devtools/build/lib/rules:repository/repository_function",
"//src/main/java/com/google/devtools/build/lib/rules:repository/resolved_file_value",
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java
index 4d40a27..d2e4e32 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java
@@ -37,7 +37,10 @@
import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule;
import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionEvent;
import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue;
+import com.google.devtools.build.lib.bazel.bzlmod.CompiledModuleFile;
+import com.google.devtools.build.lib.bazel.bzlmod.ExternalDepsException;
import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId;
+import com.google.devtools.build.lib.bazel.bzlmod.ModuleFile;
import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction;
import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue;
import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey;
@@ -633,18 +636,33 @@
"Unexpected error while reading module file after running buildozer: " + e.getMessage(),
Code.BUILDOZER_FAILED);
}
+ CompiledModuleFile compiledModuleFile;
+ try {
+ compiledModuleFile =
+ CompiledModuleFile.parseAndCompile(
+ ModuleFile.create(moduleFileContents, moduleFilePath.asPath().getPathString()),
+ ModuleKey.ROOT,
+ modTidyValue.starlarkSemantics(),
+ env.getRuntime().getRuleClassProvider().getBazelStarlarkEnvironment(),
+ env.getReporter());
+ } catch (ExternalDepsException e) {
+ return reportAndCreateFailureResult(
+ env,
+ "Unexpected error while compiling module file after running buildozer: "
+ + e.getMessage(),
+ Code.BUILDOZER_FAILED);
+ }
ModuleFileValue.RootModuleFileValue newRootModuleFileValue;
try {
newRootModuleFileValue =
ModuleFileFunction.evaluateRootModuleFile(
- moduleFileContents,
- moduleFilePath,
+ compiledModuleFile,
+ modTidyValue.includeLabelToCompiledModuleFile(),
ModuleFileFunction.getBuiltinModules(
env.getDirectories().getEmbeddedBinariesRoot()),
modTidyValue.moduleOverrides(),
modTidyValue.ignoreDevDeps(),
modTidyValue.starlarkSemantics(),
- env.getRuntime().getRuleClassProvider().getBazelStarlarkEnvironment(),
env.getReporter(),
// Not persisted to Skyframe.
SymbolGenerator.createTransient());
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/CompiledModuleFileTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/CompiledModuleFileTest.java
new file mode 100644
index 0000000..6a7125b
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/CompiledModuleFileTest.java
@@ -0,0 +1,178 @@
+// Copyright 2024 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+package com.google.devtools.build.lib.bazel.bzlmod;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.bazel.bzlmod.CompiledModuleFile.IncludeStatement;
+import net.starlark.java.syntax.Location;
+import net.starlark.java.syntax.ParserInput;
+import net.starlark.java.syntax.StarlarkFile;
+import net.starlark.java.syntax.SyntaxError;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class CompiledModuleFileTest {
+
+ private static ImmutableList<IncludeStatement> checkSyntax(String str) throws Exception {
+ return CompiledModuleFile.checkModuleFileSyntax(
+ StarlarkFile.parse(ParserInput.fromString(str, "test file")));
+ }
+
+ @Test
+ public void checkSyntax_good() throws Exception {
+ String program =
+ """
+ abc()
+ include("hullo")
+ foo = bar
+ """;
+ assertThat(checkSyntax(program))
+ .containsExactly(
+ new IncludeStatement("hullo", Location.fromFileLineColumn("test file", 2, 1)));
+ }
+
+ @Test
+ public void checkSyntax_good_multiple() throws Exception {
+ String program =
+ """
+ abc()
+ include("hullo")
+ foo = bar
+ include('world')
+ """;
+ assertThat(checkSyntax(program))
+ .containsExactly(
+ new IncludeStatement("hullo", Location.fromFileLineColumn("test file", 2, 1)),
+ new IncludeStatement("world", Location.fromFileLineColumn("test file", 4, 1)));
+ }
+
+ @Test
+ public void checkSyntax_good_multilineLiteral() throws Exception {
+ String program =
+ """
+ abc()
+ # Ludicrous as this may be, it's still valid syntax. Your funeral, etc...
+ include(\"""hullo
+ world\""")
+ """;
+ assertThat(checkSyntax(program))
+ .containsExactly(
+ new IncludeStatement("hullo\nworld", Location.fromFileLineColumn("test file", 3, 1)));
+ }
+
+ @Test
+ public void checkSyntax_good_benignUsageOfInclude() throws Exception {
+ String program =
+ """
+ myext = use_extension('whatever')
+ myext.include(include="hullo")
+ """;
+ assertThat(checkSyntax(program)).isEmpty();
+ }
+
+ @Test
+ public void checkSyntax_good_includeIdentifierReassigned() throws Exception {
+ String program =
+ """
+ include = print
+ # from this point on, we no longer check anything about `include` usage.
+ include('hello')
+ str(include)
+ exclude = include
+ """;
+ assertThat(checkSyntax(program)).isEmpty();
+ }
+
+ @Test
+ public void checkSyntax_bad_if() throws Exception {
+ String program =
+ """
+ abc()
+ if d > 3:
+ pass
+ """;
+ var ex = assertThrows(SyntaxError.Exception.class, () -> checkSyntax(program));
+ assertThat(ex)
+ .hasMessageThat()
+ .contains("`if` statements are not allowed in MODULE.bazel files");
+ }
+
+ @Test
+ public void checkSyntax_bad_assignIncludeResult() throws Exception {
+ String program =
+ """
+ foo = include('hello')
+ """;
+ var ex = assertThrows(SyntaxError.Exception.class, () -> checkSyntax(program));
+ assertThat(ex)
+ .hasMessageThat()
+ .contains("the `include` directive MUST be called directly at the top-level");
+ }
+
+ @Test
+ public void checkSyntax_bad_assignIncludeIdentifier() throws Exception {
+ String program =
+ """
+ foo = include
+ foo('hello')
+ """;
+ var ex = assertThrows(SyntaxError.Exception.class, () -> checkSyntax(program));
+ assertThat(ex)
+ .hasMessageThat()
+ .contains("the `include` directive MUST be called directly at the top-level");
+ }
+
+ @Test
+ public void checkSyntax_bad_multipleArgumentsToInclude() throws Exception {
+ String program =
+ """
+ include('hello', 'world')
+ """;
+ var ex = assertThrows(SyntaxError.Exception.class, () -> checkSyntax(program));
+ assertThat(ex)
+ .hasMessageThat()
+ .contains("the `include` directive MUST be called with exactly one positional");
+ }
+
+ @Test
+ public void checkSyntax_bad_keywordArgumentToInclude() throws Exception {
+ String program =
+ """
+ include(label='hello')
+ """;
+ var ex = assertThrows(SyntaxError.Exception.class, () -> checkSyntax(program));
+ assertThat(ex)
+ .hasMessageThat()
+ .contains("the `include` directive MUST be called with exactly one positional");
+ }
+
+ @Test
+ public void checkSyntax_bad_nonLiteralArgumentToInclude() throws Exception {
+ String program =
+ """
+ foo = 'hello'
+ include(foo)
+ """;
+ var ex = assertThrows(SyntaxError.Exception.class, () -> checkSyntax(program));
+ assertThat(ex)
+ .hasMessageThat()
+ .contains("the `include` directive MUST be called with exactly one positional");
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java
index 8e8ea8b..76d6f4e 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java
@@ -48,6 +48,10 @@
import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction;
import com.google.devtools.build.lib.skyframe.FileFunction;
import com.google.devtools.build.lib.skyframe.FileStateFunction;
+import com.google.devtools.build.lib.skyframe.IgnoredPackagePrefixesFunction;
+import com.google.devtools.build.lib.skyframe.LocalRepositoryLookupFunction;
+import com.google.devtools.build.lib.skyframe.PackageLookupFunction;
+import com.google.devtools.build.lib.skyframe.PackageLookupFunction.CrossRepositoryLabelViolationStrategy;
import com.google.devtools.build.lib.skyframe.PrecomputedFunction;
import com.google.devtools.build.lib.skyframe.PrecomputedValue;
import com.google.devtools.build.lib.skyframe.SkyFunctions;
@@ -56,6 +60,7 @@
import com.google.devtools.build.lib.testutil.TestRuleClassProvider;
import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
import com.google.devtools.build.lib.vfs.FileStateKey;
+import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.skyframe.EvaluationContext;
@@ -143,6 +148,21 @@
ruleClassProvider.getBazelStarlarkEnvironment(),
rootDirectory,
builtinModules))
+ .put(
+ SkyFunctions.PACKAGE_LOOKUP,
+ new PackageLookupFunction(
+ new AtomicReference<>(ImmutableSet.of()),
+ CrossRepositoryLabelViolationStrategy.ERROR,
+ BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY,
+ BazelSkyframeExecutorConstants.EXTERNAL_PACKAGE_HELPER))
+ .put(
+ SkyFunctions.IGNORED_PACKAGE_PREFIXES,
+ new IgnoredPackagePrefixesFunction(
+ /* ignoredPackagePrefixesFile= */ PathFragment.EMPTY_FRAGMENT))
+ .put(
+ SkyFunctions.LOCAL_REPOSITORY_LOOKUP,
+ new LocalRepositoryLookupFunction(
+ BazelSkyframeExecutorConstants.EXTERNAL_PACKAGE_HELPER))
.put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction())
.put(
SkyFunctions.REPOSITORY_DIRECTORY,
@@ -306,8 +326,178 @@
ModuleOverride bazelToolsOverride =
result.get(ModuleFileValue.KEY_FOR_ROOT_MODULE).getOverrides().get("bazel_tools");
assertThat(bazelToolsOverride).isInstanceOf(LocalPathOverride.class);
- assertThat((LocalPathOverride) bazelToolsOverride)
- .isEqualTo(LocalPathOverride.create("./bazel_tools_new"));
+ assertThat(bazelToolsOverride).isEqualTo(LocalPathOverride.create("./bazel_tools_new"));
+ }
+
+ @Test
+ public void testRootModule_include_good() throws Exception {
+ scratch.overwriteFile(
+ rootDirectory.getRelative("MODULE.bazel").getPathString(),
+ "module(name='aaa')",
+ "include('//java:MODULE.bazel.segment')",
+ "bazel_dep(name='foo', version='1.0')",
+ "register_toolchains('//:whatever')",
+ "include('//python:MODULE.bazel.segment')");
+ scratch.overwriteFile(rootDirectory.getRelative("java/BUILD").getPathString());
+ scratch.overwriteFile(
+ rootDirectory.getRelative("java/MODULE.bazel.segment").getPathString(),
+ "bazel_dep(name='java-foo', version='1.0')");
+ scratch.overwriteFile(rootDirectory.getRelative("python/BUILD").getPathString());
+ scratch.overwriteFile(
+ rootDirectory.getRelative("python/MODULE.bazel.segment").getPathString(),
+ "bazel_dep(name='py-foo', version='1.0', repo_name='python-foo')",
+ "single_version_override(module_name='java-foo', version='2.0')",
+ "include('//python:toolchains/MODULE.bazel.segment')");
+ scratch.overwriteFile(
+ rootDirectory.getRelative("python/toolchains/MODULE.bazel.segment").getPathString(),
+ "register_toolchains('//:python-whatever')");
+ FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+
+ EvaluationResult<RootModuleFileValue> result =
+ evaluator.evaluate(
+ ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext);
+ if (result.hasError()) {
+ fail(result.getError().toString());
+ }
+ RootModuleFileValue rootModuleFileValue = result.get(ModuleFileValue.KEY_FOR_ROOT_MODULE);
+ InterimModule expectedModule =
+ InterimModuleBuilder.create("aaa", "")
+ .setKey(ModuleKey.ROOT)
+ .addDep("java-foo", createModuleKey("java-foo", "1.0"))
+ .addDep("foo", createModuleKey("foo", "1.0"))
+ .addDep("python-foo", createModuleKey("py-foo", "1.0"))
+ .addToolchainsToRegister(ImmutableList.of("//:whatever", "//:python-whatever"))
+ .build();
+ assertThat(rootModuleFileValue.getModule()).isEqualTo(expectedModule);
+ // specifically assert the order of deps, which is significant; Map.equals semantics don't test
+ // this.
+ assertThat(rootModuleFileValue.getModule().getDeps())
+ .containsExactlyEntriesIn(expectedModule.getDeps())
+ .inOrder();
+ assertThat(rootModuleFileValue.getOverrides())
+ .containsExactly(
+ "java-foo",
+ SingleVersionOverride.create(
+ Version.parse("2.0"), "", ImmutableList.of(), ImmutableList.of(), 0));
+ }
+
+ @Test
+ public void testRootModule_include_bad_otherRepoLabel() throws Exception {
+ scratch.overwriteFile(
+ rootDirectory.getRelative("MODULE.bazel").getPathString(),
+ "module(name='aaa')",
+ "include('@haha//java:MODULE.bazel.segment')");
+ FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+
+ EvaluationResult<RootModuleFileValue> result =
+ evaluator.evaluate(
+ ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext);
+ assertThat(result.hasError()).isTrue();
+ assertThat(result.getError().toString()).contains("starting with double slashes");
+ }
+
+ @Test
+ public void testRootModule_include_bad_relativeLabel() throws Exception {
+ scratch.overwriteFile(
+ rootDirectory.getRelative("MODULE.bazel").getPathString(),
+ "module(name='aaa')",
+ "include(':MODULE.bazel.segment')");
+ FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+
+ EvaluationResult<RootModuleFileValue> result =
+ evaluator.evaluate(
+ ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext);
+ assertThat(result.hasError()).isTrue();
+ assertThat(result.getError().toString()).contains("starting with double slashes");
+ }
+
+ @Test
+ public void testRootModule_include_bad_badLabelSyntax() throws Exception {
+ scratch.overwriteFile(
+ rootDirectory.getRelative("MODULE.bazel").getPathString(),
+ "module(name='aaa')",
+ "include('//haha/:::')");
+ FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+
+ reporter.removeHandler(failFastHandler); // expect failures
+ EvaluationResult<RootModuleFileValue> result =
+ evaluator.evaluate(
+ ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext);
+ assertThat(result.hasError()).isTrue();
+ assertThat(result.getError().toString()).contains("bad include label");
+ }
+
+ @Test
+ public void testRootModule_include_bad_moduleAfterInclude() throws Exception {
+ scratch.overwriteFile(
+ rootDirectory.getRelative("MODULE.bazel").getPathString(),
+ "include('//java:MODULE.bazel.segment')");
+ scratch.overwriteFile(rootDirectory.getRelative("java/BUILD").getPathString());
+ scratch.overwriteFile(
+ rootDirectory.getRelative("java/MODULE.bazel.segment").getPathString(),
+ "module(name='bet-you-didnt-expect-this-didya')",
+ "bazel_dep(name='java-foo', version='1.0', repo_name='foo')");
+ FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+
+ reporter.removeHandler(failFastHandler); // expect failures
+ EvaluationResult<RootModuleFileValue> result =
+ evaluator.evaluate(
+ ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext);
+ assertThat(result.hasError()).isTrue();
+ assertContainsEvent("if module() is called, it must be called before any other functions");
+ }
+
+ @Test
+ public void testRootModule_include_bad_repoNameCollision() throws Exception {
+ scratch.overwriteFile(
+ rootDirectory.getRelative("MODULE.bazel").getPathString(),
+ "module(name='aaa')",
+ "include('//java:MODULE.bazel.segment')",
+ "include('//python:MODULE.bazel.segment')");
+ scratch.overwriteFile(rootDirectory.getRelative("java/BUILD").getPathString());
+ scratch.overwriteFile(
+ rootDirectory.getRelative("java/MODULE.bazel.segment").getPathString(),
+ "bazel_dep(name='java-foo', version='1.0', repo_name='foo')");
+ scratch.overwriteFile(rootDirectory.getRelative("python/BUILD").getPathString());
+ scratch.overwriteFile(
+ rootDirectory.getRelative("python/MODULE.bazel.segment").getPathString(),
+ "bazel_dep(name='python-foo', version='1.0', repo_name='foo')");
+ FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+
+ reporter.removeHandler(failFastHandler); // expect failures
+ EvaluationResult<RootModuleFileValue> result =
+ evaluator.evaluate(
+ ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext);
+ assertThat(result.hasError()).isTrue();
+ assertContainsEvent("The repo name 'foo' is already being used");
+ }
+
+ @Test
+ public void testRootModule_include_bad_tryingToLeakBindings() throws Exception {
+ scratch.overwriteFile(
+ rootDirectory.getRelative("MODULE.bazel").getPathString(),
+ "module(name='aaa')",
+ "FOO_NAME = 'foo'",
+ "include('//java:MODULE.bazel.segment')");
+ scratch.overwriteFile(rootDirectory.getRelative("java/BUILD").getPathString());
+ scratch.overwriteFile(
+ rootDirectory.getRelative("java/MODULE.bazel.segment").getPathString(),
+ "bazel_dep(name=FOO_NAME, version='1.0')");
+ FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+
+ reporter.removeHandler(failFastHandler); // expect failures
+ EvaluationResult<RootModuleFileValue> result =
+ evaluator.evaluate(
+ ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext);
+ assertThat(result.hasError()).isTrue();
+ assertContainsEvent("name 'FOO_NAME' is not defined");
}
@Test
@@ -359,6 +549,25 @@
}
@Test
+ public void testNonRootModuleCannotUseInclude() throws Exception {
+ FakeRegistry registry =
+ registryFactory
+ .newFakeRegistry("/foo")
+ .addModule(
+ createModuleKey("foo", "1.0"),
+ "module(name='foo',version='1.0')",
+ "include('//java:MODULE.bazel.segment')");
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+
+ EvaluationResult<RootModuleFileValue> result =
+ evaluator.evaluate(
+ ImmutableList.of(ModuleFileValue.key(createModuleKey("foo", "1.0"), null)),
+ evaluationContext);
+ assertThat(result.hasError()).isTrue();
+ assertThat(result.getError().toString()).contains("but it can only be used in the root module");
+ }
+
+ @Test
public void testLocalPathOverride() throws Exception {
// There is an override for B to use the local path "code_for_b", so we shouldn't even be
// looking at the registry.
diff --git a/src/test/py/bazel/bzlmod/bazel_module_test.py b/src/test/py/bazel/bzlmod/bazel_module_test.py
index 344d660..2b30d03 100644
--- a/src/test/py/bazel/bzlmod/bazel_module_test.py
+++ b/src/test/py/bazel/bzlmod/bazel_module_test.py
@@ -905,6 +905,44 @@
':bar',
])
+ def testInclude(self):
+ self.ScratchFile(
+ 'MODULE.bazel',
+ [
+ 'module(name="foo")',
+ 'bazel_dep(name="bbb", version="1.0")',
+ 'include("//java:MODULE.bazel.segment")',
+ ],
+ )
+ self.ScratchFile('java/BUILD')
+ self.ScratchFile(
+ 'java/MODULE.bazel.segment',
+ [
+ 'bazel_dep(name="aaa", version="1.0", repo_name="lol")',
+ ],
+ )
+ self.ScratchFile(
+ 'BUILD',
+ [
+ 'cc_binary(',
+ ' name = "main",',
+ ' srcs = ["main.cc"],',
+ ' deps = ["@lol//:lib_aaa"],',
+ ')',
+ ],
+ )
+ self.ScratchFile(
+ 'main.cc',
+ [
+ '#include "aaa.h"',
+ 'int main() {',
+ ' hello_aaa("main function");',
+ '}',
+ ],
+ )
+ _, stdout, _ = self.RunBazel(['run', '//:main'])
+ self.assertIn('main function => aaa@1.0', stdout)
+
if __name__ == '__main__':
absltest.main()
