Store remote registry file hashes in the lockfile
`MODULE.bazel`, `bazel_registry.json` and `source.json` files obtained from remote registries are stored in the repository cache and their hashes are collected in the lockfile. This speeds up incremental module resolutions, such as after adding a new `bazel_dep`.
Yanked versions are not stored in the lockfile. Their handling will be part of a follow-up PR.
Implements part of https://docs.google.com/document/d/1TjA7-M5njkI1F38IC0pm305S9EOmxcUwaCIvaSmansg/edit
Work towards #20369
Closes #21901.
PiperOrigin-RevId: 631195852
Change-Id: I35c30af7f9c3626bdbcb04c85b8c2502eeaafd3e
diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock
index ce72f16..344180e 100644
--- a/MODULE.bazel.lock
+++ b/MODULE.bazel.lock
@@ -2917,7 +2917,7 @@
"bzlTransitiveDigest": "tunTSmgwd2uvTzkCLtdbuCp0AI+WR+ftiPNqZ0rmcZk=",
"recordedFileInputs": {
"@@//MODULE.bazel": "eba5503742af5785c2d0d81d88e7407c7f23494b5162c055227435549b8774d1",
- "@@//src/test/tools/bzlmod/MODULE.bazel.lock": "4315fd0f326ba0b7493bc97ec47982b9dbdd631e12ac799f31016c72a40fdfa8"
+ "@@//src/test/tools/bzlmod/MODULE.bazel.lock": "547b1ca7af37ca0b4e7c7de36093d66b81d46440b58b41c76fe9d6df3af9ea52"
},
"recordedDirentsInputs": {},
"envVariables": {},
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
index ed54ed9..8378eed 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
@@ -15,12 +15,16 @@
package com.google.devtools.build.lib.bazel;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.CharMatcher;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
@@ -131,8 +135,8 @@
public static final String DEFAULT_CACHE_LOCATION = "cache/repos/v1";
// Default list of registries.
- public static final ImmutableList<String> DEFAULT_REGISTRIES =
- ImmutableList.of("https://bcr.bazel.build/");
+ public static final ImmutableSet<String> DEFAULT_REGISTRIES =
+ ImmutableSet.of("https://bcr.bazel.build/");
// A map of repository handlers that can be looked up by rule class name.
private final ImmutableMap<String, RepositoryFunction> repositoryHandlers;
@@ -150,7 +154,7 @@
private ImmutableMap<String, ModuleOverride> moduleOverrides = ImmutableMap.of();
private Optional<RootedPath> resolvedFileReplacingWorkspace = Optional.empty();
private FileSystem filesystem;
- private List<String> registries;
+ private ImmutableSet<String> registries;
private final AtomicBoolean ignoreDevDeps = new AtomicBoolean(false);
private CheckDirectDepsMode checkDirectDepsMode = CheckDirectDepsMode.WARNING;
private BazelCompatibilityMode bazelCompatibilityMode = BazelCompatibilityMode.ERROR;
@@ -495,7 +499,7 @@
}
if (repoOptions.registries != null && !repoOptions.registries.isEmpty()) {
- registries = repoOptions.registries;
+ registries = normalizeRegistries(repoOptions.registries);
} else {
registries = DEFAULT_REGISTRIES;
}
@@ -525,6 +529,14 @@
}
}
+ private static ImmutableSet<String> normalizeRegistries(List<String> registries) {
+ // Ensure that registries aren't duplicated even after `/modules/...` paths are appended to
+ // them.
+ return registries.stream()
+ .map(url -> CharMatcher.is('/').trimTrailingFrom(url))
+ .collect(toImmutableSet());
+ }
+
/**
* If the given path is absolute path, leave it as it is. If the given path is a relative path, it
* is relative to the current working directory. If the given path starts with '%workspace%, it is
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
index 6df8365..7729210 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
@@ -26,8 +26,6 @@
],
deps = [
"//src/main/java/com/google/devtools/build/lib/cmdline",
- "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
- "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
"//src/main/java/net/starlark/java/eval",
"//third_party:auto_value",
"//third_party:gson",
@@ -77,11 +75,12 @@
"Registry.java",
"RegistryFactory.java",
"RegistryFactoryImpl.java",
+ "RegistryFileDownloadEvent.java",
],
deps = [
":common",
+ "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache",
"//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
- "//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/events",
"//src/main/java/com/google/devtools/build/lib/profiler",
"//src/main/java/com/google/devtools/build/lib/util:os",
@@ -97,17 +96,15 @@
name = "bazel_lockfile_module",
srcs = ["BazelLockFileModule.java"],
deps = [
- ":exception",
":resolution",
":resolution_impl",
"//src/main/java/com/google/devtools/build/lib:runtime",
"//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:module_extension",
"//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options",
+ "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/skyframe:skyframe_cluster",
- "//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception",
"//src/main/java/com/google/devtools/build/lib/vfs",
- "//src/main/java/com/google/devtools/build/skyframe",
"//third_party:flogger",
"//third_party:guava",
"//third_party:jsr305",
@@ -144,6 +141,7 @@
"RegistryKey.java",
"RegistryOverride.java",
"RepoSpecKey.java",
+ "RepoSpecValue.java",
"SingleExtensionUsagesValue.java",
"SingleExtensionValue.java",
"SingleVersionOverride.java",
@@ -160,6 +158,7 @@
":root_module_file_fixup",
"//src/main/java/com/google/devtools/build/docgen/annot",
"//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories",
+ "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/events",
"//src/main/java/com/google/devtools/build/lib/packages",
@@ -226,6 +225,7 @@
"//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info",
"//src/main/java/com/google/devtools/build/lib/bazel:bazel_version",
"//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options",
+ "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache",
"//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
"//src/main/java/com/google/devtools/build/lib/bazel/repository/starlark",
"//src/main/java/com/google/devtools/build/lib/cmdline",
@@ -248,7 +248,6 @@
"//src/main/java/com/google/devtools/build/lib/util:os",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
- "//src/main/java/com/google/devtools/build/skyframe",
"//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
"//src/main/java/net/starlark/java/annot",
"//src/main/java/net/starlark/java/eval",
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java
index e5f53c6..7b11bd8 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java
@@ -19,9 +19,9 @@
import static com.google.common.collect.ImmutableBiMap.toImmutableBiMap;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static java.util.stream.Collectors.counting;
import static java.util.stream.Collectors.groupingBy;
-import static java.util.stream.Collectors.toSet;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -30,6 +30,7 @@
import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableTable;
import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue;
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode;
@@ -49,7 +50,6 @@
import com.google.devtools.build.skyframe.SkyValue;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Set;
import javax.annotation.Nullable;
/**
@@ -186,7 +186,6 @@
return null;
}
- ImmutableList<String> registries = ImmutableList.copyOf(ModuleFileFunction.REGISTRIES.get(env));
ImmutableMap<String, String> moduleOverrides =
ModuleFileFunction.MODULE_OVERRIDES.get(env).entrySet().stream()
.collect(
@@ -202,7 +201,7 @@
String envYanked = allowedYankedVersionsFromEnv.getValue();
return BzlmodFlagsAndEnvVars.create(
- registries,
+ ModuleFileFunction.REGISTRIES.get(env),
moduleOverrides,
yankedVersions,
nullToEmpty(envYanked),
@@ -257,14 +256,14 @@
ImmutableMap<ModuleKey, Module> depGraph) {
// Find modules with multiple versions in the dep graph. Currently, the only source of such
// modules is multiple_version_override.
- Set<String> multipleVersionsModules =
+ ImmutableSet<String> multipleVersionsModules =
depGraph.keySet().stream()
.collect(groupingBy(ModuleKey::getName, counting()))
.entrySet()
.stream()
.filter(entry -> entry.getValue() > 1)
.map(Entry::getKey)
- .collect(toSet());
+ .collect(toImmutableSet());
// If there is a unique version of this module in the entire dep graph, we elide the version
// from the canonical repository name. This has a number of benefits:
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java
index 500a577..20206b8 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java
@@ -19,6 +19,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import com.google.devtools.build.lib.actions.FileValue;
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode;
import com.google.devtools.build.lib.cmdline.LabelConstants;
@@ -55,7 +56,7 @@
private static final BzlmodFlagsAndEnvVars EMPTY_FLAGS =
BzlmodFlagsAndEnvVars.create(
- ImmutableList.of(), ImmutableMap.of(), ImmutableList.of(), "", false, "", "");
+ ImmutableSet.of(), ImmutableMap.of(), ImmutableList.of(), "", false, "", "");
private static final BazelLockFileValue EMPTY_LOCKFILE =
BazelLockFileValue.builder()
@@ -65,6 +66,7 @@
.setLocalOverrideHashes(ImmutableMap.of())
.setModuleDepGraph(ImmutableMap.of())
.setModuleExtensions(ImmutableMap.of())
+ .setRegistryFileHashes(ImmutableMap.of())
.build();
public BazelLockFileFunction(Path rootDirectory) {
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java
index 6116bd5..d5e75e7 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java
@@ -22,11 +22,11 @@
import com.google.common.flogger.GoogleLogger;
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions;
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode;
+import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
import com.google.devtools.build.lib.cmdline.LabelConstants;
import com.google.devtools.build.lib.runtime.BlazeModule;
import com.google.devtools.build.lib.runtime.CommandEnvironment;
import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
-import com.google.devtools.build.lib.util.AbruptExitException;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.Root;
@@ -34,6 +34,7 @@
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import javax.annotation.Nullable;
@@ -45,6 +46,7 @@
private SkyframeExecutor executor;
private Path workspaceRoot;
+ private boolean enabled;
@Nullable private BazelModuleResolutionEvent moduleResolutionEvent;
private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
@@ -53,20 +55,44 @@
public void beforeCommand(CommandEnvironment env) {
executor = env.getSkyframeExecutor();
workspaceRoot = env.getWorkspace();
- RepositoryOptions options = env.getOptions().getOptions(RepositoryOptions.class);
- if (options.lockfileMode.equals(LockfileMode.UPDATE)) {
- env.getEventBus().register(this);
- }
+
+ enabled =
+ env.getOptions().getOptions(RepositoryOptions.class).lockfileMode == LockfileMode.UPDATE;
+ moduleResolutionEvent = null;
+ env.getEventBus().register(this);
}
@Override
- public void afterCommand() throws AbruptExitException {
- if (moduleResolutionEvent == null) {
+ public void afterCommand() {
+ if (!enabled || moduleResolutionEvent == null) {
// Command does not use Bazel modules or the lockfile mode is not update.
// Since Skyframe caches events, they are replayed even when nothing has changed.
return;
}
+ BazelDepGraphValue depGraphValue;
+ BazelModuleResolutionValue moduleResolutionValue;
+ try {
+ depGraphValue =
+ (BazelDepGraphValue) executor.getEvaluator().getExistingValue(BazelDepGraphValue.KEY);
+ moduleResolutionValue =
+ (BazelModuleResolutionValue)
+ executor.getEvaluator().getExistingValue(BazelModuleResolutionValue.KEY);
+ } catch (InterruptedException e) {
+ // Not thrown in Bazel.
+ throw new IllegalStateException(e);
+ }
+
+ BazelLockFileValue oldLockfile = moduleResolutionEvent.getOnDiskLockfileValue();
+ ImmutableMap<String, Optional<Checksum>> fileHashes;
+ if (moduleResolutionValue == null) {
+ // BazelDepGraphFunction took the dep graph from the lockfile and didn't cause evaluation of
+ // BazelModuleResolutionFunction. The file hashes in the lockfile are still up-to-date.
+ fileHashes = oldLockfile.getRegistryFileHashes();
+ } else {
+ fileHashes = ImmutableSortedMap.copyOf(moduleResolutionValue.getRegistryFileHashes());
+ }
+
// All nodes corresponding to module extensions that have been evaluated in the current build
// are done at this point. Look up entries by eval keys to record results even if validation
// later fails due to invalid imports.
@@ -88,24 +114,16 @@
newExtensionInfos.put(key.argument(), value.getLockFileInfo().get());
}
});
+ var combinedExtensionInfos =
+ combineModuleExtensions(
+ oldLockfile.getModuleExtensions(), newExtensionInfos, depGraphValue);
- BazelDepGraphValue depGraphValue;
- try {
- depGraphValue =
- (BazelDepGraphValue) executor.getEvaluator().getExistingValue(BazelDepGraphValue.KEY);
- } catch (InterruptedException e) {
- // Not thrown in Bazel.
- throw new IllegalStateException(e);
- }
-
- BazelLockFileValue oldLockfile = moduleResolutionEvent.getOnDiskLockfileValue();
// Create an updated version of the lockfile, keeping only the extension results from the old
// lockfile that are still up-to-date and adding the newly resolved extension results.
BazelLockFileValue newLockfile =
moduleResolutionEvent.getResolutionOnlyLockfileValue().toBuilder()
- .setModuleExtensions(
- combineModuleExtensions(
- oldLockfile.getModuleExtensions(), newExtensionInfos, depGraphValue))
+ .setRegistryFileHashes(fileHashes)
+ .setModuleExtensions(combinedExtensionInfos)
.build();
// Write the new value to the file, but only if needed. This is not just a performance
@@ -115,7 +133,6 @@
if (!newLockfile.equals(oldLockfile)) {
updateLockfile(workspaceRoot, newLockfile);
}
- this.moduleResolutionEvent = null;
}
/**
@@ -140,8 +157,6 @@
var factorToLockedExtension = entry.getValue();
ModuleExtensionEvalFactors firstEntryFactors =
factorToLockedExtension.keySet().iterator().next();
- LockFileModuleExtension firstEntryExtension =
- factorToLockedExtension.values().iterator().next();
// All entries for a single extension share the same usages digest, so it suffices to check
// the first entry.
if (shouldKeepExtension(
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java
index 67d4db5..cd3eff7 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java
@@ -20,6 +20,7 @@
import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable;
import com.google.devtools.build.lib.skyframe.SkyFunctions;
import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant;
@@ -27,6 +28,7 @@
import com.google.devtools.build.skyframe.SkyValue;
import com.ryanharter.auto.value.gson.GenerateTypeAdapter;
import java.util.Map;
+import java.util.Optional;
/**
* The result of reading the lockfile. Contains the lockfile version, module hash, definitions of
@@ -44,7 +46,8 @@
static Builder builder() {
return new AutoValue_BazelLockFileValue.Builder()
.setLockFileVersion(LOCK_FILE_VERSION)
- .setModuleExtensions(ImmutableMap.of());
+ .setModuleExtensions(ImmutableMap.of())
+ .setRegistryFileHashes(ImmutableMap.of());
}
/** Current version of the lock file */
@@ -62,6 +65,9 @@
/** The post-selection dep graph retrieved from the lock file. */
public abstract ImmutableMap<ModuleKey, Module> getModuleDepGraph();
+ /** Hashes of files retrieved from registries. */
+ public abstract ImmutableMap<String, Optional<Checksum>> getRegistryFileHashes();
+
/** Mapping the extension id to the module extension data */
public abstract ImmutableMap<
ModuleExtensionId, ImmutableMap<ModuleExtensionEvalFactors, LockFileModuleExtension>>
@@ -82,6 +88,8 @@
public abstract Builder setModuleDepGraph(ImmutableMap<ModuleKey, Module> value);
+ public abstract Builder setRegistryFileHashes(ImmutableMap<String, Optional<Checksum>> value);
+
public abstract Builder setModuleExtensions(
ImmutableMap<
ModuleExtensionId,
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java
index 9f121a6..9d863d9 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java
@@ -31,6 +31,7 @@
import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue;
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode;
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode;
+import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.profiler.Profiler;
@@ -46,9 +47,11 @@
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import com.google.devtools.build.skyframe.SkyframeLookupResult;
+import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
+import java.util.SequencedMap;
import java.util.Set;
import javax.annotation.Nullable;
@@ -63,6 +66,14 @@
public static final Precomputed<BazelCompatibilityMode> BAZEL_COMPATIBILITY_MODE =
new Precomputed<>("bazel_compatibility_mode");
+ private record Result(
+ Selection.Result selectionResult,
+ ImmutableMap<String, Optional<Checksum>> registryFileHashes) {}
+
+ private static class ModuleResolutionComputeState implements Environment.SkyKeyComputeState {
+ Result discoverAndSelectResult;
+ }
+
@Override
@Nullable
public SkyValue compute(SkyKey skyKey, Environment env)
@@ -91,15 +102,17 @@
}
var state = env.getState(ModuleResolutionComputeState::new);
- if (state.selectionResult == null) {
- state.selectionResult = discoverAndSelect(env, root, allowedYankedVersions);
- if (state.selectionResult == null) {
+ if (state.discoverAndSelectResult == null) {
+ state.discoverAndSelectResult = discoverAndSelect(env, root, allowedYankedVersions);
+ if (state.discoverAndSelectResult == null) {
return null;
}
}
+ SequencedMap<String, Optional<Checksum>> registryFileHashes =
+ new LinkedHashMap<>(state.discoverAndSelectResult.registryFileHashes);
ImmutableSet<RepoSpecKey> repoSpecKeys =
- state.selectionResult.getResolvedDepGraph().values().stream()
+ state.discoverAndSelectResult.selectionResult.getResolvedDepGraph().values().stream()
// Modules with a null registry have a non-registry override. We don't need to
// fetch or store the repo spec in this case.
.filter(module -> module.getRegistry() != null)
@@ -108,11 +121,12 @@
SkyframeLookupResult repoSpecResults = env.getValuesAndExceptions(repoSpecKeys);
ImmutableMap.Builder<ModuleKey, RepoSpec> remoteRepoSpecs = ImmutableMap.builder();
for (RepoSpecKey repoSpecKey : repoSpecKeys) {
- RepoSpec repoSpec = (RepoSpec) repoSpecResults.get(repoSpecKey);
- if (repoSpec == null) {
+ RepoSpecValue repoSpecValue = (RepoSpecValue) repoSpecResults.get(repoSpecKey);
+ if (repoSpecValue == null) {
return null;
}
- remoteRepoSpecs.put(repoSpecKey.getModuleKey(), repoSpec);
+ remoteRepoSpecs.put(repoSpecKey.getModuleKey(), repoSpecValue.repoSpec());
+ registryFileHashes.putAll(repoSpecValue.registryFileHashes());
}
ImmutableMap<ModuleKey, Module> finalDepGraph;
@@ -120,36 +134,38 @@
Profiler.instance().profile(ProfilerTask.BZLMOD, "compute final dep graph")) {
finalDepGraph =
computeFinalDepGraph(
- state.selectionResult.getResolvedDepGraph(),
+ state.discoverAndSelectResult.selectionResult.getResolvedDepGraph(),
root.getOverrides(),
remoteRepoSpecs.buildOrThrow());
}
return BazelModuleResolutionValue.create(
- finalDepGraph, state.selectionResult.getUnprunedDepGraph());
+ finalDepGraph,
+ state.discoverAndSelectResult.selectionResult.getUnprunedDepGraph(),
+ ImmutableMap.copyOf(registryFileHashes));
}
@Nullable
- private static Selection.Result discoverAndSelect(
+ private static Result discoverAndSelect(
Environment env,
RootModuleFileValue root,
Optional<ImmutableSet<ModuleKey>> allowedYankedVersions)
throws BazelModuleResolutionFunctionException, InterruptedException {
- ImmutableMap<ModuleKey, InterimModule> initialDepGraph;
+ Discovery.Result discoveryResult;
try (SilentCloseable c = Profiler.instance().profile(ProfilerTask.BZLMOD, "discovery")) {
- initialDepGraph = Discovery.run(env, root);
+ discoveryResult = Discovery.run(env, root);
} catch (ExternalDepsException e) {
throw new BazelModuleResolutionFunctionException(e, Transience.PERSISTENT);
}
- if (initialDepGraph == null) {
+ if (discoveryResult == null) {
return null;
}
- verifyAllOverridesAreOnExistentModules(initialDepGraph, root.getOverrides());
+ verifyAllOverridesAreOnExistentModules(discoveryResult.depGraph(), root.getOverrides());
Selection.Result selectionResult;
try (SilentCloseable c = Profiler.instance().profile(ProfilerTask.BZLMOD, "selection")) {
- selectionResult = Selection.run(initialDepGraph, root.getOverrides());
+ selectionResult = Selection.run(discoveryResult.depGraph(), root.getOverrides());
} catch (ExternalDepsException e) {
throw new BazelModuleResolutionFunctionException(e, Transience.PERSISTENT);
}
@@ -176,7 +192,7 @@
try (SilentCloseable c =
Profiler.instance().profile(ProfilerTask.BZLMOD, "verify root module direct deps")) {
verifyRootModuleDirectDepsAreAccurate(
- initialDepGraph.get(ModuleKey.ROOT),
+ discoveryResult.depGraph().get(ModuleKey.ROOT),
resolvedDepGraph.get(ModuleKey.ROOT),
Objects.requireNonNull(CHECK_DIRECT_DEPENDENCIES.get(env)),
env.getListener());
@@ -195,7 +211,7 @@
checkNoYankedVersions(resolvedDepGraph, yankedVersionValues, allowedYankedVersions);
}
- return selectionResult;
+ return new Result(selectionResult, discoveryResult.registryFileHashes());
}
private static void verifyAllOverridesAreOnExistentModules(
@@ -337,10 +353,6 @@
return finalDepGraph.buildOrThrow();
}
- private static class ModuleResolutionComputeState implements Environment.SkyKeyComputeState {
- Selection.Result selectionResult;
- }
-
static class BazelModuleResolutionFunctionException extends SkyFunctionException {
BazelModuleResolutionFunctionException(ExternalDepsException e, Transience transience) {
super(e, transience);
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java
index 19c642c..6097103 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java
@@ -17,10 +17,12 @@
import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
import com.google.devtools.build.lib.skyframe.SkyFunctions;
import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
+import java.util.Optional;
/**
* The result of the selection process, containing both the pruned and the un-pruned dependency
@@ -46,9 +48,16 @@
*/
abstract ImmutableMap<ModuleKey, InterimModule> getUnprunedDepGraph();
+ /**
+ * Hashes of files obtained (or known to be missing) from registries while performing resolution.
+ */
+ abstract ImmutableMap<String, Optional<Checksum>> getRegistryFileHashes();
+
static BazelModuleResolutionValue create(
ImmutableMap<ModuleKey, Module> resolvedDepGraph,
- ImmutableMap<ModuleKey, InterimModule> unprunedDepGraph) {
- return new AutoValue_BazelModuleResolutionValue(resolvedDepGraph, unprunedDepGraph);
+ ImmutableMap<ModuleKey, InterimModule> unprunedDepGraph,
+ ImmutableMap<String, Optional<Checksum>> registryFileHashes) {
+ return new AutoValue_BazelModuleResolutionValue(
+ resolvedDepGraph, unprunedDepGraph, registryFileHashes);
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java
index a69e537..d425bc4 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java
@@ -17,6 +17,7 @@
import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import com.ryanharter.auto.value.gson.GenerateTypeAdapter;
/** Stores the values of flags and environment variables that affect the resolution */
@@ -25,7 +26,7 @@
abstract class BzlmodFlagsAndEnvVars {
public static BzlmodFlagsAndEnvVars create(
- ImmutableList<String> registries,
+ ImmutableSet<String> registries,
ImmutableMap<String, String> moduleOverrides,
ImmutableList<String> yankedVersions,
String envVarYankedVersions,
@@ -43,7 +44,7 @@
}
/** Registries provided via command line */
- public abstract ImmutableList<String> cmdRegistries();
+ public abstract ImmutableSet<String> cmdRegistries();
/** ModulesOverride provided via command line */
public abstract ImmutableMap<String, String> cmdModuleOverrides();
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Discovery.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Discovery.java
index 5da2540..6cc5eae 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Discovery.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Discovery.java
@@ -20,6 +20,7 @@
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.bazel.bzlmod.InterimModule.DepSpec;
import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue;
+import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
import com.google.devtools.build.lib.server.FailureDetails;
import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.build.skyframe.SkyKey;
@@ -28,10 +29,13 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Queue;
+import java.util.SequencedMap;
import java.util.Set;
import javax.annotation.Nullable;
@@ -43,13 +47,16 @@
final class Discovery {
private Discovery() {}
+ public record Result(
+ ImmutableMap<ModuleKey, InterimModule> depGraph,
+ ImmutableMap<String, Optional<Checksum>> registryFileHashes) {}
+
/**
* Runs module discovery. This function follows SkyFunction semantics (returns null if a Skyframe
* dependency is missing and this function needs a restart).
*/
@Nullable
- public static ImmutableMap<ModuleKey, InterimModule> run(
- Environment env, RootModuleFileValue root)
+ public static Result run(Environment env, RootModuleFileValue root)
throws InterruptedException, ExternalDepsException {
String rootModuleName = root.getModule().getName();
ImmutableMap<String, ModuleOverride> overrides = root.getOverrides();
@@ -60,9 +67,11 @@
.withDepSpecsTransformed(InterimModule.applyOverrides(overrides, rootModuleName)));
Queue<ModuleKey> unexpanded = new ArrayDeque<>();
Map<ModuleKey, ModuleKey> predecessors = new HashMap<>();
+ SequencedMap<String, Optional<Checksum>> registryFileHashes =
+ new LinkedHashMap<>(root.getRegistryFileHashes());
unexpanded.add(ModuleKey.ROOT);
while (!unexpanded.isEmpty()) {
- Set<SkyKey> unexpandedSkyKeys = new HashSet<>();
+ Set<SkyKey> unexpandedSkyKeys = new LinkedHashSet<>();
while (!unexpanded.isEmpty()) {
InterimModule module = depGraph.get(unexpanded.remove());
for (DepSpec depSpec : module.getDeps().values()) {
@@ -109,6 +118,7 @@
.getModule()
.withDepSpecsTransformed(
InterimModule.applyOverrides(overrides, rootModuleName)));
+ registryFileHashes.putAll(moduleFileValue.getRegistryFileHashes());
unexpanded.add(depKey);
}
}
@@ -116,6 +126,6 @@
if (env.valuesMissing()) {
return null;
}
- return ImmutableMap.copyOf(depGraph);
+ return new Result(ImmutableMap.copyOf(depGraph), ImmutableMap.copyOf(registryFileHashes));
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java
index a90bfb9..912b455 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java
@@ -26,6 +26,8 @@
import com.google.common.collect.ImmutableTable;
import com.google.common.collect.Table;
import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException;
+import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache;
+import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
import com.google.devtools.build.lib.cmdline.RepositoryName;
@@ -475,10 +477,62 @@
}
};
+ // This can't reuse the existing type adapter factory for Optional as we need to explicitly
+ // serialize null values but don't want to rely on GSON's serializeNulls.
+ private static final class OptionalChecksumTypeAdapterFactory implements TypeAdapterFactory {
+
+ @Nullable
+ @Override
+ public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> typeToken) {
+ if (typeToken.getRawType() != Optional.class) {
+ return null;
+ }
+ Type type = typeToken.getType();
+ if (!(type instanceof ParameterizedType)) {
+ return null;
+ }
+ Type elementType = ((ParameterizedType) type).getActualTypeArguments()[0];
+ if (elementType != Checksum.class) {
+ return null;
+ }
+ @SuppressWarnings("unchecked")
+ TypeAdapter<T> typeAdapter = (TypeAdapter<T>) new OptionalChecksumTypeAdapter();
+ return typeAdapter;
+ }
+
+ private static class OptionalChecksumTypeAdapter extends TypeAdapter<Optional<Checksum>> {
+ // This value must not be a valid checksum string.
+ private static final String NOT_FOUND_MARKER = "not found";
+
+ @Override
+ public void write(JsonWriter jsonWriter, Optional<Checksum> checksum) throws IOException {
+ if (checksum.isPresent()) {
+ jsonWriter.value(checksum.get().toString());
+ } else {
+ jsonWriter.value(NOT_FOUND_MARKER);
+ }
+ }
+
+ @Override
+ public Optional<Checksum> read(JsonReader jsonReader) throws IOException {
+ String checksumString = jsonReader.nextString();
+ if (checksumString.equals(NOT_FOUND_MARKER)) {
+ return Optional.empty();
+ }
+ try {
+ return Optional.of(Checksum.fromString(RepositoryCache.KeyType.SHA256, checksumString));
+ } catch (Checksum.InvalidChecksumException e) {
+ throw new JsonParseException(String.format("Invalid checksum: %s", checksumString), e);
+ }
+ }
+ }
+ }
+
public static Gson createLockFileGson(Path moduleFilePath, Path workspaceRoot) {
return newGsonBuilder()
.setPrettyPrinting()
.registerTypeAdapterFactory(new LocationTypeAdapterFactory(moduleFilePath, workspaceRoot))
+ .registerTypeAdapterFactory(new OptionalChecksumTypeAdapterFactory())
.create();
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
index e872524..578e78a 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
@@ -21,8 +21,10 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException;
+import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
+import com.google.devtools.build.lib.events.StoredEventHandler;
import com.google.devtools.build.lib.profiler.Profiler;
import com.google.devtools.build.lib.profiler.ProfilerTask;
import com.google.devtools.build.lib.profiler.SilentCloseable;
@@ -49,6 +51,15 @@
*/
public class IndexRegistry implements Registry {
+ /**
+ * How to handle the list of file hashes known from the lockfile when downloading files from the
+ * registry.
+ */
+ public enum KnownFileHashesMode {
+ IGNORE,
+ USE_AND_UPDATE;
+ }
+
/** The unresolved version of the url. Ex: has %workspace% placeholder */
private final String unresolvedUri;
@@ -56,7 +67,10 @@
private final DownloadManager downloadManager;
private final Map<String, String> clientEnv;
private final Gson gson;
+ private final ImmutableMap<String, Optional<Checksum>> knownFileHashes;
+ private final KnownFileHashesMode knownFileHashesMode;
private volatile Optional<BazelRegistryJson> bazelRegistryJson;
+ private volatile StoredEventHandler bazelRegistryJsonEvents;
private static final String SOURCE_JSON_FILENAME = "source.json";
@@ -64,7 +78,9 @@
URI uri,
String unresolvedUri,
DownloadManager downloadManager,
- Map<String, String> clientEnv) {
+ Map<String, String> clientEnv,
+ ImmutableMap<String, Optional<Checksum>> knownFileHashes,
+ KnownFileHashesMode knownFileHashesMode) {
this.uri = uri;
this.unresolvedUri = unresolvedUri;
this.downloadManager = downloadManager;
@@ -73,6 +89,8 @@
new GsonBuilder()
.setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
.create();
+ this.knownFileHashes = knownFileHashes;
+ this.knownFileHashesMode = knownFileHashesMode;
}
@Override
@@ -92,14 +110,46 @@
}
/** Grabs a file from the given URL. Returns {@link Optional#empty} if the file doesn't exist. */
- private Optional<byte[]> grabFile(String url, ExtendedEventHandler eventHandler)
+ private Optional<byte[]> grabFile(
+ String url, ExtendedEventHandler eventHandler, boolean useChecksum)
throws IOException, InterruptedException {
+ var maybeContent = doGrabFile(url, eventHandler, useChecksum);
+ if (knownFileHashesMode == KnownFileHashesMode.USE_AND_UPDATE && useChecksum) {
+ eventHandler.post(RegistryFileDownloadEvent.create(url, maybeContent));
+ }
+ return maybeContent;
+ }
+
+ private Optional<byte[]> doGrabFile(
+ String url, ExtendedEventHandler eventHandler, boolean useChecksum)
+ throws IOException, InterruptedException {
+ Optional<Checksum> checksum;
+ if (knownFileHashesMode != KnownFileHashesMode.IGNORE && useChecksum) {
+ Optional<Checksum> knownChecksum = knownFileHashes.get(url);
+ if (knownChecksum == null) {
+ // This is a new file, download without providing a checksum.
+ checksum = Optional.empty();
+ } else if (knownChecksum.isEmpty()) {
+ // The file is known to not exist, so don't attempt to download it.
+ return Optional.empty();
+ } else {
+ // The file is known, download with a checksum to potentially obtain a repository cache hit
+ // and ensure that the remote file hasn't changed.
+ checksum = knownChecksum;
+ }
+ } else {
+ checksum = Optional.empty();
+ }
try (SilentCloseable c =
Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "download file: " + url)) {
return Optional.of(
- downloadManager.downloadAndReadOneUrl(new URL(url), eventHandler, clientEnv));
+ downloadManager.downloadAndReadOneUrl(new URL(url), eventHandler, clientEnv, checksum));
} catch (FileNotFoundException e) {
return Optional.empty();
+ } catch (IOException e) {
+ // Include the URL in the exception message for easier debugging.
+ throw new IOException(
+ "Failed to fetch registry file %s: %s".formatted(url, e.getMessage()), e);
}
}
@@ -109,7 +159,8 @@
String url =
constructUrl(
uri.toString(), "modules", key.getName(), key.getVersion().toString(), "MODULE.bazel");
- return grabFile(url, eventHandler).map(content -> ModuleFile.create(content, url));
+ Optional<byte[]> maybeContent = grabFile(url, eventHandler, /* useChecksum= */ true);
+ return maybeContent.map(content -> ModuleFile.create(content, url));
}
/** Represents fields available in {@code bazel_registry.json} for the registry. */
@@ -153,22 +204,20 @@
* Grabs a JSON file from the given URL, and returns its content. Returns {@link Optional#empty}
* if the file doesn't exist.
*/
- private Optional<String> grabJsonFile(String url, ExtendedEventHandler eventHandler)
+ private Optional<String> grabJsonFile(
+ String url, ExtendedEventHandler eventHandler, boolean useChecksum)
throws IOException, InterruptedException {
- Optional<byte[]> bytes = grabFile(url, eventHandler);
- if (bytes.isEmpty()) {
- return Optional.empty();
- }
- return Optional.of(new String(bytes.get(), UTF_8));
+ return grabFile(url, eventHandler, useChecksum).map(value -> new String(value, UTF_8));
}
/**
* Grabs a JSON file from the given URL, and returns it as a parsed object with fields in {@code
* T}. Returns {@link Optional#empty} if the file doesn't exist.
*/
- private <T> Optional<T> grabJson(String url, Class<T> klass, ExtendedEventHandler eventHandler)
+ private <T> Optional<T> grabJson(
+ String url, Class<T> klass, ExtendedEventHandler eventHandler, boolean useChecksum)
throws IOException, InterruptedException {
- Optional<String> jsonString = grabJsonFile(url, eventHandler);
+ Optional<String> jsonString = grabJsonFile(url, eventHandler, useChecksum);
if (jsonString.isEmpty() || jsonString.get().isBlank()) {
return Optional.empty();
}
@@ -195,7 +244,7 @@
key.getName(),
key.getVersion().toString(),
SOURCE_JSON_FILENAME);
- Optional<String> jsonString = grabJsonFile(jsonUrl, eventHandler);
+ Optional<String> jsonString = grabJsonFile(jsonUrl, eventHandler, /* useChecksum= */ true);
if (jsonString.isEmpty()) {
throw new FileNotFoundException(
String.format("Module %s's %s not found in registry %s", key, SOURCE_JSON_FILENAME, uri));
@@ -232,14 +281,18 @@
if (bazelRegistryJson == null) {
synchronized (this) {
if (bazelRegistryJson == null) {
+ var storedEventHandler = new StoredEventHandler();
bazelRegistryJson =
grabJson(
constructUrl(uri.toString(), "bazel_registry.json"),
BazelRegistryJson.class,
- eventHandler);
+ storedEventHandler,
+ /* useChecksum= */ true);
+ bazelRegistryJsonEvents = storedEventHandler;
}
}
}
+ bazelRegistryJsonEvents.replayOn(eventHandler);
return bazelRegistryJson;
}
@@ -348,7 +401,9 @@
grabJson(
constructUrl(uri.toString(), "modules", moduleName, "metadata.json"),
MetadataJson.class,
- eventHandler);
+ eventHandler,
+ // metadata.json is not immutable
+ /* useChecksum= */ false);
if (metadataJson.isEmpty()) {
return Optional.empty();
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java
index 542a62e..65f8d99 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java
@@ -22,6 +22,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.devtools.build.lib.actions.FileValue;
import com.google.devtools.build.lib.bazel.bzlmod.CompiledModuleFile.IncludeStatement;
@@ -34,6 +35,7 @@
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
+import com.google.devtools.build.lib.events.StoredEventHandler;
import com.google.devtools.build.lib.packages.BazelStarlarkEnvironment;
import com.google.devtools.build.lib.packages.StarlarkExportable;
import com.google.devtools.build.lib.profiler.Profiler;
@@ -80,7 +82,8 @@
*/
public class ModuleFileFunction implements SkyFunction {
- public static final Precomputed<List<String>> REGISTRIES = new Precomputed<>("registries");
+ public static final Precomputed<ImmutableSet<String>> REGISTRIES =
+ new Precomputed<>("registries");
public static final Precomputed<Boolean> IGNORE_DEV_DEPS =
new Precomputed<>("ignore_dev_dependency");
@@ -157,6 +160,7 @@
if (getModuleFileResult == null) {
return null;
}
+ getModuleFileResult.downloadEventHandler.replayOn(env.getListener());
String moduleFileHash =
new Fingerprint().addBytes(getModuleFileResult.moduleFile.getContent()).hexDigestAndReset();
@@ -217,8 +221,11 @@
module.getVersion());
}
-
- return NonRootModuleFileValue.create(module, moduleFileHash);
+ return NonRootModuleFileValue.create(
+ module,
+ moduleFileHash,
+ RegistryFileDownloadEvent.collectToMap(
+ getModuleFileResult.downloadEventHandler.getPosts()));
}
@Nullable
@@ -529,7 +536,10 @@
*
* @param registry can be null if this module has a non-registry override.
*/
- private record GetModuleFileResult(ModuleFile moduleFile, @Nullable Registry registry) {}
+ private record GetModuleFileResult(
+ ModuleFile moduleFile,
+ @Nullable Registry registry,
+ StoredEventHandler downloadEventHandler) {}
@Nullable
private GetModuleFileResult getModuleFile(
@@ -560,7 +570,8 @@
ModuleFile.create(
readModuleFile(moduleFilePath.asPath()),
moduleFileLabel.getUnambiguousCanonicalForm()),
- /* registry= */ null);
+ /* registry= */ null,
+ new StoredEventHandler());
}
// Otherwise, we should get the module file from a registry.
@@ -573,13 +584,11 @@
+ " non-registry override?",
key.getName());
}
- // TODO(wyv): Move registry object creation to BazelRepositoryModule so we don't repeatedly
- // create them, and we can better report the error (is it a flag error or override error?).
- List<String> registries = Objects.requireNonNull(REGISTRIES.get(env));
+ ImmutableSet<String> registries = Objects.requireNonNull(REGISTRIES.get(env));
if (override instanceof RegistryOverride registryOverride) {
String overrideRegistry = registryOverride.getRegistry();
if (!overrideRegistry.isEmpty()) {
- registries = ImmutableList.of(overrideRegistry);
+ registries = ImmutableSet.of(overrideRegistry);
}
} else if (override != null) {
// This should never happen.
@@ -607,13 +616,14 @@
// Now go through the list of registries and use the first one that contains the requested
// module.
+ StoredEventHandler downloadEventHandler = new StoredEventHandler();
for (Registry registry : registryObjects) {
try {
- Optional<ModuleFile> moduleFile = registry.getModuleFile(key, env.getListener());
+ Optional<ModuleFile> moduleFile = registry.getModuleFile(key, downloadEventHandler);
if (moduleFile.isEmpty()) {
continue;
}
- return new GetModuleFileResult(moduleFile.get(), registry);
+ return new GetModuleFileResult(moduleFile.get(), registry, downloadEventHandler);
} catch (IOException e) {
throw errorf(
Code.ERROR_ACCESSING_REGISTRY, e, "Error accessing registry %s", registry.getUrl());
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java
index 7c7e38f..da64efb 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java
@@ -18,12 +18,14 @@
import com.google.auto.value.AutoValue;
import com.google.auto.value.extension.memoized.Memoized;
import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.skyframe.SkyFunctions;
import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec;
import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
+import java.util.Optional;
import javax.annotation.Nullable;
/** The result of {@link ModuleFileFunction}. */
@@ -41,12 +43,22 @@
/** The hash string of Module.bazel (using SHA256) */
public abstract String getModuleFileHash();
+ /**
+ * Hashes of files obtained (or known to be missing) from registries while obtaining this module
+ * file.
+ */
+ public abstract ImmutableMap<String, Optional<Checksum>> getRegistryFileHashes();
+
/** The {@link ModuleFileValue} for non-root modules. */
@AutoValue
public abstract static class NonRootModuleFileValue extends ModuleFileValue {
- public static NonRootModuleFileValue create(InterimModule module, String moduleFileHash) {
- return new AutoValue_ModuleFileValue_NonRootModuleFileValue(module, moduleFileHash);
+ public static NonRootModuleFileValue create(
+ InterimModule module,
+ String moduleFileHash,
+ ImmutableMap<String, Optional<Checksum>> registryFileHashes) {
+ return new AutoValue_ModuleFileValue_NonRootModuleFileValue(
+ module, moduleFileHash, registryFileHashes);
}
}
@@ -77,6 +89,12 @@
*/
public abstract ImmutableMap<String, CompiledModuleFile> getIncludeLabelToCompiledModuleFile();
+ @Override
+ public ImmutableMap<String, Optional<Checksum>> getRegistryFileHashes() {
+ // The root module is not obtained from a registry.
+ return ImmutableMap.of();
+ }
+
public static RootModuleFileValue create(
InterimModule module,
String moduleFileHash,
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java
index 03ba85d..e24b345 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java
@@ -15,7 +15,10 @@
package com.google.devtools.build.lib.bazel.bzlmod;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
import java.net.URISyntaxException;
+import java.util.Optional;
/** A factory type for {@link Registry}. */
public interface RegistryFactory {
@@ -25,5 +28,6 @@
*
* <p>Outside of tests, only {@link RegistryFunction} should call this method.
*/
- Registry createRegistry(String url) throws URISyntaxException;
+ Registry createRegistry(String url, ImmutableMap<String, Optional<Checksum>> fileHashes)
+ throws URISyntaxException;
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java
index d943e69..06efd40 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java
@@ -15,11 +15,15 @@
package com.google.devtools.build.lib.bazel.bzlmod;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.bazel.bzlmod.IndexRegistry.KnownFileHashesMode;
+import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager;
import com.google.devtools.build.lib.vfs.Path;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Map;
+import java.util.Optional;
import java.util.function.Supplier;
/** Prod implementation of {@link RegistryFactory}. */
@@ -38,7 +42,9 @@
}
@Override
- public Registry createRegistry(String unresolvedUrl) throws URISyntaxException {
+ public Registry createRegistry(
+ String unresolvedUrl, ImmutableMap<String, Optional<Checksum>> knownFileHashes)
+ throws URISyntaxException {
URI uri = new URI(unresolvedUrl.replace("%workspace%", workspacePath.getPathString()));
if (uri.getScheme() == null) {
throw new URISyntaxException(
@@ -52,10 +58,19 @@
"Registry URL path is not valid -- did you mean to use file:///foo/bar "
+ "or file:///c:/foo/bar for Windows?");
}
- return switch (uri.getScheme()) {
- case "http", "https", "file" ->
- new IndexRegistry(uri, unresolvedUrl, downloadManager, clientEnvironmentSupplier.get());
- default -> throw new URISyntaxException(uri.toString(), "Unrecognized registry URL protocol");
- };
+ var knownFileHashesMode =
+ switch (uri.getScheme()) {
+ case "http", "https" -> KnownFileHashesMode.USE_AND_UPDATE;
+ case "file" -> KnownFileHashesMode.IGNORE;
+ default ->
+ throw new URISyntaxException(uri.toString(), "Unrecognized registry URL protocol");
+ };
+ return new IndexRegistry(
+ uri,
+ unresolvedUrl,
+ downloadManager,
+ clientEnvironmentSupplier.get(),
+ knownFileHashes,
+ knownFileHashesMode);
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFileDownloadEvent.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFileDownloadEvent.java
new file mode 100644
index 0000000..941f023
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFileDownloadEvent.java
@@ -0,0 +1,53 @@
+// Copyright 2024 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package com.google.devtools.build.lib.bazel.bzlmod;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.hash.Hashing;
+import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache;
+import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
+import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable;
+import java.util.Collection;
+import java.util.Optional;
+
+/** Event that records the fact that a file has been downloaded from a remote registry. */
+public record RegistryFileDownloadEvent(String uri, Optional<Checksum> checksum)
+ implements Postable {
+
+ public static RegistryFileDownloadEvent create(String uri, Optional<byte[]> content) {
+ return new RegistryFileDownloadEvent(uri, content.map(RegistryFileDownloadEvent::computeHash));
+ }
+
+ static ImmutableMap<String, Optional<Checksum>> collectToMap(Collection<Postable> postables) {
+ ImmutableMap.Builder<String, Optional<Checksum>> builder = ImmutableMap.builder();
+ for (Postable postable : postables) {
+ if (postable instanceof RegistryFileDownloadEvent event) {
+ builder.put(event.uri(), event.checksum());
+ }
+ }
+ return builder.buildKeepingLast();
+ }
+
+ private static Checksum computeHash(byte[] bytes) {
+ try {
+ return Checksum.fromString(
+ RepositoryCache.KeyType.SHA256, Hashing.sha256().hashBytes(bytes).toString());
+ } catch (Checksum.InvalidChecksumException e) {
+ // This can't happen since HashCode.toString() always returns a valid hash.
+ throw new IllegalStateException(e);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java
index e99fc2b..0c6c12a 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java
@@ -35,9 +35,14 @@
@Nullable
public SkyValue compute(SkyKey skyKey, Environment env)
throws InterruptedException, RegistryException {
+ BazelLockFileValue lockfile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY);
+ if (lockfile == null) {
+ return null;
+ }
+
RegistryKey key = (RegistryKey) skyKey.argument();
try {
- return registryFactory.createRegistry(key.getUrl());
+ return registryFactory.createRegistry(key.getUrl(), lockfile.getRegistryFileHashes());
} catch (URISyntaxException e) {
throw new RegistryException(
ExternalDepsException.withCauseAndMessage(
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java
index 201312b..d6ad089 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java
@@ -16,7 +16,6 @@
import com.google.auto.value.AutoValue;
import com.google.common.collect.Maps;
-import com.google.devtools.build.skyframe.SkyValue;
import com.ryanharter.auto.value.gson.GenerateTypeAdapter;
import javax.annotation.Nullable;
@@ -26,7 +25,7 @@
*/
@AutoValue
@GenerateTypeAdapter
-public abstract class RepoSpec implements SkyValue {
+public abstract class RepoSpec {
/**
* The unambiguous canonical label string for the bzl file this repository rule is defined in,
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecFunction.java
index b9f4369..88d7664 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecFunction.java
@@ -15,6 +15,7 @@
package com.google.devtools.build.lib.bazel.bzlmod;
+import com.google.devtools.build.lib.events.StoredEventHandler;
import com.google.devtools.build.lib.profiler.Profiler;
import com.google.devtools.build.lib.profiler.ProfilerTask;
import com.google.devtools.build.lib.profiler.SilentCloseable;
@@ -43,10 +44,12 @@
return null;
}
+ StoredEventHandler downloadEvents = new StoredEventHandler();
+ RepoSpec repoSpec;
try (SilentCloseable c =
Profiler.instance()
.profile(ProfilerTask.BZLMOD, () -> "compute repo spec: " + key.getModuleKey())) {
- return registry.getRepoSpec(key.getModuleKey(), env.getListener());
+ repoSpec = registry.getRepoSpec(key.getModuleKey(), downloadEvents);
} catch (IOException e) {
throw new RepoSpecException(
ExternalDepsException.withCauseAndMessage(
@@ -55,6 +58,9 @@
"Unable to get module repo spec for %s from registry",
key.getModuleKey()));
}
+ downloadEvents.replayOn(env.getListener());
+ return RepoSpecValue.create(
+ repoSpec, RegistryFileDownloadEvent.collectToMap(downloadEvents.getPosts()));
}
static final class RepoSpecException extends SkyFunctionException {
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecValue.java
new file mode 100644
index 0000000..5e37bba
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecValue.java
@@ -0,0 +1,35 @@
+// Copyright 2021 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package com.google.devtools.build.lib.bazel.bzlmod;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
+import com.google.devtools.build.skyframe.SkyValue;
+import java.util.Optional;
+
+/** The value for {@link RepoSpecFunction}. */
+@AutoValue
+public abstract class RepoSpecValue implements SkyValue {
+ public abstract RepoSpec repoSpec();
+
+ public abstract ImmutableMap<String, Optional<Checksum>> registryFileHashes();
+
+ public static RepoSpecValue create(
+ RepoSpec repoSpec, ImmutableMap<String, Optional<Checksum>> registryFileHashes) {
+ return new AutoValue_RepoSpecValue(repoSpec, registryFileHashes);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionValue.java
index b0a9101..2182ef0 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionValue.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionValue.java
@@ -46,8 +46,8 @@
public abstract ImmutableBiMap<RepositoryName, String> getCanonicalRepoNameToInternalNames();
/**
- * Returns the information stored about the extension in the lockfile. Is empty if the lockfile
- * mode is not UPDATE.
+ * Returns the information stored about the extension in the lockfile. Non-empty if and only if
+ * the lockfile mode is UPDATE.
*/
public abstract Optional<LockFileModuleExtension.WithFactors> getLockFileInfo();
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java
index 402f96d..83e1a19 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java
@@ -110,9 +110,7 @@
this.useHardlinks = useHardlinks;
}
- /**
- * @return true iff the cache path is set.
- */
+ /** Returns true iff the cache path is set. */
public boolean isEnabled() {
return repositoryCachePath != null;
}
@@ -157,11 +155,46 @@
* entry with the given cacheKey was added with this String given.
* @return The Path value where the cache value has been copied to. If cache value does not exist,
* return null.
- * @throws IOException
*/
@Nullable
public Path get(String cacheKey, Path targetPath, KeyType keyType, String canonicalId)
throws IOException, InterruptedException {
+ Path cacheValue = findCacheValue(cacheKey, keyType, canonicalId);
+ if (cacheValue == null) {
+ return null;
+ }
+
+ targetPath.getParentDirectory().createDirectoryAndParents();
+ if (useHardlinks) {
+ FileSystemUtils.createHardLink(targetPath, cacheValue);
+ } else {
+ FileSystemUtils.copyFile(cacheValue, targetPath);
+ }
+
+ return targetPath;
+ }
+
+ /**
+ * Get the content of a cached value, if it exists.
+ *
+ * @param cacheKey The string key to cache the value by.
+ * @param keyType The type of key used. See: KeyType
+ * @return The bytes of the cache value. If cache value does not exist, returns null.
+ */
+ @Nullable
+ public byte[] getBytes(String cacheKey, KeyType keyType)
+ throws IOException, InterruptedException {
+ Path cacheValue = findCacheValue(cacheKey, keyType, /* canonicalId= */ null);
+ if (cacheValue == null) {
+ return null;
+ }
+
+ return FileSystemUtils.readContent(cacheValue);
+ }
+
+ @Nullable
+ private Path findCacheValue(String cacheKey, KeyType keyType, String canonicalId)
+ throws IOException, InterruptedException {
Preconditions.checkState(isEnabled());
assertKeyIsValid(cacheKey, keyType);
@@ -186,33 +219,30 @@
}
}
- targetPath.getParentDirectory().createDirectoryAndParents();
- if (useHardlinks) {
- FileSystemUtils.createHardLink(targetPath, cacheValue);
- } else {
- FileSystemUtils.copyFile(cacheValue, targetPath);
- }
-
try {
FileSystemUtils.touchFile(cacheValue);
} catch (IOException e) {
// Ignore, because the cache might be on a read-only volume.
}
- return targetPath;
+ return cacheValue;
+ }
+
+ interface FileWriter {
+ void writeTo(Path name) throws IOException;
}
/**
* Copies a value from a specified path into the cache.
*
* @param cacheKey The string key to cache the value by.
- * @param sourcePath The path of the value to be cached.
+ * @param fileWriter A function that writes the value to a given file.
* @param keyType The type of key used. See: KeyType
* @param canonicalId If set to a non-empty String associate the file with this name, allowing
* restricted cache lookups later.
- * @throws IOException
*/
- public void put(String cacheKey, Path sourcePath, KeyType keyType, String canonicalId)
+ private void storeCacheValue(
+ String cacheKey, FileWriter fileWriter, KeyType keyType, String canonicalId)
throws IOException {
Preconditions.checkState(isEnabled());
@@ -223,7 +253,7 @@
Path cacheValue = cacheEntry.getRelative(DEFAULT_CACHE_FILENAME);
Path tmpName = cacheEntry.getRelative(TMP_PREFIX + UUID.randomUUID());
cacheEntry.createDirectoryAndParents();
- FileSystemUtils.copyFile(sourcePath, tmpName);
+ fileWriter.writeTo(tmpName);
try {
tmpName.renameTo(cacheValue);
} catch (FileAccessException e) {
@@ -245,13 +275,52 @@
}
/**
+ * Copies a value from a specified path into the cache.
+ *
+ * @param cacheKey The string key to cache the value by.
+ * @param sourcePath The path of the value to be cached.
+ * @param keyType The type of key used. See: KeyType
+ * @param canonicalId If set to a non-empty String associate the file with this name, allowing
+ * restricted cache lookups later.
+ */
+ public void put(String cacheKey, Path sourcePath, KeyType keyType, String canonicalId)
+ throws IOException {
+ storeCacheValue(
+ cacheKey, tmpName -> FileSystemUtils.copyFile(sourcePath, tmpName), keyType, canonicalId);
+ }
+
+ /**
+ * Adds an in-memory value to the cache.
+ *
+ * @param content The byte content of the value to be cached.
+ * @param keyType The type of key used. See: KeyType
+ */
+ public void put(String cacheKey, byte[] content, KeyType keyType) throws IOException {
+ storeCacheValue(
+ cacheKey,
+ tmpName -> FileSystemUtils.writeContent(tmpName, content),
+ keyType,
+ /* canonicalId= */ null);
+ }
+
+ /**
+ * Adds an in-memory value to the cache.
+ *
+ * @param content The byte content of the value to be cached.
+ * @param keyType The type of key used. See: KeyType
+ */
+ public void put(byte[] content, KeyType keyType) throws IOException {
+ String cacheKey = keyType.newHasher().putBytes(content).hash().toString();
+ put(cacheKey, content, keyType);
+ }
+
+ /**
* Copies a value from a specified path into the cache, computing the cache key itself.
*
* @param sourcePath The path of the value to be cached.
* @param keyType The type of key to be used.
* @param canonicalId If set to a non-empty String associate the file with this name, allowing
* restricted cache lookups later.
- * @throws IOException
* @return The key for the cached entry.
*/
public String put(Path sourcePath, KeyType keyType, String canonicalId)
@@ -301,7 +370,6 @@
*
* @param keyType The type of hash function. e.g. SHA-1, SHA-256.
* @param path The path to the file.
- * @throws IOException
*/
public static String getChecksum(KeyType keyType, Path path)
throws IOException, InterruptedException {
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/Checksum.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/Checksum.java
index af5cf50..8a5ea43 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/Checksum.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/Checksum.java
@@ -120,6 +120,22 @@
return hashCode.toString();
}
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+ if (other instanceof Checksum c) {
+ return keyType.equals(c.keyType) && hashCode.equals(c.hashCode);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return hashCode.hashCode() * 31 + keyType.hashCode();
+ }
+
public HashCode getHashCode() {
return hashCode;
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java
index a56ce63..9737437 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java
@@ -382,16 +382,37 @@
* @param originalUrl the original URL of the file
* @param eventHandler CLI progress reporter
* @param clientEnv environment variables in shell issuing this command
+ * @param checksum checksum of the file used to verify the content and obtain repository cache
+ * hits
* @throws IllegalArgumentException on parameter badness, which should be checked beforehand
* @throws IOException if download was attempted and ended up failing
* @throws InterruptedException if this thread is being cast into oblivion
*/
public byte[] downloadAndReadOneUrl(
- URL originalUrl, ExtendedEventHandler eventHandler, Map<String, String> clientEnv)
+ URL originalUrl,
+ ExtendedEventHandler eventHandler,
+ Map<String, String> clientEnv,
+ Optional<Checksum> checksum)
throws IOException, InterruptedException {
if (Thread.interrupted()) {
throw new InterruptedException();
}
+
+ if (repositoryCache.isEnabled() && checksum.isPresent()) {
+ String cacheKey = checksum.get().toString();
+ try {
+ byte[] content = repositoryCache.getBytes(cacheKey, checksum.get().getKeyType());
+ if (content != null) {
+ // Cache hit!
+ eventHandler.post(
+ new RepositoryCacheHitEvent("Bazel module fetching", cacheKey, originalUrl));
+ return content;
+ }
+ } catch (IOException e) {
+ // Ignore error trying to get. We'll just download again.
+ }
+ }
+
Map<URI, Map<String, List<String>>> authHeaders = ImmutableMap.of();
ImmutableList<URL> rewrittenUrls = ImmutableList.of(originalUrl);
@@ -418,15 +439,27 @@
authHeaders = rewriter.updateAuthHeaders(rewrittenUrlMappings, authHeaders, netrcCreds);
}
+ if (disableDownload) {
+ throw new IOException(
+ String.format("Failed to download %s: download is disabled.", originalUrl));
+ }
+
if (rewrittenUrls.isEmpty()) {
throw new IOException(getRewriterBlockedAllUrlsMessage(ImmutableList.of(originalUrl)));
}
HttpDownloader httpDownloader = new HttpDownloader();
+ byte[] content = null;
for (int attempt = 0; attempt <= retries; ++attempt) {
try {
- return httpDownloader.downloadAndReadOneUrl(
- rewrittenUrls.get(0), credentialFactory.create(authHeaders), eventHandler, clientEnv);
+ content =
+ httpDownloader.downloadAndReadOneUrl(
+ rewrittenUrls.get(0),
+ credentialFactory.create(authHeaders),
+ checksum,
+ eventHandler,
+ clientEnv);
+ break;
} catch (ContentLengthMismatchException e) {
if (attempt == retries) {
throw e;
@@ -435,8 +468,18 @@
throw new InterruptedException(e.getMessage());
}
}
+ if (content == null) {
+ throw new IllegalStateException("Unexpected error: file should have been downloaded.");
+ }
- throw new IllegalStateException("Unexpected error: file should have been downloaded.");
+ if (repositoryCache.isEnabled()) {
+ if (checksum.isPresent()) {
+ repositoryCache.put(checksum.get().toString(), content, checksum.get().getKeyType());
+ } else {
+ repositoryCache.put(content, KeyType.SHA256);
+ }
+ }
+ return content;
}
@Nullable
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java
index 35e0ea2..796ac03 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java
@@ -147,6 +147,7 @@
public byte[] downloadAndReadOneUrl(
URL url,
Credentials credentials,
+ Optional<Checksum> checksum,
ExtendedEventHandler eventHandler,
Map<String, String> clientEnv)
throws IOException, InterruptedException {
@@ -155,8 +156,7 @@
ByteArrayOutputStream out = new ByteArrayOutputStream();
SEMAPHORE.acquire();
try (HttpStream payload =
- multiplexer.connect(
- url, Optional.empty(), ImmutableMap.of(), credentials, Optional.empty())) {
+ multiplexer.connect(url, checksum, ImmutableMap.of(), credentials, Optional.empty())) {
ByteStreams.copy(payload, out);
} catch (SocketTimeoutException e) {
// SocketTimeoutExceptions are InterruptedIOExceptions; however they do not signify
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java b/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java
index ed1ccb5..344048a 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java
@@ -185,6 +185,9 @@
ImmutableMap::of,
directories,
EXTERNAL_PACKAGE_HELPER))
+ .put(
+ SkyFunctions.BAZEL_LOCK_FILE,
+ new BazelLockFileFunction(directories.getWorkspace()))
.put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction())
.put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction())
.put(SkyFunctions.REPO_SPEC, new RepoSpecFunction())
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java
index 526e2c4..56c441f 100644
--- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java
+++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java
@@ -15,6 +15,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
@@ -208,7 +209,7 @@
RepositoryDelegatorFunction.FORCE_FETCH_DISABLED),
PrecomputedValue.injected(RepositoryDelegatorFunction.VENDOR_DIRECTORY, Optional.empty()),
PrecomputedValue.injected(RepositoryDelegatorFunction.DISABLE_NATIVE_REPO_RULES, false),
- PrecomputedValue.injected(ModuleFileFunction.REGISTRIES, ImmutableList.of()),
+ PrecomputedValue.injected(ModuleFileFunction.REGISTRIES, ImmutableSet.of()),
PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false),
PrecomputedValue.injected(ModuleFileFunction.MODULE_OVERRIDES, ImmutableMap.of()),
PrecomputedValue.injected(YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, ImmutableList.of()),
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java
index 67f0349..42c3ecc 100644
--- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java
+++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java
@@ -239,7 +239,7 @@
.setExtraPrecomputeValues(
ImmutableList.of(
PrecomputedValue.injected(
- ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())),
+ ModuleFileFunction.REGISTRIES, ImmutableSet.of(registry.getUrl())),
PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false),
PrecomputedValue.injected(
RepositoryDelegatorFunction.DISABLE_NATIVE_REPO_RULES, false),
@@ -293,7 +293,7 @@
PrecomputedValue.injected(
RepositoryDelegatorFunction.VENDOR_DIRECTORY, Optional.empty()),
PrecomputedValue.injected(
- ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())),
+ ModuleFileFunction.REGISTRIES, ImmutableSet.of(registry.getUrl())),
PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false),
PrecomputedValue.injected(RepositoryDelegatorFunction.DISABLE_NATIVE_REPO_RULES, false),
PrecomputedValue.injected(
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java
index eef29a0..89b27e1 100644
--- a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java
+++ b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java
@@ -308,7 +308,7 @@
.addAll(analysisMock.getPrecomputedValues())
.add(
PrecomputedValue.injected(
- ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())))
+ ModuleFileFunction.REGISTRIES, ImmutableSet.of(registry.getUrl())))
.addAll(extraPrecomputedValues())
.build();
PackageFactory.BuilderForTesting pkgFactoryBuilder =
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
index 2d88b05..1eef645 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
@@ -49,7 +49,6 @@
"//src/main/java/com/google/devtools/build/lib/bazel/repository/starlark",
"//src/main/java/com/google/devtools/build/lib/clock",
"//src/main/java/com/google/devtools/build/lib/cmdline",
- "//src/main/java/com/google/devtools/build/lib/collect/nestedset",
"//src/main/java/com/google/devtools/build/lib/events",
"//src/main/java/com/google/devtools/build/lib/packages",
"//src/main/java/com/google/devtools/build/lib/packages/semantics",
@@ -69,7 +68,6 @@
"//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_function",
"//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value",
"//src/main/java/com/google/devtools/build/lib/skyframe:repository_mapping_function",
- "//src/main/java/com/google/devtools/build/lib/skyframe:repository_mapping_value",
"//src/main/java/com/google/devtools/build/lib/skyframe:sky_functions",
"//src/main/java/com/google/devtools/build/lib/skyframe:skyframe_cluster",
"//src/main/java/com/google/devtools/build/lib/starlarkbuildapi/repository",
@@ -117,6 +115,8 @@
"//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:module_extension",
"//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:registry",
"//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution",
+ "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution_impl",
+ "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/events",
"//src/main/java/com/google/devtools/build/lib/packages",
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java
index 819c538..a8d61f8 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java
@@ -61,7 +61,6 @@
import com.google.devtools.build.skyframe.RecordingDifferencer;
import com.google.devtools.build.skyframe.SequencedRecordingDifferencer;
import com.google.devtools.build.skyframe.SkyFunction;
-import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
@@ -147,7 +146,7 @@
differencer,
StarlarkSemantics.builder().setBool(BuildLanguageOptions.ENABLE_BZLMOD, true).build());
ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false);
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of());
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of());
ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of());
BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set(
differencer, CheckDirectDepsMode.OFF);
@@ -364,10 +363,8 @@
@Override
@Nullable
- public SkyValue compute(SkyKey skyKey, Environment env)
- throws SkyFunctionException, InterruptedException {
-
- return BazelModuleResolutionValue.create(depGraph, ImmutableMap.of());
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ return BazelModuleResolutionValue.create(depGraph, ImmutableMap.of(), ImmutableMap.of());
}
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java
index 0aaf96e..f4dc881 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java
@@ -22,6 +22,7 @@
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import com.google.devtools.build.lib.actions.FileValue;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
@@ -192,6 +193,7 @@
.setFlags(flags)
.setLocalOverrideHashes(localOverrideHashes)
.setModuleDepGraph(key.depGraph())
+ .setRegistryFileHashes(ImmutableMap.of())
.build());
return new SkyValue() {};
@@ -203,7 +205,7 @@
PrecomputedValue.STARLARK_SEMANTICS.set(
differencer,
StarlarkSemantics.builder().setBool(BuildLanguageOptions.ENABLE_BZLMOD, true).build());
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of());
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of());
ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, true);
ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of());
YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of());
@@ -275,7 +277,7 @@
ImmutableList<String> yankedVersions = ImmutableList.of("2.4", "2.3");
LocalPathOverride override = LocalPathOverride.create("override_path");
- ImmutableList<String> registries = ImmutableList.of("registry1", "registry2");
+ ImmutableSet<String> registries = ImmutableSet.of("registry1", "registry2");
ImmutableMap<String, String> moduleOverride = ImmutableMap.of("my_dep_1", override.getPath());
ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, true);
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java
index 218a36c..2c4a7b3 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java
@@ -21,6 +21,7 @@
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import com.google.devtools.build.lib.actions.FileValue;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
@@ -344,7 +345,7 @@
.addModule(
createModuleKey("b", "1.0"),
"module(name='b', version='1.0', bazel_compatibility=['<=5.1.4', '-5.1.2']);");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
}
@Test
@@ -407,7 +408,7 @@
"bazel_dep(name='b', version='1.0')")
.addModule(createModuleKey("b", "1.0"), "module(name='b', version='1.0');")
.addYankedVersion("b", ImmutableMap.of(Version.parse("1.0"), "1.0 is a bad version!"));
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
}
@Test
@@ -437,7 +438,7 @@
"module(name='b', version='1.1')",
"bazel_dep(name='c', version='1.0')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<BazelModuleResolutionValue> result =
evaluator.evaluate(ImmutableList.of(BazelModuleResolutionValue.KEY), evaluationContext);
@@ -483,9 +484,8 @@
"bazel_dep(name='c', version='1.0')",
"print('hello from b@1.1')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
- EvaluationResult<BazelModuleResolutionValue> result =
- evaluator.evaluate(ImmutableList.of(BazelModuleResolutionValue.KEY), evaluationContext);
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
+ evaluator.evaluate(ImmutableList.of(BazelModuleResolutionValue.KEY), evaluationContext);
assertContainsEvent("hello from root module");
assertContainsEvent("hello from overridden a");
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java
index e0a1fb2..444ba83 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java
@@ -22,6 +22,7 @@
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import com.google.devtools.build.lib.actions.FileValue;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
@@ -146,7 +147,7 @@
differencer);
PrecomputedValue.STARLARK_SEMANTICS.set(differencer, StarlarkSemantics.DEFAULT);
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of());
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of());
ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false);
ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of());
YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of());
@@ -170,7 +171,7 @@
createModuleKey("bbb", "1.0"),
"module(name='bbb', version='1.0');bazel_dep(name='ccc',version='2.0')")
.addModule(createModuleKey("ccc", "2.0"), "module(name='ccc', version='2.0')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
RepositoryName repo = RepositoryName.create("ccc~");
EvaluationResult<BzlmodRepoRuleValue> result =
@@ -201,7 +202,7 @@
createModuleKey("bbb", "1.0"),
"module(name='bbb', version='1.0');bazel_dep(name='ccc',version='2.0')")
.addModule(createModuleKey("ccc", "2.0"), "module(name='ccc', version='2.0')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
RepositoryName repo = RepositoryName.create("ccc~");
EvaluationResult<BzlmodRepoRuleValue> result =
@@ -234,7 +235,7 @@
"module(name='bbb', version='1.0');bazel_dep(name='ccc',version='2.0')")
.addModule(createModuleKey("ccc", "2.0"), "module(name='ccc', version='2.0')")
.addModule(createModuleKey("ccc", "3.0"), "module(name='ccc', version='3.0')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
RepositoryName repo = RepositoryName.create("ccc~");
EvaluationResult<BzlmodRepoRuleValue> result =
@@ -270,7 +271,7 @@
"module(name='ccc', version='2.0');bazel_dep(name='ddd',version='2.0')")
.addModule(createModuleKey("ddd", "1.0"), "module(name='ddd', version='1.0')")
.addModule(createModuleKey("ddd", "2.0"), "module(name='ddd', version='2.0')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
RepositoryName repo = RepositoryName.create("ddd~v2.0");
EvaluationResult<BzlmodRepoRuleValue> result =
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java
index 742b5d6..81e4685 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java
@@ -23,6 +23,8 @@
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
import com.google.devtools.build.lib.actions.FileValue;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
@@ -30,6 +32,7 @@
import com.google.devtools.build.lib.analysis.util.AnalysisMock;
import com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil.InterimModuleBuilder;
import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue;
+import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryModule;
import com.google.devtools.build.lib.clock.BlazeClock;
import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
@@ -91,11 +94,17 @@
static final SkyFunctionName FUNCTION_NAME = SkyFunctionName.createHermetic("test_discovery");
static final SkyKey KEY = () -> FUNCTION_NAME;
- static DiscoveryValue create(ImmutableMap<ModuleKey, InterimModule> depGraph) {
- return new AutoValue_DiscoveryTest_DiscoveryValue(depGraph);
+ static DiscoveryValue create(
+ ImmutableMap<ModuleKey, InterimModule> depGraph,
+ ImmutableMap<String, Optional<String>> registryFileHashes) {
+ return new AutoValue_DiscoveryTest_DiscoveryValue(depGraph, registryFileHashes);
}
abstract ImmutableMap<ModuleKey, InterimModule> getDepGraph();
+
+ // Uses Optional<String> rather than Optional<Checksum> for easier testing (Checksum doesn't
+ // implement equals()).
+ abstract ImmutableMap<String, Optional<String>> getRegistryFileHashes();
}
static class DiscoveryFunction implements SkyFunction {
@@ -107,14 +116,21 @@
if (root == null) {
return null;
}
- ImmutableMap<ModuleKey, InterimModule> depGraph;
+ Discovery.Result discoveryResult;
try {
- depGraph = Discovery.run(env, root);
+ discoveryResult = Discovery.run(env, root);
} catch (ExternalDepsException e) {
throw new BazelModuleResolutionFunction.BazelModuleResolutionFunctionException(
e, SkyFunctionException.Transience.PERSISTENT);
}
- return depGraph == null ? null : DiscoveryValue.create(depGraph);
+ return discoveryResult == null
+ ? null
+ : DiscoveryValue.create(
+ discoveryResult.depGraph(),
+ ImmutableMap.copyOf(
+ Maps.transformValues(
+ discoveryResult.registryFileHashes(),
+ value -> value.map(Checksum::toString))));
}
}
@@ -168,6 +184,7 @@
SyscallCache.NO_CACHE,
externalFilesHelper))
.put(DiscoveryValue.FUNCTION_NAME, new DiscoveryFunction())
+ .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory))
.put(
SkyFunctions.MODULE_FILE,
new ModuleFileFunction(
@@ -236,7 +253,7 @@
createModuleKey("ddd", "3.0"),
// Add a random override here; it should be ignored
"module(name='ddd', version='3.0');local_path_override(module_name='ff',path='f')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<DiscoveryValue> result =
evaluator.evaluate(ImmutableList.of(DiscoveryValue.KEY), evaluationContext);
@@ -260,6 +277,15 @@
.setRegistry(registry)
.buildEntry(),
InterimModuleBuilder.create("ddd", "3.0").setRegistry(registry).buildEntry());
+ assertThat(discoveryValue.getRegistryFileHashes())
+ .containsExactly(
+ registry.getUrl() + "/modules/bbb/1.0/MODULE.bazel",
+ Optional.of("3f48e6d8694e0aa0d16617fd97b7d84da0e17ee9932c18cbc71888c12563372d"),
+ registry.getUrl() + "/modules/ccc/2.0/MODULE.bazel",
+ Optional.of("e613d4192495192c3d46ee444dc9882a176a9e7a243d1b5a840ab0f01553e8d6"),
+ registry.getUrl() + "/modules/ddd/3.0/MODULE.bazel",
+ Optional.of("f80d91453520d193b0b79f1501eb902b5b01a991762cc7fb659fc580b95648fd"))
+ .inOrder();
}
@Test
@@ -278,7 +304,7 @@
"bazel_dep(name='ccc',version='2.0',dev_dependency=True)")
.addModule(createModuleKey("ccc", "1.0"), "module(name='ccc', version='1.0')")
.addModule(createModuleKey("ccc", "2.0"), "module(name='ccc', version='2.0')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<DiscoveryValue> result =
evaluator.evaluate(ImmutableList.of(DiscoveryValue.KEY), evaluationContext);
@@ -313,7 +339,7 @@
"bazel_dep(name='ccc',version='2.0',dev_dependency=True)")
.addModule(createModuleKey("ccc", "1.0"), "module(name='ccc', version='1.0')")
.addModule(createModuleKey("ccc", "2.0"), "module(name='ccc', version='2.0')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, true);
EvaluationResult<DiscoveryValue> result =
@@ -346,7 +372,7 @@
.addModule(
createModuleKey("ccc", "2.0"),
"module(name='ccc', version='2.0');bazel_dep(name='bbb',version='1.0')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<DiscoveryValue> result =
evaluator.evaluate(ImmutableList.of(DiscoveryValue.KEY), evaluationContext);
@@ -383,7 +409,7 @@
createModuleKey("bbb", "1.0"),
"module(name='bbb', version='1.0');bazel_dep(name='aaa',version='2.0')")
.addModule(createModuleKey("aaa", "2.0"), "module(name='aaa', version='2.0')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<DiscoveryValue> result =
evaluator.evaluate(ImmutableList.of(DiscoveryValue.KEY), evaluationContext);
@@ -419,7 +445,7 @@
"module(name='bbb', version='0.1');bazel_dep(name='ccc',version='1.0')")
.addModule(createModuleKey("ccc", "1.0"), "module(name='ccc', version='1.0');")
.addModule(createModuleKey("ccc", "2.0"), "module(name='ccc', version='2.0');");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<DiscoveryValue> result =
evaluator.evaluate(ImmutableList.of(DiscoveryValue.KEY), evaluationContext);
@@ -461,7 +487,7 @@
"module(name='aaa',version='0.1')",
"bazel_dep(name='bbb',version='0.1')",
"single_version_override(module_name='ccc',registry='" + registry2.getUrl() + "')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry1.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry1.getUrl()));
EvaluationResult<DiscoveryValue> result =
evaluator.evaluate(ImmutableList.of(DiscoveryValue.KEY), evaluationContext);
@@ -503,7 +529,7 @@
createModuleKey("bbb", "0.1"),
"module(name='bbb', version='0.1');bazel_dep(name='ccc',version='1.0')")
.addModule(createModuleKey("ccc", "1.0"), "module(name='ccc', version='1.0');");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<DiscoveryValue> result =
evaluator.evaluate(ImmutableList.of(DiscoveryValue.KEY), evaluationContext);
@@ -525,6 +551,11 @@
InterimModuleBuilder.create("ccc", "2.0")
.setKey(createModuleKey("ccc", ""))
.buildEntry());
+ assertThat(discoveryValue.getRegistryFileHashes())
+ .containsExactly(
+ registry.getUrl() + "/modules/bbb/0.1/MODULE.bazel",
+ Optional.of("3f9e1a600b4adeee1c1a92b92df9d086eca4bbdde656c122872f48f8f3b874a3"))
+ .inOrder();
}
@Test
@@ -553,7 +584,7 @@
.newFakeRegistry("/foo")
.addModule(createModuleKey("foo", "1.0"), "module(name='foo', version='1.0')")
.addModule(createModuleKey("foo", "2.0"), "module(name='foo', version='2.0')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<DiscoveryValue> result =
evaluator.evaluate(ImmutableList.of(DiscoveryValue.KEY), evaluationContext);
@@ -587,5 +618,13 @@
.addDep("local_config_platform", createModuleKey("local_config_platform", ""))
.setRegistry(registry)
.buildEntry());
+
+ assertThat(discoveryValue.getRegistryFileHashes())
+ .containsExactly(
+ registry.getUrl() + "/modules/foo/2.0/MODULE.bazel",
+ Optional.of("76ecb05b455aecab4ec958c1deb17e4cbbe6e708d9c4e85fceda2317f6c86d7b"),
+ registry.getUrl() + "/modules/foo/1.0/MODULE.bazel",
+ Optional.of("4d887e8dfc1863861e3aa5601eeeebca5d8f110977895f1de4bdb2646e546fb5"))
+ .inOrder();
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java
index 10500fb..2bcc6d8 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java
@@ -20,6 +20,7 @@
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.util.HashMap;
@@ -63,26 +64,32 @@
@Override
public Optional<ModuleFile> getModuleFile(ModuleKey key, ExtendedEventHandler eventHandler) {
- return Optional.ofNullable(modules.get(key))
- .map(value -> value.getBytes(UTF_8))
- .map(
- content ->
- ModuleFile.create(
- content,
- String.format(
- "%s/modules/%s/%s/MODULE.bazel",
- url, key.getName(), key.getVersion().toString())));
+ String uri =
+ String.format("%s/modules/%s/%s/MODULE.bazel", url, key.getName(), key.getVersion());
+ var maybeContent = Optional.ofNullable(modules.get(key)).map(value -> value.getBytes(UTF_8));
+ eventHandler.post(RegistryFileDownloadEvent.create(uri, maybeContent));
+ return maybeContent.map(content -> ModuleFile.create(content, uri));
}
@Override
public RepoSpec getRepoSpec(ModuleKey key, ExtendedEventHandler eventHandler) {
- return RepoSpec.builder()
- .setRuleClassName("local_repository")
- .setAttributes(
- AttributeValues.create(
- ImmutableMap.of(
- "path", rootPath + "/" + key.getCanonicalRepoNameWithVersion().getName())))
- .build();
+ RepoSpec repoSpec =
+ RepoSpec.builder()
+ .setRuleClassName("local_repository")
+ .setAttributes(
+ AttributeValues.create(
+ ImmutableMap.of(
+ "path", rootPath + "/" + key.getCanonicalRepoNameWithVersion().getName())))
+ .build();
+ eventHandler.post(
+ RegistryFileDownloadEvent.create(
+ "%s/modules/%s/%s/source.json"
+ .formatted(url, key.getName(), key.getVersion().toString()),
+ Optional.of(
+ GsonTypeAdapterUtil.createSingleExtensionUsagesValueHashGson()
+ .toJson(repoSpec)
+ .getBytes(UTF_8))));
+ return repoSpec;
}
@Override
@@ -118,7 +125,8 @@
}
@Override
- public Registry createRegistry(String url) {
+ public Registry createRegistry(
+ String url, ImmutableMap<String, Optional<Checksum>> fileHashes) {
return Preconditions.checkNotNull(registries.get(url), "unknown registry url: %s", url);
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
index 2401275..a721ba6 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
@@ -15,6 +15,7 @@
package com.google.devtools.build.lib.bazel.bzlmod;
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static com.google.common.truth.Truth.assertThat;
import static com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil.createModuleKey;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -23,14 +24,17 @@
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import com.google.common.eventbus.Subscribe;
+import com.google.common.hash.Hashing;
import com.google.devtools.build.lib.authandtls.BasicHttpAuthenticationEncoder;
import com.google.devtools.build.lib.authandtls.Netrc;
import com.google.devtools.build.lib.authandtls.NetrcCredentials;
import com.google.devtools.build.lib.authandtls.NetrcParser;
import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache;
+import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager;
import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
-import com.google.devtools.build.lib.bazel.repository.downloader.UnrecoverableHttpException;
import com.google.devtools.build.lib.testutil.FoundationTestCase;
import com.google.devtools.build.lib.vfs.Path;
import java.io.ByteArrayInputStream;
@@ -38,6 +42,8 @@
import java.io.IOException;
import java.io.Writer;
import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Optional;
import org.junit.Before;
import org.junit.Rule;
@@ -49,18 +55,38 @@
/** Tests for {@link IndexRegistry}. */
@RunWith(JUnit4.class)
public class IndexRegistryTest extends FoundationTestCase {
+ private static class EventRecorder {
+ private final List<RegistryFileDownloadEvent> downloadEvents = new ArrayList<>();
+
+ @Subscribe
+ public void onRegistryFileDownloadEvent(RegistryFileDownloadEvent downloadEvent) {
+ downloadEvents.add(downloadEvent);
+ }
+
+ public ImmutableMap<String, Optional<Checksum>> getRecordedHashes() {
+ return downloadEvents.stream()
+ .collect(
+ toImmutableMap(RegistryFileDownloadEvent::uri, RegistryFileDownloadEvent::checksum));
+ }
+ }
+
private final String authToken =
BasicHttpAuthenticationEncoder.encode("rinne", "rinnepass", UTF_8);
private DownloadManager downloadManager;
+ private EventRecorder eventRecorder;
@Rule public final TestHttpServer server = new TestHttpServer(authToken);
@Rule public final TemporaryFolder tempFolder = new TemporaryFolder();
private RegistryFactory registryFactory;
+ private RepositoryCache repositoryCache;
@Before
public void setUp() throws Exception {
+ eventRecorder = new EventRecorder();
+ eventBus.register(eventRecorder);
Path workspaceRoot = scratch.dir("/ws");
- downloadManager = new DownloadManager(new RepositoryCache(), new HttpDownloader());
+ repositoryCache = new RepositoryCache();
+ downloadManager = new DownloadManager(repositoryCache, new HttpDownloader());
registryFactory =
new RegistryFactoryImpl(
workspaceRoot, downloadManager, Suppliers.ofInstance(ImmutableMap.of()));
@@ -71,7 +97,8 @@
server.serve("/myreg/modules/foo/1.0/MODULE.bazel", "lol");
server.start();
- Registry registry = registryFactory.createRegistry(server.getUrl() + "/myreg");
+ Registry registry =
+ registryFactory.createRegistry(server.getUrl() + "/myreg", ImmutableMap.of());
assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter))
.hasValue(
ModuleFile.create(
@@ -87,13 +114,18 @@
NetrcParser.parseAndClose(
new ByteArrayInputStream(
"machine [::1] login rinne password rinnepass\n".getBytes(UTF_8)));
- Registry registry = registryFactory.createRegistry(server.getUrl() + "/myreg");
+ Registry registry =
+ registryFactory.createRegistry(server.getUrl() + "/myreg", ImmutableMap.of());
- UnrecoverableHttpException e =
+ var e =
assertThrows(
- UnrecoverableHttpException.class,
+ IOException.class,
() -> registry.getModuleFile(createModuleKey("foo", "1.0"), reporter));
- assertThat(e).hasMessageThat().contains("GET returned 401 Unauthorized");
+ assertThat(e)
+ .hasMessageThat()
+ .isEqualTo(
+ "Failed to fetch registry file %s: GET returned 401 Unauthorized"
+ .formatted(server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel"));
downloadManager.setNetrcCreds(new NetrcCredentials(netrc));
assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter))
@@ -113,7 +145,7 @@
Registry registry =
registryFactory.createRegistry(
- new File(tempFolder.getRoot(), "fakereg").toURI().toString());
+ new File(tempFolder.getRoot(), "fakereg").toURI().toString(), ImmutableMap.of());
assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter))
.hasValue(ModuleFile.create("lol".getBytes(UTF_8), file.toURI().toString()));
assertThat(registry.getModuleFile(createModuleKey("bar", "1.0"), reporter)).isEmpty();
@@ -149,7 +181,7 @@
"}");
server.start();
- Registry registry = registryFactory.createRegistry(server.getUrl());
+ Registry registry = registryFactory.createRegistry(server.getUrl(), ImmutableMap.of());
assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter))
.isEqualTo(
new ArchiveRepoSpecBuilder()
@@ -193,7 +225,7 @@
"}");
server.start();
- Registry registry = registryFactory.createRegistry(server.getUrl());
+ Registry registry = registryFactory.createRegistry(server.getUrl(), ImmutableMap.of());
assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter))
.isEqualTo(
RepoSpec.builder()
@@ -215,7 +247,7 @@
" \"strip_prefix\": \"pref\"",
"}");
- Registry registry = registryFactory.createRegistry(server.getUrl());
+ Registry registry = registryFactory.createRegistry(server.getUrl(), ImmutableMap.of());
assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter))
.isEqualTo(
new ArchiveRepoSpecBuilder()
@@ -246,7 +278,7 @@
"}");
server.start();
- Registry registry = registryFactory.createRegistry(server.getUrl());
+ Registry registry = registryFactory.createRegistry(server.getUrl(), ImmutableMap.of());
assertThrows(
IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter));
}
@@ -273,7 +305,7 @@
+ " }\n"
+ "}");
server.start();
- Registry registry = registryFactory.createRegistry(server.getUrl());
+ Registry registry = registryFactory.createRegistry(server.getUrl(), ImmutableMap.of());
Optional<ImmutableMap<Version, String>> yankedVersion =
registry.getYankedVersions("red-pill", reporter);
assertThat(yankedVersion)
@@ -294,7 +326,7 @@
"}");
server.start();
- Registry registry = registryFactory.createRegistry(server.getUrl());
+ Registry registry = registryFactory.createRegistry(server.getUrl(), ImmutableMap.of());
assertThat(registry.getRepoSpec(createModuleKey("archive_type", "1.0"), reporter))
.isEqualTo(
new ArchiveRepoSpecBuilder()
@@ -306,4 +338,230 @@
.setRemotePatchStrip(0)
.build());
}
+
+ @Test
+ public void testGetModuleFileChecksums() throws Exception {
+ repositoryCache.setRepositoryCachePath(scratch.dir("cache"));
+
+ server.serve("/myreg/modules/foo/1.0/MODULE.bazel", "old");
+ server.serve("/myreg/modules/foo/2.0/MODULE.bazel", "new");
+ server.start();
+
+ var knownFiles =
+ ImmutableMap.of(
+ server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel",
+ Optional.of(sha256("old")),
+ server.getUrl() + "/myreg/modules/unused/1.0/MODULE.bazel",
+ Optional.of(sha256("unused")));
+ Registry registry = registryFactory.createRegistry(server.getUrl() + "/myreg", knownFiles);
+ assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter))
+ .hasValue(
+ ModuleFile.create(
+ "old".getBytes(UTF_8), server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel"));
+ assertThat(registry.getModuleFile(createModuleKey("foo", "2.0"), reporter))
+ .hasValue(
+ ModuleFile.create(
+ "new".getBytes(UTF_8), server.getUrl() + "/myreg/modules/foo/2.0/MODULE.bazel"));
+ assertThat(registry.getModuleFile(createModuleKey("bar", "1.0"), reporter)).isEmpty();
+
+ var recordedChecksums = eventRecorder.getRecordedHashes();
+ assertThat(
+ Maps.transformValues(
+ recordedChecksums, maybeChecksum -> maybeChecksum.map(Checksum::toString)))
+ .containsExactly(
+ server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel",
+ Optional.of(sha256("old").toString()),
+ server.getUrl() + "/myreg/modules/foo/2.0/MODULE.bazel",
+ Optional.of(sha256("new").toString()),
+ server.getUrl() + "/myreg/modules/bar/1.0/MODULE.bazel",
+ Optional.empty())
+ .inOrder();
+
+ registry = registryFactory.createRegistry(server.getUrl() + "/myreg", recordedChecksums);
+ // Test that the recorded hashes are used for repo cache hits even when the server content
+ // changes.
+ server.unserve("/myreg/modules/foo/1.0/MODULE.bazel");
+ server.unserve("/myreg/modules/foo/2.0/MODULE.bazel");
+ server.serve("/myreg/modules/bar/1.0/MODULE.bazel", "no longer 404");
+ assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter))
+ .hasValue(
+ ModuleFile.create(
+ "old".getBytes(UTF_8), server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel"));
+ assertThat(registry.getModuleFile(createModuleKey("foo", "2.0"), reporter))
+ .hasValue(
+ ModuleFile.create(
+ "new".getBytes(UTF_8), server.getUrl() + "/myreg/modules/foo/2.0/MODULE.bazel"));
+ assertThat(registry.getModuleFile(createModuleKey("bar", "1.0"), reporter)).isEmpty();
+ }
+
+ @Test
+ public void testGetModuleFileChecksumMismatch() throws Exception {
+ repositoryCache.setRepositoryCachePath(scratch.dir("cache"));
+
+ server.serve("/myreg/modules/foo/1.0/MODULE.bazel", "fake");
+ server.start();
+
+ var knownFiles =
+ ImmutableMap.of(
+ server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel",
+ Optional.of(sha256("original")));
+ Registry registry = registryFactory.createRegistry(server.getUrl() + "/myreg", knownFiles);
+ var e =
+ assertThrows(
+ IOException.class,
+ () -> registry.getModuleFile(createModuleKey("foo", "1.0"), reporter));
+ assertThat(e)
+ .hasMessageThat()
+ .isEqualTo(
+ "Failed to fetch registry file %s: Checksum was %s but wanted %s"
+ .formatted(
+ server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel",
+ sha256("fake"),
+ sha256("original")));
+ }
+
+ @Test
+ public void testGetRepoSpecChecksum() throws Exception {
+ repositoryCache.setRepositoryCachePath(scratch.dir("cache"));
+
+ String registryJson =
+ """
+ {
+ "module_base_path": "/hello/foo"
+ }
+ """;
+ server.serve("/bazel_registry.json", registryJson);
+ String sourceJson =
+ """
+ {
+ "type": "local_path",
+ "path": "../bar/project_x"
+ }
+ """;
+ server.serve("/modules/foo/1.0/source.json", sourceJson.getBytes(UTF_8));
+ server.start();
+
+ var knownFiles =
+ ImmutableMap.of(
+ server.getUrl() + "/modules/foo/2.0/source.json", Optional.of(sha256("unused")));
+ Registry registry = registryFactory.createRegistry(server.getUrl(), knownFiles);
+ assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter))
+ .isEqualTo(
+ RepoSpec.builder()
+ .setRuleClassName("local_repository")
+ .setAttributes(
+ AttributeValues.create(ImmutableMap.of("path", "/hello/bar/project_x")))
+ .build());
+
+ var recordedChecksums = eventRecorder.getRecordedHashes();
+ assertThat(
+ Maps.transformValues(recordedChecksums, checksum -> checksum.map(Checksum::toString)))
+ .containsExactly(
+ server.getUrl() + "/bazel_registry.json",
+ Optional.of(sha256(registryJson).toString()),
+ server.getUrl() + "/modules/foo/1.0/source.json",
+ Optional.of(sha256(sourceJson).toString()));
+
+ registry = registryFactory.createRegistry(server.getUrl(), recordedChecksums);
+ // Test that the recorded hashes are used for repo cache hits even when the server content
+ // changes.
+ server.unserve("/bazel_registry.json");
+ server.unserve("/modules/foo/1.0/source.json");
+ assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter))
+ .isEqualTo(
+ RepoSpec.builder()
+ .setRuleClassName("local_repository")
+ .setAttributes(
+ AttributeValues.create(ImmutableMap.of("path", "/hello/bar/project_x")))
+ .build());
+ }
+
+ @Test
+ public void testGetRepoSpecChecksumMismatch() throws Exception {
+ repositoryCache.setRepositoryCachePath(scratch.dir("cache"));
+
+ String registryJson =
+ """
+ {
+ "module_base_path": "/hello/foo"
+ }
+ """;
+ server.serve("/bazel_registry.json", registryJson.getBytes(UTF_8));
+ String sourceJson =
+ """
+ {
+ "type": "local_path",
+ "path": "../bar/project_x"
+ }
+ """;
+ String maliciousSourceJson = sourceJson.replace("project_x", "malicious");
+ server.serve("/modules/foo/1.0/source.json", maliciousSourceJson.getBytes(UTF_8));
+ server.start();
+
+ var knownFiles =
+ ImmutableMap.of(
+ server.getUrl() + "/bazel_registry.json",
+ Optional.of(sha256(registryJson)),
+ server.getUrl() + "/modules/foo/1.0/source.json",
+ Optional.of(sha256(sourceJson)));
+ Registry registry = registryFactory.createRegistry(server.getUrl(), knownFiles);
+ var e =
+ assertThrows(
+ IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter));
+ assertThat(e)
+ .hasMessageThat()
+ .isEqualTo(
+ "Failed to fetch registry file %s: Checksum was %s but wanted %s"
+ .formatted(
+ server.getUrl() + "/modules/foo/1.0/source.json",
+ sha256(maliciousSourceJson),
+ sha256(sourceJson)));
+ }
+
+ @Test
+ public void testBazelRegistryChecksumMismatch() throws Exception {
+ repositoryCache.setRepositoryCachePath(scratch.dir("cache"));
+
+ String registryJson =
+ """
+ {
+ "module_base_path": "/hello/foo"
+ }
+ """;
+ String maliciousRegistryJson = registryJson.replace("foo", "malicious");
+ server.serve("/bazel_registry.json", maliciousRegistryJson.getBytes(UTF_8));
+ String sourceJson =
+ """
+ {
+ "type": "local_path",
+ "path": "../bar/project_x"
+ }
+ """;
+ server.serve("/modules/foo/1.0/source.json", sourceJson.getBytes(UTF_8));
+ server.start();
+
+ var knownFiles =
+ ImmutableMap.of(
+ server.getUrl() + "/bazel_registry.json",
+ Optional.of(sha256(registryJson)),
+ server.getUrl() + "/modules/foo/1.0/source.json",
+ Optional.of(sha256(sourceJson)));
+ Registry registry = registryFactory.createRegistry(server.getUrl(), knownFiles);
+ var e =
+ assertThrows(
+ IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter));
+ assertThat(e)
+ .hasMessageThat()
+ .isEqualTo(
+ "Failed to fetch registry file %s: Checksum was %s but wanted %s"
+ .formatted(
+ server.getUrl() + "/bazel_registry.json",
+ sha256(maliciousRegistryJson),
+ sha256(registryJson)));
+ }
+
+ private static Checksum sha256(String content) throws Checksum.InvalidChecksumException {
+ return Checksum.fromString(
+ RepositoryCache.KeyType.SHA256, Hashing.sha256().hashString(content, UTF_8).toString());
+ }
}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java
index fbbdb8e..6607062 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java
@@ -278,7 +278,7 @@
ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false);
ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of());
YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of());
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set(
differencer, CheckDirectDepsMode.WARNING);
BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set(
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java
index cea99f3..bf4d985 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java
@@ -142,6 +142,7 @@
new TimestampGranularityMonitor(BlazeClock.instance())),
SyscallCache.NO_CACHE,
externalFilesHelper))
+ .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory))
.put(
SkyFunctions.MODULE_FILE,
new ModuleFileFunction(
@@ -223,7 +224,7 @@
"multiple_version_override(module_name='fff',versions=['1.0','2.0'])",
"archive_override(module_name='ggg',urls=['https://hello.com/world.zip'])");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<RootModuleFileValue> result =
evaluator.evaluate(
@@ -271,7 +272,7 @@
rootDirectory.getRelative("MODULE.bazel").getPathString(),
"bazel_dep(name='bbb',version='1.0')");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<RootModuleFileValue> result =
evaluator.evaluate(
@@ -297,7 +298,7 @@
"module(name='aaa')",
"single_version_override(module_name='aaa',version='7')");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<RootModuleFileValue> result =
evaluator.evaluate(
@@ -318,7 +319,7 @@
"module(name='aaa')",
"local_path_override(module_name='bazel_tools',path='./bazel_tools_new')");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<RootModuleFileValue> result =
evaluator.evaluate(
@@ -352,7 +353,7 @@
rootDirectory.getRelative("python/toolchains/toolchains.MODULE.bazel").getPathString(),
"register_toolchains('//:python-whatever')");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<RootModuleFileValue> result =
evaluator.evaluate(
@@ -389,7 +390,7 @@
"module(name='aaa')",
"include('@haha//java:java.MODULE.bazel')");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<RootModuleFileValue> result =
evaluator.evaluate(
@@ -405,7 +406,7 @@
"module(name='aaa')",
"include(':relative.MODULE.bazel')");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<RootModuleFileValue> result =
evaluator.evaluate(
@@ -421,7 +422,7 @@
"module(name='aaa')",
"include('//:MODULE.bazel.segment')");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<RootModuleFileValue> result =
evaluator.evaluate(
@@ -437,7 +438,7 @@
"module(name='aaa')",
"include('//haha/:::.MODULE.bazel')");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
reporter.removeHandler(failFastHandler); // expect failures
EvaluationResult<RootModuleFileValue> result =
@@ -458,7 +459,7 @@
"module(name='bet-you-didnt-expect-this-didya')",
"bazel_dep(name='java-foo', version='1.0', repo_name='foo')");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
reporter.removeHandler(failFastHandler); // expect failures
EvaluationResult<RootModuleFileValue> result =
@@ -484,7 +485,7 @@
rootDirectory.getRelative("python/python.MODULE.bazel").getPathString(),
"bazel_dep(name='python-foo', version='1.0', repo_name='foo')");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
reporter.removeHandler(failFastHandler); // expect failures
EvaluationResult<RootModuleFileValue> result =
@@ -506,7 +507,7 @@
rootDirectory.getRelative("java/java.MODULE.bazel").getPathString(),
"bazel_dep(name=FOO_NAME, version='1.0')");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
reporter.removeHandler(failFastHandler); // expect failures
EvaluationResult<RootModuleFileValue> result =
@@ -519,7 +520,7 @@
@Test
public void forgotVersion() throws Exception {
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
SkyKey skyKey = ModuleFileValue.key(createModuleKey("bbb", ""), null);
EvaluationResult<ModuleFileValue> result =
@@ -547,7 +548,7 @@
createModuleKey("bbb", "1.0"),
"module(name='bbb',version='1.0');bazel_dep(name='ddd',version='3.0')");
ModuleFileFunction.REGISTRIES.set(
- differencer, ImmutableList.of(registry1.getUrl(), registry2.getUrl(), registry3.getUrl()));
+ differencer, ImmutableSet.of(registry1.getUrl(), registry2.getUrl(), registry3.getUrl()));
SkyKey skyKey = ModuleFileValue.key(createModuleKey("bbb", "1.0"), null);
EvaluationResult<ModuleFileValue> result =
@@ -573,7 +574,7 @@
createModuleKey("foo", "1.0"),
"module(name='foo',version='1.0')",
"include('//java:MODULE.bazel.segment')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<RootModuleFileValue> result =
evaluator.evaluate(
@@ -602,7 +603,7 @@
.addModule(
createModuleKey("bbb", "1.0"),
"module(name='bbb',version='1.0');bazel_dep(name='ccc',version='3.0')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
// The version is empty here due to the override.
SkyKey skyKey =
@@ -653,7 +654,7 @@
.addModule(
createModuleKey("bbb", "1.0"),
"module(name='bbb',version='1.0');bazel_dep(name='ccc',version='3.0')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
// The version is empty here due to the override.
SkyKey skyKey =
@@ -688,7 +689,7 @@
createModuleKey("bbb", "1.0"),
"module(name='bbb',version='1.0',compatibility_level=6)",
"bazel_dep(name='ccc',version='3.0')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry1.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry1.getUrl()));
// Override the registry for B to be registry2 (instead of the default registry1).
SkyKey skyKey =
@@ -731,7 +732,7 @@
"maven.dep(coord='junit')",
"use_repo(maven, 'junit', 'guava')",
"maven.dep(coord='guava')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
ModuleKey myMod = createModuleKey("mymod", "1.0");
SkyKey skyKey = ModuleFileValue.key(myMod, null);
@@ -873,7 +874,7 @@
"myext4 = use_extension('//:defs.bzl','myext')",
"myext4.tag(name = 'tag4')",
"use_repo(myext4, 'delta')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of());
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of());
SkyKey skyKey = ModuleFileValue.KEY_FOR_ROOT_MODULE;
EvaluationResult<ModuleFileValue> result =
@@ -972,7 +973,7 @@
"myext4 = use_extension('//:defs.bzl','myext')",
"myext4.tag(name = 'tag4')",
"use_repo(myext4, 'delta')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
ModuleKey myMod = createModuleKey("mymod", "1.0");
SkyKey skyKey = ModuleFileValue.key(myMod, null);
@@ -1039,7 +1040,7 @@
"module(name='mymod',version='1.0')",
"myext = use_extension('//:defs.bzl','myext')",
"use_repo(myext, mymod='some_repo')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
SkyKey skyKey = ModuleFileValue.key(createModuleKey("mymod", "1.0"), null);
reporter.removeHandler(failFastHandler); // expect failures
@@ -1059,7 +1060,7 @@
"module(name='mymod',version='1.0')",
"myext = use_extension('//:defs.bzl','myext')",
"use_repo(myext, 'some_repo', again='some_repo')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
SkyKey skyKey = ModuleFileValue.key(createModuleKey("mymod", "1.0"), null);
reporter.removeHandler(failFastHandler); // expect failures
@@ -1078,7 +1079,7 @@
"http_archive = use_repo_rule('@bazel_tools//:http.bzl','http_archive')",
"http_archive(name='guava',url='guava.com')",
"http_archive(name='vuaga',url='vuaga.com',dev_dependency=True)");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of());
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of());
SkyKey skyKey = ModuleFileValue.KEY_FOR_ROOT_MODULE;
EvaluationResult<ModuleFileValue> result =
@@ -1272,7 +1273,7 @@
scratch.overwriteFile(
rootDirectory.getRelative("MODULE.bazel").getPathString(),
"bazel_dep(name='foo',version='1.0')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of());
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of());
SkyKey skyKey = ModuleFileValue.KEY_FOR_ROOT_MODULE;
EvaluationResult<RootModuleFileValue> result =
@@ -1308,7 +1309,7 @@
rootDirectory.getRelative("tools/MODULE.bazel").getPathString(),
"module(name='bazel_tools',version='1.0')",
"bazel_dep(name='foo',version='2.0')");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of());
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of());
SkyKey skyKey =
ModuleFileValue.key(createModuleKey("bazel_tools", ""), builtinModules.get("bazel_tools"));
@@ -1333,7 +1334,7 @@
rootDirectory.getRelative("MODULE.bazel").getPathString(),
"module(name='aaa',version='0.1',repo_name='bbb')");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<RootModuleFileValue> result =
evaluator.evaluate(
@@ -1357,7 +1358,7 @@
"module(name='aaa',version='0.1',repo_name='bbb')",
"bazel_dep(name='bbb',version='1.0')");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
reporter.removeHandler(failFastHandler); // expect failures
evaluator.evaluate(ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext);
@@ -1372,7 +1373,7 @@
"module(name='aaa',version='0.1',repo_name='bbb')",
"module(name='aaa',version='0.1',repo_name='bbb')");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
reporter.removeHandler(failFastHandler); // expect failures
evaluator.evaluate(ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext);
@@ -1387,7 +1388,7 @@
"use_extension('//:extensions.bzl', 'my_ext')",
"module(name='aaa',version='0.1',repo_name='bbb')");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
reporter.removeHandler(failFastHandler); // expect failures
evaluator.evaluate(ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext);
@@ -1401,7 +1402,7 @@
rootDirectory.getRelative("MODULE.bazel").getPathString(),
"if 3+5>7: module(name='aaa',version='0.1',repo_name='bbb')");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
reporter.removeHandler(failFastHandler); // expect failures
evaluator.evaluate(ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext);
@@ -1424,7 +1425,7 @@
rootDirectory.getRelative("MODULE.bazel").getPathString(),
"isolated_ext = use_extension('//:extensions.bzl', 'my_ext', isolate = True)");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
EvaluationResult<RootModuleFileValue> result =
evaluator.evaluate(
@@ -1451,7 +1452,7 @@
rootDirectory.getRelative("MODULE.bazel").getPathString(),
"use_extension('//:extensions.bzl', 'my_ext', isolate = True)");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
reporter.removeHandler(failFastHandler); // expect failures
evaluator.evaluate(ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext);
@@ -1466,7 +1467,7 @@
rootDirectory.getRelative("MODULE.bazel").getPathString(),
"use_extension('//:extensions.bzl', 'my_ext', isolate = True)");
FakeRegistry registry = registryFactory.newFakeRegistry("/foo");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
reporter.removeHandler(failFastHandler); // expect failures
evaluator.evaluate(ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext);
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java
index 4c6c632..c56ad70 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java
@@ -43,10 +43,14 @@
new DownloadManager(new RepositoryCache(), new HttpDownloader()),
Suppliers.ofInstance(ImmutableMap.of()));
Throwable exception =
- assertThrows(URISyntaxException.class, () -> registryFactory.createRegistry("/home/www"));
+ assertThrows(
+ URISyntaxException.class,
+ () -> registryFactory.createRegistry("/home/www", ImmutableMap.of()));
assertThat(exception).hasMessageThat().contains("Registry URL has no scheme");
exception =
- assertThrows(URISyntaxException.class, () -> registryFactory.createRegistry("foo://bar"));
+ assertThrows(
+ URISyntaxException.class,
+ () -> registryFactory.createRegistry("foo://bar", ImmutableMap.of()));
assertThat(exception).hasMessageThat().contains("Unrecognized registry URL protocol");
}
@@ -61,7 +65,9 @@
Throwable exception =
assertThrows(
URISyntaxException.class,
- () -> registryFactory.createRegistry("file:c:/path/to/workspace/registry"));
+ () ->
+ registryFactory.createRegistry(
+ "file:c:/path/to/workspace/registry", ImmutableMap.of()));
assertThat(exception).hasMessageThat().contains("Registry URL path is not valid");
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/TestHttpServer.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/TestHttpServer.java
index 629e4be..8a7628f 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/TestHttpServer.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/TestHttpServer.java
@@ -78,6 +78,10 @@
serve(path, JOINER.join(lines).getBytes(UTF_8));
}
+ public void unserve(String path) {
+ server.removeContext(path);
+ }
+
public String getUrl() throws MalformedURLException {
return new URL("http", "[::1]", server.getAddress().getPort(), "").toString();
}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloaderTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloaderTest.java
index 0e59d41..2a47760 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloaderTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloaderTest.java
@@ -26,6 +26,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.hash.Hashing;
import com.google.common.io.ByteStreams;
import com.google.devtools.build.lib.authandtls.StaticCredentials;
import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache;
@@ -516,6 +517,7 @@
httpDownloader.downloadAndReadOneUrl(
new URL(String.format("http://localhost:%d/foo", server.getLocalPort())),
StaticCredentials.EMPTY,
+ Optional.empty(),
eventHandler,
Collections.emptyMap()),
UTF_8))
@@ -551,12 +553,91 @@
httpDownloader.downloadAndReadOneUrl(
new URL(String.format("http://localhost:%d/foo", server.getLocalPort())),
StaticCredentials.EMPTY,
+ Optional.empty(),
eventHandler,
Collections.emptyMap()));
}
}
@Test
+ public void downloadAndReadOneUrl_checksumProvided()
+ throws IOException, Checksum.InvalidChecksumException, InterruptedException {
+ try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName(null))) {
+ @SuppressWarnings("unused")
+ Future<?> possiblyIgnoredError =
+ executor.submit(
+ () -> {
+ try (Socket socket = server.accept()) {
+ readHttpRequest(socket.getInputStream());
+ sendLines(
+ socket,
+ "HTTP/1.1 200 OK",
+ "Date: Fri, 31 Dec 1999 23:59:59 GMT",
+ "Connection: close",
+ "Content-Type: text/plain",
+ "Content-Length: 5",
+ "",
+ "hello");
+ }
+ return null;
+ });
+
+ assertThat(
+ new String(
+ httpDownloader.downloadAndReadOneUrl(
+ new URL(String.format("http://localhost:%d/foo", server.getLocalPort())),
+ StaticCredentials.EMPTY,
+ Optional.of(
+ Checksum.fromString(
+ RepositoryCache.KeyType.SHA256,
+ Hashing.sha256().hashString("hello", UTF_8).toString())),
+ eventHandler,
+ ImmutableMap.of()),
+ UTF_8))
+ .isEqualTo("hello");
+ }
+ }
+
+ @Test
+ public void downloadAndReadOneUrl_checksumMismatch() throws IOException {
+ try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName(null))) {
+ @SuppressWarnings("unused")
+ Future<?> possiblyIgnoredError =
+ executor.submit(
+ () -> {
+ try (Socket socket = server.accept()) {
+ readHttpRequest(socket.getInputStream());
+ sendLines(
+ socket,
+ "HTTP/1.1 200 OK",
+ "Date: Fri, 31 Dec 1999 23:59:59 GMT",
+ "Connection: close",
+ "Content-Type: text/plain",
+ "Content-Length: 9",
+ "",
+ "malicious");
+ }
+ return null;
+ });
+
+ var e =
+ assertThrows(
+ UnrecoverableHttpException.class,
+ () ->
+ httpDownloader.downloadAndReadOneUrl(
+ new URL(String.format("http://localhost:%d/foo", server.getLocalPort())),
+ StaticCredentials.EMPTY,
+ Optional.of(
+ Checksum.fromString(
+ RepositoryCache.KeyType.SHA256,
+ Hashing.sha256().hashUnencodedChars("hello").toString())),
+ eventHandler,
+ ImmutableMap.of()));
+ assertThat(e).hasMessageThat().contains("Checksum was");
+ }
+ }
+
+ @Test
public void download_contentLengthMismatch_propagateErrorIfNotRetry() throws Exception {
Downloader downloader = mock(Downloader.class);
DownloadManager downloadManager = new DownloadManager(repositoryCache, downloader);
diff --git a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java
index af85d6a..8f60ae2 100644
--- a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java
+++ b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java
@@ -18,6 +18,7 @@
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.actions.ActionKeyContext;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
@@ -380,7 +381,7 @@
PrecomputedValue.injected(
ModuleFileFunction.MODULE_OVERRIDES, ImmutableMap.of()),
PrecomputedValue.injected(
- ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())),
+ ModuleFileFunction.REGISTRIES, ImmutableSet.of(registry.getUrl())),
PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false),
PrecomputedValue.injected(
BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES,
@@ -417,7 +418,7 @@
PrecomputedValue.injected(
RepositoryDelegatorFunction.VENDOR_DIRECTORY, Optional.empty()),
PrecomputedValue.injected(
- ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())),
+ ModuleFileFunction.REGISTRIES, ImmutableSet.of(registry.getUrl())),
PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false),
PrecomputedValue.injected(RepositoryDelegatorFunction.DISABLE_NATIVE_REPO_RULES, false),
PrecomputedValue.injected(
diff --git a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java
index 943afea..0c88fc5 100644
--- a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java
+++ b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java
@@ -175,7 +175,7 @@
.addModule(
createModuleKey("bazel_tools", "1.0"),
"module(name='bazel_tools', version='1.0');");
- ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl()));
+ ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl()));
HashFunction hashFunction = fileSystem.getDigestFunction().getHashFunction();
evaluator =
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java
index 4d64bb2..b7da221 100644
--- a/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java
@@ -19,6 +19,7 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
import com.google.common.eventbus.EventBus;
import com.google.devtools.build.lib.analysis.util.BuildViewTestCase;
import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction;
@@ -268,7 +269,7 @@
registry = FakeRegistry.DEFAULT_FACTORY.newFakeRegistry(moduleRoot.getPathString());
return ImmutableList.of(
PrecomputedValue.injected(
- ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())),
+ ModuleFileFunction.REGISTRIES, ImmutableSet.of(registry.getUrl())),
PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false),
PrecomputedValue.injected(
BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING),
diff --git a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py
index fb04b52..6118fb8 100644
--- a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py
+++ b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py
@@ -133,26 +133,68 @@
# hence find no errors
self.RunBazel(['build', '--nobuild', '//:all'])
- def testChangeFlagWithLockfile(self):
- # Add module 'sss' to the registry with dep on 'aaa'
- self.main_registry.createCcModule('sss', '1.3', {'aaa': '1.1'})
- # Create a project with deps on 'sss'
self.ScratchFile(
'MODULE.bazel',
[
'bazel_dep(name = "sss", version = "1.3")',
+ 'bazel_dep(name = "bbb", version = "1.1")',
+ ],
+ )
+ # Shutdown bazel to empty any cache of the deps tree
+ self.RunBazel(['shutdown'])
+ # Even adding a new dependency should not fail due to the registry change
+ self.RunBazel(['build', '--nobuild', '//:all'])
+
+ def testAddModuleToRegistryWithLockfile(self):
+ # Create a project with deps on the BCR's 'platforms' module
+ self.ScratchFile(
+ 'MODULE.bazel',
+ [
+ 'bazel_dep(name = "platforms", version = "0.0.9")',
],
)
self.ScratchFile('BUILD', ['filegroup(name = "hello")'])
self.RunBazel(['build', '--nobuild', '//:all'])
- # Change registry -> update 'sss' module file (corrupt it)
- module_dir = self.main_registry.root.joinpath('modules', 'sss', '1.3')
+ # Add a broken 'platforms' module to the first registry
+ module_dir = self.main_registry.root.joinpath(
+ 'modules', 'platforms', '0.0.9'
+ )
scratchFile(module_dir.joinpath('MODULE.bazel'), ['whatever!'])
# Shutdown bazel to empty any cache of the deps tree
self.RunBazel(['shutdown'])
- # Running with the lockfile, but adding a flag should cause resolution rerun
+ # Running with the lockfile, should not recognize the registry changes
+ # hence find no errors
+ self.RunBazel(['build', '--nobuild', '//:all'])
+
+ self.ScratchFile(
+ 'MODULE.bazel',
+ [
+ 'bazel_dep(name = "platforms", version = "0.0.9")',
+ 'bazel_dep(name = "bbb", version = "1.1")',
+ ],
+ )
+ # Shutdown bazel to empty any cache of the deps tree
+ self.RunBazel(['shutdown'])
+ # Even adding a new dependency should not fail due to the registry change
+ self.RunBazel(['build', '--nobuild', '//:all'])
+
+ def testChangeFlagWithLockfile(self):
+ # Create a project with an outdated direct dep on aaa
+ self.ScratchFile(
+ 'MODULE.bazel',
+ [
+ 'bazel_dep(name = "aaa", version = "1.0")',
+ 'bazel_dep(name = "bbb", version = "1.1")',
+ ],
+ )
+ self.ScratchFile('BUILD', ['filegroup(name = "hello")'])
+ self.RunBazel(['build', '--nobuild', '//:all'])
+
+ # Shutdown bazel to empty any cache of the deps tree
+ self.RunBazel(['shutdown'])
+ # Running with the lockfile, but the changed flag value should be honored
exit_code, _, stderr = self.RunBazel(
[
'build',
@@ -163,9 +205,10 @@
allow_failure=True,
)
self.AssertExitCode(exit_code, 48, stderr)
- self.assertRegex(
+ self.assertIn(
+ "ERROR: For repository 'aaa', the root module requires module version"
+ ' aaa@1.0, but got aaa@1.1',
'\n'.join(stderr),
- "ERROR: .*/sss/1.3/MODULE.bazel:1:9: invalid character: '!'",
)
def testLockfileErrorMode(self):
@@ -1104,6 +1147,62 @@
self.assertEqual(old_data, new_data)
+ def testLockfileExtensionsUpdatedIncrementally(self):
+ self.ScratchFile(
+ 'MODULE.bazel',
+ [
+ 'lockfile_ext1 = use_extension("extension.bzl", "lockfile_ext1")',
+ 'use_repo(lockfile_ext1, "hello1")',
+ 'lockfile_ext2 = use_extension("extension.bzl", "lockfile_ext2")',
+ 'use_repo(lockfile_ext2, "hello2")',
+ ],
+ )
+ self.ScratchFile('BUILD.bazel')
+ self.ScratchFile(
+ 'extension.bzl',
+ [
+ 'def _repo_rule_impl(ctx):',
+ ' ctx.file("BUILD", "filegroup(name=\'lala\')")',
+ '',
+ 'repo_rule = repository_rule(implementation=_repo_rule_impl)',
+ '',
+ 'def _module_ext1_impl(ctx):',
+ ' print("Hello from ext1!")',
+ ' repo_rule(name="hello1")',
+ '',
+ 'lockfile_ext1 = module_extension(',
+ ' implementation=_module_ext1_impl,',
+ ')',
+ '',
+ 'def _module_ext2_impl(ctx):',
+ ' print("Hello from ext2!")',
+ ' repo_rule(name="hello2")',
+ '',
+ 'lockfile_ext2 = module_extension(',
+ ' implementation=_module_ext2_impl,',
+ ')',
+ ],
+ )
+
+ _, _, stderr = self.RunBazel(['build', '@hello1//:all'])
+ stderr = '\n'.join(stderr)
+ self.assertIn('Hello from ext1!', stderr)
+ self.assertNotIn('Hello from ext2!', stderr)
+
+ self.RunBazel(['shutdown'])
+
+ _, _, stderr = self.RunBazel(['build', '@hello1//:all', '@hello2//:all'])
+ stderr = '\n'.join(stderr)
+ self.assertNotIn('Hello from ext1!', stderr)
+ self.assertIn('Hello from ext2!', stderr)
+
+ self.RunBazel(['shutdown'])
+
+ _, _, stderr = self.RunBazel(['build', '@hello1//:all', '@hello2//:all'])
+ stderr = '\n'.join(stderr)
+ self.assertNotIn('Hello from ext1!', stderr)
+ self.assertNotIn('Hello from ext2!', stderr)
+
def testExtensionOsAndArch(self):
self.ScratchFile(
'MODULE.bazel',
diff --git a/src/test/shell/bazel/starlark_repository_test.sh b/src/test/shell/bazel/starlark_repository_test.sh
index a53803f..5742bb5 100755
--- a/src/test/shell/bazel/starlark_repository_test.sh
+++ b/src/test/shell/bazel/starlark_repository_test.sh
@@ -2327,6 +2327,9 @@
)
EOF
+ # Ensure that all Bzlmod-related downloads have happened before disabling
+ # downloads.
+ bazel mod deps || fail "Failed to cache Bazel modules"
bazel build --repository_disable_download //:it > "${TEST_log}" 2>&1 \
&& fail "Expected failure" || :
expect_log "Failed to download repository @.*: download is disabled"
diff --git a/src/test/tools/bzlmod/MODULE.bazel.lock b/src/test/tools/bzlmod/MODULE.bazel.lock
index 5f745e0..43fb80e 100644
--- a/src/test/tools/bzlmod/MODULE.bazel.lock
+++ b/src/test/tools/bzlmod/MODULE.bazel.lock
@@ -1074,6 +1074,63 @@
}
}
},
+ "registryFileHashes": {
+ "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497",
+ "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2",
+ "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589",
+ "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/source.json": "7e3a9adf473e9af076ae485ed649d5641ad50ec5c11718103f34de03170d94ad",
+ "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel": "50341a62efbc483e8a2a6aec30994a58749bd7b885e18dd96aa8c33031e558ef",
+ "https://bcr.bazel.build/modules/apple_support/1.5.0/source.json": "eb98a7627c0bc486b57f598ad8da50f6625d974c8f723e9ea71bd39f709c9862",
+ "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8",
+ "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686",
+ "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a",
+ "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5",
+ "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/source.json": "2e0e90f6788740b72f0db3c19c46155a82ec01cfc1527c35b58f3f8f0180da29",
+ "https://bcr.bazel.build/modules/buildozer/7.1.1.1/MODULE.bazel": "21c6a7d08e3171d3e13b003407caefe7ebe007693e217a053cc1f49f008ce010",
+ "https://bcr.bazel.build/modules/buildozer/7.1.1.1/source.json": "a9ced884dedcf1c45d11052d53d854e368b05aa8fbbf0f983037fbed4d3ea4c6",
+ "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4",
+ "https://bcr.bazel.build/modules/googletest/1.11.0/source.json": "c73d9ef4268c91bd0c1cd88f1f9dfa08e814b1dbe89b5f594a9f08ba0244d206",
+ "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee",
+ "https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37",
+ "https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615",
+ "https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814",
+ "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc",
+ "https://bcr.bazel.build/modules/platforms/0.0.9/source.json": "cd74d854bf16a9e002fb2ca7b1a421f4403cda29f824a765acd3a8c56f8d43e6",
+ "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7",
+ "https://bcr.bazel.build/modules/protobuf/21.7/source.json": "bbe500720421e582ff2d18b0802464205138c06056f443184de39fbb8187b09b",
+ "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0",
+ "https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858",
+ "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647",
+ "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c",
+ "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e",
+ "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5",
+ "https://bcr.bazel.build/modules/rules_cc/0.0.9/source.json": "1f1ba6fea244b616de4a554a0f4983c91a9301640c8fe0dd1d410254115c8430",
+ "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74",
+ "https://bcr.bazel.build/modules/rules_java/7.5.0/MODULE.bazel": "b329bf9aa07a58bd1ccb37bfdcd9528acf6f12712efb38c3a8553c2cc2494806",
+ "https://bcr.bazel.build/modules/rules_java/7.5.0/source.json": "72762e4e144dd1bc54e18b90be52e31a4ca9cf11d7358a06e1b87b74e839e9ad",
+ "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7",
+ "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/source.json": "a075731e1b46bc8425098512d038d416e966ab19684a10a34f4741295642fc35",
+ "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0",
+ "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d",
+ "https://bcr.bazel.build/modules/rules_license/0.0.7/source.json": "355cc5737a0f294e560d52b1b7a6492d4fff2caf0bef1a315df5a298fca2d34a",
+ "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc",
+ "https://bcr.bazel.build/modules/rules_pkg/0.7.0/source.json": "c2557066e0c0342223ba592510ad3d812d4963b9024831f7f66fd0584dd8c66c",
+ "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06",
+ "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7",
+ "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/source.json": "d57902c052424dfda0e71646cb12668d39c4620ee0544294d9d941e7d12bc3a9",
+ "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f",
+ "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel": "26114f0c0b5e93018c0c066d6673f1a2c3737c7e90af95eff30cfee38d0bbac7",
+ "https://bcr.bazel.build/modules/rules_python/0.22.1/source.json": "57226905e783bae7c37c2dd662be078728e48fa28ee4324a7eabcafb5a43d014",
+ "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c",
+ "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8",
+ "https://bcr.bazel.build/modules/stardoc/0.5.1/source.json": "a96f95e02123320aa015b956f29c00cb818fa891ef823d55148e1a362caacf29",
+ "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43",
+ "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/source.json": "f1ef7d3f9e0e26d4b23d1c39b5f5de71f584dd7d1b4ef83d9bbba6ec7a6a6459",
+ "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0",
+ "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27",
+ "https://bcr.bazel.build/modules/zlib/1.3/MODULE.bazel": "6a9c02f19a24dcedb05572b2381446e27c272cd383aed11d41d99da9e3167a72",
+ "https://bcr.bazel.build/modules/zlib/1.3/source.json": "b6b43d0737af846022636e6e255fd4a96fee0d34f08f3830e6e0bac51465c37c"
+ },
"moduleExtensions": {
"@@apple_support~//crosstool:setup.bzl%apple_cc_configure_extension": {
"general": {
