Support local search for http archives
With --experimental_repository_cache, bazel has means of avoiding
downloading the same archive again. However, this requires bazel
to first download it itself, as we make no guarantee about the
internal structure of that cache; this, in turn, does not play
well in situations where bazel has to cooperate with other tools,
e.g., because the bazel build is just one step in a larger package
building process. Therefore, add an experimental option allowing
to specify directories where the outer process may have placed
needed files and make bazel not download them if a file with correct
name and hash could be found in one of those directories. In this
way, cooperation is possible without patching all entries in the
WORSPACE file.
Change-Id: I43240b8b59bf8472ec0310661015899e46491236
PiperOrigin-RevId: 185115713
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
index 0ff341f..9012ed4 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
@@ -83,6 +83,7 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
import javax.annotation.Nullable;
/**
@@ -204,6 +205,15 @@
repositoryCache.setRepositoryCachePath(null);
}
+ if (repoOptions.experimentalDistdir != null) {
+ httpDownloader.setDistdir(
+ repoOptions
+ .experimentalDistdir
+ .stream()
+ .map(filesystem::getPath)
+ .collect(Collectors.toList()));
+ }
+
if (repoOptions.repositoryOverrides != null) {
ImmutableMap.Builder<RepositoryName, PathFragment> builder = ImmutableMap.builder();
for (RepositoryOverride override : repoOptions.repositoryOverrides) {
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java
index 72a1699..61e5010 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java
@@ -47,6 +47,20 @@
public PathFragment experimentalRepositoryCache;
@Option(
+ name = "experimental_distdir",
+ defaultValue = "null",
+ allowMultiple = true,
+ documentationCategory = OptionDocumentationCategory.BAZEL_CLIENT_OPTIONS,
+ effectTags = {OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION},
+ metadataTags = {OptionMetadataTag.EXPERIMENTAL},
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help =
+ "Additional places to search for archives before accessing the network "
+ + "to download them."
+ )
+ public List<PathFragment> experimentalDistdir;
+
+ @Option(
name = "override_repository",
defaultValue = "null",
allowMultiple = true,
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java
index 1856bbd..8bcb5eb 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java
@@ -32,6 +32,7 @@
import com.google.devtools.build.lib.syntax.Type;
import com.google.devtools.build.lib.util.JavaSleeper;
import com.google.devtools.build.lib.util.Sleeper;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
@@ -58,11 +59,16 @@
private static final Semaphore semaphore = new Semaphore(MAX_PARALLEL_DOWNLOADS, true);
protected final RepositoryCache repositoryCache;
+ private List<Path> distdir = ImmutableList.of();
public HttpDownloader(RepositoryCache repositoryCache) {
this.repositoryCache = repositoryCache;
}
+ public void setDistdir(List<Path> distdir) {
+ this.distdir = ImmutableList.copyOf(distdir);
+ }
+
/** Validates native repository rule attributes and calls the other download method. */
public Path download(
Rule rule,
@@ -182,6 +188,19 @@
return cachedDestination;
}
}
+
+ for (Path dir : distdir) {
+ Path candidate = dir.getRelative(destination.getBaseName());
+ if (RepositoryCache.getChecksum(KeyType.SHA256, candidate).equals(sha256)) {
+ // Found the archive in one of the distdirs, no need to download.
+ if (isCaching) {
+ repositoryCache.put(sha256, candidate, KeyType.SHA256);
+ }
+ FileSystemUtils.createDirectoryAndParents(destination.getParentDirectory());
+ FileSystemUtils.copyFile(candidate, destination);
+ return destination;
+ }
+ }
}
Clock clock = new JavaClock();
diff --git a/src/test/shell/bazel/external_integration_test.sh b/src/test/shell/bazel/external_integration_test.sh
index b5c5f84..cc3b18a 100755
--- a/src/test/shell/bazel/external_integration_test.sh
+++ b/src/test/shell/bazel/external_integration_test.sh
@@ -1168,6 +1168,50 @@
expect_log '@ext//:foo'
}
+function test_distdir() {
+ WRKDIR=$(mktemp -d "${TEST_TMPDIR}/testXXXXXX")
+ cd "${WRKDIR}"
+ mkdir ext
+ cat > ext/BUILD <<'EOF'
+genrule(
+ name="foo",
+ outs=["foo.txt"],
+ cmd="echo Hello World > $@",
+ visibility = ["//visibility:public"],
+)
+EOF
+ zip ext.zip ext/*
+ rm -rf ext
+ sha256=$(sha256sum ext.zip | head -c 64)
+
+ mkdir distfiles
+ mv ext.zip distfiles
+
+ mkdir main
+ cd main
+ cat > WORKSPACE <<EOF
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+http_archive(
+ name="ext",
+ strip_prefix="ext",
+ urls=["http://doesnotexist.example.com/outdatedpath/ext.zip"],
+ sha256="${sha256}",
+)
+EOF
+ cat > BUILD <<'EOF'
+genrule(
+ name = "local",
+ srcs = ["@ext//:foo"],
+ outs = ["local.txt"],
+ cmd = "cp $< $@",
+)
+EOF
+
+ bazel clean --expunge
+ bazel build --experimental_distdir="${WRKDIR}/distfiles" //:local \
+ || fail "expected success"
+}
+
function test_good_symlinks() {
WRKDIR=$(mktemp -d "${TEST_TMPDIR}/testXXXXXX")
cd "${WRKDIR}"
