Implement RemoteDownloader w/ `--experimental_remote_downloader`
This is the Bazel client implementation of https://github.com/bazelbuild/proposals/pull/160. It allows downloading of external dependencies to be delegated to a remote service.
TODOs:
- [x] Once https://github.com/bazelbuild/remote-apis/pull/112 is merged, the vendored copy of `bazelbuild/remote-apis` should be updated. I've used a [WIP] placeholder for now.
- [x] If the general approach looks reasonable then I'll add tests. Currently I've been testing with an in-house implementation of the downloader server.
R: @buchgr @dslomov
CC: @EricBurnett @sstriker @ulfjack
Closes #10622.
PiperOrigin-RevId: 300116716
diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD
index f077ea8..f5d38c2 100644
--- a/src/main/java/com/google/devtools/build/lib/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/BUILD
@@ -1252,6 +1252,7 @@
":util",
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity",
+ "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
"//src/main/java/com/google/devtools/build/lib/buildeventstream",
"//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto",
"//src/main/java/com/google/devtools/build/lib/buildeventstream/transports",
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
index 185193f..8212e47 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
@@ -33,6 +33,7 @@
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions;
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.RepositoryOverride;
import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache;
+import com.google.devtools.build.lib.bazel.repository.downloader.DelegatingDownloader;
import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager;
import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
import com.google.devtools.build.lib.bazel.repository.skylark.SkylarkRepositoryFunction;
@@ -97,8 +98,10 @@
private final SkylarkRepositoryFunction skylarkRepositoryFunction;
private final RepositoryCache repositoryCache = new RepositoryCache();
private final HttpDownloader httpDownloader = new HttpDownloader();
+ private final DelegatingDownloader delegatingDownloader =
+ new DelegatingDownloader(httpDownloader);
private final DownloadManager downloadManager =
- new DownloadManager(repositoryCache, httpDownloader);
+ new DownloadManager(repositoryCache, delegatingDownloader);
private final MutableSupplier<Map<String, String>> clientEnvironmentSupplier =
new MutableSupplier<>();
private ImmutableMap<RepositoryName, PathFragment> overrides = ImmutableMap.of();
@@ -334,6 +337,7 @@
remoteExecutor = remoteExecutorFactory.create();
}
skylarkRepositoryFunction.setRepositoryRemoteExecutor(remoteExecutor);
+ delegatingDownloader.setDelegate(env.getRuntime().getDownloaderSupplier().get());
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DelegatingDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DelegatingDownloader.java
new file mode 100644
index 0000000..6000262
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DelegatingDownloader.java
@@ -0,0 +1,64 @@
+// Copyright 2020 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository.downloader;
+
+import com.google.common.base.Optional;
+import com.google.devtools.build.lib.events.ExtendedEventHandler;
+import com.google.devtools.build.lib.vfs.Path;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URL;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/**
+ * A {@link Downloader} that delegates to another Downloader. Primarily useful for mutable
+ * dependency injection.
+ */
+public class DelegatingDownloader implements Downloader {
+ private final Downloader defaultDelegate;
+ @Nullable private Downloader delegate;
+
+ public DelegatingDownloader(Downloader defaultDelegate) {
+ this.defaultDelegate = defaultDelegate;
+ }
+
+ /**
+ * Sets the {@link Downloader} to delegate to. If setDelegate(null) is called, the default
+ * delegate passed to the constructor will be used.
+ */
+ public void setDelegate(@Nullable Downloader delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public void download(
+ List<URL> urls,
+ Map<URI, Map<String, String>> authHeaders,
+ Optional<Checksum> checksum,
+ String canonicalId,
+ Path destination,
+ ExtendedEventHandler eventHandler,
+ Map<String, String> clientEnv)
+ throws IOException, InterruptedException {
+ Downloader downloader = defaultDelegate;
+ if (delegate != null) {
+ downloader = delegate;
+ }
+ downloader.download(
+ urls, authHeaders, checksum, canonicalId, destination, eventHandler, clientEnv);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/BUILD b/src/main/java/com/google/devtools/build/lib/remote/BUILD
index 69bc4de..0410d33 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/remote/BUILD
@@ -51,12 +51,14 @@
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/analysis/platform:platform_utils",
"//src/main/java/com/google/devtools/build/lib/authandtls",
+ "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
"//src/main/java/com/google/devtools/build/lib/buildeventstream",
"//src/main/java/com/google/devtools/build/lib/collect/nestedset",
"//src/main/java/com/google/devtools/build/lib/concurrent",
"//src/main/java/com/google/devtools/build/lib/profiler",
"//src/main/java/com/google/devtools/build/lib/remote/common",
"//src/main/java/com/google/devtools/build/lib/remote/disk",
+ "//src/main/java/com/google/devtools/build/lib/remote/downloader",
"//src/main/java/com/google/devtools/build/lib/remote/http",
"//src/main/java/com/google/devtools/build/lib/remote/logging",
"//src/main/java/com/google/devtools/build/lib/remote/merkletree",
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
index 7a342f1..67d7515 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
@@ -34,6 +34,7 @@
import com.google.devtools.build.lib.analysis.test.TestProvider;
import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
+import com.google.devtools.build.lib.bazel.repository.downloader.Downloader;
import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader;
import com.google.devtools.build.lib.buildeventstream.LocalFilesArtifactUploader;
import com.google.devtools.build.lib.buildtool.BuildRequest;
@@ -43,6 +44,7 @@
import com.google.devtools.build.lib.exec.ExecutorBuilder;
import com.google.devtools.build.lib.packages.TargetUtils;
import com.google.devtools.build.lib.remote.common.RemoteCacheClient;
+import com.google.devtools.build.lib.remote.downloader.GrpcRemoteDownloader;
import com.google.devtools.build.lib.remote.logging.LoggingInterceptor;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
import com.google.devtools.build.lib.remote.options.RemoteOutputsMode;
@@ -57,6 +59,7 @@
import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutorFactory;
import com.google.devtools.build.lib.runtime.ServerBuilder;
import com.google.devtools.build.lib.skyframe.AspectValue;
+import com.google.devtools.build.lib.skyframe.MutableSupplier;
import com.google.devtools.build.lib.util.AbruptExitException;
import com.google.devtools.build.lib.util.ExitCode;
import com.google.devtools.build.lib.util.io.AsynchronousFileOutputStream;
@@ -70,6 +73,7 @@
import io.grpc.Context;
import java.io.IOException;
import java.util.List;
+import java.util.Optional;
import java.util.concurrent.Executors;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -95,11 +99,14 @@
private final RepositoryRemoteExecutorFactoryDelegate repositoryRemoteExecutorFactoryDelegate =
new RepositoryRemoteExecutorFactoryDelegate();
+ private final MutableSupplier<Downloader> remoteDownloaderSupplier = new MutableSupplier<>();
+
@Override
public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builder) {
builder.addBuildEventArtifactUploaderFactory(
buildEventArtifactUploaderFactoryDelegate, "remote");
builder.setRepositoryRemoteExecutorFactory(repositoryRemoteExecutorFactoryDelegate);
+ builder.setDownloaderSupplier(remoteDownloaderSupplier);
}
/** Returns whether remote execution should be available. */
@@ -107,6 +114,11 @@
return !Strings.isNullOrEmpty(options.remoteExecutor);
}
+ /** Returns whether remote downloading should be available. */
+ private static boolean shouldEnableRemoteDownloader(RemoteOptions options) {
+ return !Strings.isNullOrEmpty(options.remoteDownloader);
+ }
+
private void verifyServerCapabilities(
RemoteOptions remoteOptions,
ReferenceCountedChannel channel,
@@ -160,6 +172,13 @@
boolean enableHttpCache = RemoteCacheClientFactory.isHttpCache(remoteOptions);
boolean enableGrpcCache = GrpcCacheClient.isRemoteCacheOptions(remoteOptions);
boolean enableRemoteExecution = shouldEnableRemoteExecution(remoteOptions);
+ boolean enableRemoteDownloader = shouldEnableRemoteDownloader(remoteOptions);
+
+ if (enableRemoteDownloader && !enableGrpcCache) {
+ throw new AbruptExitException(
+ "The remote downloader can only be used in combination with gRPC caching",
+ ExitCode.COMMAND_LINE_ERROR);
+ }
if (!enableDiskCache && !enableHttpCache && !enableGrpcCache && !enableRemoteExecution) {
// Quit if no remote caching or execution was enabled.
@@ -208,6 +227,7 @@
ReferenceCountedChannel execChannel = null;
ReferenceCountedChannel cacheChannel = null;
+ ReferenceCountedChannel downloaderChannel = null;
if (enableRemoteExecution) {
ImmutableList.Builder<ClientInterceptor> interceptors = ImmutableList.builder();
interceptors.add(TracingMetadataUtils.newExecHeadersInterceptor(remoteOptions));
@@ -242,6 +262,25 @@
interceptors.build());
}
+ if (enableRemoteDownloader) {
+ // Create a separate channel if --remote_downloader and --remote_cache point to different
+ // endpoints.
+ if (remoteOptions.remoteDownloader.equals(remoteOptions.remoteCache)) {
+ downloaderChannel = cacheChannel.retain();
+ } else {
+ ImmutableList.Builder<ClientInterceptor> interceptors = ImmutableList.builder();
+ if (loggingInterceptor != null) {
+ interceptors.add(loggingInterceptor);
+ }
+ downloaderChannel =
+ RemoteCacheClientFactory.createGrpcChannel(
+ remoteOptions.remoteDownloader,
+ remoteOptions.remoteProxy,
+ authAndTlsOptions,
+ interceptors.build());
+ }
+ }
+
CallCredentials credentials = GoogleAuthUtils.newCallCredentials(authAndTlsOptions);
RemoteRetrier retrier =
new RemoteRetrier(
@@ -289,6 +328,9 @@
requestContext,
remoteOptions.remoteInstanceName));
+ Context repoContext =
+ TracingMetadataUtils.contextWithMetadata(buildRequestId, invocationId, "repository_rule");
+
if (enableRemoteExecution) {
RemoteRetrier execRetrier =
new RemoteRetrier(
@@ -308,9 +350,6 @@
actionContextProvider =
RemoteActionContextProvider.createForRemoteExecution(
env, remoteCache, remoteExecutor, retryScheduler, digestUtil, logDir);
- Context repoContext =
- TracingMetadataUtils.contextWithMetadata(
- buildRequestId, invocationId, "repository_rule");
repositoryRemoteExecutorFactoryDelegate.init(
new RemoteRepositoryRemoteExecutorFactory(
remoteCache,
@@ -336,6 +375,18 @@
RemoteActionContextProvider.createForRemoteCaching(
env, remoteCache, retryScheduler, digestUtil);
}
+
+ if (enableRemoteDownloader) {
+ remoteDownloaderSupplier.set(
+ new GrpcRemoteDownloader(
+ downloaderChannel.retain(),
+ Optional.ofNullable(credentials),
+ retrier,
+ repoContext,
+ cacheClient,
+ remoteOptions));
+ downloaderChannel.release();
+ }
} catch (IOException e) {
env.getReporter().handle(Event.error(e.getMessage()));
env.getBlazeModuleEnvironment()
@@ -468,6 +519,7 @@
buildEventArtifactUploaderFactoryDelegate.reset();
repositoryRemoteExecutorFactoryDelegate.reset();
+ remoteDownloaderSupplier.set(null);
actionContextProvider = null;
actionInputFetcher = null;
remoteOutputsMode = null;
diff --git a/src/main/java/com/google/devtools/build/lib/remote/downloader/BUILD b/src/main/java/com/google/devtools/build/lib/remote/downloader/BUILD
index 1035c7d..862215f 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/downloader/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/remote/downloader/BUILD
@@ -15,7 +15,6 @@
deps = [
"//src/main/java/com/google/devtools/build/lib:events",
"//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
- "//src/main/java/com/google/devtools/build/lib/remote",
"//src/main/java/com/google/devtools/build/lib/remote:ReferenceCountedChannel",
"//src/main/java/com/google/devtools/build/lib/remote:Retrier",
"//src/main/java/com/google/devtools/build/lib/remote/common",
diff --git a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java
index 4e4e27e..9c9f99f 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java
@@ -85,7 +85,16 @@
+ " https://docs.bazel.build/versions/master/remote-caching.html")
public String remoteCache;
- public final String remoteDownloader = "";
+ @Option(
+ name = "experimental_remote_downloader",
+ defaultValue = "null",
+ documentationCategory = OptionDocumentationCategory.REMOTE,
+ effectTags = {OptionEffectTag.UNKNOWN},
+ help =
+ "A URI of a remote downloader endpoint. The supported schemas are grpc and grpcs"
+ + " (grpc with TLS enabled). If no schema is provided bazel will default to grpcs."
+ + " Specify grpc:// schema to disable TLS.")
+ public String remoteDownloader;
@Option(
name = "remote_header",
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
index 1d1d640..1592416 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
@@ -34,6 +34,7 @@
import com.google.devtools.build.lib.analysis.ServerDirectories;
import com.google.devtools.build.lib.analysis.config.BuildOptions;
import com.google.devtools.build.lib.analysis.test.CoverageReportActionFactory;
+import com.google.devtools.build.lib.bazel.repository.downloader.Downloader;
import com.google.devtools.build.lib.bugreport.BugReport;
import com.google.devtools.build.lib.bugreport.BugReporter;
import com.google.devtools.build.lib.buildeventstream.BuildEvent.LocalFile.LocalFileType;
@@ -123,6 +124,7 @@
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
+import java.util.function.Supplier;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.LogRecord;
@@ -175,6 +177,7 @@
private final ImmutableMap<String, AuthHeadersProvider> authHeadersProviderMap;
private final RetainedHeapLimiter retainedHeapLimiter = new RetainedHeapLimiter();
@Nullable private final RepositoryRemoteExecutorFactory repositoryRemoteExecutorFactory;
+ private final Supplier<Downloader> downloaderSupplier;
// Workspace state (currently exactly one workspace per server)
private BlazeWorkspace workspace;
@@ -201,7 +204,8 @@
String productName,
BuildEventArtifactUploaderFactoryMap buildEventArtifactUploaderFactoryMap,
ImmutableMap<String, AuthHeadersProvider> authHeadersProviderMap,
- RepositoryRemoteExecutorFactory repositoryRemoteExecutorFactory) {
+ RepositoryRemoteExecutorFactory repositoryRemoteExecutorFactory,
+ Supplier<Downloader> downloaderSupplier) {
// Server state
this.fileSystem = fileSystem;
this.blazeModules = blazeModules;
@@ -231,6 +235,7 @@
this.authHeadersProviderMap =
Preconditions.checkNotNull(authHeadersProviderMap, "authHeadersProviderMap");
this.repositoryRemoteExecutorFactory = repositoryRemoteExecutorFactory;
+ this.downloaderSupplier = downloaderSupplier;
}
public BlazeWorkspace initWorkspace(BlazeDirectories directories, BinTools binTools)
@@ -1448,6 +1453,10 @@
return repositoryRemoteExecutorFactory;
}
+ public Supplier<Downloader> getDownloaderSupplier() {
+ return downloaderSupplier;
+ }
+
/**
* A builder for {@link BlazeRuntime} objects. The only required fields are the {@link
* BlazeDirectories}, and the {@link RuleClassProvider} (except for testing). All other fields
@@ -1589,7 +1598,8 @@
productName,
serverBuilder.getBuildEventArtifactUploaderMap(),
serverBuilder.getAuthHeadersProvidersMap(),
- serverBuilder.getRepositoryRemoteExecutorFactory());
+ serverBuilder.getRepositoryRemoteExecutorFactory(),
+ serverBuilder.getDownloaderSupplier());
}
public Builder setProductName(String productName) {
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/ServerBuilder.java b/src/main/java/com/google/devtools/build/lib/runtime/ServerBuilder.java
index b173ee5..064bdc6 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/ServerBuilder.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/ServerBuilder.java
@@ -17,6 +17,7 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.bazel.repository.downloader.Downloader;
import com.google.devtools.build.lib.packages.PackageFactory;
import com.google.devtools.build.lib.query2.QueryEnvironmentFactory;
import com.google.devtools.build.lib.query2.common.AbstractBlazeQueryEnvironment;
@@ -24,6 +25,7 @@
import com.google.devtools.build.lib.query2.query.output.OutputFormatter;
import com.google.devtools.build.lib.runtime.commands.InfoItem;
import com.google.devtools.build.lib.runtime.proto.InvocationPolicyOuterClass.InvocationPolicy;
+import java.util.function.Supplier;
/**
* Builder class to create a {@link BlazeRuntime} instance. This class is part of the module API,
@@ -44,6 +46,7 @@
private final ImmutableMap.Builder<String, AuthHeadersProvider> authHeadersProvidersMap =
ImmutableMap.builder();
private RepositoryRemoteExecutorFactory repositoryRemoteExecutorFactory;
+ private Supplier<Downloader> downloaderSupplier = () -> null;
@VisibleForTesting
public ServerBuilder() {}
@@ -88,6 +91,10 @@
return repositoryRemoteExecutorFactory;
}
+ public Supplier<Downloader> getDownloaderSupplier() {
+ return downloaderSupplier;
+ }
+
/**
* Merges the given invocation policy into the per-server invocation policy. While this can accept
* any number of policies, the end result is order-dependent if multiple policies attempt to
@@ -172,6 +179,11 @@
return this;
}
+ public ServerBuilder setDownloaderSupplier(Supplier<Downloader> downloaderSupplier) {
+ this.downloaderSupplier = downloaderSupplier;
+ return this;
+ }
+
/**
* Register a provider of authentication headers that blaze modules can use. See {@link
* AuthHeadersProvider} for more details.
