[6.0.0] Keep credentials cached across build commands. (#16881)
When using a credential helper, the lifetime of the credential cache is currently tied to an individual command, which causes the helper to be called for every command resulting in poor incremental build latency for builds using a non-trivial helper.
Since the cache must be shared by RemoteModule and BazelBuildServiceModule, I've introduced a new CredentialModule whose sole purpose is to provide access to it.
Closes #16822.
PiperOrigin-RevId: 491598103
Change-Id: Ib668954b635a0e9498f0a7418707d6a2dfae0265
Co-authored-by: kshyanashree <109167932+kshyanashree@users.noreply.github.com>
diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/AuthAndTLSOptions.java b/src/main/java/com/google/devtools/build/lib/authandtls/AuthAndTLSOptions.java
index 8f3aae8..7423b55 100644
--- a/src/main/java/com/google/devtools/build/lib/authandtls/AuthAndTLSOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/authandtls/AuthAndTLSOptions.java
@@ -175,7 +175,11 @@
converter = DurationConverter.class,
documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
effectTags = {OptionEffectTag.UNKNOWN},
- help = "Configures the duration for which credentials from Credential Helpers are cached.")
+ help =
+ "Configures the duration for which credentials from Credential Helpers are cached.\n\n"
+ + "Invoking with a different value will adjust the lifetime of preexisting entries;"
+ + " pass zero to clear the cache. A clean command always clears the cache, regardless"
+ + " of this flag.")
public Duration credentialHelperCacheTimeout;
/** One of the values of the `--credential_helper` flag. */
diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/BUILD b/src/main/java/com/google/devtools/build/lib/authandtls/BUILD
index 252e658..d220892 100644
--- a/src/main/java/com/google/devtools/build/lib/authandtls/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/authandtls/BUILD
@@ -22,6 +22,7 @@
"//src/main/java/com/google/devtools/common/options",
"//third_party:auth",
"//third_party:auto_value",
+ "//third_party:caffeine",
"//third_party:guava",
"//third_party:jsr305",
"//third_party:netty",
diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/GoogleAuthUtils.java b/src/main/java/com/google/devtools/build/lib/authandtls/GoogleAuthUtils.java
index fa4b81d..e226e6b 100644
--- a/src/main/java/com/google/devtools/build/lib/authandtls/GoogleAuthUtils.java
+++ b/src/main/java/com/google/devtools/build/lib/authandtls/GoogleAuthUtils.java
@@ -14,11 +14,14 @@
package com.google.devtools.build.lib.authandtls;
+import com.github.benmanes.caffeine.cache.Cache;
import com.google.auth.Credentials;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials;
import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment;
import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperProvider;
@@ -48,6 +51,7 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
+import java.net.URI;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -248,6 +252,7 @@
*/
public static Credentials newCredentials(
CredentialHelperEnvironment credentialHelperEnvironment,
+ Cache<URI, ImmutableMap<String, ImmutableList<String>>> credentialCache,
CommandLinePathFactory commandLinePathFactory,
FileSystem fileSystem,
AuthAndTLSOptions authAndTlsOptions)
@@ -257,12 +262,12 @@
Preconditions.checkNotNull(fileSystem);
Preconditions.checkNotNull(authAndTlsOptions);
- Optional<Credentials> credentials = newGoogleCredentials(authAndTlsOptions);
+ Optional<Credentials> fallbackCredentials = newGoogleCredentials(authAndTlsOptions);
- if (credentials.isEmpty()) {
+ if (fallbackCredentials.isEmpty()) {
// Fallback to .netrc if it exists.
try {
- credentials =
+ fallbackCredentials =
newCredentialsFromNetrc(credentialHelperEnvironment.getClientEnvironment(), fileSystem);
} catch (IOException e) {
// TODO(yannic): Make this fail the build.
@@ -276,8 +281,8 @@
commandLinePathFactory,
authAndTlsOptions.credentialHelpers),
credentialHelperEnvironment,
- credentials,
- authAndTlsOptions.credentialHelperCacheTimeout);
+ credentialCache,
+ fallbackCredentials);
}
/**
diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD
index b14a03c..275c159 100644
--- a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD
@@ -9,8 +9,22 @@
)
java_library(
+ name = "credential_module",
+ srcs = ["CredentialModule.java"],
+ deps = [
+ "//src/main/java/com/google/devtools/build/lib:runtime",
+ "//src/main/java/com/google/devtools/build/lib/authandtls",
+ "//third_party:caffeine",
+ "//third_party:guava",
+ ],
+)
+
+java_library(
name = "credentialhelper",
- srcs = glob(["*.java"]),
+ srcs = glob(
+ ["*.java"],
+ exclude = ["CredentialModule.java"],
+ ),
deps = [
"//src/main/java/com/google/devtools/build/lib/events",
"//src/main/java/com/google/devtools/build/lib/profiler",
diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java
index 2219a59..c1f0a09 100644
--- a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java
+++ b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java
@@ -67,7 +67,7 @@
* @return The response from the subprocess.
*/
public GetCredentialsResponse getCredentials(CredentialHelperEnvironment environment, URI uri)
- throws InterruptedException, IOException {
+ throws IOException {
Preconditions.checkNotNull(environment);
Preconditions.checkNotNull(uri);
@@ -81,7 +81,16 @@
GSON.toJson(GetCredentialsRequest.newBuilder().setUri(uri).build(), stdin);
}
- process.waitFor();
+ try {
+ process.waitFor();
+ } catch (InterruptedException e) {
+ throw new CredentialHelperException(
+ String.format(
+ Locale.US,
+ "Failed to get credentials for '%s' from helper '%s': process was interrupted",
+ uri,
+ path));
+ }
if (process.timedout()) {
throw new CredentialHelperException(
diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperCredentials.java b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperCredentials.java
index ecc40e1..5de7608 100644
--- a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperCredentials.java
+++ b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperCredentials.java
@@ -14,15 +14,13 @@
package com.google.devtools.build.lib.authandtls.credentialhelper;
-import com.github.benmanes.caffeine.cache.CacheLoader;
-import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.github.benmanes.caffeine.cache.Cache;
import com.google.auth.Credentials;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.net.URI;
-import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -33,29 +31,34 @@
* helper} as subprocess, falling back to another {@link Credentials} if no suitable helper exists.
*/
public class CredentialHelperCredentials extends Credentials {
+ private final CredentialHelperProvider credentialHelperProvider;
+ private final CredentialHelperEnvironment credentialHelperEnvironment;
+ private final Cache<URI, ImmutableMap<String, ImmutableList<String>>> credentialCache;
private final Optional<Credentials> fallbackCredentials;
- private final LoadingCache<URI, GetCredentialsResponse> credentialCache;
+ /** Wraps around an {@link IOException} so we can smuggle it through {@link Cache#get}. */
+ public static final class WrappedIOException extends RuntimeException {
+ private final IOException wrapped;
+
+ WrappedIOException(IOException e) {
+ super(e);
+ this.wrapped = e;
+ }
+
+ IOException getWrapped() {
+ return wrapped;
+ }
+ }
public CredentialHelperCredentials(
CredentialHelperProvider credentialHelperProvider,
CredentialHelperEnvironment credentialHelperEnvironment,
- Optional<Credentials> fallbackCredentials,
- Duration cacheTimeout) {
- Preconditions.checkNotNull(credentialHelperProvider);
- Preconditions.checkNotNull(credentialHelperEnvironment);
+ Cache<URI, ImmutableMap<String, ImmutableList<String>>> credentialCache,
+ Optional<Credentials> fallbackCredentials) {
+ this.credentialHelperProvider = Preconditions.checkNotNull(credentialHelperProvider);
+ this.credentialHelperEnvironment = Preconditions.checkNotNull(credentialHelperEnvironment);
+ this.credentialCache = Preconditions.checkNotNull(credentialCache);
this.fallbackCredentials = Preconditions.checkNotNull(fallbackCredentials);
- Preconditions.checkNotNull(cacheTimeout);
- Preconditions.checkArgument(
- !cacheTimeout.isNegative() && !cacheTimeout.isZero(),
- "Cache timeout must be greater than 0");
-
- credentialCache =
- Caffeine.newBuilder()
- .expireAfterWrite(cacheTimeout)
- .build(
- new CredentialHelperCacheLoader(
- credentialHelperProvider, credentialHelperEnvironment));
}
@Override
@@ -68,12 +71,18 @@
}
@Override
+ @SuppressWarnings("unchecked") // Map<String, ImmutableList<String>> to Map<String<List<String>>
public Map<String, List<String>> getRequestMetadata(URI uri) throws IOException {
Preconditions.checkNotNull(uri);
- Optional<Map<String, List<String>>> credentials = getRequestMetadataFromCredentialHelper(uri);
- if (credentials.isPresent()) {
- return credentials.get();
+ ImmutableMap<String, ImmutableList<String>> credentials;
+ try {
+ credentials = credentialCache.get(uri, this::getCredentialsFromHelper);
+ } catch (WrappedIOException e) {
+ throw e.getWrapped();
+ }
+ if (credentials != null) {
+ return (Map) credentials;
}
if (fallbackCredentials.isPresent()) {
@@ -83,13 +92,28 @@
return ImmutableMap.of();
}
- @SuppressWarnings("unchecked") // Map<String, ImmutableList<String>> to Map<String<List<String>>
- private Optional<Map<String, List<String>>> getRequestMetadataFromCredentialHelper(URI uri) {
+ @Nullable
+ private ImmutableMap<String, ImmutableList<String>> getCredentialsFromHelper(URI uri) {
Preconditions.checkNotNull(uri);
- GetCredentialsResponse response = credentialCache.get(uri);
+ Optional<CredentialHelper> maybeCredentialHelper =
+ credentialHelperProvider.findCredentialHelper(uri);
+ if (maybeCredentialHelper.isEmpty()) {
+ return null;
+ }
+ CredentialHelper credentialHelper = maybeCredentialHelper.get();
- return Optional.ofNullable(response).map(value -> (Map) value.getHeaders());
+ GetCredentialsResponse response;
+ try {
+ response = credentialHelper.getCredentials(credentialHelperEnvironment, uri);
+ } catch (IOException e) {
+ throw new WrappedIOException(e);
+ }
+ if (response == null) {
+ return null;
+ }
+
+ return response.getHeaders();
}
@Override
@@ -110,32 +134,4 @@
credentialCache.invalidateAll();
}
-
- private static final class CredentialHelperCacheLoader
- implements CacheLoader<URI, GetCredentialsResponse> {
- private final CredentialHelperProvider credentialHelperProvider;
- private final CredentialHelperEnvironment credentialHelperEnvironment;
-
- public CredentialHelperCacheLoader(
- CredentialHelperProvider credentialHelperProvider,
- CredentialHelperEnvironment credentialHelperEnvironment) {
- this.credentialHelperProvider = Preconditions.checkNotNull(credentialHelperProvider);
- this.credentialHelperEnvironment = Preconditions.checkNotNull(credentialHelperEnvironment);
- }
-
- @Nullable
- @Override
- public GetCredentialsResponse load(URI uri) throws IOException, InterruptedException {
- Preconditions.checkNotNull(uri);
-
- Optional<CredentialHelper> maybeCredentialHelper =
- credentialHelperProvider.findCredentialHelper(uri);
- if (maybeCredentialHelper.isEmpty()) {
- return null;
- }
- CredentialHelper credentialHelper = maybeCredentialHelper.get();
-
- return credentialHelper.getCredentials(credentialHelperEnvironment, uri);
- }
- }
}
diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialModule.java b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialModule.java
new file mode 100644
index 0000000..95af2af
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialModule.java
@@ -0,0 +1,52 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.authandtls.credentialhelper;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.CommandEnvironment;
+import java.net.URI;
+import java.time.Duration;
+
+/** A module whose sole purpose is to hold the credential cache which is shared by other modules. */
+public class CredentialModule extends BlazeModule {
+ private final Cache<URI, ImmutableMap<String, ImmutableList<String>>> credentialCache =
+ Caffeine.newBuilder().expireAfterWrite(Duration.ZERO).build();
+
+ /** Returns the credential cache. */
+ public Cache<URI, ImmutableMap<String, ImmutableList<String>>> getCredentialCache() {
+ return credentialCache;
+ }
+
+ @Override
+ public void beforeCommand(CommandEnvironment env) {
+ // Update the cache expiration policy according to the command options.
+ AuthAndTLSOptions authAndTlsOptions = env.getOptions().getOptions(AuthAndTLSOptions.class);
+ credentialCache
+ .policy()
+ .expireAfterWrite()
+ .get()
+ .setExpiresAfter(authAndTlsOptions.credentialHelperCacheTimeout);
+
+ // Clear the cache on clean.
+ if (env.getCommand().name().equals("clean")) {
+ credentialCache.invalidateAll();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/BUILD
index 499a038..550425d 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BUILD
@@ -136,6 +136,7 @@
":spawn_log_module",
"//src/main/java/com/google/devtools/build/lib:runtime",
"//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info",
+ "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module",
"//src/main/java/com/google/devtools/build/lib/bazel/coverage",
"//src/main/java/com/google/devtools/build/lib/bazel/debug:workspace-rule-module",
"//src/main/java/com/google/devtools/build/lib/bazel/repository",
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java
index fb10edd..e35b11a 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java
@@ -16,6 +16,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule;
import com.google.devtools.build.lib.runtime.BlazeModule;
import com.google.devtools.build.lib.runtime.BlazeRuntime;
import java.io.IOException;
@@ -42,6 +43,8 @@
// This module needs to be registered before any module providing a SpawnCache
// implementation.
com.google.devtools.build.lib.runtime.NoSpawnCacheModule.class,
+ // This module needs to be registered before any module that uses the credential cache.
+ CredentialModule.class,
com.google.devtools.build.lib.runtime.CommandLogModule.class,
com.google.devtools.build.lib.runtime.MemoryPressureModule.class,
com.google.devtools.build.lib.platform.SleepPreventionModule.class,
diff --git a/src/main/java/com/google/devtools/build/lib/buildeventservice/BUILD b/src/main/java/com/google/devtools/build/lib/buildeventservice/BUILD
index bde58e5..6c61f67 100644
--- a/src/main/java/com/google/devtools/build/lib/buildeventservice/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/buildeventservice/BUILD
@@ -38,9 +38,11 @@
":buildeventservice-options",
"//src/main/java/com/google/devtools/build/lib:build-request-options",
"//src/main/java/com/google/devtools/build/lib:runtime",
+ "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories",
"//src/main/java/com/google/devtools/build/lib/analysis:test/test_configuration",
"//src/main/java/com/google/devtools/build/lib/authandtls",
"//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper",
+ "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module",
"//src/main/java/com/google/devtools/build/lib/bugreport",
"//src/main/java/com/google/devtools/build/lib/buildeventservice/client",
"//src/main/java/com/google/devtools/build/lib/buildeventstream",
diff --git a/src/main/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModule.java b/src/main/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModule.java
index ac876b0..3889ce0 100644
--- a/src/main/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModule.java
+++ b/src/main/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModule.java
@@ -22,12 +22,16 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule;
import com.google.devtools.build.lib.buildeventservice.client.BuildEventServiceClient;
import com.google.devtools.build.lib.buildeventservice.client.BuildEventServiceGrpcClient;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
import com.google.devtools.build.lib.runtime.CommandEnvironment;
+import com.google.devtools.build.lib.runtime.WorkspaceBuilder;
import io.grpc.ClientInterceptor;
import io.grpc.ManagedChannel;
import io.grpc.Metadata;
@@ -68,6 +72,15 @@
private BuildEventServiceClient client;
private BackendConfig config;
+ private CredentialModule credentialModule;
+
+ @Override
+ public void workspaceInit(
+ BlazeRuntime runtime, BlazeDirectories directories, WorkspaceBuilder builder) {
+ Preconditions.checkState(credentialModule == null, "credentialModule must be null");
+ credentialModule = Preconditions.checkNotNull(runtime.getBlazeModule(CredentialModule.class));
+ }
+
@Override
protected Class<BuildEventServiceOptions> optionsClass() {
return BuildEventServiceOptions.class;
@@ -93,6 +106,7 @@
.setClientEnvironment(env.getClientEnv())
.setHelperExecutionTimeout(authAndTLSOptions.credentialHelperTimeout)
.build(),
+ credentialModule.getCredentialCache(),
env.getCommandLinePathFactory(),
env.getRuntime().getFileSystem(),
newConfig.authAndTLSOptions());
diff --git a/src/main/java/com/google/devtools/build/lib/remote/BUILD b/src/main/java/com/google/devtools/build/lib/remote/BUILD
index 6a32924..6a844d3 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/remote/BUILD
@@ -63,6 +63,7 @@
"//src/main/java/com/google/devtools/build/lib/analysis/platform:platform_utils",
"//src/main/java/com/google/devtools/build/lib/authandtls",
"//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper",
+ "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module",
"//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
"//src/main/java/com/google/devtools/build/lib/buildeventstream",
"//src/main/java/com/google/devtools/build/lib/clock",
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
index 3e6c65d..12058e0 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
@@ -21,6 +21,7 @@
import build.bazel.remote.execution.v2.DigestFunction;
import build.bazel.remote.execution.v2.ServerCapabilities;
import build.bazel.remote.execution.v2.SymlinkAbsolutePathStrategy;
+import com.github.benmanes.caffeine.cache.Cache;
import com.google.auth.Credentials;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Ascii;
@@ -28,6 +29,7 @@
import com.google.common.base.Strings;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.ListeningScheduledExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
@@ -42,6 +44,7 @@
import com.google.devtools.build.lib.authandtls.CallCredentialsProvider;
import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule;
import com.google.devtools.build.lib.bazel.repository.downloader.Downloader;
import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader;
@@ -159,6 +162,8 @@
private final MutableSupplier<Downloader> remoteDownloaderSupplier = new MutableSupplier<>();
+ private CredentialModule credentialModule;
+
@Override
public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builder) {
builder.addBuildEventArtifactUploaderFactory(
@@ -211,33 +216,16 @@
private void initHttpAndDiskCache(
CommandEnvironment env,
+ Credentials credentials,
AuthAndTLSOptions authAndTlsOptions,
RemoteOptions remoteOptions,
DigestUtil digestUtil) {
- Credentials creds;
- try {
- creds =
- newCredentials(
- CredentialHelperEnvironment.newBuilder()
- .setEventReporter(env.getReporter())
- .setWorkspacePath(env.getWorkspace())
- .setClientEnvironment(env.getClientEnv())
- .setHelperExecutionTimeout(authAndTlsOptions.credentialHelperTimeout)
- .build(),
- env.getCommandLinePathFactory(),
- env.getRuntime().getFileSystem(),
- authAndTlsOptions,
- remoteOptions);
- } catch (IOException e) {
- handleInitFailure(env, e, Code.CREDENTIALS_INIT_FAILURE);
- return;
- }
RemoteCacheClient cacheClient;
try {
cacheClient =
RemoteCacheClientFactory.create(
remoteOptions,
- creds,
+ credentials,
authAndTlsOptions,
Preconditions.checkNotNull(env.getWorkingDirectory(), "workingDirectory"),
digestUtil);
@@ -256,8 +244,10 @@
public void workspaceInit(
BlazeRuntime runtime, BlazeDirectories directories, WorkspaceBuilder builder) {
Preconditions.checkState(blockWaitingModule == null, "blockWaitingModule must be null");
+ Preconditions.checkState(credentialModule == null, "credentialModule must be null");
blockWaitingModule =
Preconditions.checkNotNull(runtime.getBlazeModule(BlockWaitingModule.class));
+ credentialModule = Preconditions.checkNotNull(runtime.getBlazeModule(CredentialModule.class));
}
@Override
@@ -358,8 +348,28 @@
executorService = Executors.newCachedThreadPool(threadFactory);
}
+ Credentials credentials;
+ try {
+ credentials =
+ createCredentials(
+ CredentialHelperEnvironment.newBuilder()
+ .setEventReporter(env.getReporter())
+ .setWorkspacePath(env.getWorkspace())
+ .setClientEnvironment(env.getClientEnv())
+ .setHelperExecutionTimeout(authAndTlsOptions.credentialHelperTimeout)
+ .build(),
+ credentialModule.getCredentialCache(),
+ env.getCommandLinePathFactory(),
+ env.getRuntime().getFileSystem(),
+ authAndTlsOptions,
+ remoteOptions);
+ } catch (IOException e) {
+ handleInitFailure(env, e, Code.CREDENTIALS_INIT_FAILURE);
+ return;
+ }
+
if ((enableHttpCache || enableDiskCache) && !enableGrpcCache) {
- initHttpAndDiskCache(env, authAndTlsOptions, remoteOptions, digestUtil);
+ initHttpAndDiskCache(env, credentials, authAndTlsOptions, remoteOptions, digestUtil);
return;
}
@@ -456,27 +466,9 @@
}
}
- CallCredentialsProvider callCredentialsProvider;
- try {
- callCredentialsProvider =
- GoogleAuthUtils.newCallCredentialsProvider(
- newCredentials(
- CredentialHelperEnvironment.newBuilder()
- .setEventReporter(env.getReporter())
- .setWorkspacePath(env.getWorkspace())
- .setClientEnvironment(env.getClientEnv())
- .setHelperExecutionTimeout(authAndTlsOptions.credentialHelperTimeout)
- .build(),
- env.getCommandLinePathFactory(),
- env.getRuntime().getFileSystem(),
- authAndTlsOptions,
- remoteOptions));
- } catch (IOException e) {
- handleInitFailure(env, e, Code.CREDENTIALS_INIT_FAILURE);
- return;
- }
-
- CallCredentials credentials = callCredentialsProvider.getCallCredentials();
+ CallCredentialsProvider callCredentialsProvider =
+ GoogleAuthUtils.newCallCredentialsProvider(credentials);
+ CallCredentials callCredentials = callCredentialsProvider.getCallCredentials();
RemoteRetrier retrier =
new RemoteRetrier(
@@ -500,7 +492,7 @@
getAndVerifyServerCapabilities(
remoteOptions,
execChannel,
- credentials,
+ callCredentials,
retrier,
env,
digestUtil,
@@ -509,7 +501,7 @@
getAndVerifyServerCapabilities(
remoteOptions,
cacheChannel,
- credentials,
+ callCredentials,
retrier,
env,
digestUtil,
@@ -519,7 +511,7 @@
getAndVerifyServerCapabilities(
remoteOptions,
execChannel,
- credentials,
+ callCredentials,
retrier,
env,
digestUtil,
@@ -530,7 +522,7 @@
getAndVerifyServerCapabilities(
remoteOptions,
cacheChannel,
- credentials,
+ callCredentials,
retrier,
env,
digestUtil,
@@ -683,7 +675,7 @@
buildRequestId,
invocationId,
downloaderChannel.retain(),
- Optional.ofNullable(credentials),
+ Optional.ofNullable(callCredentials),
retrier,
cacheClient,
remoteOptions,
@@ -1056,8 +1048,10 @@
return actionContextProvider;
}
- static Credentials newCredentials(
+ @VisibleForTesting
+ static Credentials createCredentials(
CredentialHelperEnvironment credentialHelperEnvironment,
+ Cache<URI, ImmutableMap<String, ImmutableList<String>>> credentialCache,
CommandLinePathFactory commandLinePathFactory,
FileSystem fileSystem,
AuthAndTLSOptions authAndTlsOptions,
@@ -1065,7 +1059,11 @@
throws IOException {
Credentials credentials =
GoogleAuthUtils.newCredentials(
- credentialHelperEnvironment, commandLinePathFactory, fileSystem, authAndTlsOptions);
+ credentialHelperEnvironment,
+ credentialCache,
+ commandLinePathFactory,
+ fileSystem,
+ authAndTlsOptions);
try {
if (credentials != null
diff --git a/src/test/java/com/google/devtools/build/lib/buildeventservice/BUILD b/src/test/java/com/google/devtools/build/lib/buildeventservice/BUILD
index 793238e..4aa7f32 100644
--- a/src/test/java/com/google/devtools/build/lib/buildeventservice/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/buildeventservice/BUILD
@@ -48,6 +48,7 @@
"//src/main/java/com/google/devtools/build/lib:runtime",
"//src/main/java/com/google/devtools/build/lib/actions:action_lookup_data",
"//src/main/java/com/google/devtools/build/lib/authandtls",
+ "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module",
"//src/main/java/com/google/devtools/build/lib/bugreport",
"//src/main/java/com/google/devtools/build/lib/buildeventservice",
"//src/main/java/com/google/devtools/build/lib/buildeventservice:buildeventservice-options",
diff --git a/src/test/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModuleTest.java b/src/test/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModuleTest.java
index 50b5433..4124c43 100644
--- a/src/test/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModuleTest.java
+++ b/src/test/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModuleTest.java
@@ -32,6 +32,7 @@
import com.google.devtools.build.lib.actions.ActionLookupData;
import com.google.devtools.build.lib.analysis.util.AnalysisMock;
import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule;
import com.google.devtools.build.lib.bugreport.BugReport;
import com.google.devtools.build.lib.bugreport.Crash;
import com.google.devtools.build.lib.bugreport.CrashContext;
@@ -145,6 +146,7 @@
}
})
.addBlazeModule(new NoSpawnCacheModule())
+ .addBlazeModule(new CredentialModule())
.addBlazeModule(
new BazelBuildEventServiceModule() {
@Override
diff --git a/src/test/java/com/google/devtools/build/lib/buildtool/BUILD b/src/test/java/com/google/devtools/build/lib/buildtool/BUILD
index 0733ee9..c604a80 100644
--- a/src/test/java/com/google/devtools/build/lib/buildtool/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/buildtool/BUILD
@@ -530,6 +530,7 @@
"//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
"//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster",
"//src/main/java/com/google/devtools/build/lib/analysis:configured_target",
+ "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module",
"//src/main/java/com/google/devtools/build/lib/buildeventservice",
"//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto",
"//src/main/java/com/google/devtools/build/lib/cmdline",
diff --git a/src/test/java/com/google/devtools/build/lib/buildtool/TargetCompleteEventTest.java b/src/test/java/com/google/devtools/build/lib/buildtool/TargetCompleteEventTest.java
index ea81037..9ffd40e 100644
--- a/src/test/java/com/google/devtools/build/lib/buildtool/TargetCompleteEventTest.java
+++ b/src/test/java/com/google/devtools/build/lib/buildtool/TargetCompleteEventTest.java
@@ -28,6 +28,7 @@
import com.google.devtools.build.lib.analysis.TargetCompleteEvent;
import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget;
import com.google.devtools.build.lib.analysis.util.AnalysisMock;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule;
import com.google.devtools.build.lib.buildeventservice.BazelBuildEventServiceModule;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildEvent;
@@ -71,6 +72,7 @@
protected BlazeRuntime.Builder getRuntimeBuilder() throws Exception {
return super.getRuntimeBuilder()
.addBlazeModule(new NoSpawnCacheModule())
+ .addBlazeModule(new CredentialModule())
.addBlazeModule(new BazelBuildEventServiceModule());
}
diff --git a/src/test/java/com/google/devtools/build/lib/remote/BUILD b/src/test/java/com/google/devtools/build/lib/remote/BUILD
index c043555..7cef85b 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/remote/BUILD
@@ -59,6 +59,7 @@
"//src/main/java/com/google/devtools/build/lib/analysis:server_directories",
"//src/main/java/com/google/devtools/build/lib/authandtls",
"//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper",
+ "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module",
"//src/main/java/com/google/devtools/build/lib/buildeventstream",
"//src/main/java/com/google/devtools/build/lib/clock",
"//src/main/java/com/google/devtools/build/lib/collect/nestedset",
@@ -101,6 +102,7 @@
"//src/test/java/com/google/devtools/build/lib/testutil:TestUtils",
"//third_party:api_client",
"//third_party:auth",
+ "//third_party:caffeine",
"//third_party:guava",
"//third_party:junit4",
"//third_party:mockito",
@@ -154,6 +156,7 @@
"//src/main/java/com/google/devtools/build/lib:runtime",
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/actions:artifacts",
+ "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module",
"//src/main/java/com/google/devtools/build/lib/dynamic",
"//src/main/java/com/google/devtools/build/lib/remote",
"//src/main/java/com/google/devtools/build/lib/standalone",
@@ -170,6 +173,7 @@
srcs = ["DiskCacheIntegrationTest.java"],
deps = [
"//src/main/java/com/google/devtools/build/lib:runtime",
+ "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module",
"//src/main/java/com/google/devtools/build/lib/remote",
"//src/main/java/com/google/devtools/build/lib/standalone",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
diff --git a/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTest.java b/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTest.java
index 70e138c..8c49c36 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTest.java
@@ -21,6 +21,7 @@
import com.google.common.collect.ImmutableList;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.BuildFailedException;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule;
import com.google.devtools.build.lib.dynamic.DynamicExecutionModule;
import com.google.devtools.build.lib.remote.util.IntegrationTestUtils.WorkerInstance;
import com.google.devtools.build.lib.runtime.BlazeModule;
@@ -77,6 +78,7 @@
return ImmutableList.<BlazeModule>builder()
.addAll(super.getSpawnModules())
.add(new StandaloneModule())
+ .add(new CredentialModule())
.add(new RemoteModule())
.add(new DynamicExecutionModule())
.build();
diff --git a/src/test/java/com/google/devtools/build/lib/remote/DiskCacheIntegrationTest.java b/src/test/java/com/google/devtools/build/lib/remote/DiskCacheIntegrationTest.java
index 3cf3392..ee64e17 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/DiskCacheIntegrationTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/DiskCacheIntegrationTest.java
@@ -16,6 +16,7 @@
import static com.google.devtools.build.lib.testutil.TestUtils.tmpDirFile;
import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule;
import com.google.devtools.build.lib.buildtool.util.BuildIntegrationTestCase;
import com.google.devtools.build.lib.runtime.BlazeModule;
import com.google.devtools.build.lib.runtime.BlazeRuntime;
@@ -60,6 +61,7 @@
@Override
protected BlazeRuntime.Builder getRuntimeBuilder() throws Exception {
return super.getRuntimeBuilder()
+ .addBlazeModule(new CredentialModule())
.addBlazeModule(new RemoteModule())
.addBlazeModule(new BuildSummaryStatsModule())
.addBlazeModule(new BlockWaitingModule());
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteModuleTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteModuleTest.java
index 2deeff8..a566733 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/RemoteModuleTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteModuleTest.java
@@ -26,6 +26,8 @@
import build.bazel.remote.execution.v2.GetCapabilitiesRequest;
import build.bazel.remote.execution.v2.ServerCapabilities;
import build.bazel.remote.execution.v2.SymlinkAbsolutePathStrategy;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.auth.Credentials;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -35,6 +37,7 @@
import com.google.devtools.build.lib.analysis.config.CoreOptions;
import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.exec.BinTools;
import com.google.devtools.build.lib.exec.ExecutionOptions;
@@ -103,7 +106,8 @@
CacheCapabilities.newBuilder().addDigestFunctions(Value.SHA256).build())
.build();
- private static CommandEnvironment createTestCommandEnvironment(RemoteOptions remoteOptions)
+ private static CommandEnvironment createTestCommandEnvironment(
+ RemoteModule remoteModule, RemoteOptions remoteOptions)
throws IOException, AbruptExitException {
CoreOptions coreOptions = Options.getDefaults(CoreOptions.class);
CommonCommandOptions commonCommandOptions = Options.getDefaults(CommonCommandOptions.class);
@@ -134,6 +138,8 @@
.setServerDirectories(serverDirectories)
.setStartupOptionsProvider(
OptionsParser.builder().optionsClasses(BlazeServerStartupOptions.class).build())
+ .addBlazeModule(new CredentialModule())
+ .addBlazeModule(remoteModule)
.addBlazeModule(new BlockWaitingModule())
.build();
@@ -198,7 +204,7 @@
RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
remoteOptions.remoteExecutor = executionServerName;
- CommandEnvironment env = createTestCommandEnvironment(remoteOptions);
+ CommandEnvironment env = createTestCommandEnvironment(remoteModule, remoteOptions);
remoteModule.beforeCommand(env);
@@ -226,7 +232,7 @@
RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
remoteOptions.remoteCache = cacheServerName;
- CommandEnvironment env = createTestCommandEnvironment(remoteOptions);
+ CommandEnvironment env = createTestCommandEnvironment(remoteModule, remoteOptions);
remoteModule.beforeCommand(env);
@@ -272,7 +278,7 @@
remoteOptions.remoteExecutor = executionServerName;
remoteOptions.remoteCache = cacheServerName;
- CommandEnvironment env = createTestCommandEnvironment(remoteOptions);
+ CommandEnvironment env = createTestCommandEnvironment(remoteModule, remoteOptions);
remoteModule.beforeCommand(env);
@@ -327,7 +333,7 @@
remoteOptions.remoteExecutor = executionServerName;
remoteOptions.remoteCache = cacheServerName;
- CommandEnvironment env = createTestCommandEnvironment(remoteOptions);
+ CommandEnvironment env = createTestCommandEnvironment(remoteModule, remoteOptions);
remoteModule.beforeCommand(env);
@@ -365,7 +371,7 @@
(target, proxy, options, interceptors) ->
InProcessChannelBuilder.forName(target).directExecutor().build());
- CommandEnvironment env = createTestCommandEnvironment(remoteOptions);
+ CommandEnvironment env = createTestCommandEnvironment(remoteModule, remoteOptions);
assertThrows(AbruptExitException.class, () -> remoteModule.beforeCommand(env));
} finally {
@@ -398,7 +404,7 @@
(target, proxy, options, interceptors) ->
InProcessChannelBuilder.forName(target).directExecutor().build());
- CommandEnvironment env = createTestCommandEnvironment(remoteOptions);
+ CommandEnvironment env = createTestCommandEnvironment(remoteModule, remoteOptions);
assertThrows(AbruptExitException.class, () -> remoteModule.beforeCommand(env));
} finally {
@@ -430,7 +436,7 @@
(target, proxy, options, interceptors) ->
InProcessChannelBuilder.forName(target).directExecutor().build());
- CommandEnvironment env = createTestCommandEnvironment(remoteOptions);
+ CommandEnvironment env = createTestCommandEnvironment(remoteModule, remoteOptions);
remoteModule.beforeCommand(env);
@@ -468,7 +474,7 @@
(target, proxy, options, interceptors) ->
InProcessChannelBuilder.forName(target).directExecutor().build());
- CommandEnvironment env = createTestCommandEnvironment(remoteOptions);
+ CommandEnvironment env = createTestCommandEnvironment(remoteModule, remoteOptions);
remoteModule.beforeCommand(env);
@@ -492,14 +498,18 @@
AuthAndTLSOptions authAndTLSOptions = Options.getDefaults(AuthAndTLSOptions.class);
RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
+ Cache<URI, ImmutableMap<String, ImmutableList<String>>> credentialCache =
+ Caffeine.newBuilder().build();
+
Credentials credentials =
- RemoteModule.newCredentials(
+ RemoteModule.createCredentials(
CredentialHelperEnvironment.newBuilder()
.setEventReporter(new Reporter(new EventBus()))
.setWorkspacePath(fileSystem.getPath("/workspace"))
.setClientEnvironment(ImmutableMap.of("NETRC", netrc))
.setHelperExecutionTimeout(Duration.ZERO)
.build(),
+ credentialCache,
new CommandLinePathFactory(fileSystem, ImmutableMap.of()),
fileSystem,
authAndTLSOptions,
@@ -526,7 +536,7 @@
RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
remoteOptions.remoteCache = cacheServerName;
- CommandEnvironment env = createTestCommandEnvironment(remoteOptions);
+ CommandEnvironment env = createTestCommandEnvironment(remoteModule, remoteOptions);
remoteModule.beforeCommand(env);
@@ -558,7 +568,7 @@
RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
remoteOptions.remoteExecutor = executionServerName;
- CommandEnvironment env = createTestCommandEnvironment(remoteOptions);
+ CommandEnvironment env = createTestCommandEnvironment(remoteModule, remoteOptions);
remoteModule.beforeCommand(env);
diff --git a/src/test/shell/bazel/remote/remote_execution_test.sh b/src/test/shell/bazel/remote/remote_execution_test.sh
index b9c0cbf..6d6de17 100755
--- a/src/test/shell/bazel/remote/remote_execution_test.sh
+++ b/src/test/shell/bazel/remote/remote_execution_test.sh
@@ -76,56 +76,76 @@
[[ "${charmap}" == "UTF-8" ]]
}
-function setup_credential_helper() {
+function setup_credential_helper_test() {
+ # Each helper call atomically writes one byte to this file.
+ # We can later read the file to determine how many calls were made.
+ cat > "${TEST_TMPDIR}/credhelper_log"
+
cat > "${TEST_TMPDIR}/credhelper" <<'EOF'
#!/usr/bin/env python3
+import os
+
+path = os.path.join(os.environ["TEST_TMPDIR"], "credhelper_log")
+fd = os.open(path, os.O_WRONLY|os.O_CREAT|os.O_APPEND)
+os.write(fd, b"1")
+os.close(fd)
+
print("""{"headers":{"Authorization":["Bearer secret_token"]}}""")
EOF
chmod +x "${TEST_TMPDIR}/credhelper"
+
+ mkdir -p a
+
+ cat > a/BUILD <<'EOF'
+[genrule(
+ name = x,
+ outs = [x + ".txt"],
+ cmd = "touch $(OUTS)",
+) for x in ["a", "b"]]
+EOF
+
+ stop_worker
+ start_worker --expected_authorization_token=secret_token
+}
+
+function expect_credential_helper_calls() {
+ local -r expected=$1
+ local -r actual=$(wc -c "${TEST_TMPDIR}/credhelper_log" | awk '{print $1}')
+ if [[ "$expected" != "$actual" ]]; then
+ fail "expected $expected instead of $actual credential helper calls"
+ fi
}
function test_credential_helper_remote_cache() {
- setup_credential_helper
-
- mkdir -p a
-
- cat > a/BUILD <<'EOF'
-genrule(
- name = "a",
- outs = ["a.txt"],
- cmd = "touch $(OUTS)",
-)
-EOF
-
- stop_worker
- start_worker --expected_authorization_token=secret_token
+ setup_credential_helper_test
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
//a:a >& $TEST_log && fail "Build without credentials should have failed"
expect_log "Failed to query remote execution capabilities"
+ # Helper shouldn't have been called yet.
+ expect_credential_helper_calls 0
+
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
--experimental_credential_helper="${TEST_TMPDIR}/credhelper" \
//a:a >& $TEST_log || fail "Build with credentials should have succeeded"
+
+ # First build should have called helper for 4 distinct URIs.
+ expect_credential_helper_calls 4
+
+ bazel build \
+ --remote_cache=grpc://localhost:${worker_port} \
+ --experimental_credential_helper="${TEST_TMPDIR}/credhelper" \
+ //a:b >& $TEST_log || fail "Build with credentials should have succeeded"
+
+ # Second build should have hit the credentials cache.
+ expect_credential_helper_calls 4
}
function test_credential_helper_remote_execution() {
- setup_credential_helper
-
- mkdir -p a
-
- cat > a/BUILD <<'EOF'
-genrule(
- name = "a",
- outs = ["a.txt"],
- cmd = "touch $(OUTS)",
-)
-EOF
-
- stop_worker
- start_worker --expected_authorization_token=secret_token
+ setup_credential_helper_test
bazel build \
--spawn_strategy=remote \
@@ -133,11 +153,49 @@
//a:a >& $TEST_log && fail "Build without credentials should have failed"
expect_log "Failed to query remote execution capabilities"
+ # Helper shouldn't have been called yet.
+ expect_credential_helper_calls 0
+
bazel build \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--experimental_credential_helper="${TEST_TMPDIR}/credhelper" \
//a:a >& $TEST_log || fail "Build with credentials should have succeeded"
+
+ # First build should have called helper for 5 distinct URIs.
+ expect_credential_helper_calls 5
+
+ bazel build \
+ --spawn_strategy=remote \
+ --remote_executor=grpc://localhost:${worker_port} \
+ --experimental_credential_helper="${TEST_TMPDIR}/credhelper" \
+ //a:b >& $TEST_log || fail "Build with credentials should have succeeded"
+
+ # Second build should have hit the credentials cache.
+ expect_credential_helper_calls 5
+}
+
+function test_credential_helper_clear_cache() {
+ setup_credential_helper_test
+
+ bazel build \
+ --spawn_strategy=remote \
+ --remote_executor=grpc://localhost:${worker_port} \
+ --experimental_credential_helper="${TEST_TMPDIR}/credhelper" \
+ //a:a >& $TEST_log || fail "Build with credentials should have succeeded"
+
+ expect_credential_helper_calls 5
+
+ bazel clean
+
+ bazel build \
+ --spawn_strategy=remote \
+ --remote_executor=grpc://localhost:${worker_port} \
+ --experimental_credential_helper="${TEST_TMPDIR}/credhelper" \
+ //a:b >& $TEST_log || fail "Build with credentials should have succeeded"
+
+ # Build after clean should have called helper again.
+ expect_credential_helper_calls 10
}
function test_remote_grpc_cache_with_protocol() {
