Remote: Support client TLS authentication for remote caches over HTTP

Fixes #14397.

This change augments the `SslContext` created for TLS HTTP remote caches to include the client certificate, client key, and optional root CA as specified on the command line with `--tls_client_certificate`, `--tls_client_key`, and `--tls_certificate`, respectively.

The implementation proposes propagating `AuthAndTLSOptions` through `RemoteCacheClientFactory` and `HttpCacheClient` in order to expose the relevant file paths to the logic that creates the SSL context for use by the HTTP remote cache client.

Closes #14398.

PiperOrigin-RevId: 416354592
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactory.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactory.java
index 57741a8..e5622a4 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactory.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactory.java
@@ -18,6 +18,7 @@
 import com.google.common.base.Ascii;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
 import com.google.devtools.build.lib.remote.common.RemoteCacheClient;
 import com.google.devtools.build.lib.remote.disk.DiskAndRemoteCacheClient;
 import com.google.devtools.build.lib.remote.disk.DiskCacheClient;
@@ -55,16 +56,17 @@
   public static RemoteCacheClient create(
       RemoteOptions options,
       @Nullable Credentials creds,
+      AuthAndTLSOptions authAndTlsOptions,
       Path workingDirectory,
       DigestUtil digestUtil)
       throws IOException {
     Preconditions.checkNotNull(workingDirectory, "workingDirectory");
     if (isHttpCache(options) && isDiskCache(options)) {
       return createDiskAndHttpCache(
-          workingDirectory, options.diskCache, options, creds, digestUtil);
+          workingDirectory, options.diskCache, options, creds, authAndTlsOptions, digestUtil);
     }
     if (isHttpCache(options)) {
-      return createHttp(options, creds, digestUtil);
+      return createHttp(options, creds, authAndTlsOptions, digestUtil);
     }
     if (isDiskCache(options)) {
       return createDiskCache(
@@ -80,7 +82,10 @@
   }
 
   private static RemoteCacheClient createHttp(
-      RemoteOptions options, Credentials creds, DigestUtil digestUtil) {
+      RemoteOptions options,
+      Credentials creds,
+      AuthAndTLSOptions authAndTlsOptions,
+      DigestUtil digestUtil) {
     Preconditions.checkNotNull(options.remoteCache, "remoteCache");
 
     try {
@@ -99,7 +104,8 @@
               options.remoteVerifyDownloads,
               ImmutableList.copyOf(options.remoteHeaders),
               digestUtil,
-              creds);
+              creds,
+              authAndTlsOptions);
         } else {
           throw new Exception("Remote cache proxy unsupported: " + options.remoteProxy);
         }
@@ -111,7 +117,8 @@
             options.remoteVerifyDownloads,
             ImmutableList.copyOf(options.remoteHeaders),
             digestUtil,
-            creds);
+            creds,
+            authAndTlsOptions);
       }
     } catch (Exception e) {
       throw new RuntimeException(e);
@@ -137,6 +144,7 @@
       PathFragment diskCachePath,
       RemoteOptions options,
       Credentials cred,
+      AuthAndTLSOptions authAndTlsOptions,
       DigestUtil digestUtil)
       throws IOException {
     Path cacheDir =
@@ -145,7 +153,7 @@
       cacheDir.createDirectoryAndParents();
     }
 
-    RemoteCacheClient httpCache = createHttp(options, cred, digestUtil);
+    RemoteCacheClient httpCache = createHttp(options, cred, authAndTlsOptions, digestUtil);
     return createDiskAndRemoteClient(
         workingDirectory,
         diskCachePath,
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
index d21a3ee..a26842d 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
@@ -231,6 +231,7 @@
           RemoteCacheClientFactory.create(
               remoteOptions,
               creds,
+              authAndTlsOptions,
               Preconditions.checkNotNull(env.getWorkingDirectory(), "workingDirectory"),
               digestUtil);
     } catch (IOException e) {
diff --git a/src/main/java/com/google/devtools/build/lib/remote/http/BUILD b/src/main/java/com/google/devtools/build/lib/remote/http/BUILD
index 9ce71c7..67e7bd0 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/http/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/remote/http/BUILD
@@ -19,6 +19,7 @@
     ],
     deps = [
         "//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info",
+        "//src/main/java/com/google/devtools/build/lib/authandtls",
         "//src/main/java/com/google/devtools/build/lib/remote/common",
         "//src/main/java/com/google/devtools/build/lib/remote/util",
         "//src/main/java/com/google/devtools/build/lib/vfs",
diff --git a/src/main/java/com/google/devtools/build/lib/remote/http/HttpCacheClient.java b/src/main/java/com/google/devtools/build/lib/remote/http/HttpCacheClient.java
index 3cbf6cf..cb9e806 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/http/HttpCacheClient.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/http/HttpCacheClient.java
@@ -24,6 +24,7 @@
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.common.util.concurrent.SettableFuture;
+import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
 import com.google.devtools.build.lib.remote.common.CacheNotFoundException;
 import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
 import com.google.devtools.build.lib.remote.common.RemoteCacheClient;
@@ -68,6 +69,7 @@
 import io.netty.handler.timeout.WriteTimeoutException;
 import io.netty.util.concurrent.Future;
 import io.netty.util.concurrent.Promise;
+import java.io.File;
 import java.io.FileInputStream;
 import java.io.FilterInputStream;
 import java.io.IOException;
@@ -151,7 +153,8 @@
       boolean verifyDownloads,
       ImmutableList<Entry<String, String>> extraHttpHeaders,
       DigestUtil digestUtil,
-      @Nullable final Credentials creds)
+      @Nullable final Credentials creds,
+      AuthAndTLSOptions authAndTlsOptions)
       throws Exception {
     return new HttpCacheClient(
         NioEventLoopGroup::new,
@@ -163,6 +166,7 @@
         extraHttpHeaders,
         digestUtil,
         creds,
+        authAndTlsOptions,
         null);
   }
 
@@ -174,7 +178,8 @@
       boolean verifyDownloads,
       ImmutableList<Entry<String, String>> extraHttpHeaders,
       DigestUtil digestUtil,
-      @Nullable final Credentials creds)
+      @Nullable final Credentials creds,
+      AuthAndTLSOptions authAndTlsOptions)
       throws Exception {
 
     if (KQueue.isAvailable()) {
@@ -188,6 +193,7 @@
           extraHttpHeaders,
           digestUtil,
           creds,
+          authAndTlsOptions,
           domainSocketAddress);
     } else if (Epoll.isAvailable()) {
       return new HttpCacheClient(
@@ -200,6 +206,7 @@
           extraHttpHeaders,
           digestUtil,
           creds,
+          authAndTlsOptions,
           domainSocketAddress);
     } else {
       throw new Exception("Unix domain sockets are unsupported on this platform");
@@ -216,6 +223,7 @@
       ImmutableList<Entry<String, String>> extraHttpHeaders,
       DigestUtil digestUtil,
       @Nullable final Credentials creds,
+      AuthAndTLSOptions authAndTlsOptions,
       @Nullable SocketAddress socketAddress)
       throws Exception {
     useTls = uri.getScheme().equals("https");
@@ -236,15 +244,7 @@
       socketAddress = new InetSocketAddress(uri.getHost(), uri.getPort());
     }
 
-    final SslContext sslCtx;
-    if (useTls) {
-      // OpenSsl gives us a > 2x speed improvement on fast networks, but requires netty tcnative
-      // to be there which is not available on all platforms and environments.
-      SslProvider sslProvider = OpenSsl.isAvailable() ? SslProvider.OPENSSL : SslProvider.JDK;
-      sslCtx = SslContextBuilder.forClient().sslProvider(sslProvider).build();
-    } else {
-      sslCtx = null;
-    }
+    final SslContext sslCtx = useTls ? createSSLContext(authAndTlsOptions) : null;
     final int port = uri.getPort();
     final String hostname = uri.getHost();
     this.eventLoop = newEventLoopGroup.apply(2);
@@ -269,6 +269,10 @@
             if (sslCtx != null) {
               SSLEngine engine = sslCtx.newEngine(ch.alloc(), hostname, port);
               engine.setUseClientMode(true);
+              if (authAndTlsOptions.tlsClientCertificate != null
+                  && authAndTlsOptions.tlsClientKey != null) {
+                engine.setNeedClientAuth(true);
+              }
               p.addFirst("ssl-handler", new SslHandler(engine));
             }
           }
@@ -773,4 +777,28 @@
       }
     }
   }
+
+  private static SslContext createSSLContext(AuthAndTLSOptions authAndTlsOptions)
+      throws IOException {
+    // OpenSsl gives us a > 2x speed improvement on fast networks, but requires netty tcnative
+    // to be there which is not available on all platforms and environments.
+    SslProvider sslProvider = OpenSsl.isAvailable() ? SslProvider.OPENSSL : SslProvider.JDK;
+    SslContextBuilder sslContextBuilder = SslContextBuilder.forClient().sslProvider(sslProvider);
+
+    // Root CA certificate
+    if (authAndTlsOptions.tlsCertificate != null) {
+      sslContextBuilder =
+          sslContextBuilder.trustManager(new File(authAndTlsOptions.tlsCertificate));
+    }
+
+    // Optional client TLS authentication
+    if (authAndTlsOptions.tlsClientCertificate != null && authAndTlsOptions.tlsClientKey != null) {
+      sslContextBuilder =
+          sslContextBuilder.keyManager(
+              new File(authAndTlsOptions.tlsClientCertificate),
+              new File(authAndTlsOptions.tlsClientKey));
+    }
+
+    return sslContextBuilder.build();
+  }
 }
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactoryTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactoryTest.java
index 66d74d7..5a03070 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactoryTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactoryTest.java
@@ -17,6 +17,7 @@
 import static com.google.common.truth.Truth.assertThat;
 import static org.junit.Assert.assertThrows;
 
+import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
 import com.google.devtools.build.lib.clock.JavaClock;
 import com.google.devtools.build.lib.remote.common.RemoteCacheClient;
 import com.google.devtools.build.lib.remote.disk.DiskAndRemoteCacheClient;
@@ -42,6 +43,7 @@
   private final DigestUtil digestUtil = new DigestUtil(DigestHashFunction.SHA256);
 
   private RemoteOptions remoteOptions;
+  private final AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class);
   private Path workingDirectory;
   private InMemoryFileSystem fs;
 
@@ -60,7 +62,7 @@
 
     RemoteCacheClient blobStore =
         RemoteCacheClientFactory.create(
-            remoteOptions, /* creds= */ null, workingDirectory, digestUtil);
+            remoteOptions, /* creds= */ null, authAndTlsOptions, workingDirectory, digestUtil);
 
     assertThat(blobStore).isInstanceOf(DiskAndRemoteCacheClient.class);
   }
@@ -73,7 +75,7 @@
 
     RemoteCacheClient blobStore =
         RemoteCacheClientFactory.create(
-            remoteOptions, /* creds= */ null, workingDirectory, digestUtil);
+            remoteOptions, /* creds= */ null, authAndTlsOptions, workingDirectory, digestUtil);
 
     assertThat(blobStore).isInstanceOf(DiskAndRemoteCacheClient.class);
     assertThat(workingDirectory.exists()).isTrue();
@@ -89,7 +91,11 @@
         NullPointerException.class,
         () ->
             RemoteCacheClientFactory.create(
-                remoteOptions, /* creds= */ null, /* workingDirectory= */ null, digestUtil));
+                remoteOptions,
+                /* creds= */ null,
+                authAndTlsOptions,
+                /* workingDirectory= */ null,
+                digestUtil));
   }
 
   @Test
@@ -99,7 +105,7 @@
 
     RemoteCacheClient blobStore =
         RemoteCacheClientFactory.create(
-            remoteOptions, /* creds= */ null, workingDirectory, digestUtil);
+            remoteOptions, /* creds= */ null, authAndTlsOptions, workingDirectory, digestUtil);
 
     assertThat(blobStore).isInstanceOf(HttpCacheClient.class);
   }
@@ -114,7 +120,11 @@
                 RuntimeException.class,
                 () ->
                     RemoteCacheClientFactory.create(
-                        remoteOptions, /* creds= */ null, workingDirectory, digestUtil)))
+                        remoteOptions,
+                        /* creds= */ null,
+                        authAndTlsOptions,
+                        workingDirectory,
+                        digestUtil)))
         .hasMessageThat()
         .contains("Remote cache proxy unsupported: bad-proxy");
   }
@@ -125,7 +135,7 @@
 
     RemoteCacheClient blobStore =
         RemoteCacheClientFactory.create(
-            remoteOptions, /* creds= */ null, workingDirectory, digestUtil);
+            remoteOptions, /* creds= */ null, authAndTlsOptions, workingDirectory, digestUtil);
 
     assertThat(blobStore).isInstanceOf(HttpCacheClient.class);
   }
@@ -136,7 +146,7 @@
 
     RemoteCacheClient blobStore =
         RemoteCacheClientFactory.create(
-            remoteOptions, /* creds= */ null, workingDirectory, digestUtil);
+            remoteOptions, /* creds= */ null, authAndTlsOptions, workingDirectory, digestUtil);
 
     assertThat(blobStore).isInstanceOf(DiskCacheClient.class);
   }
diff --git a/src/test/java/com/google/devtools/build/lib/remote/http/BUILD b/src/test/java/com/google/devtools/build/lib/remote/http/BUILD
index ff7064f..3c6680b 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/http/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/remote/http/BUILD
@@ -22,10 +22,12 @@
     ],
     test_class = "com.google.devtools.build.lib.AllTests",
     deps = [
+        "//src/main/java/com/google/devtools/build/lib/authandtls",
         "//src/main/java/com/google/devtools/build/lib/remote/common",
         "//src/main/java/com/google/devtools/build/lib/remote/http",
         "//src/main/java/com/google/devtools/build/lib/remote/util",
         "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/main/java/com/google/devtools/common/options",
         "//src/test/java/com/google/devtools/build/lib:test_runner",
         "//src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/http",
         "//third_party:auth",
diff --git a/src/test/java/com/google/devtools/build/lib/remote/http/HttpCacheClientTest.java b/src/test/java/com/google/devtools/build/lib/remote/http/HttpCacheClientTest.java
index c41921b..93cee67 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/http/HttpCacheClientTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/http/HttpCacheClientTest.java
@@ -32,11 +32,13 @@
 import com.google.common.base.Charsets;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
 import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
 import com.google.devtools.build.lib.remote.util.DigestUtil;
 import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
 import com.google.devtools.build.lib.vfs.DigestHashFunction;
 import com.google.devtools.build.remote.worker.http.HttpCacheServerHandler;
+import com.google.devtools.common.options.Options;
 import com.google.protobuf.ByteString;
 import io.netty.bootstrap.ServerBootstrap;
 import io.netty.buffer.ByteBuf;
@@ -258,7 +260,8 @@
       ServerChannel serverChannel,
       int timeoutSeconds,
       boolean remoteVerifyDownloads,
-      @Nullable final Credentials creds)
+      @Nullable final Credentials creds,
+      AuthAndTLSOptions authAndTlsOptions)
       throws Exception {
     SocketAddress socketAddress = serverChannel.localAddress();
     if (socketAddress instanceof DomainSocketAddress) {
@@ -272,7 +275,8 @@
           remoteVerifyDownloads,
           ImmutableList.of(),
           DIGEST_UTIL,
-          creds);
+          creds,
+          authAndTlsOptions);
     } else if (socketAddress instanceof InetSocketAddress) {
       InetSocketAddress inetSocketAddress = (InetSocketAddress) socketAddress;
       URI uri = new URI("http://localhost:" + inetSocketAddress.getPort());
@@ -283,7 +287,8 @@
           remoteVerifyDownloads,
           ImmutableList.of(),
           DIGEST_UTIL,
-          creds);
+          creds,
+          authAndTlsOptions);
     } else {
       throw new IllegalStateException(
           "unsupported socket address class " + socketAddress.getClass());
@@ -291,10 +296,13 @@
   }
 
   private HttpCacheClient createHttpBlobStore(
-      ServerChannel serverChannel, int timeoutSeconds, @Nullable final Credentials creds)
+      ServerChannel serverChannel,
+      int timeoutSeconds,
+      @Nullable final Credentials creds,
+      AuthAndTLSOptions authAndTlsOptions)
       throws Exception {
     return createHttpBlobStore(
-        serverChannel, timeoutSeconds, /* remoteVerifyDownloads= */ true, creds);
+        serverChannel, timeoutSeconds, /* remoteVerifyDownloads= */ true, creds, authAndTlsOptions);
   }
 
   @Before
@@ -313,7 +321,8 @@
       server = testServer.start(new HttpCacheServerHandler(cacheContents));
 
       HttpCacheClient blobStore =
-          createHttpBlobStore(server, /* timeoutSeconds= */ 1, /* credentials= */ null);
+          createHttpBlobStore(
+              server, /* timeoutSeconds= */ 1, /* creds= */ null, new AuthAndTLSOptions());
 
       ByteString data = ByteString.copyFrom("foo bar", StandardCharsets.UTF_8);
       Digest digest = DIGEST_UTIL.compute(data.toByteArray());
@@ -343,7 +352,9 @@
     testServer.stop(server);
 
     Credentials credentials = newCredentials();
-    HttpCacheClient blobStore = createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials);
+    AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class);
+    HttpCacheClient blobStore =
+        createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials, authAndTlsOptions);
     getFromFuture(
         blobStore.downloadBlob(remoteActionExecutionContext, DIGEST, new ByteArrayOutputStream()));
 
@@ -365,7 +376,9 @@
               });
 
       Credentials credentials = newCredentials();
-      HttpCacheClient blobStore = createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials);
+      AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class);
+      HttpCacheClient blobStore =
+          createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials, authAndTlsOptions);
       byte[] data = "File Contents".getBytes(Charsets.US_ASCII);
       getFromFuture(
           blobStore.uploadBlob(
@@ -391,7 +404,9 @@
               });
 
       Credentials credentials = newCredentials();
-      HttpCacheClient blobStore = createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials);
+      AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class);
+      HttpCacheClient blobStore =
+          createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials, authAndTlsOptions);
       getFromFuture(
           blobStore.downloadBlob(
               remoteActionExecutionContext, DIGEST, new ByteArrayOutputStream()));
@@ -425,7 +440,9 @@
               });
 
       Credentials credentials = newCredentials();
-      HttpCacheClient blobStore = createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials);
+      AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class);
+      HttpCacheClient blobStore =
+          createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials, authAndTlsOptions);
       ByteString data = ByteString.copyFrom("File Contents", Charsets.US_ASCII);
       IOException e =
           assertThrows(
@@ -465,9 +482,14 @@
               });
 
       Credentials credentials = newCredentials();
+      AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class);
       HttpCacheClient blobStore =
           createHttpBlobStore(
-              server, /* timeoutSeconds= */ 1, /* remoteVerifyDownloads= */ true, credentials);
+              server,
+              /* timeoutSeconds= */ 1,
+              /* remoteVerifyDownloads= */ true,
+              credentials,
+              authAndTlsOptions);
       Digest fooDigest = DIGEST_UTIL.compute("foo".getBytes(Charsets.UTF_8));
       try (OutputStream out = new ByteArrayOutputStream()) {
         IOException e =
@@ -507,9 +529,14 @@
               });
 
       Credentials credentials = newCredentials();
+      AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class);
       HttpCacheClient blobStore =
           createHttpBlobStore(
-              server, /* timeoutSeconds= */ 1, /* remoteVerifyDownloads= */ false, credentials);
+              server,
+              /* timeoutSeconds= */ 1,
+              /* remoteVerifyDownloads= */ false,
+              credentials,
+              authAndTlsOptions);
       Digest fooDigest = DIGEST_UTIL.compute("foo".getBytes(Charsets.UTF_8));
       try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
         getFromFuture(blobStore.downloadBlob(remoteActionExecutionContext, fooDigest, out));
@@ -535,7 +562,9 @@
       server = testServer.start(new NotAuthorizedHandler(errorType));
 
       Credentials credentials = newCredentials();
-      HttpCacheClient blobStore = createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials);
+      AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class);
+      HttpCacheClient blobStore =
+          createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials, authAndTlsOptions);
       ByteArrayOutputStream out = Mockito.spy(new ByteArrayOutputStream());
       getFromFuture(blobStore.downloadBlob(remoteActionExecutionContext, DIGEST, out));
       assertThat(out.toString(Charsets.US_ASCII.name())).isEqualTo("File Contents");
@@ -565,7 +594,9 @@
       server = testServer.start(new NotAuthorizedHandler(errorType));
 
       Credentials credentials = newCredentials();
-      HttpCacheClient blobStore = createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials);
+      AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class);
+      HttpCacheClient blobStore =
+          createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials, authAndTlsOptions);
       byte[] data = "File Contents".getBytes(Charsets.US_ASCII);
       blobStore
           .uploadBlob(
@@ -595,7 +626,9 @@
       server = testServer.start(new NotAuthorizedHandler(errorType));
 
       Credentials credentials = newCredentials();
-      HttpCacheClient blobStore = createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials);
+      AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class);
+      HttpCacheClient blobStore =
+          createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials, authAndTlsOptions);
       getFromFuture(
           blobStore.downloadBlob(
               remoteActionExecutionContext, DIGEST, new ByteArrayOutputStream()));
@@ -624,7 +657,9 @@
       server = testServer.start(new NotAuthorizedHandler(errorType));
 
       Credentials credentials = newCredentials();
-      HttpCacheClient blobStore = createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials);
+      AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class);
+      HttpCacheClient blobStore =
+          createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials, authAndTlsOptions);
       byte[] oneByte = new byte[] {0};
       getFromFuture(
           blobStore.uploadBlob(