Reorganize the //src/test/java/com/google/devtools/build/lib/remote package.
PiperOrigin-RevId: 568156390
Change-Id: If390f8535e3c574debac68ce96c3e451f1599d98
diff --git a/.bazelci/postsubmit.yml b/.bazelci/postsubmit.yml
index 6b13128..f961fc7 100644
--- a/.bazelci/postsubmit.yml
+++ b/.bazelci/postsubmit.yml
@@ -328,7 +328,7 @@
- "-//src/test/java/com/google/devtools/build/lib/query2/engine/..."
- "-//src/test/java/com/google/devtools/build/lib/versioning/..."
- "-//src/test/java/com/google/devtools/build/lib/worker/..."
- - "-//src/test/java/com/google/devtools/build/lib/remote:remote"
+ - "-//src/test/java/com/google/devtools/build/lib/remote:RemoteTests"
- "-//src/test/shell/bazel/remote/..."
- "-//tools/python:pywrapper_test"
include_json_profile:
diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml
index af9981d..5cbd96f 100644
--- a/.bazelci/presubmit.yml
+++ b/.bazelci/presubmit.yml
@@ -389,7 +389,7 @@
- "-//src/test/java/com/google/devtools/build/lib/query2/engine/..."
- "-//src/test/java/com/google/devtools/build/lib/versioning/..."
- "-//src/test/java/com/google/devtools/build/lib/worker/..."
- - "-//src/test/java/com/google/devtools/build/lib/remote:remote"
+ - "-//src/test/java/com/google/devtools/build/lib/remote:RemoteTests"
- "-//src/test/shell/bazel/remote/..."
- "-//tools/python:pywrapper_test"
include_json_profile:
diff --git a/src/test/java/com/google/devtools/build/lib/remote/BUILD b/src/test/java/com/google/devtools/build/lib/remote/BUILD
index e49e166..2496f99 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/remote/BUILD
@@ -1,6 +1,7 @@
-load("@rules_java//java:defs.bzl", "java_test")
+load("@rules_java//java:defs.bzl", "java_library", "java_test")
package(
+ default_applicable_licenses = ["//:license"],
default_testonly = 1,
default_visibility = ["//src:__subpackages__"],
)
@@ -19,32 +20,49 @@
"//src/test/java/com/google/devtools/build/lib/remote/util:srcs",
"//src/test/java/com/google/devtools/build/lib/remote/zstd:srcs",
],
- visibility = ["//src/test/java/com/google/devtools/build/lib:__pkg__"],
+ visibility = ["//src:__subpackages__"],
)
-# Do not run NativeSslTest on platforms where native SSL is not available.
+java_library(
+ name = "action_input_prefetcher_test_base",
+ srcs = ["ActionInputPrefetcherTestBase.java"],
+ deps = [
+ "//src/main/java/com/google/devtools/build/lib/actions",
+ "//src/main/java/com/google/devtools/build/lib/actions:artifacts",
+ "//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
+ "//src/main/java/com/google/devtools/build/lib/remote:abstract_action_input_prefetcher",
+ "//src/main/java/com/google/devtools/build/lib/remote/util",
+ "//src/main/java/com/google/devtools/build/lib/skyframe:tree_artifact_value",
+ "//src/main/java/com/google/devtools/build/lib/testing/vfs:spied_filesystem",
+ "//src/main/java/com/google/devtools/build/lib/util",
+ "//src/main/java/com/google/devtools/build/lib/vfs",
+ "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
+ "//src/test/java/com/google/devtools/build/lib/actions/util",
+ "//third_party:guava",
+ "//third_party:jsr305",
+ "//third_party:junit4",
+ "//third_party:mockito",
+ "//third_party:truth",
+ ],
+)
-NATIVE_SSL_TEST = ["NativeSslTest.java"]
-
-NATIVE_SSL_TEST_MAYBE = select({
- "//src/conditions:windows": NATIVE_SSL_TEST,
- "//src/conditions:darwin": NATIVE_SSL_TEST,
- "//src/conditions:linux": NATIVE_SSL_TEST,
- "//conditions:default": [],
-})
-
-java_test(
- name = "remote",
+java_library(
+ name = "RemoteTests_lib",
srcs = glob(
- ["**/*.java"],
- exclude = NATIVE_SSL_TEST + [
+ [
+ "*.java",
+ ],
+ exclude = [
+ "ActionInputPrefetcherTestBase.java",
+ "RemoteActionFileSystemTestBase.java",
"BuildWithoutTheBytesIntegrationTest.java",
"BuildWithoutTheBytesIntegrationTestBase.java",
"DiskCacheIntegrationTest.java",
],
- ) + NATIVE_SSL_TEST_MAYBE,
- test_class = "com.google.devtools.build.lib.AllTests",
+ ),
deps = [
+ ":action_input_prefetcher_test_base",
+ ":remote_action_file_system_test_base",
"//src/main/java/com/google/devtools/build/lib:runtime",
"//src/main/java/com/google/devtools/build/lib:runtime/command_line_path_factory",
"//src/main/java/com/google/devtools/build/lib/actions",
@@ -81,17 +99,15 @@
"//src/main/java/com/google/devtools/build/lib/remote/common:bulk_transfer_exception",
"//src/main/java/com/google/devtools/build/lib/remote/common:cache_not_found_exception",
"//src/main/java/com/google/devtools/build/lib/remote/disk",
- "//src/main/java/com/google/devtools/build/lib/remote/grpc",
"//src/main/java/com/google/devtools/build/lib/remote/http",
"//src/main/java/com/google/devtools/build/lib/remote/merkletree",
"//src/main/java/com/google/devtools/build/lib/remote/options",
"//src/main/java/com/google/devtools/build/lib/remote/util",
"//src/main/java/com/google/devtools/build/lib/runtime/commands",
"//src/main/java/com/google/devtools/build/lib/skyframe:tree_artifact_value",
- "//src/main/java/com/google/devtools/build/lib/testing/vfs:spied_filesystem",
- "//src/main/java/com/google/devtools/build/lib/util",
"//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception",
"//src/main/java/com/google/devtools/build/lib/util:exit_code",
+ "//src/main/java/com/google/devtools/build/lib/util:os",
"//src/main/java/com/google/devtools/build/lib/util/io",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
@@ -99,19 +115,18 @@
"//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs",
"//src/main/java/com/google/devtools/common/options",
"//src/main/protobuf:failure_details_java_proto",
- "//src/main/protobuf:remote_execution_log_java_proto",
"//src/main/protobuf:spawn_java_proto",
- "//src/test/java/com/google/devtools/build/lib:test_runner",
"//src/test/java/com/google/devtools/build/lib/actions/util",
- "//src/test/java/com/google/devtools/build/lib/analysis/util",
"//src/test/java/com/google/devtools/build/lib/exec/util",
"//src/test/java/com/google/devtools/build/lib/remote/util",
"//src/test/java/com/google/devtools/build/lib/testutil",
"//src/test/java/com/google/devtools/build/lib/testutil:TestUtils",
- "//third_party:api_client",
"//third_party:auth",
"//third_party:caffeine",
+ "//third_party:error_prone_annotations",
+ "//third_party:gson",
"//third_party:guava",
+ "//third_party:jsr305",
"//third_party:junit4",
"//third_party:mockito",
"//third_party:netty",
@@ -130,7 +145,32 @@
"@remoteapis//:build_bazel_remote_execution_v2_remote_execution_java_grpc",
"@remoteapis//:build_bazel_remote_execution_v2_remote_execution_java_proto",
"@remoteapis//:build_bazel_semver_semver_java_proto",
- "@zstd-jni//:zstd-jni",
+ "@zstd-jni",
+ ],
+)
+
+java_library(
+ name = "remote_action_file_system_test_base",
+ srcs = ["RemoteActionFileSystemTestBase.java"],
+ deps = [
+ "//src/main/java/com/google/devtools/build/lib/actions",
+ "//src/main/java/com/google/devtools/build/lib/actions:artifacts",
+ "//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
+ "//src/main/java/com/google/devtools/build/lib/remote/util",
+ "//src/main/java/com/google/devtools/build/lib/vfs",
+ "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
+ "//third_party:guava",
+ "//third_party:junit4",
+ "//third_party:truth",
+ ],
+)
+
+java_test(
+ name = "RemoteTests",
+ test_class = "com.google.devtools.build.lib.AllTests",
+ runtime_deps = [
+ ":RemoteTests_lib",
+ "//src/test/java/com/google/devtools/build/lib:test_runner",
],
)
diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java
index d0377e2..1f089b4 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java
@@ -16,6 +16,7 @@
import static com.google.common.truth.Truth.assertThat;
import static com.google.devtools.build.lib.remote.util.Utils.getFromFuture;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.concurrent.TimeUnit.SECONDS;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.fail;
import static org.mockito.AdditionalAnswers.answerVoid;
@@ -36,8 +37,11 @@
import build.bazel.remote.execution.v2.FindMissingBlobsRequest;
import build.bazel.remote.execution.v2.FindMissingBlobsResponse;
import build.bazel.remote.execution.v2.GetActionResultRequest;
+import build.bazel.remote.execution.v2.RequestMetadata;
+import build.bazel.remote.execution.v2.ServerCapabilities;
import build.bazel.remote.execution.v2.Tree;
import build.bazel.remote.execution.v2.UpdateActionResultRequest;
+import com.github.luben.zstd.Zstd;
import com.google.bytestream.ByteStreamGrpc.ByteStreamImplBase;
import com.google.bytestream.ByteStreamProto.QueryWriteStatusRequest;
import com.google.bytestream.ByteStreamProto.QueryWriteStatusResponse;
@@ -50,36 +54,74 @@
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningScheduledExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
import com.google.devtools.build.lib.actions.ActionInputHelper;
import com.google.devtools.build.lib.actions.ArtifactPathResolver;
import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
+import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
+import com.google.devtools.build.lib.authandtls.CallCredentialsProvider;
+import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
+import com.google.devtools.build.lib.clock.JavaClock;
import com.google.devtools.build.lib.events.NullEventHandler;
+import com.google.devtools.build.lib.remote.RemoteRetrier.ExponentialBackoff;
import com.google.devtools.build.lib.remote.Retrier.Backoff;
+import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
import com.google.devtools.build.lib.remote.common.RemoteCacheClient.ActionKey;
+import com.google.devtools.build.lib.remote.common.RemotePathResolver;
import com.google.devtools.build.lib.remote.merkletree.MerkleTree;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
+import com.google.devtools.build.lib.remote.util.DigestUtil;
+import com.google.devtools.build.lib.remote.util.TestUtils;
+import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
+import com.google.devtools.build.lib.testutil.Scratch;
+import com.google.devtools.build.lib.util.io.FileOutErr;
+import com.google.devtools.build.lib.vfs.DigestHashFunction;
+import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.SyscallCache;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
import com.google.devtools.common.options.Options;
+import com.google.gson.JsonObject;
import com.google.protobuf.ByteString;
import io.grpc.BindableService;
+import io.grpc.CallCredentials;
+import io.grpc.CallOptions;
+import io.grpc.Channel;
+import io.grpc.ClientCall;
+import io.grpc.ClientInterceptor;
+import io.grpc.ManagedChannel;
import io.grpc.Metadata;
+import io.grpc.MethodDescriptor;
+import io.grpc.Server;
import io.grpc.ServerCall;
import io.grpc.ServerCallHandler;
import io.grpc.ServerInterceptor;
import io.grpc.ServerInterceptors;
import io.grpc.Status;
+import io.grpc.inprocess.InProcessChannelBuilder;
+import io.grpc.inprocess.InProcessServerBuilder;
import io.grpc.stub.ServerCallStreamObserver;
import io.grpc.stub.StreamObserver;
+import io.grpc.util.MutableHandlerRegistry;
+import io.reactivex.rxjava3.core.Single;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -90,11 +132,146 @@
/** Tests for {@link GrpcCacheClient}. */
@RunWith(JUnit4.class)
-public class GrpcCacheClientTest extends GrpcCacheClientTestBase {
+public class GrpcCacheClientTest {
+ private static final DigestUtil DIGEST_UTIL =
+ new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256);
+
+ private FileSystem fs;
+ private Path execRoot;
+ private FileOutErr outErr;
+ private FakeActionInputFileCache fakeFileCache;
+ private final MutableHandlerRegistry serviceRegistry = new MutableHandlerRegistry();
+ private final String fakeServerName = "fake server for " + getClass();
+ private Server fakeServer;
+ private RemoteActionExecutionContext context;
+ private RemotePathResolver remotePathResolver;
+ private ListeningScheduledExecutorService retryService;
+ private final ArrayList<ReferenceCountedChannel> channels = new ArrayList<>();
+
private GrpcCacheClient newClient() throws IOException {
return newClient(Options.getDefaults(RemoteOptions.class));
}
+ private GrpcCacheClient newClient(RemoteOptions remoteOptions) throws IOException {
+ return newClient(remoteOptions, () -> new ExponentialBackoff(remoteOptions));
+ }
+
+ private GrpcCacheClient newClient(RemoteOptions remoteOptions, Supplier<Backoff> backoffSupplier)
+ throws IOException {
+ AuthAndTLSOptions authTlsOptions = Options.getDefaults(AuthAndTLSOptions.class);
+ authTlsOptions.useGoogleDefaultCredentials = true;
+ authTlsOptions.googleCredentials = "/execroot/main/creds.json";
+ authTlsOptions.googleAuthScopes = ImmutableList.of("dummy.scope");
+
+ JsonObject json = new JsonObject();
+ json.addProperty("type", "authorized_user");
+ json.addProperty("client_id", "some_client");
+ json.addProperty("client_secret", "foo");
+ json.addProperty("refresh_token", "bar");
+ Scratch scratch = new Scratch();
+ scratch.file(authTlsOptions.googleCredentials, json.toString());
+
+ CallCredentialsProvider callCredentialsProvider;
+ try (InputStream in = scratch.resolve(authTlsOptions.googleCredentials).getInputStream()) {
+ callCredentialsProvider =
+ GoogleAuthUtils.newCallCredentialsProvider(
+ GoogleAuthUtils.newGoogleCredentialsFromFile(in, authTlsOptions.googleAuthScopes));
+ }
+ CallCredentials creds = callCredentialsProvider.getCallCredentials();
+
+ RemoteRetrier retrier =
+ TestUtils.newRemoteRetrier(
+ backoffSupplier, RemoteRetrier.RETRIABLE_GRPC_ERRORS, retryService);
+ ReferenceCountedChannel channel =
+ new ReferenceCountedChannel(
+ new ChannelConnectionWithServerCapabilitiesFactory() {
+ @Override
+ public Single<ChannelConnectionWithServerCapabilities> create() {
+ ManagedChannel ch =
+ InProcessChannelBuilder.forName(fakeServerName)
+ .directExecutor()
+ .intercept(new CallCredentialsInterceptor(creds))
+ .intercept(TracingMetadataUtils.newCacheHeadersInterceptor(remoteOptions))
+ .build();
+ return Single.just(
+ new ChannelConnectionWithServerCapabilities(
+ ch, ServerCapabilities.getDefaultInstance()));
+ }
+
+ @Override
+ public int maxConcurrency() {
+ return 100;
+ }
+ });
+ channels.add(channel);
+ return new GrpcCacheClient(
+ channel, callCredentialsProvider, remoteOptions, retrier, DIGEST_UTIL);
+ }
+
+ private static byte[] downloadBlob(
+ RemoteActionExecutionContext context, GrpcCacheClient cacheClient, Digest digest)
+ throws IOException, InterruptedException {
+ try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+ getFromFuture(cacheClient.downloadBlob(context, digest, out));
+ return out.toByteArray();
+ }
+ }
+
+ private static class CallCredentialsInterceptor implements ClientInterceptor {
+ private final CallCredentials credentials;
+
+ public CallCredentialsInterceptor(CallCredentials credentials) {
+ this.credentials = credentials;
+ }
+
+ @Override
+ public <RequestT, ResponseT> ClientCall<RequestT, ResponseT> interceptCall(
+ MethodDescriptor<RequestT, ResponseT> method, CallOptions callOptions, Channel next) {
+ assertThat(callOptions.getCredentials()).isEqualTo(credentials);
+ // Remove the call credentials to allow testing with dummy ones.
+ return next.newCall(method, callOptions.withCallCredentials(null));
+ }
+ }
+
+ @Before
+ public final void setUp() throws Exception {
+ // Use a mutable service registry for later registering the service impl for each test case.
+ fakeServer =
+ InProcessServerBuilder.forName(fakeServerName)
+ .fallbackHandlerRegistry(serviceRegistry)
+ .directExecutor()
+ .build()
+ .start();
+ Chunker.setDefaultChunkSizeForTesting(1000); // Enough for everything to be one chunk.
+ fs = new InMemoryFileSystem(new JavaClock(), DigestHashFunction.SHA256);
+ execRoot = fs.getPath("/execroot/main");
+ execRoot.createDirectoryAndParents();
+ fakeFileCache = new FakeActionInputFileCache(execRoot);
+ remotePathResolver = RemotePathResolver.createDefault(execRoot);
+
+ Path stdout = fs.getPath("/tmp/stdout");
+ Path stderr = fs.getPath("/tmp/stderr");
+ stdout.getParentDirectory().createDirectoryAndParents();
+ stderr.getParentDirectory().createDirectoryAndParents();
+ outErr = new FileOutErr(stdout, stderr);
+ RequestMetadata metadata =
+ TracingMetadataUtils.buildMetadata(
+ "none", "none", Digest.getDefaultInstance().getHash(), null);
+ context = RemoteActionExecutionContext.create(metadata);
+ retryService = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1));
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ channels.forEach(ReferenceCountedChannel::release);
+ retryService.shutdownNow();
+ retryService.awaitTermination(
+ com.google.devtools.build.lib.testutil.TestUtils.WAIT_TIMEOUT_SECONDS, SECONDS);
+
+ fakeServer.shutdownNow();
+ fakeServer.awaitTermination();
+ }
+
@Test
public void testVirtualActionInputSupport() throws Exception {
RemoteOptions options = Options.getDefaults(RemoteOptions.class);
@@ -1020,6 +1197,86 @@
}
@Test
+ public void compressedDownloadBlobIsRetriedWithProgress()
+ throws IOException, InterruptedException {
+ RemoteOptions options = Options.getDefaults(RemoteOptions.class);
+ options.cacheCompression = true;
+ final GrpcCacheClient client = newClient(options);
+ final Digest digest = DIGEST_UTIL.computeAsUtf8("abcdefg");
+ ByteString chunk1 = ByteString.copyFrom(Zstd.compress("abc".getBytes(UTF_8)));
+ ByteString chunk2 = ByteString.copyFrom(Zstd.compress("def".getBytes(UTF_8)));
+ ByteString chunk3 = ByteString.copyFrom(Zstd.compress("g".getBytes(UTF_8)));
+ serviceRegistry.addService(
+ new ByteStreamImplBase() {
+ private boolean first = true;
+
+ @Override
+ public void read(ReadRequest request, StreamObserver<ReadResponse> responseObserver) {
+ assertThat(request.getResourceName()).contains(digest.getHash());
+ if (first) {
+ first = false;
+ responseObserver.onError(Status.DEADLINE_EXCEEDED.asException());
+ return;
+ }
+ switch (Math.toIntExact(request.getReadOffset())) {
+ case 0:
+ responseObserver.onNext(ReadResponse.newBuilder().setData(chunk1).build());
+ break;
+ case 3:
+ responseObserver.onNext(ReadResponse.newBuilder().setData(chunk2).build());
+ break;
+ case 6:
+ responseObserver.onNext(ReadResponse.newBuilder().setData(chunk3).build());
+ responseObserver.onCompleted();
+ return;
+ default:
+ throw new IllegalStateException("unexpected offset " + request.getReadOffset());
+ }
+ responseObserver.onError(Status.DEADLINE_EXCEEDED.asException());
+ }
+ });
+ assertThat(new String(downloadBlob(context, client, digest), UTF_8)).isEqualTo("abcdefg");
+ }
+
+ @Test
+ public void testCompressedDownload() throws IOException, InterruptedException {
+ RemoteOptions options = Options.getDefaults(RemoteOptions.class);
+ options.cacheCompression = true;
+ final GrpcCacheClient client = newClient(options);
+ final byte[] data = "abcdefg".getBytes(UTF_8);
+ final Digest digest = DIGEST_UTIL.compute(data);
+ final byte[] compressed = Zstd.compress(data);
+
+ serviceRegistry.addService(
+ new ByteStreamImplBase() {
+ @Override
+ public void read(ReadRequest request, StreamObserver<ReadResponse> responseObserver) {
+ assertThat(request.getResourceName()).contains(digest.getHash());
+ responseObserver.onNext(
+ ReadResponse.newBuilder()
+ .setData(ByteString.copyFrom(Arrays.copyOf(compressed, compressed.length / 3)))
+ .build());
+ responseObserver.onNext(
+ ReadResponse.newBuilder()
+ .setData(
+ ByteString.copyFrom(
+ Arrays.copyOfRange(
+ compressed, compressed.length / 3, compressed.length / 3 * 2)))
+ .build());
+ responseObserver.onNext(
+ ReadResponse.newBuilder()
+ .setData(
+ ByteString.copyFrom(
+ Arrays.copyOfRange(
+ compressed, compressed.length / 3 * 2, compressed.length)))
+ .build());
+ responseObserver.onCompleted();
+ }
+ });
+ assertThat(downloadBlob(context, client, digest)).isEqualTo(data);
+ }
+
+ @Test
public void isRemoteCacheOptionsWhenGrpcEnabled() {
RemoteOptions options = Options.getDefaults(RemoteOptions.class);
options.remoteCache = "grpc://some-host.com";
diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTestBase.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTestBase.java
deleted file mode 100644
index b674065..0000000
--- a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTestBase.java
+++ /dev/null
@@ -1,204 +0,0 @@
-// Copyright 2022 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package com.google.devtools.build.lib.remote;
-
-import static com.google.common.truth.Truth.assertThat;
-import static com.google.devtools.build.lib.remote.util.Utils.getFromFuture;
-
-import build.bazel.remote.execution.v2.Digest;
-import build.bazel.remote.execution.v2.RequestMetadata;
-import build.bazel.remote.execution.v2.ServerCapabilities;
-import com.google.api.client.json.GenericJson;
-import com.google.api.client.json.gson.GsonFactory;
-import com.google.common.collect.ImmutableList;
-import com.google.common.util.concurrent.ListeningScheduledExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
-import com.google.devtools.build.lib.authandtls.CallCredentialsProvider;
-import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
-import com.google.devtools.build.lib.clock.JavaClock;
-import com.google.devtools.build.lib.remote.RemoteRetrier.ExponentialBackoff;
-import com.google.devtools.build.lib.remote.Retrier.Backoff;
-import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
-import com.google.devtools.build.lib.remote.common.RemotePathResolver;
-import com.google.devtools.build.lib.remote.options.RemoteOptions;
-import com.google.devtools.build.lib.remote.util.DigestUtil;
-import com.google.devtools.build.lib.remote.util.TestUtils;
-import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
-import com.google.devtools.build.lib.testutil.Scratch;
-import com.google.devtools.build.lib.util.io.FileOutErr;
-import com.google.devtools.build.lib.vfs.DigestHashFunction;
-import com.google.devtools.build.lib.vfs.FileSystem;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.SyscallCache;
-import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
-import com.google.devtools.common.options.Options;
-import io.grpc.CallCredentials;
-import io.grpc.CallOptions;
-import io.grpc.Channel;
-import io.grpc.ClientCall;
-import io.grpc.ClientInterceptor;
-import io.grpc.ManagedChannel;
-import io.grpc.MethodDescriptor;
-import io.grpc.Server;
-import io.grpc.inprocess.InProcessChannelBuilder;
-import io.grpc.inprocess.InProcessServerBuilder;
-import io.grpc.util.MutableHandlerRegistry;
-import io.reactivex.rxjava3.core.Single;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Supplier;
-import org.junit.After;
-import org.junit.Before;
-
-class GrpcCacheClientTestBase {
- protected static final DigestUtil DIGEST_UTIL =
- new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256);
-
- protected FileSystem fs;
- protected Path execRoot;
- protected FileOutErr outErr;
- protected FakeActionInputFileCache fakeFileCache;
- protected final MutableHandlerRegistry serviceRegistry = new MutableHandlerRegistry();
- protected final String fakeServerName = "fake server for " + getClass();
- protected Server fakeServer;
- protected RemoteActionExecutionContext context;
- protected RemotePathResolver remotePathResolver;
- protected ListeningScheduledExecutorService retryService;
- private final ArrayList<ReferenceCountedChannel> channels = new ArrayList<>();
-
- @Before
- public final void setUp() throws Exception {
- // Use a mutable service registry for later registering the service impl for each test case.
- fakeServer =
- InProcessServerBuilder.forName(fakeServerName)
- .fallbackHandlerRegistry(serviceRegistry)
- .directExecutor()
- .build()
- .start();
- Chunker.setDefaultChunkSizeForTesting(1000); // Enough for everything to be one chunk.
- fs = new InMemoryFileSystem(new JavaClock(), DigestHashFunction.SHA256);
- execRoot = fs.getPath("/execroot/main");
- execRoot.createDirectoryAndParents();
- fakeFileCache = new FakeActionInputFileCache(execRoot);
- remotePathResolver = RemotePathResolver.createDefault(execRoot);
-
- Path stdout = fs.getPath("/tmp/stdout");
- Path stderr = fs.getPath("/tmp/stderr");
- stdout.getParentDirectory().createDirectoryAndParents();
- stderr.getParentDirectory().createDirectoryAndParents();
- outErr = new FileOutErr(stdout, stderr);
- RequestMetadata metadata =
- TracingMetadataUtils.buildMetadata(
- "none", "none", Digest.getDefaultInstance().getHash(), null);
- context = RemoteActionExecutionContext.create(metadata);
- retryService = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1));
- }
-
- @After
- public void tearDown() throws Exception {
- channels.forEach(ReferenceCountedChannel::release);
- retryService.shutdownNow();
- retryService.awaitTermination(
- com.google.devtools.build.lib.testutil.TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
-
- fakeServer.shutdownNow();
- fakeServer.awaitTermination();
- }
-
- protected GrpcCacheClient newClient(RemoteOptions remoteOptions) throws IOException {
- return newClient(remoteOptions, () -> new ExponentialBackoff(remoteOptions));
- }
-
- protected GrpcCacheClient newClient(
- RemoteOptions remoteOptions, Supplier<Backoff> backoffSupplier) throws IOException {
- AuthAndTLSOptions authTlsOptions = Options.getDefaults(AuthAndTLSOptions.class);
- authTlsOptions.useGoogleDefaultCredentials = true;
- authTlsOptions.googleCredentials = "/execroot/main/creds.json";
- authTlsOptions.googleAuthScopes = ImmutableList.of("dummy.scope");
-
- GenericJson json = new GenericJson();
- json.put("type", "authorized_user");
- json.put("client_id", "some_client");
- json.put("client_secret", "foo");
- json.put("refresh_token", "bar");
- Scratch scratch = new Scratch();
- scratch.file(authTlsOptions.googleCredentials, new GsonFactory().toString(json));
-
- CallCredentialsProvider callCredentialsProvider;
- try (InputStream in = scratch.resolve(authTlsOptions.googleCredentials).getInputStream()) {
- callCredentialsProvider =
- GoogleAuthUtils.newCallCredentialsProvider(
- GoogleAuthUtils.newGoogleCredentialsFromFile(in, authTlsOptions.googleAuthScopes));
- }
- CallCredentials creds = callCredentialsProvider.getCallCredentials();
-
- RemoteRetrier retrier =
- TestUtils.newRemoteRetrier(
- backoffSupplier, RemoteRetrier.RETRIABLE_GRPC_ERRORS, retryService);
- ReferenceCountedChannel channel =
- new ReferenceCountedChannel(
- new ChannelConnectionWithServerCapabilitiesFactory() {
- @Override
- public Single<ChannelConnectionWithServerCapabilities> create() {
- ManagedChannel ch =
- InProcessChannelBuilder.forName(fakeServerName)
- .directExecutor()
- .intercept(new CallCredentialsInterceptor(creds))
- .intercept(TracingMetadataUtils.newCacheHeadersInterceptor(remoteOptions))
- .build();
- return Single.just(
- new ChannelConnectionWithServerCapabilities(
- ch, ServerCapabilities.getDefaultInstance()));
- }
-
- @Override
- public int maxConcurrency() {
- return 100;
- }
- });
- channels.add(channel);
- return new GrpcCacheClient(
- channel, callCredentialsProvider, remoteOptions, retrier, DIGEST_UTIL);
- }
-
- protected static byte[] downloadBlob(
- RemoteActionExecutionContext context, GrpcCacheClient cacheClient, Digest digest)
- throws IOException, InterruptedException {
- try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
- getFromFuture(cacheClient.downloadBlob(context, digest, out));
- return out.toByteArray();
- }
- }
-
- private static class CallCredentialsInterceptor implements ClientInterceptor {
- private final CallCredentials credentials;
-
- public CallCredentialsInterceptor(CallCredentials credentials) {
- this.credentials = credentials;
- }
-
- @Override
- public <RequestT, ResponseT> ClientCall<RequestT, ResponseT> interceptCall(
- MethodDescriptor<RequestT, ResponseT> method, CallOptions callOptions, Channel next) {
- assertThat(callOptions.getCredentials()).isEqualTo(credentials);
- // Remove the call credentials to allow testing with dummy ones.
- return next.newCall(method, callOptions.withCallCredentials(null));
- }
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTestExtra.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTestExtra.java
deleted file mode 100644
index b840907..0000000
--- a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTestExtra.java
+++ /dev/null
@@ -1,117 +0,0 @@
-// Copyright 2021 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package com.google.devtools.build.lib.remote;
-
-import static com.google.common.truth.Truth.assertThat;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import build.bazel.remote.execution.v2.Digest;
-import com.github.luben.zstd.Zstd;
-import com.google.bytestream.ByteStreamGrpc.ByteStreamImplBase;
-import com.google.bytestream.ByteStreamProto.ReadRequest;
-import com.google.bytestream.ByteStreamProto.ReadResponse;
-import com.google.devtools.build.lib.remote.options.RemoteOptions;
-import com.google.devtools.common.options.Options;
-import com.google.protobuf.ByteString;
-import io.grpc.Status;
-import io.grpc.stub.StreamObserver;
-import java.io.IOException;
-import java.util.Arrays;
-import org.junit.Test;
-
-/** Extra tests for {@link GrpcCacheClient} that are not tested internally. */
-public class GrpcCacheClientTestExtra extends GrpcCacheClientTestBase {
-
- @Test
- public void compressedDownloadBlobIsRetriedWithProgress()
- throws IOException, InterruptedException {
- RemoteOptions options = Options.getDefaults(RemoteOptions.class);
- options.cacheCompression = true;
- final GrpcCacheClient client = newClient(options);
- final Digest digest = DIGEST_UTIL.computeAsUtf8("abcdefg");
- ByteString chunk1 = ByteString.copyFrom(Zstd.compress("abc".getBytes(UTF_8)));
- ByteString chunk2 = ByteString.copyFrom(Zstd.compress("def".getBytes(UTF_8)));
- ByteString chunk3 = ByteString.copyFrom(Zstd.compress("g".getBytes(UTF_8)));
- serviceRegistry.addService(
- new ByteStreamImplBase() {
- private boolean first = true;
-
- @Override
- public void read(ReadRequest request, StreamObserver<ReadResponse> responseObserver) {
- assertThat(request.getResourceName().contains(digest.getHash())).isTrue();
- if (first) {
- first = false;
- responseObserver.onError(Status.DEADLINE_EXCEEDED.asException());
- return;
- }
- switch (Math.toIntExact(request.getReadOffset())) {
- case 0:
- responseObserver.onNext(ReadResponse.newBuilder().setData(chunk1).build());
- break;
- case 3:
- responseObserver.onNext(ReadResponse.newBuilder().setData(chunk2).build());
- break;
- case 6:
- responseObserver.onNext(ReadResponse.newBuilder().setData(chunk3).build());
- responseObserver.onCompleted();
- return;
- default:
- throw new IllegalStateException("unexpected offset " + request.getReadOffset());
- }
- responseObserver.onError(Status.DEADLINE_EXCEEDED.asException());
- }
- });
- assertThat(new String(downloadBlob(context, client, digest), UTF_8)).isEqualTo("abcdefg");
- }
-
- @Test
- public void testCompressedDownload() throws IOException, InterruptedException {
- RemoteOptions options = Options.getDefaults(RemoteOptions.class);
- options.cacheCompression = true;
- final GrpcCacheClient client = newClient(options);
- final byte[] data = "abcdefg".getBytes(UTF_8);
- final Digest digest = DIGEST_UTIL.compute(data);
- final byte[] compressed = Zstd.compress(data);
-
- serviceRegistry.addService(
- new ByteStreamImplBase() {
- @Override
- public void read(ReadRequest request, StreamObserver<ReadResponse> responseObserver) {
- assertThat(request.getResourceName().contains(digest.getHash())).isTrue();
- responseObserver.onNext(
- ReadResponse.newBuilder()
- .setData(
- ByteString.copyFrom(
- Arrays.copyOfRange(compressed, 0, compressed.length / 3)))
- .build());
- responseObserver.onNext(
- ReadResponse.newBuilder()
- .setData(
- ByteString.copyFrom(
- Arrays.copyOfRange(
- compressed, compressed.length / 3, compressed.length / 3 * 2)))
- .build());
- responseObserver.onNext(
- ReadResponse.newBuilder()
- .setData(
- ByteString.copyFrom(
- Arrays.copyOfRange(
- compressed, compressed.length / 3 * 2, compressed.length)))
- .build());
- responseObserver.onCompleted();
- }
- });
- assertThat(downloadBlob(context, client, digest)).isEqualTo(data);
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/remote/NativeSslTest.java b/src/test/java/com/google/devtools/build/lib/remote/NativeSslTest.java
index 05da702..217b6ca 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/NativeSslTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/NativeSslTest.java
@@ -13,6 +13,10 @@
// limitations under the License.
package com.google.devtools.build.lib.remote;
+import static org.junit.Assume.assumeTrue;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.util.OS;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SslProvider;
import org.junit.Test;
@@ -22,8 +26,14 @@
/** Tests that we use OpenSSL instead of a Java implementation. */
@RunWith(JUnit4.class)
public class NativeSslTest {
+ private static final ImmutableSet<OS> OS_WITH_NATIVE_SSL =
+ ImmutableSet.of(OS.LINUX, OS.DARWIN, OS.WINDOWS);
+
@Test
public void nativeSslPresent() throws Exception {
+ // Skip the test on platforms where native SSL is not available.
+ assumeTrue(OS_WITH_NATIVE_SSL.contains(OS.getCurrent()));
+
SslContextBuilder.forClient().sslProvider(SslProvider.OPENSSL).build();
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/remote/downloader/BUILD b/src/test/java/com/google/devtools/build/lib/remote/downloader/BUILD
index 8e2cae2..d49a5aa 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/downloader/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/remote/downloader/BUILD
@@ -28,7 +28,6 @@
"//src/main/java/com/google/devtools/build/lib/remote",
"//src/main/java/com/google/devtools/build/lib/remote/common",
"//src/main/java/com/google/devtools/build/lib/remote/downloader",
- "//src/main/java/com/google/devtools/build/lib/remote/grpc",
"//src/main/java/com/google/devtools/build/lib/remote/options",
"//src/main/java/com/google/devtools/build/lib/remote/util",
"//src/main/java/com/google/devtools/build/lib/vfs",
diff --git a/src/test/java/com/google/devtools/build/lib/remote/zstd/BUILD b/src/test/java/com/google/devtools/build/lib/remote/zstd/BUILD
index 58647b4..4f80374 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/zstd/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/remote/zstd/BUILD
@@ -1,12 +1,11 @@
load("@rules_java//java:defs.bzl", "java_test")
package(
+ default_applicable_licenses = ["//:license"],
default_testonly = 1,
default_visibility = ["//src:__subpackages__"],
)
-licenses(["notice"])
-
filegroup(
name = "srcs",
testonly = 0,
@@ -24,6 +23,6 @@
"//third_party:guava",
"//third_party:junit4",
"//third_party:truth",
- "@zstd-jni//:zstd-jni",
+ "@zstd-jni",
],
)
diff --git a/src/test/java/com/google/devtools/build/lib/remote/zstd/ZstdDecompressingOutputStreamTest.java b/src/test/java/com/google/devtools/build/lib/remote/zstd/ZstdDecompressingOutputStreamTest.java
index 377bb41..4768d9f 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/zstd/ZstdDecompressingOutputStreamTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/zstd/ZstdDecompressingOutputStreamTest.java
@@ -14,8 +14,10 @@
package com.google.devtools.build.lib.remote.zstd;
import static com.google.common.truth.Truth.assertThat;
+import static java.nio.charset.StandardCharsets.UTF_8;
import com.github.luben.zstd.Zstd;
+import com.github.luben.zstd.ZstdOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Random;
@@ -41,4 +43,48 @@
assertThat(baos.toByteArray()).isEqualTo(data);
}
+
+ @Test
+ public void streamCanBeDecompressedOneByteAtATime() throws IOException {
+ Random rand = new Random();
+ byte[] data = new byte[50];
+ rand.nextBytes(data);
+ byte[] compressed = Zstd.compress(data);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try (ZstdDecompressingOutputStream zdos = new ZstdDecompressingOutputStream(baos)) {
+ for (byte b : compressed) {
+ zdos.write(b);
+ }
+ zdos.flush();
+ }
+
+ assertThat(baos.toByteArray()).isEqualTo(data);
+ }
+
+ @Test
+ public void bytesWrittenMatchesDecompressedBytes() throws IOException {
+ byte[] data = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA".getBytes(UTF_8);
+
+ ByteArrayOutputStream compressed = new ByteArrayOutputStream();
+ try (ZstdOutputStream zos = new ZstdOutputStream(compressed)) {
+ zos.setCloseFrameOnFlush(true);
+ for (int i = 0; i < data.length; i++) {
+ zos.write(data[i]);
+ if (i % 5 == 0) {
+ // Create multiple frames of 5 bytes each.
+ zos.flush();
+ }
+ }
+ }
+
+ ByteArrayOutputStream decompressed = new ByteArrayOutputStream();
+ try (ZstdDecompressingOutputStream zdos = new ZstdDecompressingOutputStream(decompressed)) {
+ for (byte b : compressed.toByteArray()) {
+ zdos.write(b);
+ zdos.flush();
+ }
+ }
+ assertThat(decompressed.toByteArray()).isEqualTo(data);
+ }
}
diff --git a/src/test/java/com/google/devtools/build/lib/remote/zstd/ZstdDecompressingOutputStreamTestExtra.java b/src/test/java/com/google/devtools/build/lib/remote/zstd/ZstdDecompressingOutputStreamTestExtra.java
deleted file mode 100644
index d5511ee..0000000
--- a/src/test/java/com/google/devtools/build/lib/remote/zstd/ZstdDecompressingOutputStreamTestExtra.java
+++ /dev/null
@@ -1,71 +0,0 @@
-// Copyright 2021 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package com.google.devtools.build.lib.remote.zstd;
-
-import static com.google.common.truth.Truth.assertThat;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import com.github.luben.zstd.Zstd;
-import com.github.luben.zstd.ZstdOutputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.Random;
-import org.junit.Test;
-
-/** Extra tests for {@link ZstdDecompressingOutputStream} that are not tested internally. */
-public class ZstdDecompressingOutputStreamTestExtra {
- @Test
- public void streamCanBeDecompressedOneByteAtATime() throws IOException {
- Random rand = new Random();
- byte[] data = new byte[50];
- rand.nextBytes(data);
- byte[] compressed = Zstd.compress(data);
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- try (ZstdDecompressingOutputStream zdos = new ZstdDecompressingOutputStream(baos)) {
- for (byte b : compressed) {
- zdos.write(b);
- }
- zdos.flush();
- }
-
- assertThat(baos.toByteArray()).isEqualTo(data);
- }
-
- @Test
- public void bytesWrittenMatchesDecompressedBytes() throws IOException {
- byte[] data = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA".getBytes(UTF_8);
-
- ByteArrayOutputStream compressed = new ByteArrayOutputStream();
- try (ZstdOutputStream zos = new ZstdOutputStream(compressed)) {
- zos.setCloseFrameOnFlush(true);
- for (int i = 0; i < data.length; i++) {
- zos.write(data[i]);
- if (i % 5 == 0) {
- // Create multiple frames of 5 bytes each.
- zos.flush();
- }
- }
- }
-
- ByteArrayOutputStream decompressed = new ByteArrayOutputStream();
- try (ZstdDecompressingOutputStream zdos = new ZstdDecompressingOutputStream(decompressed)) {
- for (byte b : compressed.toByteArray()) {
- zdos.write(b);
- zdos.flush();
- }
- }
- assertThat(decompressed.toByteArray()).isEqualTo(data);
- }
-}
