Reorganize the //src/test/java/com/google/devtools/build/lib/remote package. PiperOrigin-RevId: 568156390 Change-Id: If390f8535e3c574debac68ce96c3e451f1599d98
diff --git a/.bazelci/postsubmit.yml b/.bazelci/postsubmit.yml index 6b13128..f961fc7 100644 --- a/.bazelci/postsubmit.yml +++ b/.bazelci/postsubmit.yml
@@ -328,7 +328,7 @@ - "-//src/test/java/com/google/devtools/build/lib/query2/engine/..." - "-//src/test/java/com/google/devtools/build/lib/versioning/..." - "-//src/test/java/com/google/devtools/build/lib/worker/..." - - "-//src/test/java/com/google/devtools/build/lib/remote:remote" + - "-//src/test/java/com/google/devtools/build/lib/remote:RemoteTests" - "-//src/test/shell/bazel/remote/..." - "-//tools/python:pywrapper_test" include_json_profile:
diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index af9981d..5cbd96f 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml
@@ -389,7 +389,7 @@ - "-//src/test/java/com/google/devtools/build/lib/query2/engine/..." - "-//src/test/java/com/google/devtools/build/lib/versioning/..." - "-//src/test/java/com/google/devtools/build/lib/worker/..." - - "-//src/test/java/com/google/devtools/build/lib/remote:remote" + - "-//src/test/java/com/google/devtools/build/lib/remote:RemoteTests" - "-//src/test/shell/bazel/remote/..." - "-//tools/python:pywrapper_test" include_json_profile:
diff --git a/src/test/java/com/google/devtools/build/lib/remote/BUILD b/src/test/java/com/google/devtools/build/lib/remote/BUILD index e49e166..2496f99 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/BUILD +++ b/src/test/java/com/google/devtools/build/lib/remote/BUILD
@@ -1,6 +1,7 @@ -load("@rules_java//java:defs.bzl", "java_test") +load("@rules_java//java:defs.bzl", "java_library", "java_test") package( + default_applicable_licenses = ["//:license"], default_testonly = 1, default_visibility = ["//src:__subpackages__"], ) @@ -19,32 +20,49 @@ "//src/test/java/com/google/devtools/build/lib/remote/util:srcs", "//src/test/java/com/google/devtools/build/lib/remote/zstd:srcs", ], - visibility = ["//src/test/java/com/google/devtools/build/lib:__pkg__"], + visibility = ["//src:__subpackages__"], ) -# Do not run NativeSslTest on platforms where native SSL is not available. +java_library( + name = "action_input_prefetcher_test_base", + srcs = ["ActionInputPrefetcherTestBase.java"], + deps = [ + "//src/main/java/com/google/devtools/build/lib/actions", + "//src/main/java/com/google/devtools/build/lib/actions:artifacts", + "//src/main/java/com/google/devtools/build/lib/actions:file_metadata", + "//src/main/java/com/google/devtools/build/lib/remote:abstract_action_input_prefetcher", + "//src/main/java/com/google/devtools/build/lib/remote/util", + "//src/main/java/com/google/devtools/build/lib/skyframe:tree_artifact_value", + "//src/main/java/com/google/devtools/build/lib/testing/vfs:spied_filesystem", + "//src/main/java/com/google/devtools/build/lib/util", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", + "//src/test/java/com/google/devtools/build/lib/actions/util", + "//third_party:guava", + "//third_party:jsr305", + "//third_party:junit4", + "//third_party:mockito", + "//third_party:truth", + ], +) -NATIVE_SSL_TEST = ["NativeSslTest.java"] - -NATIVE_SSL_TEST_MAYBE = select({ - "//src/conditions:windows": NATIVE_SSL_TEST, - "//src/conditions:darwin": NATIVE_SSL_TEST, - "//src/conditions:linux": NATIVE_SSL_TEST, - "//conditions:default": [], -}) - -java_test( - name = "remote", +java_library( + name = "RemoteTests_lib", srcs = glob( - ["**/*.java"], - exclude = NATIVE_SSL_TEST + [ + [ + "*.java", + ], + exclude = [ + "ActionInputPrefetcherTestBase.java", + "RemoteActionFileSystemTestBase.java", "BuildWithoutTheBytesIntegrationTest.java", "BuildWithoutTheBytesIntegrationTestBase.java", "DiskCacheIntegrationTest.java", ], - ) + NATIVE_SSL_TEST_MAYBE, - test_class = "com.google.devtools.build.lib.AllTests", + ), deps = [ + ":action_input_prefetcher_test_base", + ":remote_action_file_system_test_base", "//src/main/java/com/google/devtools/build/lib:runtime", "//src/main/java/com/google/devtools/build/lib:runtime/command_line_path_factory", "//src/main/java/com/google/devtools/build/lib/actions", @@ -81,17 +99,15 @@ "//src/main/java/com/google/devtools/build/lib/remote/common:bulk_transfer_exception", "//src/main/java/com/google/devtools/build/lib/remote/common:cache_not_found_exception", "//src/main/java/com/google/devtools/build/lib/remote/disk", - "//src/main/java/com/google/devtools/build/lib/remote/grpc", "//src/main/java/com/google/devtools/build/lib/remote/http", "//src/main/java/com/google/devtools/build/lib/remote/merkletree", "//src/main/java/com/google/devtools/build/lib/remote/options", "//src/main/java/com/google/devtools/build/lib/remote/util", "//src/main/java/com/google/devtools/build/lib/runtime/commands", "//src/main/java/com/google/devtools/build/lib/skyframe:tree_artifact_value", - "//src/main/java/com/google/devtools/build/lib/testing/vfs:spied_filesystem", - "//src/main/java/com/google/devtools/build/lib/util", "//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception", "//src/main/java/com/google/devtools/build/lib/util:exit_code", + "//src/main/java/com/google/devtools/build/lib/util:os", "//src/main/java/com/google/devtools/build/lib/util/io", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", @@ -99,19 +115,18 @@ "//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs", "//src/main/java/com/google/devtools/common/options", "//src/main/protobuf:failure_details_java_proto", - "//src/main/protobuf:remote_execution_log_java_proto", "//src/main/protobuf:spawn_java_proto", - "//src/test/java/com/google/devtools/build/lib:test_runner", "//src/test/java/com/google/devtools/build/lib/actions/util", - "//src/test/java/com/google/devtools/build/lib/analysis/util", "//src/test/java/com/google/devtools/build/lib/exec/util", "//src/test/java/com/google/devtools/build/lib/remote/util", "//src/test/java/com/google/devtools/build/lib/testutil", "//src/test/java/com/google/devtools/build/lib/testutil:TestUtils", - "//third_party:api_client", "//third_party:auth", "//third_party:caffeine", + "//third_party:error_prone_annotations", + "//third_party:gson", "//third_party:guava", + "//third_party:jsr305", "//third_party:junit4", "//third_party:mockito", "//third_party:netty", @@ -130,7 +145,32 @@ "@remoteapis//:build_bazel_remote_execution_v2_remote_execution_java_grpc", "@remoteapis//:build_bazel_remote_execution_v2_remote_execution_java_proto", "@remoteapis//:build_bazel_semver_semver_java_proto", - "@zstd-jni//:zstd-jni", + "@zstd-jni", + ], +) + +java_library( + name = "remote_action_file_system_test_base", + srcs = ["RemoteActionFileSystemTestBase.java"], + deps = [ + "//src/main/java/com/google/devtools/build/lib/actions", + "//src/main/java/com/google/devtools/build/lib/actions:artifacts", + "//src/main/java/com/google/devtools/build/lib/actions:file_metadata", + "//src/main/java/com/google/devtools/build/lib/remote/util", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", + "//third_party:guava", + "//third_party:junit4", + "//third_party:truth", + ], +) + +java_test( + name = "RemoteTests", + test_class = "com.google.devtools.build.lib.AllTests", + runtime_deps = [ + ":RemoteTests_lib", + "//src/test/java/com/google/devtools/build/lib:test_runner", ], )
diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java index d0377e2..1f089b4 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java
@@ -16,6 +16,7 @@ import static com.google.common.truth.Truth.assertThat; import static com.google.devtools.build.lib.remote.util.Utils.getFromFuture; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertThrows; import static org.junit.Assert.fail; import static org.mockito.AdditionalAnswers.answerVoid; @@ -36,8 +37,11 @@ import build.bazel.remote.execution.v2.FindMissingBlobsRequest; import build.bazel.remote.execution.v2.FindMissingBlobsResponse; import build.bazel.remote.execution.v2.GetActionResultRequest; +import build.bazel.remote.execution.v2.RequestMetadata; +import build.bazel.remote.execution.v2.ServerCapabilities; import build.bazel.remote.execution.v2.Tree; import build.bazel.remote.execution.v2.UpdateActionResultRequest; +import com.github.luben.zstd.Zstd; import com.google.bytestream.ByteStreamGrpc.ByteStreamImplBase; import com.google.bytestream.ByteStreamProto.QueryWriteStatusRequest; import com.google.bytestream.ByteStreamProto.QueryWriteStatusResponse; @@ -50,36 +54,74 @@ import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.Maps; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import com.google.devtools.build.lib.actions.ActionInputHelper; import com.google.devtools.build.lib.actions.ArtifactPathResolver; import com.google.devtools.build.lib.actions.cache.VirtualActionInput; import com.google.devtools.build.lib.actions.util.ActionsTestUtil; +import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions; +import com.google.devtools.build.lib.authandtls.CallCredentialsProvider; +import com.google.devtools.build.lib.authandtls.GoogleAuthUtils; +import com.google.devtools.build.lib.clock.JavaClock; import com.google.devtools.build.lib.events.NullEventHandler; +import com.google.devtools.build.lib.remote.RemoteRetrier.ExponentialBackoff; import com.google.devtools.build.lib.remote.Retrier.Backoff; +import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext; import com.google.devtools.build.lib.remote.common.RemoteCacheClient.ActionKey; +import com.google.devtools.build.lib.remote.common.RemotePathResolver; import com.google.devtools.build.lib.remote.merkletree.MerkleTree; import com.google.devtools.build.lib.remote.options.RemoteOptions; +import com.google.devtools.build.lib.remote.util.DigestUtil; +import com.google.devtools.build.lib.remote.util.TestUtils; +import com.google.devtools.build.lib.remote.util.TracingMetadataUtils; +import com.google.devtools.build.lib.testutil.Scratch; +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.DigestHashFunction; +import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.SyscallCache; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import com.google.devtools.common.options.Options; +import com.google.gson.JsonObject; import com.google.protobuf.ByteString; import io.grpc.BindableService; +import io.grpc.CallCredentials; +import io.grpc.CallOptions; +import io.grpc.Channel; +import io.grpc.ClientCall; +import io.grpc.ClientInterceptor; +import io.grpc.ManagedChannel; import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import io.grpc.Server; import io.grpc.ServerCall; import io.grpc.ServerCallHandler; import io.grpc.ServerInterceptor; import io.grpc.ServerInterceptors; import io.grpc.Status; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; import io.grpc.stub.ServerCallStreamObserver; import io.grpc.stub.StreamObserver; +import io.grpc.util.MutableHandlerRegistry; +import io.reactivex.rxjava3.core.Single; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -90,11 +132,146 @@ /** Tests for {@link GrpcCacheClient}. */ @RunWith(JUnit4.class) -public class GrpcCacheClientTest extends GrpcCacheClientTestBase { +public class GrpcCacheClientTest { + private static final DigestUtil DIGEST_UTIL = + new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); + + private FileSystem fs; + private Path execRoot; + private FileOutErr outErr; + private FakeActionInputFileCache fakeFileCache; + private final MutableHandlerRegistry serviceRegistry = new MutableHandlerRegistry(); + private final String fakeServerName = "fake server for " + getClass(); + private Server fakeServer; + private RemoteActionExecutionContext context; + private RemotePathResolver remotePathResolver; + private ListeningScheduledExecutorService retryService; + private final ArrayList<ReferenceCountedChannel> channels = new ArrayList<>(); + private GrpcCacheClient newClient() throws IOException { return newClient(Options.getDefaults(RemoteOptions.class)); } + private GrpcCacheClient newClient(RemoteOptions remoteOptions) throws IOException { + return newClient(remoteOptions, () -> new ExponentialBackoff(remoteOptions)); + } + + private GrpcCacheClient newClient(RemoteOptions remoteOptions, Supplier<Backoff> backoffSupplier) + throws IOException { + AuthAndTLSOptions authTlsOptions = Options.getDefaults(AuthAndTLSOptions.class); + authTlsOptions.useGoogleDefaultCredentials = true; + authTlsOptions.googleCredentials = "/execroot/main/creds.json"; + authTlsOptions.googleAuthScopes = ImmutableList.of("dummy.scope"); + + JsonObject json = new JsonObject(); + json.addProperty("type", "authorized_user"); + json.addProperty("client_id", "some_client"); + json.addProperty("client_secret", "foo"); + json.addProperty("refresh_token", "bar"); + Scratch scratch = new Scratch(); + scratch.file(authTlsOptions.googleCredentials, json.toString()); + + CallCredentialsProvider callCredentialsProvider; + try (InputStream in = scratch.resolve(authTlsOptions.googleCredentials).getInputStream()) { + callCredentialsProvider = + GoogleAuthUtils.newCallCredentialsProvider( + GoogleAuthUtils.newGoogleCredentialsFromFile(in, authTlsOptions.googleAuthScopes)); + } + CallCredentials creds = callCredentialsProvider.getCallCredentials(); + + RemoteRetrier retrier = + TestUtils.newRemoteRetrier( + backoffSupplier, RemoteRetrier.RETRIABLE_GRPC_ERRORS, retryService); + ReferenceCountedChannel channel = + new ReferenceCountedChannel( + new ChannelConnectionWithServerCapabilitiesFactory() { + @Override + public Single<ChannelConnectionWithServerCapabilities> create() { + ManagedChannel ch = + InProcessChannelBuilder.forName(fakeServerName) + .directExecutor() + .intercept(new CallCredentialsInterceptor(creds)) + .intercept(TracingMetadataUtils.newCacheHeadersInterceptor(remoteOptions)) + .build(); + return Single.just( + new ChannelConnectionWithServerCapabilities( + ch, ServerCapabilities.getDefaultInstance())); + } + + @Override + public int maxConcurrency() { + return 100; + } + }); + channels.add(channel); + return new GrpcCacheClient( + channel, callCredentialsProvider, remoteOptions, retrier, DIGEST_UTIL); + } + + private static byte[] downloadBlob( + RemoteActionExecutionContext context, GrpcCacheClient cacheClient, Digest digest) + throws IOException, InterruptedException { + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + getFromFuture(cacheClient.downloadBlob(context, digest, out)); + return out.toByteArray(); + } + } + + private static class CallCredentialsInterceptor implements ClientInterceptor { + private final CallCredentials credentials; + + public CallCredentialsInterceptor(CallCredentials credentials) { + this.credentials = credentials; + } + + @Override + public <RequestT, ResponseT> ClientCall<RequestT, ResponseT> interceptCall( + MethodDescriptor<RequestT, ResponseT> method, CallOptions callOptions, Channel next) { + assertThat(callOptions.getCredentials()).isEqualTo(credentials); + // Remove the call credentials to allow testing with dummy ones. + return next.newCall(method, callOptions.withCallCredentials(null)); + } + } + + @Before + public final void setUp() throws Exception { + // Use a mutable service registry for later registering the service impl for each test case. + fakeServer = + InProcessServerBuilder.forName(fakeServerName) + .fallbackHandlerRegistry(serviceRegistry) + .directExecutor() + .build() + .start(); + Chunker.setDefaultChunkSizeForTesting(1000); // Enough for everything to be one chunk. + fs = new InMemoryFileSystem(new JavaClock(), DigestHashFunction.SHA256); + execRoot = fs.getPath("/execroot/main"); + execRoot.createDirectoryAndParents(); + fakeFileCache = new FakeActionInputFileCache(execRoot); + remotePathResolver = RemotePathResolver.createDefault(execRoot); + + Path stdout = fs.getPath("/tmp/stdout"); + Path stderr = fs.getPath("/tmp/stderr"); + stdout.getParentDirectory().createDirectoryAndParents(); + stderr.getParentDirectory().createDirectoryAndParents(); + outErr = new FileOutErr(stdout, stderr); + RequestMetadata metadata = + TracingMetadataUtils.buildMetadata( + "none", "none", Digest.getDefaultInstance().getHash(), null); + context = RemoteActionExecutionContext.create(metadata); + retryService = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1)); + } + + @After + public void tearDown() throws Exception { + channels.forEach(ReferenceCountedChannel::release); + retryService.shutdownNow(); + retryService.awaitTermination( + com.google.devtools.build.lib.testutil.TestUtils.WAIT_TIMEOUT_SECONDS, SECONDS); + + fakeServer.shutdownNow(); + fakeServer.awaitTermination(); + } + @Test public void testVirtualActionInputSupport() throws Exception { RemoteOptions options = Options.getDefaults(RemoteOptions.class); @@ -1020,6 +1197,86 @@ } @Test + public void compressedDownloadBlobIsRetriedWithProgress() + throws IOException, InterruptedException { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.cacheCompression = true; + final GrpcCacheClient client = newClient(options); + final Digest digest = DIGEST_UTIL.computeAsUtf8("abcdefg"); + ByteString chunk1 = ByteString.copyFrom(Zstd.compress("abc".getBytes(UTF_8))); + ByteString chunk2 = ByteString.copyFrom(Zstd.compress("def".getBytes(UTF_8))); + ByteString chunk3 = ByteString.copyFrom(Zstd.compress("g".getBytes(UTF_8))); + serviceRegistry.addService( + new ByteStreamImplBase() { + private boolean first = true; + + @Override + public void read(ReadRequest request, StreamObserver<ReadResponse> responseObserver) { + assertThat(request.getResourceName()).contains(digest.getHash()); + if (first) { + first = false; + responseObserver.onError(Status.DEADLINE_EXCEEDED.asException()); + return; + } + switch (Math.toIntExact(request.getReadOffset())) { + case 0: + responseObserver.onNext(ReadResponse.newBuilder().setData(chunk1).build()); + break; + case 3: + responseObserver.onNext(ReadResponse.newBuilder().setData(chunk2).build()); + break; + case 6: + responseObserver.onNext(ReadResponse.newBuilder().setData(chunk3).build()); + responseObserver.onCompleted(); + return; + default: + throw new IllegalStateException("unexpected offset " + request.getReadOffset()); + } + responseObserver.onError(Status.DEADLINE_EXCEEDED.asException()); + } + }); + assertThat(new String(downloadBlob(context, client, digest), UTF_8)).isEqualTo("abcdefg"); + } + + @Test + public void testCompressedDownload() throws IOException, InterruptedException { + RemoteOptions options = Options.getDefaults(RemoteOptions.class); + options.cacheCompression = true; + final GrpcCacheClient client = newClient(options); + final byte[] data = "abcdefg".getBytes(UTF_8); + final Digest digest = DIGEST_UTIL.compute(data); + final byte[] compressed = Zstd.compress(data); + + serviceRegistry.addService( + new ByteStreamImplBase() { + @Override + public void read(ReadRequest request, StreamObserver<ReadResponse> responseObserver) { + assertThat(request.getResourceName()).contains(digest.getHash()); + responseObserver.onNext( + ReadResponse.newBuilder() + .setData(ByteString.copyFrom(Arrays.copyOf(compressed, compressed.length / 3))) + .build()); + responseObserver.onNext( + ReadResponse.newBuilder() + .setData( + ByteString.copyFrom( + Arrays.copyOfRange( + compressed, compressed.length / 3, compressed.length / 3 * 2))) + .build()); + responseObserver.onNext( + ReadResponse.newBuilder() + .setData( + ByteString.copyFrom( + Arrays.copyOfRange( + compressed, compressed.length / 3 * 2, compressed.length))) + .build()); + responseObserver.onCompleted(); + } + }); + assertThat(downloadBlob(context, client, digest)).isEqualTo(data); + } + + @Test public void isRemoteCacheOptionsWhenGrpcEnabled() { RemoteOptions options = Options.getDefaults(RemoteOptions.class); options.remoteCache = "grpc://some-host.com";
diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTestBase.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTestBase.java deleted file mode 100644 index b674065..0000000 --- a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTestBase.java +++ /dev/null
@@ -1,204 +0,0 @@ -// Copyright 2022 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.lib.remote; - -import static com.google.common.truth.Truth.assertThat; -import static com.google.devtools.build.lib.remote.util.Utils.getFromFuture; - -import build.bazel.remote.execution.v2.Digest; -import build.bazel.remote.execution.v2.RequestMetadata; -import build.bazel.remote.execution.v2.ServerCapabilities; -import com.google.api.client.json.GenericJson; -import com.google.api.client.json.gson.GsonFactory; -import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.ListeningScheduledExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions; -import com.google.devtools.build.lib.authandtls.CallCredentialsProvider; -import com.google.devtools.build.lib.authandtls.GoogleAuthUtils; -import com.google.devtools.build.lib.clock.JavaClock; -import com.google.devtools.build.lib.remote.RemoteRetrier.ExponentialBackoff; -import com.google.devtools.build.lib.remote.Retrier.Backoff; -import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext; -import com.google.devtools.build.lib.remote.common.RemotePathResolver; -import com.google.devtools.build.lib.remote.options.RemoteOptions; -import com.google.devtools.build.lib.remote.util.DigestUtil; -import com.google.devtools.build.lib.remote.util.TestUtils; -import com.google.devtools.build.lib.remote.util.TracingMetadataUtils; -import com.google.devtools.build.lib.testutil.Scratch; -import com.google.devtools.build.lib.util.io.FileOutErr; -import com.google.devtools.build.lib.vfs.DigestHashFunction; -import com.google.devtools.build.lib.vfs.FileSystem; -import com.google.devtools.build.lib.vfs.Path; -import com.google.devtools.build.lib.vfs.SyscallCache; -import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; -import com.google.devtools.common.options.Options; -import io.grpc.CallCredentials; -import io.grpc.CallOptions; -import io.grpc.Channel; -import io.grpc.ClientCall; -import io.grpc.ClientInterceptor; -import io.grpc.ManagedChannel; -import io.grpc.MethodDescriptor; -import io.grpc.Server; -import io.grpc.inprocess.InProcessChannelBuilder; -import io.grpc.inprocess.InProcessServerBuilder; -import io.grpc.util.MutableHandlerRegistry; -import io.reactivex.rxjava3.core.Single; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; -import org.junit.After; -import org.junit.Before; - -class GrpcCacheClientTestBase { - protected static final DigestUtil DIGEST_UTIL = - new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); - - protected FileSystem fs; - protected Path execRoot; - protected FileOutErr outErr; - protected FakeActionInputFileCache fakeFileCache; - protected final MutableHandlerRegistry serviceRegistry = new MutableHandlerRegistry(); - protected final String fakeServerName = "fake server for " + getClass(); - protected Server fakeServer; - protected RemoteActionExecutionContext context; - protected RemotePathResolver remotePathResolver; - protected ListeningScheduledExecutorService retryService; - private final ArrayList<ReferenceCountedChannel> channels = new ArrayList<>(); - - @Before - public final void setUp() throws Exception { - // Use a mutable service registry for later registering the service impl for each test case. - fakeServer = - InProcessServerBuilder.forName(fakeServerName) - .fallbackHandlerRegistry(serviceRegistry) - .directExecutor() - .build() - .start(); - Chunker.setDefaultChunkSizeForTesting(1000); // Enough for everything to be one chunk. - fs = new InMemoryFileSystem(new JavaClock(), DigestHashFunction.SHA256); - execRoot = fs.getPath("/execroot/main"); - execRoot.createDirectoryAndParents(); - fakeFileCache = new FakeActionInputFileCache(execRoot); - remotePathResolver = RemotePathResolver.createDefault(execRoot); - - Path stdout = fs.getPath("/tmp/stdout"); - Path stderr = fs.getPath("/tmp/stderr"); - stdout.getParentDirectory().createDirectoryAndParents(); - stderr.getParentDirectory().createDirectoryAndParents(); - outErr = new FileOutErr(stdout, stderr); - RequestMetadata metadata = - TracingMetadataUtils.buildMetadata( - "none", "none", Digest.getDefaultInstance().getHash(), null); - context = RemoteActionExecutionContext.create(metadata); - retryService = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1)); - } - - @After - public void tearDown() throws Exception { - channels.forEach(ReferenceCountedChannel::release); - retryService.shutdownNow(); - retryService.awaitTermination( - com.google.devtools.build.lib.testutil.TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); - - fakeServer.shutdownNow(); - fakeServer.awaitTermination(); - } - - protected GrpcCacheClient newClient(RemoteOptions remoteOptions) throws IOException { - return newClient(remoteOptions, () -> new ExponentialBackoff(remoteOptions)); - } - - protected GrpcCacheClient newClient( - RemoteOptions remoteOptions, Supplier<Backoff> backoffSupplier) throws IOException { - AuthAndTLSOptions authTlsOptions = Options.getDefaults(AuthAndTLSOptions.class); - authTlsOptions.useGoogleDefaultCredentials = true; - authTlsOptions.googleCredentials = "/execroot/main/creds.json"; - authTlsOptions.googleAuthScopes = ImmutableList.of("dummy.scope"); - - GenericJson json = new GenericJson(); - json.put("type", "authorized_user"); - json.put("client_id", "some_client"); - json.put("client_secret", "foo"); - json.put("refresh_token", "bar"); - Scratch scratch = new Scratch(); - scratch.file(authTlsOptions.googleCredentials, new GsonFactory().toString(json)); - - CallCredentialsProvider callCredentialsProvider; - try (InputStream in = scratch.resolve(authTlsOptions.googleCredentials).getInputStream()) { - callCredentialsProvider = - GoogleAuthUtils.newCallCredentialsProvider( - GoogleAuthUtils.newGoogleCredentialsFromFile(in, authTlsOptions.googleAuthScopes)); - } - CallCredentials creds = callCredentialsProvider.getCallCredentials(); - - RemoteRetrier retrier = - TestUtils.newRemoteRetrier( - backoffSupplier, RemoteRetrier.RETRIABLE_GRPC_ERRORS, retryService); - ReferenceCountedChannel channel = - new ReferenceCountedChannel( - new ChannelConnectionWithServerCapabilitiesFactory() { - @Override - public Single<ChannelConnectionWithServerCapabilities> create() { - ManagedChannel ch = - InProcessChannelBuilder.forName(fakeServerName) - .directExecutor() - .intercept(new CallCredentialsInterceptor(creds)) - .intercept(TracingMetadataUtils.newCacheHeadersInterceptor(remoteOptions)) - .build(); - return Single.just( - new ChannelConnectionWithServerCapabilities( - ch, ServerCapabilities.getDefaultInstance())); - } - - @Override - public int maxConcurrency() { - return 100; - } - }); - channels.add(channel); - return new GrpcCacheClient( - channel, callCredentialsProvider, remoteOptions, retrier, DIGEST_UTIL); - } - - protected static byte[] downloadBlob( - RemoteActionExecutionContext context, GrpcCacheClient cacheClient, Digest digest) - throws IOException, InterruptedException { - try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - getFromFuture(cacheClient.downloadBlob(context, digest, out)); - return out.toByteArray(); - } - } - - private static class CallCredentialsInterceptor implements ClientInterceptor { - private final CallCredentials credentials; - - public CallCredentialsInterceptor(CallCredentials credentials) { - this.credentials = credentials; - } - - @Override - public <RequestT, ResponseT> ClientCall<RequestT, ResponseT> interceptCall( - MethodDescriptor<RequestT, ResponseT> method, CallOptions callOptions, Channel next) { - assertThat(callOptions.getCredentials()).isEqualTo(credentials); - // Remove the call credentials to allow testing with dummy ones. - return next.newCall(method, callOptions.withCallCredentials(null)); - } - } -}
diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTestExtra.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTestExtra.java deleted file mode 100644 index b840907..0000000 --- a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTestExtra.java +++ /dev/null
@@ -1,117 +0,0 @@ -// Copyright 2021 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.lib.remote; - -import static com.google.common.truth.Truth.assertThat; -import static java.nio.charset.StandardCharsets.UTF_8; - -import build.bazel.remote.execution.v2.Digest; -import com.github.luben.zstd.Zstd; -import com.google.bytestream.ByteStreamGrpc.ByteStreamImplBase; -import com.google.bytestream.ByteStreamProto.ReadRequest; -import com.google.bytestream.ByteStreamProto.ReadResponse; -import com.google.devtools.build.lib.remote.options.RemoteOptions; -import com.google.devtools.common.options.Options; -import com.google.protobuf.ByteString; -import io.grpc.Status; -import io.grpc.stub.StreamObserver; -import java.io.IOException; -import java.util.Arrays; -import org.junit.Test; - -/** Extra tests for {@link GrpcCacheClient} that are not tested internally. */ -public class GrpcCacheClientTestExtra extends GrpcCacheClientTestBase { - - @Test - public void compressedDownloadBlobIsRetriedWithProgress() - throws IOException, InterruptedException { - RemoteOptions options = Options.getDefaults(RemoteOptions.class); - options.cacheCompression = true; - final GrpcCacheClient client = newClient(options); - final Digest digest = DIGEST_UTIL.computeAsUtf8("abcdefg"); - ByteString chunk1 = ByteString.copyFrom(Zstd.compress("abc".getBytes(UTF_8))); - ByteString chunk2 = ByteString.copyFrom(Zstd.compress("def".getBytes(UTF_8))); - ByteString chunk3 = ByteString.copyFrom(Zstd.compress("g".getBytes(UTF_8))); - serviceRegistry.addService( - new ByteStreamImplBase() { - private boolean first = true; - - @Override - public void read(ReadRequest request, StreamObserver<ReadResponse> responseObserver) { - assertThat(request.getResourceName().contains(digest.getHash())).isTrue(); - if (first) { - first = false; - responseObserver.onError(Status.DEADLINE_EXCEEDED.asException()); - return; - } - switch (Math.toIntExact(request.getReadOffset())) { - case 0: - responseObserver.onNext(ReadResponse.newBuilder().setData(chunk1).build()); - break; - case 3: - responseObserver.onNext(ReadResponse.newBuilder().setData(chunk2).build()); - break; - case 6: - responseObserver.onNext(ReadResponse.newBuilder().setData(chunk3).build()); - responseObserver.onCompleted(); - return; - default: - throw new IllegalStateException("unexpected offset " + request.getReadOffset()); - } - responseObserver.onError(Status.DEADLINE_EXCEEDED.asException()); - } - }); - assertThat(new String(downloadBlob(context, client, digest), UTF_8)).isEqualTo("abcdefg"); - } - - @Test - public void testCompressedDownload() throws IOException, InterruptedException { - RemoteOptions options = Options.getDefaults(RemoteOptions.class); - options.cacheCompression = true; - final GrpcCacheClient client = newClient(options); - final byte[] data = "abcdefg".getBytes(UTF_8); - final Digest digest = DIGEST_UTIL.compute(data); - final byte[] compressed = Zstd.compress(data); - - serviceRegistry.addService( - new ByteStreamImplBase() { - @Override - public void read(ReadRequest request, StreamObserver<ReadResponse> responseObserver) { - assertThat(request.getResourceName().contains(digest.getHash())).isTrue(); - responseObserver.onNext( - ReadResponse.newBuilder() - .setData( - ByteString.copyFrom( - Arrays.copyOfRange(compressed, 0, compressed.length / 3))) - .build()); - responseObserver.onNext( - ReadResponse.newBuilder() - .setData( - ByteString.copyFrom( - Arrays.copyOfRange( - compressed, compressed.length / 3, compressed.length / 3 * 2))) - .build()); - responseObserver.onNext( - ReadResponse.newBuilder() - .setData( - ByteString.copyFrom( - Arrays.copyOfRange( - compressed, compressed.length / 3 * 2, compressed.length))) - .build()); - responseObserver.onCompleted(); - } - }); - assertThat(downloadBlob(context, client, digest)).isEqualTo(data); - } -}
diff --git a/src/test/java/com/google/devtools/build/lib/remote/NativeSslTest.java b/src/test/java/com/google/devtools/build/lib/remote/NativeSslTest.java index 05da702..217b6ca 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/NativeSslTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/NativeSslTest.java
@@ -13,6 +13,10 @@ // limitations under the License. package com.google.devtools.build.lib.remote; +import static org.junit.Assume.assumeTrue; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.util.OS; import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.ssl.SslProvider; import org.junit.Test; @@ -22,8 +26,14 @@ /** Tests that we use OpenSSL instead of a Java implementation. */ @RunWith(JUnit4.class) public class NativeSslTest { + private static final ImmutableSet<OS> OS_WITH_NATIVE_SSL = + ImmutableSet.of(OS.LINUX, OS.DARWIN, OS.WINDOWS); + @Test public void nativeSslPresent() throws Exception { + // Skip the test on platforms where native SSL is not available. + assumeTrue(OS_WITH_NATIVE_SSL.contains(OS.getCurrent())); + SslContextBuilder.forClient().sslProvider(SslProvider.OPENSSL).build(); } }
diff --git a/src/test/java/com/google/devtools/build/lib/remote/downloader/BUILD b/src/test/java/com/google/devtools/build/lib/remote/downloader/BUILD index 8e2cae2..d49a5aa 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/downloader/BUILD +++ b/src/test/java/com/google/devtools/build/lib/remote/downloader/BUILD
@@ -28,7 +28,6 @@ "//src/main/java/com/google/devtools/build/lib/remote", "//src/main/java/com/google/devtools/build/lib/remote/common", "//src/main/java/com/google/devtools/build/lib/remote/downloader", - "//src/main/java/com/google/devtools/build/lib/remote/grpc", "//src/main/java/com/google/devtools/build/lib/remote/options", "//src/main/java/com/google/devtools/build/lib/remote/util", "//src/main/java/com/google/devtools/build/lib/vfs",
diff --git a/src/test/java/com/google/devtools/build/lib/remote/zstd/BUILD b/src/test/java/com/google/devtools/build/lib/remote/zstd/BUILD index 58647b4..4f80374 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/zstd/BUILD +++ b/src/test/java/com/google/devtools/build/lib/remote/zstd/BUILD
@@ -1,12 +1,11 @@ load("@rules_java//java:defs.bzl", "java_test") package( + default_applicable_licenses = ["//:license"], default_testonly = 1, default_visibility = ["//src:__subpackages__"], ) -licenses(["notice"]) - filegroup( name = "srcs", testonly = 0, @@ -24,6 +23,6 @@ "//third_party:guava", "//third_party:junit4", "//third_party:truth", - "@zstd-jni//:zstd-jni", + "@zstd-jni", ], )
diff --git a/src/test/java/com/google/devtools/build/lib/remote/zstd/ZstdDecompressingOutputStreamTest.java b/src/test/java/com/google/devtools/build/lib/remote/zstd/ZstdDecompressingOutputStreamTest.java index 377bb41..4768d9f 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/zstd/ZstdDecompressingOutputStreamTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/zstd/ZstdDecompressingOutputStreamTest.java
@@ -14,8 +14,10 @@ package com.google.devtools.build.lib.remote.zstd; import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdOutputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Random; @@ -41,4 +43,48 @@ assertThat(baos.toByteArray()).isEqualTo(data); } + + @Test + public void streamCanBeDecompressedOneByteAtATime() throws IOException { + Random rand = new Random(); + byte[] data = new byte[50]; + rand.nextBytes(data); + byte[] compressed = Zstd.compress(data); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZstdDecompressingOutputStream zdos = new ZstdDecompressingOutputStream(baos)) { + for (byte b : compressed) { + zdos.write(b); + } + zdos.flush(); + } + + assertThat(baos.toByteArray()).isEqualTo(data); + } + + @Test + public void bytesWrittenMatchesDecompressedBytes() throws IOException { + byte[] data = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA".getBytes(UTF_8); + + ByteArrayOutputStream compressed = new ByteArrayOutputStream(); + try (ZstdOutputStream zos = new ZstdOutputStream(compressed)) { + zos.setCloseFrameOnFlush(true); + for (int i = 0; i < data.length; i++) { + zos.write(data[i]); + if (i % 5 == 0) { + // Create multiple frames of 5 bytes each. + zos.flush(); + } + } + } + + ByteArrayOutputStream decompressed = new ByteArrayOutputStream(); + try (ZstdDecompressingOutputStream zdos = new ZstdDecompressingOutputStream(decompressed)) { + for (byte b : compressed.toByteArray()) { + zdos.write(b); + zdos.flush(); + } + } + assertThat(decompressed.toByteArray()).isEqualTo(data); + } }
diff --git a/src/test/java/com/google/devtools/build/lib/remote/zstd/ZstdDecompressingOutputStreamTestExtra.java b/src/test/java/com/google/devtools/build/lib/remote/zstd/ZstdDecompressingOutputStreamTestExtra.java deleted file mode 100644 index d5511ee..0000000 --- a/src/test/java/com/google/devtools/build/lib/remote/zstd/ZstdDecompressingOutputStreamTestExtra.java +++ /dev/null
@@ -1,71 +0,0 @@ -// Copyright 2021 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.lib.remote.zstd; - -import static com.google.common.truth.Truth.assertThat; -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.github.luben.zstd.Zstd; -import com.github.luben.zstd.ZstdOutputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.Random; -import org.junit.Test; - -/** Extra tests for {@link ZstdDecompressingOutputStream} that are not tested internally. */ -public class ZstdDecompressingOutputStreamTestExtra { - @Test - public void streamCanBeDecompressedOneByteAtATime() throws IOException { - Random rand = new Random(); - byte[] data = new byte[50]; - rand.nextBytes(data); - byte[] compressed = Zstd.compress(data); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (ZstdDecompressingOutputStream zdos = new ZstdDecompressingOutputStream(baos)) { - for (byte b : compressed) { - zdos.write(b); - } - zdos.flush(); - } - - assertThat(baos.toByteArray()).isEqualTo(data); - } - - @Test - public void bytesWrittenMatchesDecompressedBytes() throws IOException { - byte[] data = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA".getBytes(UTF_8); - - ByteArrayOutputStream compressed = new ByteArrayOutputStream(); - try (ZstdOutputStream zos = new ZstdOutputStream(compressed)) { - zos.setCloseFrameOnFlush(true); - for (int i = 0; i < data.length; i++) { - zos.write(data[i]); - if (i % 5 == 0) { - // Create multiple frames of 5 bytes each. - zos.flush(); - } - } - } - - ByteArrayOutputStream decompressed = new ByteArrayOutputStream(); - try (ZstdDecompressingOutputStream zdos = new ZstdDecompressingOutputStream(decompressed)) { - for (byte b : compressed.toByteArray()) { - zdos.write(b); - zdos.flush(); - } - } - assertThat(decompressed.toByteArray()).isEqualTo(data); - } -}