| // Copyright 2018 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| package com.google.devtools.build.lib.remote; |
| |
| import static com.google.common.truth.Truth.assertThat; |
| import static com.google.devtools.build.lib.remote.util.Utils.getFromFuture; |
| import static com.google.devtools.build.lib.remote.util.Utils.waitForBulkTransfer; |
| import static org.junit.Assert.assertThrows; |
| import static org.mockito.ArgumentMatchers.any; |
| import static org.mockito.Mockito.doAnswer; |
| import static org.mockito.Mockito.mock; |
| import static org.mockito.Mockito.spy; |
| import static org.mockito.Mockito.verify; |
| import static org.mockito.Mockito.when; |
| |
| import build.bazel.remote.execution.v2.ActionResult; |
| import build.bazel.remote.execution.v2.CacheCapabilities; |
| import build.bazel.remote.execution.v2.Digest; |
| import build.bazel.remote.execution.v2.FastCdc2020Params; |
| import build.bazel.remote.execution.v2.RequestMetadata; |
| import build.bazel.remote.execution.v2.ServerCapabilities; |
| import com.google.common.collect.ImmutableList; |
| import com.google.common.collect.ImmutableMap; |
| import com.google.common.collect.ImmutableSet; |
| import com.google.common.collect.ImmutableSortedMap; |
| import com.google.common.collect.Maps; |
| import com.google.common.util.concurrent.Futures; |
| import com.google.common.util.concurrent.ListenableFuture; |
| import com.google.common.util.concurrent.ListeningScheduledExecutorService; |
| import com.google.common.util.concurrent.MoreExecutors; |
| import com.google.common.util.concurrent.SettableFuture; |
| import com.google.devtools.build.lib.actions.ActionInputHelper; |
| import com.google.devtools.build.lib.actions.Artifact; |
| import com.google.devtools.build.lib.actions.ArtifactRoot; |
| import com.google.devtools.build.lib.actions.ArtifactRoot.RootType; |
| import com.google.devtools.build.lib.actions.ResourceSet; |
| import com.google.devtools.build.lib.actions.SimpleSpawn; |
| import com.google.devtools.build.lib.actions.Spawn; |
| import com.google.devtools.build.lib.actions.util.ActionsTestUtil; |
| import com.google.devtools.build.lib.authandtls.CallCredentialsProvider; |
| import com.google.devtools.build.lib.clock.JavaClock; |
| import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; |
| import com.google.devtools.build.lib.collect.nestedset.Order; |
| import com.google.devtools.build.lib.exec.SpawnCheckingCacheEvent; |
| import com.google.devtools.build.lib.exec.SpawnRunner.SpawnExecutionContext; |
| import com.google.devtools.build.lib.exec.util.FakeOwner; |
| import com.google.devtools.build.lib.exec.util.SpawnBuilder; |
| import com.google.devtools.build.lib.remote.common.BulkTransferException; |
| import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext; |
| import com.google.devtools.build.lib.remote.common.RemoteCacheClient; |
| import com.google.devtools.build.lib.remote.common.RemoteCacheClient.Blob; |
| import com.google.devtools.build.lib.remote.common.RemotePathResolver; |
| import com.google.devtools.build.lib.remote.merkletree.MerkleTree; |
| import com.google.devtools.build.lib.remote.options.RemoteOptions; |
| import com.google.devtools.build.lib.remote.util.DigestUtil; |
| import com.google.devtools.build.lib.remote.util.FakeSpawnExecutionContext; |
| import com.google.devtools.build.lib.remote.util.InMemoryCacheClient; |
| import com.google.devtools.build.lib.remote.util.RxNoGlobalErrorsRule; |
| import com.google.devtools.build.lib.remote.util.TracingMetadataUtils; |
| import com.google.devtools.build.lib.testutil.TestUtils; |
| import com.google.devtools.build.lib.util.io.FileOutErr; |
| import com.google.devtools.build.lib.vfs.DigestHashFunction; |
| import com.google.devtools.build.lib.vfs.FileSystem; |
| import com.google.devtools.build.lib.vfs.FileSystemUtils; |
| import com.google.devtools.build.lib.vfs.Path; |
| import com.google.devtools.build.lib.vfs.PathFragment; |
| import com.google.devtools.build.lib.vfs.SyscallCache; |
| import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; |
| import com.google.devtools.common.options.Options; |
| import com.google.protobuf.ByteString; |
| import java.io.IOException; |
| import java.io.OutputStream; |
| import java.nio.charset.StandardCharsets; |
| import java.util.Deque; |
| import java.util.Map; |
| import java.util.SortedMap; |
| import java.util.TreeMap; |
| import java.util.concurrent.ConcurrentHashMap; |
| import java.util.concurrent.ConcurrentLinkedDeque; |
| import java.util.concurrent.ConcurrentMap; |
| import java.util.concurrent.CountDownLatch; |
| import java.util.concurrent.Executors; |
| import java.util.concurrent.TimeUnit; |
| import java.util.concurrent.atomic.AtomicBoolean; |
| import java.util.concurrent.atomic.AtomicInteger; |
| import org.junit.After; |
| import org.junit.Before; |
| import org.junit.Rule; |
| import org.junit.Test; |
| import org.junit.runner.RunWith; |
| import org.junit.runners.JUnit4; |
| import org.mockito.MockitoAnnotations; |
| |
| /** Tests for {@link CombinedCache}. */ |
| @RunWith(JUnit4.class) |
| public class CombinedCacheTest { |
| @Rule public final RxNoGlobalErrorsRule rxNoGlobalErrorsRule = new RxNoGlobalErrorsRule(); |
| |
| private RemoteActionExecutionContext remoteActionExecutionContext; |
| private FileSystem fs; |
| private Path execRoot; |
| ArtifactRoot artifactRoot; |
| private final DigestUtil digestUtil = |
| new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256); |
| private FakeActionInputFileCache fakeFileCache; |
| |
| private ListeningScheduledExecutorService retryService; |
| |
| @Before |
| public void setUp() throws Exception { |
| MockitoAnnotations.initMocks(this); |
| RequestMetadata metadata = |
| TracingMetadataUtils.buildMetadata("none", "none", "action-id", null); |
| Spawn spawn = |
| new SimpleSpawn( |
| new FakeOwner("foo", "bar", "//dummy:label"), |
| /* arguments= */ ImmutableList.of(), |
| /* environment= */ ImmutableMap.of(), |
| /* executionInfo= */ ImmutableMap.of(), |
| /* inputs= */ NestedSetBuilder.emptySet(Order.STABLE_ORDER), |
| /* outputs= */ ImmutableSet.of(), |
| ResourceSet.ZERO); |
| SpawnExecutionContext spawnExecutionContext = mock(SpawnExecutionContext.class); |
| remoteActionExecutionContext = |
| RemoteActionExecutionContext.create(spawn, spawnExecutionContext, metadata); |
| fs = new InMemoryFileSystem(new JavaClock(), DigestHashFunction.SHA256); |
| execRoot = fs.getPath("/execroot/main"); |
| execRoot.createDirectoryAndParents(); |
| fakeFileCache = new FakeActionInputFileCache(execRoot); |
| artifactRoot = ArtifactRoot.asDerivedRoot(execRoot, RootType.Output, "outputs"); |
| artifactRoot.getRoot().asPath().createDirectoryAndParents(); |
| retryService = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1)); |
| } |
| |
| @After |
| public void afterEverything() throws InterruptedException { |
| retryService.shutdownNow(); |
| retryService.awaitTermination(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); |
| } |
| |
| @Test |
| public void testDownloadEmptyBlobAndFile() throws Exception { |
| // Test that downloading an empty BLOB/file does not try to perform a download. |
| |
| // arrange |
| Path file = fs.getPath("/execroot/file"); |
| InMemoryCombinedCache combinedCache = newCombinedCache(); |
| Digest emptyDigest = digestUtil.compute(new byte[0]); |
| |
| // act and assert |
| assertThat(getFromFuture(combinedCache.downloadBlob(remoteActionExecutionContext, emptyDigest))) |
| .isEmpty(); |
| |
| try (OutputStream out = file.getOutputStream()) { |
| getFromFuture(combinedCache.downloadFile(remoteActionExecutionContext, file, emptyDigest)); |
| } |
| assertThat(file.exists()).isTrue(); |
| assertThat(file.getFileSize()).isEqualTo(0); |
| } |
| |
| @Test |
| public void downloadActionResult_reportsSpawnCheckingCacheEvent() throws Exception { |
| var combinedCache = newCombinedCache(); |
| var unused = |
| combinedCache.downloadActionResult( |
| remoteActionExecutionContext, |
| digestUtil.asActionKey(digestUtil.computeAsUtf8("key")), |
| /* inlineOutErr= */ false, |
| /* inlineOutputFiles= */ ImmutableSet.of()); |
| |
| verify(remoteActionExecutionContext.getSpawnExecutionContext()) |
| .report(SpawnCheckingCacheEvent.create("remote-cache")); |
| } |
| |
| @Test |
| public void downloadFile_cancelled_cancelDownload() throws Exception { |
| // Test that if a download future is cancelled, the download itself is also cancelled. |
| |
| // arrange |
| RemoteCacheClient remoteCacheClient = mock(RemoteCacheClient.class); |
| SettableFuture<Void> future = SettableFuture.create(); |
| // Return a future that never completes |
| doAnswer(invocationOnMock -> future).when(remoteCacheClient).downloadBlob(any(), any(), any()); |
| CombinedCache remoteCache = newCombinedCache(remoteCacheClient); |
| Digest digest = fakeFileCache.createScratchInput(ActionInputHelper.fromPath("file"), "content"); |
| Path file = execRoot.getRelative("file"); |
| |
| // act |
| ListenableFuture<Void> download = |
| remoteCache.downloadFile(remoteActionExecutionContext, file, digest); |
| download.cancel(/* mayInterruptIfRunning= */ true); |
| |
| // assert |
| assertThat(future.isCancelled()).isTrue(); |
| } |
| |
| @Test |
| public void downloadOutErr_empty_doNotPerformDownload() throws Exception { |
| // Test that downloading empty stdout/stderr does not try to perform a download. |
| |
| InMemoryCombinedCache combinedCache = newCombinedCache(); |
| Digest emptyDigest = digestUtil.compute(new byte[0]); |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| result.setStdoutDigest(emptyDigest); |
| result.setStderrDigest(emptyDigest); |
| |
| waitForBulkTransfer( |
| combinedCache.downloadOutErr( |
| remoteActionExecutionContext, |
| result.build(), |
| new FileOutErr(execRoot.getRelative("stdout"), execRoot.getRelative("stderr")))); |
| |
| assertThat(combinedCache.getNumSuccessfulDownloads()).isEqualTo(0); |
| assertThat(combinedCache.getNumFailedDownloads()).isEqualTo(0); |
| } |
| |
| @Test |
| public void testDownloadFileWithSymlinkTemplate() throws Exception { |
| // Test that when a symlink template is provided, we don't actually download files to disk. |
| // Instead, a symbolic link should be created that points to a location where the file may |
| // actually be found. That location could, for example, be backed by a FUSE file system that |
| // exposes the Content Addressable Storage. |
| |
| // arrange |
| final ConcurrentMap<Digest, byte[]> cas = new ConcurrentHashMap<>(); |
| |
| Digest helloDigest = digestUtil.computeAsUtf8("hello-contents"); |
| cas.put(helloDigest, "hello-contents".getBytes(StandardCharsets.UTF_8)); |
| |
| Path file = fs.getPath("/execroot/symlink-to-file"); |
| RemoteOptions options = Options.getDefaults(RemoteOptions.class); |
| options.remoteDownloadSymlinkTemplate = "/home/alice/cas/{hash}-{size_bytes}"; |
| InMemoryCombinedCache combinedCache = newCombinedCache(cas, options, digestUtil); |
| |
| // act |
| getFromFuture(combinedCache.downloadFile(remoteActionExecutionContext, file, helloDigest)); |
| |
| // assert |
| assertThat(file.isSymbolicLink()).isTrue(); |
| assertThat(file.readSymbolicLink()) |
| .isEqualTo( |
| PathFragment.create( |
| "/home/alice/cas/a378b939ad2e1d470a9a28b34b0e256b189e85cb236766edc1d46ec3b6ca82e5-14")); |
| } |
| |
| @Test |
| public void upload_emptyBlobAndFile_doNotPerformUpload() throws Exception { |
| // Test that uploading an empty BLOB/file does not try to perform an upload. |
| InMemoryCombinedCache combinedCache = newCombinedCache(); |
| Digest emptyDigest = fakeFileCache.createScratchInput(ActionInputHelper.fromPath("file"), ""); |
| Path file = execRoot.getRelative("file"); |
| |
| getFromFuture( |
| combinedCache.uploadBlob(remoteActionExecutionContext, emptyDigest, ByteString.EMPTY)); |
| assertThat( |
| getFromFuture( |
| combinedCache.findMissingDigests( |
| remoteActionExecutionContext, ImmutableSet.of(emptyDigest)))) |
| .containsExactly(emptyDigest); |
| |
| getFromFuture(combinedCache.uploadFile(remoteActionExecutionContext, emptyDigest, file)); |
| assertThat( |
| getFromFuture( |
| combinedCache.findMissingDigests( |
| remoteActionExecutionContext, ImmutableSet.of(emptyDigest)))) |
| .containsExactly(emptyDigest); |
| } |
| |
| @Test |
| @SuppressWarnings("FutureReturnValueIgnored") |
| public void upload_deduplicationWorks() throws IOException { |
| RemoteCacheClient remoteCacheClient = spy(new InMemoryCacheClient()); |
| AtomicInteger times = new AtomicInteger(0); |
| doAnswer( |
| invocationOnMock -> { |
| times.incrementAndGet(); |
| return SettableFuture.create(); |
| }) |
| .when(remoteCacheClient) |
| .uploadBlobImpl(any(), any(), any()); |
| CombinedCache combinedCache = newCombinedCache(remoteCacheClient); |
| Digest digest = fakeFileCache.createScratchInput(ActionInputHelper.fromPath("file"), "content"); |
| Path file = execRoot.getRelative("file"); |
| |
| var unused1 = combinedCache.uploadFile(remoteActionExecutionContext, digest, file); |
| var unused2 = combinedCache.uploadFile(remoteActionExecutionContext, digest, file); |
| |
| assertThat(times.get()).isEqualTo(1); |
| } |
| |
| @Test |
| public void upload_failedUploads_doNotDeduplicate() throws Exception { |
| AtomicBoolean failRequest = new AtomicBoolean(true); |
| RemoteCacheClient remoteCacheClient = spy(new InMemoryCacheClient()); |
| doAnswer( |
| invocationOnMock -> { |
| if (failRequest.getAndSet(false)) { |
| return Futures.immediateFailedFuture(new IOException("Failed")); |
| } |
| return invocationOnMock.callRealMethod(); |
| }) |
| .when(remoteCacheClient) |
| .uploadBlobImpl(any(), any(), any()); |
| CombinedCache combinedCache = newCombinedCache(remoteCacheClient); |
| Digest digest = fakeFileCache.createScratchInput(ActionInputHelper.fromPath("file"), "content"); |
| Path file = execRoot.getRelative("file"); |
| assertThat( |
| getFromFuture( |
| combinedCache.findMissingDigests( |
| remoteActionExecutionContext, ImmutableList.of(digest)))) |
| .containsExactly(digest); |
| |
| Exception thrown = null; |
| try { |
| getFromFuture(combinedCache.uploadFile(remoteActionExecutionContext, digest, file)); |
| } catch (IOException e) { |
| thrown = e; |
| } |
| assertThat(thrown).isNotNull(); |
| assertThat(thrown).isInstanceOf(IOException.class); |
| getFromFuture(combinedCache.uploadFile(remoteActionExecutionContext, digest, file)); |
| |
| assertThat( |
| getFromFuture( |
| combinedCache.findMissingDigests( |
| remoteActionExecutionContext, ImmutableList.of(digest)))) |
| .isEmpty(); |
| } |
| |
| @Test |
| public void ensureInputsPresent_missingInputs_exceptionHasLostInputs() throws Exception { |
| RemoteCacheClient cacheProtocol = spy(new InMemoryCacheClient()); |
| RemoteExecutionCache remoteCache = spy(newRemoteExecutionCache(cacheProtocol)); |
| remoteCache.setRemotePathChecker( |
| (context, path) -> path.relativeTo(execRoot).equals(PathFragment.create("foo"))); |
| |
| Path path = execRoot.getRelative("foo"); |
| FileSystemUtils.writeContentAsLatin1(path, "bar"); |
| SortedMap<PathFragment, Path> inputs = new TreeMap<>(); |
| inputs.put(PathFragment.create("foo"), path); |
| MerkleTree merkleTree = MerkleTree.build(inputs, digestUtil); |
| path.delete(); |
| |
| var e = |
| assertThrows( |
| BulkTransferException.class, |
| () -> |
| remoteCache.ensureInputsPresent( |
| remoteActionExecutionContext, |
| merkleTree, |
| ImmutableMap.of(), |
| false, |
| new RemotePathResolver.DefaultRemotePathResolver(execRoot))); |
| assertThat(e.getLostInputs(ActionInputHelper::fromPath)) |
| .containsExactly( |
| DigestUtil.toString(digestUtil.computeAsUtf8("bar")), |
| ActionInputHelper.fromPath("foo")); |
| } |
| |
| @Test |
| public void ensureInputsPresent_interruptedDuringUploadBlobs_cancelInProgressUploadTasks() |
| throws Exception { |
| // arrange |
| RemoteCacheClient cacheProtocol = spy(new InMemoryCacheClient()); |
| RemoteExecutionCache remoteCache = spy(newRemoteExecutionCache(cacheProtocol)); |
| |
| Deque<SettableFuture<Void>> futures = new ConcurrentLinkedDeque<>(); |
| CountDownLatch uploadBlobCalls = new CountDownLatch(2); |
| doAnswer( |
| invocationOnMock -> { |
| SettableFuture<Void> future = SettableFuture.create(); |
| futures.add(future); |
| uploadBlobCalls.countDown(); |
| return future; |
| }) |
| .when(cacheProtocol) |
| .uploadBlobImpl(any(), any(), (Blob) any()); |
| doAnswer( |
| invocationOnMock -> { |
| SettableFuture<Void> future = SettableFuture.create(); |
| futures.add(future); |
| uploadBlobCalls.countDown(); |
| return future; |
| }) |
| .when(cacheProtocol) |
| .uploadBlobImpl(any(), any(), any()); |
| |
| Path path = execRoot.getRelative("foo"); |
| FileSystemUtils.writeContentAsLatin1(path, "bar"); |
| SortedMap<PathFragment, Path> inputs = new TreeMap<>(); |
| inputs.put(PathFragment.create("foo"), path); |
| MerkleTree merkleTree = MerkleTree.build(inputs, digestUtil); |
| |
| CountDownLatch ensureInputsPresentReturned = new CountDownLatch(1); |
| Thread thread = |
| new Thread( |
| () -> { |
| try { |
| remoteCache.ensureInputsPresent( |
| remoteActionExecutionContext, |
| merkleTree, |
| ImmutableMap.of(), |
| false, |
| /* remotePathResolver= */ null); |
| } catch (IOException | InterruptedException ignored) { |
| // ignored |
| } finally { |
| ensureInputsPresentReturned.countDown(); |
| } |
| }); |
| |
| // act |
| thread.start(); |
| uploadBlobCalls.await(); |
| assertThat(futures).hasSize(2); |
| assertThat(cacheProtocol.getInProgressUploads()).isNotEmpty(); |
| |
| thread.interrupt(); |
| ensureInputsPresentReturned.await(); |
| |
| // assert |
| assertThat(cacheProtocol.getInProgressUploads()).isEmpty(); |
| assertThat(cacheProtocol.getFinishedUploads()).isEmpty(); |
| for (SettableFuture<Void> future : futures) { |
| assertThat(future.isCancelled()).isTrue(); |
| } |
| } |
| |
| @Test |
| public void |
| ensureInputsPresent_multipleConsumers_interruptedOneDuringFindMissingBlobs_keepAndFinishInProgressUploadTasks() |
| throws Exception { |
| // arrange |
| RemoteCacheClient cacheProtocol = spy(new InMemoryCacheClient()); |
| RemoteExecutionCache remoteCache = newRemoteExecutionCache(cacheProtocol); |
| |
| SettableFuture<ImmutableSet<Digest>> findMissingDigestsFuture = SettableFuture.create(); |
| CountDownLatch findMissingDigestsCalled = new CountDownLatch(1); |
| doAnswer( |
| invocationOnMock -> { |
| findMissingDigestsCalled.countDown(); |
| return findMissingDigestsFuture; |
| }) |
| .when(cacheProtocol) |
| .findMissingDigests(any(), any()); |
| Deque<SettableFuture<Void>> futures = new ConcurrentLinkedDeque<>(); |
| CountDownLatch uploadBlobCalls = new CountDownLatch(2); |
| doAnswer( |
| invocationOnMock -> { |
| SettableFuture<Void> future = SettableFuture.create(); |
| futures.add(future); |
| uploadBlobCalls.countDown(); |
| return future; |
| }) |
| .when(cacheProtocol) |
| .uploadBlobImpl(any(), any(), (Blob) any()); |
| doAnswer( |
| invocationOnMock -> { |
| SettableFuture<Void> future = SettableFuture.create(); |
| futures.add(future); |
| uploadBlobCalls.countDown(); |
| return future; |
| }) |
| .when(cacheProtocol) |
| .uploadBlobImpl(any(), any(), any()); |
| |
| Path path = execRoot.getRelative("foo"); |
| FileSystemUtils.writeContentAsLatin1(path, "bar"); |
| SortedMap<PathFragment, Path> inputs = new TreeMap<>(); |
| inputs.put(PathFragment.create("foo"), path); |
| MerkleTree merkleTree = MerkleTree.build(inputs, digestUtil); |
| |
| CountDownLatch ensureInputsPresentReturned = new CountDownLatch(2); |
| CountDownLatch ensureInterrupted = new CountDownLatch(1); |
| Runnable work = |
| () -> { |
| try { |
| remoteCache.ensureInputsPresent( |
| remoteActionExecutionContext, |
| merkleTree, |
| ImmutableMap.of(), |
| false, |
| /* remotePathResolver= */ null); |
| } catch (IOException ignored) { |
| // ignored |
| } catch (InterruptedException e) { |
| ensureInterrupted.countDown(); |
| } finally { |
| ensureInputsPresentReturned.countDown(); |
| } |
| }; |
| Thread thread1 = new Thread(work); |
| Thread thread2 = new Thread(work); |
| thread1.start(); |
| thread2.start(); |
| findMissingDigestsCalled.await(); |
| |
| // act |
| thread1.interrupt(); |
| ensureInterrupted.await(); |
| findMissingDigestsFuture.set(ImmutableSet.copyOf(merkleTree.getAllDigests())); |
| |
| uploadBlobCalls.await(); |
| assertThat(futures).hasSize(2); |
| |
| // assert |
| assertThat(cacheProtocol.getInProgressUploads()).hasSize(2); |
| assertThat(cacheProtocol.getFinishedUploads()).isEmpty(); |
| for (SettableFuture<Void> future : futures) { |
| assertThat(future.isCancelled()).isFalse(); |
| } |
| |
| for (SettableFuture<Void> future : futures) { |
| future.set(null); |
| } |
| ensureInputsPresentReturned.await(); |
| assertThat(cacheProtocol.getInProgressUploads()).isEmpty(); |
| assertThat(cacheProtocol.getFinishedUploads()).hasSize(2); |
| } |
| |
| @Test |
| public void |
| ensureInputsPresent_multipleConsumers_interruptedOneDuringUploadBlobs_keepInProgressUploadTasks() |
| throws Exception { |
| // arrange |
| RemoteCacheClient cacheProtocol = spy(new InMemoryCacheClient()); |
| RemoteExecutionCache remoteCache = spy(newRemoteExecutionCache(cacheProtocol)); |
| |
| Map<Digest, SettableFuture<Void>> uploadFutures = Maps.newConcurrentMap(); |
| // 3 unique file digests + 2 unique directory blob digests = 5 uploads total. |
| CountDownLatch uploadCalls = new CountDownLatch(5); |
| doAnswer( |
| invocationOnMock -> { |
| Digest digest = invocationOnMock.getArgument(1, Digest.class); |
| SettableFuture<Void> future = SettableFuture.create(); |
| uploadFutures.put(digest, future); |
| uploadCalls.countDown(); |
| return future; |
| }) |
| .when(cacheProtocol) |
| .uploadBlobImpl(any(), any(), (Blob) any()); |
| |
| Path foo = execRoot.getRelative("foo"); |
| FileSystemUtils.writeContentAsLatin1(foo, "foo"); |
| Path bar = execRoot.getRelative("bar"); |
| FileSystemUtils.writeContentAsLatin1(bar, "bar"); |
| Path qux = execRoot.getRelative("qux"); |
| FileSystemUtils.writeContentAsLatin1(qux, "qux"); |
| Digest fooDigest = digestUtil.computeAsUtf8("foo"); |
| Digest barDigest = digestUtil.computeAsUtf8("bar"); |
| Digest quxDigest = digestUtil.computeAsUtf8("qux"); |
| |
| SortedMap<PathFragment, Path> input1 = new TreeMap<>(); |
| input1.put(PathFragment.create("foo"), foo); |
| input1.put(PathFragment.create("bar"), bar); |
| MerkleTree merkleTree1 = MerkleTree.build(input1, digestUtil); |
| |
| SortedMap<PathFragment, Path> input2 = new TreeMap<>(); |
| input2.put(PathFragment.create("bar"), bar); |
| input2.put(PathFragment.create("qux"), qux); |
| MerkleTree merkleTree2 = MerkleTree.build(input2, digestUtil); |
| |
| CountDownLatch ensureInputsPresentReturned = new CountDownLatch(2); |
| CountDownLatch ensureInterrupted = new CountDownLatch(1); |
| Thread thread1 = |
| new Thread( |
| () -> { |
| try { |
| remoteCache.ensureInputsPresent( |
| remoteActionExecutionContext, |
| merkleTree1, |
| ImmutableMap.of(), |
| false, |
| /* remotePathResolver= */ null); |
| } catch (IOException ignored) { |
| // ignored |
| } catch (InterruptedException e) { |
| ensureInterrupted.countDown(); |
| } finally { |
| ensureInputsPresentReturned.countDown(); |
| } |
| }); |
| Thread thread2 = |
| new Thread( |
| () -> { |
| try { |
| remoteCache.ensureInputsPresent( |
| remoteActionExecutionContext, |
| merkleTree2, |
| ImmutableMap.of(), |
| false, |
| /* remotePathResolver= */ null); |
| } catch (InterruptedException | IOException ignored) { |
| // ignored |
| } finally { |
| ensureInputsPresentReturned.countDown(); |
| } |
| }); |
| |
| // act |
| thread1.start(); |
| thread2.start(); |
| uploadCalls.await(); |
| assertThat(uploadFutures).hasSize(5); |
| assertThat(cacheProtocol.getInProgressUploads()).hasSize(5); |
| |
| thread1.interrupt(); |
| ensureInterrupted.await(); |
| |
| // assert |
| assertThat(cacheProtocol.getInProgressUploads()).hasSize(3); |
| assertThat(cacheProtocol.getFinishedUploads()).isEmpty(); |
| // foo is only in tree1, so interrupting thread1 cancels it; bar is shared and qux is only in |
| // tree2, so both are kept. |
| assertThat(uploadFutures.get(fooDigest).isCancelled()).isTrue(); |
| assertThat(uploadFutures.get(barDigest).isCancelled()).isFalse(); |
| assertThat(uploadFutures.get(quxDigest).isCancelled()).isFalse(); |
| |
| for (SettableFuture<Void> future : uploadFutures.values()) { |
| future.set(null); |
| } |
| ensureInputsPresentReturned.await(); |
| assertThat(cacheProtocol.getInProgressUploads()).isEmpty(); |
| assertThat(cacheProtocol.getFinishedUploads()).hasSize(3); |
| } |
| |
| @Test |
| public void ensureInputsPresent_uploadFailed_propagateErrors() throws Exception { |
| RemoteCacheClient cacheProtocol = spy(new InMemoryCacheClient()); |
| doAnswer(invocationOnMock -> Futures.immediateFailedFuture(new IOException("upload failed"))) |
| .when(cacheProtocol) |
| .uploadBlobImpl(any(), any(), (Blob) any()); |
| doAnswer(invocationOnMock -> Futures.immediateFailedFuture(new IOException("upload failed"))) |
| .when(cacheProtocol) |
| .uploadBlobImpl(any(), any(), any()); |
| RemoteExecutionCache remoteCache = spy(newRemoteExecutionCache(cacheProtocol)); |
| Path path = execRoot.getRelative("foo"); |
| FileSystemUtils.writeContentAsLatin1(path, "bar"); |
| SortedMap<PathFragment, Path> inputs = ImmutableSortedMap.of(PathFragment.create("foo"), path); |
| MerkleTree merkleTree = MerkleTree.build(inputs, digestUtil); |
| |
| IOException e = |
| assertThrows( |
| IOException.class, |
| () -> |
| remoteCache.ensureInputsPresent( |
| remoteActionExecutionContext, |
| merkleTree, |
| ImmutableMap.of(), |
| false, |
| /* remotePathResolver= */ null)); |
| |
| assertThat(e).hasMessageThat().contains("upload failed"); |
| } |
| |
| @Test |
| public void shutdownNow_cancelInProgressUploads() throws Exception { |
| RemoteCacheClient remoteCacheClient = spy(new InMemoryCacheClient()); |
| // Return a future that never completes |
| doAnswer(invocationOnMock -> SettableFuture.create()) |
| .when(remoteCacheClient) |
| .uploadBlobImpl(any(), any(), any()); |
| CombinedCache combinedCache = newCombinedCache(remoteCacheClient); |
| Digest digest = fakeFileCache.createScratchInput(ActionInputHelper.fromPath("file"), "content"); |
| Path file = execRoot.getRelative("file"); |
| |
| ListenableFuture<Void> upload = |
| combinedCache.uploadFile(remoteActionExecutionContext, digest, file); |
| assertThat(remoteCacheClient.getInProgressUploads()).contains(digest); |
| combinedCache.shutdownNow(); |
| |
| assertThat(upload.isCancelled()).isTrue(); |
| } |
| |
| @Test |
| public void uploadFile_chunkedUpload_deduplicatesRemoteUpload() throws Exception { |
| RemoteOptions options = Options.getDefaults(RemoteOptions.class); |
| options.experimentalRemoteCacheChunking = true; |
| RemoteRetrier retrier = |
| com.google.devtools.build.lib.remote.util.TestUtils.newRemoteRetrier( |
| () -> RemoteRetrier.RETRIES_DISABLED, |
| RemoteRetrier.RETRIABLE_GRPC_ERRORS, |
| retryService); |
| GrpcCacheClient grpcCacheClient = |
| spy( |
| new GrpcCacheClient( |
| /* channel= */ null, |
| CallCredentialsProvider.NO_CREDENTIALS, |
| options, |
| retrier, |
| digestUtil)); |
| doAnswer(unused -> chunkingCapabilities()).when(grpcCacheClient).getServerCapabilities(); |
| doAnswer(unused -> Futures.immediateFuture(ImmutableSet.of())) |
| .when(grpcCacheClient) |
| .findMissingDigests(any(), any()); |
| doAnswer(unused -> null).when(grpcCacheClient).close(); |
| |
| CountDownLatch spliceStarted = new CountDownLatch(1); |
| SettableFuture<Void> spliceFuture = SettableFuture.create(); |
| doAnswer( |
| unused -> { |
| spliceStarted.countDown(); |
| return spliceFuture; |
| }) |
| .when(grpcCacheClient) |
| .spliceBlob(any(), any(), any()); |
| |
| CombinedCache combinedCache = |
| new CombinedCache(grpcCacheClient, /* diskCacheClient= */ null, options, digestUtil); |
| byte[] data = new byte[8192]; |
| Path file = execRoot.getRelative("chunked-output"); |
| try (var out = file.getOutputStream()) { |
| out.write(data); |
| } |
| Digest digest = digestUtil.compute(data); |
| |
| try { |
| ListenableFuture<Void> firstUpload = |
| combinedCache.uploadFile(remoteActionExecutionContext, digest, file); |
| assertThat(spliceStarted.await(1, TimeUnit.SECONDS)).isTrue(); |
| |
| ListenableFuture<Void> secondUpload = |
| combinedCache.uploadFile(remoteActionExecutionContext, digest, file); |
| |
| assertThat(grpcCacheClient.getUploadSubscriberCount(digest)).isEqualTo(2); |
| verify(grpcCacheClient).findMissingDigests(any(), any()); |
| verify(grpcCacheClient).spliceBlob(any(), any(), any()); |
| |
| spliceFuture.set(null); |
| getFromFuture(firstUpload); |
| getFromFuture(secondUpload); |
| } finally { |
| combinedCache.release(); |
| } |
| } |
| |
| private InMemoryCombinedCache newCombinedCache() { |
| RemoteOptions options = Options.getDefaults(RemoteOptions.class); |
| return new InMemoryCombinedCache(options, digestUtil); |
| } |
| |
| private InMemoryCombinedCache newCombinedCache( |
| Map<Digest, byte[]> casEntries, RemoteOptions options, DigestUtil digestUtil) { |
| return new InMemoryCombinedCache(casEntries, options, digestUtil); |
| } |
| |
| private CombinedCache newCombinedCache(RemoteCacheClient remoteCacheClient) { |
| return new CombinedCache( |
| remoteCacheClient, |
| /* diskCacheClient= */ null, |
| Options.getDefaults(RemoteOptions.class), |
| digestUtil); |
| } |
| |
| private RemoteExecutionCache newRemoteExecutionCache(RemoteCacheClient remoteCacheClient) { |
| return new RemoteExecutionCache( |
| remoteCacheClient, |
| /* diskCacheClient= */ null, |
| Options.getDefaults(RemoteOptions.class), |
| digestUtil); |
| } |
| |
| private static ServerCapabilities chunkingCapabilities() { |
| return ServerCapabilities.newBuilder() |
| .setCacheCapabilities( |
| CacheCapabilities.newBuilder() |
| .setFastCdc2020Params( |
| FastCdc2020Params.newBuilder().setAvgChunkSizeBytes(1024).build())) |
| .build(); |
| } |
| } |
