blob: 057d879dc1e6b58faac073b8691ba7c972187a6c [file] [log] [blame]
// Copyright 2024 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.remote.disk;
import static com.google.common.truth.Truth.assertThat;
import static com.google.devtools.build.lib.remote.util.Utils.getFromFuture;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertThrows;
import static org.junit.Assume.assumeNotNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import build.bazel.remote.execution.v2.ActionResult;
import build.bazel.remote.execution.v2.Digest;
import build.bazel.remote.execution.v2.Directory;
import build.bazel.remote.execution.v2.FileNode;
import build.bazel.remote.execution.v2.OutputDirectory;
import build.bazel.remote.execution.v2.OutputFile;
import build.bazel.remote.execution.v2.RequestMetadata;
import build.bazel.remote.execution.v2.Tree;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.exec.SpawnCheckingCacheEvent;
import com.google.devtools.build.lib.exec.SpawnRunner.SpawnExecutionContext;
import com.google.devtools.build.lib.remote.Store;
import com.google.devtools.build.lib.remote.common.CacheNotFoundException;
import com.google.devtools.build.lib.remote.common.OutputDigestMismatchException;
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
import com.google.devtools.build.lib.remote.common.RemoteCacheClient.ActionKey;
import com.google.devtools.build.lib.remote.common.RemoteCacheClient.CachedActionResult;
import com.google.devtools.build.lib.remote.util.DigestUtil;
import com.google.devtools.build.lib.vfs.DigestHashFunction;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.lib.vfs.bazel.BazelHashFunctions;
import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import com.google.protobuf.ByteString;
import com.google.protobuf.Message;
import java.io.IOException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Tests for {@link DiskCacheClient}. */
@RunWith(JUnit4.class)
public class DiskCacheClientTest {
private static final DigestUtil DIGEST_UTIL =
new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256);
private static final ExecutorService executorService =
MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
private final FileSystem fs = new InMemoryFileSystem(DigestHashFunction.SHA256);
private final Path root = fs.getPath("/disk_cache");
private DiskCacheClient client;
private RemoteActionExecutionContext context;
@Before
public void setUp() throws Exception {
client = new DiskCacheClient(root, DIGEST_UTIL, executorService, /* verifyDownloads= */ true);
context =
RemoteActionExecutionContext.create(
mock(Spawn.class),
mock(SpawnExecutionContext.class),
RequestMetadata.getDefaultInstance());
}
@Test
public void downloadActionResult_reportsSpawnCheckingCacheEvent() throws Exception {
var unused =
getFromFuture(
client.downloadActionResult(
context,
DIGEST_UTIL.asActionKey(DIGEST_UTIL.computeAsUtf8("key")),
/* inlineOutErr= */ false));
verify(context.getSpawnExecutionContext()).report(SpawnCheckingCacheEvent.create("disk-cache"));
}
@Test
public void findMissingDigests_returnsAllDigests() throws Exception {
ImmutableList<Digest> digests =
ImmutableList.of(getDigest("foo"), getDigest("bar"), getDigest("baz"));
assertThat(getFromFuture(client.findMissingDigests(context, digests)))
.containsExactlyElementsIn(digests);
}
@Test
public void toPath_forCas_forOldStyleHashFunction() throws Exception {
Digest digest = Digest.newBuilder().setHash("0123456789abcdef").setSizeBytes(42).build();
Path path = client.toPath(digest, Store.CAS);
assertThat(path).isEqualTo(root.getRelative("cas/01/0123456789abcdef"));
}
@Test
public void toPath_forAc_forOldStyleHashFunction() throws Exception {
Digest digest = Digest.newBuilder().setHash("0123456789abcdef").setSizeBytes(42).build();
Path path = client.toPath(digest, Store.AC);
assertThat(path).isEqualTo(root.getRelative("ac/01/0123456789abcdef"));
}
@Test
public void toPath_forCas_forNewStyleHashFunction() throws Exception {
assumeNotNull(BazelHashFunctions.BLAKE3); // BLAKE3 not available in Blaze.
DiskCacheClient client =
new DiskCacheClient(
root,
new DigestUtil(SyscallCache.NO_CACHE, BazelHashFunctions.BLAKE3),
executorService,
/* verifyDownloads= */ true);
Digest digest = Digest.newBuilder().setHash("0123456789abcdef").setSizeBytes(42).build();
Path path = client.toPath(digest, Store.CAS);
assertThat(path).isEqualTo(root.getRelative("blake3/cas/01/0123456789abcdef"));
}
@Test
public void toPath_forAc_forNewStyleHashFunction() throws Exception {
assumeNotNull(BazelHashFunctions.BLAKE3); // BLAKE3 not available in Blaze.
DiskCacheClient client =
new DiskCacheClient(
root,
new DigestUtil(SyscallCache.NO_CACHE, BazelHashFunctions.BLAKE3),
executorService,
/* verifyDownloads= */ true);
Digest digest = Digest.newBuilder().setHash("0123456789abcdef").setSizeBytes(42).build();
Path path = client.toPath(digest, Store.AC);
assertThat(path).isEqualTo(root.getRelative("blake3/ac/01/0123456789abcdef"));
}
@Test
public void uploadFile_whenMissing_populatesCas() throws Exception {
assertThat(root.exists()).isTrue();
Path file = fs.getPath("/file");
FileSystemUtils.writeContent(file, UTF_8, "contents");
Digest digest = getDigest("contents");
var unused = getFromFuture(client.uploadFile(context, digest, file));
assertThat(FileSystemUtils.readContent(getCasPath(digest), UTF_8)).isEqualTo("contents");
}
@Test
public void uploadFile_whenPresent_updatesMtime() throws Exception {
Path file = fs.getPath("/file");
FileSystemUtils.writeContent(file, UTF_8, "contents");
Digest digest = getDigest("contents");
// The contents would match under normal operation. This serves to check that we don't
// unnecessarily overwrite the file.
Path path = populateCas(digest, "existing contents");
var unused = getFromFuture(client.uploadFile(context, digest, file));
assertThat(FileSystemUtils.readContent(path, UTF_8)).isEqualTo("existing contents");
assertThat(path.getLastModifiedTime()).isNotEqualTo(0);
}
@Test
public void uploadBlob_whenMissing_populatesCas() throws Exception {
ByteString blob = ByteString.copyFromUtf8("contents");
Digest digest = getDigest("contents");
var unused = getFromFuture(client.uploadBlob(context, digest, blob));
assertThat(FileSystemUtils.readContent(getCasPath(digest), UTF_8)).isEqualTo("contents");
}
@Test
public void uploadBlob_whenPresent_updatesMtime() throws Exception {
ByteString blob = ByteString.copyFromUtf8("contents");
Digest digest = getDigest("contents");
// The contents would match under normal operation. This serves to check that we don't
// unnecessarily overwrite the file.
Path path = populateCas(digest, "existing contents");
var unused = getFromFuture(client.uploadBlob(context, digest, blob));
assertThat(FileSystemUtils.readContent(path, UTF_8)).isEqualTo("existing contents");
assertThat(path.getLastModifiedTime()).isNotEqualTo(0);
}
@Test
public void uploadActionResult_whenMissing_populatesAc() throws Exception {
ActionKey actionKey = new ActionKey(getDigest("key"));
ActionResult actionResult = ActionResult.newBuilder().setExitCode(42).build();
Path path = getAcPath(actionKey);
var unused = getFromFuture(client.uploadActionResult(context, actionKey, actionResult));
assertThat(FileSystemUtils.readContent(path)).isEqualTo(actionResult.toByteArray());
}
@Test
public void uploadActionResult_whenPresent_updatesMtime() throws Exception {
ActionKey actionKey = new ActionKey(getDigest("key"));
ActionResult actionResult = ActionResult.newBuilder().setExitCode(42).build();
Path path = populateAc(actionKey, actionResult);
// The contents would match under normal operation. This serves to check that we don't
// unnecessarily overwrite the file.
var unused =
getFromFuture(
client.uploadActionResult(context, actionKey, ActionResult.getDefaultInstance()));
assertThat(FileSystemUtils.readContent(path)).isEqualTo(actionResult.toByteArray());
assertThat(path.getLastModifiedTime()).isNotEqualTo(0);
}
@Test
public void downloadBlob_whenPresent_returnsContents() throws Exception {
Digest digest = getDigest("contents");
populateCas(digest, "contents");
Path out = fs.getPath("/out");
var unused = getFromFuture(client.downloadBlob(context, digest, out.getOutputStream()));
assertThat(FileSystemUtils.readContent(out, UTF_8)).isEqualTo("contents");
}
@Test
public void downloadBlob_whenMissing_throwsCacheNotFoundException() throws Exception {
Path out = fs.getPath("/out");
assertThrows(
CacheNotFoundException.class,
() ->
getFromFuture(
client.downloadBlob(context, getDigest("contents"), out.getOutputStream())));
}
@Test
public void downloadBlob_whenCorrupted_throwsOutputDigestMismatchException() throws Exception {
Digest digest = getDigest("contents");
populateCas(digest, "corrupted contents");
Path out = fs.getPath("/out");
assertThrows(
OutputDigestMismatchException.class,
() -> getFromFuture(client.downloadBlob(context, digest, out.getOutputStream())));
}
@Test
public void downloadActionResult_whenPresent_returnsCachedActionResult() throws Exception {
ActionKey actionKey = new ActionKey(getDigest("key"));
ActionResult actionResult = ActionResult.newBuilder().setExitCode(42).build();
Path path = populateAc(actionKey, actionResult);
CachedActionResult result =
getFromFuture(client.downloadActionResult(context, actionKey, /* inlineOutErr= */ false));
assertThat(result).isEqualTo(CachedActionResult.disk(actionResult));
assertThat(path.getLastModifiedTime()).isNotEqualTo(0);
}
@Test
public void downloadActionResult_whenMissing_returnsNull() throws Exception {
ActionKey actionKey = new ActionKey(getDigest("key"));
CachedActionResult result =
getFromFuture(client.downloadActionResult(context, actionKey, /* inlineOutErr= */ false));
assertThat(result).isNull();
}
@Test
public void downloadActionResult_withReferencedBlobsPresent_updatesMtimeOnBlobs()
throws Exception {
Digest stdoutDigest = getDigest("stdout contents");
Digest stderrDigest = getDigest("stderr contents");
Digest fileDigest = getDigest("file contents");
Digest treeFileDigest = getDigest("tree file contents");
Tree tree = getTreeWithFile(treeFileDigest);
Digest treeDigest = getDigest(tree);
ActionKey actionKey = new ActionKey(getDigest("key"));
ActionResult actionResult =
ActionResult.newBuilder()
.setStdoutDigest(stdoutDigest)
.setStderrDigest(stderrDigest)
.addOutputFiles(OutputFile.newBuilder().setDigest(fileDigest).build())
.addOutputDirectories(OutputDirectory.newBuilder().setTreeDigest(getDigest(tree)))
.build();
Path acPath = populateAc(actionKey, actionResult);
Path stdoutCasPath = populateCas(stdoutDigest, "stdout contents");
Path stderrCasPath = populateCas(stderrDigest, "stderr contents");
Path fileCasPath = populateCas(fileDigest, "file contents");
Path treeCasPath = populateCas(treeDigest, tree);
Path treeFileCasPath = populateCas(treeFileDigest, "tree file contents");
CachedActionResult result =
getFromFuture(client.downloadActionResult(context, actionKey, /* inlineOutErr= */ false));
assertThat(result).isEqualTo(CachedActionResult.disk(actionResult));
assertThat(acPath.getLastModifiedTime()).isNotEqualTo(0);
assertThat(stdoutCasPath.getLastModifiedTime()).isNotEqualTo(0);
assertThat(stderrCasPath.getLastModifiedTime()).isNotEqualTo(0);
assertThat(fileCasPath.getLastModifiedTime()).isNotEqualTo(0);
assertThat(treeCasPath.getLastModifiedTime()).isNotEqualTo(0);
assertThat(treeFileCasPath.getLastModifiedTime()).isNotEqualTo(0);
}
@Test
public void downloadActionResult_withReferencedFileMissing_returnsNull() throws Exception {
Digest fileDigest = getDigest("contents");
ActionKey actionKey = new ActionKey(getDigest("key"));
ActionResult actionResult =
ActionResult.newBuilder()
.addOutputFiles(OutputFile.newBuilder().setDigest(fileDigest).build())
.build();
populateAc(actionKey, actionResult);
CachedActionResult result =
getFromFuture(client.downloadActionResult(context, actionKey, /* inlineOutErr= */ false));
assertThat(result).isNull();
}
@Test
public void downloadActionResult_withReferencedTreeMissing_returnsNull() throws Exception {
Digest fileDigest = getDigest("contents");
Tree tree = getTreeWithFile(fileDigest);
Digest treeDigest = getDigest(tree);
ActionKey actionKey = new ActionKey(getDigest("key"));
ActionResult actionResult =
ActionResult.newBuilder()
.addOutputDirectories(OutputDirectory.newBuilder().setTreeDigest(treeDigest))
.build();
populateAc(actionKey, actionResult);
populateCas(fileDigest, "contents");
CachedActionResult result =
getFromFuture(client.downloadActionResult(context, actionKey, /* inlineOutErr= */ false));
assertThat(result).isNull();
}
@Test
public void downloadActionResult_withReferencedTreeFileMissing_returnsNull() throws Exception {
Digest fileDigest = getDigest("contents");
Tree tree = getTreeWithFile(fileDigest);
Digest treeDigest = getDigest(tree);
ActionKey actionKey = new ActionKey(getDigest("key"));
ActionResult actionResult =
ActionResult.newBuilder()
.addOutputDirectories(OutputDirectory.newBuilder().setTreeDigest(treeDigest))
.build();
populateAc(actionKey, actionResult);
populateCas(treeDigest, tree);
CachedActionResult result =
getFromFuture(client.downloadActionResult(context, actionKey, /* inlineOutErr= */ false));
assertThat(result).isNull();
}
private Tree getTreeWithFile(Digest fileDigest) {
return Tree.newBuilder()
.addChildren(Directory.newBuilder().addFiles(FileNode.newBuilder().setDigest(fileDigest)))
.build();
}
private Path getCasPath(Digest digest) {
return client.toPath(digest, Store.CAS);
}
@CanIgnoreReturnValue
private Path populateCas(Digest digest, String contents) throws IOException {
return populateCas(digest, contents.getBytes(UTF_8));
}
@CanIgnoreReturnValue
private Path populateCas(Digest digest, Message m) throws IOException {
return populateCas(digest, m.toByteArray());
}
private Path populateCas(Digest digest, byte[] contents) throws IOException {
Path path = getCasPath(digest);
path.getParentDirectory().createDirectoryAndParents();
FileSystemUtils.writeContent(path, contents);
path.setLastModifiedTime(0);
return path;
}
private Path getAcPath(ActionKey actionKey) {
return client.toPath(actionKey.getDigest(), Store.AC);
}
@CanIgnoreReturnValue
private Path populateAc(ActionKey actionKey, ActionResult actionResult) throws IOException {
Path path = getAcPath(actionKey);
path.getParentDirectory().createDirectoryAndParents();
FileSystemUtils.writeContent(path, actionResult.toByteArray());
path.setLastModifiedTime(0);
return path;
}
private Digest getDigest(String contents) {
return DIGEST_UTIL.computeAsUtf8(contents);
}
private Digest getDigest(Message m) {
return DIGEST_UTIL.compute(m.toByteArray());
}
}