| // Copyright 2018 The Bazel Authors. All rights reserved. | |
| // | |
| // Licensed under the Apache License, Version 2.0 (the "License"); | |
| // you may not use this file except in compliance with the License. | |
| // You may obtain a copy of the License at | |
| // | |
| // http://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, software | |
| // distributed under the License is distributed on an "AS IS" BASIS, | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| // See the License for the specific language governing permissions and | |
| // limitations under the License. | |
| package com.google.devtools.build.lib.remote; | |
| import static com.google.common.truth.Truth.assertThat; | |
| import static com.google.devtools.build.lib.remote.util.DigestUtil.toBinaryDigest; | |
| import static java.nio.charset.StandardCharsets.UTF_8; | |
| import static org.junit.Assert.assertThrows; | |
| import static org.mockito.ArgumentMatchers.anyInt; | |
| import static org.mockito.ArgumentMatchers.eq; | |
| import static org.mockito.Mockito.mock; | |
| import static org.mockito.Mockito.never; | |
| import static org.mockito.Mockito.verify; | |
| import static org.mockito.Mockito.when; | |
| import build.bazel.remote.execution.v2.Action; | |
| import build.bazel.remote.execution.v2.ActionResult; | |
| import build.bazel.remote.execution.v2.Command; | |
| import build.bazel.remote.execution.v2.Digest; | |
| import build.bazel.remote.execution.v2.Directory; | |
| import build.bazel.remote.execution.v2.DirectoryNode; | |
| import build.bazel.remote.execution.v2.FileNode; | |
| import build.bazel.remote.execution.v2.OutputDirectory; | |
| import build.bazel.remote.execution.v2.OutputFile; | |
| import build.bazel.remote.execution.v2.SymlinkNode; | |
| import build.bazel.remote.execution.v2.Tree; | |
| import com.google.common.base.Charsets; | |
| import com.google.common.base.Throwables; | |
| import com.google.common.collect.ImmutableList; | |
| import com.google.common.collect.ImmutableMap; | |
| import com.google.common.collect.ImmutableSet; | |
| import com.google.common.util.concurrent.ListeningScheduledExecutorService; | |
| import com.google.common.util.concurrent.MoreExecutors; | |
| import com.google.devtools.build.lib.actions.ActionInputHelper; | |
| import com.google.devtools.build.lib.actions.Artifact; | |
| import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact; | |
| import com.google.devtools.build.lib.actions.Artifact.SpecialArtifactType; | |
| import com.google.devtools.build.lib.actions.ArtifactRoot; | |
| import com.google.devtools.build.lib.actions.ExecException; | |
| import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue; | |
| import com.google.devtools.build.lib.actions.cache.MetadataInjector; | |
| import com.google.devtools.build.lib.actions.util.ActionsTestUtil; | |
| import com.google.devtools.build.lib.clock.JavaClock; | |
| import com.google.devtools.build.lib.remote.RemoteCache.OutputFilesLocker; | |
| import com.google.devtools.build.lib.remote.RemoteCache.UploadManifest; | |
| import com.google.devtools.build.lib.remote.common.RemoteCacheClient.ActionKey; | |
| import com.google.devtools.build.lib.remote.options.RemoteOptions; | |
| import com.google.devtools.build.lib.remote.util.DigestUtil; | |
| import com.google.devtools.build.lib.remote.util.InMemoryCacheClient; | |
| import com.google.devtools.build.lib.remote.util.Utils; | |
| import com.google.devtools.build.lib.remote.util.Utils.InMemoryOutput; | |
| import com.google.devtools.build.lib.testutil.TestUtils; | |
| import com.google.devtools.build.lib.util.io.FileOutErr; | |
| import com.google.devtools.build.lib.util.io.RecordingOutErr; | |
| import com.google.devtools.build.lib.vfs.DigestHashFunction; | |
| import com.google.devtools.build.lib.vfs.FileSystem; | |
| import com.google.devtools.build.lib.vfs.FileSystemUtils; | |
| import com.google.devtools.build.lib.vfs.Path; | |
| import com.google.devtools.build.lib.vfs.PathFragment; | |
| import com.google.devtools.build.lib.vfs.Symlinks; | |
| import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; | |
| import com.google.devtools.common.options.Options; | |
| import com.google.protobuf.ByteString; | |
| import com.google.protobuf.Message; | |
| import java.io.IOException; | |
| import java.io.OutputStream; | |
| import java.nio.charset.StandardCharsets; | |
| import java.util.Map; | |
| import java.util.concurrent.ConcurrentHashMap; | |
| import java.util.concurrent.ConcurrentMap; | |
| import java.util.concurrent.Executors; | |
| import java.util.concurrent.TimeUnit; | |
| import org.junit.After; | |
| import org.junit.Before; | |
| import org.junit.Test; | |
| import org.junit.runner.RunWith; | |
| import org.junit.runners.JUnit4; | |
| import org.mockito.Mock; | |
| import org.mockito.Mockito; | |
| import org.mockito.MockitoAnnotations; | |
| /** Tests for {@link RemoteCache}. */ | |
| @RunWith(JUnit4.class) | |
| public class RemoteCacheTests { | |
| @Mock private OutputFilesLocker outputFilesLocker; | |
| private FileSystem fs; | |
| private Path execRoot; | |
| ArtifactRoot artifactRoot; | |
| private final DigestUtil digestUtil = new DigestUtil(DigestHashFunction.SHA256); | |
| private FakeActionInputFileCache fakeFileCache; | |
| private ListeningScheduledExecutorService retryService; | |
| @Before | |
| public void setUp() throws Exception { | |
| MockitoAnnotations.initMocks(this); | |
| fs = new InMemoryFileSystem(new JavaClock(), DigestHashFunction.SHA256); | |
| execRoot = fs.getPath("/execroot"); | |
| execRoot.createDirectoryAndParents(); | |
| fakeFileCache = new FakeActionInputFileCache(execRoot); | |
| artifactRoot = ArtifactRoot.asDerivedRoot(execRoot, "outputs"); | |
| artifactRoot.getRoot().asPath().createDirectoryAndParents(); | |
| retryService = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1)); | |
| } | |
| @After | |
| public void afterEverything() throws InterruptedException { | |
| retryService.shutdownNow(); | |
| retryService.awaitTermination(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); | |
| } | |
| @Test | |
| public void uploadAbsoluteFileSymlinkAsFile() throws Exception { | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| Path link = fs.getPath("/execroot/link"); | |
| Path target = fs.getPath("/execroot/target"); | |
| FileSystemUtils.writeContent(target, new byte[] {1, 2, 3, 4, 5}); | |
| link.createSymbolicLink(target); | |
| UploadManifest um = | |
| new UploadManifest( | |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); | |
| um.addFiles(ImmutableList.of(link)); | |
| Digest digest = digestUtil.compute(target); | |
| assertThat(um.getDigestToFile()).containsExactly(digest, link); | |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); | |
| expectedResult.addOutputFilesBuilder().setPath("link").setDigest(digest); | |
| assertThat(result.build()).isEqualTo(expectedResult.build()); | |
| } | |
| @Test | |
| public void uploadAbsoluteDirectorySymlinkAsDirectory() throws Exception { | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| Path dir = fs.getPath("/execroot/dir"); | |
| dir.createDirectory(); | |
| Path foo = fs.getPath("/execroot/dir/foo"); | |
| FileSystemUtils.writeContent(foo, new byte[] {1, 2, 3, 4, 5}); | |
| Path link = fs.getPath("/execroot/link"); | |
| link.createSymbolicLink(dir); | |
| UploadManifest um = | |
| new UploadManifest( | |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); | |
| um.addFiles(ImmutableList.of(link)); | |
| Digest digest = digestUtil.compute(foo); | |
| assertThat(um.getDigestToFile()).containsExactly(digest, fs.getPath("/execroot/link/foo")); | |
| Tree tree = | |
| Tree.newBuilder() | |
| .setRoot( | |
| Directory.newBuilder() | |
| .addFiles(FileNode.newBuilder().setName("foo").setDigest(digest))) | |
| .build(); | |
| Digest treeDigest = digestUtil.compute(tree); | |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); | |
| expectedResult.addOutputDirectoriesBuilder().setPath("link").setTreeDigest(treeDigest); | |
| assertThat(result.build()).isEqualTo(expectedResult.build()); | |
| } | |
| @Test | |
| public void uploadRelativeFileSymlinkAsFile() throws Exception { | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| Path link = fs.getPath("/execroot/link"); | |
| Path target = fs.getPath("/execroot/target"); | |
| FileSystemUtils.writeContent(target, new byte[] {1, 2, 3, 4, 5}); | |
| link.createSymbolicLink(target.relativeTo(execRoot)); | |
| UploadManifest um = | |
| new UploadManifest( | |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ false, /*allowSymlinks=*/ true); | |
| um.addFiles(ImmutableList.of(link)); | |
| Digest digest = digestUtil.compute(target); | |
| assertThat(um.getDigestToFile()).containsExactly(digest, link); | |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); | |
| expectedResult.addOutputFilesBuilder().setPath("link").setDigest(digest); | |
| assertThat(result.build()).isEqualTo(expectedResult.build()); | |
| } | |
| @Test | |
| public void uploadRelativeDirectorySymlinkAsDirectory() throws Exception { | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| Path dir = fs.getPath("/execroot/dir"); | |
| dir.createDirectory(); | |
| Path foo = fs.getPath("/execroot/dir/foo"); | |
| FileSystemUtils.writeContent(foo, new byte[] {1, 2, 3, 4, 5}); | |
| Path link = fs.getPath("/execroot/link"); | |
| link.createSymbolicLink(dir.relativeTo(execRoot)); | |
| UploadManifest um = | |
| new UploadManifest( | |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ false, /*allowSymlinks=*/ true); | |
| um.addFiles(ImmutableList.of(link)); | |
| Digest digest = digestUtil.compute(foo); | |
| assertThat(um.getDigestToFile()).containsExactly(digest, fs.getPath("/execroot/link/foo")); | |
| Tree tree = | |
| Tree.newBuilder() | |
| .setRoot( | |
| Directory.newBuilder() | |
| .addFiles(FileNode.newBuilder().setName("foo").setDigest(digest))) | |
| .build(); | |
| Digest treeDigest = digestUtil.compute(tree); | |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); | |
| expectedResult.addOutputDirectoriesBuilder().setPath("link").setTreeDigest(treeDigest); | |
| assertThat(result.build()).isEqualTo(expectedResult.build()); | |
| } | |
| @Test | |
| public void uploadRelativeFileSymlink() throws Exception { | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| Path link = fs.getPath("/execroot/link"); | |
| Path target = fs.getPath("/execroot/target"); | |
| FileSystemUtils.writeContent(target, new byte[] {1, 2, 3, 4, 5}); | |
| link.createSymbolicLink(target.relativeTo(execRoot)); | |
| UploadManifest um = | |
| new UploadManifest( | |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); | |
| um.addFiles(ImmutableList.of(link)); | |
| assertThat(um.getDigestToFile()).isEmpty(); | |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); | |
| expectedResult.addOutputFileSymlinksBuilder().setPath("link").setTarget("target"); | |
| assertThat(result.build()).isEqualTo(expectedResult.build()); | |
| } | |
| @Test | |
| public void uploadRelativeDirectorySymlink() throws Exception { | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| Path dir = fs.getPath("/execroot/dir"); | |
| dir.createDirectory(); | |
| Path file = fs.getPath("/execroot/dir/foo"); | |
| FileSystemUtils.writeContent(file, new byte[] {1, 2, 3, 4, 5}); | |
| Path link = fs.getPath("/execroot/link"); | |
| link.createSymbolicLink(dir.relativeTo(execRoot)); | |
| UploadManifest um = | |
| new UploadManifest( | |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); | |
| um.addFiles(ImmutableList.of(link)); | |
| assertThat(um.getDigestToFile()).isEmpty(); | |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); | |
| expectedResult.addOutputDirectorySymlinksBuilder().setPath("link").setTarget("dir"); | |
| assertThat(result.build()).isEqualTo(expectedResult.build()); | |
| } | |
| @Test | |
| public void uploadDanglingSymlinkError() throws Exception { | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| Path link = fs.getPath("/execroot/link"); | |
| Path target = fs.getPath("/execroot/target"); | |
| link.createSymbolicLink(target.relativeTo(execRoot)); | |
| UploadManifest um = | |
| new UploadManifest( | |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); | |
| IOException e = assertThrows(IOException.class, () -> um.addFiles(ImmutableList.of(link))); | |
| assertThat(e).hasMessageThat().contains("dangling"); | |
| assertThat(e).hasMessageThat().contains("/execroot/link"); | |
| assertThat(e).hasMessageThat().contains("target"); | |
| } | |
| @Test | |
| public void uploadSymlinksNoAllowError() throws Exception { | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| Path link = fs.getPath("/execroot/link"); | |
| Path target = fs.getPath("/execroot/target"); | |
| FileSystemUtils.writeContent(target, new byte[] {1, 2, 3, 4, 5}); | |
| link.createSymbolicLink(target.relativeTo(execRoot)); | |
| UploadManifest um = | |
| new UploadManifest( | |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ false); | |
| ExecException e = assertThrows(ExecException.class, () -> um.addFiles(ImmutableList.of(link))); | |
| assertThat(e).hasMessageThat().contains("symbolic link"); | |
| assertThat(e).hasMessageThat().contains("--remote_allow_symlink_upload"); | |
| } | |
| @Test | |
| public void uploadAbsoluteFileSymlinkInDirectoryAsFile() throws Exception { | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| Path dir = fs.getPath("/execroot/dir"); | |
| dir.createDirectory(); | |
| Path target = fs.getPath("/execroot/target"); | |
| FileSystemUtils.writeContent(target, new byte[] {1, 2, 3, 4, 5}); | |
| Path link = fs.getPath("/execroot/dir/link"); | |
| link.createSymbolicLink(target); | |
| UploadManifest um = | |
| new UploadManifest( | |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); | |
| um.addFiles(ImmutableList.of(dir)); | |
| Digest digest = digestUtil.compute(target); | |
| assertThat(um.getDigestToFile()).containsExactly(digest, link); | |
| Tree tree = | |
| Tree.newBuilder() | |
| .setRoot( | |
| Directory.newBuilder() | |
| .addFiles(FileNode.newBuilder().setName("link").setDigest(digest))) | |
| .build(); | |
| Digest treeDigest = digestUtil.compute(tree); | |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); | |
| expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); | |
| assertThat(result.build()).isEqualTo(expectedResult.build()); | |
| } | |
| @Test | |
| public void uploadAbsoluteDirectorySymlinkInDirectoryAsDirectory() throws Exception { | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| Path dir = fs.getPath("/execroot/dir"); | |
| dir.createDirectory(); | |
| Path bardir = fs.getPath("/execroot/bardir"); | |
| bardir.createDirectory(); | |
| Path foo = fs.getPath("/execroot/bardir/foo"); | |
| FileSystemUtils.writeContent(foo, new byte[] {1, 2, 3, 4, 5}); | |
| Path link = fs.getPath("/execroot/dir/link"); | |
| link.createSymbolicLink(bardir); | |
| UploadManifest um = | |
| new UploadManifest( | |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); | |
| um.addFiles(ImmutableList.of(dir)); | |
| Digest digest = digestUtil.compute(foo); | |
| assertThat(um.getDigestToFile()).containsExactly(digest, fs.getPath("/execroot/dir/link/foo")); | |
| Directory barDir = | |
| Directory.newBuilder() | |
| .addFiles(FileNode.newBuilder().setName("foo").setDigest(digest)) | |
| .build(); | |
| Digest barDigest = digestUtil.compute(barDir); | |
| Tree tree = | |
| Tree.newBuilder() | |
| .setRoot( | |
| Directory.newBuilder() | |
| .addDirectories( | |
| DirectoryNode.newBuilder().setName("link").setDigest(barDigest))) | |
| .addChildren(barDir) | |
| .build(); | |
| Digest treeDigest = digestUtil.compute(tree); | |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); | |
| expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); | |
| assertThat(result.build()).isEqualTo(expectedResult.build()); | |
| } | |
| @Test | |
| public void uploadRelativeFileSymlinkInDirectoryAsFile() throws Exception { | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| Path dir = fs.getPath("/execroot/dir"); | |
| dir.createDirectory(); | |
| Path target = fs.getPath("/execroot/target"); | |
| FileSystemUtils.writeContent(target, new byte[] {1, 2, 3, 4, 5}); | |
| Path link = fs.getPath("/execroot/dir/link"); | |
| link.createSymbolicLink(PathFragment.create("../target")); | |
| UploadManifest um = | |
| new UploadManifest( | |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ false, /*allowSymlinks=*/ true); | |
| um.addFiles(ImmutableList.of(dir)); | |
| Digest digest = digestUtil.compute(target); | |
| assertThat(um.getDigestToFile()).containsExactly(digest, link); | |
| Tree tree = | |
| Tree.newBuilder() | |
| .setRoot( | |
| Directory.newBuilder() | |
| .addFiles(FileNode.newBuilder().setName("link").setDigest(digest))) | |
| .build(); | |
| Digest treeDigest = digestUtil.compute(tree); | |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); | |
| expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); | |
| assertThat(result.build()).isEqualTo(expectedResult.build()); | |
| } | |
| @Test | |
| public void uploadRelativeDirectorySymlinkInDirectoryAsDirectory() throws Exception { | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| Path dir = fs.getPath("/execroot/dir"); | |
| dir.createDirectory(); | |
| Path bardir = fs.getPath("/execroot/bardir"); | |
| bardir.createDirectory(); | |
| Path foo = fs.getPath("/execroot/bardir/foo"); | |
| FileSystemUtils.writeContent(foo, new byte[] {1, 2, 3, 4, 5}); | |
| Path link = fs.getPath("/execroot/dir/link"); | |
| link.createSymbolicLink(PathFragment.create("../bardir")); | |
| UploadManifest um = | |
| new UploadManifest( | |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ false, /*allowSymlinks=*/ true); | |
| um.addFiles(ImmutableList.of(dir)); | |
| Digest digest = digestUtil.compute(foo); | |
| assertThat(um.getDigestToFile()).containsExactly(digest, fs.getPath("/execroot/dir/link/foo")); | |
| Directory barDir = | |
| Directory.newBuilder() | |
| .addFiles(FileNode.newBuilder().setName("foo").setDigest(digest)) | |
| .build(); | |
| Digest barDigest = digestUtil.compute(barDir); | |
| Tree tree = | |
| Tree.newBuilder() | |
| .setRoot( | |
| Directory.newBuilder() | |
| .addDirectories( | |
| DirectoryNode.newBuilder().setName("link").setDigest(barDigest))) | |
| .addChildren(barDir) | |
| .build(); | |
| Digest treeDigest = digestUtil.compute(tree); | |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); | |
| expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); | |
| assertThat(result.build()).isEqualTo(expectedResult.build()); | |
| } | |
| @Test | |
| public void uploadRelativeFileSymlinkInDirectory() throws Exception { | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| Path dir = fs.getPath("/execroot/dir"); | |
| dir.createDirectory(); | |
| Path target = fs.getPath("/execroot/target"); | |
| FileSystemUtils.writeContent(target, new byte[] {1, 2, 3, 4, 5}); | |
| Path link = fs.getPath("/execroot/dir/link"); | |
| link.createSymbolicLink(PathFragment.create("../target")); | |
| UploadManifest um = | |
| new UploadManifest( | |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); | |
| um.addFiles(ImmutableList.of(dir)); | |
| assertThat(um.getDigestToFile()).isEmpty(); | |
| Tree tree = | |
| Tree.newBuilder() | |
| .setRoot( | |
| Directory.newBuilder() | |
| .addSymlinks(SymlinkNode.newBuilder().setName("link").setTarget("../target"))) | |
| .build(); | |
| Digest treeDigest = digestUtil.compute(tree); | |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); | |
| expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); | |
| assertThat(result.build()).isEqualTo(expectedResult.build()); | |
| } | |
| @Test | |
| public void uploadRelativeDirectorySymlinkInDirectory() throws Exception { | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| Path dir = fs.getPath("/execroot/dir"); | |
| dir.createDirectory(); | |
| Path bardir = fs.getPath("/execroot/bardir"); | |
| bardir.createDirectory(); | |
| Path foo = fs.getPath("/execroot/bardir/foo"); | |
| FileSystemUtils.writeContent(foo, new byte[] {1, 2, 3, 4, 5}); | |
| Path link = fs.getPath("/execroot/dir/link"); | |
| link.createSymbolicLink(PathFragment.create("../bardir")); | |
| UploadManifest um = | |
| new UploadManifest( | |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); | |
| um.addFiles(ImmutableList.of(dir)); | |
| assertThat(um.getDigestToFile()).isEmpty(); | |
| Tree tree = | |
| Tree.newBuilder() | |
| .setRoot( | |
| Directory.newBuilder() | |
| .addSymlinks(SymlinkNode.newBuilder().setName("link").setTarget("../bardir"))) | |
| .build(); | |
| Digest treeDigest = digestUtil.compute(tree); | |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); | |
| expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); | |
| assertThat(result.build()).isEqualTo(expectedResult.build()); | |
| } | |
| @Test | |
| public void uploadDanglingSymlinkInDirectoryError() throws Exception { | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| Path dir = fs.getPath("/execroot/dir"); | |
| dir.createDirectory(); | |
| Path target = fs.getPath("/execroot/target"); | |
| Path link = fs.getPath("/execroot/dir/link"); | |
| link.createSymbolicLink(target); | |
| UploadManifest um = | |
| new UploadManifest( | |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); | |
| IOException e = assertThrows(IOException.class, () -> um.addFiles(ImmutableList.of(dir))); | |
| assertThat(e).hasMessageThat().contains("dangling"); | |
| assertThat(e).hasMessageThat().contains("/execroot/dir/link"); | |
| assertThat(e).hasMessageThat().contains("/execroot/target"); | |
| } | |
| @Test | |
| public void uploadSymlinkInDirectoryNoAllowError() throws Exception { | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| Path dir = fs.getPath("/execroot/dir"); | |
| dir.createDirectory(); | |
| Path target = fs.getPath("/execroot/target"); | |
| FileSystemUtils.writeContent(target, new byte[] {1, 2, 3, 4, 5}); | |
| Path link = fs.getPath("/execroot/dir/link"); | |
| link.createSymbolicLink(target); | |
| UploadManifest um = | |
| new UploadManifest( | |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ false); | |
| ExecException e = assertThrows(ExecException.class, () -> um.addFiles(ImmutableList.of(dir))); | |
| assertThat(e).hasMessageThat().contains("symbolic link"); | |
| assertThat(e).hasMessageThat().contains("dir/link"); | |
| assertThat(e).hasMessageThat().contains("--remote_allow_symlink_upload"); | |
| } | |
| @Test | |
| public void downloadRelativeFileSymlink() throws Exception { | |
| RemoteCache cache = newRemoteCache(); | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| result.addOutputFileSymlinksBuilder().setPath("a/b/link").setTarget("../../foo"); | |
| // Doesn't check for dangling links, hence download succeeds. | |
| cache.download(result.build(), execRoot, null, outputFilesLocker); | |
| Path path = execRoot.getRelative("a/b/link"); | |
| assertThat(path.isSymbolicLink()).isTrue(); | |
| assertThat(path.readSymbolicLink()).isEqualTo(PathFragment.create("../../foo")); | |
| verify(outputFilesLocker).lock(); | |
| } | |
| @Test | |
| public void downloadRelativeDirectorySymlink() throws Exception { | |
| RemoteCache cache = newRemoteCache(); | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| result.addOutputDirectorySymlinksBuilder().setPath("a/b/link").setTarget("foo"); | |
| // Doesn't check for dangling links, hence download succeeds. | |
| cache.download(result.build(), execRoot, null, outputFilesLocker); | |
| Path path = execRoot.getRelative("a/b/link"); | |
| assertThat(path.isSymbolicLink()).isTrue(); | |
| assertThat(path.readSymbolicLink()).isEqualTo(PathFragment.create("foo")); | |
| verify(outputFilesLocker).lock(); | |
| } | |
| @Test | |
| public void downloadRelativeSymlinkInDirectory() throws Exception { | |
| InMemoryRemoteCache cache = newRemoteCache(); | |
| Tree tree = | |
| Tree.newBuilder() | |
| .setRoot( | |
| Directory.newBuilder() | |
| .addSymlinks(SymlinkNode.newBuilder().setName("link").setTarget("../foo"))) | |
| .build(); | |
| Digest treeDigest = cache.addContents(tree.toByteArray()); | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| result.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); | |
| // Doesn't check for dangling links, hence download succeeds. | |
| cache.download(result.build(), execRoot, null, outputFilesLocker); | |
| Path path = execRoot.getRelative("dir/link"); | |
| assertThat(path.isSymbolicLink()).isTrue(); | |
| assertThat(path.readSymbolicLink()).isEqualTo(PathFragment.create("../foo")); | |
| verify(outputFilesLocker).lock(); | |
| } | |
| @Test | |
| public void downloadAbsoluteDirectorySymlinkError() throws Exception { | |
| RemoteCache cache = newRemoteCache(); | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| result.addOutputDirectorySymlinksBuilder().setPath("foo").setTarget("/abs/link"); | |
| IOException expected = | |
| assertThrows( | |
| IOException.class, | |
| () -> cache.download(result.build(), execRoot, null, outputFilesLocker)); | |
| assertThat(expected).hasMessageThat().contains("/abs/link"); | |
| assertThat(expected).hasMessageThat().contains("absolute path"); | |
| verify(outputFilesLocker).lock(); | |
| } | |
| @Test | |
| public void downloadAbsoluteFileSymlinkError() throws Exception { | |
| RemoteCache cache = newRemoteCache(); | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| result.addOutputFileSymlinksBuilder().setPath("foo").setTarget("/abs/link"); | |
| IOException expected = | |
| assertThrows( | |
| IOException.class, | |
| () -> cache.download(result.build(), execRoot, null, outputFilesLocker)); | |
| assertThat(expected).hasMessageThat().contains("/abs/link"); | |
| assertThat(expected).hasMessageThat().contains("absolute path"); | |
| verify(outputFilesLocker).lock(); | |
| } | |
| @Test | |
| public void downloadAbsoluteSymlinkInDirectoryError() throws Exception { | |
| InMemoryRemoteCache cache = newRemoteCache(); | |
| Tree tree = | |
| Tree.newBuilder() | |
| .setRoot( | |
| Directory.newBuilder() | |
| .addSymlinks(SymlinkNode.newBuilder().setName("link").setTarget("/foo"))) | |
| .build(); | |
| Digest treeDigest = cache.addContents(tree.toByteArray()); | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| result.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); | |
| IOException expected = | |
| assertThrows( | |
| IOException.class, | |
| () -> cache.download(result.build(), execRoot, null, outputFilesLocker)); | |
| assertThat(expected.getSuppressed()).isEmpty(); | |
| assertThat(expected).hasMessageThat().contains("dir/link"); | |
| assertThat(expected).hasMessageThat().contains("/foo"); | |
| assertThat(expected).hasMessageThat().contains("absolute path"); | |
| verify(outputFilesLocker).lock(); | |
| } | |
| @Test | |
| public void downloadFailureMaintainsDirectories() throws Exception { | |
| InMemoryRemoteCache cache = newRemoteCache(); | |
| Tree tree = Tree.newBuilder().setRoot(Directory.newBuilder()).build(); | |
| Digest treeDigest = cache.addContents(tree.toByteArray()); | |
| Digest outputFileDigest = | |
| cache.addException("outputdir/outputfile", new IOException("download failed")); | |
| Digest otherFileDigest = cache.addContents("otherfile"); | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| result.addOutputDirectoriesBuilder().setPath("outputdir").setTreeDigest(treeDigest); | |
| result.addOutputFiles( | |
| OutputFile.newBuilder().setPath("outputdir/outputfile").setDigest(outputFileDigest)); | |
| result.addOutputFiles(OutputFile.newBuilder().setPath("otherfile").setDigest(otherFileDigest)); | |
| assertThrows( | |
| BulkTransferException.class, | |
| () -> cache.download(result.build(), execRoot, null, outputFilesLocker)); | |
| assertThat(cache.getNumFailedDownloads()).isEqualTo(1); | |
| assertThat(execRoot.getRelative("outputdir").exists()).isTrue(); | |
| assertThat(execRoot.getRelative("outputdir/outputfile").exists()).isFalse(); | |
| assertThat(execRoot.getRelative("otherfile").exists()).isFalse(); | |
| verify(outputFilesLocker, never()).lock(); | |
| } | |
| @Test | |
| public void onErrorWaitForRemainingDownloadsToComplete() throws Exception { | |
| // If one or more downloads of output files / directories fail then the code should | |
| // wait for all downloads to have been completed before it tries to clean up partially | |
| // downloaded files. | |
| Path stdout = fs.getPath("/execroot/stdout"); | |
| Path stderr = fs.getPath("/execroot/stderr"); | |
| InMemoryRemoteCache cache = newRemoteCache(); | |
| Digest digest1 = cache.addContents("file1"); | |
| Digest digest2 = cache.addException("file2", new IOException("download failed")); | |
| Digest digest3 = cache.addContents("file3"); | |
| ActionResult result = | |
| ActionResult.newBuilder() | |
| .setExitCode(0) | |
| .addOutputFiles(OutputFile.newBuilder().setPath("file1").setDigest(digest1)) | |
| .addOutputFiles(OutputFile.newBuilder().setPath("file2").setDigest(digest2)) | |
| .addOutputFiles(OutputFile.newBuilder().setPath("file3").setDigest(digest3)) | |
| .build(); | |
| BulkTransferException downloadException = | |
| assertThrows( | |
| BulkTransferException.class, | |
| () -> | |
| cache.download( | |
| result, execRoot, new FileOutErr(stdout, stderr), outputFilesLocker)); | |
| assertThat(downloadException.getSuppressed()).hasLength(1); | |
| assertThat(cache.getNumSuccessfulDownloads()).isEqualTo(2); | |
| assertThat(cache.getNumFailedDownloads()).isEqualTo(1); | |
| assertThat(downloadException.getSuppressed()[0]).isInstanceOf(IOException.class); | |
| IOException e = (IOException) downloadException.getSuppressed()[0]; | |
| assertThat(Throwables.getRootCause(e)).hasMessageThat().isEqualTo("download failed"); | |
| verify(outputFilesLocker, never()).lock(); | |
| } | |
| @Test | |
| public void downloadWithMultipleErrorsAddsThemAsSuppressed() throws Exception { | |
| Path stdout = fs.getPath("/execroot/stdout"); | |
| Path stderr = fs.getPath("/execroot/stderr"); | |
| InMemoryRemoteCache cache = newRemoteCache(); | |
| Digest digest1 = cache.addContents("file1"); | |
| Digest digest2 = cache.addException("file2", new IOException("file2 failed")); | |
| Digest digest3 = cache.addException("file3", new IOException("file3 failed")); | |
| ActionResult result = | |
| ActionResult.newBuilder() | |
| .setExitCode(0) | |
| .addOutputFiles(OutputFile.newBuilder().setPath("file1").setDigest(digest1)) | |
| .addOutputFiles(OutputFile.newBuilder().setPath("file2").setDigest(digest2)) | |
| .addOutputFiles(OutputFile.newBuilder().setPath("file3").setDigest(digest3)) | |
| .build(); | |
| BulkTransferException e = | |
| assertThrows( | |
| BulkTransferException.class, | |
| () -> | |
| cache.download( | |
| result, execRoot, new FileOutErr(stdout, stderr), outputFilesLocker)); | |
| assertThat(e.getSuppressed()).hasLength(2); | |
| assertThat(e.getSuppressed()[0]).isInstanceOf(IOException.class); | |
| assertThat(e.getSuppressed()[0]).hasMessageThat().isAnyOf("file2 failed", "file3 failed"); | |
| assertThat(e.getSuppressed()[1]).isInstanceOf(IOException.class); | |
| assertThat(e.getSuppressed()[1]).hasMessageThat().isAnyOf("file2 failed", "file3 failed"); | |
| } | |
| @Test | |
| public void downloadWithDuplicateIOErrorsDoesNotSuppress() throws Exception { | |
| Path stdout = fs.getPath("/execroot/stdout"); | |
| Path stderr = fs.getPath("/execroot/stderr"); | |
| InMemoryRemoteCache cache = newRemoteCache(); | |
| Digest digest1 = cache.addContents("file1"); | |
| IOException reusedException = new IOException("reused io exception"); | |
| Digest digest2 = cache.addException("file2", reusedException); | |
| Digest digest3 = cache.addException("file3", reusedException); | |
| ActionResult result = | |
| ActionResult.newBuilder() | |
| .setExitCode(0) | |
| .addOutputFiles(OutputFile.newBuilder().setPath("file1").setDigest(digest1)) | |
| .addOutputFiles(OutputFile.newBuilder().setPath("file2").setDigest(digest2)) | |
| .addOutputFiles(OutputFile.newBuilder().setPath("file3").setDigest(digest3)) | |
| .build(); | |
| BulkTransferException downloadException = | |
| assertThrows( | |
| BulkTransferException.class, | |
| () -> | |
| cache.download( | |
| result, execRoot, new FileOutErr(stdout, stderr), outputFilesLocker)); | |
| for (Throwable t : downloadException.getSuppressed()) { | |
| assertThat(t).isInstanceOf(IOException.class); | |
| IOException e = (IOException) t; | |
| assertThat(Throwables.getRootCause(e)).hasMessageThat().isEqualTo("reused io exception"); | |
| } | |
| } | |
| @Test | |
| public void downloadWithDuplicateInterruptionsDoesNotSuppress() throws Exception { | |
| Path stdout = fs.getPath("/execroot/stdout"); | |
| Path stderr = fs.getPath("/execroot/stderr"); | |
| InMemoryRemoteCache cache = newRemoteCache(); | |
| Digest digest1 = cache.addContents("file1"); | |
| InterruptedException reusedInterruption = new InterruptedException("reused interruption"); | |
| Digest digest2 = cache.addException("file2", reusedInterruption); | |
| Digest digest3 = cache.addException("file3", reusedInterruption); | |
| ActionResult result = | |
| ActionResult.newBuilder() | |
| .setExitCode(0) | |
| .addOutputFiles(OutputFile.newBuilder().setPath("file1").setDigest(digest1)) | |
| .addOutputFiles(OutputFile.newBuilder().setPath("file2").setDigest(digest2)) | |
| .addOutputFiles(OutputFile.newBuilder().setPath("file3").setDigest(digest3)) | |
| .build(); | |
| InterruptedException e = | |
| assertThrows( | |
| InterruptedException.class, | |
| () -> | |
| cache.download( | |
| result, execRoot, new FileOutErr(stdout, stderr), outputFilesLocker)); | |
| assertThat(e.getSuppressed()).isEmpty(); | |
| assertThat(Throwables.getRootCause(e)).hasMessageThat().isEqualTo("reused interruption"); | |
| } | |
| @Test | |
| public void testDownloadWithStdoutStderrOnSuccess() throws Exception { | |
| // Tests that fetching stdout/stderr as a digest works and that OutErr is still | |
| // writable afterwards. | |
| Path stdout = fs.getPath("/execroot/stdout"); | |
| Path stderr = fs.getPath("/execroot/stderr"); | |
| FileOutErr outErr = new FileOutErr(stdout, stderr); | |
| FileOutErr childOutErr = outErr.childOutErr(); | |
| FileOutErr spyOutErr = Mockito.spy(outErr); | |
| FileOutErr spyChildOutErr = Mockito.spy(childOutErr); | |
| when(spyOutErr.childOutErr()).thenReturn(spyChildOutErr); | |
| InMemoryRemoteCache cache = newRemoteCache(); | |
| Digest digestStdout = cache.addContents("stdout"); | |
| Digest digestStderr = cache.addContents("stderr"); | |
| ActionResult result = | |
| ActionResult.newBuilder() | |
| .setExitCode(0) | |
| .setStdoutDigest(digestStdout) | |
| .setStderrDigest(digestStderr) | |
| .build(); | |
| cache.download(result, execRoot, spyOutErr, outputFilesLocker); | |
| verify(spyOutErr, Mockito.times(2)).childOutErr(); | |
| verify(spyChildOutErr).clearOut(); | |
| verify(spyChildOutErr).clearErr(); | |
| assertThat(outErr.getOutputPath().exists()).isTrue(); | |
| assertThat(outErr.getErrorPath().exists()).isTrue(); | |
| try { | |
| outErr.getOutputStream().write(0); | |
| outErr.getErrorStream().write(0); | |
| } catch (IOException err) { | |
| throw new AssertionError("outErr should still be writable after download finished.", err); | |
| } | |
| verify(outputFilesLocker).lock(); | |
| } | |
| @Test | |
| public void testDownloadWithStdoutStderrOnFailure() throws Exception { | |
| // Test that when downloading stdout/stderr fails the OutErr is still writable | |
| // and empty. | |
| Path stdout = fs.getPath("/execroot/stdout"); | |
| Path stderr = fs.getPath("/execroot/stderr"); | |
| FileOutErr outErr = new FileOutErr(stdout, stderr); | |
| FileOutErr childOutErr = outErr.childOutErr(); | |
| FileOutErr spyOutErr = Mockito.spy(outErr); | |
| FileOutErr spyChildOutErr = Mockito.spy(childOutErr); | |
| when(spyOutErr.childOutErr()).thenReturn(spyChildOutErr); | |
| InMemoryRemoteCache cache = newRemoteCache(); | |
| // Don't add stdout/stderr as a known blob to the remote cache so that downloading it will fail | |
| Digest digestStdout = digestUtil.computeAsUtf8("stdout"); | |
| Digest digestStderr = digestUtil.computeAsUtf8("stderr"); | |
| ActionResult result = | |
| ActionResult.newBuilder() | |
| .setExitCode(0) | |
| .setStdoutDigest(digestStdout) | |
| .setStderrDigest(digestStderr) | |
| .build(); | |
| assertThrows( | |
| BulkTransferException.class, | |
| () -> cache.download(result, execRoot, spyOutErr, outputFilesLocker)); | |
| verify(spyOutErr, Mockito.times(2)).childOutErr(); | |
| verify(spyChildOutErr).clearOut(); | |
| verify(spyChildOutErr).clearErr(); | |
| assertThat(outErr.getOutputPath().exists()).isFalse(); | |
| assertThat(outErr.getErrorPath().exists()).isFalse(); | |
| try { | |
| outErr.getOutputStream().write(0); | |
| outErr.getErrorStream().write(0); | |
| } catch (IOException err) { | |
| throw new AssertionError("outErr should still be writable after download failed.", err); | |
| } | |
| verify(outputFilesLocker, never()).lock(); | |
| } | |
| @Test | |
| public void testDownloadClashes() throws Exception { | |
| // Test that injecting the metadata for a remote output file works | |
| // arrange | |
| InMemoryRemoteCache remoteCache = newRemoteCache(); | |
| Digest d1 = remoteCache.addContents("content1"); | |
| Digest d2 = remoteCache.addContents("content2"); | |
| ActionResult r = | |
| ActionResult.newBuilder() | |
| .setExitCode(0) | |
| .addOutputFiles(OutputFile.newBuilder().setPath("outputs/foo.tmp").setDigest(d1)) | |
| .addOutputFiles(OutputFile.newBuilder().setPath("outputs/foo").setDigest(d2)) | |
| .build(); | |
| Artifact a1 = ActionsTestUtil.createArtifact(artifactRoot, "foo.tmp"); | |
| Artifact a2 = ActionsTestUtil.createArtifact(artifactRoot, "foo"); | |
| // act | |
| remoteCache.download(r, execRoot, new FileOutErr(), outputFilesLocker); | |
| // assert | |
| assertThat(FileSystemUtils.readContent(a1.getPath(), StandardCharsets.UTF_8)) | |
| .isEqualTo("content1"); | |
| assertThat(FileSystemUtils.readContent(a2.getPath(), StandardCharsets.UTF_8)) | |
| .isEqualTo("content2"); | |
| verify(outputFilesLocker).lock(); | |
| } | |
| @Test | |
| public void testDownloadMinimalFiles() throws Exception { | |
| // Test that injecting the metadata for a remote output file works | |
| // arrange | |
| InMemoryRemoteCache remoteCache = newRemoteCache(); | |
| Digest d1 = remoteCache.addContents("content1"); | |
| Digest d2 = remoteCache.addContents("content2"); | |
| ActionResult r = | |
| ActionResult.newBuilder() | |
| .setExitCode(0) | |
| .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file1").setDigest(d1)) | |
| .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file2").setDigest(d2)) | |
| .build(); | |
| Artifact a1 = ActionsTestUtil.createArtifact(artifactRoot, "file1"); | |
| Artifact a2 = ActionsTestUtil.createArtifact(artifactRoot, "file2"); | |
| MetadataInjector injector = mock(MetadataInjector.class); | |
| // act | |
| InMemoryOutput inMemoryOutput = | |
| remoteCache.downloadMinimal( | |
| r, | |
| ImmutableList.of(a1, a2), | |
| /* inMemoryOutputPath= */ null, | |
| new FileOutErr(), | |
| execRoot, | |
| injector, | |
| outputFilesLocker); | |
| // assert | |
| assertThat(inMemoryOutput).isNull(); | |
| verify(injector) | |
| .injectRemoteFile(eq(a1), eq(toBinaryDigest(d1)), eq(d1.getSizeBytes()), anyInt()); | |
| verify(injector) | |
| .injectRemoteFile(eq(a2), eq(toBinaryDigest(d2)), eq(d2.getSizeBytes()), anyInt()); | |
| Path outputBase = artifactRoot.getRoot().asPath(); | |
| assertThat(outputBase.readdir(Symlinks.NOFOLLOW)).isEmpty(); | |
| verify(outputFilesLocker).lock(); | |
| } | |
| @Test | |
| public void testDownloadMinimalDirectory() throws Exception { | |
| // Test that injecting the metadata for a tree artifact / remote output directory works | |
| // arrange | |
| InMemoryRemoteCache remoteCache = newRemoteCache(); | |
| // Output Directory: | |
| // dir/file1 | |
| // dir/a/file2 | |
| Digest d1 = remoteCache.addContents("content1"); | |
| Digest d2 = remoteCache.addContents("content2"); | |
| FileNode file1 = FileNode.newBuilder().setName("file1").setDigest(d1).build(); | |
| FileNode file2 = FileNode.newBuilder().setName("file2").setDigest(d2).build(); | |
| Directory a = Directory.newBuilder().addFiles(file2).build(); | |
| Digest da = remoteCache.addContents(a); | |
| Directory root = | |
| Directory.newBuilder() | |
| .addFiles(file1) | |
| .addDirectories(DirectoryNode.newBuilder().setName("a").setDigest(da)) | |
| .build(); | |
| Tree t = Tree.newBuilder().setRoot(root).addChildren(a).build(); | |
| Digest dt = remoteCache.addContents(t); | |
| ActionResult r = | |
| ActionResult.newBuilder() | |
| .setExitCode(0) | |
| .addOutputDirectories( | |
| OutputDirectory.newBuilder().setPath("outputs/dir").setTreeDigest(dt)) | |
| .build(); | |
| SpecialArtifact dir = | |
| new SpecialArtifact( | |
| artifactRoot, | |
| PathFragment.create("outputs/dir"), | |
| ActionsTestUtil.NULL_ARTIFACT_OWNER, | |
| SpecialArtifactType.TREE); | |
| MetadataInjector injector = mock(MetadataInjector.class); | |
| // act | |
| InMemoryOutput inMemoryOutput = | |
| remoteCache.downloadMinimal( | |
| r, | |
| ImmutableList.of(dir), | |
| /* inMemoryOutputPath= */ null, | |
| new FileOutErr(), | |
| execRoot, | |
| injector, | |
| outputFilesLocker); | |
| // assert | |
| assertThat(inMemoryOutput).isNull(); | |
| Map<PathFragment, RemoteFileArtifactValue> m = | |
| ImmutableMap.<PathFragment, RemoteFileArtifactValue>builder() | |
| .put( | |
| PathFragment.create("file1"), | |
| new RemoteFileArtifactValue(toBinaryDigest(d1), d1.getSizeBytes(), 1)) | |
| .put( | |
| PathFragment.create("a/file2"), | |
| new RemoteFileArtifactValue(toBinaryDigest(d2), d2.getSizeBytes(), 1)) | |
| .build(); | |
| verify(injector).injectRemoteDirectory(eq(dir), eq(m)); | |
| Path outputBase = artifactRoot.getRoot().asPath(); | |
| assertThat(outputBase.readdir(Symlinks.NOFOLLOW)).isEmpty(); | |
| verify(outputFilesLocker).lock(); | |
| } | |
| @Test | |
| public void testDownloadMinimalDirectoryFails() throws Exception { | |
| // Test that we properly fail when downloading the metadata of an output | |
| // directory fails | |
| // arrange | |
| InMemoryRemoteCache remoteCache = newRemoteCache(); | |
| // Output Directory: | |
| // dir/file1 | |
| // dir/a/file2 | |
| Digest d1 = remoteCache.addContents("content1"); | |
| Digest d2 = remoteCache.addContents("content2"); | |
| FileNode file1 = FileNode.newBuilder().setName("file1").setDigest(d1).build(); | |
| FileNode file2 = FileNode.newBuilder().setName("file2").setDigest(d2).build(); | |
| Directory a = Directory.newBuilder().addFiles(file2).build(); | |
| Digest da = remoteCache.addContents(a); | |
| Directory root = | |
| Directory.newBuilder() | |
| .addFiles(file1) | |
| .addDirectories(DirectoryNode.newBuilder().setName("a").setDigest(da)) | |
| .build(); | |
| Tree t = Tree.newBuilder().setRoot(root).addChildren(a).build(); | |
| // Downloading the tree will fail | |
| IOException downloadTreeException = new IOException("entry not found"); | |
| Digest dt = remoteCache.addException(t, downloadTreeException); | |
| ActionResult r = | |
| ActionResult.newBuilder() | |
| .setExitCode(0) | |
| .addOutputDirectories( | |
| OutputDirectory.newBuilder().setPath("outputs/dir").setTreeDigest(dt)) | |
| .build(); | |
| SpecialArtifact dir = | |
| new SpecialArtifact( | |
| artifactRoot, | |
| PathFragment.create("outputs/dir"), | |
| ActionsTestUtil.NULL_ARTIFACT_OWNER, | |
| SpecialArtifactType.TREE); | |
| MetadataInjector injector = mock(MetadataInjector.class); | |
| // act | |
| IOException e = | |
| assertThrows( | |
| IOException.class, | |
| () -> | |
| remoteCache.downloadMinimal( | |
| r, | |
| ImmutableList.of(dir), | |
| /* inMemoryOutputPath= */ null, | |
| new FileOutErr(), | |
| execRoot, | |
| injector, | |
| outputFilesLocker)); | |
| assertThat(e).isEqualTo(downloadTreeException); | |
| verify(outputFilesLocker, never()).lock(); | |
| } | |
| @Test | |
| public void testDownloadMinimalWithStdoutStderr() throws Exception { | |
| // Test that downloading of non-embedded stdout and stderr works | |
| // arrange | |
| InMemoryRemoteCache remoteCache = newRemoteCache(); | |
| Digest dOut = remoteCache.addContents("stdout"); | |
| Digest dErr = remoteCache.addContents("stderr"); | |
| ActionResult r = | |
| ActionResult.newBuilder() | |
| .setExitCode(0) | |
| .setStdoutDigest(dOut) | |
| .setStderrDigest(dErr) | |
| .build(); | |
| RecordingOutErr outErr = new RecordingOutErr(); | |
| MetadataInjector injector = mock(MetadataInjector.class); | |
| // act | |
| InMemoryOutput inMemoryOutput = | |
| remoteCache.downloadMinimal( | |
| r, | |
| ImmutableList.of(), | |
| /* inMemoryOutputPath= */ null, | |
| outErr, | |
| execRoot, | |
| injector, | |
| outputFilesLocker); | |
| // assert | |
| assertThat(inMemoryOutput).isNull(); | |
| assertThat(outErr.outAsLatin1()).isEqualTo("stdout"); | |
| assertThat(outErr.errAsLatin1()).isEqualTo("stderr"); | |
| Path outputBase = artifactRoot.getRoot().asPath(); | |
| assertThat(outputBase.readdir(Symlinks.NOFOLLOW)).isEmpty(); | |
| verify(outputFilesLocker).lock(); | |
| } | |
| @Test | |
| public void testDownloadMinimalWithInMemoryOutput() throws Exception { | |
| // Test that downloading an in memory output works | |
| // arrange | |
| InMemoryRemoteCache remoteCache = newRemoteCache(); | |
| Digest d1 = remoteCache.addContents("content1"); | |
| Digest d2 = remoteCache.addContents("content2"); | |
| ActionResult r = | |
| ActionResult.newBuilder() | |
| .setExitCode(0) | |
| .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file1").setDigest(d1)) | |
| .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file2").setDigest(d2)) | |
| .build(); | |
| Artifact a1 = ActionsTestUtil.createArtifact(artifactRoot, "file1"); | |
| Artifact a2 = ActionsTestUtil.createArtifact(artifactRoot, "file2"); | |
| MetadataInjector injector = mock(MetadataInjector.class); | |
| // a1 should be provided as an InMemoryOutput | |
| PathFragment inMemoryOutputPathFragment = a1.getPath().relativeTo(execRoot); | |
| // act | |
| InMemoryOutput inMemoryOutput = | |
| remoteCache.downloadMinimal( | |
| r, | |
| ImmutableList.of(a1, a2), | |
| inMemoryOutputPathFragment, | |
| new FileOutErr(), | |
| execRoot, | |
| injector, | |
| outputFilesLocker); | |
| // assert | |
| assertThat(inMemoryOutput).isNotNull(); | |
| ByteString expectedContents = ByteString.copyFrom("content1", UTF_8); | |
| assertThat(inMemoryOutput.getContents()).isEqualTo(expectedContents); | |
| assertThat(inMemoryOutput.getOutput()).isEqualTo(a1); | |
| // The in memory file also needs to be injected as an output | |
| verify(injector) | |
| .injectRemoteFile(eq(a1), eq(toBinaryDigest(d1)), eq(d1.getSizeBytes()), anyInt()); | |
| verify(injector) | |
| .injectRemoteFile(eq(a2), eq(toBinaryDigest(d2)), eq(d2.getSizeBytes()), anyInt()); | |
| Path outputBase = artifactRoot.getRoot().asPath(); | |
| assertThat(outputBase.readdir(Symlinks.NOFOLLOW)).isEmpty(); | |
| verify(outputFilesLocker).lock(); | |
| } | |
| @Test | |
| public void testDownloadEmptyBlobAndFile() throws Exception { | |
| // Test that downloading an empty BLOB/file does not try to perform a download. | |
| // arrange | |
| Path file = fs.getPath("/execroot/file"); | |
| RemoteCache remoteCache = newRemoteCache(); | |
| Digest emptyDigest = digestUtil.compute(new byte[0]); | |
| // act and assert | |
| assertThat(Utils.getFromFuture(remoteCache.downloadBlob(emptyDigest))).isEmpty(); | |
| try (OutputStream out = file.getOutputStream()) { | |
| Utils.getFromFuture(remoteCache.downloadFile(file, emptyDigest)); | |
| } | |
| assertThat(file.exists()).isTrue(); | |
| assertThat(file.getFileSize()).isEqualTo(0); | |
| } | |
| @Test | |
| public void testDownloadDirectory() throws Exception { | |
| // Test that downloading an output directory works. | |
| // arrange | |
| final ConcurrentMap<Digest, byte[]> cas = new ConcurrentHashMap<>(); | |
| Digest fooDigest = digestUtil.computeAsUtf8("foo-contents"); | |
| cas.put(fooDigest, "foo-contents".getBytes(Charsets.UTF_8)); | |
| Digest quxDigest = digestUtil.computeAsUtf8("qux-contents"); | |
| cas.put(quxDigest, "qux-contents".getBytes(Charsets.UTF_8)); | |
| Tree barTreeMessage = | |
| Tree.newBuilder() | |
| .setRoot( | |
| Directory.newBuilder() | |
| .addFiles( | |
| FileNode.newBuilder() | |
| .setName("qux") | |
| .setDigest(quxDigest) | |
| .setIsExecutable(true))) | |
| .build(); | |
| Digest barTreeDigest = digestUtil.compute(barTreeMessage); | |
| cas.put(barTreeDigest, barTreeMessage.toByteArray()); | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| result.addOutputFilesBuilder().setPath("a/foo").setDigest(fooDigest); | |
| result.addOutputDirectoriesBuilder().setPath("a/bar").setTreeDigest(barTreeDigest); | |
| // act | |
| RemoteCache remoteCache = newRemoteCache(cas); | |
| remoteCache.download(result.build(), execRoot, null, /* outputFilesLocker= */ () -> {}); | |
| // assert | |
| assertThat(digestUtil.compute(execRoot.getRelative("a/foo"))).isEqualTo(fooDigest); | |
| assertThat(digestUtil.compute(execRoot.getRelative("a/bar/qux"))).isEqualTo(quxDigest); | |
| assertThat(execRoot.getRelative("a/bar/qux").isExecutable()).isTrue(); | |
| } | |
| @Test | |
| public void testDownloadEmptyDirectory() throws Exception { | |
| // Test that downloading an empty output directory works. | |
| // arrange | |
| Tree barTreeMessage = Tree.newBuilder().setRoot(Directory.newBuilder()).build(); | |
| Digest barTreeDigest = digestUtil.compute(barTreeMessage); | |
| final ConcurrentMap<Digest, byte[]> map = new ConcurrentHashMap<>(); | |
| map.put(barTreeDigest, barTreeMessage.toByteArray()); | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| result.addOutputDirectoriesBuilder().setPath("a/bar").setTreeDigest(barTreeDigest); | |
| // act | |
| RemoteCache remoteCache = newRemoteCache(map); | |
| remoteCache.download(result.build(), execRoot, null, /* outputFilesLocker= */ () -> {}); | |
| // assert | |
| assertThat(execRoot.getRelative("a/bar").isDirectory()).isTrue(); | |
| } | |
| @Test | |
| public void testDownloadNestedDirectory() throws Exception { | |
| // Test that downloading a nested output directory works. | |
| // arrange | |
| Digest fooDigest = digestUtil.computeAsUtf8("foo-contents"); | |
| Digest quxDigest = digestUtil.computeAsUtf8("qux-contents"); | |
| Directory wobbleDirMessage = | |
| Directory.newBuilder() | |
| .addFiles(FileNode.newBuilder().setName("qux").setDigest(quxDigest)) | |
| .build(); | |
| Digest wobbleDirDigest = digestUtil.compute(wobbleDirMessage); | |
| Tree barTreeMessage = | |
| Tree.newBuilder() | |
| .setRoot( | |
| Directory.newBuilder() | |
| .addFiles( | |
| FileNode.newBuilder() | |
| .setName("qux") | |
| .setDigest(quxDigest) | |
| .setIsExecutable(true)) | |
| .addDirectories( | |
| DirectoryNode.newBuilder().setName("wobble").setDigest(wobbleDirDigest))) | |
| .addChildren(wobbleDirMessage) | |
| .build(); | |
| Digest barTreeDigest = digestUtil.compute(barTreeMessage); | |
| final ConcurrentMap<Digest, byte[]> map = new ConcurrentHashMap<>(); | |
| map.put(fooDigest, "foo-contents".getBytes(Charsets.UTF_8)); | |
| map.put(barTreeDigest, barTreeMessage.toByteArray()); | |
| map.put(quxDigest, "qux-contents".getBytes(Charsets.UTF_8)); | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| result.addOutputFilesBuilder().setPath("a/foo").setDigest(fooDigest); | |
| result.addOutputDirectoriesBuilder().setPath("a/bar").setTreeDigest(barTreeDigest); | |
| // act | |
| RemoteCache remoteCache = newRemoteCache(map); | |
| remoteCache.download(result.build(), execRoot, null, /* outputFilesLocker= */ () -> {}); | |
| // assert | |
| assertThat(digestUtil.compute(execRoot.getRelative("a/foo"))).isEqualTo(fooDigest); | |
| assertThat(digestUtil.compute(execRoot.getRelative("a/bar/wobble/qux"))).isEqualTo(quxDigest); | |
| assertThat(execRoot.getRelative("a/bar/wobble/qux").isExecutable()).isFalse(); | |
| } | |
| @Test | |
| public void testDownloadDirectoryWithSameHash() throws Exception { | |
| // Test that downloading an output directory works when two Directory | |
| // protos have the same hash i.e. because they have the same name and contents or are empty. | |
| /* | |
| * /bar/foo/file | |
| * /foo/file | |
| */ | |
| // arrange | |
| Digest fileDigest = digestUtil.computeAsUtf8("file"); | |
| FileNode file = FileNode.newBuilder().setName("file").setDigest(fileDigest).build(); | |
| Directory fooDir = Directory.newBuilder().addFiles(file).build(); | |
| Digest fooDigest = digestUtil.compute(fooDir); | |
| DirectoryNode fooDirNode = | |
| DirectoryNode.newBuilder().setName("foo").setDigest(fooDigest).build(); | |
| Directory barDir = Directory.newBuilder().addDirectories(fooDirNode).build(); | |
| Digest barDigest = digestUtil.compute(barDir); | |
| DirectoryNode barDirNode = | |
| DirectoryNode.newBuilder().setName("bar").setDigest(barDigest).build(); | |
| Directory rootDir = | |
| Directory.newBuilder().addDirectories(fooDirNode).addDirectories(barDirNode).build(); | |
| Tree tree = | |
| Tree.newBuilder() | |
| .setRoot(rootDir) | |
| .addChildren(barDir) | |
| .addChildren(fooDir) | |
| .addChildren(fooDir) | |
| .build(); | |
| Digest treeDigest = digestUtil.compute(tree); | |
| final ConcurrentMap<Digest, byte[]> map = new ConcurrentHashMap<>(); | |
| map.put(fileDigest, "file".getBytes(Charsets.UTF_8)); | |
| map.put(treeDigest, tree.toByteArray()); | |
| ActionResult.Builder result = ActionResult.newBuilder(); | |
| result.addOutputDirectoriesBuilder().setPath("a/").setTreeDigest(treeDigest); | |
| // act | |
| RemoteCache remoteCache = newRemoteCache(map); | |
| remoteCache.download(result.build(), execRoot, null, /* outputFilesLocker= */ () -> {}); | |
| // assert | |
| assertThat(digestUtil.compute(execRoot.getRelative("a/bar/foo/file"))).isEqualTo(fileDigest); | |
| assertThat(digestUtil.compute(execRoot.getRelative("a/foo/file"))).isEqualTo(fileDigest); | |
| } | |
| @Test | |
| public void testUploadDirectory() throws Exception { | |
| // Test that uploading a directory works. | |
| // arrange | |
| Digest fooDigest = fakeFileCache.createScratchInput(ActionInputHelper.fromPath("a/foo"), "xyz"); | |
| Digest quxDigest = | |
| fakeFileCache.createScratchInput(ActionInputHelper.fromPath("bar/qux"), "abc"); | |
| Digest barDigest = | |
| fakeFileCache.createScratchInputDirectory( | |
| ActionInputHelper.fromPath("bar"), | |
| Tree.newBuilder() | |
| .setRoot( | |
| Directory.newBuilder() | |
| .addFiles( | |
| FileNode.newBuilder() | |
| .setIsExecutable(true) | |
| .setName("qux") | |
| .setDigest(quxDigest) | |
| .build()) | |
| .build()) | |
| .build()); | |
| Path fooFile = execRoot.getRelative("a/foo"); | |
| Path quxFile = execRoot.getRelative("bar/qux"); | |
| quxFile.setExecutable(true); | |
| Path barDir = execRoot.getRelative("bar"); | |
| Command cmd = Command.newBuilder().addOutputFiles("bla").build(); | |
| Digest cmdDigest = digestUtil.compute(cmd); | |
| Action action = Action.newBuilder().setCommandDigest(cmdDigest).build(); | |
| Digest actionDigest = digestUtil.compute(action); | |
| // act | |
| InMemoryRemoteCache remoteCache = newRemoteCache(); | |
| ActionResult result = | |
| remoteCache.upload( | |
| digestUtil.asActionKey(actionDigest), | |
| action, | |
| cmd, | |
| execRoot, | |
| ImmutableList.of(fooFile, barDir), | |
| new FileOutErr(execRoot.getRelative("stdout"), execRoot.getRelative("stderr"))); | |
| // assert | |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); | |
| expectedResult.addOutputFilesBuilder().setPath("a/foo").setDigest(fooDigest); | |
| expectedResult.addOutputDirectoriesBuilder().setPath("bar").setTreeDigest(barDigest); | |
| assertThat(result).isEqualTo(expectedResult.build()); | |
| ImmutableList<Digest> toQuery = | |
| ImmutableList.of(fooDigest, quxDigest, barDigest, cmdDigest, actionDigest); | |
| assertThat(remoteCache.findMissingDigests(toQuery)).isEmpty(); | |
| } | |
| @Test | |
| public void testUploadEmptyDirectory() throws Exception { | |
| // Test that uploading an empty directory works. | |
| // arrange | |
| final Digest barDigest = | |
| fakeFileCache.createScratchInputDirectory( | |
| ActionInputHelper.fromPath("bar"), | |
| Tree.newBuilder().setRoot(Directory.newBuilder().build()).build()); | |
| final Path barDir = execRoot.getRelative("bar"); | |
| Action action = Action.getDefaultInstance(); | |
| ActionKey actionDigest = digestUtil.computeActionKey(action); | |
| Command cmd = Command.getDefaultInstance(); | |
| // act | |
| InMemoryRemoteCache remoteCache = newRemoteCache(); | |
| ActionResult result = | |
| remoteCache.upload( | |
| actionDigest, | |
| action, | |
| cmd, | |
| execRoot, | |
| ImmutableList.of(barDir), | |
| new FileOutErr(execRoot.getRelative("stdout"), execRoot.getRelative("stderr"))); | |
| // assert | |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); | |
| expectedResult.addOutputDirectoriesBuilder().setPath("bar").setTreeDigest(barDigest); | |
| assertThat(result).isEqualTo(expectedResult.build()); | |
| assertThat(remoteCache.findMissingDigests(ImmutableList.of(barDigest))).isEmpty(); | |
| } | |
| @Test | |
| public void testUploadNestedDirectory() throws Exception { | |
| // Test that uploading a nested directory works. | |
| // arrange | |
| final Digest wobbleDigest = | |
| fakeFileCache.createScratchInput(ActionInputHelper.fromPath("bar/test/wobble"), "xyz"); | |
| final Digest quxDigest = | |
| fakeFileCache.createScratchInput(ActionInputHelper.fromPath("bar/qux"), "abc"); | |
| final Directory testDirMessage = | |
| Directory.newBuilder() | |
| .addFiles(FileNode.newBuilder().setName("wobble").setDigest(wobbleDigest).build()) | |
| .build(); | |
| final Digest testDigest = digestUtil.compute(testDirMessage); | |
| final Tree barTree = | |
| Tree.newBuilder() | |
| .setRoot( | |
| Directory.newBuilder() | |
| .addFiles( | |
| FileNode.newBuilder() | |
| .setIsExecutable(true) | |
| .setName("qux") | |
| .setDigest(quxDigest)) | |
| .addDirectories( | |
| DirectoryNode.newBuilder().setName("test").setDigest(testDigest))) | |
| .addChildren(testDirMessage) | |
| .build(); | |
| final Digest barDigest = | |
| fakeFileCache.createScratchInputDirectory(ActionInputHelper.fromPath("bar"), barTree); | |
| final Path quxFile = execRoot.getRelative("bar/qux"); | |
| quxFile.setExecutable(true); | |
| final Path barDir = execRoot.getRelative("bar"); | |
| Action action = Action.getDefaultInstance(); | |
| ActionKey actionDigest = digestUtil.computeActionKey(action); | |
| Command cmd = Command.getDefaultInstance(); | |
| // act | |
| InMemoryRemoteCache remoteCache = newRemoteCache(); | |
| ActionResult result = | |
| remoteCache.upload( | |
| actionDigest, | |
| action, | |
| cmd, | |
| execRoot, | |
| ImmutableList.of(barDir), | |
| new FileOutErr(execRoot.getRelative("stdout"), execRoot.getRelative("stderr"))); | |
| // assert | |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); | |
| expectedResult.addOutputDirectoriesBuilder().setPath("bar").setTreeDigest(barDigest); | |
| assertThat(result).isEqualTo(expectedResult.build()); | |
| ImmutableList<Digest> toQuery = ImmutableList.of(wobbleDigest, quxDigest, barDigest); | |
| assertThat(remoteCache.findMissingDigests(toQuery)).isEmpty(); | |
| } | |
| private InMemoryRemoteCache newRemoteCache(Map<Digest, byte[]> casEntries) { | |
| RemoteOptions options = Options.getDefaults(RemoteOptions.class); | |
| return new InMemoryRemoteCache(casEntries, options, digestUtil); | |
| } | |
| private InMemoryRemoteCache newRemoteCache() { | |
| RemoteOptions options = Options.getDefaults(RemoteOptions.class); | |
| return new InMemoryRemoteCache(options, digestUtil); | |
| } | |
| private static class InMemoryRemoteCache extends RemoteCache { | |
| InMemoryRemoteCache( | |
| Map<Digest, byte[]> casEntries, RemoteOptions options, DigestUtil digestUtil) { | |
| super(new InMemoryCacheClient(casEntries), options, digestUtil); | |
| } | |
| InMemoryRemoteCache(RemoteOptions options, DigestUtil digestUtil) { | |
| super(new InMemoryCacheClient(), options, digestUtil); | |
| } | |
| Digest addContents(String txt) throws IOException, InterruptedException { | |
| return addContents(txt.getBytes(UTF_8)); | |
| } | |
| Digest addContents(byte[] bytes) throws IOException, InterruptedException { | |
| Digest digest = digestUtil.compute(bytes); | |
| Utils.getFromFuture(cacheProtocol.uploadBlob(digest, ByteString.copyFrom(bytes))); | |
| return digest; | |
| } | |
| Digest addContents(Message m) throws IOException, InterruptedException { | |
| return addContents(m.toByteArray()); | |
| } | |
| Digest addException(String txt, Exception e) { | |
| Digest digest = digestUtil.compute(txt.getBytes(UTF_8)); | |
| ((InMemoryCacheClient) cacheProtocol).addDownloadFailure(digest, e); | |
| return digest; | |
| } | |
| Digest addException(Message m, Exception e) { | |
| Digest digest = digestUtil.compute(m); | |
| ((InMemoryCacheClient) cacheProtocol).addDownloadFailure(digest, e); | |
| return digest; | |
| } | |
| int getNumSuccessfulDownloads() { | |
| return ((InMemoryCacheClient) cacheProtocol).getNumSuccessfulDownloads(); | |
| } | |
| int getNumFailedDownloads() { | |
| return ((InMemoryCacheClient) cacheProtocol).getNumFailedDownloads(); | |
| } | |
| ImmutableSet<Digest> findMissingDigests(Iterable<Digest> digests) | |
| throws IOException, InterruptedException { | |
| return Utils.getFromFuture(cacheProtocol.findMissingDigests(digests)); | |
| } | |
| @Override | |
| public void close() { | |
| cacheProtocol.close(); | |
| } | |
| } | |
| } |