| // Copyright 2018 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| package com.google.devtools.build.lib.remote; |
| |
| import static com.google.common.truth.Truth.assertThat; |
| import static com.google.devtools.build.lib.remote.util.DigestUtil.toBinaryDigest; |
| import static com.google.devtools.build.lib.testutil.MoreAsserts.assertThrows; |
| import static java.nio.charset.StandardCharsets.UTF_8; |
| import static org.mockito.ArgumentMatchers.anyInt; |
| import static org.mockito.Matchers.eq; |
| import static org.mockito.Mockito.mock; |
| import static org.mockito.Mockito.never; |
| import static org.mockito.Mockito.verify; |
| import static org.mockito.Mockito.when; |
| |
| import build.bazel.remote.execution.v2.ActionResult; |
| import build.bazel.remote.execution.v2.Digest; |
| import build.bazel.remote.execution.v2.Directory; |
| import build.bazel.remote.execution.v2.DirectoryNode; |
| import build.bazel.remote.execution.v2.FileNode; |
| import build.bazel.remote.execution.v2.OutputDirectory; |
| import build.bazel.remote.execution.v2.OutputFile; |
| import build.bazel.remote.execution.v2.SymlinkNode; |
| import build.bazel.remote.execution.v2.Tree; |
| import com.google.common.base.Throwables; |
| import com.google.common.collect.ImmutableList; |
| import com.google.common.collect.ImmutableMap; |
| import com.google.common.collect.ImmutableSet; |
| import com.google.common.util.concurrent.FutureCallback; |
| import com.google.common.util.concurrent.Futures; |
| import com.google.common.util.concurrent.ListenableFuture; |
| import com.google.common.util.concurrent.ListeningScheduledExecutorService; |
| import com.google.common.util.concurrent.MoreExecutors; |
| import com.google.common.util.concurrent.SettableFuture; |
| import com.google.devtools.build.lib.actions.Artifact; |
| import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact; |
| import com.google.devtools.build.lib.actions.Artifact.SpecialArtifactType; |
| import com.google.devtools.build.lib.actions.ArtifactRoot; |
| import com.google.devtools.build.lib.actions.ExecException; |
| import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue; |
| import com.google.devtools.build.lib.actions.cache.MetadataInjector; |
| import com.google.devtools.build.lib.actions.util.ActionsTestUtil; |
| import com.google.devtools.build.lib.clock.JavaClock; |
| import com.google.devtools.build.lib.remote.AbstractRemoteActionCache.OutputFilesLocker; |
| import com.google.devtools.build.lib.remote.AbstractRemoteActionCache.UploadManifest; |
| import com.google.devtools.build.lib.remote.common.SimpleBlobStore.ActionKey; |
| import com.google.devtools.build.lib.remote.options.RemoteOptions; |
| import com.google.devtools.build.lib.remote.util.DigestUtil; |
| import com.google.devtools.build.lib.remote.util.Utils; |
| import com.google.devtools.build.lib.remote.util.Utils.InMemoryOutput; |
| import com.google.devtools.build.lib.util.io.FileOutErr; |
| import com.google.devtools.build.lib.util.io.RecordingOutErr; |
| import com.google.devtools.build.lib.vfs.DigestHashFunction; |
| import com.google.devtools.build.lib.vfs.FileSystem; |
| import com.google.devtools.build.lib.vfs.FileSystemUtils; |
| import com.google.devtools.build.lib.vfs.Path; |
| import com.google.devtools.build.lib.vfs.PathFragment; |
| import com.google.devtools.build.lib.vfs.Symlinks; |
| import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; |
| import com.google.devtools.common.options.Options; |
| import com.google.protobuf.ByteString; |
| import com.google.protobuf.Message; |
| import java.io.IOException; |
| import java.io.OutputStream; |
| import java.nio.charset.StandardCharsets; |
| import java.util.ArrayList; |
| import java.util.HashMap; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.concurrent.Executors; |
| import java.util.concurrent.atomic.AtomicInteger; |
| import javax.annotation.Nullable; |
| import org.junit.AfterClass; |
| import org.junit.Before; |
| import org.junit.BeforeClass; |
| import org.junit.Test; |
| import org.junit.runner.RunWith; |
| import org.junit.runners.JUnit4; |
| import org.mockito.Mock; |
| import org.mockito.Mockito; |
| import org.mockito.MockitoAnnotations; |
| |
| /** Tests for {@link AbstractRemoteActionCache}. */ |
| @RunWith(JUnit4.class) |
| public class AbstractRemoteActionCacheTests { |
| |
| @Mock private OutputFilesLocker outputFilesLocker; |
| |
| private FileSystem fs; |
| private Path execRoot; |
| ArtifactRoot artifactRoot; |
| private final DigestUtil digestUtil = new DigestUtil(DigestHashFunction.SHA256); |
| |
| private static ListeningScheduledExecutorService retryService; |
| |
| @BeforeClass |
| public static void beforeEverything() { |
| retryService = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1)); |
| } |
| |
| @Before |
| public void setUp() throws Exception { |
| MockitoAnnotations.initMocks(this); |
| fs = new InMemoryFileSystem(new JavaClock(), DigestHashFunction.SHA256); |
| execRoot = fs.getPath("/execroot"); |
| execRoot.createDirectoryAndParents(); |
| artifactRoot = ArtifactRoot.asDerivedRoot(execRoot, execRoot.getChild("outputs")); |
| artifactRoot.getRoot().asPath().createDirectoryAndParents(); |
| } |
| |
| @AfterClass |
| public static void afterEverything() { |
| retryService.shutdownNow(); |
| } |
| |
| @Test |
| public void uploadAbsoluteFileSymlinkAsFile() throws Exception { |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| Path link = fs.getPath("/execroot/link"); |
| Path target = fs.getPath("/execroot/target"); |
| FileSystemUtils.writeContent(target, new byte[] {1, 2, 3, 4, 5}); |
| link.createSymbolicLink(target); |
| |
| UploadManifest um = |
| new UploadManifest( |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); |
| um.addFiles(ImmutableList.of(link)); |
| Digest digest = digestUtil.compute(target); |
| assertThat(um.getDigestToFile()).containsExactly(digest, link); |
| |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); |
| expectedResult.addOutputFilesBuilder().setPath("link").setDigest(digest); |
| assertThat(result.build()).isEqualTo(expectedResult.build()); |
| } |
| |
| @Test |
| public void uploadAbsoluteDirectorySymlinkAsDirectory() throws Exception { |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| Path dir = fs.getPath("/execroot/dir"); |
| dir.createDirectory(); |
| Path foo = fs.getPath("/execroot/dir/foo"); |
| FileSystemUtils.writeContent(foo, new byte[] {1, 2, 3, 4, 5}); |
| Path link = fs.getPath("/execroot/link"); |
| link.createSymbolicLink(dir); |
| |
| UploadManifest um = |
| new UploadManifest( |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); |
| um.addFiles(ImmutableList.of(link)); |
| Digest digest = digestUtil.compute(foo); |
| assertThat(um.getDigestToFile()).containsExactly(digest, fs.getPath("/execroot/link/foo")); |
| |
| Tree tree = |
| Tree.newBuilder() |
| .setRoot( |
| Directory.newBuilder() |
| .addFiles(FileNode.newBuilder().setName("foo").setDigest(digest))) |
| .build(); |
| Digest treeDigest = digestUtil.compute(tree); |
| |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); |
| expectedResult.addOutputDirectoriesBuilder().setPath("link").setTreeDigest(treeDigest); |
| assertThat(result.build()).isEqualTo(expectedResult.build()); |
| } |
| |
| @Test |
| public void uploadRelativeFileSymlinkAsFile() throws Exception { |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| Path link = fs.getPath("/execroot/link"); |
| Path target = fs.getPath("/execroot/target"); |
| FileSystemUtils.writeContent(target, new byte[] {1, 2, 3, 4, 5}); |
| link.createSymbolicLink(target.relativeTo(execRoot)); |
| |
| UploadManifest um = |
| new UploadManifest( |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ false, /*allowSymlinks=*/ true); |
| um.addFiles(ImmutableList.of(link)); |
| Digest digest = digestUtil.compute(target); |
| assertThat(um.getDigestToFile()).containsExactly(digest, link); |
| |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); |
| expectedResult.addOutputFilesBuilder().setPath("link").setDigest(digest); |
| assertThat(result.build()).isEqualTo(expectedResult.build()); |
| } |
| |
| @Test |
| public void uploadRelativeDirectorySymlinkAsDirectory() throws Exception { |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| Path dir = fs.getPath("/execroot/dir"); |
| dir.createDirectory(); |
| Path foo = fs.getPath("/execroot/dir/foo"); |
| FileSystemUtils.writeContent(foo, new byte[] {1, 2, 3, 4, 5}); |
| Path link = fs.getPath("/execroot/link"); |
| link.createSymbolicLink(dir.relativeTo(execRoot)); |
| |
| UploadManifest um = |
| new UploadManifest( |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ false, /*allowSymlinks=*/ true); |
| um.addFiles(ImmutableList.of(link)); |
| Digest digest = digestUtil.compute(foo); |
| assertThat(um.getDigestToFile()).containsExactly(digest, fs.getPath("/execroot/link/foo")); |
| |
| Tree tree = |
| Tree.newBuilder() |
| .setRoot( |
| Directory.newBuilder() |
| .addFiles(FileNode.newBuilder().setName("foo").setDigest(digest))) |
| .build(); |
| Digest treeDigest = digestUtil.compute(tree); |
| |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); |
| expectedResult.addOutputDirectoriesBuilder().setPath("link").setTreeDigest(treeDigest); |
| assertThat(result.build()).isEqualTo(expectedResult.build()); |
| } |
| |
| @Test |
| public void uploadRelativeFileSymlink() throws Exception { |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| Path link = fs.getPath("/execroot/link"); |
| Path target = fs.getPath("/execroot/target"); |
| FileSystemUtils.writeContent(target, new byte[] {1, 2, 3, 4, 5}); |
| link.createSymbolicLink(target.relativeTo(execRoot)); |
| |
| UploadManifest um = |
| new UploadManifest( |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); |
| um.addFiles(ImmutableList.of(link)); |
| assertThat(um.getDigestToFile()).isEmpty(); |
| |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); |
| expectedResult.addOutputFileSymlinksBuilder().setPath("link").setTarget("target"); |
| assertThat(result.build()).isEqualTo(expectedResult.build()); |
| } |
| |
| @Test |
| public void uploadRelativeDirectorySymlink() throws Exception { |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| Path dir = fs.getPath("/execroot/dir"); |
| dir.createDirectory(); |
| Path file = fs.getPath("/execroot/dir/foo"); |
| FileSystemUtils.writeContent(file, new byte[] {1, 2, 3, 4, 5}); |
| Path link = fs.getPath("/execroot/link"); |
| link.createSymbolicLink(dir.relativeTo(execRoot)); |
| |
| UploadManifest um = |
| new UploadManifest( |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); |
| um.addFiles(ImmutableList.of(link)); |
| assertThat(um.getDigestToFile()).isEmpty(); |
| |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); |
| expectedResult.addOutputDirectorySymlinksBuilder().setPath("link").setTarget("dir"); |
| assertThat(result.build()).isEqualTo(expectedResult.build()); |
| } |
| |
| @Test |
| public void uploadDanglingSymlinkError() throws Exception { |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| Path link = fs.getPath("/execroot/link"); |
| Path target = fs.getPath("/execroot/target"); |
| link.createSymbolicLink(target.relativeTo(execRoot)); |
| |
| UploadManifest um = |
| new UploadManifest( |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); |
| IOException e = assertThrows(IOException.class, () -> um.addFiles(ImmutableList.of(link))); |
| assertThat(e).hasMessageThat().contains("dangling"); |
| assertThat(e).hasMessageThat().contains("/execroot/link"); |
| assertThat(e).hasMessageThat().contains("target"); |
| } |
| |
| @Test |
| public void uploadSymlinksNoAllowError() throws Exception { |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| Path link = fs.getPath("/execroot/link"); |
| Path target = fs.getPath("/execroot/target"); |
| FileSystemUtils.writeContent(target, new byte[] {1, 2, 3, 4, 5}); |
| link.createSymbolicLink(target.relativeTo(execRoot)); |
| |
| UploadManifest um = |
| new UploadManifest( |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ false); |
| ExecException e = assertThrows(ExecException.class, () -> um.addFiles(ImmutableList.of(link))); |
| assertThat(e).hasMessageThat().contains("symbolic link"); |
| assertThat(e).hasMessageThat().contains("--remote_allow_symlink_upload"); |
| } |
| |
| @Test |
| public void uploadAbsoluteFileSymlinkInDirectoryAsFile() throws Exception { |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| Path dir = fs.getPath("/execroot/dir"); |
| dir.createDirectory(); |
| Path target = fs.getPath("/execroot/target"); |
| FileSystemUtils.writeContent(target, new byte[] {1, 2, 3, 4, 5}); |
| Path link = fs.getPath("/execroot/dir/link"); |
| link.createSymbolicLink(target); |
| |
| UploadManifest um = |
| new UploadManifest( |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); |
| um.addFiles(ImmutableList.of(dir)); |
| Digest digest = digestUtil.compute(target); |
| assertThat(um.getDigestToFile()).containsExactly(digest, link); |
| |
| Tree tree = |
| Tree.newBuilder() |
| .setRoot( |
| Directory.newBuilder() |
| .addFiles(FileNode.newBuilder().setName("link").setDigest(digest))) |
| .build(); |
| Digest treeDigest = digestUtil.compute(tree); |
| |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); |
| expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); |
| assertThat(result.build()).isEqualTo(expectedResult.build()); |
| } |
| |
| @Test |
| public void uploadAbsoluteDirectorySymlinkInDirectoryAsDirectory() throws Exception { |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| Path dir = fs.getPath("/execroot/dir"); |
| dir.createDirectory(); |
| Path bardir = fs.getPath("/execroot/bardir"); |
| bardir.createDirectory(); |
| Path foo = fs.getPath("/execroot/bardir/foo"); |
| FileSystemUtils.writeContent(foo, new byte[] {1, 2, 3, 4, 5}); |
| Path link = fs.getPath("/execroot/dir/link"); |
| link.createSymbolicLink(bardir); |
| |
| UploadManifest um = |
| new UploadManifest( |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); |
| um.addFiles(ImmutableList.of(dir)); |
| Digest digest = digestUtil.compute(foo); |
| assertThat(um.getDigestToFile()).containsExactly(digest, fs.getPath("/execroot/dir/link/foo")); |
| |
| Directory barDir = |
| Directory.newBuilder() |
| .addFiles(FileNode.newBuilder().setName("foo").setDigest(digest)) |
| .build(); |
| Digest barDigest = digestUtil.compute(barDir); |
| Tree tree = |
| Tree.newBuilder() |
| .setRoot( |
| Directory.newBuilder() |
| .addDirectories( |
| DirectoryNode.newBuilder().setName("link").setDigest(barDigest))) |
| .addChildren(barDir) |
| .build(); |
| Digest treeDigest = digestUtil.compute(tree); |
| |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); |
| expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); |
| assertThat(result.build()).isEqualTo(expectedResult.build()); |
| } |
| |
| @Test |
| public void uploadRelativeFileSymlinkInDirectoryAsFile() throws Exception { |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| Path dir = fs.getPath("/execroot/dir"); |
| dir.createDirectory(); |
| Path target = fs.getPath("/execroot/target"); |
| FileSystemUtils.writeContent(target, new byte[] {1, 2, 3, 4, 5}); |
| Path link = fs.getPath("/execroot/dir/link"); |
| link.createSymbolicLink(PathFragment.create("../target")); |
| |
| UploadManifest um = |
| new UploadManifest( |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ false, /*allowSymlinks=*/ true); |
| um.addFiles(ImmutableList.of(dir)); |
| Digest digest = digestUtil.compute(target); |
| assertThat(um.getDigestToFile()).containsExactly(digest, link); |
| |
| Tree tree = |
| Tree.newBuilder() |
| .setRoot( |
| Directory.newBuilder() |
| .addFiles(FileNode.newBuilder().setName("link").setDigest(digest))) |
| .build(); |
| Digest treeDigest = digestUtil.compute(tree); |
| |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); |
| expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); |
| assertThat(result.build()).isEqualTo(expectedResult.build()); |
| } |
| |
| @Test |
| public void uploadRelativeDirectorySymlinkInDirectoryAsDirectory() throws Exception { |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| Path dir = fs.getPath("/execroot/dir"); |
| dir.createDirectory(); |
| Path bardir = fs.getPath("/execroot/bardir"); |
| bardir.createDirectory(); |
| Path foo = fs.getPath("/execroot/bardir/foo"); |
| FileSystemUtils.writeContent(foo, new byte[] {1, 2, 3, 4, 5}); |
| Path link = fs.getPath("/execroot/dir/link"); |
| link.createSymbolicLink(PathFragment.create("../bardir")); |
| |
| UploadManifest um = |
| new UploadManifest( |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ false, /*allowSymlinks=*/ true); |
| um.addFiles(ImmutableList.of(dir)); |
| Digest digest = digestUtil.compute(foo); |
| assertThat(um.getDigestToFile()).containsExactly(digest, fs.getPath("/execroot/dir/link/foo")); |
| |
| Directory barDir = |
| Directory.newBuilder() |
| .addFiles(FileNode.newBuilder().setName("foo").setDigest(digest)) |
| .build(); |
| Digest barDigest = digestUtil.compute(barDir); |
| Tree tree = |
| Tree.newBuilder() |
| .setRoot( |
| Directory.newBuilder() |
| .addDirectories( |
| DirectoryNode.newBuilder().setName("link").setDigest(barDigest))) |
| .addChildren(barDir) |
| .build(); |
| Digest treeDigest = digestUtil.compute(tree); |
| |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); |
| expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); |
| assertThat(result.build()).isEqualTo(expectedResult.build()); |
| } |
| |
| @Test |
| public void uploadRelativeFileSymlinkInDirectory() throws Exception { |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| Path dir = fs.getPath("/execroot/dir"); |
| dir.createDirectory(); |
| Path target = fs.getPath("/execroot/target"); |
| FileSystemUtils.writeContent(target, new byte[] {1, 2, 3, 4, 5}); |
| Path link = fs.getPath("/execroot/dir/link"); |
| link.createSymbolicLink(PathFragment.create("../target")); |
| |
| UploadManifest um = |
| new UploadManifest( |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); |
| um.addFiles(ImmutableList.of(dir)); |
| assertThat(um.getDigestToFile()).isEmpty(); |
| |
| Tree tree = |
| Tree.newBuilder() |
| .setRoot( |
| Directory.newBuilder() |
| .addSymlinks(SymlinkNode.newBuilder().setName("link").setTarget("../target"))) |
| .build(); |
| Digest treeDigest = digestUtil.compute(tree); |
| |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); |
| expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); |
| assertThat(result.build()).isEqualTo(expectedResult.build()); |
| } |
| |
| @Test |
| public void uploadRelativeDirectorySymlinkInDirectory() throws Exception { |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| Path dir = fs.getPath("/execroot/dir"); |
| dir.createDirectory(); |
| Path bardir = fs.getPath("/execroot/bardir"); |
| bardir.createDirectory(); |
| Path foo = fs.getPath("/execroot/bardir/foo"); |
| FileSystemUtils.writeContent(foo, new byte[] {1, 2, 3, 4, 5}); |
| Path link = fs.getPath("/execroot/dir/link"); |
| link.createSymbolicLink(PathFragment.create("../bardir")); |
| |
| UploadManifest um = |
| new UploadManifest( |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); |
| um.addFiles(ImmutableList.of(dir)); |
| assertThat(um.getDigestToFile()).isEmpty(); |
| |
| Tree tree = |
| Tree.newBuilder() |
| .setRoot( |
| Directory.newBuilder() |
| .addSymlinks(SymlinkNode.newBuilder().setName("link").setTarget("../bardir"))) |
| .build(); |
| Digest treeDigest = digestUtil.compute(tree); |
| |
| ActionResult.Builder expectedResult = ActionResult.newBuilder(); |
| expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); |
| assertThat(result.build()).isEqualTo(expectedResult.build()); |
| } |
| |
| @Test |
| public void uploadDanglingSymlinkInDirectoryError() throws Exception { |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| Path dir = fs.getPath("/execroot/dir"); |
| dir.createDirectory(); |
| Path target = fs.getPath("/execroot/target"); |
| Path link = fs.getPath("/execroot/dir/link"); |
| link.createSymbolicLink(target); |
| |
| UploadManifest um = |
| new UploadManifest( |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ true); |
| IOException e = assertThrows(IOException.class, () -> um.addFiles(ImmutableList.of(dir))); |
| assertThat(e).hasMessageThat().contains("dangling"); |
| assertThat(e).hasMessageThat().contains("/execroot/dir/link"); |
| assertThat(e).hasMessageThat().contains("/execroot/target"); |
| } |
| |
| @Test |
| public void uploadSymlinkInDirectoryNoAllowError() throws Exception { |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| Path dir = fs.getPath("/execroot/dir"); |
| dir.createDirectory(); |
| Path target = fs.getPath("/execroot/target"); |
| FileSystemUtils.writeContent(target, new byte[] {1, 2, 3, 4, 5}); |
| Path link = fs.getPath("/execroot/dir/link"); |
| link.createSymbolicLink(target); |
| |
| UploadManifest um = |
| new UploadManifest( |
| digestUtil, result, execRoot, /*uploadSymlinks=*/ true, /*allowSymlinks=*/ false); |
| ExecException e = assertThrows(ExecException.class, () -> um.addFiles(ImmutableList.of(dir))); |
| assertThat(e).hasMessageThat().contains("symbolic link"); |
| assertThat(e).hasMessageThat().contains("dir/link"); |
| assertThat(e).hasMessageThat().contains("--remote_allow_symlink_upload"); |
| } |
| |
| @Test |
| public void downloadRelativeFileSymlink() throws Exception { |
| AbstractRemoteActionCache cache = newTestCache(); |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| result.addOutputFileSymlinksBuilder().setPath("a/b/link").setTarget("../../foo"); |
| // Doesn't check for dangling links, hence download succeeds. |
| cache.download(result.build(), execRoot, null, outputFilesLocker); |
| Path path = execRoot.getRelative("a/b/link"); |
| assertThat(path.isSymbolicLink()).isTrue(); |
| assertThat(path.readSymbolicLink()).isEqualTo(PathFragment.create("../../foo")); |
| verify(outputFilesLocker).lock(); |
| } |
| |
| @Test |
| public void downloadRelativeDirectorySymlink() throws Exception { |
| AbstractRemoteActionCache cache = newTestCache(); |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| result.addOutputDirectorySymlinksBuilder().setPath("a/b/link").setTarget("foo"); |
| // Doesn't check for dangling links, hence download succeeds. |
| cache.download(result.build(), execRoot, null, outputFilesLocker); |
| Path path = execRoot.getRelative("a/b/link"); |
| assertThat(path.isSymbolicLink()).isTrue(); |
| assertThat(path.readSymbolicLink()).isEqualTo(PathFragment.create("foo")); |
| verify(outputFilesLocker).lock(); |
| } |
| |
| @Test |
| public void downloadRelativeSymlinkInDirectory() throws Exception { |
| DefaultRemoteActionCache cache = newTestCache(); |
| Tree tree = |
| Tree.newBuilder() |
| .setRoot( |
| Directory.newBuilder() |
| .addSymlinks(SymlinkNode.newBuilder().setName("link").setTarget("../foo"))) |
| .build(); |
| Digest treeDigest = cache.addContents(tree.toByteArray()); |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| result.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); |
| // Doesn't check for dangling links, hence download succeeds. |
| cache.download(result.build(), execRoot, null, outputFilesLocker); |
| Path path = execRoot.getRelative("dir/link"); |
| assertThat(path.isSymbolicLink()).isTrue(); |
| assertThat(path.readSymbolicLink()).isEqualTo(PathFragment.create("../foo")); |
| verify(outputFilesLocker).lock(); |
| } |
| |
| @Test |
| public void downloadAbsoluteDirectorySymlinkError() throws Exception { |
| AbstractRemoteActionCache cache = newTestCache(); |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| result.addOutputDirectorySymlinksBuilder().setPath("foo").setTarget("/abs/link"); |
| IOException expected = |
| assertThrows( |
| IOException.class, |
| () -> cache.download(result.build(), execRoot, null, outputFilesLocker)); |
| assertThat(expected).hasMessageThat().contains("/abs/link"); |
| assertThat(expected).hasMessageThat().contains("absolute path"); |
| verify(outputFilesLocker).lock(); |
| } |
| |
| @Test |
| public void downloadAbsoluteFileSymlinkError() throws Exception { |
| AbstractRemoteActionCache cache = newTestCache(); |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| result.addOutputFileSymlinksBuilder().setPath("foo").setTarget("/abs/link"); |
| IOException expected = |
| assertThrows( |
| IOException.class, |
| () -> cache.download(result.build(), execRoot, null, outputFilesLocker)); |
| assertThat(expected).hasMessageThat().contains("/abs/link"); |
| assertThat(expected).hasMessageThat().contains("absolute path"); |
| verify(outputFilesLocker).lock(); |
| } |
| |
| @Test |
| public void downloadAbsoluteSymlinkInDirectoryError() throws Exception { |
| DefaultRemoteActionCache cache = newTestCache(); |
| Tree tree = |
| Tree.newBuilder() |
| .setRoot( |
| Directory.newBuilder() |
| .addSymlinks(SymlinkNode.newBuilder().setName("link").setTarget("/foo"))) |
| .build(); |
| Digest treeDigest = cache.addContents(tree.toByteArray()); |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| result.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest); |
| IOException expected = |
| assertThrows( |
| IOException.class, |
| () -> cache.download(result.build(), execRoot, null, outputFilesLocker)); |
| assertThat(expected.getSuppressed()).isEmpty(); |
| assertThat(expected).hasMessageThat().contains("dir/link"); |
| assertThat(expected).hasMessageThat().contains("/foo"); |
| assertThat(expected).hasMessageThat().contains("absolute path"); |
| verify(outputFilesLocker).lock(); |
| } |
| |
| @Test |
| public void downloadFailureMaintainsDirectories() throws Exception { |
| DefaultRemoteActionCache cache = newTestCache(); |
| Tree tree = Tree.newBuilder().setRoot(Directory.newBuilder()).build(); |
| Digest treeDigest = cache.addContents(tree.toByteArray()); |
| Digest outputFileDigest = |
| cache.addException("outputdir/outputfile", new IOException("download failed")); |
| Digest otherFileDigest = cache.addContents("otherfile"); |
| |
| ActionResult.Builder result = ActionResult.newBuilder(); |
| result.addOutputDirectoriesBuilder().setPath("outputdir").setTreeDigest(treeDigest); |
| result.addOutputFiles( |
| OutputFile.newBuilder().setPath("outputdir/outputfile").setDigest(outputFileDigest)); |
| result.addOutputFiles(OutputFile.newBuilder().setPath("otherfile").setDigest(otherFileDigest)); |
| assertThrows( |
| IOException.class, () -> cache.download(result.build(), execRoot, null, outputFilesLocker)); |
| assertThat(cache.getNumFailedDownloads()).isEqualTo(1); |
| assertThat(execRoot.getRelative("outputdir").exists()).isTrue(); |
| assertThat(execRoot.getRelative("outputdir/outputfile").exists()).isFalse(); |
| assertThat(execRoot.getRelative("otherfile").exists()).isFalse(); |
| verify(outputFilesLocker, never()).lock(); |
| } |
| |
| @Test |
| public void onErrorWaitForRemainingDownloadsToComplete() throws Exception { |
| // If one or more downloads of output files / directories fail then the code should |
| // wait for all downloads to have been completed before it tries to clean up partially |
| // downloaded files. |
| |
| Path stdout = fs.getPath("/execroot/stdout"); |
| Path stderr = fs.getPath("/execroot/stderr"); |
| |
| DefaultRemoteActionCache cache = newTestCache(); |
| Digest digest1 = cache.addContents("file1"); |
| Digest digest2 = cache.addException("file2", new IOException("download failed")); |
| Digest digest3 = cache.addContents("file3"); |
| |
| ActionResult result = |
| ActionResult.newBuilder() |
| .setExitCode(0) |
| .addOutputFiles(OutputFile.newBuilder().setPath("file1").setDigest(digest1)) |
| .addOutputFiles(OutputFile.newBuilder().setPath("file2").setDigest(digest2)) |
| .addOutputFiles(OutputFile.newBuilder().setPath("file3").setDigest(digest3)) |
| .build(); |
| IOException e = |
| assertThrows( |
| IOException.class, |
| () -> |
| cache.download( |
| result, execRoot, new FileOutErr(stdout, stderr), outputFilesLocker)); |
| assertThat(e.getSuppressed()).isEmpty(); |
| assertThat(cache.getNumSuccessfulDownloads()).isEqualTo(2); |
| assertThat(cache.getNumFailedDownloads()).isEqualTo(1); |
| assertThat(cache.getDownloadQueueSize()).isEqualTo(3); |
| assertThat(Throwables.getRootCause(e)).hasMessageThat().isEqualTo("download failed"); |
| verify(outputFilesLocker, never()).lock(); |
| } |
| |
| @Test |
| public void downloadWithMultipleErrorsAddsThemAsSuppressed() throws Exception { |
| Path stdout = fs.getPath("/execroot/stdout"); |
| Path stderr = fs.getPath("/execroot/stderr"); |
| |
| DefaultRemoteActionCache cache = newTestCache(); |
| Digest digest1 = cache.addContents("file1"); |
| Digest digest2 = cache.addException("file2", new IOException("file2 failed")); |
| Digest digest3 = cache.addException("file3", new IOException("file3 failed")); |
| |
| ActionResult result = |
| ActionResult.newBuilder() |
| .setExitCode(0) |
| .addOutputFiles(OutputFile.newBuilder().setPath("file1").setDigest(digest1)) |
| .addOutputFiles(OutputFile.newBuilder().setPath("file2").setDigest(digest2)) |
| .addOutputFiles(OutputFile.newBuilder().setPath("file3").setDigest(digest3)) |
| .build(); |
| IOException e = |
| assertThrows( |
| IOException.class, |
| () -> |
| cache.download( |
| result, execRoot, new FileOutErr(stdout, stderr), outputFilesLocker)); |
| |
| assertThat(e.getSuppressed()).hasLength(1); |
| assertThat(e.getSuppressed()[0]).isInstanceOf(IOException.class); |
| assertThat(e.getSuppressed()[0]).hasMessageThat().isEqualTo("file3 failed"); |
| assertThat(Throwables.getRootCause(e)).hasMessageThat().isEqualTo("file2 failed"); |
| } |
| |
| @Test |
| public void testDownloadWithStdoutStderrOnSuccess() throws Exception { |
| // Tests that fetching stdout/stderr as a digest works and that OutErr is still |
| // writable afterwards. |
| Path stdout = fs.getPath("/execroot/stdout"); |
| Path stderr = fs.getPath("/execroot/stderr"); |
| FileOutErr outErr = new FileOutErr(stdout, stderr); |
| FileOutErr childOutErr = outErr.childOutErr(); |
| FileOutErr spyOutErr = Mockito.spy(outErr); |
| FileOutErr spyChildOutErr = Mockito.spy(childOutErr); |
| when(spyOutErr.childOutErr()).thenReturn(spyChildOutErr); |
| |
| DefaultRemoteActionCache cache = newTestCache(); |
| Digest digestStdout = cache.addContents("stdout"); |
| Digest digestStderr = cache.addContents("stderr"); |
| |
| ActionResult result = |
| ActionResult.newBuilder() |
| .setExitCode(0) |
| .setStdoutDigest(digestStdout) |
| .setStderrDigest(digestStderr) |
| .build(); |
| |
| cache.download(result, execRoot, spyOutErr, outputFilesLocker); |
| |
| verify(spyOutErr, Mockito.times(2)).childOutErr(); |
| verify(spyChildOutErr).clearOut(); |
| verify(spyChildOutErr).clearErr(); |
| assertThat(outErr.getOutputPath().exists()).isTrue(); |
| assertThat(outErr.getErrorPath().exists()).isTrue(); |
| |
| try { |
| outErr.getOutputStream().write(0); |
| outErr.getErrorStream().write(0); |
| } catch (IOException err) { |
| throw new AssertionError("outErr should still be writable after download finished.", err); |
| } |
| |
| verify(outputFilesLocker).lock(); |
| } |
| |
| @Test |
| public void testDownloadWithStdoutStderrOnFailure() throws Exception { |
| // Test that when downloading stdout/stderr fails the OutErr is still writable |
| // and empty. |
| Path stdout = fs.getPath("/execroot/stdout"); |
| Path stderr = fs.getPath("/execroot/stderr"); |
| FileOutErr outErr = new FileOutErr(stdout, stderr); |
| FileOutErr childOutErr = outErr.childOutErr(); |
| FileOutErr spyOutErr = Mockito.spy(outErr); |
| FileOutErr spyChildOutErr = Mockito.spy(childOutErr); |
| when(spyOutErr.childOutErr()).thenReturn(spyChildOutErr); |
| |
| DefaultRemoteActionCache cache = newTestCache(); |
| // Don't add stdout/stderr as a known blob to the remote cache so that downloading it will fail |
| Digest digestStdout = digestUtil.computeAsUtf8("stdout"); |
| Digest digestStderr = digestUtil.computeAsUtf8("stderr"); |
| |
| ActionResult result = |
| ActionResult.newBuilder() |
| .setExitCode(0) |
| .setStdoutDigest(digestStdout) |
| .setStderrDigest(digestStderr) |
| .build(); |
| assertThrows( |
| IOException.class, () -> cache.download(result, execRoot, spyOutErr, outputFilesLocker)); |
| verify(spyOutErr, Mockito.times(2)).childOutErr(); |
| verify(spyChildOutErr).clearOut(); |
| verify(spyChildOutErr).clearErr(); |
| assertThat(outErr.getOutputPath().exists()).isFalse(); |
| assertThat(outErr.getErrorPath().exists()).isFalse(); |
| |
| try { |
| outErr.getOutputStream().write(0); |
| outErr.getErrorStream().write(0); |
| } catch (IOException err) { |
| throw new AssertionError("outErr should still be writable after download failed.", err); |
| } |
| |
| verify(outputFilesLocker, never()).lock(); |
| } |
| |
| @Test |
| public void testDownloadClashes() throws Exception { |
| // Test that injecting the metadata for a remote output file works |
| |
| // arrange |
| DefaultRemoteActionCache remoteCache = newTestCache(); |
| Digest d1 = remoteCache.addContents("content1"); |
| Digest d2 = remoteCache.addContents("content2"); |
| ActionResult r = |
| ActionResult.newBuilder() |
| .setExitCode(0) |
| .addOutputFiles(OutputFile.newBuilder().setPath("outputs/foo.tmp").setDigest(d1)) |
| .addOutputFiles(OutputFile.newBuilder().setPath("outputs/foo").setDigest(d2)) |
| .build(); |
| |
| Artifact a1 = ActionsTestUtil.createArtifact(artifactRoot, "foo.tmp"); |
| Artifact a2 = ActionsTestUtil.createArtifact(artifactRoot, "foo"); |
| |
| // act |
| |
| remoteCache.download(r, execRoot, new FileOutErr(), outputFilesLocker); |
| |
| // assert |
| |
| assertThat(FileSystemUtils.readContent(a1.getPath(), StandardCharsets.UTF_8)) |
| .isEqualTo("content1"); |
| assertThat(FileSystemUtils.readContent(a2.getPath(), StandardCharsets.UTF_8)) |
| .isEqualTo("content2"); |
| verify(outputFilesLocker).lock(); |
| } |
| |
| @Test |
| public void testDownloadMinimalFiles() throws Exception { |
| // Test that injecting the metadata for a remote output file works |
| |
| // arrange |
| DefaultRemoteActionCache remoteCache = newTestCache(); |
| Digest d1 = remoteCache.addContents("content1"); |
| Digest d2 = remoteCache.addContents("content2"); |
| ActionResult r = |
| ActionResult.newBuilder() |
| .setExitCode(0) |
| .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file1").setDigest(d1)) |
| .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file2").setDigest(d2)) |
| .build(); |
| |
| Artifact a1 = ActionsTestUtil.createArtifact(artifactRoot, "file1"); |
| Artifact a2 = ActionsTestUtil.createArtifact(artifactRoot, "file2"); |
| |
| MetadataInjector injector = mock(MetadataInjector.class); |
| |
| // act |
| InMemoryOutput inMemoryOutput = |
| remoteCache.downloadMinimal( |
| r, |
| ImmutableList.of(a1, a2), |
| /* inMemoryOutputPath= */ null, |
| new FileOutErr(), |
| execRoot, |
| injector, |
| outputFilesLocker); |
| |
| // assert |
| assertThat(inMemoryOutput).isNull(); |
| verify(injector) |
| .injectRemoteFile(eq(a1), eq(toBinaryDigest(d1)), eq(d1.getSizeBytes()), anyInt()); |
| verify(injector) |
| .injectRemoteFile(eq(a2), eq(toBinaryDigest(d2)), eq(d2.getSizeBytes()), anyInt()); |
| |
| Path outputBase = artifactRoot.getRoot().asPath(); |
| assertThat(outputBase.readdir(Symlinks.NOFOLLOW)).isEmpty(); |
| |
| verify(outputFilesLocker).lock(); |
| } |
| |
| @Test |
| public void testDownloadMinimalDirectory() throws Exception { |
| // Test that injecting the metadata for a tree artifact / remote output directory works |
| |
| // arrange |
| DefaultRemoteActionCache remoteCache = newTestCache(); |
| // Output Directory: |
| // dir/file1 |
| // dir/a/file2 |
| Digest d1 = remoteCache.addContents("content1"); |
| Digest d2 = remoteCache.addContents("content2"); |
| FileNode file1 = FileNode.newBuilder().setName("file1").setDigest(d1).build(); |
| FileNode file2 = FileNode.newBuilder().setName("file2").setDigest(d2).build(); |
| Directory a = Directory.newBuilder().addFiles(file2).build(); |
| Digest da = remoteCache.addContents(a); |
| Directory root = |
| Directory.newBuilder() |
| .addFiles(file1) |
| .addDirectories(DirectoryNode.newBuilder().setName("a").setDigest(da)) |
| .build(); |
| Tree t = Tree.newBuilder().setRoot(root).addChildren(a).build(); |
| Digest dt = remoteCache.addContents(t); |
| ActionResult r = |
| ActionResult.newBuilder() |
| .setExitCode(0) |
| .addOutputDirectories( |
| OutputDirectory.newBuilder().setPath("outputs/dir").setTreeDigest(dt)) |
| .build(); |
| |
| SpecialArtifact dir = |
| new SpecialArtifact( |
| artifactRoot, |
| PathFragment.create("outputs/dir"), |
| ActionsTestUtil.NULL_ARTIFACT_OWNER, |
| SpecialArtifactType.TREE); |
| |
| MetadataInjector injector = mock(MetadataInjector.class); |
| |
| // act |
| InMemoryOutput inMemoryOutput = |
| remoteCache.downloadMinimal( |
| r, |
| ImmutableList.of(dir), |
| /* inMemoryOutputPath= */ null, |
| new FileOutErr(), |
| execRoot, |
| injector, |
| outputFilesLocker); |
| |
| // assert |
| assertThat(inMemoryOutput).isNull(); |
| |
| Map<PathFragment, RemoteFileArtifactValue> m = |
| ImmutableMap.<PathFragment, RemoteFileArtifactValue>builder() |
| .put( |
| PathFragment.create("file1"), |
| new RemoteFileArtifactValue(toBinaryDigest(d1), d1.getSizeBytes(), 1)) |
| .put( |
| PathFragment.create("a/file2"), |
| new RemoteFileArtifactValue(toBinaryDigest(d2), d2.getSizeBytes(), 1)) |
| .build(); |
| verify(injector).injectRemoteDirectory(eq(dir), eq(m)); |
| |
| Path outputBase = artifactRoot.getRoot().asPath(); |
| assertThat(outputBase.readdir(Symlinks.NOFOLLOW)).isEmpty(); |
| |
| verify(outputFilesLocker).lock(); |
| } |
| |
| @Test |
| public void testDownloadMinimalDirectoryFails() throws Exception { |
| // Test that we properly fail when downloading the metadata of an output |
| // directory fails |
| |
| // arrange |
| DefaultRemoteActionCache remoteCache = newTestCache(); |
| // Output Directory: |
| // dir/file1 |
| // dir/a/file2 |
| Digest d1 = remoteCache.addContents("content1"); |
| Digest d2 = remoteCache.addContents("content2"); |
| FileNode file1 = FileNode.newBuilder().setName("file1").setDigest(d1).build(); |
| FileNode file2 = FileNode.newBuilder().setName("file2").setDigest(d2).build(); |
| Directory a = Directory.newBuilder().addFiles(file2).build(); |
| Digest da = remoteCache.addContents(a); |
| Directory root = |
| Directory.newBuilder() |
| .addFiles(file1) |
| .addDirectories(DirectoryNode.newBuilder().setName("a").setDigest(da)) |
| .build(); |
| Tree t = Tree.newBuilder().setRoot(root).addChildren(a).build(); |
| // Downloading the tree will fail |
| IOException downloadTreeException = new IOException("entry not found"); |
| Digest dt = remoteCache.addException(t, downloadTreeException); |
| ActionResult r = |
| ActionResult.newBuilder() |
| .setExitCode(0) |
| .addOutputDirectories( |
| OutputDirectory.newBuilder().setPath("outputs/dir").setTreeDigest(dt)) |
| .build(); |
| SpecialArtifact dir = |
| new SpecialArtifact( |
| artifactRoot, |
| PathFragment.create("outputs/dir"), |
| ActionsTestUtil.NULL_ARTIFACT_OWNER, |
| SpecialArtifactType.TREE); |
| MetadataInjector injector = mock(MetadataInjector.class); |
| |
| // act |
| IOException e = |
| assertThrows( |
| IOException.class, |
| () -> |
| remoteCache.downloadMinimal( |
| r, |
| ImmutableList.of(dir), |
| /* inMemoryOutputPath= */ null, |
| new FileOutErr(), |
| execRoot, |
| injector, |
| outputFilesLocker)); |
| assertThat(e).isEqualTo(downloadTreeException); |
| |
| verify(outputFilesLocker, never()).lock(); |
| } |
| |
| @Test |
| public void testDownloadMinimalWithStdoutStderr() throws Exception { |
| // Test that downloading of non-embedded stdout and stderr works |
| |
| // arrange |
| DefaultRemoteActionCache remoteCache = newTestCache(); |
| Digest dOut = remoteCache.addContents("stdout"); |
| Digest dErr = remoteCache.addContents("stderr"); |
| ActionResult r = |
| ActionResult.newBuilder() |
| .setExitCode(0) |
| .setStdoutDigest(dOut) |
| .setStderrDigest(dErr) |
| .build(); |
| RecordingOutErr outErr = new RecordingOutErr(); |
| MetadataInjector injector = mock(MetadataInjector.class); |
| |
| // act |
| InMemoryOutput inMemoryOutput = |
| remoteCache.downloadMinimal( |
| r, |
| ImmutableList.of(), |
| /* inMemoryOutputPath= */ null, |
| outErr, |
| execRoot, |
| injector, |
| outputFilesLocker); |
| |
| // assert |
| assertThat(inMemoryOutput).isNull(); |
| assertThat(outErr.outAsLatin1()).isEqualTo("stdout"); |
| assertThat(outErr.errAsLatin1()).isEqualTo("stderr"); |
| |
| Path outputBase = artifactRoot.getRoot().asPath(); |
| assertThat(outputBase.readdir(Symlinks.NOFOLLOW)).isEmpty(); |
| |
| verify(outputFilesLocker).lock(); |
| } |
| |
| @Test |
| public void testDownloadMinimalWithInMemoryOutput() throws Exception { |
| // Test that downloading an in memory output works |
| |
| // arrange |
| DefaultRemoteActionCache remoteCache = newTestCache(); |
| Digest d1 = remoteCache.addContents("content1"); |
| Digest d2 = remoteCache.addContents("content2"); |
| ActionResult r = |
| ActionResult.newBuilder() |
| .setExitCode(0) |
| .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file1").setDigest(d1)) |
| .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file2").setDigest(d2)) |
| .build(); |
| Artifact a1 = ActionsTestUtil.createArtifact(artifactRoot, "file1"); |
| Artifact a2 = ActionsTestUtil.createArtifact(artifactRoot, "file2"); |
| MetadataInjector injector = mock(MetadataInjector.class); |
| // a1 should be provided as an InMemoryOutput |
| PathFragment inMemoryOutputPathFragment = a1.getPath().relativeTo(execRoot); |
| |
| // act |
| InMemoryOutput inMemoryOutput = |
| remoteCache.downloadMinimal( |
| r, |
| ImmutableList.of(a1, a2), |
| inMemoryOutputPathFragment, |
| new FileOutErr(), |
| execRoot, |
| injector, |
| outputFilesLocker); |
| |
| // assert |
| assertThat(inMemoryOutput).isNotNull(); |
| ByteString expectedContents = ByteString.copyFrom("content1", UTF_8); |
| assertThat(inMemoryOutput.getContents()).isEqualTo(expectedContents); |
| assertThat(inMemoryOutput.getOutput()).isEqualTo(a1); |
| // The in memory file also needs to be injected as an output |
| verify(injector) |
| .injectRemoteFile(eq(a1), eq(toBinaryDigest(d1)), eq(d1.getSizeBytes()), anyInt()); |
| verify(injector) |
| .injectRemoteFile(eq(a2), eq(toBinaryDigest(d2)), eq(d2.getSizeBytes()), anyInt()); |
| |
| Path outputBase = artifactRoot.getRoot().asPath(); |
| assertThat(outputBase.readdir(Symlinks.NOFOLLOW)).isEmpty(); |
| |
| verify(outputFilesLocker).lock(); |
| } |
| |
| private DefaultRemoteActionCache newTestCache() { |
| RemoteOptions options = Options.getDefaults(RemoteOptions.class); |
| return new DefaultRemoteActionCache(options, digestUtil); |
| } |
| |
| private static class DefaultRemoteActionCache extends AbstractRemoteActionCache { |
| |
| Map<Digest, ListenableFuture<byte[]>> downloadResults = new HashMap<>(); |
| List<ListenableFuture<?>> blockingDownloads = new ArrayList<>(); |
| AtomicInteger numSuccess = new AtomicInteger(); |
| AtomicInteger numFailures = new AtomicInteger(); |
| |
| public DefaultRemoteActionCache(RemoteOptions options, DigestUtil digestUtil) { |
| super(options, digestUtil); |
| } |
| |
| public Digest addContents(String txt) { |
| return addContents(txt.getBytes(UTF_8)); |
| } |
| |
| public Digest addContents(byte[] bytes) { |
| Digest digest = digestUtil.compute(bytes); |
| downloadResults.put(digest, Futures.immediateFuture(bytes)); |
| return digest; |
| } |
| |
| public Digest addContents(Message m) { |
| return addContents(m.toByteArray()); |
| } |
| |
| public Digest addException(String txt, Exception e) { |
| Digest digest = digestUtil.compute(txt.getBytes(UTF_8)); |
| downloadResults.put(digest, Futures.immediateFailedFuture(e)); |
| return digest; |
| } |
| |
| Digest addException(Message m, Exception e) { |
| Digest digest = digestUtil.compute(m); |
| downloadResults.put(digest, Futures.immediateFailedFuture(e)); |
| return digest; |
| } |
| |
| public int getNumSuccessfulDownloads() { |
| return numSuccess.get(); |
| } |
| |
| public int getNumFailedDownloads() { |
| return numFailures.get(); |
| } |
| |
| public int getDownloadQueueSize() { |
| return blockingDownloads.size(); |
| } |
| |
| @Override |
| protected <T> T getFromFuture(ListenableFuture<T> f) throws IOException, InterruptedException { |
| blockingDownloads.add(f); |
| return Utils.getFromFuture(f); |
| } |
| |
| @Nullable |
| @Override |
| ActionResult getCachedActionResult(ActionKey actionKey) { |
| throw new UnsupportedOperationException(); |
| } |
| |
| @Override |
| protected void setCachedActionResult(ActionKey actionKey, ActionResult action) { |
| throw new UnsupportedOperationException(); |
| } |
| |
| @Override |
| protected ListenableFuture<Void> uploadFile(Digest digest, Path path) { |
| throw new UnsupportedOperationException(); |
| } |
| |
| @Override |
| protected ListenableFuture<Void> uploadBlob(Digest digest, ByteString data) { |
| throw new UnsupportedOperationException(); |
| } |
| |
| @Override |
| protected ImmutableSet<Digest> getMissingDigests(Iterable<Digest> digests) { |
| throw new UnsupportedOperationException(); |
| } |
| |
| @Override |
| protected ListenableFuture<Void> downloadBlob(Digest digest, OutputStream out) { |
| SettableFuture<Void> result = SettableFuture.create(); |
| ListenableFuture<byte[]> downloadResult = downloadResults.get(digest); |
| Futures.addCallback( |
| downloadResult != null |
| ? downloadResult |
| : Futures.immediateFailedFuture(new CacheNotFoundException(digest, digestUtil)), |
| new FutureCallback<byte[]>() { |
| @Override |
| public void onSuccess(byte[] bytes) { |
| numSuccess.incrementAndGet(); |
| try { |
| out.write(bytes); |
| out.close(); |
| result.set(null); |
| } catch (IOException e) { |
| result.setException(e); |
| } |
| } |
| |
| @Override |
| public void onFailure(Throwable throwable) { |
| numFailures.incrementAndGet(); |
| result.setException(throwable); |
| } |
| }, |
| MoreExecutors.directExecutor()); |
| return result; |
| } |
| |
| @Override |
| public void close() { |
| throw new UnsupportedOperationException(); |
| } |
| } |
| } |
