remote: add directory support for remote caching and execution Add support for directory trees as artifacts. Closes #4011. PiperOrigin-RevId: 179691001
diff --git a/src/main/java/com/google/devtools/build/lib/remote/AbstractRemoteActionCache.java b/src/main/java/com/google/devtools/build/lib/remote/AbstractRemoteActionCache.java new file mode 100644 index 0000000..9401945 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/remote/AbstractRemoteActionCache.java
@@ -0,0 +1,378 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.remote; + +import com.google.devtools.build.lib.actions.EnvironmentalExecException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.concurrent.ThreadSafety; +import com.google.devtools.build.lib.remote.TreeNodeRepository.TreeNode; +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.Dirent; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.remoteexecution.v1test.ActionResult; +import com.google.devtools.remoteexecution.v1test.Command; +import com.google.devtools.remoteexecution.v1test.Digest; +import com.google.devtools.remoteexecution.v1test.Directory; +import com.google.devtools.remoteexecution.v1test.DirectoryNode; +import com.google.devtools.remoteexecution.v1test.FileNode; +import com.google.devtools.remoteexecution.v1test.OutputDirectory; +import com.google.devtools.remoteexecution.v1test.OutputFile; +import com.google.devtools.remoteexecution.v1test.Tree; +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; + +/** A cache for storing artifacts (input and output) as well as the output of running an action. */ +@ThreadSafety.ThreadSafe +public abstract class AbstractRemoteActionCache implements AutoCloseable { + protected final DigestUtil digestUtil; + + public AbstractRemoteActionCache(DigestUtil digestUtil) { + this.digestUtil = digestUtil; + } + + /** + * Ensures that the tree structure of the inputs, the input files themselves, and the command are + * available in the remote cache, such that the tree can be reassembled and executed on another + * machine given the root digest. + * + * <p>The cache may check whether files or parts of the tree structure are already present, and do + * not need to be uploaded again. + * + * <p>Note that this method is only required for remote execution, not for caching itself. + * However, remote execution uses a cache to store input files, and that may be a separate + * end-point from the executor itself, so the functionality lives here. A pure remote caching + * implementation that does not support remote execution may choose not to implement this + * function, and throw {@link UnsupportedOperationException} instead. If so, it should be clearly + * documented that it cannot be used for remote execution. + */ + public abstract void ensureInputsPresent( + TreeNodeRepository repository, Path execRoot, TreeNode root, Command command) + throws IOException, InterruptedException; + + /** + * Attempts to look up the given action in the remote cache and return its result, if present. + * Returns {@code null} if there is no such entry. Note that a successful result from this method + * does not guarantee the availability of the corresponding output files in the remote cache. + * + * @throws IOException if the remote cache is unavailable. + */ + abstract @Nullable ActionResult getCachedActionResult(DigestUtil.ActionKey actionKey) + throws IOException, InterruptedException; + + /** + * Upload the result of a locally executed action to the cache by uploading any necessary files, + * stdin / stdout, as well as adding an entry for the given action key to the cache if + * uploadAction is true. + * + * @throws IOException if the remote cache is unavailable. + */ + abstract void upload( + DigestUtil.ActionKey actionKey, + Path execRoot, + Collection<Path> files, + FileOutErr outErr, + boolean uploadAction) + throws IOException, InterruptedException; + + /** + * Download a remote blob to a local destination. + * + * @param digest The digest of the remote blob. + * @param dest The path to the local file. + * @throws IOException if download failed. + */ + protected abstract void downloadBlob(Digest digest, Path dest) + throws IOException, InterruptedException; + + /** + * Download a remote blob and store it in memory. + * + * @param digest The digest of the remote blob. + * @return The remote blob. + * @throws IOException if download failed. + */ + protected abstract byte[] downloadBlob(Digest digest) throws IOException, InterruptedException; + + /** + * Download the output files and directory trees of a remotely executed action to the local + * machine, as well stdin / stdout to the given files. + * + * <p>In case of failure, this method deletes any output files it might have already created. + * + * @throws IOException in case of a cache miss or if the remote cache is unavailable. + * @throws ExecException in case clean up after a failed download failed. + */ + // TODO(olaola): will need to amend to include the TreeNodeRepository for updating. + public void download(ActionResult result, Path execRoot, FileOutErr outErr) + throws ExecException, IOException, InterruptedException { + try { + for (OutputFile file : result.getOutputFilesList()) { + Path path = execRoot.getRelative(file.getPath()); + downloadFile(path, file.getDigest(), file.getIsExecutable(), file.getContent()); + } + for (OutputDirectory dir : result.getOutputDirectoriesList()) { + Digest treeDigest = dir.getTreeDigest(); + byte[] b = downloadBlob(treeDigest); + Digest receivedTreeDigest = digestUtil.compute(b); + if (!receivedTreeDigest.equals(treeDigest)) { + throw new IOException( + "Digest does not match " + receivedTreeDigest + " != " + treeDigest); + } + Tree tree = Tree.parseFrom(b); + Map<Digest, Directory> childrenMap = new HashMap<>(); + for (Directory child : tree.getChildrenList()) { + childrenMap.put(digestUtil.compute(child), child); + } + Path path = execRoot.getRelative(dir.getPath()); + downloadDirectory(path, tree.getRoot(), childrenMap); + } + // TODO(ulfjack): use same code as above also for stdout / stderr if applicable. + downloadOutErr(result, outErr); + } catch (IOException downloadException) { + try { + // Delete any (partially) downloaded output files, since any subsequent local execution + // of this action may expect none of the output files to exist. + for (OutputFile file : result.getOutputFilesList()) { + execRoot.getRelative(file.getPath()).delete(); + } + for (OutputDirectory directory : result.getOutputDirectoriesList()) { + execRoot.getRelative(directory.getPath()).delete(); + } + if (outErr != null) { + outErr.getOutputPath().delete(); + outErr.getErrorPath().delete(); + } + } catch (IOException e) { + // If deleting of output files failed, we abort the build with a decent error message as + // any subsequent local execution failure would likely be incomprehensible. + + // We don't propagate the downloadException, as this is a recoverable error and the cause + // of the build failure is really that we couldn't delete output files. + throw new EnvironmentalExecException( + "Failed to delete output files after incomplete " + + "download. Cannot continue with local execution.", + e, + true); + } + throw downloadException; + } + } + + /** + * Download a directory recursively. The directory is represented by a {@link Directory} protobuf + * message, and the descendant directories are in {@code childrenMap}, accessible through their + * digest. + */ + private void downloadDirectory(Path path, Directory dir, Map<Digest, Directory> childrenMap) + throws IOException, InterruptedException { + // Ensure that the directory is created here even though the directory might be empty + FileSystemUtils.createDirectoryAndParents(path); + + for (FileNode child : dir.getFilesList()) { + Path childPath = path.getRelative(child.getName()); + downloadFile(childPath, child.getDigest(), child.getIsExecutable(), null); + } + + for (DirectoryNode child : dir.getDirectoriesList()) { + Path childPath = path.getRelative(child.getName()); + Digest childDigest = child.getDigest(); + Directory childDir = childrenMap.get(childDigest); + if (childDir == null) { + throw new IOException( + "could not find subdirectory " + + child.getName() + + " of directory " + + path + + " for download: digest " + + childDigest + + "not found"); + } + downloadDirectory(childPath, childDir, childrenMap); + + // Prevent reuse. + childrenMap.remove(childDigest); + } + } + + /** + * Download a file (that is not a directory). If the {@code content} is not given, the content is + * fetched from the digest. + */ + protected void downloadFile( + Path path, Digest digest, boolean isExecutable, @Nullable ByteString content) + throws IOException, InterruptedException { + FileSystemUtils.createDirectoryAndParents(path.getParentDirectory()); + if (digest.getSizeBytes() == 0) { + // Handle empty file locally. + FileSystemUtils.writeContent(path, new byte[0]); + } else { + if (content != null && !content.isEmpty()) { + try (OutputStream stream = path.getOutputStream()) { + content.writeTo(stream); + } + } else { + downloadBlob(digest, path); + Digest receivedDigest = digestUtil.compute(path); + if (!receivedDigest.equals(digest)) { + throw new IOException("Digest does not match " + receivedDigest + " != " + digest); + } + } + } + path.setExecutable(isExecutable); + } + + private void downloadOutErr(ActionResult result, FileOutErr outErr) + throws IOException, InterruptedException { + if (!result.getStdoutRaw().isEmpty()) { + result.getStdoutRaw().writeTo(outErr.getOutputStream()); + outErr.getOutputStream().flush(); + } else if (result.hasStdoutDigest()) { + byte[] stdoutBytes = downloadBlob(result.getStdoutDigest()); + outErr.getOutputStream().write(stdoutBytes); + outErr.getOutputStream().flush(); + } + if (!result.getStderrRaw().isEmpty()) { + result.getStderrRaw().writeTo(outErr.getErrorStream()); + outErr.getErrorStream().flush(); + } else if (result.hasStderrDigest()) { + byte[] stderrBytes = downloadBlob(result.getStderrDigest()); + outErr.getErrorStream().write(stderrBytes); + outErr.getErrorStream().flush(); + } + } + + /** + * The UploadManifest is used to mutualize upload between the RemoteActionCache implementations. + */ + public class UploadManifest { + private final ActionResult.Builder result; + private final Path execRoot; + private final Map<Digest, Path> digestToFile; + private final Map<Digest, Chunker> digestToChunkers; + + /** + * Create an UploadManifest from an ActionResult builder and an exec root. The ActionResult + * builder is populated through a call to {@link #addFile(Digest, Path)}. + */ + public UploadManifest(ActionResult.Builder result, Path execRoot) { + this.result = result; + this.execRoot = execRoot; + + this.digestToFile = new HashMap<>(); + this.digestToChunkers = new HashMap<>(); + } + + /** + * Add a collection of files (and directories) to the UploadManifest. Adding a directory has the + * effect of 1) uploading a {@link Tree} protobuf message from which the whole structure of the + * directory, including the descendants, can be reconstructed and 2) uploading all the + * non-directory descendant files. + */ + public void addFiles(Collection<Path> files) throws IOException, InterruptedException { + for (Path file : files) { + // TODO(ulfjack): Maybe pass in a SpawnResult here, add a list of output files to that, and + // rely on the local spawn runner to stat the files, instead of statting here. + if (!file.exists()) { + // We ignore requested results that have not been generated by the action. + continue; + } + if (file.isDirectory()) { + addDirectory(file); + } else { + Digest digest = digestUtil.compute(file); + addFile(digest, file); + } + } + } + + /** Map of digests to file paths to upload. */ + public Map<Digest, Path> getDigestToFile() { + return digestToFile; + } + + /** + * Map of digests to chunkers to upload. When the file is a regular, non-directory file it is + * transmitted through {@link #getDigestToFile()}. When it is a directory, it is transmitted as + * a {@link Tree} protobuf message through {@link #getDigestToChunkers()}. + */ + public Map<Digest, Chunker> getDigestToChunkers() { + return digestToChunkers; + } + + private void addFile(Digest digest, Path file) throws IOException { + result + .addOutputFilesBuilder() + .setPath(file.relativeTo(execRoot).getPathString()) + .setDigest(digest) + .setIsExecutable(file.isExecutable()); + + digestToFile.put(digest, file); + } + + private void addDirectory(Path dir) throws IOException { + Tree.Builder tree = Tree.newBuilder(); + Directory root = computeDirectory(dir, tree); + tree.setRoot(root); + + byte[] blob = tree.build().toByteArray(); + Digest digest = digestUtil.compute(blob); + Chunker chunker = new Chunker(blob, blob.length, digestUtil); + + if (result != null) { + result + .addOutputDirectoriesBuilder() + .setPath(dir.relativeTo(execRoot).getPathString()) + .setTreeDigest(digest); + } + + digestToChunkers.put(chunker.digest(), chunker); + } + + private Directory computeDirectory(Path path, Tree.Builder tree) throws IOException { + Directory.Builder b = Directory.newBuilder(); + + List<Dirent> sortedDirent = new ArrayList<>(path.readdir(TreeNodeRepository.SYMLINK_POLICY)); + sortedDirent.sort(Comparator.comparing(Dirent::getName)); + + for (Dirent dirent : sortedDirent) { + String name = dirent.getName(); + Path child = path.getRelative(name); + if (dirent.getType() == Dirent.Type.DIRECTORY) { + Directory dir = computeDirectory(child, tree); + b.addDirectoriesBuilder().setName(name).setDigest(digestUtil.compute(dir)); + tree.addChildren(dir); + } else { + Digest digest = digestUtil.compute(child); + b.addFilesBuilder().setName(name).setDigest(digest).setIsExecutable(child.isExecutable()); + digestToFile.put(digest, child); + } + } + + return b.build(); + } + } + + /** Release resources associated with the cache. The cache may not be used after calling this. */ + @Override + public abstract void close(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteCache.java b/src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteCache.java index 908fb9b..d865d0c 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteCache.java +++ b/src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteCache.java
@@ -25,15 +25,12 @@ import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.devtools.build.lib.actions.ActionInput; -import com.google.devtools.build.lib.actions.EnvironmentalExecException; -import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.actions.MetadataProvider; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import com.google.devtools.build.lib.remote.DigestUtil.ActionKey; import com.google.devtools.build.lib.remote.Retrier.RetryException; import com.google.devtools.build.lib.remote.TreeNodeRepository.TreeNode; import com.google.devtools.build.lib.util.io.FileOutErr; -import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.remoteexecution.v1test.ActionCacheGrpc; import com.google.devtools.remoteexecution.v1test.ActionCacheGrpc.ActionCacheBlockingStub; @@ -46,7 +43,6 @@ import com.google.devtools.remoteexecution.v1test.FindMissingBlobsRequest; import com.google.devtools.remoteexecution.v1test.FindMissingBlobsResponse; import com.google.devtools.remoteexecution.v1test.GetActionResultRequest; -import com.google.devtools.remoteexecution.v1test.OutputFile; import com.google.devtools.remoteexecution.v1test.UpdateActionResultRequest; import io.grpc.CallCredentials; import io.grpc.Channel; @@ -57,7 +53,6 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -67,13 +62,12 @@ /** A RemoteActionCache implementation that uses gRPC calls to a remote cache server. */ @ThreadSafe -public class GrpcRemoteCache implements RemoteActionCache { +public class GrpcRemoteCache extends AbstractRemoteActionCache { private final RemoteOptions options; private final CallCredentials credentials; private final Channel channel; private final RemoteRetrier retrier; private final ByteStreamUploader uploader; - private final DigestUtil digestUtil; private final ListeningScheduledExecutorService retryScheduler = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1)); @@ -84,11 +78,11 @@ RemoteOptions options, RemoteRetrier retrier, DigestUtil digestUtil) { + super(digestUtil); this.options = options; this.credentials = credentials; this.channel = channel; this.retrier = retrier; - this.digestUtil = digestUtil; uploader = new ByteStreamUploader(options.remoteInstanceName, channel, credentials, options.remoteTimeout, retrier, retryScheduler); @@ -180,90 +174,6 @@ } /** - * Download all results of a remotely executed action locally. TODO(olaola): will need to amend to - * include the {@link com.google.devtools.build.lib.remote.TreeNodeRepository} for updating. - */ - @Override - public void download(ActionResult result, Path execRoot, FileOutErr outErr) - throws ExecException, IOException, InterruptedException { - try { - for (OutputFile file : result.getOutputFilesList()) { - Path path = execRoot.getRelative(file.getPath()); - FileSystemUtils.createDirectoryAndParents(path.getParentDirectory()); - Digest digest = file.getDigest(); - if (digest.getSizeBytes() == 0) { - // Handle empty file locally. - FileSystemUtils.writeContent(path, new byte[0]); - } else { - if (!file.getContent().isEmpty()) { - try (OutputStream stream = path.getOutputStream()) { - file.getContent().writeTo(stream); - } - } else { - retrier.execute( - () -> { - try (OutputStream stream = path.getOutputStream()) { - readBlob(digest, stream); - } - return null; - }); - Digest receivedDigest = digestUtil.compute(path); - if (!receivedDigest.equals(digest)) { - throw new IOException( - "Digest does not match " + receivedDigest + " != " + digest); - } - } - } - path.setExecutable(file.getIsExecutable()); - } - if (!result.getOutputDirectoriesList().isEmpty()) { - throw new UnsupportedOperationException(); - } - // TODO(ulfjack): use same code as above also for stdout / stderr if applicable. - downloadOutErr(result, outErr); - } catch (IOException downloadException) { - try { - // Delete any (partially) downloaded output files, since any subsequent local execution - // of this action may expect none of the output files to exist. - for (OutputFile file : result.getOutputFilesList()) { - execRoot.getRelative(file.getPath()).delete(); - } - outErr.getOutputPath().delete(); - outErr.getErrorPath().delete(); - } catch (IOException e) { - // If deleting of output files failed, we abort the build with a decent error message as - // any subsequent local execution failure would likely be incomprehensible. - - // We don't propagate the downloadException, as this is a recoverable error and the cause - // of the build failure is really that we couldn't delete output files. - throw new EnvironmentalExecException("Failed to delete output files after incomplete " - + "download. Cannot continue with local execution.", e, true); - } - throw downloadException; - } - } - - private void downloadOutErr(ActionResult result, FileOutErr outErr) - throws IOException, InterruptedException { - if (!result.getStdoutRaw().isEmpty()) { - result.getStdoutRaw().writeTo(outErr.getOutputStream()); - outErr.getOutputStream().flush(); - } else if (result.hasStdoutDigest()) { - byte[] stdoutBytes = downloadBlob(result.getStdoutDigest()); - outErr.getOutputStream().write(stdoutBytes); - outErr.getOutputStream().flush(); - } - if (!result.getStderrRaw().isEmpty()) { - result.getStderrRaw().writeTo(outErr.getErrorStream()); - outErr.getErrorStream().flush(); - } else if (result.hasStderrDigest()) { - byte[] stderrBytes = downloadBlob(result.getStderrDigest()); - outErr.getErrorStream().write(stderrBytes); - outErr.getErrorStream().flush(); - } - } - - /** * This method can throw {@link StatusRuntimeException}, but the RemoteCache interface does not * allow throwing such an exception. Any caller must make sure to catch the * {@link StatusRuntimeException}. Note that the retrier implicitly catches it, so if this is used @@ -296,6 +206,30 @@ } @Override + protected void downloadBlob(Digest digest, Path dest) throws IOException, InterruptedException { + retrier.execute( + () -> { + try (OutputStream stream = dest.getOutputStream()) { + readBlob(digest, stream); + } + return null; + }); + } + + @Override + protected byte[] downloadBlob(Digest digest) throws IOException, InterruptedException { + if (digest.getSizeBytes() == 0) { + return new byte[0]; + } + return retrier.execute( + () -> { + ByteArrayOutputStream stream = new ByteArrayOutputStream((int) digest.getSizeBytes()); + readBlob(digest, stream); + return stream.toByteArray(); + }); + } + + @Override public void upload( ActionKey actionKey, Path execRoot, @@ -329,38 +263,33 @@ void upload(Path execRoot, Collection<Path> files, FileOutErr outErr, ActionResult.Builder result) throws IOException, InterruptedException { - Map<Digest, Path> digestToFile = new HashMap<>(); - for (Path file : files) { - if (!file.exists()) { - // We ignore requested results that have not been generated by the action. - continue; - } - if (file.isDirectory()) { - // TODO(olaola): to implement this for a directory, will need to create or pass a - // TreeNodeRepository to call uploadTree. - throw new UnsupportedOperationException("Storing a directory is not yet supported."); - } + UploadManifest manifest = new UploadManifest(result, execRoot); + manifest.addFiles(files); - Digest digest = digestUtil.compute(file); - // TODO(olaola): inline small results here. - result - .addOutputFilesBuilder() - .setPath(file.relativeTo(execRoot).getPathString()) - .setDigest(digest) - .setIsExecutable(file.isExecutable()); - digestToFile.put(digest, file); - } - - ImmutableSet<Digest> digestsToUpload = getMissingDigests(digestToFile.keySet()); List<Chunker> filesToUpload = new ArrayList<>(); + + Map<Digest, Path> digestToFile = manifest.getDigestToFile(); + Map<Digest, Chunker> digestToChunkers = manifest.getDigestToChunkers(); + Collection<Digest> digests = new ArrayList<>(); + digests.addAll(digestToFile.keySet()); + digests.addAll(digestToChunkers.keySet()); + + ImmutableSet<Digest> digestsToUpload = getMissingDigests(digests); for (Digest digest : digestsToUpload) { + Chunker chunker; Path file = digestToFile.get(digest); - if (file == null) { - String message = "FindMissingBlobs call returned an unknown digest: " + digest; - throw new IOException(message); + if (file != null) { + chunker = new Chunker(file); + } else { + chunker = digestToChunkers.get(digest); + if (chunker == null) { + String message = "FindMissingBlobs call returned an unknown digest: " + digest; + throw new IOException(message); + } } - filesToUpload.add(new Chunker(file)); + filesToUpload.add(chunker); } + if (!filesToUpload.isEmpty()) { uploader.uploadBlobs(filesToUpload); } @@ -416,19 +345,6 @@ return digest; } - byte[] downloadBlob(Digest digest) - throws IOException, InterruptedException { - if (digest.getSizeBytes() == 0) { - return new byte[0]; - } - return retrier.execute( - () -> { - ByteArrayOutputStream stream = new ByteArrayOutputStream((int) digest.getSizeBytes()); - readBlob(digest, stream); - return stream.toByteArray(); - }); - } - // Execution Cache API @Override
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionCache.java deleted file mode 100644 index 6bc53aa..0000000 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionCache.java +++ /dev/null
@@ -1,91 +0,0 @@ -// Copyright 2016 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.remote; - -import com.google.devtools.build.lib.actions.ExecException; -import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; -import com.google.devtools.build.lib.remote.DigestUtil.ActionKey; -import com.google.devtools.build.lib.remote.TreeNodeRepository.TreeNode; -import com.google.devtools.build.lib.util.io.FileOutErr; -import com.google.devtools.build.lib.vfs.Path; -import com.google.devtools.remoteexecution.v1test.ActionResult; -import com.google.devtools.remoteexecution.v1test.Command; -import java.io.IOException; -import java.util.Collection; -import javax.annotation.Nullable; - -/** A cache for storing artifacts (input and output) as well as the output of running an action. */ -@ThreadCompatible -interface RemoteActionCache { - // CAS API - - // TODO(buchgr): consider removing the CacheNotFoundException, and replacing it with other - // ways to signal a cache miss. - - /** - * Ensures that the tree structure of the inputs, the input files themselves, and the command are - * available in the remote cache, such that the tree can be reassembled and executed on another - * machine given the root digest. - * - * <p>The cache may check whether files or parts of the tree structure are already present, and do - * not need to be uploaded again. - * - * <p>Note that this method is only required for remote execution, not for caching itself. - * However, remote execution uses a cache to store input files, and that may be a separate - * end-point from the executor itself, so the functionality lives here. A pure remote caching - * implementation that does not support remote execution may choose not to implement this - * function, and throw {@link UnsupportedOperationException} instead. If so, it should be clearly - * documented that it cannot be used for remote execution. - */ - void ensureInputsPresent( - TreeNodeRepository repository, Path execRoot, TreeNode root, Command command) - throws IOException, InterruptedException; - - /** - * Download the output files and directory trees of a remotely executed action to the local - * machine, as well stdin / stdout to the given files. - * - * <p>In case of failure, this method must delete any output files it might have already created. - * - * @throws CacheNotFoundException in case of a cache miss. - * @throws ExecException in case clean up after a failed download failed. - */ - // TODO(olaola): will need to amend to include the TreeNodeRepository for updating. - void download(ActionResult result, Path execRoot, FileOutErr outErr) - throws ExecException, IOException, InterruptedException; - /** - * Attempts to look up the given action in the remote cache and return its result, if present. - * Returns {@code null} if there is no such entry. Note that a successful result from this method - * does not guarantee the availability of the corresponding output files in the remote cache. - */ - @Nullable - ActionResult getCachedActionResult(ActionKey actionKey) throws IOException, InterruptedException; - - /** - * Upload the result of a locally executed action to the cache by uploading any necessary files, - * stdin / stdout, as well as adding an entry for the given action key to the cache if - * uploadAction is true. - */ - void upload( - ActionKey actionKey, - Path execRoot, - Collection<Path> files, - FileOutErr outErr, - boolean uploadAction) - throws IOException, InterruptedException; - - /** Release resources associated with the cache. The cache may not be used after calling this. */ - void close(); -}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java index c66015a..a20b51f 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java
@@ -34,13 +34,13 @@ */ final class RemoteActionContextProvider extends ActionContextProvider { private final CommandEnvironment env; - private final RemoteActionCache cache; + private final AbstractRemoteActionCache cache; private final GrpcRemoteExecutor executor; private final DigestUtil digestUtil; RemoteActionContextProvider( CommandEnvironment env, - @Nullable RemoteActionCache cache, + @Nullable AbstractRemoteActionCache cache, @Nullable GrpcRemoteExecutor executor, DigestUtil digestUtil) { this.env = env;
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java index 2cd1222..235f971 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
@@ -117,7 +117,7 @@ remoteOptions, RemoteRetrier.RETRIABLE_GRPC_ERRORS, Retrier.ALLOW_ALL_CALLS); // TODO(davido): The naming is wrong here. "Remote"-prefix in RemoteActionCache class has no // meaning. - final RemoteActionCache cache; + final AbstractRemoteActionCache cache; if (remoteOrLocalCache) { cache = new SimpleBlobStoreActionCache(
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java index b0487a7..e8af197 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
@@ -52,7 +52,7 @@ private final Path execRoot; private final RemoteOptions options; - private final RemoteActionCache remoteCache; + private final AbstractRemoteActionCache remoteCache; private final String buildRequestId; private final String commandId; private final boolean verboseFailures; @@ -67,7 +67,7 @@ RemoteSpawnCache( Path execRoot, RemoteOptions options, - RemoteActionCache remoteCache, + AbstractRemoteActionCache remoteCache, String buildRequestId, String commandId, boolean verboseFailures, @@ -95,6 +95,7 @@ Command command = RemoteSpawnRunner.buildCommand(spawn.getArguments(), spawn.getEnvironment()); Action action = RemoteSpawnRunner.buildAction( + execRoot, spawn.getOutputFiles(), digestUtil.compute(command), repository.getMerkleDigest(inputRoot),
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java index 695f6ea..55a64df 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
@@ -78,7 +78,7 @@ private final boolean verboseFailures; @Nullable private final Reporter cmdlineReporter; - @Nullable private final RemoteActionCache remoteCache; + @Nullable private final AbstractRemoteActionCache remoteCache; @Nullable private final GrpcRemoteExecutor remoteExecutor; private final String buildRequestId; private final String commandId; @@ -95,7 +95,7 @@ @Nullable Reporter cmdlineReporter, String buildRequestId, String commandId, - @Nullable RemoteActionCache remoteCache, + @Nullable AbstractRemoteActionCache remoteCache, @Nullable GrpcRemoteExecutor remoteExecutor, DigestUtil digestUtil) { this.execRoot = execRoot; @@ -127,6 +127,7 @@ Command command = buildCommand(spawn.getArguments(), spawn.getEnvironment()); Action action = buildAction( + execRoot, spawn.getOutputFiles(), digestUtil.compute(command), repository.getMerkleDigest(inputRoot), @@ -260,6 +261,7 @@ } static Action buildAction( + Path execRoot, Collection<? extends ActionInput> outputs, Digest command, Digest inputRoot, @@ -271,11 +273,17 @@ action.setCommandDigest(command); action.setInputRootDigest(inputRoot); ArrayList<String> outputPaths = new ArrayList<>(); + ArrayList<String> outputDirectoryPaths = new ArrayList<>(); for (ActionInput output : outputs) { - outputPaths.add(output.getExecPathString()); + String pathString = output.getExecPathString(); + if (execRoot.getRelative(pathString).isDirectory()) { + outputDirectoryPaths.add(pathString); + } else { + outputPaths.add(pathString); + } } Collections.sort(outputPaths); - // TODO: output directories should be handled here, when they are supported. + Collections.sort(outputDirectoryPaths); action.addAllOutputFiles(outputPaths); // Get the remote platform properties. @@ -350,7 +358,7 @@ SpawnExecutionPolicy policy, SortedMap<PathFragment, ActionInput> inputMap, boolean uploadToCache, - @Nullable RemoteActionCache remoteCache, + @Nullable AbstractRemoteActionCache remoteCache, @Nullable ActionKey actionKey) throws ExecException, IOException, InterruptedException { if (uploadToCache && remoteCache != null && actionKey != null) { @@ -364,7 +372,7 @@ Spawn spawn, SpawnExecutionPolicy policy, SortedMap<PathFragment, ActionInput> inputMap, - RemoteActionCache remoteCache, + AbstractRemoteActionCache remoteCache, ActionKey actionKey) throws ExecException, IOException, InterruptedException { Map<Path, Long> ctimesBefore = getInputCtimes(inputMap);
diff --git a/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCache.java b/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCache.java index db60846..d9c67dc 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCache.java +++ b/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCache.java
@@ -15,8 +15,6 @@ package com.google.devtools.build.lib.remote; import com.google.devtools.build.lib.actions.ActionInput; -import com.google.devtools.build.lib.actions.EnvironmentalExecException; -import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.actions.MetadataProvider; import com.google.devtools.build.lib.actions.cache.VirtualActionInput; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; @@ -32,7 +30,6 @@ import com.google.devtools.remoteexecution.v1test.Directory; import com.google.devtools.remoteexecution.v1test.DirectoryNode; import com.google.devtools.remoteexecution.v1test.FileNode; -import com.google.devtools.remoteexecution.v1test.OutputFile; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import java.io.ByteArrayInputStream; @@ -41,6 +38,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.Collection; +import java.util.Map; /** * A RemoteActionCache implementation that uses a concurrent map as a distributed storage for files @@ -51,15 +49,14 @@ * <p>Note that this class is used from src/tools/remote. */ @ThreadSafe -public final class SimpleBlobStoreActionCache implements RemoteActionCache { +public final class SimpleBlobStoreActionCache extends AbstractRemoteActionCache { private static final int MAX_BLOB_SIZE_FOR_INLINE = 10 * 1024; private final SimpleBlobStore blobStore; - private final DigestUtil digestUtil; public SimpleBlobStoreActionCache(SimpleBlobStore blobStore, DigestUtil digestUtil) { + super(digestUtil); this.blobStore = blobStore; - this.digestUtil = digestUtil; } @Override @@ -79,10 +76,11 @@ public void downloadTree(Digest rootDigest, Path rootLocation) throws IOException, InterruptedException { + FileSystemUtils.createDirectoryAndParents(rootLocation); Directory directory = Directory.parseFrom(downloadBlob(rootDigest)); for (FileNode file : directory.getFilesList()) { - downloadFileContents( - file.getDigest(), rootLocation.getRelative(file.getName()), file.getIsExecutable()); + downloadFile( + rootLocation.getRelative(file.getName()), file.getDigest(), file.getIsExecutable(), null); } for (DirectoryNode child : directory.getDirectoriesList()) { downloadTree(child.getDigest(), rootLocation.getRelative(child.getName())); @@ -109,63 +107,6 @@ } @Override - public void download(ActionResult result, Path execRoot, FileOutErr outErr) - throws ExecException, IOException, InterruptedException { - try { - for (OutputFile file : result.getOutputFilesList()) { - if (!file.getContent().isEmpty()) { - createFile( - file.getContent().toByteArray(), - execRoot.getRelative(file.getPath()), - file.getIsExecutable()); - } else { - downloadFileContents( - file.getDigest(), execRoot.getRelative(file.getPath()), file.getIsExecutable()); - } - } - if (!result.getOutputDirectoriesList().isEmpty()) { - throw new UnsupportedOperationException(); - } - downloadOutErr(result, outErr); - } catch (IOException downloadException) { - try { - // Delete any (partially) downloaded output files, since any subsequent local execution - // of this action may expect none of the output files to exist. - for (OutputFile file : result.getOutputFilesList()) { - execRoot.getRelative(file.getPath()).delete(); - } - outErr.getOutputPath().delete(); - outErr.getErrorPath().delete(); - } catch (IOException e) { - // If deleting of output files failed, we abort the build with a decent error message as - // any subsequent local execution failure would likely be incomprehensible. - - // We don't propagate the downloadException, as this is a recoverable error and the cause - // of the build failure is really that we couldn't delete output files. - throw new EnvironmentalExecException("Failed to delete output files after incomplete " - + "download. Cannot continue with local execution.", e, true); - } - throw downloadException; - } - } - - private void downloadOutErr(ActionResult result, FileOutErr outErr) - throws IOException, InterruptedException { - if (!result.getStdoutRaw().isEmpty()) { - result.getStdoutRaw().writeTo(outErr.getOutputStream()); - outErr.getOutputStream().flush(); - } else if (result.hasStdoutDigest()) { - downloadFileContents(result.getStdoutDigest(), outErr.getOutputPath(), /*executable=*/false); - } - if (!result.getStderrRaw().isEmpty()) { - result.getStderrRaw().writeTo(outErr.getErrorStream()); - outErr.getErrorStream().flush(); - } else if (result.hasStderrDigest()) { - downloadFileContents(result.getStderrDigest(), outErr.getErrorPath(), /*executable=*/false); - } - } - - @Override public void upload( ActionKey actionKey, Path execRoot, @@ -190,26 +131,17 @@ public void upload(ActionResult.Builder result, Path execRoot, Collection<Path> files) throws IOException, InterruptedException { - for (Path file : files) { - // TODO(ulfjack): Maybe pass in a SpawnResult here, add a list of output files to that, and - // rely on the local spawn runner to stat the files, instead of statting here. - if (!file.exists()) { - continue; + UploadManifest manifest = new UploadManifest(result, execRoot); + manifest.addFiles(files); + + for (Map.Entry<Digest, Path> entry : manifest.getDigestToFile().entrySet()) { + try (InputStream in = entry.getValue().getInputStream()) { + uploadStream(entry.getKey(), in); } - if (file.isDirectory()) { - // TODO(olaola): to implement this for a directory, will need to create or pass a - // TreeNodeRepository to call uploadTree. - throw new IOException("Storing a directory is not yet supported."); - } - // TODO(olaola): inline small file contents here. - // First put the file content to cache. - Digest digest = uploadFileContents(file); - // Add to protobuf. - result - .addOutputFilesBuilder() - .setPath(file.relativeTo(execRoot).getPathString()) - .setDigest(digest) - .setIsExecutable(file.isExecutable()); + } + + for (Map.Entry<Digest, Chunker> entry : manifest.getDigestToChunkers().entrySet()) { + uploadBlob(entry.getValue().next().getData().toByteArray(), entry.getKey()); } } @@ -227,23 +159,6 @@ } } - private void downloadFileContents(Digest digest, Path dest, boolean executable) - throws IOException, InterruptedException { - FileSystemUtils.createDirectoryAndParents(dest.getParentDirectory()); - try (OutputStream out = dest.getOutputStream()) { - downloadBlob(digest, out); - } - dest.setExecutable(executable); - } - - private void createFile(byte[] contents, Path dest, boolean executable) throws IOException { - FileSystemUtils.createDirectoryAndParents(dest.getParentDirectory()); - try (OutputStream stream = dest.getOutputStream()) { - stream.write(contents); - } - dest.setExecutable(executable); - } - public Digest uploadBlob(byte[] blob) throws IOException, InterruptedException { return uploadBlob(blob, digestUtil.compute(blob)); } @@ -258,27 +173,6 @@ return digest; } - private void downloadBlob(Digest digest, OutputStream out) - throws IOException, InterruptedException { - if (digest.getSizeBytes() == 0) { - return; - } - boolean success = blobStore.get(digest.getHash(), out); - if (!success) { - throw new CacheNotFoundException(digest); - } - } - - public byte[] downloadBlob(Digest digest) - throws IOException, InterruptedException { - if (digest.getSizeBytes() == 0) { - return new byte[0]; - } - ByteArrayOutputStream out = new ByteArrayOutputStream(); - downloadBlob(digest, out); - return out.toByteArray(); - } - public boolean containsKey(Digest digest) throws IOException, InterruptedException { return blobStore.containsKey(digest.getHash()); } @@ -316,4 +210,28 @@ public void close() { blobStore.close(); } + + @Override + protected void downloadBlob(Digest digest, Path dest) throws IOException, InterruptedException { + try (OutputStream out = dest.getOutputStream()) { + boolean success = blobStore.get(digest.getHash(), out); + if (!success) { + throw new CacheNotFoundException(digest); + } + } + } + + @Override + public byte[] downloadBlob(Digest digest) throws IOException, InterruptedException { + if (digest.getSizeBytes() == 0) { + return new byte[0]; + } + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + boolean success = blobStore.get(digest.getHash(), out); + if (!success) { + throw new CacheNotFoundException(digest); + } + return out.toByteArray(); + } + } }
diff --git a/src/main/java/com/google/devtools/build/lib/remote/TreeNodeRepository.java b/src/main/java/com/google/devtools/build/lib/remote/TreeNodeRepository.java index 0554682..1f836a9 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/TreeNodeRepository.java +++ b/src/main/java/com/google/devtools/build/lib/remote/TreeNodeRepository.java
@@ -24,19 +24,24 @@ import com.google.common.collect.TreeTraverser; import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.ActionInputHelper; +import com.google.devtools.build.lib.actions.cache.Metadata; import com.google.devtools.build.lib.actions.cache.VirtualActionInput; import com.google.devtools.build.lib.concurrent.BlazeInterners; import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import com.google.devtools.build.lib.exec.SpawnInputExpander; +import com.google.devtools.build.lib.vfs.Dirent; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.Symlinks; import com.google.devtools.remoteexecution.v1test.Digest; import com.google.devtools.remoteexecution.v1test.Directory; import com.google.protobuf.ByteString; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -51,10 +56,22 @@ */ @ThreadSafe public final class TreeNodeRepository extends TreeTraverser<TreeNodeRepository.TreeNode> { + // In this implementation, symlinks are NOT followed when expanding directory artifacts + public static final Symlinks SYMLINK_POLICY = Symlinks.NOFOLLOW; + /** * A single node in a hierarchical directory structure. Leaves are the Artifacts, although we only * use the ActionInput interface. We assume that the objects used for the ActionInputs are unique * (same data corresponds to a canonical object in memory). + * + * <p>There are three cases: + * + * <ol> + * <li>The node is a leaf that represents an artifact file. + * <li>The node is a directory optionally associated with an artifact (an "artifact directory"). + * <li>The node is a leaf that is the descendant of an artifact directory. In this case, the + * node is associated with a BasicActionInput, not a full Artifact. + * </ol> */ @Immutable @ThreadSafe @@ -62,7 +79,8 @@ private final int hashCode; private final ImmutableList<ChildEntry> childEntries; // no need to make it a map thus far. - @Nullable private final ActionInput actionInput; // Null iff this is a directory. + @Nullable private final ActionInput actionInput; + private final boolean isLeaf; /** A pair of path segment, TreeNode. */ @Immutable @@ -105,15 +123,23 @@ } // Should only be called by the TreeNodeRepository. - private TreeNode(Iterable<ChildEntry> childEntries) { - this.actionInput = null; + private TreeNode(Iterable<ChildEntry> childEntries, @Nullable ActionInput actionInput) { + isLeaf = false; + this.actionInput = actionInput; this.childEntries = ImmutableList.copyOf(childEntries); - hashCode = Arrays.hashCode(this.childEntries.toArray()); + if (actionInput != null) { + hashCode = actionInput.hashCode(); // This will ensure efficient interning of TreeNodes as + // long as all ActionInputs either implement data-based hashCode or are interned themselves. + } else { + hashCode = Arrays.hashCode(this.childEntries.toArray()); + } } // Should only be called by the TreeNodeRepository. private TreeNode(ActionInput actionInput) { - this.actionInput = actionInput; + isLeaf = true; + this.actionInput = + Preconditions.checkNotNull(actionInput, "a TreeNode leaf should have an ActionInput"); this.childEntries = ImmutableList.of(); hashCode = actionInput.hashCode(); // This will ensure efficient interning of TreeNodes as // long as all ActionInputs either implement data-based hashCode or are interned themselves. @@ -128,7 +154,7 @@ } public boolean isLeaf() { - return actionInput != null; + return isLeaf; } @Override @@ -176,7 +202,8 @@ } } - private static final TreeNode EMPTY_NODE = new TreeNode(ImmutableList.<TreeNode.ChildEntry>of()); + private static final TreeNode EMPTY_NODE = + new TreeNode(ImmutableList.<TreeNode.ChildEntry>of(), null); // Keep only one canonical instance of every TreeNode in the repository. private final Interner<TreeNode> interner = BlazeInterners.newWeakInterner(); @@ -184,6 +211,8 @@ // be part of the state. private final Path execRoot; private final ActionInputFileCache inputFileCache; + // For directories that are themselves artifacts, map of the ActionInput to the Merkle hash + private final Map<ActionInput, Digest> inputDirectoryDigestCache = new HashMap<>(); private final Map<TreeNode, Digest> treeNodeDigestCache = new HashMap<>(); private final Map<Digest, TreeNode> digestTreeNodeCache = new HashMap<>(); private final Map<TreeNode, Directory> directoryCache = new HashMap<>(); @@ -226,7 +255,7 @@ }); } - public TreeNode buildFromActionInputs(Iterable<? extends ActionInput> inputs) { + public TreeNode buildFromActionInputs(Iterable<? extends ActionInput> inputs) throws IOException { TreeMap<PathFragment, ActionInput> sortedMap = new TreeMap<>(); for (ActionInput input : inputs) { sortedMap.put(PathFragment.create(input.getExecPathString()), input); @@ -239,7 +268,8 @@ * of input files. TODO(olaola): switch to creating and maintaining the TreeNodeRepository based * on the build graph structure. */ - public TreeNode buildFromActionInputs(SortedMap<PathFragment, ActionInput> sortedMap) { + public TreeNode buildFromActionInputs(SortedMap<PathFragment, ActionInput> sortedMap) + throws IOException { ImmutableList.Builder<ImmutableList<String>> segments = ImmutableList.builder(); for (PathFragment path : sortedMap.keySet()) { segments.add(path.getSegments()); @@ -255,13 +285,35 @@ return buildParentNode(inputs, segments.build(), 0, inputs.size(), 0); } + // Expand the descendant of an artifact (input) directory + private List<TreeNode.ChildEntry> buildInputDirectoryEntries(Path path) throws IOException { + List<Dirent> sortedDirent = new ArrayList<>(path.readdir(SYMLINK_POLICY)); + sortedDirent.sort(Comparator.comparing(Dirent::getName)); + + List<TreeNode.ChildEntry> entries = new ArrayList<>(sortedDirent.size()); + for (Dirent dirent : sortedDirent) { + String name = dirent.getName(); + Path child = path.getRelative(name); + TreeNode childNode; + if (dirent.getType() == Dirent.Type.DIRECTORY) { + childNode = interner.intern(new TreeNode(buildInputDirectoryEntries(child), null)); + } else { + childNode = interner.intern(new TreeNode(ActionInputHelper.fromPath(child.asFragment()))); + } + entries.add(new TreeNode.ChildEntry(name, childNode)); + } + + return entries; + } + @SuppressWarnings("ReferenceEquality") // Segments are interned. private TreeNode buildParentNode( List<ActionInput> inputs, ImmutableList<ImmutableList<String>> segments, int inputsStart, int inputsEnd, - int segmentIndex) { + int segmentIndex) + throws IOException { if (segments.isEmpty()) { // We sometimes have actions with no inputs (e.g., echo "xyz" > $@), so we need to handle that // case here. @@ -273,7 +325,12 @@ Preconditions.checkArgument( inputsStart == inputsEnd - 1, "Encountered two inputs with the same path."); // TODO: check that the actionInput is a single file! - return interner.intern(new TreeNode(inputs.get(inputsStart))); + ActionInput input = inputs.get(inputsStart); + Path leafPath = execRoot.getRelative(input.getExecPathString()); + if (leafPath.isDirectory()) { + return interner.intern(new TreeNode(buildInputDirectoryEntries(leafPath), input)); + } + return interner.intern(new TreeNode(input)); } ArrayList<TreeNode.ChildEntry> entries = new ArrayList<>(); String segment = segments.get(inputsStart).get(segmentIndex); @@ -290,7 +347,7 @@ } } } - return interner.intern(new TreeNode(entries)); + return interner.intern(new TreeNode(entries, null)); } private synchronized Directory getOrComputeDirectory(TreeNode node) throws IOException { @@ -322,6 +379,9 @@ } } else { Digest childDigest = Preconditions.checkNotNull(treeNodeDigestCache.get(child)); + if (child.getActionInput() != null) { + inputDirectoryDigestCache.put(child.getActionInput(), childDigest); + } b.addDirectoriesBuilder().setName(entry.getSegment()).setDigest(childDigest); } } @@ -380,6 +440,15 @@ if (input instanceof VirtualActionInput) { return Preconditions.checkNotNull(virtualInputDigestCache.get(input)); } + Metadata metadata = Preconditions.checkNotNull(inputFileCache.getMetadata(input)); + byte[] digest = metadata.getDigest(); + if (digest == null) { + // If the artifact does not have a digest, it is because it is a directory. + // We get the digest from the set of Merkle hashes computed in this TreeNodeRepository. + return Preconditions.checkNotNull( + inputDirectoryDigestCache.get(input), + "a directory should have a precomputed Merkle hash (instead of a digest)"); + } return DigestUtil.getFromInputCache(input, inputFileCache); }
diff --git a/src/test/java/com/google/devtools/build/lib/remote/FakeActionInputFileCache.java b/src/test/java/com/google/devtools/build/lib/remote/FakeActionInputFileCache.java index 3f50968..dd37b3c 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/FakeActionInputFileCache.java +++ b/src/test/java/com/google/devtools/build/lib/remote/FakeActionInputFileCache.java
@@ -24,6 +24,7 @@ import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.remoteexecution.v1test.Digest; +import com.google.devtools.remoteexecution.v1test.Tree; import com.google.protobuf.ByteString; import java.io.IOException; import javax.annotation.Nullable; @@ -76,4 +77,12 @@ setDigest(input, digest.getHash()); return digest; } + + public Digest createScratchInputDirectory(ActionInput input, Tree content) throws IOException { + Path inputFile = execRoot.getRelative(input.getExecPath()); + FileSystemUtils.createDirectoryAndParents(inputFile); + Digest digest = digestUtil.compute(content); + setDigest(input, digest.getHash()); + return digest; + } } \ No newline at end of file
diff --git a/src/test/java/com/google/devtools/build/lib/remote/FakeImmutableCacheByteStreamImpl.java b/src/test/java/com/google/devtools/build/lib/remote/FakeImmutableCacheByteStreamImpl.java index e418a29..f8d8fe5 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/FakeImmutableCacheByteStreamImpl.java +++ b/src/test/java/com/google/devtools/build/lib/remote/FakeImmutableCacheByteStreamImpl.java
@@ -32,14 +32,25 @@ // Start returning the correct response after this number of errors is reached. private static final int MAX_ERRORS = 3; - public FakeImmutableCacheByteStreamImpl(Map<Digest, String> contents) { + public FakeImmutableCacheByteStreamImpl(Map<Digest, Object> contents) { ImmutableMap.Builder<ReadRequest, ReadResponse> b = ImmutableMap.builder(); - for (Map.Entry<Digest, String> e : contents.entrySet()) { + for (Map.Entry<Digest, Object> e : contents.entrySet()) { + Object obj = e.getValue(); + ByteString data; + if (obj instanceof String) { + data = ByteString.copyFromUtf8((String) obj); + } else if (obj instanceof ByteString) { + data = (ByteString) obj; + } else { + throw new AssertionError( + "expected object to be either a String or a ByteString, got a " + + obj.getClass().getCanonicalName()); + } b.put( ReadRequest.newBuilder() .setResourceName("blobs/" + e.getKey().getHash() + "/" + e.getKey().getSizeBytes()) .build(), - ReadResponse.newBuilder().setData(ByteString.copyFromUtf8(e.getValue())).build()); + ReadResponse.newBuilder().setData(data).build()); } cannedReplies = b.build(); numErrors = new HashMap<>(); @@ -55,7 +66,7 @@ @Override public void read(ReadRequest request, StreamObserver<ReadResponse> responseObserver) { - assertThat(cannedReplies.containsKey(request)).isTrue(); + assertThat(cannedReplies.keySet()).contains(request); int errCount = numErrors.getOrDefault(request, 0); if (errCount < MAX_ERRORS) { numErrors.put(request, errCount + 1);
diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteCacheTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteCacheTest.java index 76d3587..83d4efd 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteCacheTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteCacheTest.java
@@ -26,6 +26,7 @@ import com.google.bytestream.ByteStreamProto.WriteRequest; import com.google.bytestream.ByteStreamProto.WriteResponse; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.ActionInputHelper; import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions; import com.google.devtools.build.lib.authandtls.GoogleAuthUtils; @@ -44,9 +45,13 @@ import com.google.devtools.remoteexecution.v1test.ActionResult; import com.google.devtools.remoteexecution.v1test.ContentAddressableStorageGrpc.ContentAddressableStorageImplBase; import com.google.devtools.remoteexecution.v1test.Digest; +import com.google.devtools.remoteexecution.v1test.Directory; +import com.google.devtools.remoteexecution.v1test.DirectoryNode; +import com.google.devtools.remoteexecution.v1test.FileNode; import com.google.devtools.remoteexecution.v1test.FindMissingBlobsRequest; import com.google.devtools.remoteexecution.v1test.FindMissingBlobsResponse; import com.google.devtools.remoteexecution.v1test.GetActionResultRequest; +import com.google.devtools.remoteexecution.v1test.Tree; import com.google.devtools.remoteexecution.v1test.UpdateActionResultRequest; import com.google.protobuf.ByteString; import io.grpc.CallCredentials; @@ -233,6 +238,96 @@ } @Test + public void testDownloadDirectory() throws Exception { + GrpcRemoteCache client = newClient(); + Digest fooDigest = DIGEST_UTIL.computeAsUtf8("foo-contents"); + Digest quxDigest = DIGEST_UTIL.computeAsUtf8("qux-contents"); + Tree barTreeMessage = + Tree.newBuilder() + .setRoot( + Directory.newBuilder() + .addFiles( + FileNode.newBuilder() + .setName("qux") + .setDigest(quxDigest) + .setIsExecutable(true))) + .build(); + Digest barTreeDigest = DIGEST_UTIL.compute(barTreeMessage); + serviceRegistry.addService( + new FakeImmutableCacheByteStreamImpl( + ImmutableMap.of( + fooDigest, "foo-contents", + barTreeDigest, barTreeMessage.toByteString(), + quxDigest, "qux-contents"))); + + ActionResult.Builder result = ActionResult.newBuilder(); + result.addOutputFilesBuilder().setPath("a/foo").setDigest(fooDigest); + result.addOutputDirectoriesBuilder().setPath("a/bar").setTreeDigest(barTreeDigest); + client.download(result.build(), execRoot, null); + + assertThat(DIGEST_UTIL.compute(execRoot.getRelative("a/foo"))).isEqualTo(fooDigest); + assertThat(DIGEST_UTIL.compute(execRoot.getRelative("a/bar/qux"))).isEqualTo(quxDigest); + assertThat(execRoot.getRelative("a/bar/qux").isExecutable()).isTrue(); + } + + @Test + public void testDownloadDirectoryEmpty() throws Exception { + GrpcRemoteCache client = newClient(); + Tree barTreeMessage = Tree.newBuilder().setRoot(Directory.newBuilder()).build(); + Digest barTreeDigest = DIGEST_UTIL.compute(barTreeMessage); + serviceRegistry.addService( + new FakeImmutableCacheByteStreamImpl( + ImmutableMap.of(barTreeDigest, barTreeMessage.toByteString()))); + + ActionResult.Builder result = ActionResult.newBuilder(); + result.addOutputDirectoriesBuilder().setPath("a/bar").setTreeDigest(barTreeDigest); + client.download(result.build(), execRoot, null); + + assertThat(execRoot.getRelative("a/bar").isDirectory()).isTrue(); + } + + @Test + public void testDownloadDirectoryNested() throws Exception { + GrpcRemoteCache client = newClient(); + Digest fooDigest = DIGEST_UTIL.computeAsUtf8("foo-contents"); + Digest quxDigest = DIGEST_UTIL.computeAsUtf8("qux-contents"); + Directory wobbleDirMessage = + Directory.newBuilder() + .addFiles(FileNode.newBuilder().setName("qux").setDigest(quxDigest)) + .build(); + Digest wobbleDirDigest = DIGEST_UTIL.compute(wobbleDirMessage); + Tree barTreeMessage = + Tree.newBuilder() + .setRoot( + Directory.newBuilder() + .addFiles( + FileNode.newBuilder() + .setName("qux") + .setDigest(quxDigest) + .setIsExecutable(true)) + .addDirectories( + DirectoryNode.newBuilder().setName("wobble").setDigest(wobbleDirDigest))) + .addChildren(wobbleDirMessage) + .build(); + Digest barTreeDigest = DIGEST_UTIL.compute(barTreeMessage); + serviceRegistry.addService( + new FakeImmutableCacheByteStreamImpl( + ImmutableMap.of( + fooDigest, "foo-contents", + barTreeDigest, barTreeMessage.toByteString(), + quxDigest, "qux-contents"))); + + ActionResult.Builder result = ActionResult.newBuilder(); + result.addOutputFilesBuilder().setPath("a/foo").setDigest(fooDigest); + result.addOutputDirectoriesBuilder().setPath("a/bar").setTreeDigest(barTreeDigest); + client.download(result.build(), execRoot, null); + + assertThat(DIGEST_UTIL.compute(execRoot.getRelative("a/foo"))).isEqualTo(fooDigest); + assertThat(DIGEST_UTIL.compute(execRoot.getRelative("a/bar/wobble/qux"))).isEqualTo(quxDigest); + assertThat(execRoot.getRelative("a/bar/wobble/qux").isExecutable()).isFalse(); + } + + @Test public void testUploadBlobCacheHitWithRetries() throws Exception { final GrpcRemoteCache client = newClient(); final Digest digest = DIGEST_UTIL.computeAsUtf8("abcdefg"); @@ -391,6 +486,132 @@ } @Test + public void testUploadDirectory() throws Exception { + final GrpcRemoteCache client = newClient(); + final Digest fooDigest = + fakeFileCache.createScratchInput(ActionInputHelper.fromPath("a/foo"), "xyz"); + final Digest quxDigest = + fakeFileCache.createScratchInput(ActionInputHelper.fromPath("bar/qux"), "abc"); + final Digest barDigest = + fakeFileCache.createScratchInputDirectory( + ActionInputHelper.fromPath("bar"), + Tree.newBuilder() + .setRoot( + Directory.newBuilder() + .addFiles( + FileNode.newBuilder() + .setIsExecutable(true) + .setName("qux") + .setDigest(quxDigest) + .build()) + .build()) + .build()); + final Path fooFile = execRoot.getRelative("a/foo"); + final Path quxFile = execRoot.getRelative("bar/qux"); + quxFile.setExecutable(true); + final Path barDir = execRoot.getRelative("bar"); + serviceRegistry.addService( + new ContentAddressableStorageImplBase() { + @Override + public void findMissingBlobs( + FindMissingBlobsRequest request, + StreamObserver<FindMissingBlobsResponse> responseObserver) { + assertThat(request.getBlobDigestsList()) + .containsExactly(fooDigest, quxDigest, barDigest); + // Nothing is missing. + responseObserver.onNext(FindMissingBlobsResponse.getDefaultInstance()); + responseObserver.onCompleted(); + } + }); + + ActionResult.Builder result = ActionResult.newBuilder(); + client.upload(execRoot, ImmutableList.<Path>of(fooFile, barDir), outErr, result); + ActionResult.Builder expectedResult = ActionResult.newBuilder(); + expectedResult.addOutputFilesBuilder().setPath("a/foo").setDigest(fooDigest); + expectedResult.addOutputDirectoriesBuilder().setPath("bar").setTreeDigest(barDigest); + assertThat(result.build()).isEqualTo(expectedResult.build()); + } + + @Test + public void testUploadDirectoryEmpty() throws Exception { + final GrpcRemoteCache client = newClient(); + final Digest barDigest = + fakeFileCache.createScratchInputDirectory( + ActionInputHelper.fromPath("bar"), + Tree.newBuilder().setRoot(Directory.newBuilder().build()).build()); + final Path barDir = execRoot.getRelative("bar"); + serviceRegistry.addService( + new ContentAddressableStorageImplBase() { + @Override + public void findMissingBlobs( + FindMissingBlobsRequest request, + StreamObserver<FindMissingBlobsResponse> responseObserver) { + assertThat(request.getBlobDigestsList()).containsExactly(barDigest); + // Nothing is missing. + responseObserver.onNext(FindMissingBlobsResponse.getDefaultInstance()); + responseObserver.onCompleted(); + } + }); + + ActionResult.Builder result = ActionResult.newBuilder(); + client.upload(execRoot, ImmutableList.<Path>of(barDir), outErr, result); + ActionResult.Builder expectedResult = ActionResult.newBuilder(); + expectedResult.addOutputDirectoriesBuilder().setPath("bar").setTreeDigest(barDigest); + assertThat(result.build()).isEqualTo(expectedResult.build()); + } + + @Test + public void testUploadDirectoryNested() throws Exception { + final GrpcRemoteCache client = newClient(); + final Digest wobbleDigest = + fakeFileCache.createScratchInput(ActionInputHelper.fromPath("bar/test/wobble"), "xyz"); + final Digest quxDigest = + fakeFileCache.createScratchInput(ActionInputHelper.fromPath("bar/qux"), "abc"); + final Directory testDirMessage = + Directory.newBuilder() + .addFiles(FileNode.newBuilder().setName("wobble").setDigest(wobbleDigest).build()) + .build(); + final Digest testDigest = DIGEST_UTIL.compute(testDirMessage); + final Tree barTree = + Tree.newBuilder() + .setRoot( + Directory.newBuilder() + .addFiles( + FileNode.newBuilder() + .setIsExecutable(true) + .setName("qux") + .setDigest(quxDigest)) + .addDirectories( + DirectoryNode.newBuilder().setName("test").setDigest(testDigest))) + .addChildren(testDirMessage) + .build(); + final Digest barDigest = + fakeFileCache.createScratchInputDirectory(ActionInputHelper.fromPath("bar"), barTree); + final Path quxFile = execRoot.getRelative("bar/qux"); + quxFile.setExecutable(true); + final Path barDir = execRoot.getRelative("bar"); + serviceRegistry.addService( + new ContentAddressableStorageImplBase() { + @Override + public void findMissingBlobs( + FindMissingBlobsRequest request, + StreamObserver<FindMissingBlobsResponse> responseObserver) { + assertThat(request.getBlobDigestsList()) + .containsExactly(quxDigest, barDigest, wobbleDigest); + // Nothing is missing. + responseObserver.onNext(FindMissingBlobsResponse.getDefaultInstance()); + responseObserver.onCompleted(); + } + }); + + ActionResult.Builder result = ActionResult.newBuilder(); + client.upload(execRoot, ImmutableList.<Path>of(barDir), outErr, result); + ActionResult.Builder expectedResult = ActionResult.newBuilder(); + expectedResult.addOutputDirectoriesBuilder().setPath("bar").setTreeDigest(barDigest); + assertThat(result.build()).isEqualTo(expectedResult.build()); + } + + @Test public void testUploadCacheHits() throws Exception { final GrpcRemoteCache client = newClient(); final Digest fooDigest =
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java index 65c7d88..b14fb71 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java
@@ -87,7 +87,7 @@ private Path execRoot; private SimpleSpawn simpleSpawn; private FakeActionInputFileCache fakeFileCache; - @Mock private RemoteActionCache remoteCache; + @Mock private AbstractRemoteActionCache remoteCache; private RemoteSpawnCache cache; private FileOutErr outErr;
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java index 730244d..c2973b0 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java
@@ -87,8 +87,7 @@ private FakeActionInputFileCache fakeFileCache; private FileOutErr outErr; - @Mock - private RemoteActionCache cache; + @Mock private AbstractRemoteActionCache cache; @Mock private GrpcRemoteExecutor executor;
diff --git a/src/test/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCacheTest.java b/src/test/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCacheTest.java new file mode 100644 index 0000000..4088a83 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCacheTest.java
@@ -0,0 +1,321 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.remote; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.ActionInputHelper; +import com.google.devtools.build.lib.clock.JavaClock; +import com.google.devtools.build.lib.remote.blobstore.ConcurrentMapBlobStore; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.FileSystem.HashFunction; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; +import com.google.devtools.remoteexecution.v1test.ActionResult; +import com.google.devtools.remoteexecution.v1test.Digest; +import com.google.devtools.remoteexecution.v1test.Directory; +import com.google.devtools.remoteexecution.v1test.DirectoryNode; +import com.google.devtools.remoteexecution.v1test.FileNode; +import com.google.devtools.remoteexecution.v1test.Tree; +import io.grpc.Context; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link SimpleBlobStoreActionCache}. */ +@RunWith(JUnit4.class) +public class SimpleBlobStoreActionCacheTest { + private static final DigestUtil DIGEST_UTIL = new DigestUtil(HashFunction.SHA256); + + private FileSystem fs; + private Path execRoot; + private FakeActionInputFileCache fakeFileCache; + + @Before + public final void setUp() throws Exception { + Chunker.setDefaultChunkSizeForTesting(1000); // Enough for everything to be one chunk. + fs = new InMemoryFileSystem(new JavaClock(), HashFunction.SHA256); + execRoot = fs.getPath("/exec/root"); + FileSystemUtils.createDirectoryAndParents(execRoot); + fakeFileCache = new FakeActionInputFileCache(execRoot); + + Path stdout = fs.getPath("/tmp/stdout"); + Path stderr = fs.getPath("/tmp/stderr"); + FileSystemUtils.createDirectoryAndParents(stdout.getParentDirectory()); + FileSystemUtils.createDirectoryAndParents(stderr.getParentDirectory()); + Context withEmptyMetadata = + TracingMetadataUtils.contextWithMetadata( + "none", "none", DIGEST_UTIL.asActionKey(Digest.getDefaultInstance())); + withEmptyMetadata.attach(); + } + + private SimpleBlobStoreActionCache newClient() { + return newClient(new ConcurrentHashMap<>()); + } + + private SimpleBlobStoreActionCache newClient(ConcurrentMap<String, byte[]> map) { + return new SimpleBlobStoreActionCache(new ConcurrentMapBlobStore(map), DIGEST_UTIL); + } + + @Test + public void testDownloadEmptyBlob() throws Exception { + SimpleBlobStoreActionCache client = newClient(); + Digest emptyDigest = DIGEST_UTIL.compute(new byte[0]); + // Will not call the mock Bytestream interface at all. + assertThat(client.downloadBlob(emptyDigest)).isEmpty(); + } + + @Test + public void testDownloadBlob() throws Exception { + final ConcurrentMap<String, byte[]> map = new ConcurrentHashMap<>(); + Digest digest = DIGEST_UTIL.computeAsUtf8("abcdefg"); + map.put(digest.getHash(), "abcdefg".getBytes(Charsets.UTF_8)); + final SimpleBlobStoreActionCache client = newClient(map); + assertThat(new String(client.downloadBlob(digest), UTF_8)).isEqualTo("abcdefg"); + } + + @Test + public void testDownloadAllResults() throws Exception { + Digest fooDigest = DIGEST_UTIL.computeAsUtf8("foo-contents"); + Digest barDigest = DIGEST_UTIL.computeAsUtf8("bar-contents"); + Digest emptyDigest = DIGEST_UTIL.compute(new byte[0]); + + final ConcurrentMap<String, byte[]> map = new ConcurrentHashMap<>(); + map.put(fooDigest.getHash(), "foo-contents".getBytes(Charsets.UTF_8)); + map.put(barDigest.getHash(), "bar-contents".getBytes(Charsets.UTF_8)); + SimpleBlobStoreActionCache client = newClient(map); + + ActionResult.Builder result = ActionResult.newBuilder(); + result.addOutputFilesBuilder().setPath("a/foo").setDigest(fooDigest); + result.addOutputFilesBuilder().setPath("b/empty").setDigest(emptyDigest); + result.addOutputFilesBuilder().setPath("a/bar").setDigest(barDigest).setIsExecutable(true); + client.download(result.build(), execRoot, null); + assertThat(DIGEST_UTIL.compute(execRoot.getRelative("a/foo"))).isEqualTo(fooDigest); + assertThat(DIGEST_UTIL.compute(execRoot.getRelative("b/empty"))).isEqualTo(emptyDigest); + assertThat(DIGEST_UTIL.compute(execRoot.getRelative("a/bar"))).isEqualTo(barDigest); + assertThat(execRoot.getRelative("a/bar").isExecutable()).isTrue(); + } + + @Test + public void testDownloadDirectory() throws Exception { + Digest fooDigest = DIGEST_UTIL.computeAsUtf8("foo-contents"); + Digest quxDigest = DIGEST_UTIL.computeAsUtf8("qux-contents"); + Tree barTreeMessage = + Tree.newBuilder() + .setRoot( + Directory.newBuilder() + .addFiles( + FileNode.newBuilder() + .setName("qux") + .setDigest(quxDigest) + .setIsExecutable(true))) + .build(); + Digest barTreeDigest = DIGEST_UTIL.compute(barTreeMessage); + + final ConcurrentMap<String, byte[]> map = new ConcurrentHashMap<>(); + map.put(fooDigest.getHash(), "foo-contents".getBytes(Charsets.UTF_8)); + map.put(barTreeDigest.getHash(), barTreeMessage.toByteArray()); + map.put(quxDigest.getHash(), "qux-contents".getBytes(Charsets.UTF_8)); + SimpleBlobStoreActionCache client = newClient(map); + + ActionResult.Builder result = ActionResult.newBuilder(); + result.addOutputFilesBuilder().setPath("a/foo").setDigest(fooDigest); + result.addOutputDirectoriesBuilder().setPath("a/bar").setTreeDigest(barTreeDigest); + client.download(result.build(), execRoot, null); + + assertThat(DIGEST_UTIL.compute(execRoot.getRelative("a/foo"))).isEqualTo(fooDigest); + assertThat(DIGEST_UTIL.compute(execRoot.getRelative("a/bar/qux"))).isEqualTo(quxDigest); + assertThat(execRoot.getRelative("a/bar/qux").isExecutable()).isTrue(); + } + + @Test + public void testDownloadDirectoryEmpty() throws Exception { + Tree barTreeMessage = Tree.newBuilder().setRoot(Directory.newBuilder()).build(); + Digest barTreeDigest = DIGEST_UTIL.compute(barTreeMessage); + + final ConcurrentMap<String, byte[]> map = new ConcurrentHashMap<>(); + map.put(barTreeDigest.getHash(), barTreeMessage.toByteArray()); + SimpleBlobStoreActionCache client = newClient(map); + + ActionResult.Builder result = ActionResult.newBuilder(); + result.addOutputDirectoriesBuilder().setPath("a/bar").setTreeDigest(barTreeDigest); + client.download(result.build(), execRoot, null); + + assertThat(execRoot.getRelative("a/bar").isDirectory()).isTrue(); + } + + @Test + public void testDownloadDirectoryNested() throws Exception { + Digest fooDigest = DIGEST_UTIL.computeAsUtf8("foo-contents"); + Digest quxDigest = DIGEST_UTIL.computeAsUtf8("qux-contents"); + Directory wobbleDirMessage = + Directory.newBuilder() + .addFiles(FileNode.newBuilder().setName("qux").setDigest(quxDigest)) + .build(); + Digest wobbleDirDigest = DIGEST_UTIL.compute(wobbleDirMessage); + Tree barTreeMessage = + Tree.newBuilder() + .setRoot( + Directory.newBuilder() + .addFiles( + FileNode.newBuilder() + .setName("qux") + .setDigest(quxDigest) + .setIsExecutable(true)) + .addDirectories( + DirectoryNode.newBuilder().setName("wobble").setDigest(wobbleDirDigest))) + .addChildren(wobbleDirMessage) + .build(); + Digest barTreeDigest = DIGEST_UTIL.compute(barTreeMessage); + + final ConcurrentMap<String, byte[]> map = new ConcurrentHashMap<>(); + map.put(fooDigest.getHash(), "foo-contents".getBytes(Charsets.UTF_8)); + map.put(barTreeDigest.getHash(), barTreeMessage.toByteArray()); + map.put(quxDigest.getHash(), "qux-contents".getBytes(Charsets.UTF_8)); + SimpleBlobStoreActionCache client = newClient(map); + + ActionResult.Builder result = ActionResult.newBuilder(); + result.addOutputFilesBuilder().setPath("a/foo").setDigest(fooDigest); + result.addOutputDirectoriesBuilder().setPath("a/bar").setTreeDigest(barTreeDigest); + client.download(result.build(), execRoot, null); + + assertThat(DIGEST_UTIL.compute(execRoot.getRelative("a/foo"))).isEqualTo(fooDigest); + assertThat(DIGEST_UTIL.compute(execRoot.getRelative("a/bar/wobble/qux"))).isEqualTo(quxDigest); + assertThat(execRoot.getRelative("a/bar/wobble/qux").isExecutable()).isFalse(); + } + + @Test + public void testUploadBlob() throws Exception { + final Digest digest = DIGEST_UTIL.computeAsUtf8("abcdefg"); + + final ConcurrentMap<String, byte[]> map = new ConcurrentHashMap<>(); + final SimpleBlobStoreActionCache client = newClient(map); + + byte[] bytes = "abcdefg".getBytes(UTF_8); + assertThat(client.uploadBlob(bytes)).isEqualTo(digest); + assertThat(map.keySet()).containsExactly(digest.getHash()); + assertThat(map.entrySet()).hasSize(1); + assertThat(map.get(digest.getHash())).isEqualTo(bytes); + } + + @Test + public void testUploadDirectory() throws Exception { + final Digest fooDigest = + fakeFileCache.createScratchInput(ActionInputHelper.fromPath("a/foo"), "xyz"); + final Digest quxDigest = + fakeFileCache.createScratchInput(ActionInputHelper.fromPath("bar/qux"), "abc"); + final Digest barDigest = + fakeFileCache.createScratchInputDirectory( + ActionInputHelper.fromPath("bar"), + Tree.newBuilder() + .setRoot( + Directory.newBuilder() + .addFiles( + FileNode.newBuilder() + .setIsExecutable(true) + .setName("qux") + .setDigest(quxDigest) + .build()) + .build()) + .build()); + final Path fooFile = execRoot.getRelative("a/foo"); + final Path quxFile = execRoot.getRelative("bar/qux"); + quxFile.setExecutable(true); + final Path barDir = execRoot.getRelative("bar"); + + final ConcurrentMap<String, byte[]> map = new ConcurrentHashMap<>(); + final SimpleBlobStoreActionCache client = newClient(map); + + ActionResult.Builder result = ActionResult.newBuilder(); + client.upload(result, execRoot, ImmutableList.<Path>of(fooFile, barDir)); + ActionResult.Builder expectedResult = ActionResult.newBuilder(); + expectedResult.addOutputFilesBuilder().setPath("a/foo").setDigest(fooDigest); + expectedResult.addOutputDirectoriesBuilder().setPath("bar").setTreeDigest(barDigest); + assertThat(result.build()).isEqualTo(expectedResult.build()); + + assertThat(map.keySet()) + .containsExactly(fooDigest.getHash(), quxDigest.getHash(), barDigest.getHash()); + } + + @Test + public void testUploadDirectoryEmpty() throws Exception { + final Digest barDigest = + fakeFileCache.createScratchInputDirectory( + ActionInputHelper.fromPath("bar"), + Tree.newBuilder().setRoot(Directory.newBuilder().build()).build()); + final Path barDir = execRoot.getRelative("bar"); + + final ConcurrentMap<String, byte[]> map = new ConcurrentHashMap<>(); + final SimpleBlobStoreActionCache client = newClient(map); + + ActionResult.Builder result = ActionResult.newBuilder(); + client.upload(result, execRoot, ImmutableList.<Path>of(barDir)); + ActionResult.Builder expectedResult = ActionResult.newBuilder(); + expectedResult.addOutputDirectoriesBuilder().setPath("bar").setTreeDigest(barDigest); + assertThat(result.build()).isEqualTo(expectedResult.build()); + + assertThat(map.keySet()).containsExactly(barDigest.getHash()); + } + + @Test + public void testUploadDirectoryNested() throws Exception { + final Digest wobbleDigest = + fakeFileCache.createScratchInput(ActionInputHelper.fromPath("bar/test/wobble"), "xyz"); + final Digest quxDigest = + fakeFileCache.createScratchInput(ActionInputHelper.fromPath("bar/qux"), "abc"); + final Directory testDirMessage = + Directory.newBuilder() + .addFiles(FileNode.newBuilder().setName("wobble").setDigest(wobbleDigest).build()) + .build(); + final Digest testDigest = DIGEST_UTIL.compute(testDirMessage); + final Tree barTree = + Tree.newBuilder() + .setRoot( + Directory.newBuilder() + .addFiles( + FileNode.newBuilder() + .setIsExecutable(true) + .setName("qux") + .setDigest(quxDigest)) + .addDirectories( + DirectoryNode.newBuilder().setName("test").setDigest(testDigest))) + .addChildren(testDirMessage) + .build(); + final Digest barDigest = + fakeFileCache.createScratchInputDirectory(ActionInputHelper.fromPath("bar"), barTree); + + final ConcurrentMap<String, byte[]> map = new ConcurrentHashMap<>(); + final SimpleBlobStoreActionCache client = newClient(map); + + final Path quxFile = execRoot.getRelative("bar/qux"); + quxFile.setExecutable(true); + final Path barDir = execRoot.getRelative("bar"); + + ActionResult.Builder result = ActionResult.newBuilder(); + client.upload(result, execRoot, ImmutableList.<Path>of(barDir)); + ActionResult.Builder expectedResult = ActionResult.newBuilder(); + expectedResult.addOutputDirectoriesBuilder().setPath("bar").setTreeDigest(barDigest); + assertThat(result.build()).isEqualTo(expectedResult.build()); + + assertThat(map.keySet()) + .containsExactly(wobbleDigest.getHash(), quxDigest.getHash(), barDigest.getHash()); + } +}
diff --git a/src/test/java/com/google/devtools/build/lib/remote/TreeNodeRepositoryTest.java b/src/test/java/com/google/devtools/build/lib/remote/TreeNodeRepositoryTest.java index 9c0f43a..7ea8b0d 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/TreeNodeRepositoryTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/TreeNodeRepositoryTest.java
@@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.ActionInputHelper; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.Root; import com.google.devtools.build.lib.clock.BlazeClock; @@ -152,4 +153,54 @@ assertThat(root.getChildEntries()).isEmpty(); } + + @Test + public void testDirectoryInput() throws Exception { + Artifact foo = new Artifact(scratch.dir("/exec/root/a/foo"), rootDir); + scratch.file("/exec/root/a/foo/foo.h", "1"); + ActionInput fooH = ActionInputHelper.fromPath("/exec/root/a/foo/foo.h"); + scratch.file("/exec/root/a/foo/foo.cc", "2"); + ActionInput fooCc = ActionInputHelper.fromPath("/exec/root/a/foo/foo.cc"); + Artifact bar = new Artifact(scratch.file("/exec/root/a/bar.txt"), rootDir); + TreeNodeRepository repo = createTestTreeNodeRepository(); + + TreeNode root = repo.buildFromActionInputs(ImmutableList.<ActionInput>of(foo, bar)); + TreeNode aNode = root.getChildEntries().get(0).getChild(); + TreeNode fooNode = aNode.getChildEntries().get(1).getChild(); // foo > bar in sort order! + TreeNode barNode = aNode.getChildEntries().get(0).getChild(); + + TreeNode fooHNode = + fooNode.getChildEntries().get(1).getChild(); // foo.h > foo.cc in sort order! + TreeNode fooCcNode = fooNode.getChildEntries().get(0).getChild(); + + repo.computeMerkleDigests(root); + ImmutableCollection<Digest> digests = repo.getAllDigests(root); + Digest rootDigest = repo.getMerkleDigest(root); + Digest aDigest = repo.getMerkleDigest(aNode); + Digest fooDigest = repo.getMerkleDigest(fooNode); + Digest fooHDigest = repo.getMerkleDigest(fooHNode); + Digest fooCcDigest = repo.getMerkleDigest(fooCcNode); + Digest barDigest = repo.getMerkleDigest(barNode); + assertThat(digests) + .containsExactly(rootDigest, aDigest, barDigest, fooDigest, fooHDigest, fooCcDigest); + + ArrayList<Directory> directories = new ArrayList<>(); + ArrayList<ActionInput> actionInputs = new ArrayList<>(); + repo.getDataFromDigests(digests, actionInputs, directories); + assertThat(actionInputs).containsExactly(bar, fooH, fooCc); + assertThat(directories).hasSize(3); // root, root/a and root/a/foo + Directory rootDirectory = directories.get(0); + assertThat(rootDirectory.getDirectories(0).getName()).isEqualTo("a"); + assertThat(rootDirectory.getDirectories(0).getDigest()).isEqualTo(aDigest); + Directory aDirectory = directories.get(1); + assertThat(aDirectory.getFiles(0).getName()).isEqualTo("bar.txt"); + assertThat(aDirectory.getFiles(0).getDigest()).isEqualTo(barDigest); + assertThat(aDirectory.getDirectories(0).getName()).isEqualTo("foo"); + assertThat(aDirectory.getDirectories(0).getDigest()).isEqualTo(fooDigest); + Directory fooDirectory = directories.get(2); + assertThat(fooDirectory.getFiles(0).getName()).isEqualTo("foo.cc"); + assertThat(fooDirectory.getFiles(0).getDigest()).isEqualTo(fooCcDigest); + assertThat(fooDirectory.getFiles(1).getName()).isEqualTo("foo.h"); + assertThat(fooDirectory.getFiles(1).getDigest()).isEqualTo(fooHDigest); + } }
diff --git a/src/test/shell/bazel/remote_execution_http_test.sh b/src/test/shell/bazel/remote_execution_http_test.sh index 8605bd3..5b978bd 100755 --- a/src/test/shell/bazel/remote_execution_http_test.sh +++ b/src/test/shell/bazel/remote_execution_http_test.sh
@@ -127,4 +127,82 @@ fi } +function set_directory_artifact_testfixtures() { + mkdir -p a + cat > a/BUILD <<'EOF' +package(default_visibility = ["//visibility:public"]) +genrule( + name = "test", + srcs = ["dir"], + outs = ["qux"], + cmd = 'mkdir $@ && paste -d"\n" $(location dir)/foo.txt $(location dir)/sub2/symlinked.txt > $@/out.txt', +) +EOF + + # Need this to avoid any leftover + rm -rf a/dir + + mkdir -p a/dir + cat > a/dir/foo.txt <<EOF +Hello, world +EOF + mkdir -p a/dir/sub1 + cat > a/dir/sub1/bar.txt <<EOF +Shuffle, duffle, muzzle, muff +EOF + mkdir -p a/dir/sub2 + ln -s ../sub1/bar.txt a/dir/sub2/symlinked.txt + + cat > a/test_expected <<EOF +Hello, world +Shuffle, duffle, muzzle, muff +EOF +} + +function test_directory_artifact_local() { + set_directory_artifact_testfixtures + + bazel build //a:test >& $TEST_log \ + || fail "Failed to build //a:test without remote execution" + diff bazel-genfiles/a/qux/out.txt a/test_expected \ + || fail "Local execution generated different result" +} + +function test_directory_artifact() { + set_directory_artifact_testfixtures + + bazel build \ + --spawn_strategy=remote \ + --remote_executor=localhost:${worker_port} \ + --remote_cache=localhost:${worker_port} \ + //a:test >& $TEST_log \ + || fail "Failed to build //a:test with remote execution" + diff bazel-genfiles/a/qux/out.txt a/test_expected \ + || fail "Remote execution generated different result" +} + +function test_directory_artifact_grpc_cache() { + set_directory_artifact_testfixtures + + bazel build \ + --spawn_strategy=remote \ + --remote_cache=localhost:${worker_port} \ + //a:test >& $TEST_log \ + || fail "Failed to build //a:test with remote gRPC cache" + diff bazel-genfiles/a/qux/out.txt a/test_expected \ + || fail "Remote cache generated different result" +} + +function test_directory_artifact_rest_cache() { + set_directory_artifact_testfixtures + + bazel build \ + --spawn_strategy=remote \ + --remote_rest_cache=http://localhost:${hazelcast_port}/hazelcast/rest/maps \ + //a:test >& $TEST_log \ + || fail "Failed to build //a:test with remote REST cache" + diff bazel-genfiles/a/qux/out.txt a/test_expected \ + || fail "Remote cache generated different result" +} + run_suite "Remote execution and remote cache tests"
diff --git a/src/test/shell/bazel/remote_execution_test.sh b/src/test/shell/bazel/remote_execution_test.sh index 5eadb1d..87ad193 100755 --- a/src/test/shell/bazel/remote_execution_test.sh +++ b/src/test/shell/bazel/remote_execution_test.sh
@@ -495,7 +495,7 @@ #This will never run, because the remote side is not reachable. EOF chmod +x a/test.sh - bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 test \ + bazel test \ --spawn_strategy=remote \ --remote_executor=bazel.does.not.exist:1234 \ --noexperimental_remote_retry \
diff --git a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/ExecutionServer.java b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/ExecutionServer.java index 22761a9..f7575bd 100644 --- a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/ExecutionServer.java +++ b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/ExecutionServer.java
@@ -191,7 +191,14 @@ FileSystemUtils.createDirectoryAndParents(file.getParentDirectory()); outputs.add(file); } - // TODO(olaola): support output directories. + for (String output : action.getOutputDirectoriesList()) { + Path file = execRoot.getRelative(output); + if (file.exists()) { + throw new FileAlreadyExistsException("Output directory/file already exists: " + file); + } + FileSystemUtils.createDirectoryAndParents(file.getParentDirectory()); + outputs.add(file); + } // TODO(ulfjack): This is basically a copy of LocalSpawnRunner. Ideally, we'd use that // implementation instead of copying it.
diff --git a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/RemoteWorker.java b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/RemoteWorker.java index 8463bb00..c030397 100644 --- a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/RemoteWorker.java +++ b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/RemoteWorker.java
@@ -268,7 +268,7 @@ } else { blobStore = new ConcurrentMapBlobStore(new ConcurrentHashMap<String, byte[]>()); } - + DigestUtil digestUtil = new DigestUtil(fs.getDigestFunction()); RemoteWorker worker = new RemoteWorker(