remote: support building merkle trees from Paths
The MerkleTree builder is coupled to ActionInput, a type of the
execution phase. This change adds support for building merkle
trees from Path objects. This is needed for supporting file uploads
in repository_ctx.execute which runs before the execution phase.
Closes #11014.
PiperOrigin-RevId: 303956403
diff --git a/src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeBuilder.java b/src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeBuilder.java
index da6676c..54bbeef 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeBuilder.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeBuilder.java
@@ -37,6 +37,21 @@
/** Builder for directory trees. */
class DirectoryTreeBuilder {
+ private interface FileNodeVisitor<T> {
+
+ /**
+ * Visits an {@code input} and adds {@link FileNode}s to {@code currDir}.
+ *
+ * <p>This method mutates its parameter {@code currDir}.
+ *
+ * @param input the file or directory to add to {@code currDir}.
+ * @param path the path of {@code input} in the merkle tree.
+ * @param currDir the directory node representing {@code path} in the merkle tree.
+ * @return Returns the number of {@link FileNode}s added to {@code currDir}.
+ */
+ int visit(T input, PathFragment path, DirectoryNode currDir) throws IOException;
+ }
+
static DirectoryTree fromActionInputs(
SortedMap<PathFragment, ActionInput> inputs,
MetadataProvider metadataProvider,
@@ -44,28 +59,135 @@
DigestUtil digestUtil)
throws IOException {
Map<PathFragment, DirectoryNode> tree = new HashMap<>();
- int numFiles = fromActionInputs(inputs, metadataProvider, execRoot, digestUtil, tree);
+ int numFiles = buildFromActionInputs(inputs, metadataProvider, execRoot, digestUtil, tree);
return new DirectoryTree(tree, numFiles);
}
- private static int fromActionInputs(
+ /**
+ * Creates a tree of files and directories from a list of files.
+ *
+ * <p>This method retrieves file metadata from the filesystem. It does not use Bazel's caches.
+ * Thus, don't use this method during the execution phase. Use {@link #fromActionInputs} instead.
+ *
+ * @param inputFiles map of paths to files. The key determines the path at which the file should
+ * be mounted in the tree.
+ */
+ static DirectoryTree fromPaths(SortedMap<PathFragment, Path> inputFiles, DigestUtil digestUtil)
+ throws IOException {
+ Map<PathFragment, DirectoryNode> tree = new HashMap<>();
+ int numFiles = buildFromPaths(inputFiles, digestUtil, tree);
+ return new DirectoryTree(tree, numFiles);
+ }
+
+ /**
+ * Adds the files in {@code inputs} as nodes to {@code tree}.
+ *
+ * <p>This method mutates {@code tree}.
+ *
+ * @param inputs map of paths to files. The key determines the path at which the file should be
+ * mounted in the tree.
+ * @return the number of file nodes added to {@code tree}.
+ */
+ private static int buildFromPaths(
+ SortedMap<PathFragment, Path> inputs,
+ DigestUtil digestUtil,
+ Map<PathFragment, DirectoryNode> tree)
+ throws IOException {
+ return build(
+ inputs,
+ tree,
+ (input, path, currDir) -> {
+ if (!input.isFile(Symlinks.NOFOLLOW)) {
+ throw new IOException(String.format("Input '%s' is not a file.", input));
+ }
+ Digest d = digestUtil.compute(input);
+ currDir.addChild(new FileNode(path.getBaseName(), input, d));
+ return 1;
+ });
+ }
+
+ /**
+ * Adds the files in {@code inputs} as nodes to {@code tree}.
+ *
+ * <p>This method mutates {@code tree}.
+ *
+ * @return the number of file nodes added to {@code tree}.
+ */
+ private static int buildFromActionInputs(
SortedMap<PathFragment, ActionInput> inputs,
MetadataProvider metadataProvider,
Path execRoot,
DigestUtil digestUtil,
Map<PathFragment, DirectoryNode> tree)
throws IOException {
+ return build(
+ inputs,
+ tree,
+ (input, path, currDir) -> {
+ if (input instanceof VirtualActionInput) {
+ VirtualActionInput virtualActionInput = (VirtualActionInput) input;
+ Digest d = digestUtil.compute(virtualActionInput);
+ currDir.addChild(new FileNode(path.getBaseName(), virtualActionInput.getBytes(), d));
+ return 1;
+ }
+
+ FileArtifactValue metadata =
+ Preconditions.checkNotNull(
+ metadataProvider.getMetadata(input),
+ "missing metadata for '%s'",
+ input.getExecPathString());
+ switch (metadata.getType()) {
+ case REGULAR_FILE:
+ Digest d = DigestUtil.buildDigest(metadata.getDigest(), metadata.getSize());
+ currDir.addChild(
+ new FileNode(
+ path.getBaseName(), ActionInputHelper.toInputPath(input, execRoot), d));
+ return 1;
+
+ case DIRECTORY:
+ SortedMap<PathFragment, ActionInput> directoryInputs =
+ explodeDirectory(path, execRoot);
+ return buildFromActionInputs(
+ directoryInputs, metadataProvider, execRoot, digestUtil, tree);
+
+ case SYMLINK:
+ throw new IllegalStateException(
+ String.format(
+ "Encountered symlink input '%s', but all"
+ + " symlinks should have been resolved by SkyFrame. This is a bug.",
+ path));
+
+ case SPECIAL_FILE:
+ throw new IOException(
+ String.format(
+ "The '%s' is a special input which is not supported"
+ + " by remote caching and execution.",
+ path));
+
+ case NONEXISTENT:
+ throw new IOException(String.format("The file type of '%s' is not supported.", path));
+ }
+
+ return 0;
+ });
+ }
+
+ private static <T> int build(
+ SortedMap<PathFragment, T> inputs,
+ Map<PathFragment, DirectoryNode> tree,
+ FileNodeVisitor<T> fileNodeVisitor)
+ throws IOException {
if (inputs.isEmpty()) {
return 0;
}
PathFragment dirname = null;
DirectoryNode dir = null;
- int numFiles = inputs.size();
- for (Map.Entry<PathFragment, ActionInput> e : inputs.entrySet()) {
+ int numFiles = 0;
+ for (Map.Entry<PathFragment, T> e : inputs.entrySet()) {
// Path relative to the exec root
PathFragment path = e.getKey();
- ActionInput input = e.getValue();
+ T input = e.getValue();
if (dirname == null || !path.getParentDirectory().equals(dirname)) {
dirname = path.getParentDirectory();
dir = tree.get(dirname);
@@ -76,49 +198,9 @@
}
}
- if (input instanceof VirtualActionInput) {
- VirtualActionInput virtualActionInput = (VirtualActionInput) input;
- Digest d = digestUtil.compute(virtualActionInput);
- dir.addChild(new FileNode(path.getBaseName(), virtualActionInput.getBytes(), d));
- continue;
- }
-
- FileArtifactValue metadata =
- Preconditions.checkNotNull(
- metadataProvider.getMetadata(input),
- "missing metadata for '%s'",
- input.getExecPathString());
- switch (metadata.getType()) {
- case REGULAR_FILE:
- Digest d = DigestUtil.buildDigest(metadata.getDigest(), metadata.getSize());
- dir.addChild(
- new FileNode(path.getBaseName(), ActionInputHelper.toInputPath(input, execRoot), d));
- break;
-
- case DIRECTORY:
- SortedMap<PathFragment, ActionInput> directoryInputs = explodeDirectory(path, execRoot);
- numFiles +=
- fromActionInputs(directoryInputs, metadataProvider, execRoot, digestUtil, tree);
- break;
-
- case SYMLINK:
- throw new IllegalStateException(
- String.format(
- "Encountered symlink input '%s', but all"
- + " symlinks should have been resolved by SkyFrame. This is a bug.",
- path));
-
- case SPECIAL_FILE:
- throw new IOException(
- String.format(
- "The '%s' is a special input which is not supported"
- + " by remote caching and execution.",
- path));
-
- case NONEXISTENT:
- throw new IOException(String.format("The file type of '%s' is not supported.", path));
- }
+ numFiles += fileNodeVisitor.visit(input, path, dir);
}
+
return numFiles;
}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java b/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java
index 918e988..7115601 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java
@@ -118,13 +118,28 @@
Path execRoot,
DigestUtil digestUtil)
throws IOException {
- try (SilentCloseable c = Profiler.instance().profile("MerkleTree.build")) {
+ try (SilentCloseable c = Profiler.instance().profile("MerkleTree.build(ActionInput)")) {
DirectoryTree tree =
DirectoryTreeBuilder.fromActionInputs(inputs, metadataProvider, execRoot, digestUtil);
return build(tree, digestUtil);
}
}
+ /**
+ * Constructs a merkle tree from a lexicographically sorted map of files.
+ *
+ * @param inputFiles a map of path to files. The map is required to be sorted lexicographically by
+ * paths.
+ * @param digestUtil a hashing utility
+ */
+ public static MerkleTree build(SortedMap<PathFragment, Path> inputFiles, DigestUtil digestUtil)
+ throws IOException {
+ try (SilentCloseable c = Profiler.instance().profile("MerkleTree.build(Path)")) {
+ DirectoryTree tree = DirectoryTreeBuilder.fromPaths(inputFiles, digestUtil);
+ return build(tree, digestUtil);
+ }
+ }
+
private static MerkleTree build(DirectoryTree tree, DigestUtil digestUtil) {
Preconditions.checkNotNull(tree);
if (tree.isEmpty()) {
diff --git a/src/test/java/com/google/devtools/build/lib/remote/merkletree/ActionInputDirectoryTreeTest.java b/src/test/java/com/google/devtools/build/lib/remote/merkletree/ActionInputDirectoryTreeTest.java
new file mode 100644
index 0000000..79408f4
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/remote/merkletree/ActionInputDirectoryTreeTest.java
@@ -0,0 +1,156 @@
+// Copyright 2020 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.remote.merkletree;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.ActionInputHelper;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.FileArtifactValue;
+import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
+import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
+import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.FileNode;
+import com.google.devtools.build.lib.remote.util.StaticMetadataProvider;
+import com.google.devtools.build.lib.remote.util.StringActionInput;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import org.junit.Test;
+
+/** Tests for {@link DirectoryTreeBuilder#buildFromActionInputs}. */
+public class ActionInputDirectoryTreeTest extends DirectoryTreeTest {
+
+ @Override
+ protected DirectoryTree build(Path... paths) throws IOException {
+ SortedMap<PathFragment, ActionInput> inputFiles = new TreeMap<>();
+ Map<ActionInput, FileArtifactValue> metadata = new HashMap<>();
+
+ for (Path path : paths) {
+ PathFragment relPath = path.relativeTo(execRoot);
+ Artifact a = ActionsTestUtil.createArtifact(artifactRoot, path);
+
+ inputFiles.put(relPath, a);
+ metadata.put(a, FileArtifactValue.createForTesting(a));
+ }
+
+ return DirectoryTreeBuilder.fromActionInputs(
+ inputFiles, new StaticMetadataProvider(metadata), execRoot, digestUtil);
+ }
+
+ @Test
+ public void virtualActionInputShouldWork() throws Exception {
+ SortedMap<PathFragment, ActionInput> sortedInputs = new TreeMap<>();
+ Map<ActionInput, FileArtifactValue> metadata = new HashMap<>();
+
+ Artifact foo = addFile("srcs/foo.cc", "foo", sortedInputs, metadata);
+ VirtualActionInput bar = addVirtualFile("srcs/bar.cc", "bar", sortedInputs);
+
+ DirectoryTree tree =
+ DirectoryTreeBuilder.fromActionInputs(
+ sortedInputs, new StaticMetadataProvider(metadata), execRoot, digestUtil);
+ assertLexicographicalOrder(tree);
+
+ assertThat(directoriesAtDepth(0, tree)).containsExactly("srcs");
+ assertThat(directoriesAtDepth(1, tree)).isEmpty();
+
+ FileNode expectedFooNode =
+ new FileNode("foo.cc", foo.getPath(), digestUtil.computeAsUtf8("foo"));
+ FileNode expectedBarNode =
+ new FileNode("bar.cc", bar.getBytes(), digestUtil.computeAsUtf8("bar"));
+ assertThat(fileNodesAtDepth(tree, 0)).isEmpty();
+ assertThat(fileNodesAtDepth(tree, 1)).containsExactly(expectedFooNode, expectedBarNode);
+ }
+
+ @Test
+ public void directoryInputShouldBeExpanded() throws Exception {
+ // Test that directory inputs are fully expanded and added to the input tree.
+ // Note that this test is not about tree artifacts, but normal artifacts that point to
+ // a directory on disk.
+
+ SortedMap<PathFragment, ActionInput> sortedInputs = new TreeMap<>();
+ Map<ActionInput, FileArtifactValue> metadata = new HashMap<>();
+
+ Artifact foo = addFile("srcs/foo.cc", "foo", sortedInputs, metadata);
+
+ Path dirPath = execRoot.getRelative("srcs/dir");
+ dirPath.createDirectoryAndParents();
+
+ Path barPath = dirPath.getRelative("bar.cc");
+ FileSystemUtils.writeContentAsLatin1(barPath, "bar");
+ ActionInput bar = ActionInputHelper.fromPath(barPath.relativeTo(execRoot));
+ metadata.put(bar, FileArtifactValue.createForTesting(barPath));
+
+ dirPath.getRelative("fizz").createDirectoryAndParents();
+ Path buzzPath = dirPath.getRelative("fizz/buzz.cc");
+ FileSystemUtils.writeContentAsLatin1(dirPath.getRelative("fizz/buzz.cc"), "buzz");
+ ActionInput buzz = ActionInputHelper.fromPath(buzzPath.relativeTo(execRoot));
+ metadata.put(buzz, FileArtifactValue.createForTesting(buzzPath));
+
+ Artifact dir = ActionsTestUtil.createArtifact(artifactRoot, dirPath);
+ sortedInputs.put(dirPath.relativeTo(execRoot), dir);
+ metadata.put(dir, FileArtifactValue.createForTesting(dirPath));
+
+ DirectoryTree tree =
+ DirectoryTreeBuilder.fromActionInputs(
+ sortedInputs, new StaticMetadataProvider(metadata), execRoot, digestUtil);
+ assertLexicographicalOrder(tree);
+
+ assertThat(directoriesAtDepth(0, tree)).containsExactly("srcs");
+ assertThat(directoriesAtDepth(1, tree)).containsExactly("dir");
+ assertThat(directoriesAtDepth(2, tree)).containsExactly("fizz");
+ assertThat(directoriesAtDepth(3, tree)).isEmpty();
+
+ FileNode expectedFooNode =
+ new FileNode("foo.cc", foo.getPath(), digestUtil.computeAsUtf8("foo"));
+ FileNode expectedBarNode =
+ new FileNode(
+ "bar.cc", execRoot.getRelative(bar.getExecPath()), digestUtil.computeAsUtf8("bar"));
+ FileNode expectedBuzzNode =
+ new FileNode(
+ "buzz.cc", execRoot.getRelative(buzz.getExecPath()), digestUtil.computeAsUtf8("buzz"));
+ assertThat(fileNodesAtDepth(tree, 0)).isEmpty();
+ assertThat(fileNodesAtDepth(tree, 1)).containsExactly(expectedFooNode);
+ assertThat(fileNodesAtDepth(tree, 2)).containsExactly(expectedBarNode);
+ assertThat(fileNodesAtDepth(tree, 3)).containsExactly(expectedBuzzNode);
+ }
+
+ private static VirtualActionInput addVirtualFile(
+ String path, String content, SortedMap<PathFragment, ActionInput> sortedInputs) {
+ VirtualActionInput input = new StringActionInput(content, PathFragment.create(path));
+ sortedInputs.put(PathFragment.create(path), input);
+ return input;
+ }
+
+ private Artifact addFile(
+ String path,
+ String content,
+ SortedMap<PathFragment, ActionInput> sortedInputs,
+ Map<ActionInput, FileArtifactValue> metadata)
+ throws IOException {
+ Path p = execRoot.getRelative(path);
+ p.getParentDirectory().createDirectoryAndParents();
+ FileSystemUtils.writeContentAsLatin1(p, content);
+ Artifact a = ActionsTestUtil.createArtifact(artifactRoot, p);
+
+ sortedInputs.put(PathFragment.create(path), a);
+ metadata.put(a, FileArtifactValue.createForTesting(a));
+ return a;
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeTest.java b/src/test/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeTest.java
index bd478f4..33b1c36 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeTest.java
@@ -15,20 +15,12 @@
import static com.google.common.truth.Truth.assertThat;
-import com.google.devtools.build.lib.actions.ActionInput;
-import com.google.devtools.build.lib.actions.ActionInputHelper;
-import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.ArtifactRoot;
-import com.google.devtools.build.lib.actions.FileArtifactValue;
-import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
-import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
import com.google.devtools.build.lib.clock.JavaClock;
import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.DirectoryNode;
import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.FileNode;
import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.Node;
import com.google.devtools.build.lib.remote.util.DigestUtil;
-import com.google.devtools.build.lib.remote.util.StaticMetadataProvider;
-import com.google.devtools.build.lib.remote.util.StringActionInput;
import com.google.devtools.build.lib.vfs.DigestHashFunction;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
@@ -37,12 +29,7 @@
import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
import java.util.stream.Collectors;
import org.junit.Before;
import org.junit.Test;
@@ -51,11 +38,11 @@
/** Unit tests for {@link DirectoryTree}. */
@RunWith(JUnit4.class)
-public class DirectoryTreeTest {
+public abstract class DirectoryTreeTest {
- private Path execRoot;
- private ArtifactRoot artifactRoot;
- private DigestUtil digestUtil;
+ protected Path execRoot;
+ protected ArtifactRoot artifactRoot;
+ protected DigestUtil digestUtil;
@Before
public void setup() {
@@ -65,123 +52,36 @@
digestUtil = new DigestUtil(fs.getDigestFunction());
}
+ protected abstract DirectoryTree build(Path... paths) throws IOException;
+
@Test
public void emptyTreeShouldWork() throws Exception {
- DirectoryTree tree =
- DirectoryTreeBuilder.fromActionInputs(
- new TreeMap<>(),
- new StaticMetadataProvider(Collections.emptyMap()),
- execRoot,
- digestUtil);
+ DirectoryTree tree = build();
assertThat(directoryNodesAtDepth(tree, 0)).isEmpty();
assertThat(fileNodesAtDepth(tree, 0)).isEmpty();
}
@Test
public void buildingATreeOfFilesShouldWork() throws Exception {
- SortedMap<PathFragment, ActionInput> sortedInputs = new TreeMap<>();
- Map<ActionInput, FileArtifactValue> metadata = new HashMap<>();
+ Path foo = createFile("srcs/foo.cc", "foo");
+ Path bar = createFile("srcs/bar.cc", "bar");
+ Path buzz = createFile("srcs/fizz/buzz.cc", "buzz");
- Artifact foo = addFile("srcs/foo.cc", "foo", sortedInputs, metadata);
- Artifact bar = addFile("srcs/bar.cc", "bar", sortedInputs, metadata);
- Artifact buzz = addFile("srcs/fizz/buzz.cc", "buzz", sortedInputs, metadata);
-
- DirectoryTree tree =
- DirectoryTreeBuilder.fromActionInputs(
- sortedInputs, new StaticMetadataProvider(metadata), execRoot, digestUtil);
+ DirectoryTree tree = build(foo, bar, buzz);
assertLexicographicalOrder(tree);
assertThat(directoriesAtDepth(0, tree)).containsExactly("srcs");
assertThat(directoriesAtDepth(1, tree)).containsExactly("fizz");
assertThat(directoriesAtDepth(2, tree)).isEmpty();
- FileNode expectedFooNode =
- new FileNode("foo.cc", foo.getPath(), digestUtil.computeAsUtf8("foo"));
- FileNode expectedBarNode =
- new FileNode("bar.cc", bar.getPath(), digestUtil.computeAsUtf8("bar"));
- FileNode expectedBuzzNode =
- new FileNode("buzz.cc", buzz.getPath(), digestUtil.computeAsUtf8("buzz"));
+ FileNode expectedFooNode = new FileNode("foo.cc", foo, digestUtil.computeAsUtf8("foo"));
+ FileNode expectedBarNode = new FileNode("bar.cc", bar, digestUtil.computeAsUtf8("bar"));
+ FileNode expectedBuzzNode = new FileNode("buzz.cc", buzz, digestUtil.computeAsUtf8("buzz"));
assertThat(fileNodesAtDepth(tree, 0)).isEmpty();
assertThat(fileNodesAtDepth(tree, 1)).containsExactly(expectedFooNode, expectedBarNode);
assertThat(fileNodesAtDepth(tree, 2)).containsExactly(expectedBuzzNode);
}
- @Test
- public void virtualActionInputShouldWork() throws Exception {
- SortedMap<PathFragment, ActionInput> sortedInputs = new TreeMap<>();
- Map<ActionInput, FileArtifactValue> metadata = new HashMap<>();
-
- Artifact foo = addFile("srcs/foo.cc", "foo", sortedInputs, metadata);
- VirtualActionInput bar = addVirtualFile("srcs/bar.cc", "bar", sortedInputs);
-
- DirectoryTree tree =
- DirectoryTreeBuilder.fromActionInputs(
- sortedInputs, new StaticMetadataProvider(metadata), execRoot, digestUtil);
- assertLexicographicalOrder(tree);
-
- assertThat(directoriesAtDepth(0, tree)).containsExactly("srcs");
- assertThat(directoriesAtDepth(1, tree)).isEmpty();
-
- FileNode expectedFooNode =
- new FileNode("foo.cc", foo.getPath(), digestUtil.computeAsUtf8("foo"));
- FileNode expectedBarNode =
- new FileNode("bar.cc", bar.getBytes(), digestUtil.computeAsUtf8("bar"));
- assertThat(fileNodesAtDepth(tree, 0)).isEmpty();
- assertThat(fileNodesAtDepth(tree, 1)).containsExactly(expectedFooNode, expectedBarNode);
- }
-
- @Test
- public void directoryInputShouldBeExpanded() throws Exception {
- // Test that directory inputs are fully expanded and added to the input tree.
- // Note that this test is not about tree artifacts, but normal artifacts that point to
- // a directory on disk.
-
- SortedMap<PathFragment, ActionInput> sortedInputs = new TreeMap<>();
- Map<ActionInput, FileArtifactValue> metadata = new HashMap<>();
-
- Artifact foo = addFile("srcs/foo.cc", "foo", sortedInputs, metadata);
-
- Path dirPath = execRoot.getRelative("srcs/dir");
- dirPath.createDirectoryAndParents();
-
- Path barPath = dirPath.getRelative("bar.cc");
- FileSystemUtils.writeContentAsLatin1(barPath, "bar");
- ActionInput bar = ActionInputHelper.fromPath(barPath.relativeTo(execRoot));
- metadata.put(bar, FileArtifactValue.createForTesting(barPath));
-
- dirPath.getRelative("fizz").createDirectoryAndParents();
- Path buzzPath = dirPath.getRelative("fizz/buzz.cc");
- FileSystemUtils.writeContentAsLatin1(dirPath.getRelative("fizz/buzz.cc"), "buzz");
- ActionInput buzz = ActionInputHelper.fromPath(buzzPath.relativeTo(execRoot));
- metadata.put(buzz, FileArtifactValue.createForTesting(buzzPath));
-
- Artifact dir = ActionsTestUtil.createArtifact(artifactRoot, dirPath);
- sortedInputs.put(dirPath.relativeTo(execRoot), dir);
- metadata.put(dir, FileArtifactValue.createForTesting(dirPath));
-
- DirectoryTree tree =
- DirectoryTreeBuilder.fromActionInputs(
- sortedInputs, new StaticMetadataProvider(metadata), execRoot, digestUtil);
- assertLexicographicalOrder(tree);
-
- assertThat(directoriesAtDepth(0, tree)).containsExactly("srcs");
- assertThat(directoriesAtDepth(1, tree)).containsExactly("dir");
- assertThat(directoriesAtDepth(2, tree)).containsExactly("fizz");
- assertThat(directoriesAtDepth(3, tree)).isEmpty();
-
- FileNode expectedFooNode =
- new FileNode("foo.cc", foo.getPath(), digestUtil.computeAsUtf8("foo"));
- FileNode expectedBarNode =
- new FileNode(
- "bar.cc", execRoot.getRelative(bar.getExecPath()), digestUtil.computeAsUtf8("bar"));
- FileNode expectedBuzzNode =
- new FileNode(
- "buzz.cc", execRoot.getRelative(buzz.getExecPath()), digestUtil.computeAsUtf8("buzz"));
- assertThat(fileNodesAtDepth(tree, 0)).isEmpty();
- assertThat(fileNodesAtDepth(tree, 1)).containsExactly(expectedFooNode);
- assertThat(fileNodesAtDepth(tree, 2)).containsExactly(expectedBarNode);
- assertThat(fileNodesAtDepth(tree, 3)).containsExactly(expectedBuzzNode);
- }
@Test
public void testLexicographicalOrder() throws Exception {
@@ -196,42 +96,22 @@
//
// However, the tree node [system-root, system] is not (note the missing / suffix).
- SortedMap<PathFragment, ActionInput> sortedInputs = new TreeMap<>();
- Map<ActionInput, FileArtifactValue> metadata = new HashMap<>();
+ Path file1 = createFile("srcs/system/foo.txt", "foo");
+ Path file2 = createFile("srcs/system-root/bar.txt", "bar");
- addFile("srcs/system/foo.txt", "foo", sortedInputs, metadata);
- addFile("srcs/system-root/bar.txt", "bar", sortedInputs, metadata);
+ DirectoryTree tree = build(file1, file2);
- DirectoryTree tree =
- DirectoryTreeBuilder.fromActionInputs(
- sortedInputs, new StaticMetadataProvider(metadata), execRoot, digestUtil);
assertLexicographicalOrder(tree);
}
- private Artifact addFile(
- String path,
- String content,
- SortedMap<PathFragment, ActionInput> sortedInputs,
- Map<ActionInput, FileArtifactValue> metadata)
- throws IOException {
+ protected Path createFile(String path, String content) throws IOException {
Path p = execRoot.getRelative(path);
p.getParentDirectory().createDirectoryAndParents();
FileSystemUtils.writeContentAsLatin1(p, content);
- Artifact a = ActionsTestUtil.createArtifact(artifactRoot, p);
-
- sortedInputs.put(PathFragment.create(path), a);
- metadata.put(a, FileArtifactValue.createForTesting(a));
- return a;
+ return p;
}
- private VirtualActionInput addVirtualFile(
- String path, String content, SortedMap<PathFragment, ActionInput> sortedInputs) {
- VirtualActionInput input = new StringActionInput(content, PathFragment.create(path));
- sortedInputs.put(PathFragment.create(path), input);
- return input;
- }
-
- private static void assertLexicographicalOrder(DirectoryTree tree) {
+ static void assertLexicographicalOrder(DirectoryTree tree) {
// Assert the lexicographical order as defined by the remote execution protocol
tree.visit(
(PathFragment dirname, List<FileNode> files, List<DirectoryNode> dirs) -> {
@@ -240,7 +120,7 @@
});
}
- private static List<String> directoriesAtDepth(int depth, DirectoryTree tree) {
+ static List<String> directoriesAtDepth(int depth, DirectoryTree tree) {
return asPathSegments(directoryNodesAtDepth(tree, depth));
}
@@ -260,7 +140,7 @@
return directoryNodes;
}
- private static List<FileNode> fileNodesAtDepth(DirectoryTree tree, int depth) {
+ static List<FileNode> fileNodesAtDepth(DirectoryTree tree, int depth) {
List<FileNode> fileNodes = new ArrayList<>();
tree.visit(
(PathFragment dirname, List<FileNode> files, List<DirectoryNode> dirs) -> {
diff --git a/src/test/java/com/google/devtools/build/lib/remote/merkletree/PathDirectoryTreeTest.java b/src/test/java/com/google/devtools/build/lib/remote/merkletree/PathDirectoryTreeTest.java
new file mode 100644
index 0000000..38dd3c6
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/remote/merkletree/PathDirectoryTreeTest.java
@@ -0,0 +1,33 @@
+// Copyright 2020 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.remote.merkletree;
+
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import java.io.IOException;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+
+/** Tests for {@link DirectoryTreeBuilder#fromPaths}. */
+public class PathDirectoryTreeTest extends DirectoryTreeTest {
+
+ @Override
+ protected DirectoryTree build(Path... paths) throws IOException {
+ NavigableMap<PathFragment, Path> inputFiles = new TreeMap<>();
+ for (Path path : paths) {
+ inputFiles.put(path.relativeTo(execRoot), path);
+ }
+ return DirectoryTreeBuilder.fromPaths(inputFiles, digestUtil);
+ }
+}
