[6.0.0] Emit Tree objects in topological order (#16904)
* Emit Tree objects in topological order
remote-apis PR 230 added a way where producers of Tree messages can indicate that the directories contained within are stored in topological order. The advantage of using such an ordering is that it permits instantiation of such objects onto a local file system in a streaming fashion. The same holds for lookups of individual paths.
Even though Bazel currently does not gain from this, this change at least modifies Bazel's REv2 client to emit topologically sorted trees. This makes it possible for tools such as Buildbarn's bb-browser to process them more efficiently.
More details:
- https://github.com/bazelbuild/remote-apis/pull/229
- https://github.com/bazelbuild/remote-apis/pull/230
Closes #16463.
PiperOrigin-RevId: 487196375
Change-Id: Iafcfd617fc101fec7bfa943552113ce57ab8041b
* Emit Tree objects in topological order
remote-apis PR 230 added a way where producers of Tree messages can
indicate that the directories contained within are stored in topological
order. The advantage of using such an ordering is that it permits
instantiation of such objects onto a local file system in a streaming
fashion. The same holds for lookups of individual paths.
Even though Bazel currently does not gain from this, this change at
least modifies Bazel's REv2 client to emit topologically sorted trees.
This makes it possible for tools such as Buildbarn's bb-browser to
process them more efficiently.
More details:
- https://github.com/bazelbuild/remote-apis/pull/229
- https://github.com/bazelbuild/remote-apis/pull/230
Partial commit for third_party/*, see #16463.
Signed-off-by: Sunil Gowroji <sgowroji@google.com>
Signed-off-by: Sunil Gowroji <sgowroji@google.com>
Co-authored-by: Ed Schouten <eschouten@apple.com>
diff --git a/src/main/java/com/google/devtools/build/lib/remote/UploadManifest.java b/src/main/java/com/google/devtools/build/lib/remote/UploadManifest.java
index 742c74f..07c02ac 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/UploadManifest.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/UploadManifest.java
@@ -31,6 +31,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
import com.google.devtools.build.lib.actions.ActionExecutionMetadata;
import com.google.devtools.build.lib.actions.ActionUploadFinishedEvent;
import com.google.devtools.build.lib.actions.ActionUploadStartedEvent;
@@ -54,10 +55,12 @@
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Symlinks;
import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.Timestamp;
import io.reactivex.rxjava3.core.Completable;
import io.reactivex.rxjava3.core.Flowable;
import io.reactivex.rxjava3.core.Single;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.time.Duration;
import java.time.Instant;
@@ -65,9 +68,11 @@
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
@@ -303,25 +308,43 @@
digestToFile.put(digest, file);
}
- private void addDirectory(Path dir) throws ExecException, IOException {
- Tree.Builder tree = Tree.newBuilder();
- Directory root = computeDirectory(dir, tree);
- tree.setRoot(root);
+ // Field numbers of the 'root' and 'directory' fields in the Tree message.
+ private static final int TREE_ROOT_FIELD_NUMBER =
+ Tree.getDescriptor().findFieldByName("root").getNumber();
+ private static final int TREE_CHILDREN_FIELD_NUMBER =
+ Tree.getDescriptor().findFieldByName("children").getNumber();
- ByteString data = tree.build().toByteString();
+ private void addDirectory(Path dir) throws ExecException, IOException {
+ Set<ByteString> directories = new LinkedHashSet<>();
+ var ignored = computeDirectory(dir, directories);
+
+ // Convert individual Directory messages to a Tree message. As we want the
+ // records to be topologically sorted (parents before children), we iterate
+ // over the directories in reverse insertion order.
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ CodedOutputStream codedOutputStream = CodedOutputStream.newInstance(byteArrayOutputStream);
+ int fieldNumber = TREE_ROOT_FIELD_NUMBER;
+ for (ByteString directory : Lists.reverse(new ArrayList<ByteString>(directories))) {
+ codedOutputStream.writeBytes(fieldNumber, directory);
+ fieldNumber = TREE_CHILDREN_FIELD_NUMBER;
+ }
+ codedOutputStream.flush();
+
+ ByteString data = ByteString.copyFrom(byteArrayOutputStream.toByteArray());
Digest digest = digestUtil.compute(data.toByteArray());
if (result != null) {
result
.addOutputDirectoriesBuilder()
.setPath(remotePathResolver.localPathToOutputPath(dir))
- .setTreeDigest(digest);
+ .setTreeDigest(digest)
+ .setIsTopologicallySorted(true);
}
digestToBlobs.put(digest, data);
}
- private Directory computeDirectory(Path path, Tree.Builder tree)
+ private ByteString computeDirectory(Path path, Set<ByteString> directories)
throws ExecException, IOException {
Directory.Builder b = Directory.newBuilder();
@@ -332,9 +355,8 @@
String name = dirent.getName();
Path child = path.getRelative(name);
if (dirent.getType() == Dirent.Type.DIRECTORY) {
- Directory dir = computeDirectory(child, tree);
- b.addDirectoriesBuilder().setName(name).setDigest(digestUtil.compute(dir));
- tree.addChildren(dir);
+ ByteString dir = computeDirectory(child, directories);
+ b.addDirectoriesBuilder().setName(name).setDigest(digestUtil.compute(dir.toByteArray()));
} else if (dirent.getType() == Dirent.Type.SYMLINK) {
PathFragment target = child.readSymbolicLink();
if (!followSymlinks && !target.isAbsolute()) {
@@ -353,9 +375,8 @@
b.addFilesBuilder().setName(name).setDigest(digest).setIsExecutable(child.isExecutable());
digestToFile.put(digest, child);
} else if (statFollow.isDirectory()) {
- Directory dir = computeDirectory(child, tree);
- b.addDirectoriesBuilder().setName(name).setDigest(digestUtil.compute(dir));
- tree.addChildren(dir);
+ ByteString dir = computeDirectory(child, directories);
+ b.addDirectoriesBuilder().setName(name).setDigest(digestUtil.compute(dir.toByteArray()));
} else {
illegalOutput(child);
}
@@ -368,7 +389,9 @@
}
}
- return b.build();
+ ByteString directory = b.build().toByteString();
+ directories.add(directory);
+ return directory;
}
private void illegalOutput(Path path) throws ExecException {
diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java
index 4df12d4..9f7b551 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java
@@ -369,7 +369,11 @@
.setPath("a/foo")
.setDigest(fooDigest)
.setIsExecutable(true);
- expectedResult.addOutputDirectoriesBuilder().setPath("bar").setTreeDigest(barDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("bar")
+ .setTreeDigest(barDigest)
+ .setIsTopologicallySorted(true);
assertThat(result).isEqualTo(expectedResult.build());
}
@@ -409,7 +413,11 @@
ActionResult result = uploadDirectory(remoteCache, ImmutableList.<Path>of(barDir));
ActionResult.Builder expectedResult = ActionResult.newBuilder();
- expectedResult.addOutputDirectoriesBuilder().setPath("bar").setTreeDigest(barDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("bar")
+ .setTreeDigest(barDigest)
+ .setIsTopologicallySorted(true);
assertThat(result).isEqualTo(expectedResult.build());
}
@@ -472,7 +480,11 @@
ActionResult result = uploadDirectory(remoteCache, ImmutableList.of(barDir));
ActionResult.Builder expectedResult = ActionResult.newBuilder();
- expectedResult.addOutputDirectoriesBuilder().setPath("bar").setTreeDigest(barDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("bar")
+ .setTreeDigest(barDigest)
+ .setIsTopologicallySorted(true);
assertThat(result).isEqualTo(expectedResult.build());
}
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java
index 8341d8c..1808b86 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java
@@ -1314,7 +1314,11 @@
.setPath("outputs/a/foo")
.setDigest(fooDigest)
.setIsExecutable(true);
- expectedResult.addOutputDirectoriesBuilder().setPath("outputs/bar").setTreeDigest(barDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("outputs/bar")
+ .setTreeDigest(barDigest)
+ .setIsTopologicallySorted(true);
assertThat(manifest.getActionResult()).isEqualTo(expectedResult.build());
ImmutableList<Digest> toQuery = ImmutableList.of(fooDigest, quxDigest, barDigest);
@@ -1352,7 +1356,11 @@
// assert
ActionResult.Builder expectedResult = ActionResult.newBuilder();
- expectedResult.addOutputDirectoriesBuilder().setPath("outputs/bar").setTreeDigest(barDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("outputs/bar")
+ .setTreeDigest(barDigest)
+ .setIsTopologicallySorted(true);
assertThat(manifest.getActionResult()).isEqualTo(expectedResult.build());
assertThat(
getFromFuture(
@@ -1417,7 +1425,11 @@
// assert
ActionResult.Builder expectedResult = ActionResult.newBuilder();
- expectedResult.addOutputDirectoriesBuilder().setPath("outputs/bar").setTreeDigest(barDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("outputs/bar")
+ .setTreeDigest(barDigest)
+ .setIsTopologicallySorted(true);
assertThat(manifest.getActionResult()).isEqualTo(expectedResult.build());
ImmutableList<Digest> toQuery = ImmutableList.of(wobbleDigest, quxDigest, barDigest);
diff --git a/src/test/java/com/google/devtools/build/lib/remote/UploadManifestTest.java b/src/test/java/com/google/devtools/build/lib/remote/UploadManifestTest.java
index bc75807..7f28efb 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/UploadManifestTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/UploadManifestTest.java
@@ -14,6 +14,7 @@
package com.google.devtools.build.lib.remote;
import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat;
import static org.junit.Assert.assertThrows;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -41,6 +42,8 @@
import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -249,7 +252,11 @@
Digest treeDigest = digestUtil.compute(tree);
ActionResult.Builder expectedResult = ActionResult.newBuilder();
- expectedResult.addOutputDirectoriesBuilder().setPath("link").setTreeDigest(treeDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("link")
+ .setTreeDigest(treeDigest)
+ .setIsTopologicallySorted(true);
assertThat(result.build()).isEqualTo(expectedResult.build());
}
@@ -309,7 +316,11 @@
Digest treeDigest = digestUtil.compute(tree);
ActionResult.Builder expectedResult = ActionResult.newBuilder();
- expectedResult.addOutputDirectoriesBuilder().setPath("link").setTreeDigest(treeDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("link")
+ .setTreeDigest(treeDigest)
+ .setIsTopologicallySorted(true);
assertThat(result.build()).isEqualTo(expectedResult.build());
}
@@ -369,7 +380,11 @@
Digest treeDigest = digestUtil.compute(tree);
ActionResult.Builder expectedResult = ActionResult.newBuilder();
- expectedResult.addOutputDirectoriesBuilder().setPath("link").setTreeDigest(treeDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("link")
+ .setTreeDigest(treeDigest)
+ .setIsTopologicallySorted(true);
assertThat(result.build()).isEqualTo(expectedResult.build());
}
@@ -570,7 +585,11 @@
Digest treeDigest = digestUtil.compute(tree);
ActionResult.Builder expectedResult = ActionResult.newBuilder();
- expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("dir")
+ .setTreeDigest(treeDigest)
+ .setIsTopologicallySorted(true);
assertThat(result.build()).isEqualTo(expectedResult.build());
}
@@ -615,7 +634,11 @@
Digest treeDigest = digestUtil.compute(tree);
ActionResult.Builder expectedResult = ActionResult.newBuilder();
- expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("dir")
+ .setTreeDigest(treeDigest)
+ .setIsTopologicallySorted(true);
assertThat(result.build()).isEqualTo(expectedResult.build());
}
@@ -651,7 +674,11 @@
Digest treeDigest = digestUtil.compute(tree);
ActionResult.Builder expectedResult = ActionResult.newBuilder();
- expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("dir")
+ .setTreeDigest(treeDigest)
+ .setIsTopologicallySorted(true);
assertThat(result.build()).isEqualTo(expectedResult.build());
}
@@ -696,7 +723,11 @@
Digest treeDigest = digestUtil.compute(tree);
ActionResult.Builder expectedResult = ActionResult.newBuilder();
- expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("dir")
+ .setTreeDigest(treeDigest)
+ .setIsTopologicallySorted(true);
assertThat(result.build()).isEqualTo(expectedResult.build());
}
@@ -731,7 +762,11 @@
Digest treeDigest = digestUtil.compute(tree);
ActionResult.Builder expectedResult = ActionResult.newBuilder();
- expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("dir")
+ .setTreeDigest(treeDigest)
+ .setIsTopologicallySorted(true);
assertThat(result.build()).isEqualTo(expectedResult.build());
}
@@ -776,7 +811,11 @@
Digest treeDigest = digestUtil.compute(tree);
ActionResult.Builder expectedResult = ActionResult.newBuilder();
- expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("dir")
+ .setTreeDigest(treeDigest)
+ .setIsTopologicallySorted(true);
assertThat(result.build()).isEqualTo(expectedResult.build());
}
@@ -811,7 +850,11 @@
Digest treeDigest = digestUtil.compute(tree);
ActionResult.Builder expectedResult = ActionResult.newBuilder();
- expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("dir")
+ .setTreeDigest(treeDigest)
+ .setIsTopologicallySorted(true);
assertThat(result.build()).isEqualTo(expectedResult.build());
}
@@ -848,7 +891,11 @@
Digest treeDigest = digestUtil.compute(tree);
ActionResult.Builder expectedResult = ActionResult.newBuilder();
- expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("dir")
+ .setTreeDigest(treeDigest)
+ .setIsTopologicallySorted(true);
assertThat(result.build()).isEqualTo(expectedResult.build());
}
@@ -933,7 +980,11 @@
Digest treeDigest = digestUtil.compute(tree);
ActionResult.Builder expectedResult = ActionResult.newBuilder();
- expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("dir")
+ .setTreeDigest(treeDigest)
+ .setIsTopologicallySorted(true);
assertThat(result.build()).isEqualTo(expectedResult.build());
}
@@ -968,7 +1019,11 @@
Digest treeDigest = digestUtil.compute(tree);
ActionResult.Builder expectedResult = ActionResult.newBuilder();
- expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest);
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("dir")
+ .setTreeDigest(treeDigest)
+ .setIsTopologicallySorted(true);
assertThat(result.build()).isEqualTo(expectedResult.build());
}
@@ -1053,6 +1108,70 @@
assertThat(e).hasMessageThat().contains("dir/link");
}
+ @Test
+ public void actionResult_topologicallySortedAndDeduplicatedTree() throws Exception {
+ // Create 5^3 identical files named "dir/%d/%d/%d/file".
+ Path dir = execRoot.getRelative("dir");
+ dir.createDirectory();
+ byte[] fileContents = new byte[] {1, 2, 3, 4, 5};
+ final int childrenPerDirectory = 5;
+ for (int a = 0; a < childrenPerDirectory; a++) {
+ Path pathA = dir.getRelative(Integer.toString(a));
+ pathA.createDirectory();
+ for (int b = 0; b < childrenPerDirectory; b++) {
+ Path pathB = pathA.getRelative(Integer.toString(b));
+ pathB.createDirectory();
+ for (int c = 0; c < childrenPerDirectory; c++) {
+ Path pathC = pathB.getRelative(Integer.toString(c));
+ pathC.createDirectory();
+ Path file = pathC.getRelative("file");
+ FileSystemUtils.writeContent(file, fileContents);
+ }
+ }
+ }
+
+ ActionResult.Builder result = ActionResult.newBuilder();
+ UploadManifest um =
+ new UploadManifest(
+ digestUtil,
+ remotePathResolver,
+ result,
+ /*followSymlinks=*/ false,
+ /*allowDanglingSymlinks=*/ false,
+ /*allowAbsoluteSymlinks=*/ false);
+ um.addFiles(ImmutableList.of(dir));
+
+ // Even though we constructed 1 + 5 + 5^2 + 5^3 directories, the resulting
+ // Tree message should only contain four unique instances. The directories
+ // should also be topologically sorted.
+ List<Directory> children = new ArrayList<>();
+ Directory root =
+ Directory.newBuilder()
+ .addFiles(
+ FileNode.newBuilder().setName("file").setDigest(digestUtil.compute(fileContents)))
+ .build();
+ for (int depth = 0; depth < 3; depth++) {
+ Directory.Builder b = Directory.newBuilder();
+ Digest parentDigest = digestUtil.compute(root.toByteArray());
+ for (int i = 0; i < childrenPerDirectory; i++) {
+ b.addDirectories(
+ DirectoryNode.newBuilder().setName(Integer.toString(i)).setDigest(parentDigest));
+ }
+ children.add(0, root);
+ root = b.build();
+ }
+ Tree tree = Tree.newBuilder().setRoot(root).addAllChildren(children).build();
+ Digest treeDigest = digestUtil.compute(tree);
+
+ ActionResult.Builder expectedResult = ActionResult.newBuilder();
+ expectedResult
+ .addOutputDirectoriesBuilder()
+ .setPath("dir")
+ .setTreeDigest(treeDigest)
+ .setIsTopologicallySorted(true);
+ assertThat(result.build()).isEqualTo(expectedResult.build());
+ }
+
private Path createSpecialFile(String execPath) throws IOException {
Path special = mock(Path.class);
when(special.statIfFound(Symlinks.NOFOLLOW)).thenReturn(SPECIAL_FILE_STATUS);
diff --git a/third_party/remoteapis/build/bazel/remote/asset/v1/remote_asset.proto b/third_party/remoteapis/build/bazel/remote/asset/v1/remote_asset.proto
index e11fc7b..4d9be81 100644
--- a/third_party/remoteapis/build/bazel/remote/asset/v1/remote_asset.proto
+++ b/third_party/remoteapis/build/bazel/remote/asset/v1/remote_asset.proto
@@ -23,7 +23,7 @@
import "google/rpc/status.proto";
option csharp_namespace = "Build.Bazel.Remote.Asset.v1";
-option go_package = "remoteasset";
+option go_package = "github.com/bazelbuild/remote-apis/build/bazel/remote/asset/v1;remoteasset";
option java_multiple_files = true;
option java_outer_classname = "RemoteAssetProto";
option java_package = "build.bazel.remote.asset.v1";
diff --git a/third_party/remoteapis/build/bazel/remote/execution/v2/remote_execution.proto b/third_party/remoteapis/build/bazel/remote/execution/v2/remote_execution.proto
index 9b58064..60753f7 100644
--- a/third_party/remoteapis/build/bazel/remote/execution/v2/remote_execution.proto
+++ b/third_party/remoteapis/build/bazel/remote/execution/v2/remote_execution.proto
@@ -26,7 +26,7 @@
import "google/rpc/status.proto";
option csharp_namespace = "Build.Bazel.Remote.Execution.V2";
-option go_package = "remoteexecution";
+option go_package = "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2;remoteexecution";
option java_multiple_files = true;
option java_outer_classname = "RemoteExecutionProto";
option java_package = "build.bazel.remote.execution.v2";
@@ -255,10 +255,11 @@
//
// When attempting an upload, if another client has already completed the upload
// (which may occur in the middle of a single upload if another client uploads
-// the same blob concurrently), the request will terminate immediately with
-// a response whose `committed_size` is the full size of the uploaded file
-// (regardless of how much data was transmitted by the client). If the client
-// completes the upload but the
+// the same blob concurrently), the request will terminate immediately without
+// error, and with a response whose `committed_size` is the value `-1` if this
+// is a compressed upload, or with the full size of the uploaded file if this is
+// an uncompressed upload (regardless of how much data was transmitted by the
+// client). If the client completes the upload but the
// [Digest][build.bazel.remote.execution.v2.Digest] does not match, an
// `INVALID_ARGUMENT` error will be returned. In either case, the client should
// not attempt to retry the upload.
@@ -423,6 +424,8 @@
// CacheCapabilities and ExecutionCapabilities.
// * Execution only endpoints should return ExecutionCapabilities.
// * CAS + Action Cache only endpoints should return CacheCapabilities.
+ //
+ // There are no method-specific errors.
rpc GetCapabilities(GetCapabilitiesRequest) returns (ServerCapabilities) {
option (google.api.http) = {
get: "/v2/{instance_name=**}/capabilities"
@@ -475,6 +478,14 @@
// timeout that is longer than the server's maximum timeout, the server MUST
// reject the request.
//
+ // The timeout is only intended to cover the "execution" of the specified
+ // action and not time in queue nor any overheads before or after execution
+ // such as marshalling inputs/outputs. The server SHOULD avoid including time
+ // spent the client doesn't have control over, and MAY extend or reduce the
+ // timeout to account for delays or speedups that occur during execution
+ // itself (e.g., lazily loading data from the Content Addressable Storage,
+ // live migration of virtual machines, emulation overhead).
+ //
// The timeout is a part of the
// [Action][build.bazel.remote.execution.v2.Action] message, and
// therefore two `Actions` with different timeouts are different, even if they
@@ -529,9 +540,21 @@
string value = 2;
}
- // The arguments to the command. The first argument must be the path to the
- // executable, which must be either a relative path, in which case it is
- // evaluated with respect to the input root, or an absolute path.
+ // The arguments to the command.
+ //
+ // The first argument specifies the command to run, which may be either an
+ // absolute path, a path relative to the working directory, or an unqualified
+ // path (without path separators) which will be resolved using the operating
+ // system's equivalent of the PATH environment variable. Path separators
+ // native to the operating system running on the worker SHOULD be used. If the
+ // `environment_variables` list contains an entry for the PATH environment
+ // variable, it SHOULD be respected. If not, the resolution process is
+ // implementation-defined.
+ //
+ // Changed in v2.3. v2.2 and older require that no PATH lookups are performed,
+ // and that relative paths are resolved relative to the input root. This
+ // behavior can, however, not be relied upon, as most implementations already
+ // followed the rules described above.
repeated string arguments = 1;
// The environment variables to set when running the program. The worker may
@@ -605,10 +628,10 @@
// The type of the output (file or directory) is not specified, and will be
// determined by the server after action execution. If the resulting path is
// a file, it will be returned in an
- // [OutputFile][build.bazel.remote.execution.v2.OutputFile]) typed field.
+ // [OutputFile][build.bazel.remote.execution.v2.OutputFile] typed field.
// If the path is a directory, the entire directory structure will be returned
// as a [Tree][build.bazel.remote.execution.v2.Tree] message digest, see
- // [OutputDirectory][build.bazel.remote.execution.v2.OutputDirectory])
+ // [OutputDirectory][build.bazel.remote.execution.v2.OutputDirectory]
// Other files or directories that may be created during command execution
// are discarded.
//
@@ -942,6 +965,25 @@
// When the worker completed executing the action command.
google.protobuf.Timestamp execution_completed_timestamp = 8;
+ // New in v2.3: the amount of time the worker spent executing the action
+ // command, potentially computed using a worker-specific virtual clock.
+ //
+ // The virtual execution duration is only intended to cover the "execution" of
+ // the specified action and not time in queue nor any overheads before or
+ // after execution such as marshalling inputs/outputs. The server SHOULD avoid
+ // including time spent the client doesn't have control over, and MAY extend
+ // or reduce the execution duration to account for delays or speedups that
+ // occur during execution itself (e.g., lazily loading data from the Content
+ // Addressable Storage, live migration of virtual machines, emulation
+ // overhead).
+ //
+ // The method of timekeeping used to compute the virtual execution duration
+ // MUST be consistent with what is used to enforce the
+ // [Action][[build.bazel.remote.execution.v2.Action]'s `timeout`. There is no
+ // relationship between the virtual execution duration and the values of
+ // `execution_start_timestamp` and `execution_completed_timestamp`.
+ google.protobuf.Duration virtual_execution_duration = 12;
+
// When the worker started uploading action outputs.
google.protobuf.Timestamp output_upload_start_timestamp = 9;
@@ -1105,6 +1147,7 @@
// [GetActionResultRequest][build.bazel.remote.execution.v2.GetActionResultRequest]
// message. The server MAY omit inlining, even if requested, and MUST do so if inlining
// would cause the response to exceed message size limits.
+ // Clients SHOULD NOT populate this field when uploading to the cache.
bytes stdout_raw = 5;
// The digest for a blob containing the standard output of the action, which
@@ -1117,6 +1160,7 @@
// [GetActionResultRequest][build.bazel.remote.execution.v2.GetActionResultRequest]
// message. The server MAY omit inlining, even if requested, and MUST do so if inlining
// would cause the response to exceed message size limits.
+ // Clients SHOULD NOT populate this field when uploading to the cache.
bytes stderr_raw = 7;
// The digest for a blob containing the standard error of the action, which
@@ -1151,6 +1195,7 @@
// [GetActionResultRequest][build.bazel.remote.execution.v2.GetActionResultRequest]
// message. The server MAY omit inlining, even if requested, and MUST do so if inlining
// would cause the response to exceed message size limits.
+ // Clients SHOULD NOT populate this field when uploading to the cache.
bytes contents = 5;
// The supported node properties of the OutputFile, if requested by the Action.
@@ -1169,6 +1214,9 @@
// recursively, all its children. In order to reconstruct the directory tree,
// the client must take the digests of each of the child directories and then
// build up a tree starting from the `root`.
+ // Servers SHOULD ensure that these are ordered consistently such that two
+ // actions producing equivalent output directories on the same server
+ // implementation also produce Tree messages with matching digests.
repeated Directory children = 2;
}
@@ -1187,6 +1235,43 @@
// [Tree][build.bazel.remote.execution.v2.Tree] proto containing the
// directory's contents.
Digest tree_digest = 3;
+
+ // If set, consumers MAY make the following assumptions about the
+ // directories contained in the the Tree, so that it may be
+ // instantiated on a local file system by scanning through it
+ // sequentially:
+ //
+ // - All directories with the same binary representation are stored
+ // exactly once.
+ // - All directories, apart from the root directory, are referenced by
+ // at least one parent directory.
+ // - Directories are stored in topological order, with parents being
+ // stored before the child. The root directory is thus the first to
+ // be stored.
+ //
+ // Additionally, the Tree MUST be encoded as a stream of records,
+ // where each record has the following format:
+ //
+ // - A tag byte, having one of the following two values:
+ // - (1 << 3) | 2 == 0x0a: First record (the root directory).
+ // - (2 << 3) | 2 == 0x12: Any subsequent records (child directories).
+ // - The size of the directory, encoded as a base 128 varint.
+ // - The contents of the directory, encoded as a binary serialized
+ // Protobuf message.
+ //
+ // This encoding is a subset of the Protobuf wire format of the Tree
+ // message. As it is only permitted to store data associated with
+ // field numbers 1 and 2, the tag MUST be encoded as a single byte.
+ // More details on the Protobuf wire format can be found here:
+ // https://developers.google.com/protocol-buffers/docs/encoding
+ //
+ // It is recommended that implementations using this feature construct
+ // Tree objects manually using the specification given above, as
+ // opposed to using a Protobuf library to marshal a full Tree message.
+ // As individual Directory messages already need to be marshaled to
+ // compute their digests, constructing the Tree object manually avoids
+ // redundant marshaling.
+ bool is_topologically_sorted = 4;
}
// An `OutputSymlink` is similar to a
@@ -1334,6 +1419,17 @@
}
// The current stage of action execution.
+//
+// Even though these stages are numbered according to the order in which
+// they generally occur, there is no requirement that the remote
+// execution system reports events along this order. For example, an
+// operation MAY transition from the EXECUTING stage back to QUEUED
+// in case the hardware on which the operation executes fails.
+//
+// If and only if the remote execution system reports that an operation
+// has reached the COMPLETED stage, it MUST set the [done
+// field][google.longrunning.Operation.done] of the
+// [Operation][google.longrunning.Operation] and terminate the stream.
message ExecutionStage {
enum Value {
// Invalid value.
@@ -1469,6 +1565,12 @@
// The raw binary data.
bytes data = 2;
+
+ // The format of `data`. Must be `IDENTITY`/unspecified, or one of the
+ // compressors advertised by the
+ // [CacheCapabilities.supported_batch_compressors][build.bazel.remote.execution.v2.CacheCapabilities.supported_batch_compressors]
+ // field.
+ Compressor.Value compressor = 3;
}
// The instance of the execution system to operate against. A server may
@@ -1510,6 +1612,10 @@
// The individual blob digests.
repeated Digest digests = 2;
+
+ // A list of acceptable encodings for the returned inlined data, in no
+ // particular order. `IDENTITY` is always allowed even if not specified here.
+ repeated Compressor.Value acceptable_compressors = 3;
}
// A response message for
@@ -1523,6 +1629,10 @@
// The raw binary data.
bytes data = 2;
+ // The format the data is encoded in. MUST be `IDENTITY`/unspecified,
+ // or one of the acceptable compressors specified in the `BatchReadBlobsRequest`.
+ Compressor.Value compressor = 4;
+
// The result of attempting to download that blob.
google.rpc.Status status = 3;
}
@@ -1724,6 +1834,11 @@
// Note that this does not imply which if any compressors are supported by
// the server at the gRPC level.
repeated Compressor.Value supported_compressors = 6;
+
+ // Compressors supported for inlined data in
+ // [BatchUpdateBlobs][build.bazel.remote.execution.v2.ContentAddressableStorage.BatchUpdateBlobs]
+ // requests.
+ repeated Compressor.Value supported_batch_update_compressors = 7;
}
// Capabilities of the remote execution system.
diff --git a/third_party/remoteapis/build/bazel/semver/semver.proto b/third_party/remoteapis/build/bazel/semver/semver.proto
index 3b626b7..44f83f8 100644
--- a/third_party/remoteapis/build/bazel/semver/semver.proto
+++ b/third_party/remoteapis/build/bazel/semver/semver.proto
@@ -17,7 +17,7 @@
package build.bazel.semver;
option csharp_namespace = "Build.Bazel.Semver";
-option go_package = "semver";
+option go_package = "github.com/bazelbuild/remote-apis/build/bazel/semver";
option java_multiple_files = true;
option java_outer_classname = "SemverProto";
option java_package = "build.bazel.semver";
