Store the spawn digest in the execution log.
The digest is the unique identifier for a remote execution request and comes in handy when replaying the action locally for debugging (it dispenses with inspecting the grpc log).
PiperOrigin-RevId: 446170315
diff --git a/src/main/java/com/google/devtools/build/lib/actions/SpawnResult.java b/src/main/java/com/google/devtools/build/lib/actions/SpawnResult.java
index 962db41..7205dcb 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/SpawnResult.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/SpawnResult.java
@@ -13,6 +13,7 @@
// limitations under the License.
package com.google.devtools.build.lib.actions;
+import com.google.auto.value.AutoValue;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.devtools.build.lib.bugreport.BugReport;
@@ -268,6 +269,23 @@
/** Whether the spawn result was obtained through remote strategy. */
boolean wasRemote();
+ /** A unique identifier for the spawn. */
+ @AutoValue
+ @Immutable
+ public abstract class Digest {
+ public abstract String getHash();
+
+ public abstract Long getSizeBytes();
+
+ public static Digest of(String hash, Long sizeBytes) {
+ return new AutoValue_SpawnResult_Digest(hash, sizeBytes);
+ }
+ }
+
+ default Optional<Digest> getDigest() {
+ return Optional.empty();
+ }
+
/** Basic implementation of {@link SpawnResult}. */
@Immutable
@ThreadSafe
@@ -294,7 +312,9 @@
// Invariant: Either both have a value or both are null.
@Nullable private final ActionInput inMemoryOutputFile;
@Nullable private final ByteString inMemoryContents;
+
private final boolean remote;
+ private final Optional<Digest> digest;
SimpleSpawnResult(Builder builder) {
this.exitCode = builder.exitCode;
@@ -320,6 +340,7 @@
this.inMemoryContents = builder.inMemoryContents;
this.actionMetadataLog = builder.actionMetadataLog;
this.remote = builder.remote;
+ this.digest = builder.digest;
}
@Override
@@ -456,6 +477,11 @@
public boolean wasRemote() {
return remote;
}
+
+ @Override
+ public Optional<Digest> getDigest() {
+ return digest;
+ }
}
/** Builder class for {@link SpawnResult}. */
@@ -482,7 +508,9 @@
// Invariant: Either both have a value or both are null.
@Nullable private ActionInput inMemoryOutputFile;
@Nullable private ByteString inMemoryContents;
+
private boolean remote;
+ private Optional<Digest> digest = Optional.empty();
public SpawnResult build() {
Preconditions.checkArgument(!runnerName.isEmpty());
@@ -617,6 +645,11 @@
this.remote = remote;
return this;
}
+
+ public Builder setDigest(Optional<Digest> digest) {
+ this.digest = digest;
+ return this;
+ }
}
/** A {@link Spawn}'s metadata name and {@link Path}. */
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java b/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java
index 8da6e88..bd954c4 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java
@@ -150,6 +150,13 @@
builder.setExitCode(result.exitCode());
builder.setRemoteCacheHit(result.isCacheHit());
builder.setRunner(result.getRunnerName());
+ if (result.getDigest().isPresent()) {
+ builder
+ .getDigestBuilder()
+ .setHash(result.getDigest().get().getHash())
+ .setSizeBytes(result.getDigest().get().getSizeBytes());
+ }
+
String progressMessage = spawn.getResourceOwner().getProgressMessage();
if (progressMessage != null) {
builder.setProgressMessage(progressMessage);
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
index 67d0201..03e8825 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
@@ -122,6 +122,7 @@
.setNetworkTime(action.getNetworkTime().getDuration());
SpawnResult spawnResult =
createSpawnResult(
+ action.getActionKey(),
result.getExitCode(),
/*cacheHit=*/ true,
result.cacheName(),
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
index a74d6ac..01c01ae 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
@@ -422,6 +422,7 @@
// subtract network time consumed here to ensure wall clock during fetch is not double
// counted, and metrics time computation does not exceed total time
return createSpawnResult(
+ action.getActionKey(),
result.getExitCode(),
cacheHit,
cacheName,
diff --git a/src/main/java/com/google/devtools/build/lib/remote/util/Utils.java b/src/main/java/com/google/devtools/build/lib/remote/util/Utils.java
index c1b10e1..524b96b 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/util/Utils.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/util/Utils.java
@@ -76,6 +76,7 @@
import java.util.Collection;
import java.util.Locale;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.function.BiFunction;
@@ -143,6 +144,7 @@
/** Constructs a {@link SpawnResult}. */
public static SpawnResult createSpawnResult(
+ ActionKey actionKey,
int exitCode,
boolean cacheHit,
String runnerName,
@@ -164,7 +166,11 @@
timestampToInstant(executionStartTimestamp),
timestampToInstant(executionCompletedTimestamp)))
.setSpawnMetrics(spawnMetrics)
- .setRemote(true);
+ .setRemote(true)
+ .setDigest(
+ Optional.of(
+ SpawnResult.Digest.of(
+ actionKey.getDigest().getHash(), actionKey.getDigest().getSizeBytes())));
if (exitCode != 0) {
builder.setFailureDetail(
FailureDetail.newBuilder()
diff --git a/src/main/protobuf/spawn.proto b/src/main/protobuf/spawn.proto
index 1507add..0f55812 100644
--- a/src/main/protobuf/spawn.proto
+++ b/src/main/protobuf/spawn.proto
@@ -136,4 +136,7 @@
// Canonical label of the target that emitted this spawn, may not always be
// set.
string target_label = 18;
+
+ // A unique identifier for this Spawn.
+ Digest digest = 19;
}
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java
index 4503118..c926c23 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java
@@ -88,11 +88,13 @@
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
+import java.util.Optional;
import java.util.SortedMap;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
+import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.mockito.invocation.InvocationOnMock;
@@ -263,10 +265,11 @@
// arrange
RemoteSpawnCache cache = createRemoteSpawnCache();
RemoteExecutionService service = cache.getRemoteExecutionService();
+ ArgumentCaptor<ActionKey> actionKeyCaptor = ArgumentCaptor.forClass(ActionKey.class);
ActionResult actionResult = ActionResult.getDefaultInstance();
when(remoteCache.downloadActionResult(
any(RemoteActionExecutionContext.class),
- any(ActionKey.class),
+ actionKeyCaptor.capture(),
/* inlineOutErr= */ eq(false)))
.thenAnswer(
new Answer<CachedActionResult>() {
@@ -304,6 +307,12 @@
.downloadOutputs(
any(), eq(RemoteActionResult.createFromCache(CachedActionResult.remote(actionResult))));
verify(service, never()).uploadOutputs(any(), any());
+ assertThat(result.getDigest())
+ .isEqualTo(
+ Optional.of(
+ SpawnResult.Digest.of(
+ actionKeyCaptor.getValue().getDigest().getHash(),
+ actionKeyCaptor.getValue().getDigest().getSizeBytes())));
assertThat(result.setupSuccess()).isTrue();
assertThat(result.exitCode()).isEqualTo(0);
assertThat(result.isCacheHit()).isTrue();
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java
index 22afbf5..bee8104 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java
@@ -113,6 +113,7 @@
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.List;
+import java.util.Optional;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
@@ -1212,6 +1213,40 @@
}
@Test
+ public void testDigest() throws Exception {
+ RemoteSpawnRunner runner = newSpawnRunner();
+ RemoteExecutionService service = runner.getRemoteExecutionService();
+
+ ExecuteResponse resp =
+ ExecuteResponse.newBuilder()
+ .setResult(ActionResult.newBuilder().setExitCode(0).build())
+ .build();
+ when(executor.executeRemotely(
+ any(RemoteActionExecutionContext.class),
+ any(ExecuteRequest.class),
+ any(OperationObserver.class)))
+ .thenReturn(resp);
+
+ Spawn spawn = newSimpleSpawn();
+ FakeSpawnExecutionContext policy = getSpawnContext(spawn);
+
+ SpawnResult res = runner.exec(spawn, policy);
+ assertThat(res.status()).isEqualTo(Status.SUCCESS);
+
+ ArgumentCaptor<RemoteAction> requestCaptor = ArgumentCaptor.forClass(RemoteAction.class);
+
+ verify(service)
+ .executeRemotely(requestCaptor.capture(), anyBoolean(), any(OperationObserver.class));
+
+ assertThat(res.getDigest())
+ .isEqualTo(
+ Optional.of(
+ SpawnResult.Digest.of(
+ requestCaptor.getValue().getActionKey().getDigest().getHash(),
+ requestCaptor.getValue().getActionKey().getDigest().getSizeBytes())));
+ }
+
+ @Test
public void accountingDisabledWithoutWorker() {
SpawnMetrics.Builder spawnMetrics = Mockito.mock(SpawnMetrics.Builder.class);
RemoteSpawnRunner.spawnMetricsAccounting(
