Populate SpawnMetrics and improve CP output
Prevent non-remote executions from remote stat output in
CriticalPathComponent.
Populate metrics recorded in remote execution.
Emit SpawnExecutedEvent to supply metrics to CriticalPathComputer
subscriber after 'Actual execution' in AbstractSpawnStrategy.
Closes #10361.
PiperOrigin-RevId: 304132580
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
index 54271d2..88023c1 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
@@ -25,11 +25,13 @@
import build.bazel.remote.execution.v2.Digest;
import build.bazel.remote.execution.v2.Platform;
import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableSet;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.actions.Spawn;
+import com.google.devtools.build.lib.actions.SpawnMetrics;
import com.google.devtools.build.lib.actions.SpawnResult;
import com.google.devtools.build.lib.actions.SpawnResult.Status;
import com.google.devtools.build.lib.actions.Spawns;
@@ -50,6 +52,7 @@
import com.google.devtools.build.lib.remote.options.RemoteOptions;
import com.google.devtools.build.lib.remote.options.RemoteOutputsMode;
import com.google.devtools.build.lib.remote.util.DigestUtil;
+import com.google.devtools.build.lib.remote.util.NetworkTime;
import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
import com.google.devtools.build.lib.remote.util.Utils;
import com.google.devtools.build.lib.remote.util.Utils.InMemoryOutput;
@@ -116,9 +119,16 @@
return SpawnCache.NO_RESULT_NO_STORE;
}
+ NetworkTime networkTime = new NetworkTime();
+ Stopwatch totalTime = Stopwatch.createStarted();
+
SortedMap<PathFragment, ActionInput> inputMap = context.getInputMapping(true);
MerkleTree merkleTree =
MerkleTree.build(inputMap, context.getMetadataProvider(), execRoot, digestUtil);
+ SpawnMetrics.Builder spawnMetrics =
+ new SpawnMetrics.Builder()
+ .setInputBytes(merkleTree.getInputBytes())
+ .setInputFiles(merkleTree.getInputFiles());
Digest merkleTreeRoot = merkleTree.getRootDigest();
// Get the remote platform properties.
@@ -138,7 +148,8 @@
// Look up action cache, and reuse the action output if it is found.
ActionKey actionKey = digestUtil.computeActionKey(action);
Context withMetadata =
- TracingMetadataUtils.contextWithMetadata(buildRequestId, commandId, actionKey);
+ TracingMetadataUtils.contextWithMetadata(buildRequestId, commandId, actionKey)
+ .withValue(NetworkTime.CONTEXT_KEY, networkTime);
Profiler prof = Profiler.instance();
if (options.remoteAcceptCached
@@ -161,6 +172,7 @@
remoteOutputsMode,
/* exitCode = */ 0,
hasFilesToDownload(spawn.getOutputFiles(), filesToDownload));
+ Stopwatch fetchTime = Stopwatch.createStarted();
if (downloadOutputs) {
try (SilentCloseable c =
prof.profile(ProfilerTask.REMOTE_DOWNLOAD, "download outputs")) {
@@ -183,9 +195,19 @@
context::lockOutputFiles);
}
}
+ fetchTime.stop();
+ totalTime.stop();
+ spawnMetrics
+ .setFetchTime(fetchTime.elapsed())
+ .setTotalTime(totalTime.elapsed())
+ .setNetworkTime(networkTime.getDuration());
SpawnResult spawnResult =
createSpawnResult(
- result.getExitCode(), /* cacheHit= */ true, "remote", inMemoryOutput);
+ result.getExitCode(),
+ /* cacheHit= */ true,
+ "remote",
+ inMemoryOutput,
+ spawnMetrics.build());
return SpawnCache.success(spawnResult);
}
} catch (CacheNotFoundException e) {