remote: improve profiler support
Add profiler tasks for common code paths and add user readable names.
Closes #7892.
PiperOrigin-RevId: 241890706
diff --git a/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java
index f0f9837..0504f1b 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java
@@ -39,6 +39,8 @@
import com.google.devtools.build.lib.exec.SpawnCache.CacheHandle;
import com.google.devtools.build.lib.exec.SpawnRunner.ProgressStatus;
import com.google.devtools.build.lib.exec.SpawnRunner.SpawnExecutionContext;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.SilentCloseable;
import com.google.devtools.build.lib.util.CommandFailureUtils;
import com.google.devtools.build.lib.util.io.FileOutErr;
import com.google.devtools.build.lib.vfs.Path;
@@ -245,13 +247,16 @@
public SortedMap<PathFragment, ActionInput> getInputMapping(
boolean expandTreeArtifactsInRunfiles) throws IOException {
if (lazyInputMapping == null) {
- lazyInputMapping =
- spawnInputExpander.getInputMapping(
- spawn,
- actionExecutionContext.getArtifactExpander(),
- actionExecutionContext.getPathResolver(),
- actionExecutionContext.getMetadataProvider(),
- expandTreeArtifactsInRunfiles);
+ try (SilentCloseable c =
+ Profiler.instance().profile("AbstractSpawnStrategy.getInputMapping")) {
+ lazyInputMapping =
+ spawnInputExpander.getInputMapping(
+ spawn,
+ actionExecutionContext.getArtifactExpander(),
+ actionExecutionContext.getPathResolver(),
+ actionExecutionContext.getMetadataProvider(),
+ expandTreeArtifactsInRunfiles);
+ }
}
return lazyInputMapping;
}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SpawnInputExpander.java b/src/main/java/com/google/devtools/build/lib/exec/SpawnInputExpander.java
index 81a8170..baad566 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/SpawnInputExpander.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/SpawnInputExpander.java
@@ -203,6 +203,7 @@
MetadataProvider actionInputFileCache,
boolean expandTreeArtifactsInRunfiles)
throws IOException {
+
TreeMap<PathFragment, ActionInput> inputMap = new TreeMap<>();
addInputs(inputMap, spawn, artifactExpander);
addRunfilesToInputs(
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java b/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java
index d26b081..c74b30e 100644
--- a/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java
+++ b/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java
@@ -89,6 +89,8 @@
STARLARK_BUILTIN_FN("Starlark builtin function call", 0x990033),
STARLARK_USER_COMPILED_FN("Starlark compiled user function call", 0xCC0033),
ACTION_FS_STAGING("Staging per-action file system", 0x000000),
+ REMOTE_CACHE_CHECK("remote action cache check", 0x9999CC),
+ REMOTE_DOWNLOAD("remote output download", 0x9999CC),
UNKNOWN("Unknown event", 0x339966);
// Size of the ProfilerTask value space.
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcher.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcher.java
index 7f3960a..1c57b64 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcher.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcher.java
@@ -27,6 +27,7 @@
import com.google.devtools.build.lib.actions.MetadataProvider;
import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
import com.google.devtools.build.lib.profiler.SilentCloseable;
import com.google.devtools.build.lib.remote.util.DigestUtil;
import com.google.devtools.build.lib.remote.util.Utils;
@@ -86,7 +87,8 @@
public void prefetchFiles(
Iterable<? extends ActionInput> inputs, MetadataProvider metadataProvider)
throws IOException, InterruptedException {
- try (SilentCloseable c = Profiler.instance().profile("Remote.fetchInputs")) {
+ try (SilentCloseable c =
+ Profiler.instance().profile(ProfilerTask.REMOTE_DOWNLOAD, "stage remote inputs")) {
Map<Path, ListenableFuture<Void>> downloadsToWaitFor = new HashMap<>();
for (ActionInput input : inputs) {
if (input instanceof VirtualActionInput) {
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
index a482f1b..3b1732d 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
@@ -38,6 +38,7 @@
import com.google.devtools.build.lib.exec.SpawnRunner.ProgressStatus;
import com.google.devtools.build.lib.exec.SpawnRunner.SpawnExecutionContext;
import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
import com.google.devtools.build.lib.profiler.SilentCloseable;
import com.google.devtools.build.lib.remote.merkletree.MerkleTree;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
@@ -127,13 +128,14 @@
Context withMetadata =
TracingMetadataUtils.contextWithMetadata(buildRequestId, commandId, actionKey);
+ Profiler prof = Profiler.instance();
if (checkCache) {
// Metadata will be available in context.current() until we detach.
// This is done via a thread-local variable.
Context previous = withMetadata.attach();
try {
ActionResult result;
- try (SilentCloseable c = Profiler.instance().profile("RemoteCache.getCachedActionResult")) {
+ try (SilentCloseable c = prof.profile(ProfilerTask.REMOTE_CACHE_CHECK, "check cache hit")) {
result = remoteCache.getCachedActionResult(actionKey);
}
if (result != null && result.getExitCode() == 0) {
@@ -143,7 +145,8 @@
InMemoryOutput inMemoryOutput = null;
switch (remoteOutputsMode) {
case MINIMAL:
- try (SilentCloseable c = Profiler.instance().profile("RemoteCache.downloadMinimal")) {
+ try (SilentCloseable c =
+ prof.profile(ProfilerTask.REMOTE_DOWNLOAD, "download outputs minimal")) {
inMemoryOutput =
remoteCache.downloadMinimal(
result,
@@ -155,7 +158,8 @@
}
break;
case ALL:
- try (SilentCloseable c = Profiler.instance().profile("RemoteCache.download")) {
+ try (SilentCloseable c =
+ prof.profile(ProfilerTask.REMOTE_DOWNLOAD, "download outputs")) {
remoteCache.download(result, execRoot, context.getFileOutErr());
}
break;
@@ -206,8 +210,7 @@
}
if (options.experimentalGuardAgainstConcurrentChanges) {
- try (SilentCloseable c =
- Profiler.instance().profile("RemoteCache.checkForConcurrentModifications")) {
+ try (SilentCloseable c = prof.profile("RemoteCache.checkForConcurrentModifications")) {
checkForConcurrentModifications();
} catch (IOException e) {
report(Event.warn(e.getMessage()));
@@ -218,7 +221,7 @@
Context previous = withMetadata.attach();
Collection<Path> files =
RemoteSpawnRunner.resolveActionInputs(execRoot, spawn.getOutputFiles());
- try (SilentCloseable c = Profiler.instance().profile("RemoteCache.upload")) {
+ try (SilentCloseable c = prof.profile(ProfilerTask.UPLOAD_TIME, "upload outputs")) {
remoteCache.upload(
actionKey, action, command, execRoot, files, context.getFileOutErr());
} catch (IOException e) {
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
index e03cd4c..464f373 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
@@ -14,6 +14,9 @@
package com.google.devtools.build.lib.remote;
+import static com.google.devtools.build.lib.profiler.ProfilerTask.REMOTE_DOWNLOAD;
+import static com.google.devtools.build.lib.profiler.ProfilerTask.REMOTE_EXECUTION;
+import static com.google.devtools.build.lib.profiler.ProfilerTask.UPLOAD_TIME;
import static com.google.devtools.build.lib.remote.util.Utils.createSpawnResult;
import static com.google.devtools.build.lib.remote.util.Utils.getFromFuture;
import static com.google.devtools.build.lib.remote.util.Utils.getInMemoryOutputPath;
@@ -52,6 +55,7 @@
import com.google.devtools.build.lib.exec.SpawnExecException;
import com.google.devtools.build.lib.exec.SpawnRunner;
import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
import com.google.devtools.build.lib.profiler.SilentCloseable;
import com.google.devtools.build.lib.remote.merkletree.MerkleTree;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
@@ -177,6 +181,7 @@
Context withMetadata =
TracingMetadataUtils.contextWithMetadata(buildRequestId, commandId, actionKey);
Context previous = withMetadata.attach();
+ Profiler prof = Profiler.instance();
try {
boolean acceptCachedResult = remoteOptions.remoteAcceptCached && spawnCachable;
boolean uploadLocalResults = remoteOptions.remoteUploadLocalResults && spawnCachable;
@@ -184,7 +189,7 @@
try {
// Try to lookup the action in the action cache.
ActionResult cachedResult;
- try (SilentCloseable c = Profiler.instance().profile("Remote.getCachedActionResult")) {
+ try (SilentCloseable c = prof.profile(ProfilerTask.REMOTE_CACHE_CHECK, "check cache hit")) {
cachedResult = acceptCachedResult ? remoteCache.getCachedActionResult(actionKey) : null;
}
if (cachedResult != null) {
@@ -236,14 +241,14 @@
ExecuteRequest request = requestBuilder.build();
// Upload the command and all the inputs into the remote cache.
- try (SilentCloseable c = Profiler.instance().profile("Remote.uploadInputs")) {
+ try (SilentCloseable c = prof.profile(UPLOAD_TIME, "upload missing inputs")) {
Map<Digest, Message> additionalInputs = Maps.newHashMapWithExpectedSize(2);
additionalInputs.put(actionKey.getDigest(), action);
additionalInputs.put(commandHash, command);
remoteCache.ensureInputsPresent(merkleTree, additionalInputs, execRoot);
}
ExecuteResponse reply;
- try (SilentCloseable c = Profiler.instance().profile("Remote.executeRemotely")) {
+ try (SilentCloseable c = prof.profile(REMOTE_EXECUTION, "execute remotely")) {
reply = remoteExecutor.executeRemotely(request);
}
@@ -256,8 +261,7 @@
outErr.printErr(message + "\n");
}
- try (SilentCloseable c =
- Profiler.instance().profile("Remote.maybeDownloadServerLogs")) {
+ try (SilentCloseable c = prof.profile(REMOTE_DOWNLOAD, "download server logs")) {
maybeDownloadServerLogs(reply, actionKey);
}
@@ -297,7 +301,8 @@
InMemoryOutput inMemoryOutput = null;
switch (effectiveOutputsStrategy) {
case MINIMAL:
- try (SilentCloseable c = Profiler.instance().profile("Remote.downloadMinimal")) {
+ try (SilentCloseable c =
+ Profiler.instance().profile(REMOTE_DOWNLOAD, "download outputs minimal")) {
inMemoryOutput =
remoteCache.downloadMinimal(
actionResult,
@@ -310,7 +315,7 @@
break;
case ALL:
- try (SilentCloseable c = Profiler.instance().profile("Remote.downloadRemoteResults")) {
+ try (SilentCloseable c = Profiler.instance().profile(REMOTE_DOWNLOAD, "download outputs")) {
remoteCache.download(actionResult, execRoot, context.getFileOutErr());
}
break;
@@ -573,7 +578,7 @@
}
Collection<Path> outputFiles = resolveActionInputs(execRoot, spawn.getOutputFiles());
- try (SilentCloseable c = Profiler.instance().profile("Remote.upload")) {
+ try (SilentCloseable c = Profiler.instance().profile(UPLOAD_TIME, "upload outputs")) {
remoteCache.upload(
actionKey, action, command, execRoot, outputFiles, context.getFileOutErr());
} catch (IOException e) {
