remote: introduce --experimental_remote_download_outputs=minimal
This implements the first milestone of #6862. When specifying
--experimental_remote_download_outputs=minimal Bazel will avoid
downloading most action outputs when used with remote caching
and remote execution. That is, it will only download an action's
stdout/stderr as well as .d and .jdeps output files when building
C++ and Java. All other output files will only be downloaded on
demand, that is if a local action declares a remote output as an
input.
Enabling this mode currently requires specifying three additional
flags to the set of the remote caching/execution flags:
$ bazel build ... \
--experimental_inmemory_jdeps_files \
--experimental_inmemory_dotd_files \
--experimental_remote_download_outputs=minimal
This mode does not yet interact well with the Build Event
Service (--bes_backend=...). Specifically, local file uploads
and remote file name conversions are disabled when
--experimental_remote_download_outputs=minimal is specified.
Lastly, this mode also does not at all interact with Bazel's
on disk action cache. That is, all (local) cached state will
be lost after restarting the Bazel server. Also changing the
value of --experimental_remote_download_outputs between builds
will invalidate all (cached) actions.
Closes #7905.
PiperOrigin-RevId: 241681028
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
index 2da78bf..a482f1b 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
@@ -14,6 +14,8 @@
package com.google.devtools.build.lib.remote;
import static com.google.common.base.Strings.isNullOrEmpty;
+import static com.google.devtools.build.lib.remote.util.Utils.createSpawnResult;
+import static com.google.devtools.build.lib.remote.util.Utils.getInMemoryOutputPath;
import build.bazel.remote.execution.v2.Action;
import build.bazel.remote.execution.v2.ActionResult;
@@ -39,9 +41,11 @@
import com.google.devtools.build.lib.profiler.SilentCloseable;
import com.google.devtools.build.lib.remote.merkletree.MerkleTree;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
+import com.google.devtools.build.lib.remote.options.RemoteOutputsMode;
import com.google.devtools.build.lib.remote.util.DigestUtil;
import com.google.devtools.build.lib.remote.util.DigestUtil.ActionKey;
import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
+import com.google.devtools.build.lib.remote.util.Utils.InMemoryOutput;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import io.grpc.Context;
@@ -114,12 +118,12 @@
Command command =
RemoteSpawnRunner.buildCommand(
spawn.getOutputFiles(), spawn.getArguments(), spawn.getEnvironment(), platform);
+ RemoteOutputsMode remoteOutputsMode = options.remoteOutputsMode;
Action action =
RemoteSpawnRunner.buildAction(
digestUtil.compute(command), merkleTreeRoot, context.getTimeout(), true);
// Look up action cache, and reuse the action output if it is found.
ActionKey actionKey = digestUtil.computeActionKey(action);
-
Context withMetadata =
TracingMetadataUtils.contextWithMetadata(buildRequestId, commandId, actionKey);
@@ -135,16 +139,30 @@
if (result != null && result.getExitCode() == 0) {
// In case if failed action returned (exit code != 0) we treat it as a cache miss
// Otherwise, we know that result exists.
- try (SilentCloseable c = Profiler.instance().profile("RemoteCache.download")) {
- remoteCache.download(result, execRoot, context.getFileOutErr());
+ PathFragment inMemoryOutputPath = getInMemoryOutputPath(spawn);
+ InMemoryOutput inMemoryOutput = null;
+ switch (remoteOutputsMode) {
+ case MINIMAL:
+ try (SilentCloseable c = Profiler.instance().profile("RemoteCache.downloadMinimal")) {
+ inMemoryOutput =
+ remoteCache.downloadMinimal(
+ result,
+ spawn.getOutputFiles(),
+ inMemoryOutputPath,
+ context.getFileOutErr(),
+ execRoot,
+ context.getMetadataInjector());
+ }
+ break;
+ case ALL:
+ try (SilentCloseable c = Profiler.instance().profile("RemoteCache.download")) {
+ remoteCache.download(result, execRoot, context.getFileOutErr());
+ }
+ break;
}
SpawnResult spawnResult =
- new SpawnResult.Builder()
- .setStatus(Status.SUCCESS)
- .setExitCode(result.getExitCode())
- .setCacheHit(true)
- .setRunnerName("remote cache hit")
- .build();
+ createSpawnResult(
+ result.getExitCode(), /* cacheHit= */ true, "remote", inMemoryOutput);
return SpawnCache.success(spawnResult);
}
} catch (CacheNotFoundException e) {
@@ -160,6 +178,9 @@
withMetadata.detach(previous);
}
}
+
+ context.prefetchInputs();
+
if (options.remoteUploadLocalResults) {
return new CacheHandle() {
@Override
