Remote API v2 migration.
Major differences between v1test and v2 that are implemented here:
- Execute call streams Operation updates
- WaitExecution call replaces Watcher API
- Action is no longer part of the Execute request, and must be uploaded separately
- output spec and platform moved from Action to Command
Also, adding retries to operations lost on the server, resolving a TODO.
TESTED=unit tests, LRE, RBE.
TYPE_CHANGE_OK=this is a breaking change by design, will update proto parsing tool as well
TAG_CHANGE_OK=this way give us flexibility to update the parsing tool to support both versions
PiperOrigin-RevId: 208990450
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
index e99cc6e..65570b6 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
@@ -16,6 +16,14 @@
import static com.google.devtools.build.lib.remote.util.Utils.getFromFuture;
+import build.bazel.remote.execution.v2.Action;
+import build.bazel.remote.execution.v2.ActionResult;
+import build.bazel.remote.execution.v2.Command;
+import build.bazel.remote.execution.v2.Digest;
+import build.bazel.remote.execution.v2.ExecuteRequest;
+import build.bazel.remote.execution.v2.ExecuteResponse;
+import build.bazel.remote.execution.v2.LogFile;
+import build.bazel.remote.execution.v2.Platform;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
@@ -48,14 +56,6 @@
import com.google.devtools.build.lib.util.io.FileOutErr;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.devtools.remoteexecution.v1test.Action;
-import com.google.devtools.remoteexecution.v1test.ActionResult;
-import com.google.devtools.remoteexecution.v1test.Command;
-import com.google.devtools.remoteexecution.v1test.Digest;
-import com.google.devtools.remoteexecution.v1test.ExecuteRequest;
-import com.google.devtools.remoteexecution.v1test.ExecuteResponse;
-import com.google.devtools.remoteexecution.v1test.LogFile;
-import com.google.devtools.remoteexecution.v1test.Platform;
import com.google.protobuf.TextFormat;
import com.google.protobuf.TextFormat.ParseException;
import io.grpc.Context;
@@ -136,7 +136,7 @@
public SpawnResult exec(Spawn spawn, SpawnExecutionContext context)
throws ExecException, InterruptedException, IOException {
if (!Spawns.mayBeExecutedRemotely(spawn) || remoteCache == null) {
- return fallbackRunner.get().exec(spawn, context);
+ return execLocally(spawn, context);
}
context.report(ProgressStatus.EXECUTING, getName());
@@ -147,13 +147,15 @@
TreeNode inputRoot = repository.buildFromActionInputs(inputMap);
repository.computeMerkleDigests(inputRoot);
maybeWriteParamFilesLocally(spawn);
- Command command = buildCommand(spawn.getArguments(), spawn.getEnvironment());
+ Command command = buildCommand(
+ spawn.getOutputFiles(),
+ spawn.getArguments(),
+ spawn.getEnvironment(),
+ spawn.getExecutionPlatform());
Action action =
buildAction(
- spawn.getOutputFiles(),
digestUtil.compute(command),
repository.getMerkleDigest(inputRoot),
- spawn.getExecutionPlatform(),
context.getTimeout(),
Spawns.mayBeCached(spawn));
@@ -190,26 +192,27 @@
}
}
} catch (IOException e) {
- return execLocallyOrFail(spawn, context, inputMap, actionKey, uploadLocalResults, e);
+ return execLocallyAndUploadOrFail(
+ spawn, context, inputMap, actionKey, action, command, uploadLocalResults, e);
}
if (remoteExecutor == null) {
// Remote execution is disabled and so execute the spawn on the local machine.
- return execLocally(spawn, context, inputMap, uploadLocalResults, remoteCache, actionKey);
- }
+ return execLocallyAndUpload(
+ spawn, context, inputMap, remoteCache, actionKey, action, command, uploadLocalResults);
+ }
ExecuteRequest request =
ExecuteRequest.newBuilder()
.setInstanceName(remoteOptions.remoteInstanceName)
- .setAction(action)
+ .setActionDigest(actionKey.getDigest())
.setSkipCacheLookup(!acceptCachedResult)
.build();
try {
return retrier.execute(
() -> {
// Upload the command and all the inputs into the remote cache.
- remoteCache.ensureInputsPresent(repository, execRoot, inputRoot, command);
-
+ remoteCache.ensureInputsPresent(repository, execRoot, inputRoot, action, command);
ExecuteResponse reply = remoteExecutor.executeRemotely(request);
maybeDownloadServerLogs(reply, actionKey);
@@ -219,8 +222,10 @@
.build();
});
} catch (IOException e) {
- return execLocallyOrFail(spawn, context, inputMap, actionKey, uploadLocalResults, e);
+ return execLocallyAndUploadOrFail(
+ spawn, context, inputMap, actionKey, action, command, uploadLocalResults, e);
}
+
} finally {
withMetadata.detach(previous);
}
@@ -258,7 +263,7 @@
logPath = parent.getRelative(e.getKey());
logCount++;
try {
- getFromFuture(remoteCache.downloadFile(logPath, e.getValue().getDigest(), null));
+ getFromFuture(remoteCache.downloadFile(logPath, e.getValue().getDigest()));
} catch (IOException ex) {
reportOnce(Event.warn("Failed downloading server logs from the remote cache."));
}
@@ -280,11 +285,18 @@
.setExitCode(exitCode);
}
- private SpawnResult execLocallyOrFail(
+ private SpawnResult execLocally(Spawn spawn, SpawnExecutionContext context)
+ throws ExecException, InterruptedException, IOException {
+ return fallbackRunner.get().exec(spawn, context);
+ }
+
+ private SpawnResult execLocallyAndUploadOrFail(
Spawn spawn,
SpawnExecutionContext context,
SortedMap<PathFragment, ActionInput> inputMap,
ActionKey actionKey,
+ Action action,
+ Command command,
boolean uploadLocalResults,
IOException cause)
throws ExecException, InterruptedException, IOException {
@@ -296,7 +308,8 @@
if (remoteOptions.remoteLocalFallback
&& !(cause instanceof RetryException
&& RemoteRetrierUtils.causedByExecTimeout((RetryException) cause))) {
- return execLocally(spawn, context, inputMap, uploadLocalResults, remoteCache, actionKey);
+ return execLocallyAndUpload(
+ spawn, context, inputMap, remoteCache, actionKey, action, command, uploadLocalResults);
}
return handleError(cause, context.getFileOutErr(), actionKey);
}
@@ -343,38 +356,14 @@
}
static Action buildAction(
- Collection<? extends ActionInput> outputs,
Digest command,
Digest inputRoot,
- @Nullable PlatformInfo executionPlatform,
Duration timeout,
boolean cacheable) {
Action.Builder action = Action.newBuilder();
action.setCommandDigest(command);
action.setInputRootDigest(inputRoot);
- ArrayList<String> outputPaths = new ArrayList<>();
- ArrayList<String> outputDirectoryPaths = new ArrayList<>();
- for (ActionInput output : outputs) {
- String pathString = output.getExecPathString();
- if (output instanceof Artifact && ((Artifact) output).isTreeArtifact()) {
- outputDirectoryPaths.add(pathString);
- } else {
- outputPaths.add(pathString);
- }
- }
- Collections.sort(outputPaths);
- Collections.sort(outputDirectoryPaths);
- action.addAllOutputFiles(outputPaths);
- action.addAllOutputDirectories(outputDirectoryPaths);
-
- // Get the remote platform properties.
- if (executionPlatform != null) {
- Platform platform =
- parsePlatform(executionPlatform.label(), executionPlatform.remoteExecutionProperties());
- action.setPlatform(platform);
- }
-
if (!timeout.isZero()) {
action.setTimeout(com.google.protobuf.Duration.newBuilder().setSeconds(timeout.getSeconds()));
}
@@ -399,8 +388,33 @@
return platformBuilder.build();
}
- static Command buildCommand(List<String> arguments, ImmutableMap<String, String> env) {
+ static Command buildCommand(
+ Collection<? extends ActionInput> outputs,
+ List<String> arguments,
+ ImmutableMap<String, String> env,
+ @Nullable PlatformInfo executionPlatform) {
Command.Builder command = Command.newBuilder();
+ ArrayList<String> outputFiles = new ArrayList<>();
+ ArrayList<String> outputDirectories = new ArrayList<>();
+ for (ActionInput output : outputs) {
+ String pathString = output.getExecPathString();
+ if (output instanceof Artifact && ((Artifact) output).isTreeArtifact()) {
+ outputDirectories.add(pathString);
+ } else {
+ outputFiles.add(pathString);
+ }
+ }
+ Collections.sort(outputFiles);
+ Collections.sort(outputDirectories);
+ command.addAllOutputFiles(outputFiles);
+ command.addAllOutputDirectories(outputDirectories);
+
+ // Get the remote platform properties.
+ if (executionPlatform != null) {
+ Platform platform =
+ parsePlatform(executionPlatform.label(), executionPlatform.remoteExecutionProperties());
+ command.setPlatform(platform);
+ }
command.addAllArguments(arguments);
// Sorting the environment pairs by variable name.
TreeSet<String> variables = new TreeSet<>(env.keySet());
@@ -430,35 +444,19 @@
return ctimes;
}
- /**
- * Execute a {@link Spawn} locally, using {@link #fallbackRunner}.
- *
- * <p>If possible also upload the {@link SpawnResult} to a remote cache.
- */
- private SpawnResult execLocally(
- Spawn spawn,
- SpawnExecutionContext context,
- SortedMap<PathFragment, ActionInput> inputMap,
- boolean uploadToCache,
- @Nullable AbstractRemoteActionCache remoteCache,
- @Nullable ActionKey actionKey)
- throws ExecException, IOException, InterruptedException {
- if (uploadToCache && remoteCache != null && actionKey != null) {
- return execLocallyAndUpload(spawn, context, inputMap, remoteCache, actionKey);
- }
- return fallbackRunner.get().exec(spawn, context);
- }
-
@VisibleForTesting
SpawnResult execLocallyAndUpload(
Spawn spawn,
SpawnExecutionContext context,
SortedMap<PathFragment, ActionInput> inputMap,
AbstractRemoteActionCache remoteCache,
- ActionKey actionKey)
+ ActionKey actionKey,
+ Action action,
+ Command command,
+ boolean uploadLocalResults)
throws ExecException, IOException, InterruptedException {
Map<Path, Long> ctimesBefore = getInputCtimes(inputMap);
- SpawnResult result = fallbackRunner.get().exec(spawn, context);
+ SpawnResult result = execLocally(spawn, context);
Map<Path, Long> ctimesAfter = getInputCtimes(inputMap);
for (Map.Entry<Path, Long> e : ctimesBefore.entrySet()) {
// Skip uploading to remote cache, because an input was modified during execution.
@@ -466,13 +464,17 @@
return result;
}
}
+ if (!uploadLocalResults) {
+ return result;
+ }
boolean uploadAction =
Spawns.mayBeCached(spawn)
&& Status.SUCCESS.equals(result.status())
&& result.exitCode() == 0;
Collection<Path> outputFiles = resolveActionInputs(execRoot, spawn.getOutputFiles());
try {
- remoteCache.upload(actionKey, execRoot, outputFiles, context.getFileOutErr(), uploadAction);
+ remoteCache.upload(
+ actionKey, action, command, execRoot, outputFiles, context.getFileOutErr(), uploadAction);
} catch (IOException e) {
if (verboseFailures) {
report(Event.debug("Upload to remote cache failed: " + e.getMessage()));
