Encode remaining SimpleSpawnResult failures with FailureDetails
RELNOTES: None.
PiperOrigin-RevId: 318514673
diff --git a/src/main/java/com/google/devtools/build/lib/exec/BUILD b/src/main/java/com/google/devtools/build/lib/exec/BUILD
index 9251478..eabf79f 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/exec/BUILD
@@ -216,6 +216,7 @@
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code",
"//src/main/java/com/google/devtools/build/lib/util:exit_code",
+ "//src/main/protobuf:failure_details_java_proto",
"//third_party:guava",
],
)
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SpawnExecException.java b/src/main/java/com/google/devtools/build/lib/exec/SpawnExecException.java
index 7c77636..ae9b7a9 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/SpawnExecException.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/SpawnExecException.java
@@ -22,6 +22,9 @@
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.SpawnResult;
import com.google.devtools.build.lib.actions.SpawnResult.Status;
+import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
+import com.google.devtools.build.lib.server.FailureDetails.Spawn;
+import com.google.devtools.build.lib.server.FailureDetails.Spawn.Code;
import com.google.devtools.build.lib.util.DetailedExitCode;
import com.google.devtools.build.lib.util.ExitCode;
@@ -77,7 +80,12 @@
ExitCode exitCode =
result.status().isConsideredUserError() ? ExitCode.BUILD_FAILURE : ExitCode.REMOTE_ERROR;
if (result.failureDetail() == null) {
- return DetailedExitCode.justExitCode(exitCode);
+ return DetailedExitCode.of(
+ exitCode,
+ FailureDetail.newBuilder()
+ .setMessage("spawn failed")
+ .setSpawn(Spawn.newBuilder().setCode(Code.UNSPECIFIED_EXECUTION_FAILURE))
+ .build());
}
return DetailedExitCode.of(exitCode, result.failureDetail());
}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
index 7966e08..649e24a 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
@@ -74,6 +74,8 @@
import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
import com.google.devtools.build.lib.remote.util.Utils;
import com.google.devtools.build.lib.remote.util.Utils.InMemoryOutput;
+import com.google.devtools.build.lib.server.FailureDetails;
+import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.util.ExitCode;
import com.google.devtools.build.lib.util.io.FileOutErr;
import com.google.devtools.build.lib.vfs.Path;
@@ -602,16 +604,31 @@
.setRunnerName(getName())
.setStatus(Status.TIMEOUT)
.setExitCode(SpawnResult.POSIX_TIMEOUT_EXIT_CODE)
+ .setFailureDetail(
+ FailureDetail.newBuilder()
+ .setMessage("remote spawn timed out")
+ .setSpawn(
+ FailureDetails.Spawn.newBuilder()
+ .setCode(FailureDetails.Spawn.Code.TIMEOUT))
+ .build())
.build();
}
}
final Status status;
+ FailureDetails.Spawn.Code detailedCode;
+ boolean catastrophe;
if (RemoteRetrierUtils.causedByStatus(exception, Code.UNAVAILABLE)) {
status = Status.EXECUTION_FAILED_CATASTROPHICALLY;
+ detailedCode = FailureDetails.Spawn.Code.EXECUTION_FAILED;
+ catastrophe = true;
} else if (remoteCacheFailed) {
status = Status.REMOTE_CACHE_FAILED;
+ detailedCode = FailureDetails.Spawn.Code.REMOTE_CACHE_FAILED;
+ catastrophe = false;
} else {
status = Status.EXECUTION_FAILED;
+ detailedCode = FailureDetails.Spawn.Code.EXECUTION_FAILED;
+ catastrophe = false;
}
final String errorMessage;
@@ -627,6 +644,14 @@
.setStatus(status)
.setExitCode(ExitCode.REMOTE_ERROR.getNumericExitCode())
.setFailureMessage(errorMessage)
+ .setFailureDetail(
+ FailureDetail.newBuilder()
+ .setMessage("remote spawn failed: " + errorMessage)
+ .setSpawn(
+ FailureDetails.Spawn.newBuilder()
+ .setCode(detailedCode)
+ .setCatastrophic(catastrophe))
+ .build())
.build();
}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/util/BUILD b/src/main/java/com/google/devtools/build/lib/remote/util/BUILD
index 7612bda..e727163 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/util/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/remote/util/BUILD
@@ -22,6 +22,7 @@
"//src/main/java/com/google/devtools/build/lib/remote/options",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
+ "//src/main/protobuf:failure_details_java_proto",
"//third_party:guava",
"//third_party:jsr305",
"//third_party/grpc:grpc-jar",
diff --git a/src/main/java/com/google/devtools/build/lib/remote/util/Utils.java b/src/main/java/com/google/devtools/build/lib/remote/util/Utils.java
index c6b6357..71847e7 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/util/Utils.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/util/Utils.java
@@ -29,6 +29,9 @@
import com.google.devtools.build.lib.remote.common.CacheNotFoundException;
import com.google.devtools.build.lib.remote.common.RemoteCacheClient.ActionKey;
import com.google.devtools.build.lib.remote.options.RemoteOutputsMode;
+import com.google.devtools.build.lib.server.FailureDetails;
+import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
+import com.google.devtools.build.lib.server.FailureDetails.Spawn.Code;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
@@ -98,6 +101,13 @@
.setCacheHit(cacheHit)
.setSpawnMetrics(spawnMetrics)
.setRemote(true);
+ if (exitCode != 0) {
+ builder.setFailureDetail(
+ FailureDetail.newBuilder()
+ .setMessage("remote spawn failed")
+ .setSpawn(FailureDetails.Spawn.newBuilder().setCode(Code.NON_ZERO_EXIT))
+ .build());
+ }
if (inMemoryOutput != null) {
builder.setInMemoryOutput(inMemoryOutput.getOutput(), inMemoryOutput.getContents());
}
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/AbstractSandboxSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/sandbox/AbstractSandboxSpawnRunner.java
index 886f82c..8807993 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/AbstractSandboxSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/AbstractSandboxSpawnRunner.java
@@ -36,6 +36,7 @@
import com.google.devtools.build.lib.profiler.Profiler;
import com.google.devtools.build.lib.profiler.SilentCloseable;
import com.google.devtools.build.lib.runtime.CommandEnvironment;
+import com.google.devtools.build.lib.server.FailureDetails;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.server.FailureDetails.Sandbox;
import com.google.devtools.build.lib.server.FailureDetails.Sandbox.Code;
@@ -193,11 +194,13 @@
.getErrorStream()
.write(("Action failed to execute: java.io.IOException: " + msg + "\n").getBytes(UTF_8));
outErr.getErrorStream().flush();
+ String message = makeFailureMessage(originalSpawn, sandbox);
return new SpawnResult.Builder()
.setRunnerName(getName())
.setStatus(Status.EXECUTION_FAILED)
.setExitCode(LOCAL_EXEC_ERROR)
- .setFailureMessage(makeFailureMessage(originalSpawn, sandbox))
+ .setFailureMessage(message)
+ .setFailureDetail(createFailureDetail(message, Code.EXECUTION_FAILED))
.build();
}
@@ -206,12 +209,40 @@
boolean wasTimeout =
(useSubprocessTimeout && terminationStatus.timedOut())
|| (!useSubprocessTimeout && wasTimeout(timeout, wallTime));
- int exitCode =
- wasTimeout ? SpawnResult.POSIX_TIMEOUT_EXIT_CODE : terminationStatus.getRawExitCode();
- Status status =
- wasTimeout
- ? Status.TIMEOUT
- : (exitCode == 0) ? Status.SUCCESS : Status.NON_ZERO_EXIT;
+
+ int exitCode;
+ Status status;
+ String failureMessage;
+ FailureDetail failureDetail;
+ if (wasTimeout) {
+ exitCode = SpawnResult.POSIX_TIMEOUT_EXIT_CODE;
+ status = Status.TIMEOUT;
+ failureMessage = makeFailureMessage(originalSpawn, sandbox);
+ failureDetail =
+ FailureDetail.newBuilder()
+ .setMessage(failureMessage)
+ .setSpawn(
+ FailureDetails.Spawn.newBuilder().setCode(FailureDetails.Spawn.Code.TIMEOUT))
+ .build();
+ } else {
+ exitCode = terminationStatus.getRawExitCode();
+ if (exitCode == 0) {
+ status = Status.SUCCESS;
+ failureMessage = "";
+ failureDetail = null;
+ } else {
+ status = Status.NON_ZERO_EXIT;
+ failureMessage = makeFailureMessage(originalSpawn, sandbox);
+ failureDetail =
+ FailureDetail.newBuilder()
+ .setMessage(failureMessage)
+ .setSpawn(
+ FailureDetails.Spawn.newBuilder()
+ .setCode(FailureDetails.Spawn.Code.NON_ZERO_EXIT)
+ .setSpawnExitCode(exitCode))
+ .build();
+ }
+ }
SpawnResult.Builder spawnResultBuilder =
new SpawnResult.Builder()
@@ -219,10 +250,11 @@
.setStatus(status)
.setExitCode(exitCode)
.setWallTime(wallTime)
- .setFailureMessage(
- status != Status.SUCCESS || exitCode != 0
- ? makeFailureMessage(originalSpawn, sandbox)
- : "");
+ .setFailureMessage(failureMessage);
+
+ if (failureDetail != null) {
+ spawnResultBuilder.setFailureDetail(failureDetail);
+ }
Path statisticsPath = sandbox.getStatisticsPath();
if (statisticsPath != null) {
diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java
index 898a6ac..bdb9b1d 100644
--- a/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java
@@ -33,6 +33,7 @@
import com.google.devtools.build.lib.actions.SpawnExecutedEvent;
import com.google.devtools.build.lib.actions.SpawnMetrics;
import com.google.devtools.build.lib.actions.SpawnResult;
+import com.google.devtools.build.lib.actions.SpawnResult.Status;
import com.google.devtools.build.lib.actions.Spawns;
import com.google.devtools.build.lib.actions.UserExecException;
import com.google.devtools.build.lib.events.Event;
@@ -209,19 +210,28 @@
response.getOutputBytes().writeTo(outErr.getErrorStream());
int exitCode = response.getExitCode();
- SpawnResult result =
+ SpawnResult.Builder builder =
new SpawnResult.Builder()
.setRunnerName(getName())
.setExitCode(exitCode)
- .setStatus(
- exitCode == 0 ? SpawnResult.Status.SUCCESS : SpawnResult.Status.NON_ZERO_EXIT)
+ .setStatus(exitCode == 0 ? Status.SUCCESS : Status.NON_ZERO_EXIT)
.setWallTime(wallTime)
.setSpawnMetrics(
SpawnMetrics.Builder.forWorkerExec()
.setTotalTime(wallTime)
.setExecutionWallTime(wallTime)
- .build())
- .build();
+ .build());
+ if (exitCode != 0) {
+ builder.setFailureDetail(
+ FailureDetail.newBuilder()
+ .setMessage("worker spawn failed")
+ .setSpawn(
+ FailureDetails.Spawn.newBuilder()
+ .setCode(FailureDetails.Spawn.Code.NON_ZERO_EXIT)
+ .setSpawnExitCode(exitCode))
+ .build());
+ }
+ SpawnResult result = builder.build();
reporter.post(new SpawnExecutedEvent(spawn, result, startTime));
return result;
}
diff --git a/src/main/protobuf/failure_details.proto b/src/main/protobuf/failure_details.proto
index 7beafc6..d125f39 100644
--- a/src/main/protobuf/failure_details.proto
+++ b/src/main/protobuf/failure_details.proto
@@ -205,6 +205,9 @@
INVALID_TIMEOUT = 9 [(metadata) = { exit_code: 1 }];
INVALID_REMOTE_EXECUTION_PROPERTIES = 10 [(metadata) = { exit_code: 1 }];
NO_USABLE_STRATEGY_FOUND = 11 [(metadata) = { exit_code: 1 }];
+ // TODO(b/138456686): this code should be deprecated when SpawnResult is
+ // refactored to prohibit undetailed failures
+ UNSPECIFIED_EXECUTION_FAILURE = 12 [(metadata) = { exit_code: 1 }];
}
Code code = 1;
@@ -653,6 +656,7 @@
MOUNT_SOURCE_DOES_NOT_EXIST = 7 [(metadata) = { exit_code: 1 }];
MOUNT_SOURCE_TARGET_TYPE_MISMATCH = 8 [(metadata) = { exit_code: 1 }];
MOUNT_TARGET_DOES_NOT_EXIST = 9 [(metadata) = { exit_code: 1 }];
+ EXECUTION_FAILED = 10 [(metadata) = { exit_code: 1 }];
}
Code code = 1;
