Use a MetadataConsumer implementation that assists with tree artifact aggregation.
Remove IOException from MetadataConsumer since it is never thrown.
PiperOrigin-RevId: 320714944
diff --git a/src/main/java/com/google/devtools/build/lib/actions/MetadataConsumer.java b/src/main/java/com/google/devtools/build/lib/actions/MetadataConsumer.java
index d737d54..56313f5 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/MetadataConsumer.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/MetadataConsumer.java
@@ -13,7 +13,7 @@
// limitations under the License.
package com.google.devtools.build.lib.actions;
-import java.io.IOException;
+import java.util.function.BiConsumer;
/**
* Consumes metadata for artifacts.
@@ -21,6 +21,4 @@
* <p>May be called when metadata isn't otherwise propagated at the spawn level.
*/
@FunctionalInterface
-public interface MetadataConsumer {
- void accept(Artifact artifact, FileArtifactValue value) throws IOException;
-}
+public interface MetadataConsumer extends BiConsumer<Artifact, FileArtifactValue> {}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java
index ac888d6..b60c2ed 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java
@@ -53,6 +53,7 @@
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.Artifact.ArtifactExpanderImpl;
import com.google.devtools.build.lib.actions.Artifact.OwnerlessArtifactWrapper;
+import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
import com.google.devtools.build.lib.actions.ArtifactPathResolver;
import com.google.devtools.build.lib.actions.CachedActionEvent;
import com.google.devtools.build.lib.actions.EnvironmentalExecException;
@@ -70,6 +71,7 @@
import com.google.devtools.build.lib.actions.StoppedScanningActionEvent;
import com.google.devtools.build.lib.actions.UserExecException;
import com.google.devtools.build.lib.actions.cache.MetadataHandler;
+import com.google.devtools.build.lib.actions.cache.MetadataInjector;
import com.google.devtools.build.lib.buildtool.BuildRequestOptions;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
@@ -105,6 +107,7 @@
import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.common.options.OptionsProvider;
+import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Collection;
@@ -407,13 +410,13 @@
ActionPostprocessing postprocessing,
boolean hasDiscoveredInputs)
throws ActionExecutionException, InterruptedException {
+ MetadataAggregator metadataAggregator;
if (actionFileSystem != null) {
+ metadataAggregator = new MetadataAggregator(metadataHandler);
updateActionFileSystemContext(
- action,
- actionFileSystem,
- env,
- metadataHandler.getOutputStore()::injectOutputData,
- expandedFilesets);
+ action, actionFileSystem, env, metadataAggregator, expandedFilesets);
+ } else {
+ metadataAggregator = null;
}
ActionExecutionContext actionExecutionContext =
@@ -454,7 +457,8 @@
actionStartTime,
actionExecutionContext,
actionLookupData,
- postprocessing)));
+ postprocessing,
+ metadataAggregator)));
SharedActionCallback callback =
getSharedActionCallback(env.getListener(), hasDiscoveredInputs, action, actionLookupData);
@@ -535,17 +539,10 @@
Action action,
@Nullable ActionExecutionException finalException)
throws ActionExecutionException {
- if (finalException != null) {
- try {
- context.close();
- } catch (IOException | RuntimeException e) {
- finalException.addSuppressed(e);
+ try (Closeable c = context) {
+ if (finalException != null) {
+ throw finalException;
}
- throw finalException;
- }
-
- try {
- context.close();
} catch (IOException e) {
String message = "Failed to close action output: " + e.getMessage();
DetailedExitCode code = createDetailedExitCode(message, Code.ACTION_OUTPUT_CLOSE_FAILURE);
@@ -867,6 +864,7 @@
private final ActionLookupData actionLookupData;
private final ActionExecutionStatusReporter statusReporter;
private final ActionPostprocessing postprocessing;
+ @Nullable private final MetadataAggregator metadataAggregator;
ActionRunner(
Action action,
@@ -874,7 +872,8 @@
long actionStartTime,
ActionExecutionContext actionExecutionContext,
ActionLookupData actionLookupData,
- ActionPostprocessing postprocessing) {
+ ActionPostprocessing postprocessing,
+ @Nullable MetadataAggregator metadataAggregator) {
this.action = action;
this.metadataHandler = metadataHandler;
this.actionStartTime = actionStartTime;
@@ -882,6 +881,7 @@
this.actionLookupData = actionLookupData;
this.statusReporter = statusReporterRef.get();
this.postprocessing = postprocessing;
+ this.metadataAggregator = metadataAggregator;
}
@SuppressWarnings("LogAndThrow") // Thrown exception shown in user output, not info logs.
@@ -1101,6 +1101,10 @@
Preconditions.checkState(action.inputsDiscovered(),
"Action %s successfully executed, but inputs still not known", action);
+ if (metadataAggregator != null) {
+ metadataAggregator.finish();
+ }
+
if (!checkOutputs(action, metadataHandler)) {
throw toActionExecutionException(
"not all outputs were created or valid",
@@ -1753,4 +1757,31 @@
return input != null ? input : perBuildFileCache.getInput(execPath);
}
}
+
+ /**
+ * Assists with aggregation of tree artifacts for an action file system which is only aware of
+ * individual outputs.
+ */
+ private static final class MetadataAggregator implements MetadataConsumer {
+ private final TreeArtifactValue.MultiBuilder treeArtifacts =
+ TreeArtifactValue.newConcurrentMultiBuilder();
+ private final MetadataInjector metadataInjector;
+
+ MetadataAggregator(MetadataInjector metadataInjector) {
+ this.metadataInjector = metadataInjector;
+ }
+
+ @Override
+ public void accept(Artifact output, FileArtifactValue metadata) {
+ if (output.isChildOfDeclaredDirectory()) {
+ treeArtifacts.putChild((TreeFileArtifact) output, metadata);
+ } else {
+ metadataInjector.injectFile(output, metadata);
+ }
+ }
+
+ void finish() {
+ treeArtifacts.injectTo(metadataInjector);
+ }
+ }
}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileInfo.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileInfo.java
index 6f148aa..c4fe9a8 100644
--- a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileInfo.java
+++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileInfo.java
@@ -25,6 +25,7 @@
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.ReadableByteChannel;
+import java.util.function.Consumer;
/**
* InMemoryFileInfo manages file contents by storing them entirely in memory.
@@ -111,10 +112,10 @@
* A {@link ByteArrayOutputStream} which notifiers a callback when it has flushed its data.
*/
public static class InMemoryOutputStream extends ByteArrayOutputStream {
- private final IOByteReceiver receiver;
+ private final Consumer<byte[]> receiver;
private boolean closed = false;
- public InMemoryOutputStream(IOByteReceiver receiver) {
+ public InMemoryOutputStream(Consumer<byte[]> receiver) {
this.receiver = receiver;
}
@@ -137,22 +138,14 @@
}
@Override
- public synchronized void close() throws IOException {
+ public synchronized void close() {
flush();
closed = true;
}
@Override
- public synchronized void flush() throws IOException {
+ public synchronized void flush() {
receiver.accept(toByteArray().clone());
}
}
-
- /**
- * Similar to {@link com.google.common.base.Receiver}, but allows implementations to throw
- * {@link IOException}.
- */
- public interface IOByteReceiver {
- void accept(byte[] bytes) throws IOException;
- }
}
