[9.3.0] Bust caches for rewound actions (#31300) (#31420) ### Description When an action is rewound, it needs to bust both intra-build Bazel caches as well as remote caches to ensure that the rewinding is effective: * When a remote cache doesn't verify that the `ActionResult`s it returns reference existing CAS blobs, rewinding can get into a loop and fail when the maximum number of retries is exhausted. `knownMissingCasDigests` handled this for build rewinding, but by design can't accommodate intra-build recovery (as observed in https://github.com/bazelbuild/bazel/pull/31251). Instead, just disable remote caching for rewound actions until a better signal is available (see the TODOs and https://github.com/bazelbuild/remote-apis/pull/386). * Bazel's upload deduplication and caching could result in an action that's been rewound multiple times not reuploading its local outputs. Such caches and deduplication are now ignored for rewound actions. `LostInputsEvent` and `knownMissingCasDigests` remain in use only for the legacy whole-invocation retry path when action rewinding is disabled. The extra wiring required to propagate `LostInputsEvent` for a rewound action has been removed. ### Motivation ### Build API Changes No ### Checklist - [x] I have added tests for the new use cases (if any). - [ ] I have updated the documentation (if applicable). ### Release Notes RELNOTES: None Closes #31300 COPYBARA_INTEGRATE_REVIEW=https://github.com/bazelbuild/bazel/pull/31300 from fmeum:fix-missing-cas-digest-lifecycle 9055476e5fb1a2af007bb6d33077af4b7099fe44 PiperOrigin-RevId: 990587446 Change-Id: I8e14f5bea2fbfb584756e7e16ccd925f8d57c945 <!-- Thank you for contributing to Bazel! Please read the contribution guidelines: https://bazel.build/contribute --> ### Description <!-- Please provide a brief summary of the changes in this PR. --> ### Motivation <!-- Why is this change important? Does it fix a specific bug or add a new feature? If this PR fixes an existing issue, please link it here (e.g. "Fixes #1234"). --> ### Build API Changes <!-- Does this PR affect the Build API? (e.g. Starlark API, providers, command-line flags, native rules) If yes, please answer the following: 1. Has this been discussed in a design doc or issue? (Please link it) 2. Is the change backward compatible? 3. If it's a breaking change, what is the migration plan? --> No ### Checklist - [ ] I have added tests for the new use cases (if any). - [ ] I have updated the documentation (if applicable). ### Release Notes <!-- If this is a new feature, please add 'RELNOTES[NEW]: <description>' here. If this is a breaking change, please add 'RELNOTES[INC]: <reason>' here. If this change should be mentioned in release notes, please add 'RELNOTES: <reason>' here. --> RELNOTES: None Commit https://github.com/bazelbuild/bazel/commit/f653dd6d08351de86caafc684af6ed6abc975df8 Co-authored-by: Fabian Meumertzheim <fabian@meumertzhe.im>
diff --git a/src/main/java/com/google/devtools/build/lib/remote/ChunkedBlobUploader.java b/src/main/java/com/google/devtools/build/lib/remote/ChunkedBlobUploader.java index 384b4cdf..13de181 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/ChunkedBlobUploader.java +++ b/src/main/java/com/google/devtools/build/lib/remote/ChunkedBlobUploader.java
@@ -91,8 +91,12 @@ * Uploads a blob in content-defined chunks. The file is chunked with the configured chunking * function, missing chunks are uploaded, and {@code SpliceBlob} is called to register the blob as * the concatenation of its chunks. + * + * @param force whether to upload missing chunks even if the cache client has already completed + * uploads of them */ - public void uploadChunked(RemoteActionExecutionContext context, Digest blobDigest, Path file) + public void uploadChunked( + RemoteActionExecutionContext context, Digest blobDigest, Path file, boolean force) throws IOException, InterruptedException { List<Digest> chunkDigests; try (InputStream input = file.getInputStream()) { @@ -104,7 +108,7 @@ ImmutableSet<Digest> missingDigests = getFromFuture(grpcCacheClient.findMissingDigests(context, chunkDigests)); - uploadMissingChunks(context, missingDigests, chunkDigests, file); + uploadMissingChunks(context, missingDigests, chunkDigests, file, force); getFromFuture(grpcCacheClient.spliceBlob(context, blobDigest, chunkDigests, chunkingFunction)); } @@ -112,12 +116,13 @@ RemoteActionExecutionContext context, ImmutableSet<Digest> missingDigests, List<Digest> chunkDigests, - Path file) + Path file, + boolean force) throws IOException, InterruptedException { if (missingDigests.isEmpty()) { return; } - new UploadSession(context, missingDigests, chunkDigests).run(file); + new UploadSession(context, missingDigests, chunkDigests, force).run(file); } private final class UploadSession { @@ -129,14 +134,17 @@ private final RemoteActionExecutionContext context; private final ImmutableSet<Digest> missingDigests; private final List<Digest> chunkDigests; + private final boolean force; UploadSession( RemoteActionExecutionContext context, ImmutableSet<Digest> missingDigests, - List<Digest> chunkDigests) { + List<Digest> chunkDigests, + boolean force) { this.context = context; this.missingDigests = missingDigests; this.chunkDigests = chunkDigests; + this.force = force; } void run(Path file) throws IOException, InterruptedException { @@ -169,7 +177,7 @@ private void startUpload(Path file, long chunkOffset, Digest chunkDigest) { ListenableFuture<Void> upload = combinedCache.uploadBlob( - context, chunkDigest, new ChunkBlob(file, chunkOffset, chunkDigest)); + context, chunkDigest, new ChunkBlob(file, chunkOffset, chunkDigest), force); inFlightUploads.add(upload); upload.addListener(() -> completedUploads.add(upload), directExecutor()); }
diff --git a/src/main/java/com/google/devtools/build/lib/remote/CombinedCache.java b/src/main/java/com/google/devtools/build/lib/remote/CombinedCache.java index 27f56a9..f6e353b 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/CombinedCache.java +++ b/src/main/java/com/google/devtools/build/lib/remote/CombinedCache.java
@@ -357,18 +357,28 @@ .call(() -> null, directExecutor()); } + /** Upload a local file to the remote and/or disk cache. */ + public ListenableFuture<Void> uploadFile( + RemoteActionExecutionContext context, Digest digest, Path file) { + return uploadFile(context, digest, file, /* force= */ false); + } + /** * Upload a local file to the remote cache. * * <p>Trying to upload the same file multiple times concurrently, results in only one upload being - * performed. + * performed. An upload to the remote cache is also skipped if this instance has already completed + * an upload of the same blob, unless {@code force} is set. * * @param context the context for the action. * @param digest the digest of the file. * @param file the file to upload. + * @param force whether to upload to the remote cache even if this instance has already completed + * an upload of the same blob, e.g. because it may have been evicted since. Concurrent uploads + * are still deduplicated. */ public ListenableFuture<Void> uploadFile( - RemoteActionExecutionContext context, Digest digest, Path file) { + RemoteActionExecutionContext context, Digest digest, Path file, boolean force) { if (digest.getSizeBytes() == 0) { return COMPLETED_SUCCESS; } @@ -390,9 +400,9 @@ if (chunkingSupported && digest.getSizeBytes() > chunking.config().chunkingThreshold()) { remoteCacheFuture = remoteCacheClient.dedupUpload( - digest, () -> uploadChunked(context, digest, file), /* force= */ false); + digest, () -> uploadChunked(context, digest, file, force), force); } else { - remoteCacheFuture = remoteCacheClient.uploadFile(context, digest, file, /* force= */ false); + remoteCacheFuture = remoteCacheClient.uploadFile(context, digest, file, force); } } @@ -401,10 +411,10 @@ } private ListenableFuture<Void> uploadChunked( - RemoteActionExecutionContext context, Digest digest, Path file) { + RemoteActionExecutionContext context, Digest digest, Path file, boolean force) { return virtualThreadExecutor.submit( () -> { - chunking.uploader().uploadChunked(context, digest, file); + chunking.uploader().uploadChunked(context, digest, file, force); return null; }); } @@ -432,6 +442,21 @@ */ public ListenableFuture<Void> uploadBlob( RemoteActionExecutionContext context, Digest digest, Blob blob) { + return uploadBlob(context, digest, blob, /* force= */ false); + } + + /** + * Uploads a blob to the cache from a repeatable stream supplier. + * + * <p>The supplier may be opened more than once, including concurrently when both disk and remote + * cache writes are enabled. + * + * @param force whether to upload to the remote cache even if this instance has already completed + * an upload of the same blob, e.g. because it may have been evicted since. Concurrent uploads + * are still deduplicated. + */ + public ListenableFuture<Void> uploadBlob( + RemoteActionExecutionContext context, Digest digest, Blob blob, boolean force) { if (digest.getSizeBytes() == 0) { return COMPLETED_SUCCESS; } @@ -443,7 +468,7 @@ ListenableFuture<Void> remoteCacheFuture = Futures.immediateVoidFuture(); if (remoteCacheClient != null && context.getWriteCachePolicy().allowRemoteCache()) { - remoteCacheFuture = remoteCacheClient.uploadBlob(context, digest, blob, /* force= */ false); + remoteCacheFuture = remoteCacheClient.uploadBlob(context, digest, blob, force); } return Futures.whenAllSucceed(diskCacheFuture, remoteCacheFuture)
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java index 19b902c..a8efcc0 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java
@@ -184,7 +184,8 @@ captureCorruptedOutputsDir, remoteOutputChecker, outputService, - knownMissingCasDigests); + knownMissingCasDigests, + env.getSkyframeExecutor()::wasActionRewound); env.getEventBus().register(remoteExecutionService); }
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java index 42d0a7a..e0f56e6 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java
@@ -67,6 +67,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; +import com.google.devtools.build.lib.actions.ActionAnalysisMetadata; import com.google.devtools.build.lib.actions.ActionExecutionMetadata; import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.Artifact; @@ -162,6 +163,7 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; import java.util.stream.Stream; import javax.annotation.Nullable; @@ -209,6 +211,7 @@ @Nullable private final Scrubber scrubber; private final Set<Digest> knownMissingCasDigests; + private final Predicate<ActionAnalysisMetadata> wasRewound; private Boolean useOutputPaths; @@ -229,7 +232,8 @@ @Nullable Path captureCorruptedOutputsDir, @Nullable RemoteOutputChecker remoteOutputChecker, OutputService outputService, - Set<Digest> knownMissingCasDigests) { + Set<Digest> knownMissingCasDigests, + Predicate<ActionAnalysisMetadata> wasRewound) { this.reporter = reporter; this.verboseFailures = verboseFailures; this.execRoot = execRoot; @@ -260,6 +264,7 @@ this.remoteOutputChecker = remoteOutputChecker; this.outputService = outputService; this.knownMissingCasDigests = knownMissingCasDigests; + this.wasRewound = wasRewound; } private Command buildCommand( @@ -768,6 +773,15 @@ action.getRemoteActionExecutionContext().getReadCachePolicy().allowAnyCache(), "spawn doesn't accept cached result"); + // A rewound action must regenerate its lost outputs even if the cache still serves its stale + // action result. Treat this as a cache miss. + // TODO(https://github.com/bazelbuild/remote-apis/pull/386): Allow cache lookup for rewound + // actions when CacheCapabilities.verifies_action_results is advertised, while still bypassing + // unverified cache layers. + if (wasRewound.test(action.getSpawn().getResourceOwner())) { + return null; + } + ImmutableSet<String> inlineOutputFiles = ImmutableSet.of(); PathFragment inMemoryOutputPath = getInMemoryOutputPath(action.getSpawn()); if (inMemoryOutputPath != null) { @@ -788,9 +802,9 @@ var result = RemoteActionResult.createFromCache(cachedActionResult); - // We only add digests to `knownMissingCasDigests` when LostInputsEvent occurs which will cause - // the build to abort and rewind, so there is no data race here. This allows us to avoid the - // check until cache eviction happens. + // The legacy whole-invocation retry path uses knownMissingCasDigests to prevent a retried + // invocation from accepting the same stale action result. Action rewinding bypasses cache + // lookup above without populating this set. if (!knownMissingCasDigests.isEmpty()) { ActionResultMetadata metadata; try { @@ -1769,11 +1783,16 @@ return; } + // The output blobs of a rewound action may have been evicted after this invocation uploaded + // them, so bypass the deduplication of completed uploads. Otherwise, the refreshed action + // result could still reference missing blobs. Evaluate this here rather than in the upload + // task, which may run after the build's rewinding state has been discarded. + boolean force = wasRewound.test(action.getSpawn().getResourceOwner()); if (remoteOptions.remoteCacheAsync && !action.getSpawn().getResourceOwner().mayModifySpawnOutputsAfterExecution()) { - new OutputUploadTask(action, spawnResult, onUploadComplete).start(); + new OutputUploadTask(action, spawnResult, force, onUploadComplete).start(); } else { - doUploadOutputs(action, spawnResult, onUploadComplete); + doUploadOutputs(action, spawnResult, force, onUploadComplete); } } @@ -1807,12 +1826,13 @@ // happens-before edge to the reads in run() on the executor thread. private Runnable unregisterHandle = () -> {}; - OutputUploadTask(RemoteAction action, SpawnResult spawnResult, Runnable onUploadComplete) { + OutputUploadTask( + RemoteAction action, SpawnResult spawnResult, boolean force, Runnable onUploadComplete) { this.upload = new CancellableTask<>( () -> { try { - doUploadOutputs(action, spawnResult, /* onUploadComplete= */ () -> {}); + doUploadOutputs(action, spawnResult, force, /* onUploadComplete= */ () -> {}); } catch (ExecException e) { reportUploadError(e); } @@ -1871,13 +1891,13 @@ } private void doUploadOutputs( - RemoteAction action, SpawnResult spawnResult, Runnable onUploadComplete) + RemoteAction action, SpawnResult spawnResult, boolean force, Runnable onUploadComplete) throws ExecException, InterruptedException { try (SilentCloseable c = Profiler.instance().profile(ProfilerTask.UPLOAD_TIME, "upload outputs")) { UploadManifest manifest = buildUploadManifest(action, spawnResult); var unused = - manifest.upload(action.getRemoteActionExecutionContext(), combinedCache, reporter); + manifest.upload(action.getRemoteActionExecutionContext(), combinedCache, reporter, force); } catch (IOException e) { reportUploadError(e); } finally { @@ -2005,7 +2025,11 @@ .setInstanceName(remoteOptions.remoteInstanceName) .setDigestFunction(digestUtil.getDigestFunction()) .setActionDigest(action.getActionKey().digest()) - .setSkipCacheLookup(!acceptCachedResult); + // TODO(https://github.com/bazelbuild/remote-apis/pull/386): Allow cached execution + // results for rewound actions when ExecutionCapabilities.verifies_action_results is + // advertised. + .setSkipCacheLookup( + !acceptCachedResult || wasRewound.test(action.getSpawn().getResourceOwner())); if (remoteOptions.remoteResultCachePriority != 0) { requestBuilder .getResultsCachePolicyBuilder()
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java index 954018b..633cae1 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
@@ -141,6 +141,8 @@ MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1)); private final ThreadPoolExecutor executorService; + // TODO: Remove this set when whole-invocation retries no longer handle lost action inputs. + // RemoteExecutionService skips cache lookup for rewound actions instead. private final Set<Digest> knownMissingCasDigests = Sets.newConcurrentHashSet(); private boolean useRemoteRepoContentsCache;
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteRepoContentsCacheImpl.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteRepoContentsCacheImpl.java index 0bdc1ce..c389493 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteRepoContentsCacheImpl.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteRepoContentsCacheImpl.java
@@ -213,7 +213,7 @@ /* startTime= */ Instant.now(), /* wallTimeInMs= */ 0, /* preserveExecutableBit= */ true) - .upload(context, cache, reporter); + .upload(context, cache, reporter, /* force= */ false); } catch (ExecException | IOException e) { reporter.handle( Event.warn(
diff --git a/src/main/java/com/google/devtools/build/lib/remote/UploadManifest.java b/src/main/java/com/google/devtools/build/lib/remote/UploadManifest.java index 220e771..0a70fca 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/UploadManifest.java +++ b/src/main/java/com/google/devtools/build/lib/remote/UploadManifest.java
@@ -600,11 +600,17 @@ return result.build(); } - /** Uploads outputs and action result (if exit code is 0) to the remote and/or disk cache. */ + /** + * Uploads outputs and action result (if exit code is 0) to the remote and/or disk cache. + * + * @param force whether to upload blobs to the remote cache even if {@code combinedCache} has + * already completed uploads of them, e.g. because they may have been evicted since + */ public ActionResult upload( RemoteActionExecutionContext context, CombinedCache combinedCache, - ExtendedEventHandler reporter) + ExtendedEventHandler reporter, + boolean force) throws IOException, InterruptedException, ExecException { ActionExecutionMetadata action = context.getSpawnOwner(); var allDigests = Sets.union(digestToBlobs.keySet(), digestToFile.keySet()).immutableCopy(); @@ -621,7 +627,7 @@ for (var digest : missingDigests) { uploadFutures.add( decorateUploadFuture( - uploadSingleDigest(context, combinedCache, digest), + uploadSingleDigest(context, combinedCache, digest, force), reporter, action, Store.CAS, @@ -650,10 +656,13 @@ } private ListenableFuture<Void> uploadSingleDigest( - RemoteActionExecutionContext context, CombinedCache combinedCache, Digest digest) { + RemoteActionExecutionContext context, + CombinedCache combinedCache, + Digest digest, + boolean force) { Path file = digestToFile.get(digest); if (file != null) { - return combinedCache.uploadFile(context, digest, file); + return combinedCache.uploadFile(context, digest, file, force); } ByteString blob = digestToBlobs.get(digest); @@ -662,7 +671,7 @@ new IOException("FindMissingBlobs call returned an unknown digest: " + digest)); } - return combinedCache.uploadBlob(context, digest, blob); + return combinedCache.uploadBlob(context, digest, blob::newInput, force); } @CanIgnoreReturnValue
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/BUILD index d31e913..9c1919b 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/BUILD +++ b/src/main/java/com/google/devtools/build/lib/skyframe/BUILD
@@ -2245,10 +2245,7 @@ java_library( name = "progress_suppressing_event_handler", srcs = ["ProgressSuppressingEventHandler.java"], - deps = [ - "//src/main/java/com/google/devtools/build/lib/events", - "//src/main/java/com/google/devtools/build/lib/remote/common:lost_inputs_event", - ], + deps = ["//src/main/java/com/google/devtools/build/lib/events"], ) java_library(
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ProgressSuppressingEventHandler.java b/src/main/java/com/google/devtools/build/lib/skyframe/ProgressSuppressingEventHandler.java index cb297c2..3fbcff1 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/ProgressSuppressingEventHandler.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ProgressSuppressingEventHandler.java
@@ -16,7 +16,6 @@ import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.ExtendedEventHandler; -import com.google.devtools.build.lib.remote.common.LostInputsEvent; /** * Suppresses {@link #post} when the provided {@link Postable} represents a progress event (denoted @@ -32,9 +31,7 @@ @Override public void post(Postable obj) { - // A LostInputsEvent isn't replayable, but still has to be delivered to ensure that rewound - // actions can recover their lost inputs by triggering further rewinding. - if (obj.storeForReplay() || obj instanceof LostInputsEvent) { + if (obj.storeForReplay()) { delegate.post(obj); } }
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java index 8194c8a..93059ab 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java
@@ -499,7 +499,10 @@ /** Determines whether the given action was rewound during the current build. */ public boolean wasRewound(ActionAnalysisMetadata action) { - return rewoundActions.contains(new OwnerlessArtifactWrapper(action.getPrimaryOutput())); + Artifact primaryOutput = action.getPrimaryOutput(); + // Only GrepIncludesAction (from include scanning) has a null primary output. + return primaryOutput != null + && rewoundActions.contains(new OwnerlessArtifactWrapper(primaryOutput)); } /**
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java index ef4d0a9..d089137 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
@@ -3102,6 +3102,11 @@ this.statusReporterRef.set(statusReporter); } + /** Determines whether the given action was rewound during the current build. */ + public boolean wasActionRewound(ActionAnalysisMetadata action) { + return skyframeActionExecutor.wasRewound(action); + } + public abstract void detectModifiedOutputFiles( ModifiedFileSet modifiedOutputFiles, @Nullable Range<Long> lastExecutionTimeRange,
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/rewinding/ActionRewindStrategy.java b/src/main/java/com/google/devtools/build/lib/skyframe/rewinding/ActionRewindStrategy.java index a7b46d0..a035a89 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/rewinding/ActionRewindStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/rewinding/ActionRewindStrategy.java
@@ -292,15 +292,19 @@ LostType lostType, ExtendedEventHandler listener) throws ActionRewindException { - // Bazel's (but not Blaze's) remote implementation needs to learn about lost digests to - // invalidate caches, both for rewinding and for invocation retries in BlazeCommandDispatcher. - listener.post(new LostInputsEvent(lostArtifacts.keySet())); if (skyframeActionExecutor.rewindingEnabled()) { + // Action rewinding takes precedence over whole-invocation retries when both are enabled. + // Remote execution skips cache lookup for rewound actions without tracking lost digests. + // Other transient cache errors, including evicted repository files read during loading, + // can still trigger invocation retries in BlazeCommandDispatcher without passing here. return; } if (skyframeActionExecutor.invocationRetriesEnabled()) { - // If rewinding failed, Bazel may still be able to recover by retrying the invocation in - // BlazeCommandDispatcher if retries are enabled. + // Bazel's (but not Blaze's) remote implementation needs to learn about lost digests so that + // the retried invocation doesn't accept the same stale action result. + listener.post(new LostInputsEvent(lostArtifacts.keySet())); + // When action rewinding is disabled, recover by retrying the invocation in + // BlazeCommandDispatcher instead. throw new FallbackToBuildRewindingException( lostArtifacts.entries().stream() .limit(MAX_LOST_INPUTS_RECORDED)
diff --git a/src/test/java/com/google/devtools/build/lib/remote/BUILD b/src/test/java/com/google/devtools/build/lib/remote/BUILD index 59a3f2e..95f39f2 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/BUILD +++ b/src/test/java/com/google/devtools/build/lib/remote/BUILD
@@ -296,6 +296,7 @@ ":build_without_the_bytes_integration_test_base", "//src/main/java/com/google/devtools/build/lib:runtime", "//src/main/java/com/google/devtools/build/lib/actions", + "//src/main/java/com/google/devtools/build/lib/actions:action_lookup_data", "//src/main/java/com/google/devtools/build/lib/actions:artifacts", "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module", "//src/main/java/com/google/devtools/build/lib/dynamic", @@ -305,8 +306,11 @@ "//src/main/java/com/google/devtools/build/lib/util:os", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", + "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", "//src/main/protobuf:failure_details_java_proto", "//src/test/java/com/google/devtools/build/lib/remote/util:integration_test_utils", + "//src/test/java/com/google/devtools/build/lib/skyframe/rewinding:rewinding_tests_helper", + "//src/test/java/com/google/devtools/build/lib/testutil:action_event_recorder", "//third_party:guava", "//third_party:junit4", "//third_party:truth",
diff --git a/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTest.java b/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTest.java index f0e081b..58e12dc 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTest.java
@@ -22,8 +22,10 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; +import com.google.devtools.build.lib.actions.ActionLookupData; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.BuildFailedException; +import com.google.devtools.build.lib.actions.SpawnResult; import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule; import com.google.devtools.build.lib.dynamic.DynamicExecutionModule; import com.google.devtools.build.lib.remote.util.IntegrationTestUtils; @@ -33,15 +35,19 @@ import com.google.devtools.build.lib.runtime.BlockWaitingModule; import com.google.devtools.build.lib.runtime.BuildSummaryStatsModule; import com.google.devtools.build.lib.server.FailureDetails; +import com.google.devtools.build.lib.skyframe.rewinding.RewindingTestsHelper; import com.google.devtools.build.lib.standalone.StandaloneModule; +import com.google.devtools.build.lib.testutil.ActionEventRecorder; import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.vfs.Symlinks; +import com.google.devtools.build.skyframe.SkyKey; import com.google.testing.junit.testparameterinjector.TestParameter; import com.google.testing.junit.testparameterinjector.TestParameterInjector; import java.io.IOException; +import java.util.List; import java.util.UUID; import org.junit.ClassRule; import org.junit.Rule; @@ -53,6 +59,22 @@ public class BuildWithoutTheBytesIntegrationTest extends BuildWithoutTheBytesIntegrationTestBase { @ClassRule @Rule public static final WorkerInstance worker = IntegrationTestUtils.createWorker(); + private final RewindingTestsHelper rewindingTestsHelper = + new RewindingTestsHelper(this, new ActionEventRecorder()); + + private static void assertRewoundActions(List<SkyKey> rewoundKeys, String... expectedLabels) { + assertThat( + rewoundKeys.stream() + .filter(key -> key instanceof ActionLookupData) + .map( + key -> + ((ActionLookupData) key) + .getActionLookupKey() + .getLabel() + .getCanonicalForm())) + .containsExactlyElementsIn(ImmutableList.copyOf(expectedLabels)); + } + @TestParameter public boolean useDiskCache; @Override @@ -982,18 +1004,19 @@ @Test public void actionRewinding_chainedLostInputsWithStaleActionCacheEntries_recovers( - @TestParameter boolean actionCacheIntegrityCheck) throws Exception { - // A rewound action that itself observes a lost input must report the lost digest just like an - // action that hasn't been rewound. Otherwise, if the worker serves cached action results even - // if the blobs they reference are missing from the CAS, the rewound generating action accepts - // the stale action result and the two actions keep rewinding each other until the limit on - // repeated lost inputs fails the build. + @TestParameter boolean actionCacheIntegrityCheck, @TestParameter boolean invocationRetries) + throws Exception { + // A rewound action that itself observes a lost input must rewind that input's producer too. + // Each rewound action must skip its stale cache entry to avoid repeatedly losing the same + // input. var chainWorker = IntegrationTestUtils.createWorker( "--action_cache_integrity_check=" + actionCacheIntegrityCheck); try (var ignored = chainWorker.start()) { addOptions("--remote_executor=grpc://localhost:" + chainWorker.getPort()); enableActionRewinding(); + // Action rewinding takes precedence, even if whole-invocation retries are also enabled. + addOptions("--experimental_remote_cache_eviction_retries=" + (invocationRetries ? 5 : 0)); write( "a/BUILD", """ @@ -1038,9 +1061,283 @@ // which in turn discovers the other lost input and rewinds //a:foo. write("a/baz.in", "baz2"); setDownloadToplevel(); + var rewoundKeys = rewindingTestsHelper.collectOrderedRewoundKeys(); buildTarget("//a:baz"); assertValidOutputFile("a/baz.out", "foobarbaz2\n"); + // Cache lookup is bypassed for each rewound action. Otherwise //a:baz would discover bar.out + // lost again and rewind //a:bar once more. + assertRewoundActions(rewoundKeys, "//a:bar", "//a:foo"); + } + } + + @Test + public void actionRewinding_cacheOnlyLocalExecution_skipsStaleCacheEntry( + @TestParameter boolean uploadLocalResults) throws Exception { + // Exercise cache-only local execution after clean(), including recovery without uploads. + var unverifiedWorker = IntegrationTestUtils.createWorker("--noaction_cache_integrity_check"); + try (var ignored = unverifiedWorker.start()) { + addOptions( + "--remote_executor=", "--remote_cache=grpc://localhost:" + unverifiedWorker.getPort()); + enableActionRewinding(); + write( + "a/BUILD", + """ + genrule( + name = "foo", + srcs = [], + outs = ["foo.out"], + cmd = "echo -n foo > $@", + ) + + genrule( + name = "bar", + srcs = [ + ":foo.out", + "bar.in", + ], + outs = ["bar.out"], + cmd = "cat $(location :foo.out) $(location bar.in) > $@", + ) + """); + write("a/bar.in", "one"); + + buildTarget("//a:bar"); + + // Delete foo.out locally. + clean(); + // Delete foo.out remotely, but keep the AC for foo. + unverifiedWorker.evictBlob("foo".getBytes(UTF_8)); + // Invalidate //a:bar, so its execution finds foo.out lost and rewinds //a:foo. + write("a/bar.in", "two"); + setDownloadToplevel(); + if (useDiskCache) { + // Prevent the disk cache from restoring the deleted blobs. + addOptions("--disk_cache=" + UUID.randomUUID()); + } + addOptions("--remote_upload_local_results=" + uploadLocalResults); + var rewoundKeys = rewindingTestsHelper.collectOrderedRewoundKeys(); + + buildTarget("//a:bar"); + + assertRewoundActions(rewoundKeys, "//a:foo"); + assertValidOutputFile("a/bar.out", "footwo\n"); + assertThat(unverifiedWorker.hasCasBlob("foo".getBytes(UTF_8))).isEqualTo(uploadLocalResults); + } + } + + @Test + public void actionRewinding_unrelatedFailure_reusesRepairedCacheEntry() throws Exception { + // After rewinding and an unrelated build failure, restoring the lost blob should let the next + // build accept the cache entry again. + assumeFalse(useDiskCache); + var unverifiedWorker = IntegrationTestUtils.createWorker("--noaction_cache_integrity_check"); + try (var ignored = unverifiedWorker.start()) { + addOptions("--remote_executor=grpc://localhost:" + unverifiedWorker.getPort()); + enableActionRewinding(); + write( + "a/BUILD", + """ + genrule( + name = "bar", + srcs = [], + outs = ["bar.out"], + cmd = "echo -n bar > $@", + ) + + genrule( + name = "consumer", + srcs = [ + ":bar.out", + "consumer.in", + ], + outs = ["consumer.out"], + cmd = "cat $(location :bar.out) $(location consumer.in) > $@", + ) + + genrule( + name = "fail", + srcs = [":consumer.out"], + outs = ["fail.out"], + cmd = "exit 1", + ) + """); + write("a/consumer.in", "one"); + + buildTarget("//a:consumer"); + + // Delete the blob backing bar.out, keeping //a:bar's action cache entry. + byte[] barContents = "bar".getBytes(UTF_8); + unverifiedWorker.evictBlob(barContents); + + // Invalidate only //a:consumer, so its execution finds bar.out lost and rewinds //a:bar. + write("a/consumer.in", "two"); + setDownloadToplevel(); + addOptions( + "--strategy_regexp=.*=standalone", + "--notrack_incremental_state", + "--remote_upload_local_results=false"); + var rewoundKeys = rewindingTestsHelper.collectOrderedRewoundKeys(); + assertThrows(BuildFailedException.class, () -> buildTarget("//a:fail")); + assertRewoundActions(rewoundKeys, "//a:bar"); + + // Put the blob back, as another build writing the same output would. + Path restoredBlob = getFileSystem().getPath(unverifiedWorker.getCasBlobPath(barContents)); + restoredBlob.getParentDirectory().createDirectoryAndParents(); + FileSystemUtils.writeContent(restoredBlob, barContents); + + // Delete the locally regenerated outputs and change //a:consumer's input, so the next build + // must re-evaluate and cannot reuse local state, forcing //a:bar's entry to be looked up. + getOutputPath("a/bar.out").delete(); + getOutputPath("a/consumer.out").delete(); + write("a/consumer.in", "three"); + ActionEventRecorder actionEventRecorder = new ActionEventRecorder(); + getRuntimeWrapper().registerSubscriber(actionEventRecorder); + buildTarget("//a:consumer"); + assertThat( + actionEventRecorder.getActionResultReceivedEvents().stream() + .filter( + event -> + !event.getActionResult().spawnResults().isEmpty() + && event.getActionResult().spawnResults().stream() + .allMatch(SpawnResult::isCacheHit)) + .map(event -> event.getAction().getOwner().getLabel().getCanonicalForm())) + .containsExactly("//a:bar"); + assertValidOutputFile("a/consumer.out", "barthree\n"); + } + } + + @Test + public void actionRewinding_localExecution_bustsStaleCacheEntry( + @TestParameter boolean actionCacheIntegrityCheck, @TestParameter boolean uploadLocalResults) + throws Exception { + // A rewound action that is executed locally rather than remotely must bypass its stale action + // result independently of whether that result is served by the cache. A cache that checks the + // integrity of action results doesn't serve it at all. + var cacheWorker = + IntegrationTestUtils.createWorker( + "--action_cache_integrity_check=" + actionCacheIntegrityCheck); + try (var ignored = cacheWorker.start()) { + addOptions("--remote_executor=grpc://localhost:" + cacheWorker.getPort()); + enableActionRewinding(); + write( + "a/BUILD", + """ + genrule( + name = "foo", + srcs = [], + outs = ["foo.out"], + cmd = "echo -n foo > $@", + ) + + genrule( + name = "bar", + srcs = [ + ":foo.out", + "bar.in", + ], + outs = ["bar.out"], + cmd = "cat $(location :foo.out) $(location bar.in) > $@", + ) + """); + write("a/bar.in", "one"); + + // Execute remotely without downloading outputs, so that foo.out only exists in the CAS. + buildTarget("//a:bar"); + + // Delete the blob backing foo.out from the CAS while keeping all action cache entries. + cacheWorker.evictBlob("foo".getBytes(UTF_8)); + if (useDiskCache) { + // Prevent the disk cache from restoring the deleted blob. + addOptions("--disk_cache=" + UUID.randomUUID()); + } + // Execute locally from now on. Not by unsetting --remote_executor, as that would invalidate + // //a:foo and thus have it executed instead of rewound. + addOptions( + "--strategy_regexp=.*=local", "--remote_upload_local_results=" + uploadLocalResults); + // Invalidate only //a:bar so that its execution discovers the lost input and rewinds //a:foo. + write("a/bar.in", "two"); + var rewoundKeys = rewindingTestsHelper.collectOrderedRewoundKeys(); + buildTarget("//a:bar"); + + assertRewoundActions(rewoundKeys, "//a:foo"); + assertValidOutputFile("a/bar.out", "footwo\n"); + assertThat(cacheWorker.hasCasBlob("foo".getBytes(UTF_8))).isEqualTo(uploadLocalResults); + } + } + + @Test + public void actionRewinding_localExecution_reuploadsBlobUploadedEarlierInBuild() + throws Exception { + // A rewound action that is executed locally may regenerate a blob that the same build has + // already uploaded before it was evicted. Its upload must not be deduplicated against the + // earlier one, or the refreshed action result would still reference a missing blob. + // With a disk cache, :bar would fetch foo.out from the copy uploaded by :same instead of + // discovering it lost. + assumeFalse(useDiskCache); + var cacheWorker = IntegrationTestUtils.createWorker("--noaction_cache_integrity_check"); + try (var ignored = cacheWorker.start()) { + addOptions("--remote_executor=grpc://localhost:" + cacheWorker.getPort()); + enableActionRewinding(); + byte[] fooContents = "foo".getBytes(UTF_8); + write( + "a/BUILD", + """ + genrule( + name = "foo", + srcs = [], + outs = ["foo.out"], + cmd = "echo -n foo > $@", + ) + + # Produces the same blob as foo.out. + genrule( + name = "same", + srcs = [], + outs = ["same.out"], + cmd = "echo -n foo > $@", + ) + + # Evicts that blob from the CAS after :same has uploaded it. + genrule( + name = "evict", + srcs = [":same.out"], + outs = ["evict.out"], + cmd = "rm -f '%s' && touch $@", + tags = ["no-cache"], + ) + + genrule( + name = "bar", + srcs = [ + ":foo.out", + ":evict.out", + ], + outs = ["bar.out"], + cmd = "cat $(location :foo.out) > $@", + ) + """ + .formatted(cacheWorker.getCasBlobPath(fooContents))); + + // Execute remotely without downloading outputs, so that foo.out only exists in the CAS. + buildTarget("//a:foo"); + + // Delete the blob backing foo.out from the CAS while keeping //a:foo's action cache entry, so + // that :same has to upload it again. + cacheWorker.evictBlob(fooContents); + // Execute locally from now on. Not by unsetting --remote_executor, as that would invalidate + // //a:foo and thus have it executed instead of rewound. Upload synchronously so that :evict + // only runs after :same has uploaded its output. + addOptions("--strategy_regexp=.*=local", "--noremote_cache_async"); + setDownloadToplevel(); + var rewoundKeys = rewindingTestsHelper.collectOrderedRewoundKeys(); + // :bar discovers foo.out lost after :evict has deleted it and rewinds //a:foo, which + // regenerates the blob locally. + buildTarget("//a:bar"); + + assertRewoundActions(rewoundKeys, "//a:foo"); + assertValidOutputFile("a/bar.out", "foo"); + assertThat(cacheWorker.hasCasBlob(fooContents)).isTrue(); } } @@ -1334,9 +1631,11 @@ addOptions("--strategy_regexp=.*=local"); } enableActionRewinding(); + var rewoundKeys = rewindingTestsHelper.collectOrderedRewoundKeys(); buildTarget("//a:bar"); + assertRewoundActions(rewoundKeys, "//a:foo.out"); assertValidOutputFile("a/bar.out", "file-inside\nupdated bar\n"); } }
diff --git a/src/test/java/com/google/devtools/build/lib/remote/ChunkedBlobUploaderTest.java b/src/test/java/com/google/devtools/build/lib/remote/ChunkedBlobUploaderTest.java index 496c442..9a3548a 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/ChunkedBlobUploaderTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/ChunkedBlobUploaderTest.java
@@ -18,6 +18,7 @@ import static com.google.common.util.concurrent.Futures.immediateVoidFuture; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -117,11 +118,11 @@ List<Digest> digests = invocation.getArgument(1); return immediateFuture(ImmutableSet.copyOf(digests)); }); - when(combinedCache.uploadBlob(any(), any(Digest.class), any(Blob.class))) + when(combinedCache.uploadBlob(any(), any(Digest.class), any(Blob.class), anyBoolean())) .thenReturn(immediateVoidFuture()); when(grpcCacheClient.spliceBlob(any(), any(), any(), any())).thenReturn(immediateVoidFuture()); - uploader.uploadChunked(context, blobDigest, file); + uploader.uploadChunked(context, blobDigest, file, /* force= */ false); List<Digest> chunkDigests = digestsCaptor.getValue(); assertThat(chunkDigests.size()).isGreaterThan(1); @@ -131,6 +132,32 @@ @Test @SuppressWarnings("unchecked") + public void uploadChunked_force_forwardsForceToChunkUploads() throws Exception { + Path file = execRoot.getRelative("test_force.txt"); + byte[] data = new byte[8192]; + new Random(42).nextBytes(data); + writeFile(file, data); + Digest blobDigest = DIGEST_UTIL.compute(data); + + ArgumentCaptor<List<Digest>> digestsCaptor = ArgumentCaptor.forClass(List.class); + when(grpcCacheClient.findMissingDigests(any(), digestsCaptor.capture())) + .thenAnswer( + invocation -> { + List<Digest> digests = invocation.getArgument(1); + return immediateFuture(ImmutableSet.copyOf(digests)); + }); + when(combinedCache.uploadBlob(any(), any(Digest.class), any(Blob.class), eq(true))) + .thenReturn(immediateVoidFuture()); + when(grpcCacheClient.spliceBlob(any(), any(), any(), any())).thenReturn(immediateVoidFuture()); + + uploader.uploadChunked(context, blobDigest, file, /* force= */ true); + + verify(combinedCache, times(digestsCaptor.getValue().size())) + .uploadBlob(any(), any(Digest.class), any(Blob.class), eq(true)); + } + + @Test + @SuppressWarnings("unchecked") public void uploadChunked_noChunksMissing_skipsChunkUpload() throws Exception { Path file = execRoot.getRelative("test.txt"); byte[] data = new byte[8192]; @@ -142,9 +169,10 @@ .thenReturn(immediateFuture(ImmutableSet.of())); when(grpcCacheClient.spliceBlob(any(), any(), any(), any())).thenReturn(immediateVoidFuture()); - uploader.uploadChunked(context, blobDigest, file); + uploader.uploadChunked(context, blobDigest, file, /* force= */ false); - verify(combinedCache, never()).uploadBlob(any(), any(Digest.class), any(Blob.class)); + verify(combinedCache, never()) + .uploadBlob(any(), any(Digest.class), any(Blob.class), anyBoolean()); verify(grpcCacheClient).spliceBlob(any(), eq(blobDigest), any(), any()); } @@ -188,7 +216,7 @@ when(grpcCacheClient.findMissingDigests(any(), any())) .thenReturn(immediateFuture(ImmutableSet.copyOf(digestsToReportMissing))); Map<Digest, ByteString> actualUploads = new HashMap<>(); - when(combinedCache.uploadBlob(any(), any(Digest.class), any(Blob.class))) + when(combinedCache.uploadBlob(any(), any(Digest.class), any(Blob.class), anyBoolean())) .thenAnswer( invocation -> { Digest d = invocation.getArgument(1); @@ -200,7 +228,7 @@ }); when(grpcCacheClient.spliceBlob(any(), any(), any(), any())).thenReturn(immediateVoidFuture()); - uploader.uploadChunked(context, blobDigest, file); + uploader.uploadChunked(context, blobDigest, file, /* force= */ false); assertThat(actualUploads.keySet()).isEqualTo(expectedChunkData.keySet()); for (Map.Entry<Digest, ByteString> entry : expectedChunkData.entrySet()) { @@ -248,7 +276,7 @@ CountDownLatch firstWindowRequested = new CountDownLatch(MAX_IN_FLIGHT_CHUNK_UPLOADS); CountDownLatch overflowUploadRequested = new CountDownLatch(1); - when(combinedCache.uploadBlob(any(), any(Digest.class), any(Blob.class))) + when(combinedCache.uploadBlob(any(), any(Digest.class), any(Blob.class), anyBoolean())) .thenAnswer( invocation -> { Digest digest = invocation.getArgument(1); @@ -266,7 +294,7 @@ .unstarted( () -> { try { - uploader.uploadChunked(context, blobDigest, file); + uploader.uploadChunked(context, blobDigest, file, /* force= */ false); } catch (IOException | InterruptedException e) { throw new RuntimeException(e); } @@ -324,7 +352,7 @@ SettableFuture<Void> failedUpload = SettableFuture.create(); SettableFuture<Void> cancelledUpload = SettableFuture.create(); CountDownLatch uploadsStarted = new CountDownLatch(2); - when(combinedCache.uploadBlob(any(), any(Digest.class), any(Blob.class))) + when(combinedCache.uploadBlob(any(), any(Digest.class), any(Blob.class), anyBoolean())) .thenAnswer( invocation -> { Digest digest = invocation.getArgument(1); @@ -343,7 +371,7 @@ .unstarted( () -> { try { - uploader.uploadChunked(context, blobDigest, file); + uploader.uploadChunked(context, blobDigest, file, /* force= */ false); } catch (IOException | InterruptedException e) { throw new RuntimeException(e); } @@ -382,11 +410,12 @@ SettableFuture<Void> cancelledUpload = SettableFuture.create(); cancelledUpload.cancel(/* mayInterruptIfRunning= */ true); - when(combinedCache.uploadBlob(any(), eq(firstChunkDigest), any(Blob.class))) + when(combinedCache.uploadBlob(any(), eq(firstChunkDigest), any(Blob.class), anyBoolean())) .thenReturn(cancelledUpload); assertThrows( - InterruptedException.class, () -> uploader.uploadChunked(context, blobDigest, file)); + InterruptedException.class, + () -> uploader.uploadChunked(context, blobDigest, file, /* force= */ false)); verify(grpcCacheClient, never()).spliceBlob(any(), any(), any(), any()); } @@ -414,7 +443,7 @@ SettableFuture<Void> failedUpload = SettableFuture.create(); failedUpload.setException(new IOException("upload failed")); - when(combinedCache.uploadBlob(any(), eq(chunkDigests.get(0)), any(Blob.class))) + when(combinedCache.uploadBlob(any(), eq(chunkDigests.get(0)), any(Blob.class), anyBoolean())) .thenReturn(failedUpload); Thread uploadThread = @@ -422,7 +451,7 @@ .unstarted( () -> { try { - uploader.uploadChunked(context, blobDigest, file); + uploader.uploadChunked(context, blobDigest, file, /* force= */ false); } catch (IOException | InterruptedException e) { throw new RuntimeException(e); } @@ -457,7 +486,7 @@ .thenReturn(new ByteArrayInputStream(data), new ByteArrayInputStream(new byte[0])); when(grpcCacheClient.findMissingDigests(any(), any())) .thenReturn(immediateFuture(ImmutableSet.of(secondChunkDigest))); - when(combinedCache.uploadBlob(any(), eq(secondChunkDigest), any(Blob.class))) + when(combinedCache.uploadBlob(any(), eq(secondChunkDigest), any(Blob.class), anyBoolean())) .thenAnswer( invocation -> { Blob blob = invocation.getArgument(2); @@ -468,7 +497,9 @@ }); IOException e = - assertThrows(IOException.class, () -> uploader.uploadChunked(context, blobDigest, file)); + assertThrows( + IOException.class, + () -> uploader.uploadChunked(context, blobDigest, file, /* force= */ false)); assertThat(e).hasMessageThat().contains("file was concurrently modified during upload"); assertThat(e).hasCauseThat().isInstanceOf(EOFException.class);
diff --git a/src/test/java/com/google/devtools/build/lib/remote/ChunkedTransferBenchmark.java b/src/test/java/com/google/devtools/build/lib/remote/ChunkedTransferBenchmark.java index 43c19a2..88d364a 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/ChunkedTransferBenchmark.java +++ b/src/test/java/com/google/devtools/build/lib/remote/ChunkedTransferBenchmark.java
@@ -15,6 +15,7 @@ package com.google.devtools.build.lib.remote; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -78,7 +79,7 @@ @Benchmark public void uploadChunked(UploadState state) throws Exception { - state.uploader.uploadChunked(CONTEXT, state.blobDigest, state.file); + state.uploader.uploadChunked(CONTEXT, state.blobDigest, state.file, /* force= */ false); } @State(Scope.Thread) @@ -210,7 +211,7 @@ .thenReturn(Futures.immediateFuture(ImmutableSet.copyOf(chunkDigests))); when(grpcCacheClient.spliceBlob(any(), any(Digest.class), any(), any())) .thenReturn(Futures.immediateVoidFuture()); - when(combinedCache.uploadBlob(any(), any(Digest.class), any(Blob.class))) + when(combinedCache.uploadBlob(any(), any(Digest.class), any(Blob.class), anyBoolean())) .thenAnswer( invocation -> delayedFuture(null, delayMillis, jitterMillis, latencyJitter, scheduler));
diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java index aad6345..12d0e36 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java
@@ -844,7 +844,8 @@ /* startTime= */ null, /* wallTimeInMs= */ 0, /* preserveExecutableBit= */ false); - return uploadManifest.upload(context, combinedCache, NullEventHandler.INSTANCE); + return uploadManifest.upload( + context, combinedCache, NullEventHandler.INSTANCE, /* force= */ false); } private ActionResult uploadDirectory(CombinedCache combinedCache, List<Path> outputs)
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java index 4c3a423..6e5304d 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java
@@ -2564,6 +2564,7 @@ .new OutputUploadTask( action, spawnResult, + /* force= */ false, () -> { completionStarted.release(); completionMayFinish.acquireUninterruptibly(); @@ -2629,7 +2630,9 @@ .setRunnerName("test") .build(); var uploadComplete = new CountDownLatch(1); - var task = service.new OutputUploadTask(action, spawnResult, uploadComplete::countDown); + var task = + service + .new OutputUploadTask(action, spawnResult, /* force= */ false, uploadComplete::countDown); var spawnOwner = action.getRemoteActionExecutionContext().getSpawnOwner(); task.start(); @@ -2660,7 +2663,10 @@ .setRunnerName("test") .build(); AtomicInteger completionCalls = new AtomicInteger(); - var task = service.new OutputUploadTask(action, spawnResult, completionCalls::incrementAndGet); + var task = + service + .new OutputUploadTask( + action, spawnResult, /* force= */ false, completionCalls::incrementAndGet); task.requestCancellation(); task.awaitCompletion(); @@ -3314,7 +3320,8 @@ null, remoteOutputChecker, outputService, - Sets.newConcurrentHashSet()); + Sets.newConcurrentHashSet(), + /* wasRewound= */ action -> false); } private RunfilesTree createRunfilesTree(String root, Collection<Artifact> artifacts) {
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java index bf0d0c1..8cafcf9 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java
@@ -44,6 +44,7 @@ import com.google.common.eventbus.EventBus; import com.google.common.io.ByteStreams; import com.google.common.util.concurrent.ListenableFuture; +import com.google.devtools.build.lib.actions.ActionAnalysisMetadata; import com.google.devtools.build.lib.actions.ActionContext; import com.google.devtools.build.lib.actions.ActionExecutionMetadata; import com.google.devtools.build.lib.actions.ActionInput; @@ -109,6 +110,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; import javax.annotation.Nullable; import org.junit.Before; import org.junit.Test; @@ -132,6 +134,8 @@ private static final String BUILD_REQUEST_ID = "build-req-id"; private static final String COMMAND_ID = "command-id"; + @Mock private Predicate<ActionAnalysisMetadata> wasRewound; + private FileSystem fs; private DigestUtil digestUtil; private Path execRoot; @@ -317,7 +321,8 @@ /* captureCorruptedOutputsDir= */ null, DUMMY_REMOTE_OUTPUT_CHECKER, mock(OutputService.class), - Sets.newConcurrentHashSet())); + Sets.newConcurrentHashSet(), + wasRewound)); return new RemoteSpawnCache(options, /* verboseFailures= */ true, service, digestUtil); } @@ -817,6 +822,15 @@ @Test public void pathMappedActionIsDeduplicated() throws Exception { + pathMappedActionIsDeduplicated(/* rewound= */ false); + } + + @Test + public void rewoundPathMappedActionIsDeduplicated() throws Exception { + pathMappedActionIsDeduplicated(/* rewound= */ true); + } + + private void pathMappedActionIsDeduplicated(boolean rewound) throws Exception { // arrange RemoteSpawnCache cache = createRemoteSpawnCache(); @@ -832,6 +846,9 @@ SpawnExecutionContext secondPolicy = createSpawnExecutionContext(secondSpawn, execRoot, secondFakeFileCache, outErr); + when(wasRewound.test(firstSpawn.getResourceOwner())).thenReturn(rewound); + when(wasRewound.test(secondSpawn.getResourceOwner())).thenReturn(rewound); + RemoteExecutionService remoteExecutionService = cache.getRemoteExecutionService(); Mockito.doCallRealMethod().when(remoteExecutionService).waitForAndReuseOutputs(any(), any()); // Simulate a very slow upload to the remote cache to ensure that the second spawn is @@ -869,6 +886,9 @@ assertThat(secondCacheHandle.willStore()).isFalse(); onUploadComplete.get().run(); assertThat(cache.getInFlightExecutionsSize()).isEqualTo(0); + if (rewound) { + verify(combinedCache, never()).downloadActionResult(any(), any(), anyBoolean(), any()); + } } @Test
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java index b485d13..af2d2ac 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java
@@ -54,6 +54,7 @@ import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; +import com.google.devtools.build.lib.actions.ActionAnalysisMetadata; import com.google.devtools.build.lib.actions.ActionContext; import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.ActionInputHelper; @@ -128,6 +129,7 @@ import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; import javax.annotation.Nullable; import org.junit.After; import org.junit.Before; @@ -155,6 +157,8 @@ ImmutableMap.of(ExecutionRequirements.NO_CACHE, ""); private ListeningScheduledExecutorService retryService; + @Mock private Predicate<ActionAnalysisMetadata> wasRewound; + private FileSystem fs; private Path execRoot; private ArtifactRoot artifactRoot; @@ -218,6 +222,24 @@ } @Test + public void rewoundActionSkipsCacheLookup() throws Exception { + Spawn spawn = newSimpleSpawn(); + when(wasRewound.test(spawn.getResourceOwner())).thenReturn(true); + when(executor.executeRemotely(any(), any(), any())) + .thenReturn( + ExecuteResponse.newBuilder().setResult(ActionResult.getDefaultInstance()).build()); + + SpawnResult result = newSpawnRunner().exec(spawn, getSpawnContext(spawn)); + + assertThat(result.status()).isEqualTo(Status.SUCCESS); + assertThat(result.isCacheHit()).isFalse(); + verify(cache, never()).downloadActionResult(any(), any(), anyBoolean(), any()); + ArgumentCaptor<ExecuteRequest> requestCaptor = ArgumentCaptor.forClass(ExecuteRequest.class); + verify(executor).executeRemotely(any(), requestCaptor.capture(), any()); + assertThat(requestCaptor.getValue().getSkipCacheLookup()).isTrue(); + } + + @Test public void nonCachableSpawnsShouldNotBeCached_remote() throws Exception { // Test that if a spawn is marked "NO_CACHE" then it's not fetched from a remote cache. // It should be executed remotely, but marked non-cacheable to remote execution, so that @@ -1375,7 +1397,8 @@ /* captureCorruptedOutputsDir= */ null, remoteOutputChecker, mock(OutputService.class), - Sets.newConcurrentHashSet()); + Sets.newConcurrentHashSet(), + wasRewound); RemoteSpawnRunner runner = new RemoteSpawnRunner( remoteOptions, @@ -1911,7 +1934,8 @@ /* captureCorruptedOutputsDir= */ null, remoteOutputChecker, mock(OutputService.class), - Sets.newConcurrentHashSet())); + Sets.newConcurrentHashSet(), + wasRewound)); return new RemoteSpawnRunner( remoteOptions,
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java index e3acde4..c6d90de 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java
@@ -359,7 +359,8 @@ /* captureCorruptedOutputsDir= */ null, remoteOutputChecker, mock(OutputService.class), - Sets.newConcurrentHashSet()); + Sets.newConcurrentHashSet(), + /* wasRewound= */ action -> false); client = new RemoteSpawnRunner( remoteOptions,
diff --git a/src/test/java/com/google/devtools/build/lib/remote/util/IntegrationTestUtils.java b/src/test/java/com/google/devtools/build/lib/remote/util/IntegrationTestUtils.java index 472b48b..2d80be1 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/util/IntegrationTestUtils.java +++ b/src/test/java/com/google/devtools/build/lib/remote/util/IntegrationTestUtils.java
@@ -280,6 +280,11 @@ return PathFragment.create(casBlobPath(digest).toString()); } + /** Returns whether the blob with the given contents is present in the worker's CAS. */ + public boolean hasCasBlob(byte[] contents) { + return Files.exists(Path.of(getCasBlobPath(contents).getPathString())); + } + /** * Deletes every blob from the worker's CAS, while leaving the AC entries that reference them * intact.
diff --git a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/ExecutionServer.java b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/ExecutionServer.java index 45c7186c..3caa2a1 100644 --- a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/ExecutionServer.java +++ b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/ExecutionServer.java
@@ -422,7 +422,7 @@ startTime, (int) wallTime.toMillis(), /* preserveExecutableBit= */ false); - result = manifest.upload(context, cache, NullEventHandler.INSTANCE); + result = manifest.upload(context, cache, NullEventHandler.INSTANCE, /* force= */ false); } catch (ExecException e) { if (errStatus == null) { errStatus =