| // Copyright 2022 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| package com.google.devtools.build.lib.skyframe.rewinding; |
| |
| import static com.google.common.base.Preconditions.checkArgument; |
| import static com.google.common.base.Preconditions.checkNotNull; |
| import static com.google.common.base.Preconditions.checkState; |
| import static com.google.common.collect.ImmutableList.toImmutableList; |
| import static com.google.common.collect.ImmutableSetMultimap.toImmutableSetMultimap; |
| import static com.google.common.truth.Truth.assertThat; |
| import static com.google.devtools.build.lib.buildtool.util.BuildIntegrationTestCase.assertAndClearBugReporterStoredCrash; |
| import static com.google.devtools.build.lib.vfs.FileSystemUtils.readContentAsLatin1; |
| import static java.util.concurrent.TimeUnit.SECONDS; |
| import static org.junit.Assert.assertThrows; |
| import static org.junit.Assert.fail; |
| import static org.mockito.ArgumentMatchers.any; |
| import static org.mockito.Mockito.mock; |
| import static org.mockito.Mockito.verify; |
| |
| import com.google.common.collect.ImmutableList; |
| import com.google.common.collect.ImmutableMap; |
| import com.google.common.collect.ImmutableSetMultimap; |
| import com.google.common.collect.Iterables; |
| import com.google.common.collect.Multimap; |
| import com.google.common.eventbus.Subscribe; |
| import com.google.common.util.concurrent.Uninterruptibles; |
| import com.google.devtools.build.lib.actions.Action; |
| import com.google.devtools.build.lib.actions.ActionExecutionContext; |
| import com.google.devtools.build.lib.actions.ActionInput; |
| import com.google.devtools.build.lib.actions.ActionInputDepOwnerMap; |
| import com.google.devtools.build.lib.actions.ActionInputDepOwners; |
| import com.google.devtools.build.lib.actions.ActionLookupData; |
| import com.google.devtools.build.lib.actions.Artifact; |
| import com.google.devtools.build.lib.actions.BuildFailedException; |
| import com.google.devtools.build.lib.actions.LostInputsExecException; |
| import com.google.devtools.build.lib.actions.MiddlemanType; |
| import com.google.devtools.build.lib.actions.Spawn; |
| import com.google.devtools.build.lib.actions.SpawnResult; |
| import com.google.devtools.build.lib.analysis.config.CoreOptions; |
| import com.google.devtools.build.lib.bugreport.BugReporter; |
| import com.google.devtools.build.lib.buildtool.buildevent.BuildStartingEvent; |
| import com.google.devtools.build.lib.buildtool.util.BuildIntegrationTestCase; |
| import com.google.devtools.build.lib.cmdline.Label; |
| import com.google.devtools.build.lib.exec.SpawnExecException; |
| import com.google.devtools.build.lib.runtime.KeepGoingOption; |
| import com.google.devtools.build.lib.server.FailureDetails; |
| import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; |
| import com.google.devtools.build.lib.server.FailureDetails.Spawn.Code; |
| import com.google.devtools.build.lib.skyframe.ArtifactNestedSetKey; |
| import com.google.devtools.build.lib.testutil.ActionEventRecorder; |
| import com.google.devtools.build.lib.testutil.ActionEventRecorder.ActionRewindEventAsserter; |
| import com.google.devtools.build.lib.testutil.ControllableActionStrategyModule; |
| import com.google.devtools.build.lib.testutil.SpawnController; |
| import com.google.devtools.build.lib.testutil.SpawnController.ExecResult; |
| import com.google.devtools.build.lib.testutil.SpawnController.SpawnShim; |
| import com.google.devtools.build.lib.testutil.SpawnInputUtils; |
| import com.google.devtools.build.lib.testutil.TestConstants; |
| import com.google.devtools.build.lib.testutil.TestUtils; |
| import com.google.devtools.build.skyframe.NodeEntry.DirtyType; |
| import com.google.devtools.build.skyframe.NotifyingHelper; |
| import com.google.devtools.build.skyframe.NotifyingHelper.EventType; |
| import com.google.devtools.build.skyframe.NotifyingHelper.Order; |
| import com.google.devtools.build.skyframe.QueryableGraph.Reason; |
| import com.google.devtools.build.skyframe.SkyKey; |
| import com.google.devtools.build.skyframe.TrackingAwaiter; |
| import com.google.devtools.build.skyframe.proto.GraphInconsistency.Inconsistency; |
| import com.google.errorprone.annotations.ForOverride; |
| import com.google.errorprone.annotations.Keep; |
| import java.io.IOException; |
| import java.util.ArrayList; |
| import java.util.Collections; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.concurrent.CountDownLatch; |
| import java.util.concurrent.atomic.AtomicInteger; |
| import java.util.concurrent.atomic.AtomicReference; |
| |
| /** |
| * Implements rewinding-specific infrastructure and test logic used for rewinding tests. Search for |
| * the callers of these methods to find the {@link BuildIntegrationTestCase} classes where the |
| * build-specific infrastructure and the actual tests are implemented. |
| * |
| * <p>In this class, methods whose names begin with {@code run} implement test logic. Generally, |
| * these tests have the following structure: |
| * |
| * <ol> |
| * <li>{@code BUILD}, {@code bzl}, and other source file setup. |
| * <li>Injection of one or more {@linkplain SpawnShim spawn shims}, giving the test control of |
| * whether execution succeeds or fails with either an appropriately structured {@link |
| * LostInputsExecException} or an {@link IOException}. These shims also occasionally capture |
| * input file contents to be checked after the build is done. |
| * <li>Injection of a {@link |
| * com.google.devtools.build.skyframe.MemoizingEvaluator.GraphTransformerForTesting} so that |
| * the invalidated Skyframe nodes can be tracked, along with the order they're invalidated in. |
| * <li>The build itself. |
| * <li>Assertions of (some subset of): |
| * <ul> |
| * <li>the build's output's contents |
| * <li>what spawn actions were run (and, when possible, in what order) |
| * <li>what events were emitted |
| * <li>what Skyframe nodes were invalidated and in what order |
| * </ul> |
| * </ol> |
| * |
| * <p>Tests are largely of two structurally similar but distinguishable categories: |
| * |
| * <ol> |
| * <li>Tests that check the rewinding strategy's behavior and how it interacts with build logic |
| * under varying circumstances, like {@link #runActionFromPreviousBuildReevaluated}, {@link |
| * #runMultiplyLosingInputsFails}, {@link #runInterruptedDuringRewindStopsNormally}. |
| * <li>Tests that check the behavior of the execution strategy and Skyframe action execution |
| * machinery to make sure they collaborate to give the action rewinding strategy the |
| * information it needs to figure out what Skyframe nodes need to be rewound. Examples of |
| * these include {@link #runDependentActionsReevaluated}, {@link #runTreeFileArtifactRewound}, |
| * and others, which test different combinations of types of action inputs which can get lost. |
| * </ol> |
| */ |
| public class RewindingTestsHelper { |
| |
| final ActionEventRecorder recorder; |
| final BuildIntegrationTestCase testCase; |
| private final SpawnController spawnController = new SpawnController(); |
| |
| RewindingTestsHelper(BuildIntegrationTestCase testCase, ActionEventRecorder recorder) { |
| this.testCase = checkNotNull(testCase); |
| this.recorder = checkNotNull(recorder); |
| } |
| |
| /** Disables remote caching if the execution strategy supports it. */ |
| @ForOverride |
| void disableRemoteCaching() {} |
| |
| /** |
| * Returns whether the execution strategy can handle rewinding happening concurrently with another |
| * action consuming the rewound action's outputs. |
| * |
| * <p>When an action is rewound, it executes a second time, including the {@link Action#prepare} |
| * step which deletes previous outputs on disk. Another action which observed its dependency to be |
| * done (before rewinding was initiated) may simultaneously attempt to consume these deleted |
| * outputs, leading to a flaky build failure. If this method returns {@code false}, test cases |
| * which exercise the described scenario will set {@code --jobs=1} to avoid the race condition. |
| */ |
| @ForOverride |
| boolean supportsConcurrentRewinding() { |
| return false; |
| } |
| |
| /** |
| * Converts a file digest to a hex string compatible with the test's active {@link |
| * com.google.devtools.build.lib.vfs.DigestHashFunction}. |
| */ |
| @ForOverride |
| String toHex(byte[] digest) { |
| StringBuilder hex = new StringBuilder(); |
| for (byte b : digest) { |
| hex.append(String.format("%02x", b)); |
| } |
| return hex.toString(); |
| } |
| |
| public final ControllableActionStrategyModule makeControllableActionStrategyModule( |
| String identifier) { |
| return new ControllableActionStrategyModule(spawnController, identifier); |
| } |
| |
| final ImmutableList<String> getExecutedSpawnDescriptions() { |
| return spawnController.getExecutedSpawnDescriptions(); |
| } |
| |
| public final void addSpawnShim(String spawnDescription, SpawnShim spawnShim) { |
| spawnController.addSpawnShim(spawnDescription, spawnShim); |
| } |
| |
| public final void verifyAllSpawnShimsConsumed() { |
| spawnController.verifyAllShimsConsumed(); |
| } |
| |
| public final ExecResult createLostInputsExecException( |
| ActionExecutionContext context, |
| ImmutableList<ActionInput> lostInputs, |
| ActionInputDepOwners owners) |
| throws IOException { |
| ImmutableMap.Builder<String, ActionInput> builder = ImmutableMap.builder(); |
| for (ActionInput lostInput : lostInputs) { |
| builder.put(getHexDigest(lostInput, context), lostInput); |
| } |
| return ExecResult.ofException(new LostInputsExecException(builder.buildOrThrow(), owners)); |
| } |
| |
| private String getHexDigest(ActionInput input, ActionExecutionContext context) |
| throws IOException { |
| return toHex(context.getMetadataProvider().getMetadata(input).getDigest()); |
| } |
| |
| static ActionInputDepOwners getInputOwners(Multimap<ActionInput, Artifact> mappings) { |
| ActionInputDepOwnerMap owners = new ActionInputDepOwnerMap(mappings.keySet()); |
| mappings.forEach(owners::addOwner); |
| return owners; |
| } |
| |
| public final List<SkyKey> collectOrderedDirtiedKeys() { |
| return collectOrderedDirtiedKeys(/*exactNestedSets=*/ false); |
| } |
| |
| final List<SkyKey> collectOrderedDirtiedKeys(boolean exactNestedSets) { |
| List<SkyKey> dirtiedKeys = new ArrayList<>(); |
| injectListenerAtStartOfNextBuild(collectOrderedDirtyKeysListener(dirtiedKeys, exactNestedSets)); |
| return dirtiedKeys; |
| } |
| |
| /** |
| * Creates a {@link NotifyingHelper.Listener} that collects keys dirtied by rewinding. |
| * |
| * <p>This can be called instead of {@link #collectOrderedDirtiedKeys} if it needs to be combined |
| * with other listeners, but then it must be {@linkplain |
| * com.google.devtools.build.skyframe.MemoizingEvaluator#injectGraphTransformerForTesting |
| * injected} manually. |
| * |
| * <p>To avoid brittle assertions on the number of keys dirtied, {@link ArtifactNestedSetKey} may |
| * not be collected (depending on {@code exactNestedSets}), though it may be dirtied. Its |
| * {@linkplain ArtifactNestedSetKey#getSet set} may contain multiple paths (of varying length) to |
| * a lost artifact, any of which would be a correct chain for rewinding. |
| */ |
| public static NotifyingHelper.Listener collectOrderedDirtyKeysListener( |
| List<SkyKey> dirtiedKeys, boolean exactNestedSets) { |
| return (key, type, order, context) -> { |
| if (type.equals(NotifyingHelper.EventType.MARK_DIRTY) && order.equals(Order.AFTER)) { |
| NotifyingHelper.MarkDirtyAfterContext markDirtyAfterContext = |
| (NotifyingHelper.MarkDirtyAfterContext) context; |
| checkState( |
| markDirtyAfterContext.dirtyType().equals(DirtyType.FORCE_REBUILD), |
| "Unexpected DirtyType %s for key %s", |
| context, |
| key); |
| if (markDirtyAfterContext.actuallyDirtied() |
| && (exactNestedSets || !(key instanceof ArtifactNestedSetKey))) { |
| synchronized (dirtiedKeys) { |
| dirtiedKeys.add(key); |
| } |
| } |
| } |
| }; |
| } |
| |
| /** |
| * Lazily injects the given listener at the start of the next build. |
| * |
| * <p>Injecting the listener immediately would reach the <em>current</em> evaluator, but the next |
| * build may create a new evaluator, which happens when {@link |
| * com.google.devtools.build.lib.skyframe.SequencedSkyframeExecutor} is not tracking incremental |
| * state. |
| */ |
| private void injectListenerAtStartOfNextBuild(NotifyingHelper.Listener listener) { |
| testCase |
| .getRuntimeWrapper() |
| .registerSubscriber( |
| new Object() { |
| private boolean injected = false; |
| |
| @Subscribe |
| @Keep |
| void buildStarting(@SuppressWarnings("unused") BuildStartingEvent event) { |
| if (!injected) { |
| testCase |
| .getSkyframeExecutor() |
| .getEvaluator() |
| .injectGraphTransformerForTesting( |
| NotifyingHelper.makeNotifyingTransformer(listener)); |
| injected = true; |
| } |
| } |
| }); |
| } |
| |
| static void assertOnlyActionsDirtied(List<SkyKey> dirtiedKeys) { |
| for (SkyKey key : dirtiedKeys) { |
| if (!(key instanceof ArtifactNestedSetKey)) { |
| assertThat(key).isInstanceOf(ActionLookupData.class); |
| } |
| } |
| } |
| |
| static ImmutableList<String> dirtiedArtifactOwnerLabels(List<SkyKey> dirtiedKeys) { |
| return dirtiedKeys.stream() |
| .filter(k -> k instanceof ActionLookupData) |
| .map(k -> ((ActionLookupData) k).getActionLookupKey().getLabel().getCanonicalForm()) |
| .collect(toImmutableList()); |
| } |
| |
| static void assertArtifactKey(SkyKey skyKey, String path) { |
| assertThat(skyKey).isInstanceOf(Artifact.class); |
| assertThat(((Artifact) skyKey).getRootRelativePathString()).isEqualTo(path); |
| } |
| |
| static void assertActionKey(SkyKey skyKey, String label, int index) { |
| assertThat(skyKey).isInstanceOf(ActionLookupData.class); |
| assertThat(((ActionLookupData) skyKey).getLabel().getCanonicalForm()).isEqualTo(label); |
| assertThat(((ActionLookupData) skyKey).getActionIndex()).isEqualTo(index); |
| } |
| |
| static String latin1StringFromActionInput(ActionExecutionContext context, ActionInput input) |
| throws IOException { |
| // Test logic implemented here requires that files whose contents will be read locally have the |
| // suffix ".inlined". Tests using remote execution should be configured to eagerly fetch these |
| // artifacts. |
| checkArgument( |
| input.getExecPathString().endsWith(".inlined"), |
| "Only inputs ending in .inlined are guaranteed readable. Tried to read: %s", |
| input); |
| return new String(readContentAsLatin1(context.getInputPath(input))); |
| } |
| |
| /** |
| * Builds the genrule "//{@code pkg}:consume_output", which must specify "//{@code |
| * pkg}:output.inlined" as a "srcs" dep. Returns the contents of {@code output.inlined} as a |
| * latin1 {@link String}. |
| * |
| * <p>This is useful for builds that do not write output files to disk, and so those files' |
| * contents can't be verified via regular filesystem operations. This method extracts {@code |
| * output.inlined}'s contents during evaluation. |
| */ |
| final String buildAndGetOutput(String pkg, BuildIntegrationTestCase testCase) throws Exception { |
| AtomicReference<String> invocationOutput = new AtomicReference<>(null); |
| addSpawnShim( |
| String.format("Executing genrule //%s:consume_output", pkg), |
| (spawn, context) -> { |
| ActionInput actionInput = SpawnInputUtils.getInputWithName(spawn, "output.inlined"); |
| invocationOutput.set(latin1StringFromActionInput(context, actionInput)); |
| return ExecResult.delegate(); |
| }); |
| testCase.buildTarget(String.format("//%s:consume_output", pkg)); |
| return invocationOutput.get(); |
| } |
| |
| public final void runNoLossSmokeTest() throws Exception { |
| testCase.write( |
| "test/BUILD", |
| "genrule(name = 'rule1',", |
| " srcs = ['source.txt'],", |
| " outs = ['intermediate.txt'],", |
| " cmd = '(cat $< && echo from rule1) > $@')", |
| "", |
| "genrule(name = 'rule2',", |
| " srcs = ['intermediate.txt'],", |
| " outs = ['output.inlined'],", |
| " cmd = '(cat $< && echo from rule2) > $@')", |
| "", |
| "genrule(name = 'consume_output',", |
| " srcs = [':output.inlined'],", |
| " outs = ['dummy.out'],", |
| " cmd = 'touch $@')"); |
| testCase.write("test/source.txt", "source"); |
| |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| String outputFileContent = buildAndGetOutput("test", testCase); |
| |
| assertThat(outputFileContent).isEqualTo("source\nfrom rule1\nfrom rule2\n"); |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactly( |
| "Executing genrule //test:rule1", |
| "Executing genrule //test:rule2", |
| "Executing genrule //test:consume_output") |
| .inOrder(); |
| |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of( |
| "Executing genrule //test:rule1", "Executing genrule //test:rule2"), |
| /*completedRewound=*/ ImmutableList.of(), |
| /*failedRewound=*/ ImmutableList.of(), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of(), |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of(0)); |
| |
| assertThat(dirtiedKeys).isEmpty(); |
| } |
| |
| /** |
| * Tests that {@link Inconsistency#BUILDING_PARENT_FOUND_UNDONE_CHILD} is not tolerated if there |
| * has not been any rewinding. |
| */ |
| public final void runBuildingParentFoundUndoneChildNotToleratedWithoutRewinding() |
| throws Exception { |
| BugReporter bugReporter = mock(BugReporter.class); |
| testCase.setCustomBugReporterAndReinitialize(bugReporter); |
| testCase.write( |
| "foo/BUILD", |
| "genrule(name = 'top', outs = ['top.out'], srcs = [':dep'], cmd = 'cp $< $@')", |
| "genrule(name = 'dep', outs = ['dep.out'], cmd = 'touch $@')"); |
| injectListenerAtStartOfNextBuild( |
| (key, type, order, context) -> { |
| if (type == EventType.GET_BATCH |
| && order == Order.BEFORE |
| && context == Reason.PREFETCH |
| && isActionExecutionKey(key, Label.parseAbsoluteUnchecked("//foo:dep"))) { |
| try { |
| testCase |
| .getSkyframeExecutor() |
| .getEvaluator() |
| .getExistingEntryAtCurrentlyEvaluatingVersion(key) |
| .markDirty(DirtyType.FORCE_REBUILD); |
| } catch (InterruptedException e) { |
| throw new IllegalStateException(e); |
| } |
| } |
| }); |
| |
| Exception e = |
| assertThrows(IllegalStateException.class, () -> testCase.buildTarget("//foo:top")); |
| assertThat(e).hasMessageThat().contains("Unexpected undone children"); |
| verify(bugReporter).handleCrash(any(), any()); |
| } |
| |
| public final void runDependentActionsReevaluated_spawnFailed() throws Exception { |
| // The first time rule2 is executed, the execution strategy fails, saying that rule2's two input |
| // files are missing. |
| runDependentActionsReevaluated( |
| (spawn, context) -> |
| createLostInputsExecException( |
| context, |
| getIntermediate1And2LostInputs(spawn), |
| new ActionInputDepOwnerMap(getIntermediate1And2LostInputs(spawn)))); |
| } |
| |
| static ImmutableList<ActionInput> getIntermediate1And2LostInputs(Spawn spawn) { |
| return ImmutableList.of( |
| SpawnInputUtils.getInputWithName(spawn, "intermediate_1.txt"), |
| SpawnInputUtils.getInputWithName(spawn, "intermediate_2.txt")); |
| } |
| |
| final void runDependentActionsReevaluated(SpawnShim shim) throws Exception { |
| // This test sets up a genrule, rule2, that consumes the outputs of two other genrules. |
| testCase.write( |
| "test/BUILD", |
| "genrule(name = 'rule1_1',", |
| " srcs = ['source_1.txt'],", |
| " outs = ['intermediate_1.txt'],", |
| " cmd = '(cat $< && echo from rule1_1) > $@')", |
| "genrule(name = 'rule1_2',", |
| " srcs = ['source_2.txt'],", |
| " outs = ['intermediate_2.txt'],", |
| " cmd = '(cat $< && echo from rule1_2) > $@')", |
| "", |
| "genrule(name = 'rule2',", |
| " srcs = ['intermediate_1.txt', 'intermediate_2.txt', 'source_3.txt'],", |
| " outs = ['output.inlined'],", |
| " cmd = '(cat $(SRCS) && echo from rule2) > $@')", |
| "", |
| "genrule(name = 'consume_output',", |
| " srcs = [':output.inlined'],", |
| " outs = ['dummy.out'],", |
| " cmd = 'touch $@')"); |
| |
| testCase.write("test/source_1.txt", "source_1"); |
| testCase.write("test/source_2.txt", "source_2"); |
| testCase.write("test/source_3.txt", "source_3"); |
| |
| addSpawnShim("Executing genrule //test:rule2", shim); |
| |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| String outputFileContents = buildAndGetOutput("test", testCase); |
| |
| // The evaluation succeeds, producing the expected output, after re-executing rule1_1's and |
| // rule1_2's actions. |
| assertThat(outputFileContents) |
| .isEqualTo("source_1\nfrom rule1_1\nsource_2\nfrom rule1_2\nsource_3\nfrom rule2\n"); |
| |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactly( |
| "Executing genrule //test:rule1_1", |
| "Executing genrule //test:rule1_2", |
| "Executing genrule //test:rule2", |
| "Executing genrule //test:rule1_1", |
| "Executing genrule //test:rule1_2", |
| "Executing genrule //test:rule2", |
| "Executing genrule //test:consume_output"); |
| |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of(), |
| /*completedRewound=*/ ImmutableList.of( |
| "Executing genrule //test:rule1_1", "Executing genrule //test:rule1_2"), |
| /*failedRewound=*/ ImmutableList.of("Executing genrule //test:rule2"), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of(), |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of(2)); |
| |
| assertOnlyActionsDirtied(dirtiedKeys); |
| assertThat(dirtiedArtifactOwnerLabels(dirtiedKeys)) |
| .containsExactly("//test:rule1_1", "//test:rule1_2"); |
| } |
| |
| /** |
| * Tests that intra-evaluation ActionExecutionState state remains consistent with restarted deps: |
| * the sources for an action should be updated if the sources non-deterministically change after |
| * they are regenerated. |
| * |
| * <p>To produce this scenario, we rewind the source-generating action after the C++ action |
| * discovers its inputs but before it actually executes. We then release the C++ action, at which |
| * point it encounters an undone-previously-done input. If the C++ action uses its stale state, it |
| * will use a stale input digest. |
| * |
| * <p>The sequence of events is: |
| * |
| * <ol> |
| * <li>loser requests a dependency on its loose header, marker_include.h |
| * <li>restarter claims that source.cc is a lost input and forces a rewind of gensrc |
| * <li>gensrc is marked dirty because it is rewound |
| * <li>loser restarts evaluation |
| * <li>loser notices gensrc is no longer done and resets itself |
| * <li>gensrc re-executes |
| * <li>restarter notes the new digest of source.cc |
| * <li>loser executes, we check that the digest matches the new one |
| * </ol> |
| */ |
| public final void runInputDiscoveringActionNoticesMissingDep() throws Exception { |
| testCase.write( |
| "test/BUILD", |
| "genrule(name = 'gensrc', outs = ['source.cc'], srcs = ['gensrc.sh'],", |
| " cmd = '$< > $@ && cat $@')", |
| "cc_library(name = 'loser', srcs = ['source.cc'], hdrs_check = 'loose')", |
| "genrule(name = 'reader', srcs = ['output.inlined'], outs = ['out'], cmd = 'touch $@')", |
| "genrule(name = 'restarter', srcs = ['source.cc'], outs = ['output.inlined'],", |
| " cmd = 'cp $< $@')"); |
| testCase |
| .write("test/gensrc.sh", "echo '#include \"test/marker_include.h\" //'$RANDOM") |
| .setExecutable(true); |
| testCase.write("test/marker_include.h"); |
| |
| CountDownLatch looseHeaderDepDeclared = new CountDownLatch(1); |
| CountDownLatch gensrcRestarted = new CountDownLatch(1); |
| CountDownLatch cppRestarted = new CountDownLatch(1); |
| Label cppLabel = Label.parseAbsoluteUnchecked("//test:loser"); |
| Label gensrcLabel = Label.parseAbsoluteUnchecked("//test:gensrc"); |
| injectListenerAtStartOfNextBuild( |
| (key, type, order, context) -> { |
| if (EventType.ADD_REVERSE_DEP.equals(type) |
| && isActionExecutionKey(context, cppLabel) |
| && (key instanceof Artifact) |
| && ((Artifact) key).getFilename().equals("marker_include.h")) { |
| looseHeaderDepDeclared.countDown(); |
| } else if (EventType.MARK_DIRTY.equals(type) |
| && Order.AFTER.equals(order) |
| && isActionExecutionKey(key, gensrcLabel)) { |
| gensrcRestarted.countDown(); |
| } else if (EventType.IS_READY.equals(type) |
| && isActionExecutionKey(key, cppLabel) |
| && looseHeaderDepDeclared.getCount() == 0) { |
| TrackingAwaiter.INSTANCE.awaitLatchAndTrackExceptions( |
| gensrcRestarted, "Gensrc not restarted in time for waiting C++ action"); |
| } else if (EventType.RESET_FOR_RESTART_FROM_SCRATCH.equals(type) |
| && isActionExecutionKey(key, cppLabel)) { |
| cppRestarted.countDown(); |
| } else if (EventType.SET_VALUE.equals(type) |
| && isActionExecutionKey(key, gensrcLabel) |
| && gensrcRestarted.getCount() == 0) { |
| TrackingAwaiter.INSTANCE.awaitLatchAndTrackExceptions( |
| cppRestarted, "Cpp not restarted in time for waiting gensrc"); |
| } |
| }); |
| addSpawnShim( |
| "Executing genrule //test:restarter", |
| (spawn, context) -> { |
| TrackingAwaiter.INSTANCE.awaitLatchAndTrackExceptions(looseHeaderDepDeclared, "include"); |
| ImmutableList<ActionInput> ccInputList = |
| ImmutableList.of(SpawnInputUtils.getInputWithName(spawn, "source.cc")); |
| return createLostInputsExecException( |
| context, ccInputList, new ActionInputDepOwnerMap(ccInputList)); |
| }); |
| AtomicReference<String> sourceDigest = new AtomicReference<>(); |
| CountDownLatch digestRead = new CountDownLatch(1); |
| addSpawnShim( |
| "Executing genrule //test:restarter", |
| (spawn, context) -> { |
| sourceDigest.set( |
| getHexDigest(SpawnInputUtils.getInputWithName(spawn, "source.cc"), context)); |
| digestRead.countDown(); |
| return ExecResult.delegate(); |
| }); |
| addSpawnShim( |
| "Compiling test/source.cc", |
| (spawn, context) -> { |
| if (cppRestarted.getCount() == 0) { |
| TrackingAwaiter.INSTANCE.awaitLatchAndTrackExceptions(digestRead, "digest read"); |
| assertThat(getHexDigest(SpawnInputUtils.getInputWithName(spawn, "source.cc"), context)) |
| .isEqualTo(sourceDigest.get()); |
| } |
| return ExecResult.delegate(); |
| }); |
| |
| // Make gensrc actually execute twice: simulates output falling out of remote cache. |
| disableRemoteCaching(); |
| |
| testCase.buildTarget("//test:loser", "//test:restarter"); |
| |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsAtLeast( |
| "Executing genrule //test:gensrc", |
| "Executing genrule //test:restarter", |
| "Executing genrule //test:gensrc", |
| "Compiling test/source.cc") |
| .inOrder(); |
| |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of(), |
| /*completedRewound=*/ ImmutableList.of("Executing genrule //test:gensrc"), |
| /*failedRewound=*/ ImmutableList.of("Executing genrule //test:restarter"), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of(), |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of(1)); |
| |
| // TODO(b/228090759): Verify that deps are correct (see assertion that used to be here). |
| |
| TrackingAwaiter.INSTANCE.assertNoErrors(); |
| } |
| |
| private static void writeTwoGenrulePackage(BuildIntegrationTestCase testCase) throws IOException { |
| testCase.write( |
| "test/BUILD", |
| "genrule(name = 'rule1',", |
| " srcs = ['source_1.txt'],", |
| " outs = ['intermediate.txt'],", |
| " cmd = '(cat $< && echo from rule1) > $@')", |
| "genrule(name = 'rule2',", |
| " srcs = ['intermediate.txt', 'source_2.txt'],", |
| " outs = ['output.inlined'],", |
| " cmd = '(cat $(SRCS) && echo from rule2) > $@')", |
| "", |
| "genrule(name = 'consume_output',", |
| " srcs = [':output.inlined'],", |
| " outs = ['dummy.out'],", |
| " cmd = 'touch $@')"); |
| |
| testCase.write("test/source_1.txt", "source_1"); |
| testCase.write("test/source_2.txt", "source_2"); |
| } |
| |
| public final void runActionFromPreviousBuildReevaluated() throws Exception { |
| // This test sets up a genrule, rule2, that consumes the outputs of rule1. rule1 is requested on |
| // the first build, so that on the second build, when rule2 discovers its missing input, rule1 |
| // is cached. |
| writeTwoGenrulePackage(testCase); |
| |
| testCase.buildTarget("//test:rule1"); |
| assertThat(getExecutedSpawnDescriptions()).containsExactly("Executing genrule //test:rule1"); |
| |
| spawnController.clearExecutedSpawnDescriptions(); |
| // The first time rule2 is executed, the execution strategy fails, saying that rule2's input |
| // file is missing. |
| addSpawnShim( |
| "Executing genrule //test:rule2", |
| (spawn, context) -> { |
| ImmutableList<ActionInput> lostInputs = |
| ImmutableList.of(SpawnInputUtils.getInputWithName(spawn, "intermediate.txt")); |
| return createLostInputsExecException( |
| context, lostInputs, new ActionInputDepOwnerMap(lostInputs)); |
| }); |
| |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| String outputFileContents = buildAndGetOutput("test", testCase); |
| |
| // The evaluation succeeds, producing the expected output, after re-executing rule1's action. |
| assertThat(outputFileContents).isEqualTo("source_1\nfrom rule1\nsource_2\nfrom rule2\n"); |
| |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactly( |
| "Executing genrule //test:rule2", |
| "Executing genrule //test:rule1", |
| "Executing genrule //test:rule2", |
| "Executing genrule //test:consume_output"); |
| |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of(), |
| /*completedRewound=*/ ImmutableList.of("Executing genrule //test:rule1"), |
| /*failedRewound=*/ ImmutableList.of("Executing genrule //test:rule2"), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of(), |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of(0, 1)); |
| |
| assertOnlyActionsDirtied(dirtiedKeys); |
| assertThat(dirtiedArtifactOwnerLabels(dirtiedKeys)).containsExactly("//test:rule1"); |
| } |
| |
| public final void runMultiplyLosingInputsFails() throws Exception { |
| // This test sets up two genrules, and makes the several execution attempts of rule2 fail, |
| // saying that the file produced by rule1 is missing. The last time rule2 fails because of the |
| // same lost input, rewinding is not attempted, and the build fails with a crash because |
| // BugReport throws in tests. |
| writeTwoGenrulePackage(testCase); |
| |
| // Store a reference to the input so that we can match the exception message. The output |
| // directory name (and hence the string representation) varies by platform. |
| AtomicReference<ActionInput> intermediate = new AtomicReference<>(); |
| for (int i = 0; i <= ActionRewindStrategy.MAX_REPEATED_LOST_INPUTS; i++) { |
| addSpawnShim( |
| "Executing genrule //test:rule2", |
| (spawn, context) -> { |
| intermediate.set(SpawnInputUtils.getInputWithName(spawn, "intermediate.txt")); |
| return ExecResult.ofException( |
| new LostInputsExecException( |
| ImmutableMap.of("fakedigest", intermediate.get()), |
| new ActionInputDepOwnerMap(ImmutableList.of(intermediate.get())))); |
| }); |
| } |
| |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| RuntimeException e = |
| assertThrows(RuntimeException.class, () -> testCase.buildTarget("//test:rule2")); |
| IllegalStateException illegalStateException = (IllegalStateException) e.getCause(); |
| // Command thread is not interrupted because crash propagates up and is finally processed on |
| // command thread. |
| assertThat(Thread.currentThread().isInterrupted()).isFalse(); |
| |
| String errorDetail = |
| String.format( |
| "lost input too many times (#%s) for the same action. lostInput: %s, " |
| + "lostInput digest: fakedigest, " |
| + "failedAction: action 'Executing genrule //test:rule2'", |
| ActionRewindStrategy.MAX_REPEATED_LOST_INPUTS + 1, intermediate.get()); |
| assertThat(illegalStateException).hasMessageThat().contains(errorDetail); |
| |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactlyElementsIn( |
| Iterables.concat( |
| Collections.nCopies( |
| ActionRewindStrategy.MAX_REPEATED_LOST_INPUTS + 1, |
| ImmutableList.of( |
| "Executing genrule //test:rule1", "Executing genrule //test:rule2")))) |
| .inOrder(); |
| |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of(), |
| /*completedRewound=*/ ImmutableList.of("Executing genrule //test:rule1"), |
| /*failedRewound=*/ ImmutableList.of(), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of(), |
| /*expectResultReceivedForFailedRewound=*/ false, |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of( |
| ActionRewindStrategy.MAX_REPEATED_LOST_INPUTS + 1), |
| /*lostInputAndActionsPosts=*/ ImmutableList.of( |
| ImmutableList.copyOf( |
| Collections.nCopies( |
| // 3 invalidated nodes: |
| // 1. The failed action itself. |
| // 2. The ArtifactNestedSet node for [srcs] in |
| // action.getInputs() = {genrule-setup.sh, [srcs]}. |
| // 3. The lost input's generating action. |
| 5, ActionRewindEventAsserter.create("Genrule", "//test:rule2", 1, 3))))); |
| |
| assertOnlyActionsDirtied(dirtiedKeys); |
| assertThat(Iterables.frequency(dirtiedArtifactOwnerLabels(dirtiedKeys), "//test:rule1")) |
| .isEqualTo(ActionRewindStrategy.MAX_REPEATED_LOST_INPUTS); |
| assertAndClearBugReporterStoredCrash(RuntimeException.class); |
| } |
| |
| /** |
| * Create N genrules that are dependent on a static source file. And then create another N |
| * genrules that will consume the previous genrules equal to its index from 1 to N. For example, |
| * if N = 3, the first consume genrule will contain 'rule1', the second consume genrule will |
| * contain 'rule1' and 'rule2', and the third consume genrule will contain 'rule1', 'rule2', and |
| * 'rule3'. Lastly there is a genrule that will have an output: 'output.inlined' that contains all |
| * N consume genrules as sources; this is used to assert that no output file remains after the |
| * test case. |
| */ |
| private void writeNGenrulePackages(int n) throws IOException { |
| List<String> lines = new ArrayList<>(); |
| for (int i = 1; i <= n; i++) { |
| testCase.write("test/source_" + i + ".txt", "source_" + i); |
| lines.add("genrule("); |
| lines.add(" name = 'rule" + i + "',"); |
| lines.add(" srcs = ['source_" + i + ".txt'],"); |
| lines.add(" outs = ['out_" + i + ".txt'],"); |
| lines.add(" cmd = '(cat $(SRCS) && echo from rule" + i + ") > $@') "); |
| lines.add(""); |
| } |
| StringBuilder outs = new StringBuilder(); |
| for (int i = 1; i <= n; i++) { |
| String out = " 'consume_" + i + ".out', "; |
| outs.append(out); |
| StringBuilder entries = new StringBuilder(); |
| for (int e = 1; e <= i; e++) { |
| entries.append("':out_").append(e).append(".txt', "); |
| } |
| lines.add("genrule("); |
| lines.add(" name = 'consume_" + i + "',"); |
| lines.add(" srcs = [" + entries + "],"); |
| lines.add(" outs = [" + out + "],"); |
| lines.add(" cmd = '(cat $(SRCS) && echo from consume_" + i + ") > $@') "); |
| lines.add(""); |
| } |
| lines.add("genrule("); |
| lines.add(" name = 'consume_output',"); |
| lines.add(" srcs = [" + outs + "],"); |
| lines.add(" outs = ['output.inlined'],"); |
| lines.add(" cmd = 'touch $@')"); |
| String[] writeLines = new String[lines.size()]; |
| for (int i = 0; i < lines.size(); i++) { |
| writeLines[i] = lines.get(i); |
| } |
| testCase.write("test/BUILD", writeLines); |
| } |
| |
| /** |
| * This test sets up {@link ActionRewindStrategy#MAX_ACTION_REWIND_EVENTS} + 1 (N) genrules that |
| * consume 1 ... N inputs respectively and will build each of the genrules. All N inputs will be |
| * lost and throw a {@link LostInputsExecException} such that all of the genrule actions will |
| * rewind. The {@link ActionRewindingStats} event will contain the top {@link |
| * ActionRewindStrategy#MAX_ACTION_REWIND_EVENTS} action rewind events based on the maximum number |
| * of nodes invalidated for each rewind action plan. The expected action rewind events logged will |
| * not contain the genrule action with one input. |
| */ |
| public final void runMultipleLostInputsForRewindPlan() throws Exception { |
| if (!supportsConcurrentRewinding()) { |
| testCase.addOptions("--jobs=1"); |
| } |
| writeNGenrulePackages(ActionRewindStrategy.MAX_ACTION_REWIND_EVENTS + 1); |
| for (int i = 1; i <= ActionRewindStrategy.MAX_ACTION_REWIND_EVENTS + 1; i++) { |
| final int target = i; |
| addSpawnShim( |
| "Executing genrule //test:consume_" + target, |
| (spawn, context) -> { |
| ImmutableMap.Builder<String, ActionInput> inputMapBuilder = ImmutableMap.builder(); |
| for (int e = 1; e <= target; e++) { |
| ActionInput input = SpawnInputUtils.getInputWithName(spawn, "out_" + e + ".txt"); |
| inputMapBuilder.put("fake_digest_" + target + "_" + e, input); |
| } |
| ImmutableMap<String, ActionInput> inputMap = inputMapBuilder.buildOrThrow(); |
| return ExecResult.ofException( |
| new LostInputsExecException( |
| inputMap, new ActionInputDepOwnerMap(inputMap.values()))); |
| }); |
| } |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| testCase.buildTarget( |
| "//test:consume_1", |
| "//test:consume_2", |
| "//test:consume_3", |
| "//test:consume_4", |
| "//test:consume_5", |
| "//test:consume_6"); |
| assertOnlyActionsDirtied(dirtiedKeys); |
| recorder.assertRewindActionStats( |
| /*totalLostInputCounts=*/ ImmutableList.of(21), |
| /*lostInputAndActionsPosts=*/ ImmutableList.of( |
| ImmutableList.of( |
| // All invalidated node counts are len(srcs) + 2 (+1 for the failed action itself, |
| // and +1 for the ArtifactNestedSet node for [srcs] in |
| // action.getInputs() = {genrule-setup.sh, [srcs]}. |
| ActionRewindEventAsserter.create("Genrule", "//test:consume_6", 6, 8), |
| ActionRewindEventAsserter.create("Genrule", "//test:consume_5", 5, 7), |
| ActionRewindEventAsserter.create("Genrule", "//test:consume_4", 4, 6), |
| ActionRewindEventAsserter.create("Genrule", "//test:consume_3", 3, 5), |
| ActionRewindEventAsserter.create("Genrule", "//test:consume_2", 2, 4)))); |
| } |
| |
| public final void runInterruptedDuringRewindStopsNormally() throws Exception { |
| // This test sets up two genrules, and makes the first execution of rule2 fail, saying that the |
| // file produced by rule1 is missing. Before rule1 is re-executed, the test interrupts the |
| // build. The build should stop with an interrupt normally (and not crash). |
| writeTwoGenrulePackage(testCase); |
| |
| Thread mainThread = Thread.currentThread(); |
| addSpawnShim( |
| "Executing genrule //test:rule2", |
| (spawn, context) -> { |
| addSpawnShim( |
| "Executing genrule //test:rule1", |
| (ignoredSpawn, ignoredContext) -> { |
| mainThread.interrupt(); |
| return ExecResult.delegate(); |
| }); |
| |
| ImmutableList<ActionInput> lostInputs = |
| ImmutableList.of(SpawnInputUtils.getInputWithName(spawn, "intermediate.txt")); |
| return createLostInputsExecException( |
| context, lostInputs, new ActionInputDepOwnerMap(lostInputs)); |
| }); |
| |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| assertThrows(InterruptedException.class, () -> testCase.buildTarget("//test:rule2")); |
| |
| assertOutputForStopBeforeRewoundReexecution(); |
| |
| assertOnlyActionsDirtied(dirtiedKeys); |
| assertThat(dirtiedArtifactOwnerLabels(dirtiedKeys)).containsExactly("//test:rule1"); |
| } |
| |
| private void assertOutputForStopBeforeRewoundReexecution() { |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactly( |
| "Executing genrule //test:rule1", |
| "Executing genrule //test:rule2", |
| "Executing genrule //test:rule1") |
| .inOrder(); |
| |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of("Executing genrule //test:rule1"), |
| /*completedRewound=*/ ImmutableList.of(), |
| /*failedRewound=*/ ImmutableList.of(), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of(), |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of(1)); |
| assertThat( |
| recorder.getActionStartedEvents().stream() |
| .map(e -> ActionEventRecorder.progressMessageOrPrettyPrint(e.getAction())) |
| .filter("Executing genrule //test:rule2"::equals) |
| .count()) |
| .isEqualTo(1); |
| assertThat( |
| recorder.getActionCompletionEvents().stream() |
| .map(e -> ActionEventRecorder.progressMessageOrPrettyPrint(e.getAction())) |
| .filter("Executing genrule //test:rule2"::equals) |
| .count()) |
| .isEqualTo(0); |
| assertThat( |
| recorder.getActionExecutedEvents().stream() |
| .map(e -> ActionEventRecorder.progressMessageOrPrettyPrint(e.getAction())) |
| .filter("Executing genrule //test:rule2"::equals) |
| .count()) |
| .isEqualTo(0); |
| assertThat( |
| recorder.getActionResultReceivedEvents().stream() |
| .map(e -> ActionEventRecorder.progressMessageOrPrettyPrint(e.getAction())) |
| .filter("Executing genrule //test:rule2"::equals) |
| .count()) |
| .isEqualTo(0); |
| assertThat( |
| recorder.getActionRewoundEvents().stream() |
| .map( |
| e -> |
| ActionEventRecorder.progressMessageOrPrettyPrint( |
| e.getFailedRewoundAction())) |
| .filter("Executing genrule //test:rule2"::equals) |
| .count()) |
| .isEqualTo(1); |
| } |
| |
| private static final SpawnResult FAILED_RESULT = |
| new SpawnResult.Builder() |
| .setStatus(SpawnResult.Status.NON_ZERO_EXIT) |
| .setExitCode(1) |
| .setFailureDetail( |
| FailureDetail.newBuilder() |
| .setSpawn(FailureDetails.Spawn.newBuilder().setCode(Code.NON_ZERO_EXIT)) |
| .build()) |
| .setRunnerName("remote") |
| .build(); |
| |
| public final void runFailureDuringRewindStopsNormally() throws Exception { |
| // This test sets up two genrules, and makes the first execution of rule2 fail, saying that the |
| // file produced by rule1 is missing. The execution of rule1 fails. The build should stop with |
| // that failure (and not crash). |
| writeTwoGenrulePackage(testCase); |
| |
| addSpawnShim( |
| "Executing genrule //test:rule2", |
| (spawn, context) -> { |
| addSpawnShim( |
| "Executing genrule //test:rule1", |
| (ignoredSpawn, ignoredContext) -> |
| ExecResult.ofException( |
| new SpawnExecException( |
| "kaboom", |
| FAILED_RESULT, |
| /*forciblyRunRemotely=*/ false, |
| /*catastrophe=*/ false))); |
| |
| ImmutableList<ActionInput> lostInputs = |
| ImmutableList.of(SpawnInputUtils.getInputWithName(spawn, "intermediate.txt")); |
| return createLostInputsExecException( |
| context, lostInputs, new ActionInputDepOwnerMap(lostInputs)); |
| }); |
| |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| BuildFailedException buildFailedException = |
| assertThrows(BuildFailedException.class, () -> testCase.buildTarget("//test:rule2")); |
| |
| String errorDetail = "Executing genrule //test:rule1 failed: (Exit 1)"; |
| if (keepGoing()) { |
| assertThat(buildFailedException).hasMessageThat().isNull(); |
| } else { |
| assertThat(buildFailedException).hasMessageThat().contains(errorDetail); |
| } |
| testCase.assertContainsError(errorDetail); |
| assertOutputForStopBeforeRewoundReexecution(); |
| assertOnlyActionsDirtied(dirtiedKeys); |
| assertThat(dirtiedArtifactOwnerLabels(dirtiedKeys)).containsExactly("//test:rule1"); |
| } |
| |
| public final void runIntermediateActionRewound() throws Exception { |
| // This test sets up three genrules, and makes the first execution of rule3 fail, saying that |
| // the file produced by rule2 is missing but the file produced by rule1 is not. |
| // Rule2 executes twice, consuming the file output from rule1 both times. Rule1 is not executed |
| // a second time. |
| testCase.write( |
| "test/BUILD", |
| "genrule(name = 'rule1',", |
| " srcs = ['source_1.txt'],", |
| " outs = ['intermediate_1.txt'],", |
| " cmd = '(cat $< && echo from rule1) > $@')", |
| "genrule(name = 'rule2',", |
| " srcs = ['intermediate_1.txt', 'source_2.txt'],", |
| " outs = ['intermediate_2.txt'],", |
| " cmd = '(cat $(SRCS) && echo from rule2) > $@')", |
| "genrule(name = 'rule3',", |
| " srcs = ['intermediate_1.txt', 'intermediate_2.txt', 'source_3.txt'],", |
| " outs = ['output.inlined'],", |
| " cmd = '(cat $(SRCS) && echo from rule3) > $@')", |
| "", |
| "genrule(name = 'consume_output',", |
| " srcs = [':output.inlined'],", |
| " outs = ['dummy.out'],", |
| " cmd = 'touch $@')"); |
| |
| testCase.write("test/source_1.txt", "source_1"); |
| testCase.write("test/source_2.txt", "source_2"); |
| testCase.write("test/source_3.txt", "source_3"); |
| |
| addSpawnShim( |
| "Executing genrule //test:rule3", |
| (spawn, context) -> { |
| ImmutableList<ActionInput> lostInputs = |
| ImmutableList.of(SpawnInputUtils.getInputWithName(spawn, "intermediate_2.txt")); |
| return createLostInputsExecException( |
| context, lostInputs, new ActionInputDepOwnerMap(lostInputs)); |
| }); |
| |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| String outputFileContents = buildAndGetOutput("test", testCase); |
| |
| assertThat(outputFileContents) |
| .isEqualTo( |
| "source_1\nfrom rule1\nsource_1\nfrom rule1\nsource_2\nfrom rule2\nsource_3\n" |
| + "from rule3\n"); |
| |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactly( |
| "Executing genrule //test:rule1", |
| "Executing genrule //test:rule2", |
| "Executing genrule //test:rule3", |
| "Executing genrule //test:rule2", |
| "Executing genrule //test:rule3", |
| "Executing genrule //test:consume_output") |
| .inOrder(); |
| |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of("Executing genrule //test:rule1"), |
| /*completedRewound=*/ ImmutableList.of("Executing genrule //test:rule2"), |
| /*failedRewound=*/ ImmutableList.of("Executing genrule //test:rule3"), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of(), |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of(1)); |
| |
| assertOnlyActionsDirtied(dirtiedKeys); |
| assertThat(dirtiedArtifactOwnerLabels(dirtiedKeys)).containsExactly("//test:rule2"); |
| } |
| |
| public final void runChainOfActionsRewound() throws Exception { |
| // This test exercises recursive rewinding. It sets up three genrules that depend on each other |
| // in a chain. Rule1 and rule2 execute successfully their first time. When rule3 executes, it |
| // fails, saying that the file produced by rule2 is missing. When rule2 is executed for the |
| // second time, it fails, saying that the file produced by rule1 is missing. Thereafter, all |
| // executions succeed. |
| testCase.write( |
| "test/BUILD", |
| "genrule(name = 'rule1',", |
| " srcs = ['source_1.txt'],", |
| " outs = ['intermediate_1.txt'],", |
| " cmd = '(cat $< && echo from rule1) > $@')", |
| "genrule(name = 'rule2',", |
| " srcs = ['intermediate_1.txt', 'source_2.txt'],", |
| " outs = ['intermediate_2.txt'],", |
| " cmd = '(cat $(SRCS) && echo from rule2) > $@')", |
| "genrule(name = 'rule3',", |
| " srcs = ['intermediate_2.txt', 'source_3.txt'],", |
| " outs = ['output.inlined'],", |
| " cmd = '(cat $(SRCS) && echo from rule3) > $@')", |
| "", |
| "genrule(name = 'consume_output',", |
| " srcs = [':output.inlined'],", |
| " outs = ['dummy.out'],", |
| " cmd = 'touch $@')"); |
| |
| testCase.write("test/source_1.txt", "source_1"); |
| testCase.write("test/source_2.txt", "source_2"); |
| testCase.write("test/source_3.txt", "source_3"); |
| |
| addSpawnShim( |
| "Executing genrule //test:rule3", |
| (spawn, context) -> { |
| addSpawnShim( |
| "Executing genrule //test:rule2", |
| (otherSpawn, otherContext) -> { |
| ImmutableList<ActionInput> lostInputs = |
| ImmutableList.of( |
| SpawnInputUtils.getInputWithName(otherSpawn, "intermediate_1.txt")); |
| return createLostInputsExecException( |
| otherContext, lostInputs, new ActionInputDepOwnerMap(lostInputs)); |
| }); |
| |
| ImmutableList<ActionInput> lostInputs = |
| ImmutableList.of(SpawnInputUtils.getInputWithName(spawn, "intermediate_2.txt")); |
| return createLostInputsExecException( |
| context, lostInputs, new ActionInputDepOwnerMap(lostInputs)); |
| }); |
| |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| String outputFileContents = buildAndGetOutput("test", testCase); |
| |
| assertThat(outputFileContents) |
| .isEqualTo("source_1\nfrom rule1\nsource_2\nfrom rule2\nsource_3\nfrom rule3\n"); |
| |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactly( |
| "Executing genrule //test:rule1", |
| "Executing genrule //test:rule2", |
| "Executing genrule //test:rule3", |
| "Executing genrule //test:rule2", |
| "Executing genrule //test:rule1", |
| "Executing genrule //test:rule2", |
| "Executing genrule //test:rule3", |
| "Executing genrule //test:consume_output") |
| .inOrder(); |
| |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of(), |
| /*completedRewound=*/ ImmutableList.of( |
| "Executing genrule //test:rule1", "Executing genrule //test:rule2"), |
| /*failedRewound=*/ ImmutableList.of("Executing genrule //test:rule3"), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of(), |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of(2)); |
| |
| assertOnlyActionsDirtied(dirtiedKeys); |
| assertThat(dirtiedArtifactOwnerLabels(dirtiedKeys)) |
| .containsExactly("//test:rule2", "//test:rule1"); |
| } |
| |
| public final void runNondeterministicActionRewound() throws Exception { |
| // This test demonstrates that rewinding works when rewound actions are nondeterministic. |
| // |
| // A nondeterministic genrule, rule1, produces output which is lost. The genrule rule2 uses this |
| // output and fails. The rewound nondeterministic action generates a new output. This test |
| // asserts that rule2 uses the new output on its second try, by checking rule2's output when |
| // it's used by rule3. |
| |
| testCase.write( |
| "test/BUILD", |
| "genrule(name = 'rule1',", |
| " srcs = ['source_1.txt'],", |
| " outs = ['intermediate_1.inlined'],", |
| " cmd = '(cat $(location source_1.txt) && echo $$RANDOM) > $@',", |
| " tags = ['no-cache'])", |
| "", |
| "genrule(name = 'rule2',", |
| " srcs = ['source_2.txt', 'intermediate_1.inlined'],", |
| " outs = ['intermediate_2.inlined'],", |
| " cmd = '(cat $(SRCS) && echo from rule2) > $@')", |
| "", |
| "genrule(name = 'rule3',", |
| " srcs = ['intermediate_2.inlined'],", |
| " outs = ['output.txt'],", |
| " cmd = '(cat $< && echo from rule3) > $@')"); |
| testCase.write("test/source_1.txt", "source_1"); |
| testCase.write("test/source_2.txt", "source_2"); |
| |
| AtomicReference<String> intermediate1FirstContent = new AtomicReference<>(null); |
| addSpawnShim( |
| "Executing genrule //test:rule2", |
| (spawn, context) -> { |
| ActionInput intermediate1 = |
| SpawnInputUtils.getInputWithName(spawn, "intermediate_1.inlined"); |
| intermediate1FirstContent.set(latin1StringFromActionInput(context, intermediate1)); |
| ImmutableList<ActionInput> lostInputs = ImmutableList.of(intermediate1); |
| return createLostInputsExecException( |
| context, lostInputs, new ActionInputDepOwnerMap(lostInputs)); |
| }); |
| |
| AtomicReference<String> intermediate1SecondContent = new AtomicReference<>(null); |
| addSpawnShim( |
| "Executing genrule //test:rule2", |
| (spawn, context) -> { |
| ActionInput intermediate1 = |
| SpawnInputUtils.getInputWithName(spawn, "intermediate_1.inlined"); |
| intermediate1SecondContent.set(latin1StringFromActionInput(context, intermediate1)); |
| return ExecResult.delegate(); |
| }); |
| |
| AtomicReference<String> intermediate2Content = new AtomicReference<>(null); |
| addSpawnShim( |
| "Executing genrule //test:rule3", |
| (spawn, context) -> { |
| ActionInput intermediate2 = |
| SpawnInputUtils.getInputWithName(spawn, "intermediate_2.inlined"); |
| intermediate2Content.set(latin1StringFromActionInput(context, intermediate2)); |
| return ExecResult.delegate(); |
| }); |
| |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| testCase.buildTarget("//test:rule3"); |
| |
| assertThat(intermediate1SecondContent.get()).isNotEqualTo(intermediate1FirstContent.get()); |
| assertThat(intermediate2Content.get()) |
| .isEqualTo(String.format("source_2\n%sfrom rule2\n", intermediate1SecondContent.get())); |
| verifyAllSpawnShimsConsumed(); |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactly( |
| "Executing genrule //test:rule1", |
| "Executing genrule //test:rule2", |
| "Executing genrule //test:rule1", |
| "Executing genrule //test:rule2", |
| "Executing genrule //test:rule3") |
| .inOrder(); |
| |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of("Executing genrule //test:rule3"), |
| /*completedRewound=*/ ImmutableList.of("Executing genrule //test:rule1"), |
| /*failedRewound=*/ ImmutableList.of("Executing genrule //test:rule2"), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of(), |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of(1)); |
| |
| assertOnlyActionsDirtied(dirtiedKeys); |
| assertThat(dirtiedArtifactOwnerLabels(dirtiedKeys)).containsExactly("//test:rule1"); |
| } |
| |
| private void setUpParallelTrackSharedActionPackage() throws IOException { |
| testCase.write( |
| "shared/shared.bzl", |
| "def _shared_impl(ctx):", |
| " in_file = ctx.file.src", |
| " a_shared_out = ctx.actions.declare_file('A-shared.out')", |
| " ctx.actions.run_shell(", |
| " inputs = [in_file],", |
| " outputs = [a_shared_out],", |
| " progress_message = 'Copying %s input %s to A-shared.out' % (ctx.attr.name," |
| + " in_file.short_path),", |
| " command = 'cp %s %s' % (in_file.path, a_shared_out.path),", |
| " )", |
| " b_shared_out = ctx.actions.declare_file('B-shared.out')", |
| " ctx.actions.run_shell(", |
| " inputs = [a_shared_out],", |
| " outputs = [b_shared_out],", |
| " progress_message = 'Copying A-shared.out to B-shared.out on behalf of %s' % " |
| + "(ctx.attr.name),", |
| " command = 'cp %s %s' % (a_shared_out.path, b_shared_out.path),", |
| " )", |
| " out = ctx.outputs.out", |
| " ctx.actions.run_shell(", |
| " inputs = [b_shared_out],", |
| " outputs = [out],", |
| " progress_message = 'Copying B-shared.out to %s output %s' % (ctx.attr.name," |
| + " out.short_path),", |
| " command = 'cp %s %s' % (b_shared_out.path, out.path),", |
| " )", |
| " return [DefaultInfo(files = depset([out]))]", |
| "", |
| "shared = rule(", |
| " implementation = _shared_impl,", |
| " attrs = {", |
| " 'src': attr.label(", |
| " mandatory = True,", |
| " allow_single_file = True,", |
| " ),", |
| " 'out': attr.output(", |
| " mandatory = True", |
| " ),", |
| " }", |
| ")"); |
| testCase.write( |
| "shared/BUILD", |
| "load('//shared:shared.bzl', 'shared')", |
| "", |
| "genrule(", |
| " name = 'shared_input',", |
| " srcs = [],", |
| " outs = ['shared_input.txt'],", |
| " cmd = 'echo \"hi i am a shared input\" > $@')", |
| "", |
| "shared(", |
| " name = 'shared_1',", |
| " src = 'shared_input.txt',", |
| " out = 'shared_1.out')", |
| "", |
| "shared(", |
| " name = 'shared_2',", |
| " src = 'shared_input.txt',", |
| " out = 'shared_2.out')", |
| "", |
| "genrule(name = 'merge_shared_rules',", |
| " srcs = ['shared_1.out', 'shared_2.out'],", |
| " outs = ['output.inlined'],", |
| " cmd = '(cat $(location shared_1.out) && cat $(location shared_2.out)) > $@')", |
| "", |
| "genrule(name = 'consume_output',", |
| " srcs = [':output.inlined'],", |
| " outs = ['dummy.out'],", |
| " cmd = 'touch $@')"); |
| } |
| |
| private static boolean actionHasLabelAndIndex( |
| ActionLookupData actionLookupData, String labelName, int index) { |
| Label label = actionLookupData.getLabel(); |
| return label != null |
| && label.getName().equals(labelName) |
| && actionLookupData.getActionIndex() == index; |
| } |
| |
| public final void runParallelTrackSharedActionsRewound() throws Exception { |
| // This test demonstrates that, given a pair of parallel sequences of shared actions like so: |
| // |
| // 1B 2B (higher actions depend on lower actions) |
| // | | |
| // 1A 2A |
| // |
| // in which 1A and 2A are shared, 1B and 2B are shared, and xB depends on an output of xA, |
| // when 1B rewinds because the 1A output it depends on is lost, and 2B ran simultaneously with |
| // the first, failed, evaluation of 1B and registers itself as depending on 1B's completion |
| // future, then 2B gets restarted when 1B clears its ActionExecutionState. Re-evaluations of |
| // dep actions may proceed non-deterministically, but this test makes 2A win the "rewound A" |
| // race, and then 1B win the "rewound B" race. |
| |
| setUpParallelTrackSharedActionPackage(); |
| |
| addSpawnShim( |
| "Copying A-shared.out to B-shared.out on behalf of shared_1", |
| (spawn, context) -> { |
| ImmutableList<ActionInput> sharedInput = |
| ImmutableList.of(SpawnInputUtils.getInputWithName(spawn, "A-shared.out")); |
| return createLostInputsExecException( |
| context, sharedInput, new ActionInputDepOwnerMap(sharedInput)); |
| }); |
| addSpawnShim( |
| "Copying A-shared.out to B-shared.out on behalf of shared_2", |
| (spawn, context) -> { |
| ImmutableList<ActionInput> sharedInput = |
| ImmutableList.of(SpawnInputUtils.getInputWithName(spawn, "A-shared.out")); |
| return createLostInputsExecException( |
| context, sharedInput, new ActionInputDepOwnerMap(sharedInput)); |
| }); |
| |
| // This code controls the evaluation of the shared actions belonging to shared_1 and shared_2 |
| // so that the following events occur in the specified order. Each non-final step is associated |
| // with a latch which prevents the subsequent step from happening before the preceding step |
| // happens. |
| // |
| // 1. shared_1's B-shared.out generating action (hereafter referred to as "shared_1B", and |
| // likewise for other actions) emits an ActionStartedEvent, discovers its input is lost, |
| // emits an ActionRewoundEvent, but does not yet clear its ActionExecutionState from |
| // SkyframeActionExecutor. |
| CountDownLatch shared1BEmittedRewoundEvent = new CountDownLatch(1); |
| |
| // 2. shared_2A coalesces with shared_1A's done ActionExecutionState. shared_2B coalesces with |
| // action_1B's not-done ActionExecutionState. It declares a Future dependency, and waits. |
| CountDownLatch shared2BDeclaresFutureDep = new CountDownLatch(1); |
| |
| // 3. shared_1B clears its ActionExecutionState from SkyframeActionExecutor, triggering |
| // shared_2B's re-evaluation. shared_1B also clears shared_1A's ActionExecutionState. |
| // shared_2B does not find a matching ActionExecutionState, proceeds with its own evaluation, |
| // and discovers its input is lost also (which would be realistic, given that neither |
| // shared_1A nor shared_2A have re-evaluated). |
| // |
| // shared_2B clears its ActionExecutionState, attempts to clear any ActionExecutionState |
| // associated with shared_2A (but there is none), requests shared_2A's re-evaluation, |
| // shared_2A re-evaluates, and shared_2B is ready to evaluate for its fifth(*) time. |
| // |
| // (*) Count: |
| // 1. before shared_2A is first evaluated |
| // 2. after shared_2A is first evaluated |
| // 3. restarted by shared_1B's state clearing |
| // 4. restarted by its own rewinding, before shared_2A is again evaluated |
| // 5. after shared_2A is again evaluated |
| CountDownLatch shared2BReadyForFifthTime = new CountDownLatch(1); |
| |
| // 4. shared_1A coalesces with the done ActionExecutionState from shared_2A's second evaluation, |
| // and shared_1B successfully re-evaluates. |
| CountDownLatch shared1BDone = new CountDownLatch(1); |
| |
| // 5. shared_2B coalesces with shared_1B's done ActionExecutionState, and the build successfully |
| // completes. |
| |
| AtomicInteger shared1ADirtied = new AtomicInteger(0); |
| AtomicInteger shared2ADirtied = new AtomicInteger(0); |
| AtomicInteger shared2AReady = new AtomicInteger(0); |
| AtomicInteger shared2BReady = new AtomicInteger(0); |
| injectListenerAtStartOfNextBuild( |
| (key, type, order, context) -> { |
| // Count the times shared_1{A,B} are dirtied. |
| if (type.equals(NotifyingHelper.EventType.MARK_DIRTY) && order.equals(Order.AFTER)) { |
| NotifyingHelper.MarkDirtyAfterContext markDirtyAfterContext = |
| (NotifyingHelper.MarkDirtyAfterContext) context; |
| checkState( |
| markDirtyAfterContext.dirtyType().equals(DirtyType.FORCE_REBUILD), |
| "Unexpected DirtyType %s for key %s", |
| context, |
| key); |
| checkState(key instanceof ActionLookupData, "dirtied key not an action: %s", key); |
| if (actionHasLabelAndIndex((ActionLookupData) key, "shared_1", 0)) { |
| checkState(shared1ADirtied.incrementAndGet() == 1, "shared_1A dirtied twice"); |
| } else if (actionHasLabelAndIndex((ActionLookupData) key, "shared_2", 0)) { |
| checkState(shared2ADirtied.incrementAndGet() == 1, "shared_2A dirtied twice"); |
| } else { |
| throw new IllegalStateException( |
| String.format("dirtied key has unexpected address: %s", key)); |
| } |
| } |
| |
| if (type.equals(EventType.IS_READY) |
| && key instanceof ActionLookupData |
| && actionHasLabelAndIndex((ActionLookupData) key, "shared_2", 0)) { |
| int shared2AReadiedCount = shared2AReady.incrementAndGet(); |
| if (shared2AReadiedCount == 1) { |
| awaitUninterruptibly(shared1BEmittedRewoundEvent); |
| } |
| } |
| |
| if (type.equals(EventType.IS_READY) |
| && key instanceof ActionLookupData |
| && actionHasLabelAndIndex((ActionLookupData) key, "shared_2", 1)) { |
| int shared2BReadiedCount = shared2BReady.incrementAndGet(); |
| if (shared2BReadiedCount == 5) { |
| // Wait to attempt final evaluation of shared_2B until after shared_1B is done. |
| shared2BReadyForFifthTime.countDown(); |
| awaitUninterruptibly(shared1BDone); |
| } |
| } |
| |
| // When shared_2B declares a future dep, allow shared_1B's Skyframe execution attempt to |
| // clear its ActionExecutionState and reset its node. |
| if (type.equals(EventType.ADD_EXTERNAL_DEP) |
| && key instanceof ActionLookupData |
| && actionHasLabelAndIndex((ActionLookupData) key, "shared_2", 1)) { |
| shared2BDeclaresFutureDep.countDown(); |
| } |
| |
| // Wait to attempt the rewound evaluation of shared_1A until after shared_2A finishes its |
| // rewound evaluation and shared_2B is ready again. |
| if (type.equals(EventType.IS_READY) |
| && key instanceof ActionLookupData |
| && actionHasLabelAndIndex((ActionLookupData) key, "shared_1", 0)) { |
| if (shared1ADirtied.get() == 1) { |
| awaitUninterruptibly(shared2BReadyForFifthTime); |
| } |
| } |
| |
| if (type.equals(EventType.SET_VALUE) |
| && key instanceof ActionLookupData |
| && actionHasLabelAndIndex((ActionLookupData) key, "shared_1", 1)) { |
| shared1BDone.countDown(); |
| } |
| }); |
| |
| recorder.setActionRewoundEventSubscriber( |
| rewoundEvent -> { |
| String progressMessage = rewoundEvent.getFailedRewoundAction().getProgressMessage(); |
| if (progressMessage.equals( |
| "Copying A-shared.out to B-shared.out on behalf of shared_1")) { |
| shared1BEmittedRewoundEvent.countDown(); |
| awaitUninterruptibly(shared2BDeclaresFutureDep); |
| } |
| }); |
| |
| String output = buildAndGetOutput("shared", testCase); |
| |
| assertThat(output).isEqualTo("hi i am a shared input\nhi i am a shared input\n"); |
| |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactly( |
| "Executing genrule //shared:shared_input", |
| "Copying shared_1 input shared/shared_input.txt to A-shared.out", |
| "Copying A-shared.out to B-shared.out on behalf of shared_1", |
| "Copying A-shared.out to B-shared.out on behalf of shared_2", |
| "Copying shared_2 input shared/shared_input.txt to A-shared.out", |
| "Copying A-shared.out to B-shared.out on behalf of shared_1", |
| "Copying B-shared.out to shared_1 output shared/shared_1.out", |
| "Copying B-shared.out to shared_2 output shared/shared_2.out", |
| "Executing genrule //shared:merge_shared_rules", |
| "Executing genrule //shared:consume_output"); |
| assertThat(shared1ADirtied.get()).isEqualTo(1); |
| assertThat(shared2ADirtied.get()).isEqualTo(1); |
| } |
| |
| /** |
| * This method defines a package with a Starlark rule "make_cc" and a cc_library rule |
| * "consumes_tree" which depends on "make_cc". The Starlark rule generates a tree artifact. The |
| * cc_library rule class knows how to consume tree artifacts: it uses a separate compilation |
| * action for each file in the tree, and then one linking action for the tree. |
| */ |
| private static void setUpTreeArtifactPackage(BuildIntegrationTestCase testCase) throws Exception { |
| testCase.write( |
| "tree/tree.bzl", |
| "def _tree_impl(ctx):", |
| " tree_artifact = ctx.actions.declare_directory(ctx.attr.name + '_dir.cc')", |
| " ctx.actions.run_shell(", |
| " inputs = ctx.files.srcs,", |
| " outputs = [tree_artifact],", |
| " command = 'touch $1/file1.cc && touch $1/file2.cc',", |
| " arguments = [tree_artifact.path],", |
| " )", |
| " return DefaultInfo(files=depset(direct=[tree_artifact]))", |
| "", |
| "tree = rule(", |
| " implementation = _tree_impl, ", |
| " attrs = {'srcs': attr.label_list(allow_files = True)})"); |
| |
| testCase.write( |
| "tree/BUILD", |
| "load(':tree.bzl', 'tree')", |
| "", |
| "tree(name = 'make_cc', srcs = ['source_1.txt'])", |
| "", |
| "cc_library(name = 'consumes_tree', srcs = [':make_cc', 'source_2.cc'])"); |
| |
| testCase.write("tree/source_1.txt", "source_1"); |
| testCase.write("tree/source_2.cc", "#define FOO"); |
| // Don't want to have to track inclusion extraction for tree file artifacts. |
| testCase.addOptions("--features=-cc_include_scanning"); |
| } |
| |
| public final void runTreeFileArtifactRewound_spawnFailed() throws Exception { |
| runTreeFileArtifactRewound( |
| (spawn, context) -> { |
| ImmutableList<ActionInput> lostInputs = getTreeFileArtifactRewoundLostInputs(spawn); |
| return createLostInputsExecException( |
| context, lostInputs, new ActionInputDepOwnerMap(lostInputs)); |
| }); |
| } |
| |
| static ImmutableList<ActionInput> getTreeFileArtifactRewoundLostInputs(Spawn spawn) { |
| return ImmutableList.of(SpawnInputUtils.getInputWithName(spawn, "make_cc_dir.cc/file1.cc")); |
| } |
| |
| final void runTreeFileArtifactRewound(SpawnShim shim) throws Exception { |
| // This test demonstrates that rewinding works when an action fails due to a lost input which is |
| // a generated TreeFileArtifact that is directly depended on. To emphasize: the failed action |
| // directly depends on a file *contained in the tree*, and does *not* directly depend on the |
| // tree itself. |
| // |
| // The compilation action "Compiling tree/make_cc_dir.cc/file1.cc" fails, saying that |
| // "make_cc_dir.cc/file1.cc", one of the output files in the tree outputted by the "make_cc" |
| // rule, is lost. The action that generated that tree, "Action tree/make_cc_dir.cc", is |
| // restarted along with the failed compilation action. |
| // |
| // This test also confirms that rewinding is compatible with critical-path tracking when a |
| // non-shared action (like this test's compiling actions) fails and is run a second time. |
| |
| setUpTreeArtifactPackage(testCase); |
| |
| addSpawnShim("Compiling tree/make_cc_dir.cc/file1.cc", shim); |
| |
| if (!supportsConcurrentRewinding()) { |
| testCase.addOptions("--jobs=1"); |
| } |
| |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| testCase.buildTarget("//tree:consumes_tree"); |
| |
| verifyAllSpawnShimsConsumed(); |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactly( |
| "Action tree/make_cc_dir.cc", |
| "Compiling tree/make_cc_dir.cc/file1.cc", |
| "Action tree/make_cc_dir.cc", |
| "Compiling tree/make_cc_dir.cc/file1.cc", |
| "Compiling tree/make_cc_dir.cc/file2.cc", |
| "Compiling tree/source_2.cc", |
| "Linking tree/libconsumes_tree.so", |
| "Linking tree/libconsumes_tree.a"); |
| |
| // Rules like cc_library depend on middleman inputs which are part of the toolchain, but none |
| // are involved in rewinding. |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of( |
| "Compiling tree/make_cc_dir.cc/file2.cc", |
| "Linking tree/libconsumes_tree.so", |
| "Linking tree/libconsumes_tree.a"), |
| /*completedRewound=*/ ImmutableList.of("Action tree/make_cc_dir.cc"), |
| /*failedRewound=*/ ImmutableList.of("Compiling tree/make_cc_dir.cc/file1.cc"), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of(), |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of(1)); |
| |
| assertThat(dirtiedKeys).hasSize(1); |
| assertActionKey(dirtiedKeys.get(0), "//tree:make_cc", /*index=*/ 0); |
| } |
| |
| public final void runTreeArtifactRewound_allFilesLost_spawnFailed() throws Exception { |
| // This test demonstrates that rewinding works when an action fails due to a lost input which is |
| // a generated TreeFileArtifact that is *indirectly* depended on. In contrast to what |
| // testTreeFileArtifactRewound tests, in this test the failed action directly depends on the |
| // tree, not the file contained in the tree. |
| // |
| // The linking action "Linking tree/libconsumes_tree.so" fails, saying that the "*.pic.o" files |
| // produced by the compilation actions are lost. The linking action which failed is restarted |
| // along with those compilation actions. |
| // |
| // This test also confirms that rewinding is compatible with critical-path tracking when a |
| // previously completed non-shared action (like this test's compiling actions) is rerun. |
| |
| ImmutableList<String> lostTreeFileArtifactNames = |
| ImmutableList.of("make_cc_dir/file1.pic.o", "make_cc_dir/file2.pic.o"); |
| |
| SpawnShim shim = |
| getTreeArtifactRewoundWhenTreeFilesLostSpawnFailedShim(lostTreeFileArtifactNames); |
| |
| runTreeArtifactRewoundWhenTreeFilesLost(lostTreeFileArtifactNames, shim); |
| } |
| |
| public final void runTreeArtifactRewound_oneFileLost_spawnFailed() throws Exception { |
| // This test is like runTreeArtifactRewound_allFilesLost_spawnFailed, except it loses only one |
| // of the files in the tree that "Linking tree/libconsumes_tree.so" depends on. By doing so it |
| // exercises the case when only a subset of a tree's files are lost. |
| // |
| // The linking action which failed is restarted, and *all* the compilation actions whose outputs |
| // are included by the tree are also restarted. |
| // |
| // It would be better if only the compilation action responsible for the lost file was |
| // restarted, but rewinding is expected to be uncommon, so the overkill effort shouldn't be a |
| // problem in practice. |
| |
| ImmutableList<String> lostTreeFileArtifactNames = ImmutableList.of("make_cc_dir/file1.pic.o"); |
| |
| SpawnShim shim = |
| getTreeArtifactRewoundWhenTreeFilesLostSpawnFailedShim(lostTreeFileArtifactNames); |
| |
| runTreeArtifactRewoundWhenTreeFilesLost(lostTreeFileArtifactNames, shim); |
| } |
| |
| private SpawnShim getTreeArtifactRewoundWhenTreeFilesLostSpawnFailedShim( |
| ImmutableList<String> lostTreeFileArtifactNames) { |
| return (spawn, context) -> { |
| Artifact treeArtifact = getTreeArtifactRewoundWhenTreeFilesLostTree(spawn); |
| ImmutableList<ActionInput> lostTreeFileArtifacts = |
| getTreeArtifactRewoundWhenTreeFilesLostInputs( |
| lostTreeFileArtifactNames, spawn, context, treeArtifact); |
| |
| return createLostInputsExecException( |
| context, |
| lostTreeFileArtifacts, |
| getInputOwners( |
| lostTreeFileArtifacts.stream() |
| .collect(toImmutableSetMultimap(a -> a, a -> treeArtifact)))); |
| }; |
| } |
| |
| static Artifact getTreeArtifactRewoundWhenTreeFilesLostTree(Spawn spawn) { |
| return SpawnInputUtils.getTreeArtifactWithName(spawn, "make_cc_dir"); |
| } |
| |
| static ImmutableList<ActionInput> getTreeArtifactRewoundWhenTreeFilesLostInputs( |
| ImmutableList<String> lostTreeFileArtifactNames, |
| Spawn spawn, |
| ActionExecutionContext context, |
| Artifact treeArtifact) { |
| return lostTreeFileArtifactNames.stream() |
| .map(n -> SpawnInputUtils.getExpandedToArtifact(n, treeArtifact, spawn, context)) |
| .collect(toImmutableList()); |
| } |
| |
| final void runTreeArtifactRewoundWhenTreeFilesLost( |
| ImmutableList<String> lostTreeFileArtifactNames, SpawnShim shim) throws Exception { |
| setUpTreeArtifactPackage(testCase); |
| |
| addSpawnShim("Linking tree/libconsumes_tree.so", shim); |
| |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| testCase.buildTarget("//tree:consumes_tree"); |
| verifyAllSpawnShimsConsumed(); |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactly( |
| "Action tree/make_cc_dir.cc", |
| "Compiling tree/make_cc_dir.cc/file1.cc", |
| "Compiling tree/make_cc_dir.cc/file2.cc", |
| "Compiling tree/source_2.cc", |
| "Linking tree/libconsumes_tree.so", |
| "Compiling tree/make_cc_dir.cc/file1.cc", |
| "Compiling tree/make_cc_dir.cc/file2.cc", |
| "Linking tree/libconsumes_tree.so", |
| "Linking tree/libconsumes_tree.a"); |
| |
| // Rules like cc_library depend on middleman inputs which are part of the toolchain, but none |
| // are involved in rewinding. |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of( |
| "Action tree/make_cc_dir.cc", "Linking tree/libconsumes_tree.a"), |
| /*completedRewound=*/ ImmutableList.of( |
| "Compiling tree/make_cc_dir.cc/file1.cc", "Compiling tree/make_cc_dir.cc/file2.cc"), |
| /*failedRewound=*/ ImmutableList.of("Linking tree/libconsumes_tree.so"), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of(), |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of(lostTreeFileArtifactNames.size())); |
| |
| assertThat(dirtiedKeys).hasSize(3); |
| HashSet<Integer> treeActionIndices = new HashSet<>(ImmutableList.of(0, 1)); |
| for (int i = 0; i < 2; i++) { |
| assertThat(dirtiedKeys.get(i)).isInstanceOf(ActionLookupData.class); |
| assertThat(((ActionLookupData) dirtiedKeys.get(i)).getLabel().getCanonicalForm()) |
| .isEqualTo("//tree:consumes_tree"); |
| assertThat(treeActionIndices.remove(((ActionLookupData) dirtiedKeys.get(i)).getActionIndex())) |
| .isTrue(); |
| } |
| assertArtifactKey(dirtiedKeys.get(2), "tree/_pic_objs/consumes_tree/make_cc_dir"); |
| } |
| |
| public final void runGeneratedRunfilesRewound_allFilesLost_spawnFailed() throws Exception { |
| // This test demonstrates that rewinding works when an action fails due to lost inputs which are |
| // generated files in the action's runfiles. Rewinding must propagate across the middleman |
| // artifacts and actions associated with the runfiles. |
| |
| ImmutableList<String> lostRunfiles = ImmutableList.of("gen1.dat", "gen2.dat"); |
| |
| SpawnShim shim = getGeneratedRunfilesRewoundSpawnFailedShim(lostRunfiles); |
| |
| runGeneratedRunfilesRewound(lostRunfiles, shim); |
| } |
| |
| public final void runGeneratedRunfilesRewound_oneFileLost_spawnFailed() throws Exception { |
| // This test is like runGeneratedRunfilesRewound_allFilesLost_spawnFailed, except it loses only |
| // one of the two generated runfiles that "Executing genrule //middle:tool_user" depends on. |
| // |
| // Like with runTreeArtifactRewound_oneFileLost_spawnFailed, it would be better if only the one |
| // action responsible for the lost input was restarted, but rewinding is expected to be |
| // uncommon, so the overkill effort isn't expected to be a problem in practice. |
| |
| ImmutableList<String> lostRunfiles = ImmutableList.of("gen1.dat"); |
| |
| SpawnShim shim = getGeneratedRunfilesRewoundSpawnFailedShim(lostRunfiles); |
| |
| runGeneratedRunfilesRewound(lostRunfiles, shim); |
| } |
| |
| private SpawnShim getGeneratedRunfilesRewoundSpawnFailedShim(ImmutableList<String> lostRunfiles) { |
| return (spawn, context) -> { |
| ImmutableList<ActionInput> lostRunfileArtifacts = |
| getGeneratedRunfilesRewoundLostRunfiles(lostRunfiles, spawn); |
| return createLostInputsExecException( |
| context, lostRunfileArtifacts, new ActionInputDepOwnerMap(lostRunfileArtifacts)); |
| }; |
| } |
| |
| static ImmutableList<ActionInput> getGeneratedRunfilesRewoundLostRunfiles( |
| ImmutableList<String> lostRunfiles, Spawn spawn) { |
| return lostRunfiles.stream() |
| .map(n -> SpawnInputUtils.getRunfilesArtifactWithName(spawn, n)) |
| .collect(toImmutableList()); |
| } |
| |
| final void runGeneratedRunfilesRewound(ImmutableList<String> lostRunfiles, SpawnShim shim) |
| throws Exception { |
| testCase.write( |
| "middle/BUILD", |
| "genrule(name = 'gen1',", |
| " srcs = [],", |
| " outs = ['gen1.dat'],", |
| " cmd = 'echo \"made by gen1\" > $@')", |
| "genrule(name = 'gen2',", |
| " srcs = [],", |
| " outs = ['gen2.dat'],", |
| " cmd = 'echo \"made by gen2\" > $@')", |
| "sh_binary(name = 'tool',", |
| " srcs = ['tool.sh'],", |
| " data = ['gen1.dat', 'gen2.dat', 'source_1.txt'])", |
| "genrule(name = 'tool_user',", |
| " outs = ['tool_user.out'],", |
| " srcs = [],", |
| " tools = ['tool'],", |
| " cmd = 'touch $(OUTS)')"); |
| testCase.write("middle/tool.sh", "#!/bin/bash").setExecutable(true); |
| testCase.write("middle/source_1.txt", "source_1"); |
| |
| addSpawnShim("Executing genrule //middle:tool_user", shim); |
| |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| testCase.buildTarget("//middle:tool_user"); |
| verifyAllSpawnShimsConsumed(); |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactly( |
| "Executing genrule //middle:gen1 [for tool]", |
| "Executing genrule //middle:gen2 [for tool]", |
| "Executing genrule //middle:tool_user", |
| "Executing genrule //middle:gen1 [for tool]", |
| "Executing genrule //middle:gen2 [for tool]", |
| "Executing genrule //middle:tool_user"); |
| |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of(), |
| /*completedRewound=*/ ImmutableList.of( |
| "Executing genrule //middle:gen1 [for tool]", |
| "Executing genrule //middle:gen2 [for tool]"), |
| /*failedRewound=*/ ImmutableList.of("Executing genrule //middle:tool_user"), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of( |
| middlemanEvent -> |
| ActionEventRecorder.progressMessageOrPrettyPrint(middlemanEvent.getAction()) |
| .equals("runfiles for //middle:tool")), |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of(lostRunfiles.size())); |
| |
| if (buildRunfileManifests()) { |
| assertThat(dirtiedKeys).hasSize(7); |
| HashSet<String> expectedDirtiedGenrules = |
| new HashSet<>(ImmutableList.of("//middle:gen1", "//middle:gen2")); |
| int i = 0; |
| while (i < 5) { |
| assertThat(dirtiedKeys.get(i)).isInstanceOf(ActionLookupData.class); |
| ActionLookupData actionKey = (ActionLookupData) dirtiedKeys.get(i); |
| String actionLabel = actionKey.getLabel().getCanonicalForm(); |
| i++; |
| if (actionLabel.equals("//middle:tool")) { |
| switch (actionKey.getActionIndex()) { |
| case 0: // SymlinkAction |
| break; |
| case 1: // SourceManifestAction |
| assertActionKey(dirtiedKeys.get(i), "//middle:tool", 2); |
| i++; |
| break; |
| default: |
| fail(String.format("Unexpected action index. actionKey: %s, i: %s", actionKey, i)); |
| } |
| } else { |
| assertThat(expectedDirtiedGenrules.remove(actionLabel)).isTrue(); |
| } |
| } |
| |
| assertActionKey(dirtiedKeys.get(i++), "//middle:tool", /*index=*/ 3); |
| assertArtifactKey(dirtiedKeys.get(i), "_middlemen/middle_Stool-runfiles"); |
| } else { |
| assertThat(dirtiedKeys).hasSize(5); |
| HashSet<String> expectedDirtiedGenrules = |
| new HashSet<>(ImmutableList.of("//middle:gen1", "//middle:gen2")); |
| int i = 0; |
| while (i < 3) { |
| assertThat(dirtiedKeys.get(i)).isInstanceOf(ActionLookupData.class); |
| ActionLookupData actionKey = (ActionLookupData) dirtiedKeys.get(i); |
| String actionLabel = actionKey.getLabel().getCanonicalForm(); |
| i++; |
| if (actionLabel.equals("//middle:tool")) { |
| assertThat(actionKey.getActionIndex()).isEqualTo(0); |
| } else { |
| assertThat(expectedDirtiedGenrules.remove(actionLabel)).isTrue(); |
| } |
| } |
| |
| assertActionKey(dirtiedKeys.get(i++), "//middle:tool", /*index=*/ 1); |
| assertArtifactKey(dirtiedKeys.get(i), "_middlemen/middle_Stool-runfiles"); |
| } |
| } |
| |
| public final void runDupeDirectAndRunfilesDependencyRewound_spawnFailed() throws Exception { |
| AtomicReference<String> intermediate1FirstContent = new AtomicReference<>(null); |
| SpawnShim shim = |
| (spawn, context) -> { |
| ActionInput lostInput = getDupeDirectAndRunfilesDependencyRewoundLostInput(spawn); |
| intermediate1FirstContent.set(latin1StringFromActionInput(context, lostInput)); |
| ImmutableList<ActionInput> lostInputs = ImmutableList.of(lostInput); |
| return createLostInputsExecException( |
| context, lostInputs, new ActionInputDepOwnerMap(lostInputs)); |
| }; |
| runDupeDirectAndRunfilesDependencyRewound(intermediate1FirstContent, shim); |
| } |
| |
| static ActionInput getDupeDirectAndRunfilesDependencyRewoundLostInput(Spawn spawn) { |
| return SpawnInputUtils.getRunfilesArtifactWithName(spawn, "intermediate_1.inlined"); |
| } |
| |
| /** |
| * Runs a test which demonstrates that rewinding works when a lost input is both directly depended |
| * on and, via runfiles, indirectly depended on by an action. Rewinding must invalidate both paths |
| * from the failed action to the rewound generating action. |
| * |
| * <p>This checks that the correct nodes were invalidated in the correct order. In particular, the |
| * runfiles action and output artifact must have been invalidated after the artifact corresponding |
| * to the lost input. Otherwise, their evaluation could race with the invalidation of the |
| * generating action and its output artifact. If the runfiles nodes won, they could propagate |
| * stale values for the lost input. |
| */ |
| final void runDupeDirectAndRunfilesDependencyRewound( |
| AtomicReference<String> intermediate1FirstContent, SpawnShim shim) throws Exception { |
| testCase.write( |
| "test/BUILD", |
| "genrule(name = 'rule1',", |
| " srcs = [],", |
| " outs = ['intermediate_1.inlined'],", |
| " cmd = 'echo $$RANDOM > $@',", |
| " tags = ['no-cache'])", |
| "", |
| "sh_binary(name = 'tool',", |
| " srcs = ['tool.sh'],", |
| " data = ['intermediate_1.inlined'])", |
| "", |
| "genrule(name = 'rule2',", |
| " srcs = [],", |
| " outs = ['intermediate_2.inlined'],", |
| " tools = ['tool', 'intermediate_1.inlined'],", |
| " cmd = '($(location tool) && cat $(location intermediate_1.inlined) && ' + ", |
| " 'echo from rule2) > $@')", |
| "", |
| "genrule(name = 'rule3',", |
| " srcs = ['intermediate_2.inlined'],", |
| " outs = ['output.txt'],", |
| " cmd = '(cat $< && echo from rule3) > $@')"); |
| testCase |
| .write( |
| "test/tool.sh", |
| "#!/bin/bash", |
| String.format( |
| "cat ${0}.runfiles/%s/test/intermediate_1.inlined", TestConstants.WORKSPACE_NAME), |
| "echo 'from tool'") |
| .setExecutable(true); |
| |
| addSpawnShim("Executing genrule //test:rule2", shim); |
| |
| AtomicReference<String> intermediate1SecondContent = new AtomicReference<>(null); |
| addSpawnShim( |
| "Executing genrule //test:rule2", |
| (spawn, context) -> { |
| Artifact intermediate1 = |
| SpawnInputUtils.getRunfilesArtifactWithName(spawn, "intermediate_1.inlined"); |
| intermediate1SecondContent.set(latin1StringFromActionInput(context, intermediate1)); |
| return ExecResult.delegate(); |
| }); |
| |
| AtomicReference<String> intermediate2Content = new AtomicReference<>(null); |
| addSpawnShim( |
| "Executing genrule //test:rule3", |
| (spawn, context) -> { |
| ActionInput intermediate2 = |
| SpawnInputUtils.getInputWithName(spawn, "intermediate_2.inlined"); |
| intermediate2Content.set(latin1StringFromActionInput(context, intermediate2)); |
| return ExecResult.delegate(); |
| }); |
| |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| testCase.buildTarget("//test:rule3"); |
| |
| assertThat(intermediate1SecondContent.get()).isNotEqualTo(intermediate1FirstContent.get()); |
| assertThat(intermediate2Content.get()) |
| .isEqualTo( |
| String.format( |
| "%sfrom tool\n%sfrom rule2\n", |
| intermediate1SecondContent.get(), intermediate1SecondContent.get())); |
| verifyAllSpawnShimsConsumed(); |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactly( |
| "Executing genrule //test:rule1 [for tool]", |
| "Executing genrule //test:rule2", |
| "Executing genrule //test:rule1 [for tool]", |
| "Executing genrule //test:rule2", |
| "Executing genrule //test:rule3") |
| .inOrder(); |
| |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of("Executing genrule //test:rule3"), |
| /*completedRewound=*/ ImmutableList.of("Executing genrule //test:rule1 [for tool]"), |
| /*failedRewound=*/ ImmutableList.of("Executing genrule //test:rule2"), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of( |
| middlemanEvent -> |
| ActionEventRecorder.progressMessageOrPrettyPrint(middlemanEvent.getAction()) |
| .equals("runfiles for //test:tool")), |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of(1)); |
| |
| if (buildRunfileManifests()) { |
| assertThat(dirtiedKeys).hasSize(6); |
| int i = 0; |
| while (i < 4) { |
| assertThat(dirtiedKeys.get(i)).isInstanceOf(ActionLookupData.class); |
| ActionLookupData actionKey = (ActionLookupData) dirtiedKeys.get(i); |
| String actionLabel = actionKey.getLabel().getCanonicalForm(); |
| i++; |
| if (actionLabel.equals("//test:tool")) { |
| switch (actionKey.getActionIndex()) { |
| case 0: // SymlinkAction |
| break; |
| case 1: // SourceManifestAction |
| assertActionKey(dirtiedKeys.get(i), "//test:tool", /*index=*/ 2); |
| i++; |
| break; |
| default: |
| fail( |
| String.format( |
| "Unexpected action index. actionKey: %s, dirtiedKeys: %s", |
| actionKey, dirtiedKeys)); |
| } |
| } else { |
| assertThat(actionLabel).isEqualTo("//test:rule1"); |
| } |
| } |
| |
| assertActionKey(dirtiedKeys.get(i++), "//test:tool", /*index=*/ 3); |
| assertArtifactKey(dirtiedKeys.get(i), "_middlemen/test_Stool-runfiles"); |
| } else { |
| assertThat(dirtiedKeys).hasSize(4); |
| int i = 0; |
| while (i < 2) { |
| assertThat(dirtiedKeys.get(i)).isInstanceOf(ActionLookupData.class); |
| ActionLookupData actionKey = (ActionLookupData) dirtiedKeys.get(i); |
| String actionLabel = actionKey.getLabel().getCanonicalForm(); |
| i++; |
| if (actionLabel.equals("//test:tool")) { |
| assertThat(actionKey.getActionIndex()).isEqualTo(0); |
| } else { |
| assertThat(actionLabel).isEqualTo("//test:rule1"); |
| } |
| } |
| |
| assertActionKey(dirtiedKeys.get(i++), "//test:tool", /*index=*/ 1); |
| assertArtifactKey(dirtiedKeys.get(i), "_middlemen/test_Stool-runfiles"); |
| } |
| } |
| |
| public final void runTreeInRunfilesRewound_spawnFailed() throws Exception { |
| SpawnShim shim = |
| (spawn, context) -> { |
| Artifact treeArtifact = getTreeInRunfilesRewoundTree(spawn); |
| ImmutableList<ActionInput> lostInputs = |
| getTreeInRunfilesRewoundLostInputs(spawn, context, treeArtifact); |
| return createLostInputsExecException( |
| context, |
| lostInputs, |
| getInputOwners( |
| ImmutableSetMultimap.of( |
| lostInputs.get(0), treeArtifact, lostInputs.get(1), treeArtifact))); |
| }; |
| |
| runTreeInRunfilesRewound(shim); |
| } |
| |
| static Artifact getTreeInRunfilesRewoundTree(Spawn spawn) { |
| return SpawnInputUtils.getRunfilesArtifactWithName(spawn, "gen_tree"); |
| } |
| |
| static ImmutableList<ActionInput> getTreeInRunfilesRewoundLostInputs( |
| Spawn spawn, ActionExecutionContext context, Artifact treeArtifact) { |
| return ImmutableList.of( |
| SpawnInputUtils.getExpandedToArtifact("gen1.out", treeArtifact, spawn, context), |
| SpawnInputUtils.getExpandedToArtifact("gen2.out", treeArtifact, spawn, context)); |
| } |
| |
| final void runTreeInRunfilesRewound(SpawnShim shim) throws Exception { |
| testCase.write( |
| "middle/tree.bzl", |
| "def _tree_impl(ctx):", |
| " tree_artifact = ctx.actions.declare_directory(ctx.attr.name + '_dir')", |
| " ctx.actions.run_shell(", |
| " inputs = ctx.files.srcs,", |
| " outputs = [tree_artifact],", |
| " command = '(echo \"tree1\" > $1/gen1.out) && (echo \"tree2\" > $1/gen2.out)',", |
| " arguments = [tree_artifact.path],", |
| " )", |
| " return DefaultInfo(files=depset(direct=[tree_artifact]),", |
| " runfiles = ctx.runfiles(files = [tree_artifact]))", |
| "", |
| "tree = rule(", |
| " implementation = _tree_impl, ", |
| " attrs = {'srcs': attr.label_list(allow_files = True)})"); |
| |
| testCase.write( |
| "middle/BUILD", |
| "load(':tree.bzl', 'tree')", |
| "", |
| "tree(name = 'gen_tree', srcs = ['source_1.txt'])", |
| "", |
| "sh_binary(name = 'tool',", |
| " srcs = ['tool.sh'],", |
| " data = [':gen_tree', 'source_2.txt'])", |
| "genrule(name = 'tool_user',", |
| " outs = ['tool_user.out'],", |
| " srcs = [],", |
| " tools = ['tool'],", |
| " cmd = 'touch $(OUTS)')"); |
| testCase.write("middle/tool.sh", "#!/bin/bash").setExecutable(true); |
| testCase.write("middle/source_1.txt", "source_1"); |
| testCase.write("middle/source_2.txt", "source_2"); |
| |
| addSpawnShim("Executing genrule //middle:tool_user", shim); |
| |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| testCase.buildTarget("//middle:tool_user"); |
| verifyAllSpawnShimsConsumed(); |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactly( |
| "Action middle/gen_tree_dir [for tool]", |
| "Executing genrule //middle:tool_user", |
| "Action middle/gen_tree_dir [for tool]", |
| "Executing genrule //middle:tool_user") |
| .inOrder(); |
| |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of(), |
| /*completedRewound=*/ ImmutableList.of("Action middle/gen_tree_dir [for tool]"), |
| /*failedRewound=*/ ImmutableList.of("Executing genrule //middle:tool_user"), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of( |
| middlemanEvent -> |
| ActionEventRecorder.progressMessageOrPrettyPrint(middlemanEvent.getAction()) |
| .equals("runfiles for //middle:tool")), |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of(2)); |
| |
| if (buildRunfileManifests()) { |
| assertThat(dirtiedKeys).hasSize(7); |
| int i = 0; |
| while (i < 5) { |
| assertThat(dirtiedKeys.get(i)).isInstanceOf(ActionLookupData.class); |
| ActionLookupData actionKey = (ActionLookupData) dirtiedKeys.get(i); |
| String actionLabel = actionKey.getLabel().getCanonicalForm(); |
| i++; |
| if (actionLabel.equals("//middle:tool")) { |
| switch (actionKey.getActionIndex()) { |
| case 0: // SymlinkAction |
| break; |
| case 1: // SourceManifestAction |
| assertActionKey(dirtiedKeys.get(i), "//middle:tool", 2); |
| i++; |
| break; |
| default: |
| fail(String.format("Unexpected action index. actionKey: %s", actionKey)); |
| } |
| } else { |
| assertThat(actionLabel).isEqualTo("//middle:gen_tree"); |
| assertArtifactKey(dirtiedKeys.get(i), "middle/gen_tree_dir"); |
| i++; |
| } |
| } |
| |
| assertActionKey(dirtiedKeys.get(i++), "//middle:tool", /*index=*/ 3); |
| assertArtifactKey(dirtiedKeys.get(i), "_middlemen/middle_Stool-runfiles"); |
| } else { |
| assertThat(dirtiedKeys).hasSize(5); |
| int i = 0; |
| while (i < 3) { |
| assertThat(dirtiedKeys.get(i)).isInstanceOf(ActionLookupData.class); |
| ActionLookupData actionKey = (ActionLookupData) dirtiedKeys.get(i); |
| String actionLabel = actionKey.getLabel().getCanonicalForm(); |
| i++; |
| if (actionLabel.equals("//middle:tool")) { |
| assertThat(actionKey.getActionIndex()).isEqualTo(0); |
| } else { |
| assertThat(actionLabel).isEqualTo("//middle:gen_tree"); |
| assertArtifactKey(dirtiedKeys.get(i), "middle/gen_tree_dir"); |
| i++; |
| } |
| } |
| |
| assertActionKey(dirtiedKeys.get(i++), "//middle:tool", /*index=*/ 1); |
| assertArtifactKey(dirtiedKeys.get(i), "_middlemen/middle_Stool-runfiles"); |
| } |
| } |
| |
| /** |
| * Regression test for b/181884247. |
| * |
| * <p>The action for {@code //test:consumer} has three inputs all generated by {@code //test:gen}. |
| * However, the action's {@code depset} of inputs is arranged such that the three artifacts are |
| * split among its children. This tests that rewinding properly handles the case of a requested |
| * {@link ArtifactNestedSetKey} containing only some of the inputs for a particular generating |
| * action. |
| */ |
| public final void runInputsFromSameGeneratingActionSplitAmongNestedSetChildren() |
| throws Exception { |
| testCase.write( |
| "test/defs.bzl", |
| "def _consumer_impl(ctx):", |
| " in1, in2, in3 = ctx.attr.three_output_genrule.files.to_list()", |
| " out = ctx.actions.declare_file('consumer.out')", |
| " ctx.actions.run_shell(", |
| " outputs = [out],", |
| // Arrange the inputs such that they are split among the depset's children. |
| " inputs = depset([in1], transitive = [depset([in2, in3])]),", |
| " command = 'touch %s' % out.path,", |
| " progress_message = 'Running consumer',", |
| " )", |
| " return DefaultInfo(files = depset([out]))", |
| "", |
| "consumer = rule(", |
| " implementation = _consumer_impl,", |
| " attrs = {'three_output_genrule': attr.label(mandatory = True)}", |
| ")"); |
| testCase.write( |
| "test/BUILD", |
| "load(':defs.bzl', 'consumer')", |
| "genrule(name = 'gen', outs = ['gen.out1', 'gen.out2', 'gen.out3'], cmd = 'touch $(OUTS)')", |
| "consumer(name = 'consumer', three_output_genrule = ':gen')"); |
| |
| addSpawnShim( |
| "Running consumer", |
| (spawn, context) -> { |
| ActionInput genOut1 = SpawnInputUtils.getInputWithName(spawn, "gen.out1"); |
| ImmutableList<ActionInput> lostInputs = ImmutableList.of(genOut1); |
| return createLostInputsExecException( |
| context, lostInputs, new ActionInputDepOwnerMap(lostInputs)); |
| }); |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| |
| testCase.buildTarget("//test:consumer"); |
| |
| assertThat(dirtiedKeys).hasSize(1); |
| assertThat(dirtiedArtifactOwnerLabels(dirtiedKeys)).containsExactly("//test:gen"); |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactly( |
| "Executing genrule //test:gen", |
| "Running consumer", |
| "Executing genrule //test:gen", |
| "Running consumer") |
| .inOrder(); |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of(), |
| /*completedRewound=*/ ImmutableList.of("Executing genrule //test:gen"), |
| /*failedRewound=*/ ImmutableList.of("Running consumer"), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of(), |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of(1)); |
| } |
| |
| private static void writeGeneratedHeaderDirectDepPackage(BuildIntegrationTestCase testCase) |
| throws IOException { |
| testCase.write( |
| "genheader/BUILD", |
| "genrule(name = 'gen_header',", |
| " srcs = [],", |
| " outs = ['gen.h'],", |
| " cmd = 'touch $@')", |
| "", |
| "cc_binary(name = 'consumes_header',", |
| " srcs = ['consumes.cc', 'gen.h'])"); |
| testCase.write( |
| "genheader/consumes.cc", |
| "#include \"genheader/gen.h\"", |
| "int main() {", |
| " return 0;", |
| "}"); |
| } |
| |
| public final void runGeneratedHeaderRewound_lostInInputDiscovery_spawnFailed() throws Exception { |
| SpawnShim shim = |
| (spawn, context) -> { |
| ActionInput header = getGeneratedHeaderRewoundLostInput(spawn); |
| ImmutableList<ActionInput> lostInputs = ImmutableList.of(header); |
| return createLostInputsExecException( |
| context, lostInputs, new ActionInputDepOwnerMap(lostInputs)); |
| }; |
| |
| runGeneratedHeaderRewound_lostInInputDiscovery(shim); |
| } |
| |
| static ActionInput getGeneratedHeaderRewoundLostInput(Spawn spawn) { |
| return SpawnInputUtils.getInputWithName(spawn, "genheader/gen.h"); |
| } |
| |
| final void runGeneratedHeaderRewound_lostInInputDiscovery(SpawnShim shim) throws Exception { |
| // This test checks that rewinding works when the lost input is a generated header, and the loss |
| // is found by remote include scanning, which happens in input discovery. |
| writeGeneratedHeaderDirectDepPackage(testCase); |
| |
| addSpawnShim( |
| String.format( |
| "Extracting include lines from %s-out/k8-fastbuild/bin/genheader/gen.h", |
| TestConstants.PRODUCT_NAME), |
| shim); |
| |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| testCase.buildTarget("//genheader:consumes_header"); |
| verifyAllSpawnShimsConsumed(); |
| |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactly( |
| "Executing genrule //genheader:gen_header", |
| "Extracting include lines from genheader/consumes.cc", |
| "Extracting include lines from tools/cpp/malloc.cc", |
| "Compiling tools/cpp/malloc.cc", |
| String.format( |
| "Extracting include lines from %s-out/k8-fastbuild/bin/genheader/gen.h", |
| TestConstants.PRODUCT_NAME), |
| "Executing genrule //genheader:gen_header", |
| String.format( |
| "Extracting include lines from %s-out/k8-fastbuild/bin/genheader/gen.h", |
| TestConstants.PRODUCT_NAME), |
| "Compiling genheader/consumes.cc", |
| "Linking genheader/consumes_header"); |
| |
| // Input discovery actions do not result in action lifecycle events. E.g., the "Extracting |
| // [...]" action is run, but results in no ActionStartedEvent/ActionCompletionEvent/etc. |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of( |
| "Linking genheader/consumes_header", "Compiling genheader/consumes.cc"), |
| /*completedRewound=*/ ImmutableList.of("Executing genrule //genheader:gen_header"), |
| /*failedRewound=*/ ImmutableList.of(), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of( |
| middlemanEvent -> |
| isSchedulingDepMiddlemanForTarget( |
| middlemanEvent.getAction(), "//genheader:consumes_header")), |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of(1)); |
| |
| assertOnlyActionsDirtied(dirtiedKeys); |
| assertThat(dirtiedArtifactOwnerLabels(dirtiedKeys)).containsExactly("//genheader:gen_header"); |
| } |
| |
| static boolean isSchedulingDepMiddlemanForTarget(Action middlemanAction, String expectedTarget) { |
| return ActionEventRecorder.progressMessageOrPrettyPrint(middlemanAction) |
| .contains(expectedTarget) |
| && middlemanAction.getActionType().equals(MiddlemanType.SCHEDULING_DEPENDENCY_MIDDLEMAN); |
| } |
| |
| public final void runGeneratedHeaderRewound_lostInActionExecution_spawnFailed() throws Exception { |
| SpawnShim shim = |
| (spawn, context) -> { |
| ActionInput header = getGeneratedHeaderRewoundLostInput(spawn); |
| ImmutableList<ActionInput> lostInputs = ImmutableList.of(header); |
| return createLostInputsExecException( |
| context, lostInputs, new ActionInputDepOwnerMap(lostInputs)); |
| }; |
| |
| runGeneratedHeaderRewound_lostInActionExecution(shim); |
| } |
| |
| final void runGeneratedHeaderRewound_lostInActionExecution(SpawnShim shim) throws Exception { |
| // This test checks that rewinding works when the lost input is a generated header, and the loss |
| // is found during action execution (after input discovery) |
| // |
| // This test also confirms that rewinding is compatible with critical-path tracking when a |
| // non-shared action (like this test's compiling action) fails and is run a second time. |
| writeGeneratedHeaderDirectDepPackage(testCase); |
| |
| addSpawnShim("Compiling genheader/consumes.cc", shim); |
| |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| testCase.buildTarget("//genheader:consumes_header"); |
| verifyAllSpawnShimsConsumed(); |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactly( |
| "Executing genrule //genheader:gen_header", |
| "Extracting include lines from genheader/consumes.cc", |
| "Extracting include lines from tools/cpp/malloc.cc", |
| "Compiling tools/cpp/malloc.cc", |
| String.format( |
| "Extracting include lines from %s-out/k8-fastbuild/bin/genheader/gen.h", |
| TestConstants.PRODUCT_NAME), |
| "Compiling genheader/consumes.cc", |
| "Executing genrule //genheader:gen_header", |
| "Compiling genheader/consumes.cc", |
| "Linking genheader/consumes_header"); |
| |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of("Linking genheader/consumes_header"), |
| /*completedRewound=*/ ImmutableList.of("Executing genrule //genheader:gen_header"), |
| /*failedRewound=*/ ImmutableList.of("Compiling genheader/consumes.cc"), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of( |
| middlemanEvent -> |
| isSchedulingDepMiddlemanForTarget( |
| middlemanEvent.getAction(), "//genheader:consumes_header")), |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of(1)); |
| |
| assertOnlyActionsDirtied(dirtiedKeys); |
| assertThat(dirtiedArtifactOwnerLabels(dirtiedKeys)).containsExactly("//genheader:gen_header"); |
| } |
| |
| private static void writeGeneratedHeaderIndirectDepPackage(BuildIntegrationTestCase testCase) |
| throws IOException { |
| testCase.write( |
| "genheader/BUILD", |
| "", |
| "genrule(name = 'gen_header',", |
| " srcs = [],", |
| " outs = ['gen.h'], ", |
| " cmd = 'echo \"int f(int x);\" > $@')", |
| "", |
| "cc_library(name = 'intermediate', ", |
| " srcs = ['intermediate.cc'],", |
| " hdrs = ['gen.h'])", |
| "", |
| "cc_binary(name = 'consumes_header', ", |
| " srcs = ['consumes.cc'], ", |
| " deps = ['intermediate'])"); |
| testCase.write("genheader/intermediate.cc", "int f(int x) { return x + 1; }"); |
| testCase.write( |
| "genheader/consumes.cc", |
| "#include \"genheader/gen.h\"", |
| "int main() {", |
| " return f(1);", |
| "}"); |
| } |
| |
| public final void runGeneratedTransitiveHeaderRewound_lostInInputDiscovery_spawnFailed() |
| throws Exception { |
| SpawnShim shim = |
| (discoverySpawn, discoveryContext) -> { |
| ActionInput header = getGeneratedHeaderRewoundLostInput(discoverySpawn); |
| ImmutableList<ActionInput> lostInputs = ImmutableList.of(header); |
| return createLostInputsExecException( |
| discoveryContext, lostInputs, new ActionInputDepOwnerMap(lostInputs)); |
| }; |
| |
| runGeneratedTransitiveHeaderRewound_lostInInputDiscovery(shim); |
| } |
| |
| final void runGeneratedTransitiveHeaderRewound_lostInInputDiscovery(SpawnShim shim) |
| throws Exception { |
| // Like runGeneratedHeaderRewound_lostInInputDiscovery, this test checks that rewinding works |
| // when the lost input is a generated header, except in this test, the header is indirectly |
| // depended on. |
| // |
| // Note that only the target-graph dependency is indirect (i.e. the dependency between |
| // ":consumes_header" and ":gen.h"). The Skyframe node corresponding to the compiling action of |
| // ":consumes_header" directly depends on the "gen.h" artifact, though that dependency is |
| // discovered during execution. |
| writeGeneratedHeaderIndirectDepPackage(testCase); |
| |
| addSpawnShim( |
| String.format( |
| "Extracting include lines from %s-out/k8-fastbuild/bin/genheader/gen.h", |
| TestConstants.PRODUCT_NAME), |
| shim); |
| |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| testCase.buildTarget("//genheader:consumes_header"); |
| verifyAllSpawnShimsConsumed(); |
| |
| // Note that because intermediate.cc does not have an include directive for gen.h, the |
| // "Extracting [...]/gen.h" action is first attempted just prior to the first attempt of |
| // "Compiling genheader/consumes.cc". |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactly( |
| "Executing genrule //genheader:gen_header", |
| "Extracting include lines from genheader/intermediate.cc", |
| "Extracting include lines from genheader/consumes.cc", |
| "Extracting include lines from tools/cpp/malloc.cc", |
| "Compiling tools/cpp/malloc.cc", |
| "Compiling genheader/intermediate.cc", |
| String.format( |
| "Extracting include lines from %s-out/k8-fastbuild/bin/genheader/gen.h", |
| TestConstants.PRODUCT_NAME), |
| "Executing genrule //genheader:gen_header", |
| String.format( |
| "Extracting include lines from %s-out/k8-fastbuild/bin/genheader/gen.h", |
| TestConstants.PRODUCT_NAME), |
| "Compiling genheader/consumes.cc", |
| "Linking genheader/consumes_header"); |
| |
| // Input discovery actions do not result in action lifecycle events. E.g., the "Extracting |
| // [...]" action is run, but results in no ActionStartedEvent/ActionCompletionEvent/etc. |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of( |
| "Compiling genheader/intermediate.cc", |
| "Compiling genheader/consumes.cc", |
| "Linking genheader/consumes_header"), |
| /*completedRewound=*/ ImmutableList.of("Executing genrule //genheader:gen_header"), |
| /*failedRewound=*/ ImmutableList.of(), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of( |
| middlemanEvent -> |
| isSchedulingDepMiddlemanForTarget( |
| middlemanEvent.getAction(), "//genheader:consumes_header"), |
| middlemanEvent -> |
| isSchedulingDepMiddlemanForTarget( |
| middlemanEvent.getAction(), "//genheader:intermediate")), |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of(1)); |
| |
| assertOnlyActionsDirtied(dirtiedKeys); |
| assertThat(dirtiedArtifactOwnerLabels(dirtiedKeys)).containsExactly("//genheader:gen_header"); |
| } |
| |
| public final void runGeneratedTransitiveHeaderRewound_lostInActionExecution_spawnFailed() |
| throws Exception { |
| SpawnShim shim = |
| (spawn, context) -> { |
| ActionInput header = getGeneratedHeaderRewoundLostInput(spawn); |
| ImmutableList<ActionInput> lostInputs = ImmutableList.of(header); |
| return createLostInputsExecException( |
| context, lostInputs, new ActionInputDepOwnerMap(lostInputs)); |
| }; |
| |
| runGeneratedTransitiveHeaderRewound_lostInActionExecution(shim); |
| } |
| |
| final void runGeneratedTransitiveHeaderRewound_lostInActionExecution(SpawnShim shim) |
| throws Exception { |
| // Like runGeneratedHeaderRewound_lostInActionExecution, this test checks that rewinding works |
| // when the lost input is a generated header, except in this test, the header is indirectly |
| // depended on. |
| // |
| // Note that only the target-graph dependency is indirect (i.e. the dependency between |
| // ":consumes_header" and ":gen.h"). The Skyframe node corresponding to the compiling action of |
| // ":consumes_header" directly depends on the "gen.h" artifact, though that dependency is |
| // discovered during execution. |
| writeGeneratedHeaderIndirectDepPackage(testCase); |
| |
| addSpawnShim("Compiling genheader/consumes.cc", shim); |
| |
| List<SkyKey> dirtiedKeys = collectOrderedDirtiedKeys(); |
| testCase.buildTarget("//genheader:consumes_header"); |
| verifyAllSpawnShimsConsumed(); |
| assertThat(getExecutedSpawnDescriptions()) |
| .containsExactly( |
| "Executing genrule //genheader:gen_header", |
| "Extracting include lines from genheader/intermediate.cc", |
| "Extracting include lines from tools/cpp/malloc.cc", |
| "Compiling tools/cpp/malloc.cc", |
| "Extracting include lines from genheader/consumes.cc", |
| "Compiling genheader/intermediate.cc", |
| String.format( |
| "Extracting include lines from %s-out/k8-fastbuild/bin/genheader/gen.h", |
| TestConstants.PRODUCT_NAME), |
| "Compiling genheader/consumes.cc", |
| "Executing genrule //genheader:gen_header", |
| "Compiling genheader/consumes.cc", |
| "Linking genheader/consumes_header"); |
| |
| recorder.assertEvents( |
| /*runOnce=*/ ImmutableList.of( |
| "Linking genheader/consumes_header", "Compiling genheader/intermediate.cc"), |
| /*completedRewound=*/ ImmutableList.of("Executing genrule //genheader:gen_header"), |
| /*failedRewound=*/ ImmutableList.of("Compiling genheader/consumes.cc"), |
| /*exactlyOneMiddlemanEventChecks=*/ ImmutableList.of( |
| middlemanEvent -> |
| isSchedulingDepMiddlemanForTarget( |
| middlemanEvent.getAction(), "//genheader:consumes_header"), |
| middlemanEvent -> |
| isSchedulingDepMiddlemanForTarget( |
| middlemanEvent.getAction(), "//genheader:intermediate")), |
| /*actionRewindingPostLostInputCounts=*/ ImmutableList.of(1)); |
| |
| assertOnlyActionsDirtied(dirtiedKeys); |
| assertThat(dirtiedArtifactOwnerLabels(dirtiedKeys)).containsExactly("//genheader:gen_header"); |
| } |
| |
| /** |
| * Regression test for b/242179728. |
| * |
| * <p>Exercises a scenario where a failing action depends on another action which is rewound |
| * between the time that the action fails and the dep is looked up for signaling. The order of |
| * events in this scenario (synchronized so that they execute sequentially) is: |
| * |
| * <ol> |
| * <li>{@code //foo:dep} executes successfully and produces two outputs, {@code dep.out1} and |
| * {@code dep.out2}. |
| * <li>{@code //foo:fail}, which depends on {@code dep.out1}, executes and fails due to a |
| * regular action execution failure (not a lost input). |
| * <li>{@code //foo:other}, which depends on {@code dep.out2}, executes and observes a lost |
| * input. {@code //foo:dep} is rewound. |
| * <li>{@code //foo:fail} looks up {@code //foo:dep} for {@link Reason#RDEP_ADDITION} and |
| * observes it to be dirty. |
| * </ol> |
| */ |
| public final void runDoneToDirtyDepForNodeInError() throws Exception { |
| testCase.write( |
| "foo/BUILD", |
| "genrule(name = 'other', srcs = [':dep.out2'], outs = ['other.out'], cmd = 'cp $< $@')", |
| "genrule(name = 'fail', srcs = [':dep.out1'], outs = ['fail.out'], cmd = 'false')", |
| "genrule(name = 'dep', outs = ['dep.out1', 'dep.out2'], cmd = 'touch $(OUTS)')"); |
| CountDownLatch depDone = new CountDownLatch(1); |
| CountDownLatch failExecuting = new CountDownLatch(1); |
| CountDownLatch depRewound = new CountDownLatch(1); |
| Label fail = Label.parseAbsoluteUnchecked("//foo:fail"); |
| Label dep = Label.parseAbsoluteUnchecked("//foo:dep"); |
| addSpawnShim( |
| "Executing genrule //foo:fail", |
| (spawn, context) -> { |
| failExecuting.countDown(); |
| return ExecResult.delegate(); |
| }); |
| addSpawnShim( |
| "Executing genrule //foo:other", |
| (spawn, context) -> { |
| ImmutableList<ActionInput> lostInputs = |
| ImmutableList.of(SpawnInputUtils.getInputWithName(spawn, "dep.out2")); |
| return createLostInputsExecException( |
| context, lostInputs, new ActionInputDepOwnerMap(lostInputs)); |
| }); |
| injectListenerAtStartOfNextBuild( |
| (key, type, order, context) -> { |
| if (isActionExecutionKey(key, fail) && type == EventType.CREATE_IF_ABSENT) { |
| awaitUninterruptibly(depDone); |
| } else if (isActionExecutionKey(key, dep) |
| && type == EventType.SET_VALUE |
| && order == Order.AFTER) { |
| depDone.countDown(); |
| } else if (isActionExecutionKey(key, dep) |
| && type == EventType.ADD_REVERSE_DEP |
| && order == Order.BEFORE |
| && isActionExecutionKey(context, fail)) { |
| awaitUninterruptibly(depRewound); |
| } else if (isActionExecutionKey(key, dep) |
| && type == EventType.MARK_DIRTY |
| && order == Order.AFTER) { |
| depRewound.countDown(); |
| } |
| }); |
| |
| assertThrows(BuildFailedException.class, () -> testCase.buildTarget("//foo:all")); |
| testCase.assertContainsError("Executing genrule //foo:fail failed"); |
| } |
| |
| private static boolean isActionExecutionKey(Object key, Label label) { |
| return key instanceof ActionLookupData && label.equals(((ActionLookupData) key).getLabel()); |
| } |
| |
| private static void awaitUninterruptibly(CountDownLatch latch) { |
| assertThat( |
| Uninterruptibles.awaitUninterruptibly(latch, TestUtils.WAIT_TIMEOUT_SECONDS, SECONDS)) |
| .isTrue(); |
| } |
| |
| private boolean keepGoing() { |
| return testCase.getRuntimeWrapper().getOptions(KeepGoingOption.class).keepGoing; |
| } |
| |
| final boolean buildRunfileManifests() { |
| return testCase.getRuntimeWrapper().getOptions(CoreOptions.class).buildRunfilesManifests; |
| } |
| } |