Ensure that Skyframe graph is cleaned after preloading a query if it might have nodes that need to be deleted. Not doing it eagerly means that, in a non-SkyQuery query, LabelVisitor may request single-package evaluation that will trigger deletion, but because of our usage of ConcurrentHashMap#forEachEntry in deletion, and its usage of the currently active ForkJoinPool, that deletion may attempt a re-entrant Skyframe evaluation.
This is too hard to trigger in a full test (it requires work-stealing by the ForkJoinPool thread), so I added a few unit tests.
PiperOrigin-RevId: 439407123
diff --git a/src/main/java/com/google/devtools/build/lib/query2/common/QueryTransitivePackagePreloader.java b/src/main/java/com/google/devtools/build/lib/query2/common/QueryTransitivePackagePreloader.java
index 72ee308..fb7dc43 100644
--- a/src/main/java/com/google/devtools/build/lib/query2/common/QueryTransitivePackagePreloader.java
+++ b/src/main/java/com/google/devtools/build/lib/query2/common/QueryTransitivePackagePreloader.java
@@ -13,8 +13,10 @@
// limitations under the License.
package com.google.devtools.build.lib.query2.common;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import com.google.devtools.build.lib.bugreport.BugReport;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.bugreport.BugReporter;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.query2.engine.QueryException;
@@ -42,12 +44,15 @@
public class QueryTransitivePackagePreloader {
private final Supplier<MemoizingEvaluator> memoizingEvaluatorSupplier;
private final Supplier<EvaluationContext.Builder> evaluationContextBuilderSupplier;
+ private final BugReporter bugReporter;
public QueryTransitivePackagePreloader(
Supplier<MemoizingEvaluator> memoizingEvaluatorSupplier,
- Supplier<EvaluationContext.Builder> evaluationContextBuilderSupplier) {
+ Supplier<EvaluationContext.Builder> evaluationContextBuilderSupplier,
+ BugReporter bugReporter) {
this.memoizingEvaluatorSupplier = memoizingEvaluatorSupplier;
this.evaluationContextBuilderSupplier = evaluationContextBuilderSupplier;
+ this.bugReporter = bugReporter;
}
/**
@@ -60,9 +65,21 @@
QueryExpression caller,
String operation)
throws QueryException {
+ maybeThrowQueryExceptionForResultWithError(
+ result, roots, caller, operation, BugReporter.defaultInstance());
+ }
+
+ @VisibleForTesting
+ static void maybeThrowQueryExceptionForResultWithError(
+ EvaluationResult<SkyValue> result,
+ Iterable<? extends SkyKey> roots,
+ QueryExpression caller,
+ String operation,
+ BugReporter bugReporter)
+ throws QueryException {
Exception exception = result.getCatastrophe();
if (exception != null) {
- throw throwException(exception, caller, operation, result);
+ throw throwException(exception, caller, operation, result, bugReporter);
}
// Catastrophe not present: look at top-level keys now.
@@ -79,7 +96,7 @@
}
if (exception != null) {
- throw throwException(exception, caller, operation, result);
+ throw throwException(exception, caller, operation, result, bugReporter);
}
Preconditions.checkState(
foundCycle, "No cycle or exception found in result with error: %s %s", result, roots);
@@ -89,11 +106,12 @@
Exception exception,
QueryExpression caller,
String operation,
- EvaluationResult<SkyValue> resultForDebugging)
+ EvaluationResult<SkyValue> resultForDebugging,
+ BugReporter bugReporter)
throws QueryException {
FailureDetails.FailureDetail failureDetail;
if (!(exception instanceof DetailedException)) {
- BugReport.sendBugReport(
+ bugReporter.sendBugReport(
new IllegalStateException(
"Non-detailed exception found for " + operation + ": " + resultForDebugging,
exception));
@@ -132,9 +150,25 @@
.build();
EvaluationResult<SkyValue> result =
memoizingEvaluatorSupplier.get().evaluate(valueNames, evaluationContext);
- if (callerForError != null && result.hasError()) {
- maybeThrowQueryExceptionForResultWithError(
- result, labelsToVisit, callerForError, "preloading transitive closure");
+ if (!result.hasError()) {
+ return;
}
+ if (callerForError != null) {
+ maybeThrowQueryExceptionForResultWithError(
+ result, labelsToVisit, callerForError, "preloading transitive closure", bugReporter);
+ return;
+ }
+ if (keepGoing && result.getCatastrophe() == null) {
+ // keep-going must have completed every in-flight node if there was no catastrophe.
+ return;
+ }
+
+ // At the beginning of every Skyframe evaluation, the evaluator first deletes nodes that were
+ // incomplete in the previous evaluation. The query may do later Skyframe evaluations (possibly
+ // because this pre-evaluation failed!), so we prevent the first such evaluation from doing
+ // unexpected deletions, which can lead to subtle threadpool issues.
+ //
+ // This is unnecessary in case there is a cycle, but not worth optimizing for.
+ memoizingEvaluatorSupplier.get().evaluate(ImmutableList.of(), evaluationContext);
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
index 1a80f7e..954ac1c 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
@@ -435,7 +435,7 @@
this.workspaceStatusActionFactory = workspaceStatusActionFactory;
this.queryTransitivePackagePreloader =
new QueryTransitivePackagePreloader(
- () -> memoizingEvaluator, this::newEvaluationContextBuilder);
+ () -> memoizingEvaluator, this::newEvaluationContextBuilder, bugReporter);
this.packageManager =
new SkyframePackageManager(
new SkyframePackageLoader(),
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageManager.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageManager.java
index 519020b..4dfeee9 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageManager.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageManager.java
@@ -42,7 +42,7 @@
private final Supplier<PathPackageLocator> pkgLocator;
private final AtomicInteger numPackagesSuccessfullyLoaded;
- public SkyframePackageManager(
+ SkyframePackageManager(
SkyframePackageLoader packageLoader,
SyscallCache syscallCache,
Supplier<PathPackageLocator> pkgLocator,
diff --git a/src/main/java/com/google/devtools/build/skyframe/ErrorInfo.java b/src/main/java/com/google/devtools/build/skyframe/ErrorInfo.java
index c2a754b..35bbb04 100644
--- a/src/main/java/com/google/devtools/build/skyframe/ErrorInfo.java
+++ b/src/main/java/com/google/devtools/build/skyframe/ErrorInfo.java
@@ -26,8 +26,7 @@
*
* <p>This is intended only for use in alternative {@code MemoizingEvaluator} implementations.
*/
-public class ErrorInfo {
-
+public final class ErrorInfo {
/** Create an ErrorInfo from a {@link ReifiedSkyFunctionException}. */
public static ErrorInfo fromException(
ReifiedSkyFunctionException skyFunctionException, boolean isTransitivelyTransient) {
diff --git a/src/test/java/com/google/devtools/build/lib/query2/common/BUILD b/src/test/java/com/google/devtools/build/lib/query2/common/BUILD
index 8277a25..1276433 100644
--- a/src/test/java/com/google/devtools/build/lib/query2/common/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/query2/common/BUILD
@@ -68,3 +68,26 @@
"//third_party:truth",
],
)
+
+java_test(
+ name = "QueryTransitivePackagePreloaderTest",
+ srcs = ["QueryTransitivePackagePreloaderTest.java"],
+ deps = [
+ "//src/main/java/com/google/devtools/build/lib/bugreport",
+ "//src/main/java/com/google/devtools/build/lib/cmdline",
+ "//src/main/java/com/google/devtools/build/lib/events",
+ "//src/main/java/com/google/devtools/build/lib/query2/common:QueryTransitivePackagePreloader",
+ "//src/main/java/com/google/devtools/build/lib/query2/engine",
+ "//src/main/java/com/google/devtools/build/lib/skyframe:detailed_exceptions",
+ "//src/main/java/com/google/devtools/build/lib/skyframe:transitive_target_key",
+ "//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code",
+ "//src/main/java/com/google/devtools/build/skyframe",
+ "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
+ "//src/main/protobuf:failure_details_java_proto",
+ "//third_party:guava",
+ "//third_party:junit4",
+ "//third_party:mockito",
+ "//third_party:truth",
+ "@com_google_testparameterinjector//:testparameterinjector",
+ ],
+)
diff --git a/src/test/java/com/google/devtools/build/lib/query2/common/QueryTransitivePackagePreloaderTest.java b/src/test/java/com/google/devtools/build/lib/query2/common/QueryTransitivePackagePreloaderTest.java
new file mode 100644
index 0000000..0c33206
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/query2/common/QueryTransitivePackagePreloaderTest.java
@@ -0,0 +1,350 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.query2.common;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat;
+import static org.junit.Assert.assertThrows;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.bugreport.BugReporter;
+import com.google.devtools.build.lib.cmdline.Label;
+import com.google.devtools.build.lib.events.ExtendedEventHandler;
+import com.google.devtools.build.lib.query2.engine.QueryException;
+import com.google.devtools.build.lib.query2.engine.QueryExpression;
+import com.google.devtools.build.lib.server.FailureDetails;
+import com.google.devtools.build.lib.skyframe.DetailedException;
+import com.google.devtools.build.lib.skyframe.TransitiveTargetKey;
+import com.google.devtools.build.lib.util.DetailedExitCode;
+import com.google.devtools.build.skyframe.CycleInfo;
+import com.google.devtools.build.skyframe.ErrorInfo;
+import com.google.devtools.build.skyframe.EvaluationContext;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.MemoizingEvaluator;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyValue;
+import com.google.testing.junit.testparameterinjector.TestParameter;
+import com.google.testing.junit.testparameterinjector.TestParameterInjector;
+import java.util.List;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+/** Tests for {@link QueryTransitivePackagePreloader}. */
+@RunWith(TestParameterInjector.class)
+public class QueryTransitivePackagePreloaderTest {
+ private static final Label LABEL = Label.parseAbsoluteUnchecked("//my:label");
+ private static final Label LABEL2 = Label.parseAbsoluteUnchecked("//my:label2");
+ private static final Label LABEL3 = Label.parseAbsoluteUnchecked("//my:label3");
+ private static final TransitiveTargetKey KEY = TransitiveTargetKey.of(LABEL);
+ private static final TransitiveTargetKey KEY2 = TransitiveTargetKey.of(LABEL2);
+ private static final TransitiveTargetKey KEY3 = TransitiveTargetKey.of(LABEL3);
+
+ private static final ErrorInfo DETAILED_ERROR =
+ ErrorInfo.fromException(
+ new SkyFunctionException.ReifiedSkyFunctionException(
+ new SkyFunctionException(
+ new MyDetailedException("bork"), SkyFunctionException.Transience.PERSISTENT) {}),
+ /*isTransitivelyTransient=*/ false);
+ private static final ErrorInfo UNDETAILED_ERROR =
+ ErrorInfo.fromException(
+ new SkyFunctionException.ReifiedSkyFunctionException(
+ new SkyFunctionException(
+ new UndetailedException("bork"), SkyFunctionException.Transience.PERSISTENT) {}),
+ /*isTransitivelyTransient=*/ false);
+ private static final ErrorInfo CYCLE_ERROR =
+ ErrorInfo.fromCycle(new CycleInfo(ImmutableList.of(KEY)));
+
+ @Mock MemoizingEvaluator memoizingEvaluator;
+ @Mock EvaluationContext.Builder contextBuilder;
+ @Mock EvaluationContext context;
+ private final BugReporter bugReporter = mock(BugReporter.class);
+
+ private final QueryTransitivePackagePreloader underTest =
+ new QueryTransitivePackagePreloader(
+ () -> memoizingEvaluator, () -> contextBuilder, bugReporter);
+ private AutoCloseable closeable;
+
+ @Before
+ public void setUpMocks() {
+ closeable = MockitoAnnotations.openMocks(this);
+ when(contextBuilder.setKeepGoing(ArgumentMatchers.anyBoolean())).thenReturn(contextBuilder);
+ when(contextBuilder.setNumThreads(ArgumentMatchers.anyInt())).thenReturn(contextBuilder);
+ when(contextBuilder.setEventHandler(ArgumentMatchers.any())).thenReturn(contextBuilder);
+ when(contextBuilder.setUseForkJoinPool(ArgumentMatchers.anyBoolean()))
+ .thenReturn(contextBuilder);
+ when(contextBuilder.build()).thenReturn(context);
+ }
+
+ @After
+ public void releaseMocks() throws Exception {
+ verifyNoMoreInteractions(memoizingEvaluator);
+ verifyNoMoreInteractions(bugReporter);
+ closeable.close();
+ }
+
+ @Test
+ public void preloadTransitiveTargets_noError() throws Exception {
+ List<TransitiveTargetKey> roots = Lists.newArrayList(KEY);
+
+ when(memoizingEvaluator.evaluate(roots, context))
+ .thenReturn(EvaluationResult.builder().build());
+
+ underTest.preloadTransitiveTargets(
+ mock(ExtendedEventHandler.class),
+ ImmutableList.of(LABEL),
+ /*keepGoing=*/ true,
+ 1,
+ /*callerForError=*/ null);
+
+ verify(memoizingEvaluator).evaluate(roots, context);
+ }
+
+ @Test
+ public void preloadTransitiveTargets_errorWithNullCallerKeepGoing_doesntCleanGraph()
+ throws Exception {
+ List<TransitiveTargetKey> roots = Lists.newArrayList(KEY);
+
+ when(memoizingEvaluator.evaluate(roots, context))
+ .thenReturn(EvaluationResult.builder().addError(KEY, UNDETAILED_ERROR).build());
+
+ underTest.preloadTransitiveTargets(
+ mock(ExtendedEventHandler.class),
+ ImmutableList.of(LABEL),
+ /*keepGoing=*/ true,
+ 1,
+ /*callerForError=*/ null);
+
+ verify(memoizingEvaluator).evaluate(roots, context);
+ }
+
+ @Test
+ public void preloadTransitiveTargets_errorWithNullCallerKeepGoingCatastrophe_cleansGraph()
+ throws Exception {
+ List<TransitiveTargetKey> roots = Lists.newArrayList(KEY);
+
+ when(memoizingEvaluator.evaluate(roots, context))
+ .thenReturn(
+ EvaluationResult.builder()
+ .setCatastrophe(new UndetailedException("catas"))
+ .addError(KEY, UNDETAILED_ERROR)
+ .build());
+
+ underTest.preloadTransitiveTargets(
+ mock(ExtendedEventHandler.class),
+ ImmutableList.of(LABEL),
+ /*keepGoing=*/ true,
+ 1,
+ /*callerForError=*/ null);
+
+ verify(memoizingEvaluator).evaluate(roots, context);
+ verify(memoizingEvaluator).evaluate(ImmutableList.of(), context);
+ }
+
+ @Test
+ public void preloadTransitiveTargets_errorWithNullCallerNoKeepGoing_cleansGraph()
+ throws Exception {
+ List<TransitiveTargetKey> roots = Lists.newArrayList(KEY);
+
+ when(memoizingEvaluator.evaluate(roots, context))
+ .thenReturn(EvaluationResult.builder().addError(KEY, UNDETAILED_ERROR).build());
+
+ underTest.preloadTransitiveTargets(
+ mock(ExtendedEventHandler.class),
+ ImmutableList.of(LABEL),
+ /*keepGoing=*/ false,
+ 1,
+ /*callerForError=*/ null);
+
+ verify(memoizingEvaluator).evaluate(roots, context);
+ verify(memoizingEvaluator).evaluate(ImmutableList.of(), context);
+ }
+
+ @Test
+ public void preloadTransitiveTargets_detailedErrorWithCaller_throwsError(
+ @TestParameter boolean keepGoing) throws Exception {
+ List<TransitiveTargetKey> roots = Lists.newArrayList(KEY);
+
+ when(memoizingEvaluator.evaluate(roots, context))
+ .thenReturn(EvaluationResult.builder().addError(KEY, DETAILED_ERROR).build());
+
+ var e =
+ assertThrows(
+ QueryException.class,
+ () ->
+ underTest.preloadTransitiveTargets(
+ mock(ExtendedEventHandler.class),
+ ImmutableList.of(LABEL),
+ keepGoing,
+ 1,
+ /*callerForError=*/ mock(QueryExpression.class)));
+ assertThat(e).hasMessageThat().contains("failed: bork");
+ assertThat(e.getFailureDetail())
+ .isSameInstanceAs(MyDetailedException.DETAILED_EXIT_CODE.getFailureDetail());
+
+ verify(memoizingEvaluator).evaluate(roots, context);
+ }
+
+ @Test
+ public void preloadTransitiveTargets_undetailedErrorWithCaller_throwsErrorAndFilesBugReport(
+ @TestParameter boolean keepGoing) throws Exception {
+ List<TransitiveTargetKey> roots = Lists.newArrayList(KEY);
+
+ when(memoizingEvaluator.evaluate(roots, context))
+ .thenReturn(EvaluationResult.builder().addError(KEY, UNDETAILED_ERROR).build());
+
+ var e =
+ assertThrows(
+ QueryException.class,
+ () ->
+ underTest.preloadTransitiveTargets(
+ mock(ExtendedEventHandler.class),
+ ImmutableList.of(LABEL),
+ keepGoing,
+ 1,
+ /*callerForError=*/ mock(QueryExpression.class)));
+ assertThat(e).hasMessageThat().contains("failed: bork");
+ assertThat(e.getFailureDetail())
+ .comparingExpectedFieldsOnly()
+ .isEqualTo(
+ FailureDetails.FailureDetail.newBuilder()
+ .setQuery(
+ FailureDetails.Query.newBuilder()
+ .setCode(FailureDetails.Query.Code.NON_DETAILED_ERROR)
+ .build())
+ .build());
+
+ verify(memoizingEvaluator).evaluate(roots, context);
+ verify(bugReporter).sendBugReport(ArgumentMatchers.any());
+ }
+
+ @Test
+ public void
+ preloadTransitiveTargets_undetailedCatastropheAndDetailedExceptionWithCaller_throwsErrorAndFilesBugReport(
+ @TestParameter boolean keepGoing) throws Exception {
+ List<TransitiveTargetKey> roots = Lists.newArrayList(KEY);
+
+ when(memoizingEvaluator.evaluate(roots, context))
+ .thenReturn(
+ EvaluationResult.builder()
+ .addError(KEY, DETAILED_ERROR)
+ .setCatastrophe(new UndetailedException("undetailed bok"))
+ .build());
+
+ var e =
+ assertThrows(
+ QueryException.class,
+ () ->
+ underTest.preloadTransitiveTargets(
+ mock(ExtendedEventHandler.class),
+ ImmutableList.of(LABEL),
+ keepGoing,
+ 1,
+ /*callerForError=*/ mock(QueryExpression.class)));
+ assertThat(e).hasMessageThat().contains("failed: undetailed bok");
+ assertThat(e.getFailureDetail())
+ .comparingExpectedFieldsOnly()
+ .isEqualTo(
+ FailureDetails.FailureDetail.newBuilder()
+ .setQuery(
+ FailureDetails.Query.newBuilder()
+ .setCode(FailureDetails.Query.Code.NON_DETAILED_ERROR)
+ .build())
+ .build());
+
+ verify(memoizingEvaluator).evaluate(roots, context);
+ verify(bugReporter).sendBugReport(ArgumentMatchers.any());
+ }
+
+ @Test
+ public void preloadTransitiveTargets_undetailedAndDetailedExceptionsWithCaller_throwsError(
+ @TestParameter boolean keepGoing, @TestParameter boolean includeCycle) throws Exception {
+ List<TransitiveTargetKey> roots = Lists.newArrayList(KEY, KEY2, KEY3);
+
+ EvaluationResult.Builder<SkyValue> resultBuilder =
+ EvaluationResult.builder().addError(KEY, UNDETAILED_ERROR).addError(KEY2, DETAILED_ERROR);
+ if (includeCycle) {
+ resultBuilder.addError(KEY3, CYCLE_ERROR);
+ }
+ when(memoizingEvaluator.evaluate(roots, context)).thenReturn(resultBuilder.build());
+
+ var e =
+ assertThrows(
+ QueryException.class,
+ () ->
+ underTest.preloadTransitiveTargets(
+ mock(ExtendedEventHandler.class),
+ ImmutableList.of(LABEL, LABEL2, LABEL3),
+ keepGoing,
+ 1,
+ /*callerForError=*/ mock(QueryExpression.class)));
+ assertThat(e).hasMessageThat().contains("failed: bork");
+ assertThat(e.getFailureDetail())
+ .isSameInstanceAs(MyDetailedException.DETAILED_EXIT_CODE.getFailureDetail());
+
+ verify(memoizingEvaluator).evaluate(roots, context);
+ }
+
+ @Test
+ public void preloadTransitiveTargets_cycleOnly_returns() throws Exception {
+ List<TransitiveTargetKey> roots = Lists.newArrayList(KEY);
+
+ when(memoizingEvaluator.evaluate(roots, context))
+ .thenReturn(EvaluationResult.builder().addError(KEY, CYCLE_ERROR).build());
+
+ underTest.preloadTransitiveTargets(
+ mock(ExtendedEventHandler.class),
+ ImmutableList.of(LABEL),
+ /*keepGoing=*/ true,
+ 1,
+ /*callerForError=*/ null);
+
+ verify(memoizingEvaluator).evaluate(roots, context);
+ }
+
+ private static final class UndetailedException extends Exception {
+ UndetailedException(String message) {
+ super(message);
+ }
+ }
+
+ private static final class MyDetailedException extends Exception implements DetailedException {
+ private static final DetailedExitCode DETAILED_EXIT_CODE =
+ DetailedExitCode.of(
+ FailureDetails.FailureDetail.newBuilder()
+ .setQuery(
+ FailureDetails.Query.newBuilder()
+ .setCode(FailureDetails.Query.Code.BUILD_FILE_ERROR))
+ .build());
+
+ MyDetailedException(String message) {
+ super(message);
+ }
+
+ @Override
+ public DetailedExitCode getDetailedExitCode() {
+ return DETAILED_EXIT_CODE;
+ }
+ }
+}
