Refactor test case summarization
This improves accuracy in cases where no or limited test case
information is available. I also simplified the API and made the
implementation of TestSummary only do a single transitive traversal over
the TestCase tree.
Fixes #8188.
PiperOrigin-RevId: 289859356
diff --git a/src/test/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifierTest.java b/src/test/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifierTest.java
index 37293cf..4fdaa45 100644
--- a/src/test/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifierTest.java
+++ b/src/test/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifierTest.java
@@ -49,14 +49,15 @@
@RunWith(JUnit4.class)
public final class TerminalTestResultNotifierTest {
- private static final String ALL_TEST_CASES_PASSED_BUT_TARGET_FAILED_DISCLAIMER =
- "however note that at least one target failed";
+ private static final String SOME_TARGETS_ARE_MISSING_TEST_CASES_DISCLAIMER =
+ "some targets did not have test case information";
private final OptionsParsingResult optionsParsingResult = mock(OptionsParsingResult.class);
private final AnsiTerminalPrinter ansiTerminalPrinter = mock(AnsiTerminalPrinter.class);
private BlazeTestStatus targetStatus;
private int numFailedTestCases;
+ private int numUnknownTestCases;
private int numTotalTestCases;
private TestSummaryFormat testSummaryFormat;
@@ -73,7 +74,7 @@
assertThat(printed).contains(info("10 passing"));
assertThat(printed).contains("0 failing");
assertThat(printed).contains("out of 10 test cases");
- assertThat(printed).doesNotContain(ALL_TEST_CASES_PASSED_BUT_TARGET_FAILED_DISCLAIMER);
+ assertThat(printed).doesNotContain(SOME_TARGETS_ARE_MISSING_TEST_CASES_DISCLAIMER);
assertThat(printed).doesNotContain(AnsiTerminalPrinter.Mode.ERROR.toString());
}
@@ -81,16 +82,17 @@
public void testCaseOption_allPassButTargetFails() throws Exception {
testSummaryFormat = TestSummaryFormat.TESTCASE;
numFailedTestCases = 0;
+ numUnknownTestCases = 10;
numTotalTestCases = 10;
targetStatus = BlazeTestStatus.FAILED;
printTestCaseSummary();
String printed = getPrintedMessage();
- assertThat(printed).contains(info("10 passing"));
+ assertThat(printed).contains("0 passing");
assertThat(printed).contains("0 failing");
assertThat(printed).contains("out of 10 test cases");
- assertThat(printed).contains(ALL_TEST_CASES_PASSED_BUT_TARGET_FAILED_DISCLAIMER);
+ assertThat(printed).contains(SOME_TARGETS_ARE_MISSING_TEST_CASES_DISCLAIMER);
assertThat(printed).doesNotContain(AnsiTerminalPrinter.Mode.ERROR.toString());
}
@@ -98,6 +100,7 @@
public void testCaseOption_someFail() throws Exception {
testSummaryFormat = TestSummaryFormat.TESTCASE;
numFailedTestCases = 2;
+ numUnknownTestCases = 0;
numTotalTestCases = 10;
targetStatus = BlazeTestStatus.FAILED;
@@ -107,7 +110,7 @@
assertThat(printed).contains(info("8 passing"));
assertThat(printed).contains(error("2 failing"));
assertThat(printed).contains("out of 10 test cases");
- assertThat(printed).doesNotContain(ALL_TEST_CASES_PASSED_BUT_TARGET_FAILED_DISCLAIMER);
+ assertThat(printed).doesNotContain(SOME_TARGETS_ARE_MISSING_TEST_CASES_DISCLAIMER);
}
@Test
@@ -115,6 +118,7 @@
testSummaryFormat = TestSummaryFormat.SHORT;
numFailedTestCases = 0;
int numFailedToBuildTestCases = TerminalTestResultNotifier.NUM_FAILED_TO_BUILD + 1;
+ numUnknownTestCases = 0;
numTotalTestCases = 10;
targetStatus = BlazeTestStatus.FAILED_TO_BUILD;
@@ -142,6 +146,7 @@
public void testCaseOption_allFail() throws Exception {
testSummaryFormat = TestSummaryFormat.TESTCASE;
numFailedTestCases = 10;
+ numUnknownTestCases = 0;
numTotalTestCases = 10;
targetStatus = BlazeTestStatus.FAILED;
@@ -151,7 +156,7 @@
assertThat(printed).contains("0 passing");
assertThat(printed).contains(error("10 failing"));
assertThat(printed).contains("out of 10 test cases");
- assertThat(printed).doesNotContain(ALL_TEST_CASES_PASSED_BUT_TARGET_FAILED_DISCLAIMER);
+ assertThat(printed).doesNotContain(SOME_TARGETS_ARE_MISSING_TEST_CASES_DISCLAIMER);
assertThat(printed).doesNotContain(AnsiTerminalPrinter.Mode.INFO.toString());
}
@@ -159,6 +164,7 @@
public void detailedOption_allPass() throws Exception {
testSummaryFormat = TestSummaryFormat.DETAILED;
numFailedTestCases = 0;
+ numUnknownTestCases = 0;
numTotalTestCases = 10;
targetStatus = BlazeTestStatus.PASSED;
@@ -168,7 +174,7 @@
assertThat(printed).contains(info("10 passing"));
assertThat(printed).contains("0 failing");
assertThat(printed).contains("out of 10 test cases");
- assertThat(printed).doesNotContain(ALL_TEST_CASES_PASSED_BUT_TARGET_FAILED_DISCLAIMER);
+ assertThat(printed).doesNotContain(SOME_TARGETS_ARE_MISSING_TEST_CASES_DISCLAIMER);
assertThat(printed).doesNotContain(AnsiTerminalPrinter.Mode.ERROR.toString());
}
@@ -176,16 +182,17 @@
public void detailedOption_allPassButTargetFails() throws Exception {
testSummaryFormat = TestSummaryFormat.DETAILED;
numFailedTestCases = 0;
+ numUnknownTestCases = 10;
numTotalTestCases = 10;
targetStatus = BlazeTestStatus.FAILED;
printTestCaseSummary();
String printed = getPrintedMessage();
- assertThat(printed).contains(info("10 passing"));
+ assertThat(printed).contains("0 passing");
assertThat(printed).contains("0 failing");
assertThat(printed).contains("out of 10 test cases");
- assertThat(printed).contains(ALL_TEST_CASES_PASSED_BUT_TARGET_FAILED_DISCLAIMER);
+ assertThat(printed).contains(SOME_TARGETS_ARE_MISSING_TEST_CASES_DISCLAIMER);
assertThat(printed).doesNotContain(AnsiTerminalPrinter.Mode.ERROR.toString());
}
@@ -193,6 +200,7 @@
public void detailedOption_someFail() throws Exception {
testSummaryFormat = TestSummaryFormat.DETAILED;
numFailedTestCases = 2;
+ numUnknownTestCases = 0;
numTotalTestCases = 10;
targetStatus = BlazeTestStatus.FAILED;
@@ -202,13 +210,14 @@
assertThat(printed).contains(info("8 passing"));
assertThat(printed).contains(error("2 failing"));
assertThat(printed).contains("out of 10 test cases");
- assertThat(printed).doesNotContain(ALL_TEST_CASES_PASSED_BUT_TARGET_FAILED_DISCLAIMER);
+ assertThat(printed).doesNotContain(SOME_TARGETS_ARE_MISSING_TEST_CASES_DISCLAIMER);
}
@Test
public void detailedOption_allFail() throws Exception {
testSummaryFormat = TestSummaryFormat.DETAILED;
numFailedTestCases = 10;
+ numUnknownTestCases = 0;
numTotalTestCases = 10;
targetStatus = BlazeTestStatus.FAILED;
@@ -218,7 +227,7 @@
assertThat(printed).contains("0 passing");
assertThat(printed).contains(error("10 failing"));
assertThat(printed).contains("out of 10 test cases");
- assertThat(printed).doesNotContain(ALL_TEST_CASES_PASSED_BUT_TARGET_FAILED_DISCLAIMER);
+ assertThat(printed).doesNotContain(SOME_TARGETS_ARE_MISSING_TEST_CASES_DISCLAIMER);
assertThat(printed).doesNotContain(AnsiTerminalPrinter.Mode.INFO.toString());
}
@@ -226,6 +235,7 @@
public void shortOption_noSummaryPrinted() throws Exception {
testSummaryFormat = TestSummaryFormat.SHORT;
numFailedTestCases = 2;
+ numUnknownTestCases = 0;
numTotalTestCases = 10;
targetStatus = BlazeTestStatus.FAILED;
@@ -238,6 +248,7 @@
public void terseOption_noSummaryPrinted() throws Exception {
testSummaryFormat = TestSummaryFormat.TERSE;
numFailedTestCases = 2;
+ numUnknownTestCases = 0;
numTotalTestCases = 10;
targetStatus = BlazeTestStatus.FAILED;
@@ -250,6 +261,7 @@
public void noneOption_noSummaryPrinted() throws Exception {
testSummaryFormat = TestSummaryFormat.NONE;
numFailedTestCases = 2;
+ numUnknownTestCases = 0;
numTotalTestCases = 10;
targetStatus = BlazeTestStatus.FAILED;
@@ -296,6 +308,7 @@
TestSummary testSummary = mock(TestSummary.class);
when(testSummary.getTotalTestCases()).thenReturn(numTotalTestCases);
+ when(testSummary.getUnkownTestCases()).thenReturn(numUnknownTestCases);
TestCase failedTestCase = TestCase.newBuilder().setStatus(Status.FAILED).build();
List<TestCase> failedTestCases = Collections.nCopies(numFailedTestCases, failedTestCase);
