include test-case resolution at the end of the build log
Added test-case resolution in all log modes.
https://github.com/bazelbuild/bazel/issues/1506
Closes #5429.
PiperOrigin-RevId: 210362746
diff --git a/src/main/java/com/google/devtools/build/lib/exec/TestStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/TestStrategy.java
index f3662bb..277bd01 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/TestStrategy.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/TestStrategy.java
@@ -101,7 +101,9 @@
SHORT, // Print information only about tests.
TERSE, // Like "SHORT", but even shorter: Do not print PASSED and NO STATUS tests.
DETAILED, // Print information only about failed test cases.
- NONE; // Do not print summary.
+ NONE, // Do not print summary.
+ TESTCASE; // Print summary in test case resolution, do not print detailed information about
+ // failed test cases.
/** Converts to {@link TestSummaryFormat}. */
public static class Converter extends EnumConverter<TestSummaryFormat> {
@@ -286,7 +288,8 @@
protected TestCase parseTestResult(Path resultFile) {
/* xml files. We avoid parsing it unnecessarily, since test results can potentially consume
a large amount of memory. */
- if (executionOptions.testSummary != TestSummaryFormat.DETAILED) {
+ if ((executionOptions.testSummary != TestSummaryFormat.DETAILED)
+ && (executionOptions.testSummary != TestSummaryFormat.TESTCASE)) {
return null;
}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifier.java b/src/main/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifier.java
index 74e041b..2a93cde 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifier.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifier.java
@@ -13,6 +13,9 @@
// limitations under the License.
package com.google.devtools.build.lib.runtime;
+import static com.google.devtools.build.lib.exec.TestStrategy.TestSummaryFormat.DETAILED;
+import static com.google.devtools.build.lib.exec.TestStrategy.TestSummaryFormat.TESTCASE;
+
import com.google.devtools.build.lib.analysis.test.TestResult;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.exec.ExecutionOptions;
@@ -46,6 +49,9 @@
int noStatusCount;
int numberOfExecutedTargets;
boolean wasUnreportedWrongSize;
+
+ int totalTestCases;
+ int totalFailedTestCases;
}
/**
@@ -175,6 +181,9 @@
if (summary.wasUnreportedWrongSize()) {
stats.wasUnreportedWrongSize = true;
}
+
+ stats.totalFailedTestCases += summary.getFailedTestCases().size();
+ stats.totalTestCases += summary.getTotalTestCases();
}
stats.failedCount = summaries.size() - stats.passCount;
@@ -193,6 +202,7 @@
printShortSummary(summaries, /* showPassingTests= */ false);
break;
+ case TESTCASE:
case NONE:
break;
}
@@ -213,6 +223,21 @@
}
private void printStats(TestResultStats stats) {
+ TestSummaryFormat testSummaryFormat = options.getOptions(ExecutionOptions.class).testSummary;
+ if ((testSummaryFormat == DETAILED) || (testSummaryFormat == TESTCASE)) {
+ Integer passCount = stats.totalTestCases - stats.totalFailedTestCases;
+ printer.printLn(
+ String.format(
+ "Test cases: finished with %s%d passing%s and %s%d failing%s out of %d test cases",
+ passCount > 0 ? AnsiTerminalPrinter.Mode.INFO : "",
+ passCount,
+ AnsiTerminalPrinter.Mode.DEFAULT,
+ stats.totalFailedTestCases > 0 ? AnsiTerminalPrinter.Mode.ERROR : "",
+ stats.totalFailedTestCases,
+ AnsiTerminalPrinter.Mode.DEFAULT,
+ stats.totalTestCases));
+ }
+
if (!optionCheckTestsUpToDate()) {
List<String> results = new ArrayList<>();
if (stats.passCount == 1) {
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TestResultAnalyzer.java b/src/main/java/com/google/devtools/build/lib/runtime/TestResultAnalyzer.java
index 5333d55..871f2dd 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/TestResultAnalyzer.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/TestResultAnalyzer.java
@@ -265,6 +265,7 @@
.addFailedLogs(failed)
.addWarnings(result.getData().getWarningList())
.collectFailedTests(result.getData().getTestCase())
+ .countTotalTestCases(result.getData().getTestCase())
.setRanRemotely(result.getData().getIsRemoteStrategy());
List<String> warnings = new ArrayList<>();
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TestSummary.java b/src/main/java/com/google/devtools/build/lib/runtime/TestSummary.java
index 32a5c84..486c833 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/TestSummary.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/TestSummary.java
@@ -36,6 +36,7 @@
import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
import com.google.devtools.build.lib.view.test.TestStatus.FailedTestCasesStatus;
import com.google.devtools.build.lib.view.test.TestStatus.TestCase;
+import com.google.devtools.build.lib.view.test.TestStatus.TestCase.Type;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -74,6 +75,7 @@
addCoverageFiles(existingSummary.coverageFiles);
addPassedLogs(existingSummary.passedLogs);
addFailedLogs(existingSummary.failedLogs);
+ addTotalTestCases(existingSummary.totalTestCases);
if (existingSummary.failedTestCasesStatus != null) {
addFailedTestCases(existingSummary.getFailedTestCases(),
@@ -143,6 +145,12 @@
return this;
}
+ public Builder addTotalTestCases(int totalTestCases) {
+ checkMutation(totalTestCases);
+ summary.totalTestCases += totalTestCases;
+ return this;
+ }
+
public Builder collectFailedTests(TestCase testCase) {
if (testCase == null) {
summary.failedTestCasesStatus = FailedTestCasesStatus.NOT_AVAILABLE;
@@ -172,6 +180,26 @@
return this;
}
+ public Builder countTotalTestCases(TestCase testCase) {
+ if (testCase != null) {
+ summary.totalTestCases = traverseCountTotalTestCases(testCase);
+ }
+ return this;
+ }
+
+ private int traverseCountTotalTestCases(TestCase testCase) {
+ if (testCase.getChildCount() > 0) {
+ // don't count container of test cases as test
+ int res = 0;
+ for (TestCase child : testCase.getChildList()) {
+ res += traverseCountTotalTestCases(child);
+ }
+ return res;
+ } else {
+ return testCase.getType() == Type.TEST_CASE ? 1 : 0;
+ }
+ }
+
public Builder addFailedTestCases(List<TestCase> testCases, FailedTestCasesStatus status) {
checkMutation(status);
checkMutation(testCases);
@@ -315,6 +343,7 @@
private List<Path> coverageFiles = new ArrayList<>();
private List<Long> testTimes = new ArrayList<>();
private FailedTestCasesStatus failedTestCasesStatus = null;
+ private int totalTestCases;
// Don't allow public instantiation; go through the Builder.
private TestSummary() {
@@ -402,6 +431,10 @@
return failedTestCases;
}
+ public int getTotalTestCases() {
+ return totalTestCases;
+ }
+
public List<Path> getCoverageFiles() {
return coverageFiles;
}
@@ -439,6 +472,7 @@
.compare(
this.getTarget().getConfigurationChecksum(),
that.getTarget().getConfigurationChecksum())
+ .compare(this.getTotalTestCases(), that.getTotalTestCases())
.result();
}
diff --git a/src/test/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifierTest.java b/src/test/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifierTest.java
new file mode 100644
index 0000000..af2e211
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifierTest.java
@@ -0,0 +1,129 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.cmdline.Label;
+import com.google.devtools.build.lib.exec.ExecutionOptions;
+import com.google.devtools.build.lib.exec.TestStrategy.TestSummaryFormat;
+import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier.TestSummaryOptions;
+import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter;
+import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
+import com.google.devtools.build.lib.view.test.TestStatus.TestCase;
+import com.google.devtools.build.lib.view.test.TestStatus.TestCase.Status;
+import com.google.devtools.common.options.OptionsParsingResult;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Random;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mockito;
+
+/** Tests {@link TerminalTestResultNotifier}. */
+@RunWith(JUnit4.class)
+public class TerminalTestResultNotifierTest {
+
+ private final OptionsParsingResult optionsParsingResult =
+ Mockito.mock(OptionsParsingResult.class);
+ private final AnsiTerminalPrinter ansiTerminalPrinter = Mockito.mock(AnsiTerminalPrinter.class);
+ private final Random random = new Random();
+
+ private void givenExecutionOption(TestSummaryFormat format) {
+ ExecutionOptions executionOptions = ExecutionOptions.DEFAULTS;
+ executionOptions.testSummary = format;
+ when(optionsParsingResult.getOptions(ExecutionOptions.class)).thenReturn(executionOptions);
+
+ TestSummaryOptions testSummaryOptions = new TestSummaryOptions();
+ testSummaryOptions.verboseSummary = true;
+ when(optionsParsingResult.getOptions(TestSummaryOptions.class)).thenReturn(testSummaryOptions);
+ }
+
+ private void runTest(Boolean shouldPrintTestCaseSummary) throws Exception {
+ int numOfTotalTestCases = random.nextInt(10) + 1;
+ int numOfFailedCases = random.nextInt(numOfTotalTestCases);
+ int numOfSuccessfulTestCases = numOfTotalTestCases - numOfFailedCases;
+
+ TerminalTestResultNotifier terminalTestResultNotifier =
+ new TerminalTestResultNotifier(ansiTerminalPrinter, optionsParsingResult);
+
+ TestSummary testSummary = Mockito.mock(TestSummary.class);
+ when(testSummary.getTotalTestCases()).thenReturn(numOfTotalTestCases);
+ TestCase failedTestCase = TestCase.newBuilder().setStatus(Status.FAILED).build();
+ ArrayList<TestCase> testCases =
+ new ArrayList<>(Collections.nCopies(numOfFailedCases, failedTestCase));
+
+ Label labelA = Label.parseAbsolute("//foo/bar:baz", ImmutableMap.of());
+ when(testSummary.getFailedTestCases()).thenReturn(testCases);
+ when(testSummary.getStatus()).thenReturn(BlazeTestStatus.FAILED);
+ when(testSummary.getLabel()).thenReturn(labelA);
+
+ HashSet<TestSummary> testSummaries = new HashSet<>();
+ testSummaries.add(testSummary);
+ terminalTestResultNotifier.notify(testSummaries, 1);
+
+ String summaryMessage =
+ String.format(
+ "Test cases: finished with %s%d passing%s and %s%d failing%s out of %d test cases",
+ numOfSuccessfulTestCases > 0 ? AnsiTerminalPrinter.Mode.INFO : "",
+ numOfSuccessfulTestCases,
+ AnsiTerminalPrinter.Mode.DEFAULT,
+ numOfFailedCases > 0 ? AnsiTerminalPrinter.Mode.ERROR : "",
+ numOfFailedCases,
+ AnsiTerminalPrinter.Mode.DEFAULT,
+ numOfTotalTestCases);
+
+ if (shouldPrintTestCaseSummary) {
+ verify(ansiTerminalPrinter).printLn(summaryMessage);
+ } else {
+ verify(ansiTerminalPrinter, never()).printLn(summaryMessage);
+ }
+ }
+
+ @Test
+ public void testCasesDataVisibleInTestCaseOption() throws Exception {
+ givenExecutionOption(TestSummaryFormat.TESTCASE);
+ runTest(true);
+ }
+
+ @Test
+ public void testCasesDataVisibleInDetailedOption() throws Exception {
+ givenExecutionOption(TestSummaryFormat.DETAILED);
+ runTest(true);
+ }
+
+ @Test
+ public void testCasesDataInVisibleInShortOption() throws Exception {
+ givenExecutionOption(TestSummaryFormat.SHORT);
+ runTest(false);
+ }
+
+ @Test
+ public void testCasesDataInVisibleInTerseOption() throws Exception {
+ givenExecutionOption(TestSummaryFormat.TERSE);
+ runTest(false);
+ }
+
+ @Test
+ public void testCasesDataInVisibleInNoneOption() throws Exception {
+ givenExecutionOption(TestSummaryFormat.NONE);
+ runTest(false);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/runtime/TestSummaryTest.java b/src/test/java/com/google/devtools/build/lib/runtime/TestSummaryTest.java
index ee0f08c..c48ebbc 100644
--- a/src/test/java/com/google/devtools/build/lib/runtime/TestSummaryTest.java
+++ b/src/test/java/com/google/devtools/build/lib/runtime/TestSummaryTest.java
@@ -108,7 +108,7 @@
TestSummary summary = createTestSummary(stubTarget, BlazeTestStatus.PASSED, NOT_CACHED);
TestSummaryPrinter.print(summary, terminalPrinter, true, false);
- terminalPrinter.print(find(expectedString));
+ verify(terminalPrinter).print(find(expectedString));
}
@Test
@@ -224,6 +224,7 @@
.build();
assertThat(failedCacheTemplate.numCached()).isEqualTo(50);
assertThat(failedCacheTemplate.getStatus()).isEqualTo(BlazeTestStatus.FAILED);
+ assertThat(failedCacheTemplate.getTotalTestCases()).isEqualTo(fiftyCached.getTotalTestCases());
}
@Test
@@ -437,6 +438,7 @@
public void testCollectingFailedDetails() throws Exception {
TestCase rootCase = TestCase.newBuilder()
.setName("tests")
+ .setClassName("testclass")
.setRunDurationMillis(5000L)
.addChild(newDetail("apple", TestCase.Status.FAILED, 1000L))
.addChild(newDetail("banana", TestCase.Status.PASSED, 1000L))
@@ -455,6 +457,57 @@
verify(printer).print(find("ERROR.*cherry"));
}
+ @Test
+ public void countTotalTestCases() throws Exception {
+ TestCase rootCase =
+ TestCase.newBuilder()
+ .setName("tests")
+ .setRunDurationMillis(5000L)
+ .addChild(newDetail("apple", TestCase.Status.FAILED, 1000L))
+ .addChild(newDetail("banana", TestCase.Status.PASSED, 1000L))
+ .addChild(newDetail("cherry", TestCase.Status.ERROR, 1000L))
+ .build();
+
+ TestSummary summary =
+ getTemplateBuilder()
+ .countTotalTestCases(rootCase)
+ .setStatus(BlazeTestStatus.FAILED)
+ .build();
+
+ assertThat(summary.getTotalTestCases()).isEqualTo(3);
+ }
+
+ @Test
+ public void countTotalTestCasesInNestedTree() throws Exception {
+ TestCase aCase =
+ TestCase.newBuilder()
+ .setName("tests-1")
+ .setRunDurationMillis(5000L)
+ .addChild(newDetail("apple", TestCase.Status.FAILED, 1000L))
+ .addChild(newDetail("banana", TestCase.Status.PASSED, 1000L))
+ .addChild(newDetail("cherry", TestCase.Status.ERROR, 1000L))
+ .build();
+ TestCase anotherCase =
+ TestCase.newBuilder()
+ .setName("tests-2")
+ .setRunDurationMillis(5000L)
+ .addChild(newDetail("apple", TestCase.Status.FAILED, 1000L))
+ .addChild(newDetail("banana", TestCase.Status.PASSED, 1000L))
+ .addChild(newDetail("cherry", TestCase.Status.ERROR, 1000L))
+ .build();
+
+ TestCase rootCase =
+ TestCase.newBuilder().setName("tests").addChild(aCase).addChild(anotherCase).build();
+
+ TestSummary summary =
+ getTemplateBuilder()
+ .countTotalTestCases(rootCase)
+ .setStatus(BlazeTestStatus.FAILED)
+ .build();
+
+ assertThat(summary.getTotalTestCases()).isEqualTo(6);
+ }
+
private ConfiguredTarget target(String path, String targetName) throws Exception {
ConfiguredTarget target = Mockito.mock(ConfiguredTarget.class);
when(target.getLabel()).thenReturn(Label.create(path, targetName));
