Adds capability of parsing files in parallel to CoverageOutputGenerator/LcovMerger.
- Adds capability of parsing files in parallel to CoverageOutputGenerator/LcovMerger.
- Uses immutable objects and java parallel stream to keep the code as simple as possible.
- Adds functional tests to cross-validate parallel vs sequential test output
- Makes parallel parsing the default behavior in Bazel report generator (overridable with --combined_report_opt=--parse_sequentially)
PiperOrigin-RevId: 293646734
diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Coverage.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Coverage.java
index 25c1851..aa3d87f 100644
--- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Coverage.java
+++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Coverage.java
@@ -18,6 +18,7 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
+import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Set;
@@ -51,6 +52,18 @@
return merged;
}
+ static Coverage create(SourceFileCoverage... sourceFilesCoverage) {
+ return create(Arrays.asList(sourceFilesCoverage));
+ }
+
+ static Coverage create(List<SourceFileCoverage> sourceFilesCoverage) {
+ Coverage coverage = new Coverage();
+ for (SourceFileCoverage sourceFileCoverage : sourceFilesCoverage) {
+ coverage.add(sourceFileCoverage);
+ }
+ return coverage;
+ }
+
/**
* Returns {@link Coverage} only for the given CC source filenames, filtering out every other CC
* sources of the given coverage. Other types of source files (e.g. Java) will not be filtered
diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/LcovMergerFlags.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/LcovMergerFlags.java
index 9b5363c..eaaf560 100644
--- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/LcovMergerFlags.java
+++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/LcovMergerFlags.java
@@ -55,6 +55,9 @@
@Parameter(names = "--sources_to_replace_file")
private String sourcesToReplaceFile;
+ @Parameter(names = "--parse_sequentially")
+ private boolean parseSequentially;
+
public String coverageDir() {
return coverageDir;
}
@@ -83,6 +86,10 @@
return sourceFileManifest != null;
}
+ boolean parseSequentially() {
+ return parseSequentially;
+ }
+
static LcovMergerFlags parseFlags(String[] args) {
LcovMergerFlags flags = new LcovMergerFlags();
JCommander jCommander = new JCommander(flags);
diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java
index 9b61a22..cd2ca7d 100644
--- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java
+++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java
@@ -35,6 +35,7 @@
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
+import java.util.Objects;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -45,7 +46,7 @@
public class Main {
private static final Logger logger = Logger.getLogger(Main.class.getName());
- public static void main(String[] args) {
+ public static void main(String... args) {
LcovMergerFlags flags = null;
try {
flags = LcovMergerFlags.parseFlags(args);
@@ -62,8 +63,14 @@
: Collections.emptyList();
Coverage coverage =
Coverage.merge(
- parseFiles(getTracefiles(flags, filesInCoverageDir), LcovParser::parse),
- parseFiles(getGcovInfoFiles(filesInCoverageDir), GcovParser::parse));
+ parseFiles(
+ getTracefiles(flags, filesInCoverageDir),
+ LcovParser::parse,
+ flags.parseSequentially()),
+ parseFiles(
+ getGcovInfoFiles(filesInCoverageDir),
+ GcovParser::parse,
+ flags.parseSequentially()));
if (flags.sourcesToReplaceFile() != null) {
coverage.maybeReplaceSourceFileNames(getMapFromFile(flags.sourcesToReplaceFile()));
@@ -246,7 +253,15 @@
return mapBuilder.build();
}
- private static Coverage parseFiles(List<File> files, Parser parser) {
+ static Coverage parseFiles(List<File> files, Parser parser, boolean parseSequentially) {
+ if (parseSequentially) {
+ return parseFilesSequentially(files, parser);
+ } else {
+ return parseFilesInParallel(files, parser);
+ }
+ }
+
+ static Coverage parseFilesSequentially(List<File> files, Parser parser) {
Coverage coverage = new Coverage();
for (File file : files) {
try {
@@ -265,6 +280,31 @@
return coverage;
}
+ static Coverage parseFilesInParallel(List<File> files, Parser parser) {
+ return files.stream()
+ .parallel()
+ .map(
+ file -> {
+ try (FileInputStream inputStream = new FileInputStream(file)) {
+ logger.log(Level.INFO, "Parsing file " + file);
+ return parser.parse(inputStream);
+ } catch (IOException e) {
+ logger.log(
+ Level.SEVERE,
+ "File "
+ + file.getAbsolutePath()
+ + " could not be parsed due to: "
+ + e.getMessage());
+ System.exit(1);
+ }
+ return null;
+ })
+ .filter(Objects::nonNull)
+ .map(Coverage::create)
+ .reduce(Coverage::merge)
+ .orElse(Coverage.create());
+ }
+
/**
* Returns a list of all the files with the given extension found recursively under the given dir.
*/
diff --git a/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/BUILD b/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/BUILD
index 4f597de..7acaa7e 100644
--- a/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/BUILD
+++ b/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/BUILD
@@ -122,6 +122,9 @@
"//third_party:junit4",
"//third_party:truth",
"//tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:Constants",
+ "//tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:Coverage",
+ "//tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:LcovParser",
+ "//tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:LcovPrinter",
"//tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:MainLibrary",
],
)
diff --git a/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/LcovMergerTestUtils.java b/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/LcovMergerTestUtils.java
index cfb6486..10f0dda 100644
--- a/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/LcovMergerTestUtils.java
+++ b/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/LcovMergerTestUtils.java
@@ -17,6 +17,12 @@
import static com.google.common.truth.Truth.assertThat;
import com.google.common.collect.ImmutableList;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import java.util.TreeMap;
@@ -400,4 +406,41 @@
assertThat(merged.nrOfLinesWithNonZeroExecution()).isEqualTo(14);
assertThat(merged.nrOfInstrumentedLines()).isEqualTo(14);
}
+
+ static List<String> generateLcovContents(
+ String srcPrefix, int numSourceFiles, int numLinesPerSourceFile) {
+ ArrayList<String> lines = new ArrayList<>();
+ for (int i = 0; i < numSourceFiles; i++) {
+ lines.add(String.format("SF:%s%s.cc", srcPrefix, i));
+ lines.add("FNF:0");
+ lines.add("FNH:0");
+ for (int srcLineNum = 1; srcLineNum <= numLinesPerSourceFile; srcLineNum += 4) {
+ lines.add(String.format("BA:%s,2", srcLineNum));
+ }
+ lines.add("BRF:" + numLinesPerSourceFile / 4);
+ lines.add("BRH:" + numLinesPerSourceFile / 4);
+ for (int srcLineNum = 1; srcLineNum <= numLinesPerSourceFile; srcLineNum++) {
+ lines.add(String.format("DA:%s,%s", srcLineNum, srcLineNum % 2));
+ }
+ lines.add("LH:" + numLinesPerSourceFile / 2);
+ lines.add("LF:" + numLinesPerSourceFile);
+ lines.add("end_of_record");
+ }
+ return lines;
+ }
+
+ static List<Path> generateLcovFiles(
+ String srcPrefix, int numLcovFiles, int numSrcFiles, int numLinesPerSrcFile, Path coverageDir)
+ throws IOException {
+ Path lcovFile = Files.createFile(Paths.get(coverageDir.toString(), "coverage0.dat"));
+ List<Path> lcovFiles = new ArrayList<>();
+ Files.write(lcovFile, generateLcovContents(srcPrefix, numSrcFiles, numLinesPerSrcFile));
+ lcovFiles.add(lcovFile);
+ for (int i = 1; i < numLcovFiles; i++) {
+ lcovFiles.add(
+ Files.createSymbolicLink(
+ Paths.get(coverageDir.toString(), String.format("coverage%s.dat", i)), lcovFile));
+ }
+ return lcovFiles;
+ }
}
diff --git a/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/MainTest.java b/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/MainTest.java
index 52e90bb..84b650a 100644
--- a/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/MainTest.java
+++ b/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/MainTest.java
@@ -17,13 +17,16 @@
import static com.google.common.truth.Truth.assertThat;
import static com.google.devtools.coverageoutputgenerator.Constants.TRACEFILE_EXTENSION;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -31,11 +34,12 @@
@RunWith(JUnit4.class)
public class MainTest {
+ @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
private Path coverageDir;
@Before
public void createCoverageDirectory() throws IOException {
- coverageDir = Files.createTempDirectory("coverage-dir");
+ coverageDir = temporaryFolder.newFolder("coverage-dir").toPath();
}
@Test
@@ -55,4 +59,34 @@
List<File> tracefiles = Main.getFilesWithExtension(coverageFiles, TRACEFILE_EXTENSION);
assertThat(tracefiles).hasSize(2);
}
+
+ @Test
+ public void testParallelParse_1KLoC_1KLcovFiles() throws IOException {
+ assertParallelParse(1024, 4, 256);
+ }
+
+ @Test
+ public void testParallelParse_1MLoC_4LcovFiles() throws IOException {
+ assertParallelParse(4, 1024, 1024);
+ }
+
+ private void assertParallelParse(int numLcovFiles, int numSourceFiles, int numLinesPerSourceFile)
+ throws IOException {
+
+ ByteArrayOutputStream sequentialOutput = new ByteArrayOutputStream();
+ ByteArrayOutputStream parallelOutput = new ByteArrayOutputStream();
+
+ LcovMergerTestUtils.generateLcovFiles(
+ "test_data/simple_test", numLcovFiles, numSourceFiles, numLinesPerSourceFile, coverageDir);
+
+ List<File> coverageFiles = Main.getCoverageFilesInDir(coverageDir.toAbsolutePath().toString());
+
+ Coverage sequentialCoverage = Main.parseFilesSequentially(coverageFiles, LcovParser::parse);
+ LcovPrinter.print(sequentialOutput, sequentialCoverage);
+
+ Coverage parallelCoverage = Main.parseFilesInParallel(coverageFiles, LcovParser::parse);
+ LcovPrinter.print(parallelOutput, parallelCoverage);
+
+ assertThat(parallelOutput.toString()).isEqualTo(sequentialOutput.toString());
+ }
}
