Limit the number of threads used to parallel parse and merge lcov files
PiperOrigin-RevId: 300786351
diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/LcovMergerFlags.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/LcovMergerFlags.java
index eaaf560..15e2dd8 100644
--- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/LcovMergerFlags.java
+++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/LcovMergerFlags.java
@@ -26,6 +26,7 @@
@Parameters(separators = "= ", optionPrefixes = "--")
class LcovMergerFlags {
private static final Logger logger = Logger.getLogger(LcovMergerFlags.class.getName());
+ private static final int DEFAULT_PARSE_FILE_PARALLELISM = 8;
@Parameter(names = "--coverage_dir")
private String coverageDir;
@@ -55,8 +56,8 @@
@Parameter(names = "--sources_to_replace_file")
private String sourcesToReplaceFile;
- @Parameter(names = "--parse_sequentially")
- private boolean parseSequentially;
+ @Parameter(names = "--parse_parallelism")
+ private Integer parseParallelism;
public String coverageDir() {
return coverageDir;
@@ -86,8 +87,8 @@
return sourceFileManifest != null;
}
- boolean parseSequentially() {
- return parseSequentially;
+ int parseParallelism() {
+ return parseParallelism == null ? DEFAULT_PARSE_FILE_PARALLELISM : parseParallelism;
}
static LcovMergerFlags parseFlags(String[] args) {
diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java
index 4e50e29..21f3565 100644
--- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java
+++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java
@@ -35,8 +35,9 @@
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
-import java.util.Objects;
import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ForkJoinPool;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
@@ -56,7 +57,7 @@
}
}
- static int runWithArgs(String... args) {
+ static int runWithArgs(String... args) throws ExecutionException, InterruptedException {
LcovMergerFlags flags = null;
try {
flags = LcovMergerFlags.parseFlags(args);
@@ -76,11 +77,9 @@
parseFiles(
getTracefiles(flags, filesInCoverageDir),
LcovParser::parse,
- flags.parseSequentially()),
+ flags.parseParallelism()),
parseFiles(
- getGcovInfoFiles(filesInCoverageDir),
- GcovParser::parse,
- flags.parseSequentially()));
+ getGcovInfoFiles(filesInCoverageDir), GcovParser::parse, flags.parseParallelism()));
if (flags.sourcesToReplaceFile() != null) {
coverage.maybeReplaceSourceFileNames(getMapFromFile(flags.sourcesToReplaceFile()));
@@ -285,11 +284,12 @@
return mapBuilder.build();
}
- static Coverage parseFiles(List<File> files, Parser parser, boolean parseSequentially) {
- if (parseSequentially) {
+ static Coverage parseFiles(List<File> files, Parser parser, int parallelism)
+ throws ExecutionException, InterruptedException {
+ if (parallelism == 1) {
return parseFilesSequentially(files, parser);
} else {
- return parseFilesInParallel(files, parser);
+ return parseFilesInParallel(files, parser, parallelism);
}
}
@@ -312,29 +312,31 @@
return coverage;
}
- static Coverage parseFilesInParallel(List<File> files, Parser parser) {
- return files.stream()
- .parallel()
- .map(
- file -> {
- try (FileInputStream inputStream = new FileInputStream(file)) {
- logger.log(Level.INFO, "Parsing file " + file);
- return parser.parse(inputStream);
- } catch (IOException e) {
- logger.log(
- Level.SEVERE,
- "File "
- + file.getAbsolutePath()
- + " could not be parsed due to: "
- + e.getMessage());
- System.exit(1);
- }
- return null;
- })
- .filter(Objects::nonNull)
- .map(Coverage::create)
- .reduce(Coverage::merge)
- .orElse(Coverage.create());
+ static Coverage parseFilesInParallel(List<File> files, Parser parser, int parallelism)
+ throws ExecutionException, InterruptedException {
+ ForkJoinPool pool = new ForkJoinPool(parallelism);
+ return pool.submit(
+ () ->
+ files.parallelStream()
+ .map(
+ file -> {
+ try (FileInputStream inputStream = new FileInputStream(file)) {
+ logger.log(Level.INFO, "Parsing file " + file);
+ return Coverage.create(parser.parse(inputStream));
+ } catch (IOException e) {
+ logger.log(
+ Level.SEVERE,
+ "File "
+ + file.getAbsolutePath()
+ + " could not be parsed due to: "
+ + e.getMessage());
+ System.exit(1);
+ }
+ return null;
+ })
+ .reduce(Coverage::merge)
+ .orElse(Coverage.create()))
+ .get();
}
/**
diff --git a/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/MainTest.java b/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/MainTest.java
index 370458f..e39d190 100644
--- a/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/MainTest.java
+++ b/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/MainTest.java
@@ -36,6 +36,7 @@
@RunWith(JUnit4.class)
public class MainTest {
+ private static final int TEST_PARSE_PARALLELISM = 8;
@Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
@Rule public TestName testName = new TestName();
private Path coverageDir;
@@ -64,17 +65,17 @@
}
@Test
- public void testParallelParse_1KLoC_1KLcovFiles() throws IOException {
+ public void testParallelParse_1KLoC_1KLcovFiles() throws Exception {
assertParallelParse(1024, 4, 256);
}
@Test
- public void testParallelParse_1MLoC_4LcovFiles() throws IOException {
+ public void testParallelParse_1MLoC_4LcovFiles() throws Exception {
assertParallelParse(4, 1024, 1024);
}
@Test
- public void testEmptyInputProducesEmptyOutput() throws IOException {
+ public void testEmptyInputProducesEmptyOutput() throws Exception {
Path output =
Paths.get(
temporaryFolder.getRoot().getAbsolutePath(),
@@ -89,7 +90,7 @@
}
@Test
- public void testNonEmptyInputProducesNonEmptyOutput() throws IOException {
+ public void testNonEmptyInputProducesNonEmptyOutput() throws Exception {
LcovMergerTestUtils.generateLcovFiles("test_data/simple_test", 8, 8, 8, coverageDir);
Path output =
Paths.get(
@@ -105,7 +106,7 @@
}
private void assertParallelParse(int numLcovFiles, int numSourceFiles, int numLinesPerSourceFile)
- throws IOException {
+ throws Exception {
ByteArrayOutputStream sequentialOutput = new ByteArrayOutputStream();
ByteArrayOutputStream parallelOutput = new ByteArrayOutputStream();
@@ -118,7 +119,8 @@
Coverage sequentialCoverage = Main.parseFilesSequentially(coverageFiles, LcovParser::parse);
LcovPrinter.print(sequentialOutput, sequentialCoverage);
- Coverage parallelCoverage = Main.parseFilesInParallel(coverageFiles, LcovParser::parse);
+ Coverage parallelCoverage =
+ Main.parseFilesInParallel(coverageFiles, LcovParser::parse, TEST_PARSE_PARALLELISM);
LcovPrinter.print(parallelOutput, parallelCoverage);
assertThat(parallelOutput.toString()).isEqualTo(sequentialOutput.toString());
