Fix GCC 9 Code coverage by adding its JSON format
GCC 9 changed the intermediate format to a JSON based format and with it
changed the meaning of the `-i` flag. Because of these changes it was not
possible to generate code coverage with GCC 9. This patch addresses that
by adding its format next to the existing GCov parser.
Addresses: #9406
Closes #11538.
PiperOrigin-RevId: 316641962
diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD
index db22b5c..d9f59a4 100644
--- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD
+++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD
@@ -49,6 +49,7 @@
srcs = glob(["*.java"]),
deps = [
"//third_party:auto_value",
+ "//third_party:gson",
"//third_party:guava",
"//third_party:jsr305",
"//third_party/java/jcommander",
@@ -118,6 +119,21 @@
)
java_library(
+ name = "GcovJsonParser",
+ srcs = [
+ "GcovJsonParser.java",
+ "Parser.java",
+ ],
+ deps = [
+ ":BranchCoverage",
+ ":Constants",
+ ":LineCoverage",
+ ":SourceFileCoverage",
+ "//third_party:gson",
+ ],
+)
+
+java_library(
name = "LcovParser",
srcs = [
"LcovParser.java",
@@ -158,6 +174,7 @@
deps = [
":Constants",
":Coverage",
+ ":GcovJsonParser",
":GcovParser",
":LcovMergerFlags",
":LcovParser",
diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Constants.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Constants.java
index a4b1b53..808e257 100644
--- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Constants.java
+++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Constants.java
@@ -39,6 +39,7 @@
static final String TAKEN = "-";
static final String TRACEFILE_EXTENSION = ".dat";
static final String GCOV_EXTENSION = ".gcov";
+ static final String GCOV_JSON_EXTENSION = ".gcov.json.gz";
static final String PROFDATA_EXTENSION = ".profdata";
static final String DELIMITER = ",";
static final String GCOV_VERSION_MARKER = "version:";
diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Coverage.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Coverage.java
index aa3d87f..afd3899 100644
--- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Coverage.java
+++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Coverage.java
@@ -41,13 +41,12 @@
}
}
- static Coverage merge(Coverage c1, Coverage c2) {
+ static Coverage merge(Coverage... coverages) {
Coverage merged = new Coverage();
- for (SourceFileCoverage sourceFile : c1.getAllSourceFiles()) {
- merged.add(sourceFile);
- }
- for (SourceFileCoverage sourceFile : c2.getAllSourceFiles()) {
- merged.add(sourceFile);
+ for (Coverage c : coverages) {
+ for (SourceFileCoverage sourceFile : c.getAllSourceFiles()) {
+ merged.add(sourceFile);
+ }
}
return merged;
}
diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/GcovJsonParser.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/GcovJsonParser.java
new file mode 100644
index 0000000..72ac862
--- /dev/null
+++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/GcovJsonParser.java
@@ -0,0 +1,128 @@
+// Copyright 2020 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.coverageoutputgenerator;
+
+import com.google.gson.Gson;
+import com.google.gson.annotations.SerializedName;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.zip.GZIPInputStream;
+
+/**
+ * A {@link Parser} for gcov intermediate json format introduced in GCC 9.1. See the flag {@code
+ * --intermediate-format} in <a
+ * href="https://gcc.gnu.org/onlinedocs/gcc-9.3.0/gcc/Invoking-Gcov.html">gcov documentation</a>.
+ */
+public class GcovJsonParser {
+ private static final Logger logger = Logger.getLogger(GcovJsonParser.class.getName());
+ private final InputStream inputStream;
+
+ private GcovJsonParser(InputStream inputStream) {
+ this.inputStream = inputStream;
+ }
+
+ public static List<SourceFileCoverage> parse(InputStream inputStream) throws IOException {
+ return new GcovJsonParser(inputStream).parse();
+ }
+
+ private List<SourceFileCoverage> parse() throws IOException {
+ ArrayList<SourceFileCoverage> allSourceFiles = new ArrayList<>();
+ try (InputStream gzipStream = new GZIPInputStream(inputStream)) {
+ ByteArrayOutputStream contents = new ByteArrayOutputStream();
+ byte[] buffer = new byte[1024];
+ int length;
+ while ((length = gzipStream.read(buffer)) != -1) {
+ contents.write(buffer, 0, length);
+ }
+ Gson gson = new Gson();
+ GcovJsonFormat document = gson.fromJson(contents.toString(), GcovJsonFormat.class);
+ if (!document.format_version.equals("1")) {
+ logger.log(
+ Level.WARNING,
+ "Expect GCov JSON format version 1, got format version " + document.format_version);
+ }
+ for (GcovJsonFile file : document.files) {
+ SourceFileCoverage currentFileCoverage = new SourceFileCoverage(file.file);
+ for (GcovJsonFunction function : file.functions) {
+ currentFileCoverage.addLineNumber(function.name, function.start_line);
+ currentFileCoverage.addFunctionExecution(function.name, function.execution_count);
+ }
+ for (GcovJsonLine line : file.lines) {
+ currentFileCoverage.addLine(
+ line.line_number, LineCoverage.create(line.line_number, line.count, null));
+ for (GcovJsonBranch branch : line.branches) {
+ currentFileCoverage.addBranch(
+ line.line_number, BranchCoverage.create(line.line_number, branch.count));
+ }
+ }
+ allSourceFiles.add(currentFileCoverage);
+ }
+ }
+
+ return allSourceFiles;
+ }
+
+ // Classes for the Gson data mapper representing the structure of the GCov JSON format
+ // These do not follow the Java naming styleguide as they need to match the JSON field names
+ // Documentation can be found in GCov's manpage, of which the source is available at
+ // https://gcc.gnu.org/git/?p=gcc.git;a=blob;f=gcc/doc/gcov.texi;h=dcdd7831ff063483d43e5347af0b67083c85ecc4;hb=4212a6a3e44f870412d9025eeb323fd4f50a61da#l184
+
+ static class GcovJsonFormat {
+ String gcc_version;
+ GcovJsonFile[] files;
+ String format_version;
+ String current_working_directory;
+ String data_file;
+ }
+
+ static class GcovJsonFile {
+ String file;
+ GcovJsonFunction[] functions;
+ GcovJsonLine[] lines;
+ }
+
+ static class GcovJsonFunction {
+ int blocks;
+ int end_column;
+ int start_line;
+ String name;
+ int blocks_executed;
+ int execution_count;
+ String demangled_name;
+ int start_column;
+ int end_line;
+ }
+
+ static class GcovJsonLine {
+ GcovJsonBranch[] branches;
+ int count;
+ int line_number;
+ boolean unexecuted_block;
+ String function_name;
+ }
+
+ static class GcovJsonBranch {
+ boolean fallthrough;
+ int count;
+
+ @SerializedName("throw")
+ boolean _throw;
+ }
+}
diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java
index 21f3565..7f24d32 100644
--- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java
+++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java
@@ -15,6 +15,7 @@
package com.google.devtools.coverageoutputgenerator;
import static com.google.devtools.coverageoutputgenerator.Constants.GCOV_EXTENSION;
+import static com.google.devtools.coverageoutputgenerator.Constants.GCOV_JSON_EXTENSION;
import static com.google.devtools.coverageoutputgenerator.Constants.PROFDATA_EXTENSION;
import static com.google.devtools.coverageoutputgenerator.Constants.TRACEFILE_EXTENSION;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -79,7 +80,11 @@
LcovParser::parse,
flags.parseParallelism()),
parseFiles(
- getGcovInfoFiles(filesInCoverageDir), GcovParser::parse, flags.parseParallelism()));
+ getGcovInfoFiles(filesInCoverageDir), GcovParser::parse, flags.parseParallelism()),
+ parseFiles(
+ getGcovJsonInfoFiles(filesInCoverageDir),
+ GcovJsonParser::parse,
+ flags.parseParallelism()));
if (flags.sourcesToReplaceFile() != null) {
coverage.maybeReplaceSourceFileNames(getMapFromFile(flags.sourcesToReplaceFile()));
@@ -221,6 +226,16 @@
return gcovFiles;
}
+ private static List<File> getGcovJsonInfoFiles(List<File> filesInCoverageDir) {
+ List<File> gcovJsonFiles = getFilesWithExtension(filesInCoverageDir, GCOV_JSON_EXTENSION);
+ if (gcovJsonFiles.isEmpty()) {
+ logger.log(Level.INFO, "No gcov json file found.");
+ } else {
+ logger.log(Level.INFO, "Found " + gcovJsonFiles.size() + " gcov json files.");
+ }
+ return gcovJsonFiles;
+ }
+
/**
* Returns a .profdata file from the given files or null if none or more profdata files were
* found.
@@ -352,6 +367,7 @@
p ->
p.toString().endsWith(TRACEFILE_EXTENSION)
|| p.toString().endsWith(GCOV_EXTENSION)
+ || p.toString().endsWith(GCOV_JSON_EXTENSION)
|| p.toString().endsWith(PROFDATA_EXTENSION))
.map(path -> path.toFile())
.collect(Collectors.toList());
diff --git a/tools/test/collect_cc_coverage.sh b/tools/test/collect_cc_coverage.sh
index 181433b..1d50238 100755
--- a/tools/test/collect_cc_coverage.sh
+++ b/tools/test/collect_cc_coverage.sh
@@ -125,10 +125,23 @@
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84879).
"${GCOV}" -i $COVERAGE_GCOV_OPTIONS -o "$(dirname ${gcda})" "${gcda}"
- # Append all .gcov files in the current directory to the output file.
- cat *.gcov >> "$output_file"
- # Delete the .gcov files.
- rm *.gcov
+ # Extract gcov's version: the output of `gcov --version` contains the
+ # version as a set of major-minor-patch numbers, of which we extract
+ # the major version.
+ gcov_major_version=$("${GCOV}" --version | sed -n -E -e 's/^.*\s([0-9]+)\.[0-9]+\.[0-9]+\s?.*$/\1/p')
+
+ # Check the gcov version so we can process the data correctly
+ if [[ $gcov_major_version -ge 9 ]]; then
+ # gcov 9 or higher use a JSON based format for their coverage reports.
+ # The output is generated into multiple files: "$(basename ${gcda}).gcov.json.gz"
+ # Concatenating JSON documents does not yield a valid document, so they are moved individually
+ mv -- *.gcov.json.gz "$(dirname "$output_file")"
+ else
+ # Append all .gcov files in the current directory to the output file.
+ cat -- *.gcov >> "$output_file"
+ # Delete the .gcov files.
+ rm -- *.gcov
+ fi
fi
fi
done < "${COVERAGE_MANIFEST}"
