scripts/coverage/compare_gcov_json.py

   1 #!/usr/bin/env python3
   2 #
   3 # Compare output of two gcovr JSON reports and report differences. To
   4 # generate the required output first:
   5 #   - create two build dirs with --enable-gcov
   6 #   - run set of tests in each
   7 #   - run make coverage-html in each
   8 #   - run gcovr --json --exclude-unreachable-branches \
   9 #           --print-summary -o coverage.json --root ../../ . *.p
  10 #
  11 # Author: Alex Bennée <alex.bennee@linaro.org>
  12 #
  13 # SPDX-License-Identifier: GPL-2.0-or-later
  14 #
  15
  16 import argparse
  17 import json
  18 import sys
  19 from pathlib import Path
  20
  21 def create_parser():
  22     parser = argparse.ArgumentParser(
  23         prog='compare_gcov_json',
  24         description='analyse the differences in coverage between two runs')
  25
  26     parser.add_argument('-a', type=Path, default=None,
  27                         help=('First file to check'))
  28
  29     parser.add_argument('-b', type=Path, default=None,
  30                         help=('Second file to check'))
  31
  32     parser.add_argument('--verbose', action='store_true', default=False,
  33                         help=('A minimal verbosity level that prints the '
  34                               'overall result of the check/wait'))
  35     return parser
  36
  37
  38 # See https://gcovr.com/en/stable/output/json.html#json-format-reference
  39 def load_json(json_file_path: Path, verbose = False) -> dict[str, set[int]]:
  40
  41     with open(json_file_path) as f:
  42         data = json.load(f)
  43
  44     root_dir = json_file_path.absolute().parent
  45     covered_lines = dict()
  46
  47     for filecov in data["files"]:
  48         file_path = Path(filecov["file"])
  49
  50         # account for generated files - map into src tree
  51         resolved_path = Path(file_path).absolute()
  52         if resolved_path.is_relative_to(root_dir):
  53             file_path = resolved_path.relative_to(root_dir)
  54             # print(f"remapped {resolved_path} to {file_path}")
  55
  56         lines = filecov["lines"]
  57
  58         executed_lines = set(
  59             linecov["line_number"]
  60             for linecov in filecov["lines"]
  61             if linecov["count"] != 0 and not linecov["gcovr/noncode"]
  62         )
  63
  64         # if this file has any coverage add it to the system
  65         if len(executed_lines) > 0:
  66             if verbose:
  67                 print(f"file {file_path} {len(executed_lines)}/{len(lines)}")
  68             covered_lines[str(file_path)] = executed_lines
  69
  70     return covered_lines
  71
  72 def find_missing_files(first, second):
  73     """
  74     Return a list of files not covered in the second set
  75     """
  76     missing_files = []
  77     for f in sorted(first):
  78         file_a = first[f]
  79         try:
  80             file_b = second[f]
  81         except KeyError:
  82             missing_files.append(f)
  83
  84     return missing_files
  85
  86 def main():
  87     """
  88     Script entry point
  89     """
  90     parser = create_parser()
  91     args = parser.parse_args()
  92
  93     if not args.a or not args.b:
  94         print("We need two files to compare")
  95         sys.exit(1)
  96
  97     first_coverage = load_json(args.a, args.verbose)
  98     second_coverage = load_json(args.b, args.verbose)
  99
 100     first_missing = find_missing_files(first_coverage,
 101                                        second_coverage)
 102
 103     second_missing = find_missing_files(second_coverage,
 104                                         first_coverage)
 105
 106     a_name = args.a.parent.name
 107     b_name = args.b.parent.name
 108
 109     print(f"{b_name} missing coverage in {len(first_missing)} files")
 110     for f in first_missing:
 111         print(f"  {f}")
 112
 113     print(f"{a_name} missing coverage in {len(second_missing)} files")
 114     for f in second_missing:
 115         print(f"  {f}")
 116
 117
 118 if __name__ == '__main__':
 119     main()