testing/testinfo.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, division, print_function
   6
   7 import datetime
   8 import errno
   9 import json
  10 import os
  11 import posixpath
  12 import re
  13 import requests
  14 import six.moves.urllib_parse as urlparse
  15 import subprocess
  16 import mozpack.path as mozpath
  17 from moztest.resolve import TestResolver, TestManifestLoader
  18 from mozfile import which
  19
  20 from mozbuild.base import MozbuildObject, MachCommandConditions as conditions
  21
  22 REFERER = "https://wiki.developer.mozilla.org/en-US/docs/Mozilla/Test-Info"
  23
  24
  25 class TestInfo(object):
  26     """
  27     Support 'mach test-info'.
  28     """
  29
  30     def __init__(self, verbose):
  31         self.verbose = verbose
  32         here = os.path.abspath(os.path.dirname(__file__))
  33         self.build_obj = MozbuildObject.from_environment(cwd=here)
  34
  35     def log_verbose(self, what):
  36         if self.verbose:
  37             print(what)
  38
  39
  40 class TestInfoTests(TestInfo):
  41     """
  42     Support 'mach test-info tests': Detailed report of specified tests.
  43     """
  44
  45     def __init__(self, verbose):
  46         TestInfo.__init__(self, verbose)
  47
  48         self._hg = None
  49         if conditions.is_hg(self.build_obj):
  50             self._hg = which("hg")
  51             if not self._hg:
  52                 raise OSError(errno.ENOENT, "Could not find 'hg' on PATH.")
  53
  54         self._git = None
  55         if conditions.is_git(self.build_obj):
  56             self._git = which("git")
  57             if not self._git:
  58                 raise OSError(errno.ENOENT, "Could not find 'git' on PATH.")
  59
  60     def find_in_hg_or_git(self, test_name):
  61         if self._hg:
  62             cmd = [self._hg, "files", "-I", test_name]
  63         elif self._git:
  64             cmd = [self._git, "ls-files", test_name]
  65         else:
  66             return None
  67         try:
  68             out = subprocess.check_output(cmd, universal_newlines=True).splitlines()
  69         except subprocess.CalledProcessError:
  70             out = None
  71         return out
  72
  73     def set_test_name(self):
  74         # Generating a unified report for a specific test is complicated
  75         # by differences in the test name used in various data sources.
  76         # Consider:
  77         #   - It is often convenient to request a report based only on
  78         #     a short file name, rather than the full path;
  79         #   - Bugs may be filed in bugzilla against a simple, short test
  80         #     name or the full path to the test;
  81         # This function attempts to find appropriate names for different
  82         # queries based on the specified test name.
  83
  84         # full_test_name is full path to file in hg (or git)
  85         self.full_test_name = None
  86         out = self.find_in_hg_or_git(self.test_name)
  87         if out and len(out) == 1:
  88             self.full_test_name = out[0]
  89         elif out and len(out) > 1:
  90             print("Ambiguous test name specified. Found:")
  91             for line in out:
  92                 print(line)
  93         else:
  94             out = self.find_in_hg_or_git("**/%s*" % self.test_name)
  95             if out and len(out) == 1:
  96                 self.full_test_name = out[0]
  97             elif out and len(out) > 1:
  98                 print("Ambiguous test name. Found:")
  99                 for line in out:
 100                     print(line)
 101         if self.full_test_name:
 102             self.full_test_name.replace(os.sep, posixpath.sep)
 103             print("Found %s in source control." % self.full_test_name)
 104         else:
 105             print("Unable to validate test name '%s'!" % self.test_name)
 106             self.full_test_name = self.test_name
 107
 108         # search for full_test_name in test manifests
 109         here = os.path.abspath(os.path.dirname(__file__))
 110         resolver = TestResolver.from_environment(
 111             cwd=here, loader_cls=TestManifestLoader
 112         )
 113         relpath = self.build_obj._wrap_path_argument(self.full_test_name).relpath()
 114         tests = list(resolver.resolve_tests(paths=[relpath]))
 115         if len(tests) == 1:
 116             relpath = self.build_obj._wrap_path_argument(tests[0]["manifest"]).relpath()
 117             print("%s found in manifest %s" % (self.full_test_name, relpath))
 118             if tests[0].get("flavor"):
 119                 print("  flavor: %s" % tests[0]["flavor"])
 120             if tests[0].get("skip-if"):
 121                 print("  skip-if: %s" % tests[0]["skip-if"])
 122             if tests[0].get("fail-if"):
 123                 print("  fail-if: %s" % tests[0]["fail-if"])
 124         elif len(tests) == 0:
 125             print("%s not found in any test manifest!" % self.full_test_name)
 126         else:
 127             print("%s found in more than one manifest!" % self.full_test_name)
 128
 129         # short_name is full_test_name without path
 130         self.short_name = None
 131         name_idx = self.full_test_name.rfind("/")
 132         if name_idx > 0:
 133             self.short_name = self.full_test_name[name_idx + 1 :]
 134         if self.short_name and self.short_name == self.test_name:
 135             self.short_name = None
 136
 137     def get_platform(self, record):
 138         if "platform" in record["build"]:
 139             platform = record["build"]["platform"]
 140         else:
 141             platform = "-"
 142         platform_words = platform.split("-")
 143         types_label = ""
 144         # combine run and build types and eliminate duplicates
 145         run_types = []
 146         if "run" in record and "type" in record["run"]:
 147             run_types = record["run"]["type"]
 148             run_types = run_types if isinstance(run_types, list) else [run_types]
 149         build_types = []
 150         if "build" in record and "type" in record["build"]:
 151             build_types = record["build"]["type"]
 152             build_types = (
 153                 build_types if isinstance(build_types, list) else [build_types]
 154             )
 155         run_types = list(set(run_types + build_types))
 156         # '1proc' is used as a treeherder label but does not appear in run types
 157         if "e10s" not in run_types:
 158             run_types = run_types + ["1proc"]
 159         for run_type in run_types:
 160             # chunked is not interesting
 161             if run_type == "chunked":
 162                 continue
 163             # e10s is the default: implied
 164             if run_type == "e10s":
 165                 continue
 166             # sometimes a build/run type is already present in the build platform
 167             if run_type in platform_words:
 168                 continue
 169             if types_label:
 170                 types_label += "-"
 171             types_label += run_type
 172         return "%s/%s:" % (platform, types_label)
 173
 174     def report_bugs(self):
 175         # Report open bugs matching test name
 176         search = self.full_test_name
 177         if self.test_name:
 178             search = "%s,%s" % (search, self.test_name)
 179         if self.short_name:
 180             search = "%s,%s" % (search, self.short_name)
 181         payload = {"quicksearch": search, "include_fields": "id,summary"}
 182         response = requests.get("https://bugzilla.mozilla.org/rest/bug", payload)
 183         response.raise_for_status()
 184         json_response = response.json()
 185         print("\nBugzilla quick search for '%s':" % search)
 186         if "bugs" in json_response:
 187             for bug in json_response["bugs"]:
 188                 print("Bug %s: %s" % (bug["id"], bug["summary"]))
 189         else:
 190             print("No bugs found.")
 191
 192     def report(
 193         self,
 194         test_names,
 195         start,
 196         end,
 197         show_info,
 198         show_bugs,
 199     ):
 200         self.start = start
 201         self.end = end
 202         self.show_info = show_info
 203
 204         if not self.show_info and not show_bugs:
 205             # by default, show everything
 206             self.show_info = True
 207             show_bugs = True
 208
 209         for test_name in test_names:
 210             print("===== %s =====" % test_name)
 211             self.test_name = test_name
 212             if len(self.test_name) < 6:
 213                 print("'%s' is too short for a test name!" % self.test_name)
 214                 continue
 215             self.set_test_name()
 216             if show_bugs:
 217                 self.report_bugs()
 218
 219
 220 class TestInfoReport(TestInfo):
 221     """
 222     Support 'mach test-info report': Report of test runs summarized by
 223     manifest and component.
 224     """
 225
 226     def __init__(self, verbose):
 227         TestInfo.__init__(self, verbose)
 228         self.threads = []
 229
 230     def update_report(self, by_component, result, path_mod):
 231         def update_item(item, label, value):
 232             # It is important to include any existing item value in case ActiveData
 233             # returns multiple records for the same test; that can happen if the report
 234             # sometimes maps more than one ActiveData record to the same path.
 235             new_value = item.get(label, 0) + value
 236             if type(new_value) == int:
 237                 item[label] = new_value
 238             else:
 239                 item[label] = float(round(new_value, 2))  # pylint: disable=W1633
 240
 241         if "test" in result and "tests" in by_component:
 242             test = result["test"]
 243             if path_mod:
 244                 test = path_mod(test)
 245             for bc in by_component["tests"]:
 246                 for item in by_component["tests"][bc]:
 247                     if test == item["test"]:
 248                         # pylint: disable=W1633
 249                         seconds = float(round(result.get("duration", 0), 2))
 250                         update_item(item, "total run time, seconds", seconds)
 251                         update_item(item, "total runs", result.get("count", 0))
 252                         update_item(item, "skipped runs", result.get("skips", 0))
 253                         update_item(item, "failed runs", result.get("failures", 0))
 254                         return True
 255         return False
 256
 257     def path_mod_reftest(self, path):
 258         # "<path1> == <path2>" -> "<path1>"
 259         path = path.split(" ")[0]
 260         # "<path>?<params>" -> "<path>"
 261         path = path.split("?")[0]
 262         # "<path>#<fragment>" -> "<path>"
 263         path = path.split("#")[0]
 264         return path
 265
 266     def path_mod_jsreftest(self, path):
 267         # "<path>;assert" -> "<path>"
 268         path = path.split(";")[0]
 269         return path
 270
 271     def path_mod_marionette(self, path):
 272         # "<path> <test-name>" -> "<path>"
 273         path = path.split(" ")[0]
 274         # "part1\part2" -> "part1/part2"
 275         path = path.replace("\\", os.path.sep)
 276         return path
 277
 278     def path_mod_wpt(self, path):
 279         if path[0] == os.path.sep:
 280             # "/<path>" -> "<path>"
 281             path = path[1:]
 282         # "<path>" -> "testing/web-platform/tests/<path>"
 283         path = os.path.join("testing", "web-platform", "tests", path)
 284         # "<path>?<params>" -> "<path>"
 285         path = path.split("?")[0]
 286         return path
 287
 288     def path_mod_jittest(self, path):
 289         # "part1\part2" -> "part1/part2"
 290         path = path.replace("\\", os.path.sep)
 291         # "<path>" -> "js/src/jit-test/tests/<path>"
 292         return os.path.join("js", "src", "jit-test", "tests", path)
 293
 294     def path_mod_xpcshell(self, path):
 295         # <manifest>.ini:<path> -> "<path>"
 296         path = path.split(".ini:")[-1]
 297         return path
 298
 299     def description(
 300         self,
 301         components,
 302         flavor,
 303         subsuite,
 304         paths,
 305         show_manifests,
 306         show_tests,
 307         show_summary,
 308         show_annotations,
 309         filter_values,
 310         filter_keys,
 311         start_date,
 312         end_date,
 313     ):
 314         # provide a natural language description of the report options
 315         what = []
 316         if show_manifests:
 317             what.append("test manifests")
 318         if show_tests:
 319             what.append("tests")
 320         if show_annotations:
 321             what.append("test manifest annotations")
 322         if show_summary and len(what) == 0:
 323             what.append("summary of tests only")
 324         if len(what) > 1:
 325             what[-1] = "and " + what[-1]
 326         what = ", ".join(what)
 327         d = "Test summary report for " + what
 328         if components:
 329             d += ", in specified components (%s)" % components
 330         else:
 331             d += ", in all components"
 332         if flavor:
 333             d += ", in specified flavor (%s)" % flavor
 334         if subsuite:
 335             d += ", in specified subsuite (%s)" % subsuite
 336         if paths:
 337             d += ", on specified paths (%s)" % paths
 338         if filter_values:
 339             d += ", containing '%s'" % filter_values
 340             if filter_keys:
 341                 d += " in manifest keys '%s'" % filter_keys
 342             else:
 343                 d += " in any part of manifest entry"
 344         d += ", including historical run-time data for the last "
 345
 346         start = datetime.datetime.strptime(start_date, "%Y-%m-%d")
 347         end = datetime.datetime.strptime(end_date, "%Y-%m-%d")
 348         d += "%s days on trunk (autoland/m-c)" % ((end - start).days)
 349         d += " as of %s." % end_date
 350         return d
 351
 352     # TODO: this is hacked for now and very limited
 353     def parse_test(self, summary):
 354         if summary.endswith("single tracking bug"):
 355             name_part = summary.split("|")[0]  # remove 'single tracking bug'
 356             name_part.strip()
 357             return name_part.split()[-1]  # get just the test name, not extra words
 358         return None
 359
 360     def get_intermittent_failure_data(self, start, end):
 361         retVal = {}
 362
 363         # get IFV bug list
 364         # i.e. https://th.m.o/api/failures/?startday=2022-06-22&endday=2022-06-29&tree=all
 365         url = (
 366             "https://treeherder.mozilla.org/api/failures/?startday=%s&endday=%s&tree=trunk"
 367             % (start, end)
 368         )
 369         r = requests.get(url, headers={"User-agent": "mach-test-info/1.0"})
 370         if_data = r.json()
 371         buglist = [x["bug_id"] for x in if_data]
 372
 373         # get bug data for summary, 800 bugs at a time
 374         # i.e. https://b.m.o/rest/bug?include_fields=id,product,component,summary&id=1,2,3...
 375         max_bugs = 800
 376         bug_data = []
 377         fields = ["id", "product", "component", "summary"]
 378         for bug_index in range(0, len(buglist), max_bugs):
 379             bugs = [str(x) for x in buglist[bug_index:max_bugs]]
 380             url = "https://bugzilla.mozilla.org/rest/bug?include_fields=%s&id=%s" % (
 381                 ",".join(fields),
 382                 ",".join(bugs),
 383             )
 384             r = requests.get(url, headers={"User-agent": "mach-test-info/1.0"})
 385             data = r.json()
 386             if data and "bugs" in data.keys():
 387                 bug_data.extend(data["bugs"])
 388
 389         # for each summary, parse filename, store component
 390         # IF we find >1 bug with same testname, for now summarize as one
 391         for bug in bug_data:
 392             test_name = self.parse_test(bug["summary"])
 393             if not test_name:
 394                 continue
 395
 396             c = int([x["bug_count"] for x in if_data if x["bug_id"] == bug["id"]][0])
 397             if test_name not in retVal.keys():
 398                 retVal[test_name] = {
 399                     "id": bug["id"],
 400                     "count": 0,
 401                     "product": bug["product"],
 402                     "component": bug["component"],
 403                 }
 404             retVal[test_name]["count"] += c
 405
 406             if bug["product"] != retVal[test_name]["product"]:
 407                 print(
 408                     "ERROR | %s | mismatched bugzilla product, bugzilla (%s) != repo (%s)"
 409                     % (bug["id"], bug["product"], retVal[test_name]["product"])
 410                 )
 411             if bug["component"] != retVal[test_name]["component"]:
 412                 print(
 413                     "ERROR | %s | mismatched bugzilla component, bugzilla (%s) != repo (%s)"
 414                     % (bug["id"], bug["component"], retVal[test_name]["component"])
 415                 )
 416         return retVal
 417
 418     def report(
 419         self,
 420         components,
 421         flavor,
 422         subsuite,
 423         paths,
 424         show_manifests,
 425         show_tests,
 426         show_summary,
 427         show_annotations,
 428         filter_values,
 429         filter_keys,
 430         show_components,
 431         output_file,
 432         start,
 433         end,
 434     ):
 435         def matches_filters(test):
 436             """
 437             Return True if all of the requested filter_values are found in this test;
 438             if filter_keys are specified, restrict search to those test keys.
 439             """
 440             for value in filter_values:
 441                 value_found = False
 442                 for key in test:
 443                     if not filter_keys or key in filter_keys:
 444                         if re.search(value, test[key]):
 445                             value_found = True
 446                             break
 447                 if not value_found:
 448                     return False
 449             return True
 450
 451         start_time = datetime.datetime.now()
 452
 453         # Ensure useful report by default
 454         if (
 455             not show_manifests
 456             and not show_tests
 457             and not show_summary
 458             and not show_annotations
 459         ):
 460             show_manifests = True
 461             show_summary = True
 462
 463         by_component = {}
 464         if components:
 465             components = components.split(",")
 466         if filter_keys:
 467             filter_keys = filter_keys.split(",")
 468         if filter_values:
 469             filter_values = filter_values.split(",")
 470         else:
 471             filter_values = []
 472         display_keys = (filter_keys or []) + ["skip-if", "fail-if", "fails-if"]
 473         display_keys = set(display_keys)
 474         ifd = self.get_intermittent_failure_data(start, end)
 475
 476         print("Finding tests...")
 477         here = os.path.abspath(os.path.dirname(__file__))
 478         resolver = TestResolver.from_environment(
 479             cwd=here, loader_cls=TestManifestLoader
 480         )
 481         tests = list(
 482             resolver.resolve_tests(paths=paths, flavor=flavor, subsuite=subsuite)
 483         )
 484
 485         manifest_paths = set()
 486         for t in tests:
 487             if t.get("manifest", None):
 488                 manifest_path = t["manifest"]
 489                 if t.get("ancestor_manifest", None):
 490                     manifest_path = "%s:%s" % (t["ancestor_manifest"], t["manifest"])
 491                 manifest_paths.add(manifest_path)
 492         manifest_count = len(manifest_paths)
 493         print(
 494             "Resolver found {} tests, {} manifests".format(len(tests), manifest_count)
 495         )
 496
 497         if show_manifests:
 498             topsrcdir = self.build_obj.topsrcdir
 499             by_component["manifests"] = {}
 500             manifest_paths = list(manifest_paths)
 501             manifest_paths.sort()
 502             relpaths = []
 503             for manifest_path in manifest_paths:
 504                 relpath = mozpath.relpath(manifest_path, topsrcdir)
 505                 if mozpath.commonprefix((manifest_path, topsrcdir)) != topsrcdir:
 506                     continue
 507                 relpaths.append(relpath)
 508             reader = self.build_obj.mozbuild_reader(config_mode="empty")
 509             files_info = reader.files_info(relpaths)
 510             for manifest_path in manifest_paths:
 511                 relpath = mozpath.relpath(manifest_path, topsrcdir)
 512                 if mozpath.commonprefix((manifest_path, topsrcdir)) != topsrcdir:
 513                     continue
 514                 manifest_info = None
 515                 if relpath in files_info:
 516                     bug_component = files_info[relpath].get("BUG_COMPONENT")
 517                     if bug_component:
 518                         key = "{}::{}".format(
 519                             bug_component.product, bug_component.component
 520                         )
 521                     else:
 522                         key = "<unknown bug component>"
 523                     if (not components) or (key in components):
 524                         manifest_info = {"manifest": relpath, "tests": 0, "skipped": 0}
 525                         rkey = key if show_components else "all"
 526                         if rkey in by_component["manifests"]:
 527                             by_component["manifests"][rkey].append(manifest_info)
 528                         else:
 529                             by_component["manifests"][rkey] = [manifest_info]
 530                 if manifest_info:
 531                     for t in tests:
 532                         if t["manifest"] == manifest_path:
 533                             manifest_info["tests"] += 1
 534                             if t.get("skip-if"):
 535                                 manifest_info["skipped"] += 1
 536             for key in by_component["manifests"]:
 537                 by_component["manifests"][key].sort(key=lambda k: k["manifest"])
 538
 539         if show_tests:
 540             by_component["tests"] = {}
 541
 542         if show_tests or show_summary or show_annotations:
 543             test_count = 0
 544             failed_count = 0
 545             skipped_count = 0
 546             annotation_count = 0
 547             condition_count = 0
 548             component_set = set()
 549             relpaths = []
 550             conditions = {}
 551             known_unconditional_annotations = ["skip", "fail", "asserts", "random"]
 552             known_conditional_annotations = [
 553                 "skip-if",
 554                 "fail-if",
 555                 "run-if",
 556                 "fails-if",
 557                 "fuzzy-if",
 558                 "random-if",
 559                 "asserts-if",
 560             ]
 561             for t in tests:
 562                 relpath = t.get("srcdir_relpath")
 563                 relpaths.append(relpath)
 564             reader = self.build_obj.mozbuild_reader(config_mode="empty")
 565             files_info = reader.files_info(relpaths)
 566             for t in tests:
 567                 if not matches_filters(t):
 568                     continue
 569                 if "referenced-test" in t:
 570                     # Avoid double-counting reftests: disregard reference file entries
 571                     continue
 572                 if show_annotations:
 573                     for key in t:
 574                         if key in known_unconditional_annotations:
 575                             annotation_count += 1
 576                         if key in known_conditional_annotations:
 577                             annotation_count += 1
 578                             # Here 'key' is a manifest annotation type like 'skip-if' and t[key]
 579                             # is the associated condition. For example, the manifestparser
 580                             # manifest annotation, "skip-if = os == 'win'", is expected to be
 581                             # encoded as t['skip-if'] = "os == 'win'".
 582                             # To allow for reftest manifests, t[key] may have multiple entries
 583                             # separated by ';', each corresponding to a condition for that test
 584                             # and annotation type. For example,
 585                             # "skip-if(Android&&webrender) skip-if(OSX)", would be
 586                             # encoded as t['skip-if'] = "Android&&webrender;OSX".
 587                             annotation_conditions = t[key].split(";")
 588
 589                             # if key has \n in it, we need to strip it. for manifestparser format
 590                             #  1) from the beginning of the line
 591                             #  2) different conditions if in the middle of the line
 592                             annotation_conditions = [
 593                                 x.strip("\n") for x in annotation_conditions
 594                             ]
 595                             temp = []
 596                             for condition in annotation_conditions:
 597                                 temp.extend(condition.split("\n"))
 598                             annotation_conditions = temp
 599
 600                             for condition in annotation_conditions:
 601                                 condition_count += 1
 602                                 # Trim reftest fuzzy-if ranges: everything after the first comma
 603                                 # eg. "Android,0-2,1-3" -> "Android"
 604                                 condition = condition.split(",")[0]
 605                                 if condition not in conditions:
 606                                     conditions[condition] = 0
 607                                 conditions[condition] += 1
 608                 test_count += 1
 609                 relpath = t.get("srcdir_relpath")
 610                 if relpath in files_info:
 611                     bug_component = files_info[relpath].get("BUG_COMPONENT")
 612                     if bug_component:
 613                         key = "{}::{}".format(
 614                             bug_component.product, bug_component.component
 615                         )
 616                     else:
 617                         key = "<unknown bug component>"
 618                     if (not components) or (key in components):
 619                         component_set.add(key)
 620                         test_info = {"test": relpath}
 621                         for test_key in display_keys:
 622                             value = t.get(test_key)
 623                             if value:
 624                                 test_info[test_key] = value
 625                         if t.get("fail-if"):
 626                             failed_count += 1
 627                         if t.get("fails-if"):
 628                             failed_count += 1
 629                         if t.get("skip-if"):
 630                             skipped_count += 1
 631
 632                         # add in intermittent failure data
 633                         if ifd.get(relpath):
 634                             if_data = ifd.get(relpath)
 635                             test_info["failure_count"] = if_data["count"]
 636
 637                         if "manifest_relpath" in t and "manifest" in t:
 638                             if "web-platform" in t["manifest_relpath"]:
 639                                 test_info["manifest"] = [t["manifest"]]
 640                             else:
 641                                 test_info["manifest"] = [t["manifest_relpath"]]
 642
 643                             # handle included manifests as ancestor:child
 644                             if t.get("ancestor_manifest", None):
 645                                 test_info["manifest"] = [
 646                                     "%s:%s"
 647                                     % (t["ancestor_manifest"], test_info["manifest"][0])
 648                                 ]
 649
 650                         if show_tests:
 651                             rkey = key if show_components else "all"
 652                             if rkey in by_component["tests"]:
 653                                 # Avoid duplicates: Some test paths have multiple TestResolver
 654                                 # entries, as when a test is included by multiple manifests.
 655                                 found = False
 656                                 for ctest in by_component["tests"][rkey]:
 657                                     if ctest["test"] == test_info["test"]:
 658                                         found = True
 659                                         break
 660                                 if not found:
 661                                     by_component["tests"][rkey].append(test_info)
 662                                 else:
 663                                     for ti in by_component["tests"][rkey]:
 664                                         if ti["test"] == test_info["test"]:
 665                                             if (
 666                                                 test_info["manifest"][0]
 667                                                 not in ti["manifest"]
 668                                             ):
 669                                                 ti_manifest = test_info["manifest"]
 670                                                 if test_info.get(
 671                                                     "ancestor_manifest", None
 672                                                 ):
 673                                                     ti_manifest = "%s:%s" % (
 674                                                         test_info["ancestor_manifest"],
 675                                                         ti_manifest,
 676                                                     )
 677                                                 ti["manifest"].extend(ti_manifest)
 678                             else:
 679                                 by_component["tests"][rkey] = [test_info]
 680             if show_tests:
 681                 for key in by_component["tests"]:
 682                     by_component["tests"][key].sort(key=lambda k: k["test"])
 683
 684         by_component["description"] = self.description(
 685             components,
 686             flavor,
 687             subsuite,
 688             paths,
 689             show_manifests,
 690             show_tests,
 691             show_summary,
 692             show_annotations,
 693             filter_values,
 694             filter_keys,
 695             start,
 696             end,
 697         )
 698
 699         if show_summary:
 700             by_component["summary"] = {}
 701             by_component["summary"]["components"] = len(component_set)
 702             by_component["summary"]["manifests"] = manifest_count
 703             by_component["summary"]["tests"] = test_count
 704             by_component["summary"]["failed tests"] = failed_count
 705             by_component["summary"]["skipped tests"] = skipped_count
 706
 707         if show_annotations:
 708             by_component["annotations"] = {}
 709             by_component["annotations"]["total annotations"] = annotation_count
 710             by_component["annotations"]["total conditions"] = condition_count
 711             by_component["annotations"]["unique conditions"] = len(conditions)
 712             by_component["annotations"]["conditions"] = conditions
 713
 714         self.write_report(by_component, output_file)
 715
 716         end_time = datetime.datetime.now()
 717         self.log_verbose(
 718             "%d seconds total to generate report"
 719             % (end_time - start_time).total_seconds()
 720         )
 721
 722     def write_report(self, by_component, output_file):
 723         json_report = json.dumps(by_component, indent=2, sort_keys=True)
 724         if output_file:
 725             output_file = os.path.abspath(output_file)
 726             output_dir = os.path.dirname(output_file)
 727             if not os.path.isdir(output_dir):
 728                 os.makedirs(output_dir)
 729
 730             with open(output_file, "w") as f:
 731                 f.write(json_report)
 732         else:
 733             print(json_report)
 734
 735     def report_diff(self, before, after, output_file):
 736         """
 737         Support for 'mach test-info report-diff'.
 738         """
 739
 740         def get_file(path_or_url):
 741             if urlparse.urlparse(path_or_url).scheme:
 742                 response = requests.get(path_or_url)
 743                 response.raise_for_status()
 744                 return json.loads(response.text)
 745             with open(path_or_url) as f:
 746                 return json.load(f)
 747
 748         report1 = get_file(before)
 749         report2 = get_file(after)
 750
 751         by_component = {"tests": {}, "summary": {}}
 752         self.diff_summaries(by_component, report1["summary"], report2["summary"])
 753         self.diff_all_components(by_component, report1["tests"], report2["tests"])
 754         self.write_report(by_component, output_file)
 755
 756     def diff_summaries(self, by_component, summary1, summary2):
 757         """
 758         Update by_component with comparison of summaries.
 759         """
 760         all_keys = set(summary1.keys()) | set(summary2.keys())
 761         for key in all_keys:
 762             delta = summary2.get(key, 0) - summary1.get(key, 0)
 763             by_component["summary"]["%s delta" % key] = delta
 764
 765     def diff_all_components(self, by_component, tests1, tests2):
 766         """
 767         Update by_component with any added/deleted tests, for all components.
 768         """
 769         self.added_count = 0
 770         self.deleted_count = 0
 771         for component in tests1:
 772             component1 = tests1[component]
 773             component2 = [] if component not in tests2 else tests2[component]
 774             self.diff_component(by_component, component, component1, component2)
 775         for component in tests2:
 776             if component not in tests1:
 777                 component2 = tests2[component]
 778                 self.diff_component(by_component, component, [], component2)
 779         by_component["summary"]["added tests"] = self.added_count
 780         by_component["summary"]["deleted tests"] = self.deleted_count
 781
 782     def diff_component(self, by_component, component, component1, component2):
 783         """
 784         Update by_component[component] with any added/deleted tests for the
 785         named component.
 786         "added": tests found in component2 but missing from component1.
 787         "deleted": tests found in component1 but missing from component2.
 788         """
 789         tests1 = set([t["test"] for t in component1])
 790         tests2 = set([t["test"] for t in component2])
 791         deleted = tests1 - tests2
 792         added = tests2 - tests1
 793         if deleted or added:
 794             by_component["tests"][component] = {}
 795             if deleted:
 796                 by_component["tests"][component]["deleted"] = sorted(list(deleted))
 797             if added:
 798                 by_component["tests"][component]["added"] = sorted(list(added))
 799         self.added_count += len(added)
 800         self.deleted_count += len(deleted)
 801         common = len(tests1.intersection(tests2))
 802         self.log_verbose(
 803             "%s: %d deleted, %d added, %d common"
 804             % (component, len(deleted), len(added), common)
 805         )