testing/skipfails.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import gzip
   6 import io
   7 import json
   8 import logging
   9 import os
  10 import os.path
  11 import pprint
  12 import re
  13 import sys
  14 import tempfile
  15 import urllib.parse
  16 from copy import deepcopy
  17 from pathlib import Path
  18 from statistics import median
  19 from xmlrpc.client import Fault
  20
  21 from yaml import load
  22
  23 try:
  24     from yaml import CLoader as Loader
  25 except ImportError:
  26     from yaml import Loader
  27
  28 import bugzilla
  29 import mozci.push
  30 import requests
  31 from manifestparser import ManifestParser
  32 from manifestparser.toml import add_skip_if, alphabetize_toml_str, sort_paths
  33 from mozci.task import TestTask
  34 from mozci.util.taskcluster import get_task
  35
  36 from taskcluster.exceptions import TaskclusterRestFailure
  37
  38 TASK_LOG = "live_backing.log"
  39 TASK_ARTIFACT = "public/logs/" + TASK_LOG
  40 ATTACHMENT_DESCRIPTION = "Compressed " + TASK_ARTIFACT + " for task "
  41 ATTACHMENT_REGEX = (
  42     r".*Created attachment ([0-9]+)\n.*"
  43     + ATTACHMENT_DESCRIPTION
  44     + "([A-Za-z0-9_-]+)\n.*"
  45 )
  46
  47 BUGZILLA_AUTHENTICATION_HELP = "Must create a Bugzilla API key per https://github.com/mozilla/mozci-tools/blob/main/citools/test_triage_bug_filer.py"
  48
  49 MS_PER_MINUTE = 60 * 1000  # ms per minute
  50 DEBUG_THRESHOLD = 40 * MS_PER_MINUTE  # 40 minutes in ms
  51 OPT_THRESHOLD = 20 * MS_PER_MINUTE  # 20 minutes in ms
  52
  53 ANYJS = "anyjs"
  54 CC = "classification"
  55 DEF = "DEFAULT"
  56 DIFFERENCE = "difference"
  57 DURATIONS = "durations"
  58 EQEQ = "=="
  59 ERROR = "error"
  60 FAIL = "FAIL"
  61 FAILED_RUNS = "runs_failed"
  62 FAILURE_RATIO = 0.4  # more than this fraction of failures will disable
  63 INTERMITTENT_RATIO_REFTEST = 0.4  # reftest low frequency intermittent
  64 FAILURE_RATIO_REFTEST = 0.8  # disable ratio for reftest (high freq intermittent)
  65 GROUP = "group"
  66 KIND = "kind"
  67 LINENO = "lineno"
  68 LL = "label"
  69 MEDIAN_DURATION = "duration_median"
  70 MINIMUM_RUNS = 3  # mininum number of runs to consider success/failure
  71 MOCK_BUG_DEFAULTS = {"blocks": [], "comments": []}
  72 MOCK_TASK_DEFAULTS = {"extra": {}, "failure_types": {}, "results": []}
  73 MOCK_TASK_INITS = ["results"]
  74 MODIFIERS = "modifiers"
  75 NOTEQ = "!="
  76 OPT = "opt"
  77 PASS = "PASS"
  78 PIXELS = "pixels"
  79 PP = "path"
  80 QUERY = "query"
  81 RR = "result"
  82 RUNS = "runs"
  83 STATUS = "status"
  84 SUBTEST = "subtest"
  85 SUBTEST_REGEX = (
  86     r"image comparison, max difference: ([0-9]+), number of differing pixels: ([0-9]+)"
  87 )
  88 SUM_BY_LABEL = "sum_by_label"
  89 TEST = "test"
  90 TEST_TYPES = [EQEQ, NOTEQ]
  91 TOTAL_DURATION = "duration_total"
  92 TOTAL_RUNS = "runs_total"
  93 WP = "testing/web-platform/"
  94 WPT0 = WP + "tests/infrastructure"
  95 WPT_META0 = WP + "tests/infrastructure/metadata"
  96 WPT_META0_CLASSIC = WP + "meta/infrastructure"
  97 WPT1 = WP + "tests"
  98 WPT_META1 = WPT1.replace("tests", "meta")
  99 WPT2 = WP + "mozilla/tests"
 100 WPT_META2 = WPT2.replace("tests", "meta")
 101 WPT_MOZILLA = "/_mozilla"
 102
 103
 104 class Mock(object):
 105     def __init__(self, data, defaults={}, inits=[]):
 106         self._data = data
 107         self._defaults = defaults
 108         for name in inits:
 109             values = self._data.get(name, [])  # assume type is an array
 110             values = [Mock(value, defaults, inits) for value in values]
 111             self._data[name] = values
 112
 113     def __getattr__(self, name):
 114         if name in self._data:
 115             return self._data[name]
 116         if name in self._defaults:
 117             return self._defaults[name]
 118         return ""
 119
 120
 121 class Classification(object):
 122     "Classification of the failure (not the task result)"
 123
 124     DISABLE_INTERMITTENT = "disable_intermittent"  # reftest [40%, 80%)
 125     DISABLE_FAILURE = "disable_failure"  # reftest (80%,100%] failure
 126     DISABLE_MANIFEST = "disable_manifest"  # crash found
 127     DISABLE_RECOMMENDED = "disable_recommended"  # disable first failing path
 128     DISABLE_TOO_LONG = "disable_too_long"  # runtime threshold exceeded
 129     INTERMITTENT = "intermittent"
 130     SECONDARY = "secondary"  # secondary failing path
 131     SUCCESS = "success"  # path always succeeds
 132     UNKNOWN = "unknown"
 133
 134
 135 class Kind(object):
 136     "Kind of manifest"
 137
 138     LIST = "list"
 139     TOML = "toml"
 140     UNKNOWN = "unknown"
 141     WPT = "wpt"
 142
 143
 144 class Skipfails(object):
 145     "mach manifest skip-fails implementation: Update manifests to skip failing tests"
 146
 147     REPO = "repo"
 148     REVISION = "revision"
 149     TREEHERDER = "treeherder.mozilla.org"
 150     BUGZILLA_SERVER_DEFAULT = "bugzilla.allizom.org"
 151
 152     def __init__(
 153         self,
 154         command_context=None,
 155         try_url="",
 156         verbose=True,
 157         bugzilla=None,
 158         dry_run=False,
 159         turbo=False,
 160         implicit_vars=False,
 161     ):
 162         self.command_context = command_context
 163         if self.command_context is not None:
 164             self.topsrcdir = self.command_context.topsrcdir
 165         else:
 166             self.topsrcdir = Path(__file__).parent.parent
 167         self.topsrcdir = os.path.normpath(self.topsrcdir)
 168         if isinstance(try_url, list) and len(try_url) == 1:
 169             self.try_url = try_url[0]
 170         else:
 171             self.try_url = try_url
 172         self.dry_run = dry_run
 173         self.implicit_vars = implicit_vars
 174         self.verbose = verbose
 175         self.turbo = turbo
 176         if bugzilla is not None:
 177             self.bugzilla = bugzilla
 178         elif "BUGZILLA" in os.environ:
 179             self.bugzilla = os.environ["BUGZILLA"]
 180         else:
 181             self.bugzilla = Skipfails.BUGZILLA_SERVER_DEFAULT
 182         if self.bugzilla == "disable":
 183             self.bugzilla = None  # Bug filing disabled
 184         self.component = "skip-fails"
 185         self._bzapi = None
 186         self._attach_rx = None
 187         self.variants = {}
 188         self.tasks = {}
 189         self.pp = None
 190         self.headers = {}  # for Treeherder requests
 191         self.headers["Accept"] = "application/json"
 192         self.headers["User-Agent"] = "treeherder-pyclient"
 193         self.jobs_url = "https://treeherder.mozilla.org/api/jobs/"
 194         self.push_ids = {}
 195         self.job_ids = {}
 196         self.extras = {}
 197         self.bugs = []  # preloaded bugs, currently not an updated cache
 198         self.error_summary = {}
 199         self._subtest_rx = None
 200         self.lmp = None
 201         self.failure_types = None
 202
 203     def _initialize_bzapi(self):
 204         """Lazily initializes the Bugzilla API (returns True on success)"""
 205         if self._bzapi is None and self.bugzilla is not None:
 206             self._bzapi = bugzilla.Bugzilla(self.bugzilla)
 207             self._attach_rx = re.compile(ATTACHMENT_REGEX, flags=re.M)
 208         return self._bzapi is not None
 209
 210     def pprint(self, obj):
 211         if self.pp is None:
 212             self.pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)
 213         self.pp.pprint(obj)
 214         sys.stderr.flush()
 215
 216     def error(self, e):
 217         if self.command_context is not None:
 218             self.command_context.log(
 219                 logging.ERROR, self.component, {ERROR: str(e)}, "ERROR: {error}"
 220             )
 221         else:
 222             print(f"ERROR: {e}", file=sys.stderr, flush=True)
 223
 224     def warning(self, e):
 225         if self.command_context is not None:
 226             self.command_context.log(
 227                 logging.WARNING, self.component, {ERROR: str(e)}, "WARNING: {error}"
 228             )
 229         else:
 230             print(f"WARNING: {e}", file=sys.stderr, flush=True)
 231
 232     def info(self, e):
 233         if self.command_context is not None:
 234             self.command_context.log(
 235                 logging.INFO, self.component, {ERROR: str(e)}, "INFO: {error}"
 236             )
 237         else:
 238             print(f"INFO: {e}", file=sys.stderr, flush=True)
 239
 240     def vinfo(self, e):
 241         if self.verbose:
 242             self.info(e)
 243
 244     def full_path(self, filename):
 245         """Returns full path for the relative filename"""
 246
 247         return os.path.join(self.topsrcdir, os.path.normpath(filename))
 248
 249     def isdir(self, filename):
 250         """Returns True if filename is a directory"""
 251
 252         return os.path.isdir(self.full_path(filename))
 253
 254     def exists(self, filename):
 255         """Returns True if filename exists"""
 256
 257         return os.path.exists(self.full_path(filename))
 258
 259     def run(
 260         self,
 261         meta_bug_id=None,
 262         save_tasks=None,
 263         use_tasks=None,
 264         save_failures=None,
 265         use_failures=None,
 266         max_failures=-1,
 267     ):
 268         "Run skip-fails on try_url, return True on success"
 269
 270         try_url = self.try_url
 271         revision, repo = self.get_revision(try_url)
 272         if use_tasks is not None:
 273             tasks = self.read_tasks(use_tasks)
 274             self.vinfo(f"use tasks: {use_tasks}")
 275             self.failure_types = None  # do NOT cache failure_types
 276         else:
 277             tasks = self.get_tasks(revision, repo)
 278             self.failure_types = {}  # cache failure_types
 279         if use_failures is not None:
 280             failures = self.read_failures(use_failures)
 281             self.vinfo(f"use failures: {use_failures}")
 282         else:
 283             failures = self.get_failures(tasks)
 284             if save_failures is not None:
 285                 self.write_json(save_failures, failures)
 286                 self.vinfo(f"save failures: {save_failures}")
 287         if save_tasks is not None:
 288             self.write_tasks(save_tasks, tasks)
 289             self.vinfo(f"save tasks: {save_tasks}")
 290         num_failures = 0
 291         self.vinfo(
 292             f"skip-fails assumes implicit-vars for reftest: {self.implicit_vars}"
 293         )
 294         for manifest in failures:
 295             kind = failures[manifest][KIND]
 296             for label in failures[manifest][LL]:
 297                 for path in failures[manifest][LL][label][PP]:
 298                     classification = failures[manifest][LL][label][PP][path][CC]
 299                     if classification.startswith("disable_") or (
 300                         self.turbo and classification == Classification.SECONDARY
 301                     ):
 302                         anyjs = {}  # anyjs alternate basename = False
 303                         differences = []
 304                         pixels = []
 305                         status = FAIL
 306                         lineno = failures[manifest][LL][label][PP][path].get(LINENO, 0)
 307                         for task_id in failures[manifest][LL][label][PP][path][RUNS]:
 308                             if kind == Kind.TOML:
 309                                 break  # just use the first task_id
 310                             elif kind == Kind.LIST:
 311                                 difference = failures[manifest][LL][label][PP][path][
 312                                     RUNS
 313                                 ][task_id].get(DIFFERENCE, 0)
 314                                 if difference > 0:
 315                                     differences.append(difference)
 316                                 pixel = failures[manifest][LL][label][PP][path][RUNS][
 317                                     task_id
 318                                 ].get(PIXELS, 0)
 319                                 if pixel > 0:
 320                                     pixels.append(pixel)
 321                                 status = failures[manifest][LL][label][PP][path][RUNS][
 322                                     task_id
 323                                 ].get(STATUS, FAIL)
 324                             elif kind == Kind.WPT:
 325                                 filename = os.path.basename(path)
 326                                 anyjs[filename] = False
 327                                 if (
 328                                     QUERY
 329                                     in failures[manifest][LL][label][PP][path][RUNS][
 330                                         task_id
 331                                     ]
 332                                 ):
 333                                     query = failures[manifest][LL][label][PP][path][
 334                                         RUNS
 335                                     ][task_id][QUERY]
 336                                     anyjs[filename + query] = False
 337                                 else:
 338                                     query = None
 339                                 if (
 340                                     ANYJS
 341                                     in failures[manifest][LL][label][PP][path][RUNS][
 342                                         task_id
 343                                     ]
 344                                 ):
 345                                     any_filename = os.path.basename(
 346                                         failures[manifest][LL][label][PP][path][RUNS][
 347                                             task_id
 348                                         ][ANYJS]
 349                                     )
 350                                     anyjs[any_filename] = False
 351                                     if query is not None:
 352                                         anyjs[any_filename + query] = False
 353                         self.skip_failure(
 354                             manifest,
 355                             kind,
 356                             path,
 357                             anyjs,
 358                             differences,
 359                             pixels,
 360                             lineno,
 361                             status,
 362                             label,
 363                             classification,
 364                             task_id,
 365                             try_url,
 366                             revision,
 367                             repo,
 368                             meta_bug_id,
 369                         )
 370                         num_failures += 1
 371                         if max_failures >= 0 and num_failures >= max_failures:
 372                             self.warning(
 373                                 f"max_failures={max_failures} threshold reached: stopping."
 374                             )
 375                             return True
 376         return True
 377
 378     def get_revision(self, url):
 379         parsed = urllib.parse.urlparse(url)
 380         if parsed.scheme != "https":
 381             raise ValueError("try_url scheme not https")
 382         if parsed.netloc != Skipfails.TREEHERDER:
 383             raise ValueError(f"try_url server not {Skipfails.TREEHERDER}")
 384         if len(parsed.query) == 0:
 385             raise ValueError("try_url query missing")
 386         query = urllib.parse.parse_qs(parsed.query)
 387         if Skipfails.REVISION not in query:
 388             raise ValueError("try_url query missing revision")
 389         revision = query[Skipfails.REVISION][0]
 390         if Skipfails.REPO in query:
 391             repo = query[Skipfails.REPO][0]
 392         else:
 393             repo = "try"
 394         self.vinfo(f"considering {repo} revision={revision}")
 395         return revision, repo
 396
 397     def get_tasks(self, revision, repo):
 398         push = mozci.push.Push(revision, repo)
 399         return push.tasks
 400
 401     def get_failures(self, tasks):
 402         """
 403         find failures and create structure comprised of runs by path:
 404            result:
 405             * False (failed)
 406             * True (passed)
 407            classification: Classification
 408             * unknown (default) < 3 runs
 409             * intermittent (not enough failures)
 410             * disable_recommended (enough repeated failures) >3 runs >= 4
 411             * disable_manifest (disable DEFAULT if no other failures)
 412             * secondary (not first failure in group)
 413             * success
 414         """
 415
 416         ff = {}
 417         manifest_paths = {}
 418         manifest_ = {
 419             KIND: Kind.UNKNOWN,
 420             LL: {},
 421         }
 422         label_ = {
 423             DURATIONS: {},
 424             MEDIAN_DURATION: 0,
 425             OPT: False,
 426             PP: {},
 427             SUM_BY_LABEL: {},  # All sums implicitly zero
 428             TOTAL_DURATION: 0,
 429         }
 430         path_ = {
 431             CC: Classification.UNKNOWN,
 432             FAILED_RUNS: 0,
 433             RUNS: {},
 434             TOTAL_RUNS: 0,
 435         }
 436         run_ = {
 437             RR: False,
 438         }
 439
 440         for task in tasks:  # add explicit failures
 441             try:
 442                 if len(task.results) == 0:
 443                     continue  # ignore aborted tasks
 444                 failure_types = task.failure_types  # call magic property once
 445                 if self.failure_types is not None:
 446                     self.failure_types[task.id] = failure_types
 447                 self.vinfo(f"Getting failure_types from task: {task.id}")
 448                 for manifest in failure_types:
 449                     mm = manifest
 450                     ll = task.label
 451                     kind = Kind.UNKNOWN
 452                     if mm.endswith(".ini"):
 453                         self.warning(
 454                             f"cannot analyze skip-fails on INI manifests: {mm}"
 455                         )
 456                         continue
 457                     elif mm.endswith(".list"):
 458                         kind = Kind.LIST
 459                     elif mm.endswith(".toml"):
 460                         kind = Kind.TOML
 461                     else:
 462                         kind = Kind.WPT
 463                         path, mm, _query, _anyjs = self.wpt_paths(mm)
 464                         if path is None:  # not WPT
 465                             self.warning(
 466                                 f"cannot analyze skip-fails on unknown manifest type: {manifest}"
 467                             )
 468                             continue
 469                     if kind != Kind.WPT:
 470                         if mm not in manifest_paths:
 471                             manifest_paths[mm] = []
 472                         if mm not in ff:
 473                             ff[mm] = deepcopy(manifest_)
 474                             ff[mm][KIND] = kind
 475                         if ll not in ff[mm][LL]:
 476                             ff[mm][LL][ll] = deepcopy(label_)
 477                     for path_type in failure_types[manifest]:
 478                         path, _type = path_type
 479                         query = None
 480                         anyjs = None
 481                         allpaths = []
 482                         if kind == Kind.WPT:
 483                             path, mmpath, query, anyjs = self.wpt_paths(path)
 484                             if path is None:
 485                                 self.warning(
 486                                     f"non existant failure path: {path_type[0]}"
 487                                 )
 488                                 break
 489                             allpaths = [path]
 490                             mm = os.path.dirname(mmpath)
 491                             if mm not in manifest_paths:
 492                                 manifest_paths[mm] = []
 493                             if mm not in ff:
 494                                 ff[mm] = deepcopy(manifest_)
 495                                 ff[mm][KIND] = kind
 496                             if ll not in ff[mm][LL]:
 497                                 ff[mm][LL][ll] = deepcopy(label_)
 498                         elif kind == Kind.LIST:
 499                             words = path.split()
 500                             if len(words) != 3 or words[1] not in TEST_TYPES:
 501                                 self.warning(f"reftest type not supported: {path}")
 502                                 continue
 503                             allpaths = self.get_allpaths(task.id, mm, path)
 504                         elif kind == Kind.TOML:
 505                             if path == mm:
 506                                 path = DEF  # refers to the manifest itself
 507                             allpaths = [path]
 508                         for path in allpaths:
 509                             if path in manifest_paths[mm]:
 510                                 continue  # duplicate path for this task
 511                             manifest_paths[mm].append(path)
 512                             self.vinfo(
 513                                 f"Getting failure info in manifest: {mm}, path: {path}"
 514                             )
 515                             if path not in ff[mm][LL][ll][PP]:
 516                                 ff[mm][LL][ll][PP][path] = deepcopy(path_)
 517                             if task.id not in ff[mm][LL][ll][PP][path][RUNS]:
 518                                 ff[mm][LL][ll][PP][path][RUNS][task.id] = deepcopy(run_)
 519                             ff[mm][LL][ll][PP][path][RUNS][task.id][RR] = False
 520                             if query is not None:
 521                                 ff[mm][LL][ll][PP][path][RUNS][task.id][QUERY] = query
 522                             if anyjs is not None:
 523                                 ff[mm][LL][ll][PP][path][RUNS][task.id][ANYJS] = anyjs
 524                             ff[mm][LL][ll][PP][path][TOTAL_RUNS] += 1
 525                             ff[mm][LL][ll][PP][path][FAILED_RUNS] += 1
 526                             if kind == Kind.LIST:
 527                                 (
 528                                     lineno,
 529                                     difference,
 530                                     pixels,
 531                                     status,
 532                                 ) = self.get_lineno_difference_pixels_status(
 533                                     task.id, mm, path
 534                                 )
 535                                 if lineno > 0:
 536                                     ff[mm][LL][ll][PP][path][LINENO] = lineno
 537                                 else:
 538                                     self.vinfo(f"ERROR no lineno for {path}")
 539                                 if status != FAIL:
 540                                     ff[mm][LL][ll][PP][path][RUNS][task.id][
 541                                         STATUS
 542                                     ] = status
 543                                 if status == FAIL and difference == 0 and pixels == 0:
 544                                     # intermittent, not error
 545                                     ff[mm][LL][ll][PP][path][RUNS][task.id][RR] = True
 546                                     ff[mm][LL][ll][PP][path][FAILED_RUNS] -= 1
 547                                 elif difference > 0:
 548                                     ff[mm][LL][ll][PP][path][RUNS][task.id][
 549                                         DIFFERENCE
 550                                     ] = difference
 551                                 if pixels > 0:
 552                                     ff[mm][LL][ll][PP][path][RUNS][task.id][
 553                                         PIXELS
 554                                     ] = pixels
 555             except AttributeError:
 556                 pass  # self.warning(f"unknown attribute in task (#1): {ae}")
 557
 558         for task in tasks:  # add results
 559             try:
 560                 if len(task.results) == 0:
 561                     continue  # ignore aborted tasks
 562                 self.vinfo(f"Getting results from task: {task.id}")
 563                 for result in task.results:
 564                     mm = result.group
 565                     ll = task.label
 566                     kind = Kind.UNKNOWN
 567                     if mm.endswith(".ini"):
 568                         self.warning(
 569                             f"cannot analyze skip-fails on INI manifests: {mm}"
 570                         )
 571                         continue
 572                     elif mm.endswith(".list"):
 573                         kind = Kind.LIST
 574                     elif mm.endswith(".toml"):
 575                         kind = Kind.TOML
 576                     else:
 577                         kind = Kind.WPT
 578                         path, mm, _query, _anyjs = self.wpt_paths(mm)
 579                         if path is None:  # not WPT
 580                             self.warning(
 581                                 f"cannot analyze skip-fails on unknown manifest type: {result.group}"
 582                             )
 583                             continue
 584                     if mm not in manifest_paths:
 585                         continue
 586                     if mm not in ff:
 587                         ff[mm] = deepcopy(manifest_)
 588                     if ll not in ff[mm][LL]:
 589                         ff[mm][LL][ll] = deepcopy(label_)
 590                     if task.id not in ff[mm][LL][ll][DURATIONS]:
 591                         # duration may be None !!!
 592                         ff[mm][LL][ll][DURATIONS][task.id] = result.duration or 0
 593                         if ff[mm][LL][ll][OPT] is None:
 594                             ff[mm][LL][ll][OPT] = self.get_opt_for_task(task.id)
 595                     for path in manifest_paths[mm]:  # all known paths
 596                         self.vinfo(f"Getting result for manifest: {mm}, path: {path}")
 597                         if path not in ff[mm][LL][ll][PP]:
 598                             ff[mm][LL][ll][PP][path] = deepcopy(path_)
 599                         if task.id not in ff[mm][LL][ll][PP][path][RUNS]:
 600                             ff[mm][LL][ll][PP][path][RUNS][task.id] = deepcopy(run_)
 601                             ff[mm][LL][ll][PP][path][RUNS][task.id][RR] = result.ok
 602                             ff[mm][LL][ll][PP][path][TOTAL_RUNS] += 1
 603                             if not result.ok:
 604                                 ff[mm][LL][ll][PP][path][FAILED_RUNS] += 1
 605                             if kind == Kind.LIST:
 606                                 (
 607                                     lineno,
 608                                     difference,
 609                                     pixels,
 610                                     status,
 611                                 ) = self.get_lineno_difference_pixels_status(
 612                                     task.id, mm, path
 613                                 )
 614                                 if lineno > 0:
 615                                     ff[mm][LL][ll][PP][path][LINENO] = lineno
 616                                 else:
 617                                     self.vinfo(f"ERROR no lineno for {path}")
 618                                 if status != FAIL:
 619                                     ff[mm][LL][ll][PP][path][RUNS][task.id][
 620                                         STATUS
 621                                     ] = status
 622                                 if (
 623                                     status == FAIL
 624                                     and difference == 0
 625                                     and pixels == 0
 626                                     and not result.ok
 627                                 ):
 628                                     # intermittent, not error
 629                                     ff[mm][LL][ll][PP][path][RUNS][task.id][RR] = True
 630                                     ff[mm][LL][ll][PP][path][FAILED_RUNS] -= 1
 631                                 if difference > 0:
 632                                     ff[mm][LL][ll][PP][path][RUNS][task.id][
 633                                         DIFFERENCE
 634                                     ] = difference
 635                                 if pixels > 0:
 636                                     ff[mm][LL][ll][PP][path][RUNS][task.id][
 637                                         PIXELS
 638                                     ] = pixels
 639             except AttributeError:
 640                 pass  # self.warning(f"unknown attribute in task (#2): {ae}")
 641
 642         for mm in ff:  # determine classifications
 643             kind = ff[mm][KIND]
 644             for label in ff[mm][LL]:
 645                 ll = label
 646                 opt = ff[mm][LL][ll][OPT]
 647                 durations = []  # summarize durations
 648                 for task_id in ff[mm][LL][ll][DURATIONS]:
 649                     duration = ff[mm][LL][ll][DURATIONS][task_id]
 650                     durations.append(duration)
 651                 if len(durations) > 0:
 652                     total_duration = sum(durations)
 653                     median_duration = median(durations)
 654                     ff[mm][LL][ll][TOTAL_DURATION] = total_duration
 655                     ff[mm][LL][ll][MEDIAN_DURATION] = median_duration
 656                     if (opt and median_duration > OPT_THRESHOLD) or (
 657                         (not opt) and median_duration > DEBUG_THRESHOLD
 658                     ):
 659                         if kind == Kind.TOML:
 660                             paths = [DEF]
 661                         else:
 662                             paths = ff[mm][LL][ll][PP].keys()
 663                         for path in paths:
 664                             if path not in ff[mm][LL][ll][PP]:
 665                                 ff[mm][LL][ll][PP][path] = deepcopy(path_)
 666                             if task_id not in ff[mm][LL][ll][PP][path][RUNS]:
 667                                 ff[mm][LL][ll][PP][path][RUNS][task.id] = deepcopy(run_)
 668                                 ff[mm][LL][ll][PP][path][RUNS][task.id][RR] = False
 669                                 ff[mm][LL][ll][PP][path][TOTAL_RUNS] += 1
 670                                 ff[mm][LL][ll][PP][path][FAILED_RUNS] += 1
 671                             ff[mm][LL][ll][PP][path][
 672                                 CC
 673                             ] = Classification.DISABLE_TOO_LONG
 674                 primary = True  # we have not seen the first failure
 675                 for path in sort_paths(ff[mm][LL][ll][PP]):
 676                     classification = ff[mm][LL][ll][PP][path][CC]
 677                     if classification == Classification.UNKNOWN:
 678                         failed_runs = ff[mm][LL][ll][PP][path][FAILED_RUNS]
 679                         total_runs = ff[mm][LL][ll][PP][path][TOTAL_RUNS]
 680                         status = FAIL  # default status, only one run could be PASS
 681                         for task_id in ff[mm][LL][ll][PP][path][RUNS]:
 682                             status = ff[mm][LL][ll][PP][path][RUNS][task_id].get(
 683                                 STATUS, status
 684                             )
 685                         if kind == Kind.LIST:
 686                             failure_ratio = INTERMITTENT_RATIO_REFTEST
 687                         else:
 688                             failure_ratio = FAILURE_RATIO
 689                         if total_runs >= MINIMUM_RUNS:
 690                             if failed_runs / total_runs < failure_ratio:
 691                                 if failed_runs == 0:
 692                                     classification = Classification.SUCCESS
 693                                 else:
 694                                     classification = Classification.INTERMITTENT
 695                             elif kind == Kind.LIST:
 696                                 if failed_runs / total_runs < FAILURE_RATIO_REFTEST:
 697                                     classification = Classification.DISABLE_INTERMITTENT
 698                                 else:
 699                                     classification = Classification.DISABLE_FAILURE
 700                             elif primary:
 701                                 if path == DEF:
 702                                     classification = Classification.DISABLE_MANIFEST
 703                                 else:
 704                                     classification = Classification.DISABLE_RECOMMENDED
 705                                 primary = False
 706                             else:
 707                                 classification = Classification.SECONDARY
 708                         ff[mm][LL][ll][PP][path][CC] = classification
 709                     if classification not in ff[mm][LL][ll][SUM_BY_LABEL]:
 710                         ff[mm][LL][ll][SUM_BY_LABEL][classification] = 0
 711                     ff[mm][LL][ll][SUM_BY_LABEL][classification] += 1
 712         return ff
 713
 714     def _get_os_version(self, os, platform):
 715         """Return the os_version given the label platform string"""
 716         i = platform.find(os)
 717         j = i + len(os)
 718         yy = platform[j : j + 2]
 719         mm = platform[j + 2 : j + 4]
 720         return yy + "." + mm
 721
 722     def get_bug_by_id(self, id):
 723         """Get bug by bug id"""
 724
 725         bug = None
 726         for b in self.bugs:
 727             if b.id == id:
 728                 bug = b
 729                 break
 730         if bug is None and self._initialize_bzapi():
 731             bug = self._bzapi.getbug(id)
 732         return bug
 733
 734     def get_bugs_by_summary(self, summary):
 735         """Get bug by bug summary"""
 736
 737         bugs = []
 738         for b in self.bugs:
 739             if b.summary == summary:
 740                 bugs.append(b)
 741         if len(bugs) > 0:
 742             return bugs
 743         if self._initialize_bzapi():
 744             query = self._bzapi.build_query(short_desc=summary)
 745             query["include_fields"] = [
 746                 "id",
 747                 "product",
 748                 "component",
 749                 "status",
 750                 "resolution",
 751                 "summary",
 752                 "blocks",
 753             ]
 754             bugs = self._bzapi.query(query)
 755         return bugs
 756
 757     def create_bug(
 758         self,
 759         summary="Bug short description",
 760         description="Bug description",
 761         product="Testing",
 762         component="General",
 763         version="unspecified",
 764         bugtype="task",
 765     ):
 766         """Create a bug"""
 767
 768         bug = None
 769         if self._initialize_bzapi():
 770             if not self._bzapi.logged_in:
 771                 self.error(
 772                     "Must create a Bugzilla API key per https://github.com/mozilla/mozci-tools/blob/main/citools/test_triage_bug_filer.py"
 773                 )
 774                 raise PermissionError(f"Not authenticated for Bugzilla {self.bugzilla}")
 775             createinfo = self._bzapi.build_createbug(
 776                 product=product,
 777                 component=component,
 778                 summary=summary,
 779                 version=version,
 780                 description=description,
 781             )
 782             createinfo["type"] = bugtype
 783             bug = self._bzapi.createbug(createinfo)
 784         return bug
 785
 786     def add_bug_comment(self, id, comment, meta_bug_id=None):
 787         """Add a comment to an existing bug"""
 788
 789         if self._initialize_bzapi():
 790             if not self._bzapi.logged_in:
 791                 self.error(BUGZILLA_AUTHENTICATION_HELP)
 792                 raise PermissionError("Not authenticated for Bugzilla")
 793             if meta_bug_id is not None:
 794                 blocks_add = [meta_bug_id]
 795             else:
 796                 blocks_add = None
 797             updateinfo = self._bzapi.build_update(
 798                 comment=comment, blocks_add=blocks_add
 799             )
 800             self._bzapi.update_bugs([id], updateinfo)
 801
 802     def skip_failure(
 803         self,
 804         manifest,
 805         kind,
 806         path,
 807         anyjs,
 808         differences,
 809         pixels,
 810         lineno,
 811         status,
 812         label,
 813         classification,
 814         task_id,
 815         try_url,
 816         revision,
 817         repo,
 818         meta_bug_id=None,
 819     ):
 820         """
 821         Skip a failure (for TOML, WPT and REFTEST manifests)
 822         For wpt anyjs is a dictionary mapping from alternate basename to
 823         a boolean (indicating if the basename has been handled in the manifest)
 824         """
 825
 826         self.vinfo(f"\n\n===== Skip failure in manifest: {manifest} =====")
 827         self.vinfo(f"    path: {path}")
 828         if task_id is None:
 829             skip_if = "true"
 830         else:
 831             skip_if = self.task_to_skip_if(task_id, kind)
 832         if skip_if is None:
 833             self.warning(
 834                 f"Unable to calculate skip-if condition from manifest={manifest} from failure label={label}"
 835             )
 836             return
 837         if kind == Kind.TOML:
 838             filename = DEF
 839         elif kind == Kind.WPT:
 840             _path, manifest, _query, _anyjs = self.wpt_paths(path)
 841             filename = os.path.basename(path)
 842         elif kind == Kind.LIST:
 843             filename = path
 844             if status == PASS:
 845                 self.info(f"Unexpected status: {status}")
 846             if status == PASS or classification == Classification.DISABLE_INTERMITTENT:
 847                 zero = True  # refest lower ranges should include zero
 848             else:
 849                 zero = False
 850         bug_reference = ""
 851         if classification == Classification.DISABLE_MANIFEST:
 852             comment = "Disabled entire manifest due to crash result"
 853         elif classification == Classification.DISABLE_TOO_LONG:
 854             comment = "Disabled entire manifest due to excessive run time"
 855         else:
 856             if kind == Kind.TOML:
 857                 filename = self.get_filename_in_manifest(manifest, path)
 858             comment = f'Disabled test due to failures in test file: "{filename}"'
 859             if classification == Classification.SECONDARY:
 860                 comment += " (secondary)"
 861                 if kind != Kind.WPT:
 862                     bug_reference = " (secondary)"
 863         if kind != Kind.LIST:
 864             self.vinfo(f"filename: {filename}")
 865         if kind == Kind.WPT and len(anyjs) > 1:
 866             comment += "\nAdditional WPT wildcard paths:"
 867             for p in sorted(anyjs.keys()):
 868                 if p != filename:
 869                     comment += f'\n  "{p}"'
 870         platform, testname = self.label_to_platform_testname(label)
 871         if platform is not None:
 872             comment += "\nCommand line to reproduce (experimental):\n"
 873             comment += f"  \"mach try fuzzy -q '{platform}' {testname}\""
 874         comment += f"\nTry URL = {try_url}"
 875         comment += f"\nrevision = {revision}"
 876         comment += f"\nrepo = {repo}"
 877         comment += f"\nlabel = {label}"
 878         if task_id is not None:
 879             comment += f"\ntask_id = {task_id}"
 880             if kind != Kind.LIST:
 881                 push_id = self.get_push_id(revision, repo)
 882                 if push_id is not None:
 883                     comment += f"\npush_id = {push_id}"
 884                     job_id = self.get_job_id(push_id, task_id)
 885                     if job_id is not None:
 886                         comment += f"\njob_id = {job_id}"
 887                         (
 888                             suggestions_url,
 889                             line_number,
 890                             line,
 891                             log_url,
 892                         ) = self.get_bug_suggestions(repo, job_id, path, anyjs)
 893                         if log_url is not None:
 894                             comment += f"\nBug suggestions: {suggestions_url}"
 895                             comment += f"\nSpecifically see at line {line_number} in the attached log: {log_url}"
 896                             comment += f'\n\n  "{line}"\n'
 897         bug_summary = f"MANIFEST {manifest}"
 898         attachments = {}
 899         bugid = "TBD"
 900         if self.bugzilla is None:
 901             self.vinfo("Bugzilla has been disabled: no bugs created or updated")
 902         else:
 903             bugs = self.get_bugs_by_summary(bug_summary)
 904             if len(bugs) == 0:
 905                 description = (
 906                     f"This bug covers excluded failing tests in the MANIFEST {manifest}"
 907                 )
 908                 description += "\n(generated by `mach manifest skip-fails`)"
 909                 product, component = self.get_file_info(path)
 910                 if self.dry_run:
 911                     self.warning(
 912                         f'Dry-run NOT creating bug: {product}::{component} "{bug_summary}"'
 913                     )
 914                 else:
 915                     bug = self.create_bug(bug_summary, description, product, component)
 916                     bugid = bug.id
 917                     self.vinfo(
 918                         f'Created Bug {bugid} {product}::{component} : "{bug_summary}"'
 919                     )
 920             elif len(bugs) == 1:
 921                 bugid = bugs[0].id
 922                 product = bugs[0].product
 923                 component = bugs[0].component
 924                 self.vinfo(f'Found Bug {bugid} {product}::{component} "{bug_summary}"')
 925                 if meta_bug_id is not None:
 926                     if meta_bug_id in bugs[0].blocks:
 927                         self.vinfo(
 928                             f"  Bug {bugid} already blocks meta bug {meta_bug_id}"
 929                         )
 930                         meta_bug_id = None  # no need to add again
 931                 comments = bugs[0].getcomments()
 932                 for i in range(len(comments)):
 933                     text = comments[i]["text"]
 934                     m = self._attach_rx.findall(text)
 935                     if len(m) == 1:
 936                         a_task_id = m[0][1]
 937                         attachments[a_task_id] = m[0][0]
 938                         if a_task_id == task_id:
 939                             self.vinfo(
 940                                 f"  Bug {bugid} already has the compressed log attached for this task"
 941                             )
 942             else:
 943                 self.error(f'More than one bug found for summary: "{bug_summary}"')
 944                 return
 945         bug_reference = f"Bug {bugid}" + bug_reference
 946         extra = self.get_extra(task_id)
 947         json.dumps(extra)
 948         if kind == Kind.LIST:
 949             comment += (
 950                 f"\nfuzzy-if condition on line {lineno}: {skip_if} # {bug_reference}"
 951             )
 952         else:
 953             comment += f"\nskip-if condition: {skip_if} # {bug_reference}"
 954         manifest_path = self.full_path(manifest)
 955         manifest_str = ""
 956         additional_comment = ""
 957         if kind == Kind.WPT:
 958             if os.path.exists(manifest_path):
 959                 manifest_str = io.open(manifest_path, "r", encoding="utf-8").read()
 960             else:
 961                 # ensure parent directories exist
 962                 os.makedirs(os.path.dirname(manifest_path), exist_ok=True)
 963             manifest_str, additional_comment = self.wpt_add_skip_if(
 964                 manifest_str, anyjs, skip_if, bug_reference
 965             )
 966         elif kind == Kind.TOML:
 967             mp = ManifestParser(use_toml=True, document=True)
 968             mp.read(manifest_path)
 969             document = mp.source_documents[manifest_path]
 970             additional_comment = add_skip_if(document, filename, skip_if, bug_reference)
 971             manifest_str = alphabetize_toml_str(document)
 972         elif kind == Kind.LIST:
 973             if lineno == 0:
 974                 self.error(
 975                     f"cannot determine line to edit in manifest: {manifest_path}"
 976                 )
 977             # elif skip_if.find("winWidget") >= 0 and skip_if.find("!is64Bit") >= 0:
 978             #     self.error(
 979             #         "Skipping failures for Windows 32-bit are temporarily disabled"
 980             #     )
 981             elif not os.path.exists(manifest_path):
 982                 self.error(f"manifest does not exist: {manifest_path}")
 983             else:
 984                 manifest_str = io.open(manifest_path, "r", encoding="utf-8").read()
 985                 manifest_str, additional_comment = self.reftest_add_fuzzy_if(
 986                     manifest_str,
 987                     filename,
 988                     skip_if,
 989                     differences,
 990                     pixels,
 991                     lineno,
 992                     zero,
 993                     bug_reference,
 994                 )
 995                 if not manifest_str and additional_comment:
 996                     self.warning(additional_comment)
 997         if additional_comment:
 998             comment += "\n" + additional_comment
 999         if len(manifest_str) > 0:
1000             fp = io.open(manifest_path, "w", encoding="utf-8", newline="\n")
1001             fp.write(manifest_str)
1002             fp.close()
1003             self.info(f'Edited ["{filename}"] in manifest: "{manifest}"')
1004             if kind != Kind.LIST:
1005                 self.info(f'added skip-if condition: "{skip_if}" # {bug_reference}')
1006             if self.dry_run:
1007                 self.info(f"Dry-run NOT adding comment to Bug {bugid}:\n{comment}")
1008                 self.info(
1009                     f'Dry-run NOT editing ["{filename}"] in manifest: "{manifest}"'
1010                 )
1011                 self.info(f'would add skip-if condition: "{skip_if}" # {bug_reference}')
1012                 if task_id is not None and task_id not in attachments:
1013                     self.info("would add compressed log for this task")
1014                 return
1015             elif self.bugzilla is None:
1016                 self.warning(f"NOT adding comment to Bug {bugid}:\n{comment}")
1017             else:
1018                 self.add_bug_comment(bugid, comment, meta_bug_id)
1019                 self.info(f"Added comment to Bug {bugid}:\n{comment}")
1020                 if meta_bug_id is not None:
1021                     self.info(f"  Bug {bugid} blocks meta Bug: {meta_bug_id}")
1022                 if task_id is not None and task_id not in attachments:
1023                     self.add_attachment_log_for_task(bugid, task_id)
1024                     self.info("Added compressed log for this task")
1025         else:
1026             self.error(f'Error editing ["{filename}"] in manifest: "{manifest}"')
1027
1028     def get_variants(self):
1029         """Get mozinfo for each test variants"""
1030
1031         if len(self.variants) == 0:
1032             variants_file = "taskcluster/kinds/test/variants.yml"
1033             variants_path = self.full_path(variants_file)
1034             fp = io.open(variants_path, "r", encoding="utf-8")
1035             raw_variants = load(fp, Loader=Loader)
1036             fp.close()
1037             for k, v in raw_variants.items():
1038                 mozinfo = k
1039                 if "mozinfo" in v:
1040                     mozinfo = v["mozinfo"]
1041                 self.variants[k] = mozinfo
1042         return self.variants
1043
1044     def get_task_details(self, task_id):
1045         """Download details for task task_id"""
1046
1047         if task_id in self.tasks:  # if cached
1048             task = self.tasks[task_id]
1049         else:
1050             self.vinfo(f"get_task_details for task: {task_id}")
1051             try:
1052                 task = get_task(task_id)
1053             except TaskclusterRestFailure:
1054                 self.warning(f"Task {task_id} no longer exists.")
1055                 return None
1056             self.tasks[task_id] = task
1057         return task
1058
1059     def get_extra(self, task_id):
1060         """Calculate extra for task task_id"""
1061
1062         if task_id in self.extras:  # if cached
1063             extra = self.extras[task_id]
1064         else:
1065             self.get_variants()
1066             task = self.get_task_details(task_id) or {}
1067             arch = None
1068             bits = None
1069             build = None
1070             build_types = []
1071             display = None
1072             os = None
1073             os_version = None
1074             runtimes = []
1075             test_setting = task.get("extra", {}).get("test-setting", {})
1076             platform = test_setting.get("platform", {})
1077             platform_os = platform.get("os", {})
1078             opt = False
1079             debug = False
1080             if "name" in platform_os:
1081                 os = platform_os["name"]
1082                 if os == "windows":
1083                     os = "win"
1084                 if os == "macosx":
1085                     os = "mac"
1086             if "version" in platform_os:
1087                 os_version = platform_os["version"]
1088                 if len(os_version) == 4:
1089                     os_version = os_version[0:2] + "." + os_version[2:4]
1090             if "build" in platform_os:
1091                 build = platform_os["build"]
1092             if "arch" in platform:
1093                 arch = platform["arch"]
1094                 if arch == "x86" or arch.find("32") >= 0:
1095                     bits = "32"
1096                     arch = "x86"
1097                 else:
1098                     bits = "64"
1099                     if arch != "aarch64" and arch != "ppc":
1100                         arch = "x86_64"
1101             if "display" in platform:
1102                 display = platform["display"]
1103             if "runtime" in test_setting:
1104                 for k in test_setting["runtime"]:
1105                     if k == "no-fission" and test_setting["runtime"][k]:
1106                         runtimes.append("no-fission")
1107                     elif k in self.variants:  # draw-snapshot -> snapshot
1108                         runtimes.append(self.variants[k])  # adds mozinfo
1109             if "build" in test_setting:
1110                 tbuild = test_setting["build"]
1111                 for k in tbuild:
1112                     if k == "type":
1113                         if tbuild[k] == "opt":
1114                             opt = True
1115                         elif tbuild[k] == "debug":
1116                             debug = True
1117                         build_types.append(tbuild[k])
1118                     else:
1119                         build_types.append(k)
1120             unknown = None
1121             extra = {
1122                 "arch": arch or unknown,
1123                 "bits": bits or unknown,
1124                 "build": build or unknown,
1125                 "build_types": build_types,
1126                 "debug": debug,
1127                 "display": display or unknown,
1128                 "opt": opt,
1129                 "os": os or unknown,
1130                 "os_version": os_version or unknown,
1131                 "runtimes": runtimes,
1132             }
1133         self.extras[task_id] = extra
1134         return extra
1135
1136     def get_opt_for_task(self, task_id):
1137         extra = self.get_extra(task_id)
1138         return extra["opt"]
1139
1140     def task_to_skip_if(self, task_id, kind):
1141         """Calculate the skip-if condition for failing task task_id"""
1142
1143         if kind == Kind.WPT:
1144             qq = '"'
1145             aa = " and "
1146             nn = "not "
1147         elif kind == Kind.LIST:
1148             qq = "'"
1149             aa = "&&"
1150             nn = "!"
1151         else:
1152             qq = "'"
1153             aa = " && "
1154             nn = "!"
1155         eq = " == "
1156         arch = "processor"
1157         version = "os_version"
1158         extra = self.get_extra(task_id)
1159         skip_if = None
1160         os = extra.get("os")
1161         if os is not None:
1162             if kind == Kind.LIST:
1163                 if os == "linux":
1164                     skip_if = "gtkWidget"
1165                 elif os == "win":
1166                     skip_if = "winWidget"
1167                 elif os == "mac":
1168                     skip_if = "cocoaWidget"
1169                 elif os == "android":
1170                     skip_if = "Android"
1171                 else:
1172                     self.error(f"cannot calculate skip-if for unknown OS: '{os}'")
1173                     return None
1174             elif extra.get("os_version") is not None:
1175                 if (
1176                     extra.get("build") is not None
1177                     and os == "win"
1178                     and extra["os_version"] == "11"
1179                     and extra["build"] == "2009"
1180                 ):
1181                     skip_if = "win11_2009"  # mozinfo.py:137
1182                 else:
1183                     skip_if = "os" + eq + qq + os + qq
1184                     skip_if += aa + version + eq + qq + extra["os_version"] + qq
1185             if kind != Kind.LIST and extra.get("arch") is not None:
1186                 skip_if += aa + arch + eq + qq + extra["arch"] + qq
1187             # since we always give arch/processor, bits are not required
1188             # if extra["bits"] is not None:
1189             #     skip_if += aa + "bits" + eq + extra["bits"]
1190             debug = extra.get("debug", False)
1191             runtimes = extra.get("runtimes", [])
1192             fission = "no-fission" not in runtimes
1193             snapshot = "snapshot" in runtimes
1194             swgl = "swgl" in runtimes
1195             build_types = extra.get("build_types", [])
1196             asan = "asan" in build_types
1197             ccov = "ccov" in build_types
1198             tsan = "tsan" in build_types
1199             optimized = (not debug) and (not ccov) and (not asan) and (not tsan)
1200             skip_if += aa
1201             if kind == Kind.LIST:
1202                 if optimized:
1203                     skip_if += "optimized"
1204                 elif debug:
1205                     skip_if += "isDebugBuild"
1206                 elif ccov:
1207                     skip_if += "isCoverageBuild"
1208                 elif asan:
1209                     skip_if += "AddressSanitizer"
1210                 elif tsan:
1211                     skip_if += "ThreadSanitizer"
1212                 # See implicit VARIANT_DEFAULTS in
1213                 # https://searchfox.org/mozilla-central/source/layout/tools/reftest/manifest.sys.mjs#30
1214                 if not self.implicit_vars and fission:
1215                     skip_if += aa + "fission"
1216                 elif not fission:  # implicit default: fission
1217                     skip_if += aa + nn + "fission"
1218                 if extra.get("bits") is not None:
1219                     if extra["bits"] == "32":
1220                         skip_if += aa + nn + "is64Bit"  # override implicit is64Bit
1221                     elif not self.implicit_vars and os == "winWidget":
1222                         skip_if += aa + "is64Bit"
1223                 if not self.implicit_vars and not swgl:
1224                     skip_if += aa + nn + "swgl"
1225                 elif swgl:  # implicit default: !swgl
1226                     skip_if += aa + "swgl"
1227                 if os == "gtkWidget":
1228                     if not self.implicit_vars and not snapshot:
1229                         skip_if += aa + nn + "useDrawSnapshot"
1230                     elif snapshot:  # implicit default: !useDrawSnapshot
1231                         skip_if += aa + "useDrawSnapshot"
1232             else:
1233                 if not debug:
1234                     skip_if += nn
1235                 skip_if += "debug"
1236                 if extra.get("display") is not None:
1237                     skip_if += aa + "display" + eq + qq + extra["display"] + qq
1238                 for runtime in extra.get("runtimes", []):
1239                     skip_if += aa + runtime
1240                 for build_type in extra.get("build_types", []):
1241                     # note: lite will not evaluate on non-android platforms
1242                     if build_type not in ["debug", "lite", "opt", "shippable"]:
1243                         skip_if += aa + build_type
1244         return skip_if
1245
1246     def get_file_info(self, path, product="Testing", component="General"):
1247         """
1248         Get bugzilla product and component for the path.
1249         Provide defaults (in case command_context is not defined
1250         or there isn't file info available).
1251         """
1252         if path != DEF and self.command_context is not None:
1253             reader = self.command_context.mozbuild_reader(config_mode="empty")
1254             info = reader.files_info([path])
1255             cp = info[path]["BUG_COMPONENT"]
1256             product = cp.product
1257             component = cp.component
1258         return product, component
1259
1260     def get_filename_in_manifest(self, manifest, path):
1261         """return relative filename for path in manifest"""
1262
1263         filename = os.path.basename(path)
1264         if filename == DEF:
1265             return filename
1266         manifest_dir = os.path.dirname(manifest)
1267         i = 0
1268         j = min(len(manifest_dir), len(path))
1269         while i < j and manifest_dir[i] == path[i]:
1270             i += 1
1271         if i < len(manifest_dir):
1272             for _ in range(manifest_dir.count("/", i) + 1):
1273                 filename = "../" + filename
1274         elif i < len(path):
1275             filename = path[i + 1 :]
1276         return filename
1277
1278     def get_push_id(self, revision, repo):
1279         """Return the push_id for revision and repo (or None)"""
1280
1281         self.vinfo(f"Retrieving push_id for {repo} revision: {revision} ...")
1282         if revision in self.push_ids:  # if cached
1283             push_id = self.push_ids[revision]
1284         else:
1285             push_id = None
1286             push_url = f"https://treeherder.mozilla.org/api/project/{repo}/push/"
1287             params = {}
1288             params["full"] = "true"
1289             params["count"] = 10
1290             params["revision"] = revision
1291             r = requests.get(push_url, headers=self.headers, params=params)
1292             if r.status_code != 200:
1293                 self.warning(f"FAILED to query Treeherder = {r} for {r.url}")
1294             else:
1295                 response = r.json()
1296                 if "results" in response:
1297                     results = response["results"]
1298                     if len(results) > 0:
1299                         r0 = results[0]
1300                         if "id" in r0:
1301                             push_id = r0["id"]
1302             self.push_ids[revision] = push_id
1303         return push_id
1304
1305     def get_job_id(self, push_id, task_id):
1306         """Return the job_id for push_id, task_id (or None)"""
1307
1308         self.vinfo(f"Retrieving job_id for push_id: {push_id}, task_id: {task_id} ...")
1309         k = f"{push_id}:{task_id}"
1310         if k in self.job_ids:  # if cached
1311             job_id = self.job_ids[k]
1312         else:
1313             job_id = None
1314             params = {}
1315             params["push_id"] = push_id
1316             r = requests.get(self.jobs_url, headers=self.headers, params=params)
1317             if r.status_code != 200:
1318                 self.warning(f"FAILED to query Treeherder = {r} for {r.url}")
1319             else:
1320                 response = r.json()
1321                 if "results" in response:
1322                     results = response["results"]
1323                     if len(results) > 0:
1324                         for result in results:
1325                             if len(result) > 14:
1326                                 if result[14] == task_id:
1327                                     job_id = result[1]
1328                                     break
1329             self.job_ids[k] = job_id
1330         return job_id
1331
1332     def get_bug_suggestions(self, repo, job_id, path, anyjs=None):
1333         """
1334         Return the (suggestions_url, line_number, line, log_url)
1335         for the given repo and job_id
1336         """
1337         self.vinfo(
1338             f"Retrieving bug_suggestions for {repo} job_id: {job_id}, path: {path} ..."
1339         )
1340         suggestions_url = f"https://treeherder.mozilla.org/api/project/{repo}/jobs/{job_id}/bug_suggestions/"
1341         line_number = None
1342         line = None
1343         log_url = None
1344         r = requests.get(suggestions_url, headers=self.headers)
1345         if r.status_code != 200:
1346             self.warning(f"FAILED to query Treeherder = {r} for {r.url}")
1347         else:
1348             if anyjs is not None:
1349                 pathdir = os.path.dirname(path) + "/"
1350                 paths = [pathdir + f for f in anyjs.keys()]
1351             else:
1352                 paths = [path]
1353             response = r.json()
1354             if len(response) > 0:
1355                 for sugg in response:
1356                     for p in paths:
1357                         path_end = sugg.get("path_end", None)
1358                         # handles WPT short paths
1359                         if path_end is not None and p.endswith(path_end):
1360                             line_number = sugg["line_number"] + 1
1361                             line = sugg["search"]
1362                             log_url = f"https://treeherder.mozilla.org/logviewer?repo={repo}&job_id={job_id}&lineNumber={line_number}"
1363                             break
1364         rv = (suggestions_url, line_number, line, log_url)
1365         return rv
1366
1367     def read_json(self, filename):
1368         """read data as JSON from filename"""
1369
1370         fp = io.open(filename, "r", encoding="utf-8")
1371         data = json.load(fp)
1372         fp.close()
1373         return data
1374
1375     def read_tasks(self, filename):
1376         """read tasks as JSON from filename"""
1377
1378         if not os.path.exists(filename):
1379             msg = f"use-tasks JSON file does not exist: {filename}"
1380             raise OSError(2, msg, filename)
1381         tasks = self.read_json(filename)
1382         tasks = [Mock(task, MOCK_TASK_DEFAULTS, MOCK_TASK_INITS) for task in tasks]
1383         for task in tasks:
1384             if len(task.extra) > 0:  # pre-warm cache for extra information
1385                 self.extras[task.id] = task.extra
1386         return tasks
1387
1388     def read_failures(self, filename):
1389         """read failures as JSON from filename"""
1390
1391         if not os.path.exists(filename):
1392             msg = f"use-failures JSON file does not exist: {filename}"
1393             raise OSError(2, msg, filename)
1394         failures = self.read_json(filename)
1395         return failures
1396
1397     def read_bugs(self, filename):
1398         """read bugs as JSON from filename"""
1399
1400         if not os.path.exists(filename):
1401             msg = f"bugs JSON file does not exist: {filename}"
1402             raise OSError(2, msg, filename)
1403         bugs = self.read_json(filename)
1404         bugs = [Mock(bug, MOCK_BUG_DEFAULTS) for bug in bugs]
1405         return bugs
1406
1407     def write_json(self, filename, data):
1408         """saves data as JSON to filename"""
1409         fp = io.open(filename, "w", encoding="utf-8")
1410         json.dump(data, fp, indent=2, sort_keys=True)
1411         fp.close()
1412
1413     def write_tasks(self, save_tasks, tasks):
1414         """saves tasks as JSON to save_tasks"""
1415         jtasks = []
1416         for task in tasks:
1417             if not isinstance(task, TestTask):
1418                 continue
1419             jtask = {}
1420             jtask["id"] = task.id
1421             jtask["label"] = task.label
1422             jtask["duration"] = task.duration
1423             jtask["result"] = task.result
1424             jtask["state"] = task.state
1425             jtask["extra"] = self.get_extra(task.id)
1426             jtags = {}
1427             for k, v in task.tags.items():
1428                 if k == "createdForUser":
1429                     jtags[k] = "ci@mozilla.com"
1430                 else:
1431                     jtags[k] = v
1432             jtask["tags"] = jtags
1433             jtask["tier"] = task.tier
1434             jtask["results"] = [
1435                 {"group": r.group, "ok": r.ok, "duration": r.duration}
1436                 for r in task.results
1437             ]
1438             jtask["errors"] = None  # Bug with task.errors property??
1439             jft = {}
1440             if self.failure_types is not None and task.id in self.failure_types:
1441                 failure_types = self.failure_types[task.id]  # use cache
1442             else:
1443                 failure_types = task.failure_types
1444             for k in failure_types:
1445                 jft[k] = [[f[0], f[1].value] for f in task.failure_types[k]]
1446             jtask["failure_types"] = jft
1447             jtasks.append(jtask)
1448         self.write_json(save_tasks, jtasks)
1449
1450     def label_to_platform_testname(self, label):
1451         """convert from label to platform, testname for mach command line"""
1452         platform = None
1453         testname = None
1454         platform_details = label.split("/")
1455         if len(platform_details) == 2:
1456             platform, details = platform_details
1457             words = details.split("-")
1458             if len(words) > 2:
1459                 platform += "/" + words.pop(0)  # opt or debug
1460                 try:
1461                     _chunk = int(words[-1])
1462                     words.pop()
1463                 except ValueError:
1464                     pass
1465                 words.pop()  # remove test suffix
1466                 testname = "-".join(words)
1467             else:
1468                 platform = None
1469         return platform, testname
1470
1471     def add_attachment_log_for_task(self, bugid, task_id):
1472         """Adds compressed log for this task to bugid"""
1473
1474         log_url = f"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/{task_id}/artifacts/public/logs/live_backing.log"
1475         r = requests.get(log_url, headers=self.headers)
1476         if r.status_code != 200:
1477             self.error(f"Unable to get log for task: {task_id}")
1478             return
1479         attach_fp = tempfile.NamedTemporaryFile()
1480         fp = gzip.open(attach_fp, "wb")
1481         fp.write(r.text.encode("utf-8"))
1482         fp.close()
1483         if self._initialize_bzapi():
1484             description = ATTACHMENT_DESCRIPTION + task_id
1485             file_name = TASK_LOG + ".gz"
1486             comment = "Added compressed log"
1487             content_type = "application/gzip"
1488             try:
1489                 self._bzapi.attachfile(
1490                     [bugid],
1491                     attach_fp.name,
1492                     description,
1493                     file_name=file_name,
1494                     comment=comment,
1495                     content_type=content_type,
1496                     is_private=False,
1497                 )
1498             except Fault:
1499                 pass  # Fault expected: Failed to fetch key 9372091 from network storage: The specified key does not exist.
1500
1501     def get_wpt_path_meta(self, shortpath):
1502         if shortpath.startswith(WPT0):
1503             path = shortpath
1504             meta = shortpath.replace(WPT0, WPT_META0, 1)
1505         elif shortpath.startswith(WPT1):
1506             path = shortpath
1507             meta = shortpath.replace(WPT1, WPT_META1, 1)
1508         elif shortpath.startswith(WPT2):
1509             path = shortpath
1510             meta = shortpath.replace(WPT2, WPT_META2, 1)
1511         elif shortpath.startswith(WPT_MOZILLA):
1512             shortpath = shortpath[len(WPT_MOZILLA) :]
1513             path = WPT2 + shortpath
1514             meta = WPT_META2 + shortpath
1515         else:
1516             path = WPT1 + shortpath
1517             meta = WPT_META1 + shortpath
1518         return (path, meta)
1519
1520     def wpt_paths(self, shortpath):
1521         """
1522         Analyzes the WPT short path for a test and returns
1523         (path, manifest, query, anyjs) where
1524         path is the relative path to the test file
1525         manifest is the relative path to the file metadata
1526         query is the test file query paramters (or None)
1527         anyjs is the html test file as reported by mozci (or None)
1528         """
1529         query = None
1530         anyjs = None
1531         i = shortpath.find("?")
1532         if i > 0:
1533             query = shortpath[i:]
1534             shortpath = shortpath[0:i]
1535         path, manifest = self.get_wpt_path_meta(shortpath)
1536         failure_type = not self.isdir(path)
1537         if failure_type:
1538             i = path.find(".any.")
1539             if i > 0:
1540                 anyjs = path  # orig path
1541                 manifest = manifest.replace(path[i:], ".any.js")
1542                 path = path[0:i] + ".any.js"
1543             else:
1544                 i = path.find(".window.")
1545                 if i > 0:
1546                     anyjs = path  # orig path
1547                     manifest = manifest.replace(path[i:], ".window.js")
1548                     path = path[0:i] + ".window.js"
1549                 else:
1550                     i = path.find(".worker.")
1551                     if i > 0:
1552                         anyjs = path  # orig path
1553                         manifest = manifest.replace(path[i:], ".worker.js")
1554                         path = path[0:i] + ".worker.js"
1555             manifest += ".ini"
1556         manifest_classic = ""
1557         if manifest.startswith(WPT_META0):
1558             manifest_classic = manifest.replace(WPT_META0, WPT_META0_CLASSIC, 1)
1559             if self.exists(manifest_classic):
1560                 if self.exists(manifest):
1561                     self.warning(
1562                         f"Both classic {manifest_classic} and metadata {manifest} manifests exist"
1563                     )
1564                 else:
1565                     self.warning(
1566                         f"Using the classic {manifest_classic} manifest as the metadata manifest {manifest} does not exist"
1567                     )
1568                     manifest = manifest_classic
1569         if not self.exists(path):
1570             return (None, None, None, None)
1571         return (path, manifest, query, anyjs)
1572
1573     def wpt_add_skip_if(self, manifest_str, anyjs, skip_if, bug_reference):
1574         """
1575         Edits a WPT manifest string to add disabled condition
1576         anyjs is a dictionary mapping from filename and any alternate basenames to
1577         a boolean (indicating if the file has been handled in the manifest).
1578         Returns additional_comment (if any)
1579         """
1580
1581         additional_comment = ""
1582         disabled_key = False
1583         disabled = "  disabled:"
1584         condition_start = "    if "
1585         condition = condition_start + skip_if + ": " + bug_reference
1586         lines = manifest_str.splitlines()
1587         section = None  # name of the section
1588         i = 0
1589         n = len(lines)
1590         while i < n:
1591             line = lines[i]
1592             if line.startswith("["):
1593                 if section is not None and not anyjs[section]:  # not yet handled
1594                     if not disabled_key:
1595                         lines.insert(i, disabled)
1596                         i += 1
1597                     lines.insert(i, condition)
1598                     lines.insert(i + 1, "")  # blank line after condition
1599                     i += 2
1600                     n += 2
1601                     anyjs[section] = True
1602                 section = line[1:-1]
1603                 if section in anyjs and not anyjs[section]:
1604                     disabled_key = False
1605                 else:
1606                     section = None  # ignore section we are not interested in
1607             elif section is not None:
1608                 if line == disabled:
1609                     disabled_key = True
1610                 elif line.startswith("  ["):
1611                     if i > 0 and i - 1 < n and lines[i - 1] == "":
1612                         del lines[i - 1]
1613                         i -= 1
1614                         n -= 1
1615                     if not disabled_key:
1616                         lines.insert(i, disabled)
1617                         i += 1
1618                         n += 1
1619                     lines.insert(i, condition)
1620                     lines.insert(i + 1, "")  # blank line after condition
1621                     i += 2
1622                     n += 2
1623                     anyjs[section] = True
1624                     section = None
1625                 elif line.startswith("  ") and not line.startswith("    "):
1626                     if disabled_key:  # insert condition above new key
1627                         lines.insert(i, condition)
1628                         i += 1
1629                         n += 1
1630                         anyjs[section] = True
1631                         section = None
1632                         disabled_key = False
1633                 elif line.startswith("    "):
1634                     if disabled_key and line == condition:
1635                         anyjs[section] = True  # condition already present
1636                         section = None
1637             i += 1
1638         if section is not None and not anyjs[section]:  # not yet handled
1639             if i > 0 and i - 1 < n and lines[i - 1] == "":
1640                 del lines[i - 1]
1641             if not disabled_key:
1642                 lines.append(disabled)
1643                 i += 1
1644                 n += 1
1645             lines.append(condition)
1646             lines.append("")  # blank line after condition
1647             i += 2
1648             n += 2
1649             anyjs[section] = True
1650         for section in anyjs:
1651             if not anyjs[section]:
1652                 if i > 0 and i - 1 < n and lines[i - 1] != "":
1653                     lines.append("")  # blank line before condition
1654                     i += 1
1655                     n += 1
1656                 lines.append("[" + section + "]")
1657                 lines.append(disabled)
1658                 lines.append(condition)
1659                 lines.append("")  # blank line after condition
1660                 i += 4
1661                 n += 4
1662         manifest_str = "\n".join(lines) + "\n"
1663         return manifest_str, additional_comment
1664
1665     def reftest_add_fuzzy_if(
1666         self,
1667         manifest_str,
1668         filename,
1669         fuzzy_if,
1670         differences,
1671         pixels,
1672         lineno,
1673         zero,
1674         bug_reference,
1675     ):
1676         """
1677         Edits a reftest manifest string to add disabled condition
1678         """
1679
1680         if self.lmp is None:
1681             from parse_reftest import ListManifestParser
1682
1683             self.lmp = ListManifestParser(
1684                 self.implicit_vars, self.verbose, self.error, self.warning, self.info
1685             )
1686         manifest_str, additional_comment = self.lmp.reftest_add_fuzzy_if(
1687             manifest_str,
1688             filename,
1689             fuzzy_if,
1690             differences,
1691             pixels,
1692             lineno,
1693             zero,
1694             bug_reference,
1695         )
1696         return manifest_str, additional_comment
1697
1698     def get_lineno_difference_pixels_status(self, task_id, manifest, allmods):
1699         """
1700         Returns
1701         - lineno in manifest
1702         - image comparison, max *difference*
1703         - number of differing *pixels*
1704         - status (PASS or FAIL)
1705         as cached from reftest_errorsummary.log for a task
1706         """
1707
1708         manifest_obj = self.error_summary.get(manifest, {})
1709         allmods_obj = manifest_obj.get(allmods, {})
1710         lineno = allmods_obj.get(LINENO, 0)
1711         runs_obj = allmods_obj.get(RUNS, {})
1712         task_obj = runs_obj.get(task_id, {})
1713         difference = task_obj.get(DIFFERENCE, 0)
1714         pixels = task_obj.get(PIXELS, 0)
1715         status = task_obj.get(STATUS, FAIL)
1716         return lineno, difference, pixels, status
1717
1718     def reftest_find_lineno(self, manifest, modifiers, allmods):
1719         """
1720         Return the line number with modifiers in manifest (else 0)
1721         """
1722
1723         lineno = 0
1724         mods = []
1725         prefs = []
1726         for i in range(len(modifiers)):
1727             if modifiers[i].find("pref(") >= 0 or modifiers[i].find("skip-if(") >= 0:
1728                 prefs.append(modifiers[i])
1729             else:
1730                 mods.append(modifiers[i])
1731         m = len(mods)
1732         manifest_str = io.open(manifest, "r", encoding="utf-8").read()
1733         lines = manifest_str.splitlines()
1734         defaults = []
1735         found = False
1736         alt_lineno = 0
1737         for linenum in range(len(lines)):
1738             line = lines[linenum]
1739             if len(line) > 0 and line[0] == "#":
1740                 continue
1741             comment_start = line.find(" #")  # MUST NOT match anchors!
1742             if comment_start > 0:
1743                 line = line[0:comment_start].strip()
1744             words = line.split()
1745             n = len(words)
1746             if n > 1 and words[0] == "defaults":
1747                 defaults = words[1:].copy()
1748                 continue
1749             line_defaults = defaults.copy()
1750             i = 0
1751             while i < n:
1752                 if words[i].find("pref(") >= 0 or words[i].find("skip-if(") >= 0:
1753                     line_defaults.append(words[i])
1754                     del words[i]
1755                     n -= 1
1756                 else:
1757                     i += 1
1758             if (len(prefs) == 0 or prefs == line_defaults) and words == mods:
1759                 found = True
1760                 lineno = linenum + 1
1761                 break
1762             elif m > 2 and n > 2:
1763                 if words[-3:] == mods[-3:]:
1764                     alt_lineno = linenum + 1
1765                 else:
1766                     bwords = [os.path.basename(f) for f in words[-2:]]
1767                     bmods = [os.path.basename(f) for f in mods[-2:]]
1768                     if bwords == bmods:
1769                         alt_lineno = linenum + 1
1770         if not found:
1771             if alt_lineno > 0:
1772                 lineno = alt_lineno
1773                 self.warning(
1774                     f"manifest '{manifest}' found lineno: {lineno}, but it does not contain all the prefs from modifiers,\nSEARCH: {allmods}\nFOUND : {lines[alt_lineno - 1]}"
1775                 )
1776             else:
1777                 lineno = 0
1778                 self.error(
1779                     f"manifest '{manifest}' does not contain line with modifiers: {allmods}"
1780                 )
1781         return lineno
1782
1783     def get_allpaths(self, task_id, manifest, path):
1784         """
1785         Looks up the reftest_errorsummary.log for a task
1786         and caches the details in self.error_summary by
1787            task_id, manifest, allmods
1788         where allmods is the concatenation of all modifiers
1789         and the details include
1790         - image comparison, max *difference*
1791         - number of differing *pixels*
1792         - status: unexpected PASS or FAIL
1793
1794         The list iof unique modifiers (allmods) for the given path are returned
1795         """
1796
1797         allpaths = []
1798         words = path.split()
1799         if len(words) != 3 or words[1] not in TEST_TYPES:
1800             self.warning(
1801                 f"reftest_errorsummary.log for task: {task_id} has unsupported test type '{path}'"
1802             )
1803             return allpaths
1804         if manifest in self.error_summary:
1805             for allmods in self.error_summary[manifest]:
1806                 if self.error_summary[manifest][allmods][
1807                     TEST
1808                 ] == path and task_id in self.error_summary[manifest][allmods].get(
1809                     RUNS, {}
1810                 ):
1811                     allpaths.append(allmods)
1812             if len(allpaths) > 0:
1813                 return allpaths  # cached (including self tests)
1814         error_url = f"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/{task_id}/artifacts/public/test_info/reftest_errorsummary.log"
1815         self.vinfo(f"Requesting reftest_errorsummary.log for task: {task_id}")
1816         r = requests.get(error_url, headers=self.headers)
1817         if r.status_code != 200:
1818             self.error(f"Unable to get reftest_errorsummary.log for task: {task_id}")
1819             return allpaths
1820         for line in r.text.encode("utf-8").splitlines():
1821             summary = json.loads(line)
1822             group = summary.get(GROUP, "")
1823             if not group or not os.path.exists(group):  # not error line
1824                 continue
1825             test = summary.get(TEST, None)
1826             if test is None:
1827                 continue
1828             if not MODIFIERS in summary:
1829                 self.warning(
1830                     f"reftest_errorsummary.log for task: {task_id} does not have modifiers for '{test}'"
1831                 )
1832                 continue
1833             words = test.split()
1834             if len(words) != 3 or words[1] not in TEST_TYPES:
1835                 self.warning(
1836                     f"reftest_errorsummary.log for task: {task_id} has unsupported test '{test}'"
1837                 )
1838                 continue
1839             status = summary.get(STATUS, "")
1840             if status not in [FAIL, PASS]:
1841                 self.warning(
1842                     f"reftest_errorsummary.log for task: {task_id} has unknown status: {status} for '{test}'"
1843                 )
1844                 continue
1845             error = summary.get(SUBTEST, "")
1846             mods = summary[MODIFIERS]
1847             allmods = " ".join(mods)
1848             if group not in self.error_summary:
1849                 self.error_summary[group] = {}
1850             if allmods not in self.error_summary[group]:
1851                 self.error_summary[group][allmods] = {}
1852             self.error_summary[group][allmods][TEST] = test
1853             lineno = self.error_summary[group][allmods].get(LINENO, 0)
1854             if lineno == 0:
1855                 lineno = self.reftest_find_lineno(group, mods, allmods)
1856                 if lineno > 0:
1857                     self.error_summary[group][allmods][LINENO] = lineno
1858             if RUNS not in self.error_summary[group][allmods]:
1859                 self.error_summary[group][allmods][RUNS] = {}
1860             if task_id not in self.error_summary[group][allmods][RUNS]:
1861                 self.error_summary[group][allmods][RUNS][task_id] = {}
1862             self.error_summary[group][allmods][RUNS][task_id][ERROR] = error
1863             if self._subtest_rx is None:
1864                 self._subtest_rx = re.compile(SUBTEST_REGEX)
1865             m = self._subtest_rx.findall(error)
1866             if len(m) == 1:
1867                 difference = int(m[0][0])
1868                 pixels = int(m[0][1])
1869             else:
1870                 difference = 0
1871                 pixels = 0
1872             if difference > 0:
1873                 self.error_summary[group][allmods][RUNS][task_id][
1874                     DIFFERENCE
1875                 ] = difference
1876             if pixels > 0:
1877                 self.error_summary[group][allmods][RUNS][task_id][PIXELS] = pixels
1878             if status != FAIL:
1879                 self.error_summary[group][allmods][RUNS][task_id][STATUS] = status
1880             if test == path:
1881                 allpaths.append(allmods)
1882         return allpaths