testing/skipfails.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import gzip
   6 import io
   7 import json
   8 import logging
   9 import os
  10 import os.path
  11 import pprint
  12 import re
  13 import sys
  14 import tempfile
  15 import urllib.parse
  16 from copy import deepcopy
  17 from enum import Enum
  18 from pathlib import Path
  19 from statistics import median
  20 from xmlrpc.client import Fault
  21
  22 from yaml import load
  23
  24 try:
  25     from yaml import CLoader as Loader
  26 except ImportError:
  27     from yaml import Loader
  28
  29 import bugzilla
  30 import mozci.push
  31 import requests
  32 from manifestparser import ManifestParser
  33 from manifestparser.toml import add_skip_if, alphabetize_toml_str, sort_paths
  34 from mozci.task import TestTask
  35 from mozci.util.taskcluster import get_task
  36
  37 from taskcluster.exceptions import TaskclusterRestFailure
  38
  39 TASK_LOG = "live_backing.log"
  40 TASK_ARTIFACT = "public/logs/" + TASK_LOG
  41 ATTACHMENT_DESCRIPTION = "Compressed " + TASK_ARTIFACT + " for task "
  42 ATTACHMENT_REGEX = (
  43     r".*Created attachment ([0-9]+)\n.*"
  44     + ATTACHMENT_DESCRIPTION
  45     + "([A-Za-z0-9_-]+)\n.*"
  46 )
  47
  48 BUGZILLA_AUTHENTICATION_HELP = "Must create a Bugzilla API key per https://github.com/mozilla/mozci-tools/blob/main/citools/test_triage_bug_filer.py"
  49
  50 MS_PER_MINUTE = 60 * 1000  # ms per minute
  51 DEBUG_THRESHOLD = 40 * MS_PER_MINUTE  # 40 minutes in ms
  52 OPT_THRESHOLD = 20 * MS_PER_MINUTE  # 20 minutes in ms
  53
  54 ANYJS = "anyjs"
  55 CC = "classification"
  56 DEF = "DEFAULT"
  57 DURATIONS = "durations"
  58 FAILED_RUNS = "runs_failed"
  59 FAILURE_RATIO = 0.4  # more than this fraction of failures will disable
  60 LL = "label"
  61 MEDIAN_DURATION = "duration_median"
  62 MINIMUM_RUNS = 3  # mininum number of runs to consider success/failure
  63 MOCK_BUG_DEFAULTS = {"blocks": [], "comments": []}
  64 MOCK_TASK_DEFAULTS = {"extra": {}, "failure_types": {}, "results": []}
  65 MOCK_TASK_INITS = ["results"]
  66 OPT = "opt"
  67 PP = "path"
  68 QUERY = "query"
  69 RR = "result"
  70 RUNS = "runs"
  71 SUM_BY_LABEL = "sum_by_label"
  72 TOTAL_DURATION = "duration_total"
  73 TOTAL_RUNS = "runs_total"
  74 WP = "testing/web-platform/"
  75 WPT = "wpt"
  76 WPT0 = WP + "tests/infrastructure"
  77 WPT_META0 = WP + "tests/infrastructure/metadata"
  78 WPT_META0_CLASSIC = WP + "meta/infrastructure"
  79 WPT1 = WP + "tests"
  80 WPT_META1 = WPT1.replace("tests", "meta")
  81 WPT2 = WP + "mozilla/tests"
  82 WPT_META2 = WPT2.replace("tests", "meta")
  83 WPT_MOZILLA = "/_mozilla"
  84
  85
  86 class Mock(object):
  87     def __init__(self, data, defaults={}, inits=[]):
  88         self._data = data
  89         self._defaults = defaults
  90         for name in inits:
  91             values = self._data.get(name, [])  # assume type is an array
  92             values = [Mock(value, defaults, inits) for value in values]
  93             self._data[name] = values
  94
  95     def __getattr__(self, name):
  96         if name in self._data:
  97             return self._data[name]
  98         if name in self._defaults:
  99             return self._defaults[name]
 100         return ""
 101
 102
 103 class Classification(object):
 104     "Classification of the failure (not the task result)"
 105
 106     DISABLE_MANIFEST = "disable_manifest"  # crash found
 107     DISABLE_RECOMMENDED = "disable_recommended"  # disable first failing path
 108     DISABLE_TOO_LONG = "disable_too_long"  # runtime threshold exceeded
 109     INTERMITTENT = "intermittent"
 110     SECONDARY = "secondary"  # secondary failing path
 111     SUCCESS = "success"  # path always succeeds
 112     UNKNOWN = "unknown"
 113
 114
 115 class Run(Enum):
 116     """
 117     constant indexes for attributes of a run
 118     """
 119
 120     MANIFEST = 0
 121     TASK_ID = 1
 122     TASK_LABEL = 2
 123     RESULT = 3
 124     CLASSIFICATION = 4
 125
 126
 127 class Skipfails(object):
 128     "mach manifest skip-fails implementation: Update manifests to skip failing tests"
 129
 130     REPO = "repo"
 131     REVISION = "revision"
 132     TREEHERDER = "treeherder.mozilla.org"
 133     BUGZILLA_SERVER_DEFAULT = "bugzilla.allizom.org"
 134
 135     def __init__(
 136         self,
 137         command_context=None,
 138         try_url="",
 139         verbose=False,
 140         bugzilla=None,
 141         dry_run=False,
 142         turbo=False,
 143     ):
 144         self.command_context = command_context
 145         if self.command_context is not None:
 146             self.topsrcdir = self.command_context.topsrcdir
 147         else:
 148             self.topsrcdir = Path(__file__).parent.parent
 149         self.topsrcdir = os.path.normpath(self.topsrcdir)
 150         if isinstance(try_url, list) and len(try_url) == 1:
 151             self.try_url = try_url[0]
 152         else:
 153             self.try_url = try_url
 154         self.dry_run = dry_run
 155         self.verbose = verbose
 156         self.turbo = turbo
 157         if bugzilla is not None:
 158             self.bugzilla = bugzilla
 159         elif "BUGZILLA" in os.environ:
 160             self.bugzilla = os.environ["BUGZILLA"]
 161         else:
 162             self.bugzilla = Skipfails.BUGZILLA_SERVER_DEFAULT
 163         if self.bugzilla == "disable":
 164             self.bugzilla = None  # Bug filing disabled
 165         self.component = "skip-fails"
 166         self._bzapi = None
 167         self._attach_rx = None
 168         self.variants = {}
 169         self.tasks = {}
 170         self.pp = None
 171         self.headers = {}  # for Treeherder requests
 172         self.headers["Accept"] = "application/json"
 173         self.headers["User-Agent"] = "treeherder-pyclient"
 174         self.jobs_url = "https://treeherder.mozilla.org/api/jobs/"
 175         self.push_ids = {}
 176         self.job_ids = {}
 177         self.extras = {}
 178         self.bugs = []  # preloaded bugs, currently not an updated cache
 179
 180     def _initialize_bzapi(self):
 181         """Lazily initializes the Bugzilla API (returns True on success)"""
 182         if self._bzapi is None and self.bugzilla is not None:
 183             self._bzapi = bugzilla.Bugzilla(self.bugzilla)
 184             self._attach_rx = re.compile(ATTACHMENT_REGEX, flags=re.M)
 185         return self._bzapi is not None
 186
 187     def pprint(self, obj):
 188         if self.pp is None:
 189             self.pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)
 190         self.pp.pprint(obj)
 191         sys.stderr.flush()
 192
 193     def error(self, e):
 194         if self.command_context is not None:
 195             self.command_context.log(
 196                 logging.ERROR, self.component, {"error": str(e)}, "ERROR: {error}"
 197             )
 198         else:
 199             print(f"ERROR: {e}", file=sys.stderr, flush=True)
 200
 201     def warning(self, e):
 202         if self.command_context is not None:
 203             self.command_context.log(
 204                 logging.WARNING, self.component, {"error": str(e)}, "WARNING: {error}"
 205             )
 206         else:
 207             print(f"WARNING: {e}", file=sys.stderr, flush=True)
 208
 209     def info(self, e):
 210         if self.command_context is not None:
 211             self.command_context.log(
 212                 logging.INFO, self.component, {"error": str(e)}, "INFO: {error}"
 213             )
 214         else:
 215             print(f"INFO: {e}", file=sys.stderr, flush=True)
 216
 217     def vinfo(self, e):
 218         if self.verbose:
 219             self.info(e)
 220
 221     def full_path(self, filename):
 222         """Returns full path for the relative filename"""
 223
 224         return os.path.join(self.topsrcdir, os.path.normpath(filename))
 225
 226     def isdir(self, filename):
 227         """Returns True if filename is a directory"""
 228
 229         return os.path.isdir(self.full_path(filename))
 230
 231     def exists(self, filename):
 232         """Returns True if filename exists"""
 233
 234         return os.path.exists(self.full_path(filename))
 235
 236     def run(
 237         self,
 238         meta_bug_id=None,
 239         save_tasks=None,
 240         use_tasks=None,
 241         save_failures=None,
 242         use_failures=None,
 243         max_failures=-1,
 244     ):
 245         "Run skip-fails on try_url, return True on success"
 246
 247         try_url = self.try_url
 248         revision, repo = self.get_revision(try_url)
 249         if use_tasks is not None:
 250             tasks = self.read_tasks(use_tasks)
 251             self.vinfo(f"use tasks: {use_tasks}")
 252         else:
 253             tasks = self.get_tasks(revision, repo)
 254         if use_failures is not None:
 255             failures = self.read_failures(use_failures)
 256             self.vinfo(f"use failures: {use_failures}")
 257         else:
 258             failures = self.get_failures(tasks)
 259             if save_failures is not None:
 260                 self.write_json(save_failures, failures)
 261                 self.vinfo(f"save failures: {save_failures}")
 262         if save_tasks is not None:
 263             self.write_tasks(save_tasks, tasks)
 264             self.vinfo(f"save tasks: {save_tasks}")
 265         num_failures = 0
 266         for manifest in failures:
 267             if manifest.endswith(".toml") or manifest.startswith(WP):
 268                 wpt = failures[manifest][WPT]
 269                 for label in failures[manifest][LL]:
 270                     for path in failures[manifest][LL][label][PP]:
 271                         classification = failures[manifest][LL][label][PP][path][CC]
 272                         if classification.startswith("disable_") or (
 273                             self.turbo and classification == Classification.SECONDARY
 274                         ):
 275                             anyjs = {}  # anyjs alternate basename = False
 276                             for task_id in failures[manifest][LL][label][PP][path][
 277                                 RUNS
 278                             ]:
 279                                 if not wpt:
 280                                     break  # just use the first task_id
 281                                 filename = os.path.basename(path)
 282                                 anyjs[filename] = False
 283                                 if (
 284                                     QUERY
 285                                     in failures[manifest][LL][label][PP][path][RUNS][
 286                                         task_id
 287                                     ]
 288                                 ):
 289                                     query = failures[manifest][LL][label][PP][path][
 290                                         RUNS
 291                                     ][task_id][QUERY]
 292                                     anyjs[filename + query] = False
 293                                 else:
 294                                     query = None
 295                                 if (
 296                                     ANYJS
 297                                     in failures[manifest][LL][label][PP][path][RUNS][
 298                                         task_id
 299                                     ]
 300                                 ):
 301                                     any_filename = os.path.basename(
 302                                         failures[manifest][LL][label][PP][path][RUNS][
 303                                             task_id
 304                                         ][ANYJS]
 305                                     )
 306                                     anyjs[any_filename] = False
 307                                     if query is not None:
 308                                         anyjs[any_filename + query] = False
 309                             self.skip_failure(
 310                                 manifest,
 311                                 path,
 312                                 anyjs,
 313                                 label,
 314                                 classification,
 315                                 task_id,
 316                                 try_url,
 317                                 revision,
 318                                 repo,
 319                                 meta_bug_id,
 320                             )
 321                             num_failures += 1
 322                             if max_failures >= 0 and num_failures >= max_failures:
 323                                 self.warning(
 324                                     f"max_failures={max_failures} threshold reached. stopping."
 325                                 )
 326                                 return True
 327             elif manifest.endswith(".ini"):
 328                 self.warning(f"cannot process skip-fails on INI manifest: {manifest}")
 329             elif manifest.endswith(".list"):
 330                 self.warning(f"cannot process skip-fails on LIST manifest: {manifest}")
 331             else:
 332                 self.warning(
 333                     f"cannot process skip-fails on unknown manifest: {manifest}"
 334                 )
 335         return True
 336
 337     def get_revision(self, url):
 338         parsed = urllib.parse.urlparse(url)
 339         if parsed.scheme != "https":
 340             raise ValueError("try_url scheme not https")
 341         if parsed.netloc != Skipfails.TREEHERDER:
 342             raise ValueError(f"try_url server not {Skipfails.TREEHERDER}")
 343         if len(parsed.query) == 0:
 344             raise ValueError("try_url query missing")
 345         query = urllib.parse.parse_qs(parsed.query)
 346         if Skipfails.REVISION not in query:
 347             raise ValueError("try_url query missing revision")
 348         revision = query[Skipfails.REVISION][0]
 349         if Skipfails.REPO in query:
 350             repo = query[Skipfails.REPO][0]
 351         else:
 352             repo = "try"
 353         self.vinfo(f"considering {repo} revision={revision}")
 354         return revision, repo
 355
 356     def get_tasks(self, revision, repo):
 357         push = mozci.push.Push(revision, repo)
 358         return push.tasks
 359
 360     def get_failures(self, tasks):
 361         """
 362         find failures and create structure comprised of runs by path:
 363            result:
 364             * False (failed)
 365             * True (passed)
 366            classification: Classification
 367             * unknown (default) < 3 runs
 368             * intermittent (not enough failures)
 369             * disable_recommended (enough repeated failures) >3 runs >= 4
 370             * disable_manifest (disable DEFAULT if no other failures)
 371             * secondary (not first failure in group)
 372             * success
 373         """
 374
 375         ff = {}
 376         manifest_paths = {}
 377         manifest_ = {
 378             WPT: False,
 379             LL: {},
 380         }
 381         label_ = {
 382             DURATIONS: {},
 383             MEDIAN_DURATION: 0,
 384             OPT: False,
 385             PP: {},
 386             SUM_BY_LABEL: {
 387                 Classification.DISABLE_MANIFEST: 0,
 388                 Classification.DISABLE_RECOMMENDED: 0,
 389                 Classification.DISABLE_TOO_LONG: 0,
 390                 Classification.INTERMITTENT: 0,
 391                 Classification.SECONDARY: 0,
 392                 Classification.SUCCESS: 0,
 393                 Classification.UNKNOWN: 0,
 394             },
 395             TOTAL_DURATION: 0,
 396         }
 397         path_ = {
 398             CC: Classification.UNKNOWN,
 399             FAILED_RUNS: 0,
 400             RUNS: {},
 401             TOTAL_RUNS: 0,
 402         }
 403         run_ = {
 404             RR: False,
 405         }
 406
 407         for task in tasks:  # add explicit failures
 408             try:
 409                 if len(task.results) == 0:
 410                     continue  # ignore aborted tasks
 411                 for manifest in task.failure_types:
 412                     mm = manifest
 413                     ll = task.label
 414                     wpt = False
 415                     if mm.endswith(".ini"):
 416                         self.warning(
 417                             f"cannot analyze skip-fails on INI manifests: {mm}"
 418                         )
 419                         continue
 420                     elif mm.endswith(".list"):
 421                         self.warning(
 422                             f"cannot analyze skip-fails on LIST manifests: {mm}"
 423                         )
 424                         continue
 425                     elif not mm.endswith(".toml"):
 426                         path, mm, _query, _anyjs = self.wpt_paths(mm)
 427                         if path is None:  # not WPT
 428                             self.warning(
 429                                 f"cannot analyze skip-fails on unknown manifest type: {manifest}"
 430                             )
 431                             continue
 432                         wpt = True
 433                     if not wpt:
 434                         if mm not in manifest_paths:
 435                             manifest_paths[mm] = []
 436                         if mm not in ff:
 437                             ff[mm] = deepcopy(manifest_)
 438                             ff[mm][WPT] = wpt
 439                         if ll not in ff[mm][LL]:
 440                             ff[mm][LL][ll] = deepcopy(label_)
 441                     for path_type in task.failure_types[manifest]:
 442                         path, _type = path_type
 443                         if wpt:
 444                             path, mmpath, query, anyjs = self.wpt_paths(path)
 445                             if path is None:
 446                                 self.warning(
 447                                     f"non existant failure path: {path_type[0]}"
 448                                 )
 449                                 break
 450                             mm = os.path.dirname(mmpath)
 451                             if mm not in manifest_paths:
 452                                 manifest_paths[mm] = []
 453                             if mm not in ff:
 454                                 ff[mm] = deepcopy(manifest_)
 455                                 ff[mm][WPT] = wpt
 456                             if ll not in ff[mm][LL]:
 457                                 ff[mm][LL][ll] = deepcopy(label_)
 458                         else:
 459                             query = None
 460                             anyjs = None
 461                         if not wpt and path == mm:
 462                             path = DEF  # refers to the manifest itself
 463                         if path not in manifest_paths[mm]:
 464                             manifest_paths[mm].append(path)
 465                         if path not in ff[mm][LL][ll][PP]:
 466                             ff[mm][LL][ll][PP][path] = deepcopy(path_)
 467                         if task.id not in ff[mm][LL][ll][PP][path][RUNS]:
 468                             ff[mm][LL][ll][PP][path][RUNS][task.id] = deepcopy(run_)
 469                             ff[mm][LL][ll][PP][path][RUNS][task.id][RR] = False
 470                             if query is not None:
 471                                 ff[mm][LL][ll][PP][path][RUNS][task.id][QUERY] = query
 472                             if anyjs is not None:
 473                                 ff[mm][LL][ll][PP][path][RUNS][task.id][ANYJS] = anyjs
 474                             ff[mm][LL][ll][PP][path][TOTAL_RUNS] += 1
 475                             ff[mm][LL][ll][PP][path][FAILED_RUNS] += 1
 476             except AttributeError as ae:
 477                 self.warning(f"unknown attribute in task (#1): {ae}")
 478
 479         for task in tasks:  # add results
 480             try:
 481                 if len(task.results) == 0:
 482                     continue  # ignore aborted tasks
 483                 for result in task.results:
 484                     wpt = False
 485                     mm = result.group
 486                     if mm.endswith(".ini"):
 487                         self.warning(
 488                             f"cannot analyze skip-fails on INI manifests: {mm}"
 489                         )
 490                         continue
 491                     elif mm.endswith(".list"):
 492                         self.warning(
 493                             f"cannot analyze skip-fails on LIST manifests: {mm}"
 494                         )
 495                         continue
 496                     elif not mm.endswith(".toml"):
 497                         path, mm, _query, _anyjs = self.wpt_paths(mm)
 498                         if path is None:  # not WPT
 499                             self.warning(
 500                                 f"cannot analyze skip-fails on unknown manifest type: {result.group}"
 501                             )
 502                             continue
 503                         wpt = True
 504                     if mm not in manifest_paths:
 505                         continue
 506                     if mm not in ff:
 507                         ff[mm] = deepcopy(manifest_)
 508                     ll = task.label
 509                     if ll not in ff[mm][LL]:
 510                         ff[mm][LL][ll] = deepcopy(label_)
 511                     if task.id not in ff[mm][LL][ll][DURATIONS]:
 512                         # duration may be None !!!
 513                         ff[mm][LL][ll][DURATIONS][task.id] = result.duration or 0
 514                         if ff[mm][LL][ll][OPT] is None:
 515                             ff[mm][LL][ll][OPT] = self.get_opt_for_task(task.id)
 516                     for path in manifest_paths[mm]:  # all known paths
 517                         if path not in ff[mm][LL][ll][PP]:
 518                             ff[mm][LL][ll][PP][path] = deepcopy(path_)
 519                         if task.id not in ff[mm][LL][ll][PP][path][RUNS]:
 520                             ff[mm][LL][ll][PP][path][RUNS][task.id] = deepcopy(run_)
 521                             ff[mm][LL][ll][PP][path][RUNS][task.id][RR] = result.ok
 522                             ff[mm][LL][ll][PP][path][TOTAL_RUNS] += 1
 523                             if not result.ok:
 524                                 ff[mm][LL][ll][PP][path][FAILED_RUNS] += 1
 525             except AttributeError as ae:
 526                 self.warning(f"unknown attribute in task (#2): {ae}")
 527
 528         for mm in ff:  # determine classifications
 529             wpt = ff[mm][WPT]
 530             for label in ff[mm][LL]:
 531                 ll = label
 532                 opt = ff[mm][LL][ll][OPT]
 533                 durations = []  # summarize durations
 534                 for task_id in ff[mm][LL][ll][DURATIONS]:
 535                     duration = ff[mm][LL][ll][DURATIONS][task_id]
 536                     durations.append(duration)
 537                 if len(durations) > 0:
 538                     total_duration = sum(durations)
 539                     median_duration = median(durations)
 540                     ff[mm][LL][ll][TOTAL_DURATION] = total_duration
 541                     ff[mm][LL][ll][MEDIAN_DURATION] = median_duration
 542                     if (opt and median_duration > OPT_THRESHOLD) or (
 543                         (not opt) and median_duration > DEBUG_THRESHOLD
 544                     ):
 545                         if wpt:
 546                             paths = ff[mm][LL][ll][PP].keys()
 547                         else:
 548                             paths = [DEF]
 549                         for path in paths:
 550                             if path not in ff[mm][LL][ll][PP]:
 551                                 ff[mm][LL][ll][PP][path] = deepcopy(path_)
 552                             if task_id not in ff[mm][LL][ll][PP][path][RUNS]:
 553                                 ff[mm][LL][ll][PP][path][RUNS][task.id] = deepcopy(run_)
 554                                 ff[mm][LL][ll][PP][path][RUNS][task.id][RR] = False
 555                                 ff[mm][LL][ll][PP][path][TOTAL_RUNS] += 1
 556                                 ff[mm][LL][ll][PP][path][FAILED_RUNS] += 1
 557                             ff[mm][LL][ll][PP][path][
 558                                 CC
 559                             ] = Classification.DISABLE_TOO_LONG
 560                 primary = True  # we have not seen the first failure
 561                 for path in sort_paths(ff[mm][LL][ll][PP]):
 562                     classification = ff[mm][LL][ll][PP][path][CC]
 563                     if classification == Classification.UNKNOWN:
 564                         failed_runs = ff[mm][LL][ll][PP][path][FAILED_RUNS]
 565                         total_runs = ff[mm][LL][ll][PP][path][TOTAL_RUNS]
 566                         if total_runs >= MINIMUM_RUNS:
 567                             if failed_runs / total_runs < FAILURE_RATIO:
 568                                 if failed_runs == 0:
 569                                     classification = Classification.SUCCESS
 570                                 else:
 571                                     classification = Classification.INTERMITTENT
 572                             elif primary:
 573                                 if path == DEF:
 574                                     classification = Classification.DISABLE_MANIFEST
 575                                 else:
 576                                     classification = Classification.DISABLE_RECOMMENDED
 577                                 primary = False
 578                             else:
 579                                 classification = Classification.SECONDARY
 580                         ff[mm][LL][ll][PP][path][CC] = classification
 581                     ff[mm][LL][ll][SUM_BY_LABEL][classification] += 1
 582         return ff
 583
 584     def _get_os_version(self, os, platform):
 585         """Return the os_version given the label platform string"""
 586         i = platform.find(os)
 587         j = i + len(os)
 588         yy = platform[j : j + 2]
 589         mm = platform[j + 2 : j + 4]
 590         return yy + "." + mm
 591
 592     def get_bug_by_id(self, id):
 593         """Get bug by bug id"""
 594
 595         bug = None
 596         for b in self.bugs:
 597             if b.id == id:
 598                 bug = b
 599                 break
 600         if bug is None and self._initialize_bzapi():
 601             bug = self._bzapi.getbug(id)
 602         return bug
 603
 604     def get_bugs_by_summary(self, summary):
 605         """Get bug by bug summary"""
 606
 607         bugs = []
 608         for b in self.bugs:
 609             if b.summary == summary:
 610                 bugs.append(b)
 611         if len(bugs) > 0:
 612             return bugs
 613         if self._initialize_bzapi():
 614             query = self._bzapi.build_query(short_desc=summary)
 615             query["include_fields"] = [
 616                 "id",
 617                 "product",
 618                 "component",
 619                 "status",
 620                 "resolution",
 621                 "summary",
 622                 "blocks",
 623             ]
 624             bugs = self._bzapi.query(query)
 625         return bugs
 626
 627     def create_bug(
 628         self,
 629         summary="Bug short description",
 630         description="Bug description",
 631         product="Testing",
 632         component="General",
 633         version="unspecified",
 634         bugtype="task",
 635     ):
 636         """Create a bug"""
 637
 638         bug = None
 639         if self._initialize_bzapi():
 640             if not self._bzapi.logged_in:
 641                 self.error(
 642                     "Must create a Bugzilla API key per https://github.com/mozilla/mozci-tools/blob/main/citools/test_triage_bug_filer.py"
 643                 )
 644                 raise PermissionError(f"Not authenticated for Bugzilla {self.bugzilla}")
 645             createinfo = self._bzapi.build_createbug(
 646                 product=product,
 647                 component=component,
 648                 summary=summary,
 649                 version=version,
 650                 description=description,
 651             )
 652             createinfo["type"] = bugtype
 653             bug = self._bzapi.createbug(createinfo)
 654         return bug
 655
 656     def add_bug_comment(self, id, comment, meta_bug_id=None):
 657         """Add a comment to an existing bug"""
 658
 659         if self._initialize_bzapi():
 660             if not self._bzapi.logged_in:
 661                 self.error(BUGZILLA_AUTHENTICATION_HELP)
 662                 raise PermissionError("Not authenticated for Bugzilla")
 663             if meta_bug_id is not None:
 664                 blocks_add = [meta_bug_id]
 665             else:
 666                 blocks_add = None
 667             updateinfo = self._bzapi.build_update(
 668                 comment=comment, blocks_add=blocks_add
 669             )
 670             self._bzapi.update_bugs([id], updateinfo)
 671
 672     def skip_failure(
 673         self,
 674         manifest,
 675         path,
 676         anyjs,
 677         label,
 678         classification,
 679         task_id,
 680         try_url,
 681         revision,
 682         repo,
 683         meta_bug_id=None,
 684     ):
 685         """
 686         Skip a failure (for TOML and WPT manifests)
 687         For wpt anyjs is a dictionary mapping from alternate basename to
 688         a boolean (indicating if the basename has been handled in the manifest)
 689         """
 690
 691         if manifest.endswith(".toml"):
 692             wpt = False
 693             filename = DEF
 694         else:
 695             wpt = True
 696             _path, manifest, _query, _anyjs = self.wpt_paths(path)
 697             filename = os.path.basename(path)
 698         self.vinfo(f"===== Skip failure in manifest: {manifest} =====")
 699         if task_id is None:
 700             skip_if = "true"
 701         else:
 702             skip_if = self.task_to_skip_if(task_id, wpt)
 703         if skip_if is None:
 704             self.warning(
 705                 f"Unable to calculate skip-if condition from manifest={manifest} from failure label={label}"
 706             )
 707             return
 708         bug_reference = ""
 709         if classification == Classification.DISABLE_MANIFEST:
 710             comment = "Disabled entire manifest due to crash result"
 711         elif classification == Classification.DISABLE_TOO_LONG:
 712             comment = "Disabled entire manifest due to excessive run time"
 713         else:
 714             if not wpt:
 715                 filename = self.get_filename_in_manifest(manifest, path)
 716             comment = f'Disabled test due to failures in test file: "{filename}"'
 717             if classification == Classification.SECONDARY:
 718                 comment += " (secondary)"
 719                 if not wpt:
 720                     bug_reference = " (secondary)"
 721         if wpt and len(anyjs) > 1:
 722             comment += "\nAdditional WPT wildcard paths:"
 723             for p in sorted(anyjs.keys()):
 724                 if p != filename:
 725                     comment += f'\n  "{p}"'
 726         platform, testname = self.label_to_platform_testname(label)
 727         if platform is not None:
 728             comment += "\nCommand line to reproduce (experimental):\n"
 729             comment += f"  \"mach try fuzzy -q '{platform}' {testname}\"\n"
 730         comment += f"\nTry URL = {try_url}"
 731         comment += f"\nrevision = {revision}"
 732         comment += f"\nrepo = {repo}"
 733         comment += f"\nlabel = {label}"
 734         if task_id is not None:
 735             comment += f"\ntask_id = {task_id}"
 736             push_id = self.get_push_id(revision, repo)
 737             if push_id is not None:
 738                 comment += f"\npush_id = {push_id}"
 739                 job_id = self.get_job_id(push_id, task_id)
 740                 if job_id is not None:
 741                     comment += f"\njob_id = {job_id}"
 742                     (
 743                         suggestions_url,
 744                         line_number,
 745                         line,
 746                         log_url,
 747                     ) = self.get_bug_suggestions(repo, job_id, path, anyjs)
 748                     if log_url is not None:
 749                         comment += f"\nBug suggestions: {suggestions_url}"
 750                         comment += f"\nSpecifically see at line {line_number} in the attached log: {log_url}"
 751                         comment += f'\n\n  "{line}"\n'
 752         bug_summary = f"MANIFEST {manifest}"
 753         attachments = {}
 754         bugid = "TBD"
 755         if self.bugzilla is None:
 756             self.warning("Bugzilla has been disabled: no bugs created or updated")
 757         else:
 758             bugs = self.get_bugs_by_summary(bug_summary)
 759             if len(bugs) == 0:
 760                 description = (
 761                     f"This bug covers excluded failing tests in the MANIFEST {manifest}"
 762                 )
 763                 description += "\n(generated by `mach manifest skip-fails`)"
 764                 product, component = self.get_file_info(path)
 765                 if self.dry_run:
 766                     self.warning(
 767                         f'Dry-run NOT creating bug: {product}::{component} "{bug_summary}"'
 768                     )
 769                 else:
 770                     bug = self.create_bug(bug_summary, description, product, component)
 771                     bugid = bug.id
 772                     self.vinfo(
 773                         f'Created Bug {bugid} {product}::{component} : "{bug_summary}"'
 774                     )
 775             elif len(bugs) == 1:
 776                 bugid = bugs[0].id
 777                 product = bugs[0].product
 778                 component = bugs[0].component
 779                 self.vinfo(f'Found Bug {bugid} {product}::{component} "{bug_summary}"')
 780                 if meta_bug_id is not None:
 781                     if meta_bug_id in bugs[0].blocks:
 782                         self.vinfo(
 783                             f"  Bug {bugid} already blocks meta bug {meta_bug_id}"
 784                         )
 785                         meta_bug_id = None  # no need to add again
 786                 comments = bugs[0].getcomments()
 787                 for i in range(len(comments)):
 788                     text = comments[i]["text"]
 789                     m = self._attach_rx.findall(text)
 790                     if len(m) == 1:
 791                         a_task_id = m[0][1]
 792                         attachments[a_task_id] = m[0][0]
 793                         if a_task_id == task_id:
 794                             self.vinfo(
 795                                 f"  Bug {bugid} already has the compressed log attached for this task"
 796                             )
 797             else:
 798                 self.error(f'More than one bug found for summary: "{bug_summary}"')
 799                 return
 800         bug_reference = f"Bug {bugid}" + bug_reference
 801         comment += f"\nskip-if condition: {skip_if} # {bug_reference}"
 802         if self.dry_run:
 803             self.warning(f"Dry-run NOT adding comment to Bug {bugid}:\n{comment}")
 804             self.info(f'Dry-run NOT editing ["{filename}"] manifest: "{manifest}"')
 805             self.info(f'would add skip-if condition: "{skip_if}" # {bug_reference}')
 806             if task_id is not None and task_id not in attachments:
 807                 self.info("would add compressed log for this task")
 808             return
 809         elif self.bugzilla is None:
 810             self.warning(f"NOT adding comment to Bug {bugid}:\n{comment}")
 811         else:
 812             self.add_bug_comment(bugid, comment, meta_bug_id)
 813             self.info(f"Added comment to Bug {bugid}:\n{comment}")
 814             if meta_bug_id is not None:
 815                 self.info(f"  Bug {bugid} blocks meta Bug: {meta_bug_id}")
 816             if task_id is not None and task_id not in attachments:
 817                 self.add_attachment_log_for_task(bugid, task_id)
 818                 self.info("Added compressed log for this task")
 819         manifest_path = self.full_path(manifest)
 820         if wpt:
 821             if os.path.exists(manifest_path):
 822                 manifest_str = io.open(manifest_path, "r", encoding="utf-8").read()
 823             else:
 824                 manifest_str = ""
 825                 # ensure parent directories exist
 826                 os.makedirs(os.path.dirname(manifest_path), exist_ok=True)
 827             manifest_str = self.wpt_add_skip_if(
 828                 manifest_str, anyjs, skip_if, bug_reference
 829             )
 830         else:
 831             mp = ManifestParser(use_toml=True, document=True)
 832             mp.read(manifest_path)
 833             document = mp.source_documents[manifest_path]
 834             add_skip_if(document, filename, skip_if, bug_reference)
 835             manifest_str = alphabetize_toml_str(document)
 836         fp = io.open(manifest_path, "w", encoding="utf-8", newline="\n")
 837         fp.write(manifest_str)
 838         fp.close()
 839         self.info(f'Edited ["{filename}"] in manifest: "{manifest}"')
 840         self.info(f'added skip-if condition: "{skip_if}" # {bug_reference}')
 841
 842     def get_variants(self):
 843         """Get mozinfo for each test variants"""
 844
 845         if len(self.variants) == 0:
 846             variants_file = "taskcluster/ci/test/variants.yml"
 847             variants_path = self.full_path(variants_file)
 848             fp = io.open(variants_path, "r", encoding="utf-8")
 849             raw_variants = load(fp, Loader=Loader)
 850             fp.close()
 851             for k, v in raw_variants.items():
 852                 mozinfo = k
 853                 if "mozinfo" in v:
 854                     mozinfo = v["mozinfo"]
 855                 self.variants[k] = mozinfo
 856         return self.variants
 857
 858     def get_task_details(self, task_id):
 859         """Download details for task task_id"""
 860
 861         if task_id in self.tasks:  # if cached
 862             task = self.tasks[task_id]
 863         else:
 864             try:
 865                 task = get_task(task_id)
 866             except TaskclusterRestFailure:
 867                 self.warning(f"Task {task_id} no longer exists.")
 868                 return None
 869             self.tasks[task_id] = task
 870         return task
 871
 872     def get_extra(self, task_id):
 873         """Calculate extra for task task_id"""
 874
 875         if task_id in self.extras:  # if cached
 876             extra = self.extras[task_id]
 877         else:
 878             self.get_variants()
 879             task = self.get_task_details(task_id) or {}
 880             arch = None
 881             bits = None
 882             build = None
 883             build_types = []
 884             display = None
 885             os = None
 886             os_version = None
 887             runtimes = []
 888             test_setting = task.get("extra", {}).get("test-setting", {})
 889             platform = test_setting.get("platform", {})
 890             platform_os = platform.get("os", {})
 891             opt = False
 892             debug = False
 893             if "name" in platform_os:
 894                 os = platform_os["name"]
 895                 if os == "windows":
 896                     os = "win"
 897                 if os == "macosx":
 898                     os = "mac"
 899             if "version" in platform_os:
 900                 os_version = platform_os["version"]
 901                 if len(os_version) == 4:
 902                     os_version = os_version[0:2] + "." + os_version[2:4]
 903             if "build" in platform_os:
 904                 build = platform_os["build"]
 905             if "arch" in platform:
 906                 arch = platform["arch"]
 907                 if arch == "x86" or arch.find("32") >= 0:
 908                     bits = "32"
 909                     arch = "x86"
 910                 else:
 911                     bits = "64"
 912                     if arch != "aarch64" and arch != "ppc":
 913                         arch = "x86_64"
 914             if "display" in platform:
 915                 display = platform["display"]
 916             if "runtime" in test_setting:
 917                 for k in test_setting["runtime"]:
 918                     if k in self.variants:
 919                         runtimes.append(self.variants[k])  # adds mozinfo
 920             if "build" in test_setting:
 921                 tbuild = test_setting["build"]
 922                 for k in tbuild:
 923                     if k == "type":
 924                         if tbuild[k] == "opt":
 925                             opt = True
 926                         elif tbuild[k] == "debug":
 927                             debug = True
 928                         build_types.append(tbuild[k])
 929                     else:
 930                         build_types.append(k)
 931             unknown = None
 932             extra = {
 933                 "arch": arch or unknown,
 934                 "bits": bits or unknown,
 935                 "build": build or unknown,
 936                 "build_types": build_types,
 937                 "debug": debug,
 938                 "display": display or unknown,
 939                 "opt": opt,
 940                 "os": os or unknown,
 941                 "os_version": os_version or unknown,
 942                 "runtimes": runtimes,
 943             }
 944         self.extras[task_id] = extra
 945         return extra
 946
 947     def get_opt_for_task(self, task_id):
 948         extra = self.get_extra(task_id)
 949         return extra["opt"]
 950
 951     def task_to_skip_if(self, task_id, wpt=False):
 952         """Calculate the skip-if condition for failing task task_id"""
 953
 954         if wpt:
 955             qq = '"'
 956             aa = " and "
 957             nn = "not "
 958         else:
 959             qq = "'"
 960             aa = " && "
 961             nn = "!"
 962         eq = " == "
 963         arch = "processor"
 964         version = "os_version"
 965         extra = self.get_extra(task_id)
 966         skip_if = None
 967         if extra["os"] is not None:
 968             if extra["os_version"] is not None:
 969                 skip_if = "os" + eq + qq + extra["os"] + qq
 970                 if (
 971                     extra["build"] is not None
 972                     and extra["os"] == "win"
 973                     and extra["os_version"] == "11"
 974                     and extra["build"] == "2009"
 975                 ):
 976                     skip_if = "win11_2009"  # mozinfo.py:137
 977                 else:
 978                     skip_if += aa + version + eq + qq + extra["os_version"] + qq
 979             if extra["arch"] is not None:
 980                 skip_if += aa + arch + eq + qq + extra["arch"] + qq
 981             # since we always give arch/processor, bits are not required
 982             # if extra["bits"] is not None:
 983             #     skip_if += aa + "bits" + eq + extra["bits"]
 984             if extra["debug"]:
 985                 skip_if += aa + "debug"
 986             else:
 987                 skip_if += aa + nn + "debug"
 988             if extra["display"] is not None:
 989                 skip_if += aa + "display" + eq + qq + extra["display"] + qq
 990             for runtime in extra["runtimes"]:
 991                 skip_if += aa + runtime
 992             for build_type in extra["build_types"]:
 993                 # note: lite will not evaluate on non-android platforms
 994                 if build_type not in ["debug", "lite", "opt", "shippable"]:
 995                     skip_if += aa + build_type
 996         return skip_if
 997
 998     def get_file_info(self, path, product="Testing", component="General"):
 999         """
1000         Get bugzilla product and component for the path.
1001         Provide defaults (in case command_context is not defined
1002         or there isn't file info available).
1003         """
1004         if path != DEF and self.command_context is not None:
1005             reader = self.command_context.mozbuild_reader(config_mode="empty")
1006             info = reader.files_info([path])
1007             cp = info[path]["BUG_COMPONENT"]
1008             product = cp.product
1009             component = cp.component
1010         return product, component
1011
1012     def get_filename_in_manifest(self, manifest, path):
1013         """return relative filename for path in manifest"""
1014
1015         filename = os.path.basename(path)
1016         if filename == DEF:
1017             return filename
1018         manifest_dir = os.path.dirname(manifest)
1019         i = 0
1020         j = min(len(manifest_dir), len(path))
1021         while i < j and manifest_dir[i] == path[i]:
1022             i += 1
1023         if i < len(manifest_dir):
1024             for _ in range(manifest_dir.count("/", i) + 1):
1025                 filename = "../" + filename
1026         elif i < len(path):
1027             filename = path[i + 1 :]
1028         return filename
1029
1030     def get_push_id(self, revision, repo):
1031         """Return the push_id for revision and repo (or None)"""
1032
1033         self.vinfo(f"Retrieving push_id for {repo} revision: {revision} ...")
1034         if revision in self.push_ids:  # if cached
1035             push_id = self.push_ids[revision]
1036         else:
1037             push_id = None
1038             push_url = f"https://treeherder.mozilla.org/api/project/{repo}/push/"
1039             params = {}
1040             params["full"] = "true"
1041             params["count"] = 10
1042             params["revision"] = revision
1043             r = requests.get(push_url, headers=self.headers, params=params)
1044             if r.status_code != 200:
1045                 self.warning(f"FAILED to query Treeherder = {r} for {r.url}")
1046             else:
1047                 response = r.json()
1048                 if "results" in response:
1049                     results = response["results"]
1050                     if len(results) > 0:
1051                         r0 = results[0]
1052                         if "id" in r0:
1053                             push_id = r0["id"]
1054             self.push_ids[revision] = push_id
1055         return push_id
1056
1057     def get_job_id(self, push_id, task_id):
1058         """Return the job_id for push_id, task_id (or None)"""
1059
1060         self.vinfo(f"Retrieving job_id for push_id: {push_id}, task_id: {task_id} ...")
1061         k = f"{push_id}:{task_id}"
1062         if k in self.job_ids:  # if cached
1063             job_id = self.job_ids[k]
1064         else:
1065             job_id = None
1066             params = {}
1067             params["push_id"] = push_id
1068             r = requests.get(self.jobs_url, headers=self.headers, params=params)
1069             if r.status_code != 200:
1070                 self.warning(f"FAILED to query Treeherder = {r} for {r.url}")
1071             else:
1072                 response = r.json()
1073                 if "results" in response:
1074                     results = response["results"]
1075                     if len(results) > 0:
1076                         for result in results:
1077                             if len(result) > 14:
1078                                 if result[14] == task_id:
1079                                     job_id = result[1]
1080                                     break
1081             self.job_ids[k] = job_id
1082         return job_id
1083
1084     def get_bug_suggestions(self, repo, job_id, path, anyjs=None):
1085         """
1086         Return the (suggestions_url, line_number, line, log_url)
1087         for the given repo and job_id
1088         """
1089         self.vinfo(
1090             f"Retrieving bug_suggestions for {repo} job_id: {job_id}, path: {path} ..."
1091         )
1092         suggestions_url = f"https://treeherder.mozilla.org/api/project/{repo}/jobs/{job_id}/bug_suggestions/"
1093         line_number = None
1094         line = None
1095         log_url = None
1096         r = requests.get(suggestions_url, headers=self.headers)
1097         if r.status_code != 200:
1098             self.warning(f"FAILED to query Treeherder = {r} for {r.url}")
1099         else:
1100             if anyjs is not None:
1101                 pathdir = os.path.dirname(path) + "/"
1102                 paths = [pathdir + f for f in anyjs.keys()]
1103             else:
1104                 paths = [path]
1105             response = r.json()
1106             if len(response) > 0:
1107                 for sugg in response:
1108                     for p in paths:
1109                         path_end = sugg.get("path_end", None)
1110                         # handles WPT short paths
1111                         if path_end is not None and p.endswith(path_end):
1112                             line_number = sugg["line_number"] + 1
1113                             line = sugg["search"]
1114                             log_url = f"https://treeherder.mozilla.org/logviewer?repo={repo}&job_id={job_id}&lineNumber={line_number}"
1115                             break
1116         rv = (suggestions_url, line_number, line, log_url)
1117         return rv
1118
1119     def read_json(self, filename):
1120         """read data as JSON from filename"""
1121
1122         fp = io.open(filename, "r", encoding="utf-8")
1123         data = json.load(fp)
1124         fp.close()
1125         return data
1126
1127     def read_tasks(self, filename):
1128         """read tasks as JSON from filename"""
1129
1130         if not os.path.exists(filename):
1131             msg = f"use-tasks JSON file does not exist: {filename}"
1132             raise OSError(2, msg, filename)
1133         tasks = self.read_json(filename)
1134         tasks = [Mock(task, MOCK_TASK_DEFAULTS, MOCK_TASK_INITS) for task in tasks]
1135         for task in tasks:
1136             if len(task.extra) > 0:  # pre-warm cache for extra information
1137                 self.extras[task.id] = task.extra
1138         return tasks
1139
1140     def read_failures(self, filename):
1141         """read failures as JSON from filename"""
1142
1143         if not os.path.exists(filename):
1144             msg = f"use-failures JSON file does not exist: {filename}"
1145             raise OSError(2, msg, filename)
1146         failures = self.read_json(filename)
1147         return failures
1148
1149     def read_bugs(self, filename):
1150         """read bugs as JSON from filename"""
1151
1152         if not os.path.exists(filename):
1153             msg = f"bugs JSON file does not exist: {filename}"
1154             raise OSError(2, msg, filename)
1155         bugs = self.read_json(filename)
1156         bugs = [Mock(bug, MOCK_BUG_DEFAULTS) for bug in bugs]
1157         return bugs
1158
1159     def write_json(self, filename, data):
1160         """saves data as JSON to filename"""
1161         fp = io.open(filename, "w", encoding="utf-8")
1162         json.dump(data, fp, indent=2, sort_keys=True)
1163         fp.close()
1164
1165     def write_tasks(self, save_tasks, tasks):
1166         """saves tasks as JSON to save_tasks"""
1167         jtasks = []
1168         for task in tasks:
1169             if not isinstance(task, TestTask):
1170                 continue
1171             jtask = {}
1172             jtask["id"] = task.id
1173             jtask["label"] = task.label
1174             jtask["duration"] = task.duration
1175             jtask["result"] = task.result
1176             jtask["state"] = task.state
1177             jtask["extra"] = self.get_extra(task.id)
1178             jtags = {}
1179             for k, v in task.tags.items():
1180                 if k == "createdForUser":
1181                     jtags[k] = "ci@mozilla.com"
1182                 else:
1183                     jtags[k] = v
1184             jtask["tags"] = jtags
1185             jtask["tier"] = task.tier
1186             jtask["results"] = [
1187                 {"group": r.group, "ok": r.ok, "duration": r.duration}
1188                 for r in task.results
1189             ]
1190             jtask["errors"] = None  # Bug with task.errors property??
1191             jft = {}
1192             for k in task.failure_types:
1193                 jft[k] = [[f[0], f[1].value] for f in task.failure_types[k]]
1194             jtask["failure_types"] = jft
1195             jtasks.append(jtask)
1196         self.write_json(save_tasks, jtasks)
1197
1198     def label_to_platform_testname(self, label):
1199         """convert from label to platform, testname for mach command line"""
1200         platform = None
1201         testname = None
1202         platform_details = label.split("/")
1203         if len(platform_details) == 2:
1204             platform, details = platform_details
1205             words = details.split("-")
1206             if len(words) > 2:
1207                 platform += "/" + words.pop(0)  # opt or debug
1208                 try:
1209                     _chunk = int(words[-1])
1210                     words.pop()
1211                 except ValueError:
1212                     pass
1213                 words.pop()  # remove test suffix
1214                 testname = "-".join(words)
1215             else:
1216                 platform = None
1217         return platform, testname
1218
1219     def add_attachment_log_for_task(self, bugid, task_id):
1220         """Adds compressed log for this task to bugid"""
1221
1222         log_url = f"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/{task_id}/artifacts/public/logs/live_backing.log"
1223         r = requests.get(log_url, headers=self.headers)
1224         if r.status_code != 200:
1225             self.error(f"Unable get log for task: {task_id}")
1226             return
1227         attach_fp = tempfile.NamedTemporaryFile()
1228         fp = gzip.open(attach_fp, "wb")
1229         fp.write(r.text.encode("utf-8"))
1230         fp.close()
1231         if self._initialize_bzapi():
1232             description = ATTACHMENT_DESCRIPTION + task_id
1233             file_name = TASK_LOG + ".gz"
1234             comment = "Added compressed log"
1235             content_type = "application/gzip"
1236             try:
1237                 self._bzapi.attachfile(
1238                     [bugid],
1239                     attach_fp.name,
1240                     description,
1241                     file_name=file_name,
1242                     comment=comment,
1243                     content_type=content_type,
1244                     is_private=False,
1245                 )
1246             except Fault:
1247                 pass  # Fault expected: Failed to fetch key 9372091 from network storage: The specified key does not exist.
1248
1249     def get_wpt_path_meta(self, shortpath):
1250         if shortpath.startswith(WPT0):
1251             path = shortpath
1252             meta = shortpath.replace(WPT0, WPT_META0, 1)
1253         elif shortpath.startswith(WPT1):
1254             path = shortpath
1255             meta = shortpath.replace(WPT1, WPT_META1, 1)
1256         elif shortpath.startswith(WPT2):
1257             path = shortpath
1258             meta = shortpath.replace(WPT2, WPT_META2, 1)
1259         elif shortpath.startswith(WPT_MOZILLA):
1260             shortpath = shortpath[len(WPT_MOZILLA) :]
1261             path = WPT2 + shortpath
1262             meta = WPT_META2 + shortpath
1263         else:
1264             path = WPT1 + shortpath
1265             meta = WPT_META1 + shortpath
1266         return (path, meta)
1267
1268     def wpt_paths(self, shortpath):
1269         """
1270         Analyzes the WPT short path for a test and returns
1271         (path, manifest, query, anyjs) where
1272         path is the relative path to the test file
1273         manifest is the relative path to the file metadata
1274         query is the test file query paramters (or None)
1275         anyjs is the html test file as reported by mozci (or None)
1276         """
1277         query = None
1278         anyjs = None
1279         i = shortpath.find("?")
1280         if i > 0:
1281             query = shortpath[i:]
1282             shortpath = shortpath[0:i]
1283         path, manifest = self.get_wpt_path_meta(shortpath)
1284         failure_type = not self.isdir(path)
1285         if failure_type:
1286             i = path.find(".any.")
1287             if i > 0:
1288                 anyjs = path  # orig path
1289                 manifest = manifest.replace(path[i:], ".any.js")
1290                 path = path[0:i] + ".any.js"
1291             else:
1292                 i = path.find(".window.")
1293                 if i > 0:
1294                     anyjs = path  # orig path
1295                     manifest = manifest.replace(path[i:], ".window.js")
1296                     path = path[0:i] + ".window.js"
1297                 else:
1298                     i = path.find(".worker.")
1299                     if i > 0:
1300                         anyjs = path  # orig path
1301                         manifest = manifest.replace(path[i:], ".worker.js")
1302                         path = path[0:i] + ".worker.js"
1303             manifest += ".ini"
1304         manifest_classic = ""
1305         if manifest.startswith(WPT_META0):
1306             manifest_classic = manifest.replace(WPT_META0, WPT_META0_CLASSIC, 1)
1307             if self.exists(manifest_classic):
1308                 if self.exists(manifest):
1309                     self.warning(
1310                         f"Both classic {manifest_classic} and metadata {manifest} manifests exist"
1311                     )
1312                 else:
1313                     self.warning(
1314                         f"Using the classic {manifest_classic} manifest as the metadata manifest {manifest} does not exist"
1315                     )
1316                     manifest = manifest_classic
1317         if not self.exists(path):
1318             return (None, None, None, None)
1319         return (path, manifest, query, anyjs)
1320
1321     def wpt_add_skip_if(self, manifest_str, anyjs, skip_if, bug_reference):
1322         """
1323         Edits a WPT manifest string to add disabled condition
1324         anyjs is a dictionary mapping from filename and any alternate basenames to
1325         a boolean (indicating if the file has been handled in the manifest)
1326         """
1327
1328         disabled_key = False
1329         disabled = "  disabled:"
1330         condition_start = "    if "
1331         condition = condition_start + skip_if + ": " + bug_reference
1332         lines = manifest_str.splitlines()
1333         section = None  # name of the section
1334         i = 0
1335         n = len(lines)
1336         while i < n:
1337             line = lines[i]
1338             if line.startswith("["):
1339                 if section is not None and not anyjs[section]:  # not yet handled
1340                     if not disabled_key:
1341                         lines.insert(i, disabled)
1342                         i += 1
1343                     lines.insert(i, condition)
1344                     lines.insert(i + 1, "")  # blank line after condition
1345                     i += 2
1346                     n += 2
1347                     anyjs[section] = True
1348                 section = line[1:-1]
1349                 if section in anyjs and not anyjs[section]:
1350                     disabled_key = False
1351                 else:
1352                     section = None  # ignore section we are not interested in
1353             elif section is not None:
1354                 if line == disabled:
1355                     disabled_key = True
1356                 elif line.startswith("  ["):
1357                     if i > 0 and i - 1 < n and lines[i - 1] == "":
1358                         del lines[i - 1]
1359                         i -= 1
1360                         n -= 1
1361                     if not disabled_key:
1362                         lines.insert(i, disabled)
1363                         i += 1
1364                         n += 1
1365                     lines.insert(i, condition)
1366                     lines.insert(i + 1, "")  # blank line after condition
1367                     i += 2
1368                     n += 2
1369                     anyjs[section] = True
1370                     section = None
1371                 elif line.startswith("  ") and not line.startswith("    "):
1372                     if disabled_key:  # insert condition above new key
1373                         lines.insert(i, condition)
1374                         i += 1
1375                         n += 1
1376                         anyjs[section] = True
1377                         section = None
1378                         disabled_key = False
1379                 elif line.startswith("    "):
1380                     if disabled_key and line == condition:
1381                         anyjs[section] = True  # condition already present
1382                         section = None
1383             i += 1
1384         if section is not None and not anyjs[section]:  # not yet handled
1385             if i > 0 and i - 1 < n and lines[i - 1] == "":
1386                 del lines[i - 1]
1387             if not disabled_key:
1388                 lines.append(disabled)
1389                 i += 1
1390                 n += 1
1391             lines.append(condition)
1392             lines.append("")  # blank line after condition
1393             i += 2
1394             n += 2
1395             anyjs[section] = True
1396         for section in anyjs:
1397             if not anyjs[section]:
1398                 if i > 0 and i - 1 < n and lines[i - 1] != "":
1399                     lines.append("")  # blank line before condition
1400                     i += 1
1401                     n += 1
1402                 lines.append("[" + section + "]")
1403                 lines.append(disabled)
1404                 lines.append(condition)
1405                 lines.append("")  # blank line after condition
1406                 i += 4
1407                 n += 4
1408         manifest_str = "\n".join(lines) + "\n"
1409         return manifest_str