testing/skipfails.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import gzip
   6 import io
   7 import json
   8 import logging
   9 import os
  10 import os.path
  11 import pprint
  12 import re
  13 import sys
  14 import tempfile
  15 import urllib.parse
  16 from enum import Enum
  17 from pathlib import Path
  18 from xmlrpc.client import Fault
  19
  20 from yaml import load
  21
  22 try:
  23     from yaml import CLoader as Loader
  24 except ImportError:
  25     from yaml import Loader
  26
  27 import bugzilla
  28 import mozci.push
  29 import requests
  30 from manifestparser import ManifestParser
  31 from manifestparser.toml import add_skip_if, alphabetize_toml_str, sort_paths
  32 from mozci.task import TestTask
  33 from mozci.util.taskcluster import get_task
  34
  35 BUGZILLA_AUTHENTICATION_HELP = "Must create a Bugzilla API key per https://github.com/mozilla/mozci-tools/blob/main/citools/test_triage_bug_filer.py"
  36 TASK_LOG = "live_backing.log"
  37 TASK_ARTIFACT = "public/logs/" + TASK_LOG
  38 ATTACHMENT_DESCRIPTION = "Compressed " + TASK_ARTIFACT + " for task "
  39 ATTACHMENT_REGEX = (
  40     r".*Created attachment ([0-9]+)\n.*"
  41     + ATTACHMENT_DESCRIPTION
  42     + "([A-Za-z0-9_-]+)\n.*"
  43 )
  44
  45
  46 class MockResult(object):
  47     def __init__(self, result):
  48         self.result = result
  49
  50     @property
  51     def group(self):
  52         return self.result["group"]
  53
  54     @property
  55     def ok(self):
  56         _ok = self.result["ok"]
  57         return _ok
  58
  59
  60 class MockTask(object):
  61     def __init__(self, task):
  62         self.task = task
  63         if "results" in self.task:
  64             self.task["results"] = [
  65                 MockResult(result) for result in self.task["results"]
  66             ]
  67         else:
  68             self.task["results"] = []
  69
  70     @property
  71     def failure_types(self):
  72         if "failure_types" in self.task:
  73             return self.task["failure_types"]
  74         else:  # note no failure_types in Task object
  75             return {}
  76
  77     @property
  78     def id(self):
  79         return self.task["id"]
  80
  81     @property
  82     def label(self):
  83         return self.task["label"]
  84
  85     @property
  86     def results(self):
  87         return self.task["results"]
  88
  89
  90 class Classification(object):
  91     "Classification of the failure (not the task result)"
  92
  93     DISABLE_MANIFEST = "disable_manifest"  # crash found
  94     DISABLE_RECOMMENDED = "disable_recommended"  # disable first failing path
  95     INTERMITTENT = "intermittent"
  96     SECONDARY = "secondary"  # secondary failing path
  97     SUCCESS = "success"  # path always succeeds
  98     UNKNOWN = "unknown"
  99
 100
 101 class Run(Enum):
 102     """
 103     constant indexes for attributes of a run
 104     """
 105
 106     MANIFEST = 0
 107     TASK_ID = 1
 108     TASK_LABEL = 2
 109     RESULT = 3
 110     CLASSIFICATION = 4
 111
 112
 113 class Skipfails(object):
 114     "mach manifest skip-fails implementation: Update manifests to skip failing tests"
 115
 116     REPO = "repo"
 117     REVISION = "revision"
 118     TREEHERDER = "treeherder.mozilla.org"
 119     BUGZILLA_SERVER_DEFAULT = "bugzilla.allizom.org"
 120
 121     def __init__(
 122         self,
 123         command_context=None,
 124         try_url="",
 125         verbose=False,
 126         bugzilla=None,
 127         dry_run=False,
 128         turbo=False,
 129     ):
 130         self.command_context = command_context
 131         if self.command_context is not None:
 132             self.topsrcdir = self.command_context.topsrcdir
 133         else:
 134             self.topsrcdir = Path(__file__).parent.parent
 135         self.topsrcdir = os.path.normpath(self.topsrcdir)
 136         if isinstance(try_url, list) and len(try_url) == 1:
 137             self.try_url = try_url[0]
 138         else:
 139             self.try_url = try_url
 140         self.dry_run = dry_run
 141         self.verbose = verbose
 142         self.turbo = turbo
 143         if bugzilla is not None:
 144             self.bugzilla = bugzilla
 145         else:
 146             if "BUGZILLA" in os.environ:
 147                 self.bugzilla = os.environ["BUGZILLA"]
 148             else:
 149                 self.bugzilla = Skipfails.BUGZILLA_SERVER_DEFAULT
 150         self.component = "skip-fails"
 151         self._bzapi = None
 152         self._attach_rx = None
 153         self.variants = {}
 154         self.tasks = {}
 155         self.pp = None
 156         self.headers = {}  # for Treeherder requests
 157         self.headers["Accept"] = "application/json"
 158         self.headers["User-Agent"] = "treeherder-pyclient"
 159         self.jobs_url = "https://treeherder.mozilla.org/api/jobs/"
 160         self.push_ids = {}
 161         self.job_ids = {}
 162
 163     def _initialize_bzapi(self):
 164         """Lazily initializes the Bugzilla API"""
 165         if self._bzapi is None:
 166             self._bzapi = bugzilla.Bugzilla(self.bugzilla)
 167             self._attach_rx = re.compile(ATTACHMENT_REGEX, flags=re.M)
 168
 169     def pprint(self, obj):
 170         if self.pp is None:
 171             self.pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)
 172         self.pp.pprint(obj)
 173         sys.stderr.flush()
 174
 175     def error(self, e):
 176         if self.command_context is not None:
 177             self.command_context.log(
 178                 logging.ERROR, self.component, {"error": str(e)}, "ERROR: {error}"
 179             )
 180         else:
 181             print(f"ERROR: {e}", file=sys.stderr, flush=True)
 182
 183     def warning(self, e):
 184         if self.command_context is not None:
 185             self.command_context.log(
 186                 logging.WARNING, self.component, {"error": str(e)}, "WARNING: {error}"
 187             )
 188         else:
 189             print(f"WARNING: {e}", file=sys.stderr, flush=True)
 190
 191     def info(self, e):
 192         if self.command_context is not None:
 193             self.command_context.log(
 194                 logging.INFO, self.component, {"error": str(e)}, "INFO: {error}"
 195             )
 196         else:
 197             print(f"INFO: {e}", file=sys.stderr, flush=True)
 198
 199     def vinfo(self, e):
 200         if self.verbose:
 201             self.info(e)
 202
 203     def run(
 204         self,
 205         meta_bug_id=None,
 206         save_tasks=None,
 207         use_tasks=None,
 208         save_failures=None,
 209         use_failures=None,
 210         max_failures=-1,
 211     ):
 212         "Run skip-fails on try_url, return True on success"
 213
 214         try_url = self.try_url
 215         revision, repo = self.get_revision(try_url)
 216
 217         if use_tasks is not None:
 218             if os.path.exists(use_tasks):
 219                 self.vinfo(f"use tasks: {use_tasks}")
 220                 tasks = self.read_json(use_tasks)
 221                 tasks = [MockTask(task) for task in tasks]
 222             else:
 223                 self.error(f"uses tasks JSON file does not exist: {use_tasks}")
 224                 return False
 225         else:
 226             tasks = self.get_tasks(revision, repo)
 227
 228         if use_failures is not None:
 229             if os.path.exists(use_failures):
 230                 self.vinfo(f"use failures: {use_failures}")
 231                 failures = self.read_json(use_failures)
 232             else:
 233                 self.error(f"use failures JSON file does not exist: {use_failures}")
 234                 return False
 235         else:
 236             failures = self.get_failures(tasks)
 237             if save_failures is not None:
 238                 self.vinfo(f"save failures: {save_failures}")
 239                 self.write_json(save_failures, failures)
 240
 241         if save_tasks is not None:
 242             self.vinfo(f"save tasks: {save_tasks}")
 243             self.write_tasks(save_tasks, tasks)
 244
 245         num_failures = 0
 246         for manifest in failures:
 247             if not manifest.endswith(".toml"):
 248                 self.warning(f"cannot process skip-fails on INI manifests: {manifest}")
 249             else:
 250                 for path in failures[manifest]["path"]:
 251                     for label in failures[manifest]["path"][path]:
 252                         classification = failures[manifest]["path"][path][label][
 253                             "classification"
 254                         ]
 255                         if classification.startswith("disable_") or (
 256                             self.turbo and classification == Classification.SECONDARY
 257                         ):
 258                             for task_id in failures[manifest]["path"][path][label][
 259                                 "runs"
 260                             ].keys():
 261                                 self.skip_failure(
 262                                     manifest,
 263                                     path,
 264                                     label,
 265                                     classification,
 266                                     task_id,
 267                                     try_url,
 268                                     revision,
 269                                     repo,
 270                                     meta_bug_id,
 271                                 )
 272                                 num_failures += 1
 273                                 if max_failures >= 0 and num_failures >= max_failures:
 274                                     self.warning(
 275                                         f"max_failures={max_failures} threshold reached. stopping."
 276                                     )
 277                                     return True
 278                                 break  # just use the first task_id
 279         return True
 280
 281     def get_revision(self, url):
 282         parsed = urllib.parse.urlparse(url)
 283         if parsed.scheme != "https":
 284             raise ValueError("try_url scheme not https")
 285         if parsed.netloc != Skipfails.TREEHERDER:
 286             raise ValueError(f"try_url server not {Skipfails.TREEHERDER}")
 287         if len(parsed.query) == 0:
 288             raise ValueError("try_url query missing")
 289         query = urllib.parse.parse_qs(parsed.query)
 290         if Skipfails.REVISION not in query:
 291             raise ValueError("try_url query missing revision")
 292         revision = query[Skipfails.REVISION][0]
 293         if Skipfails.REPO in query:
 294             repo = query[Skipfails.REPO][0]
 295         else:
 296             repo = "try"
 297         self.vinfo(f"considering {repo} revision={revision}")
 298         return revision, repo
 299
 300     def get_tasks(self, revision, repo):
 301         push = mozci.push.Push(revision, repo)
 302         return push.tasks
 303
 304     def get_failures(self, tasks):
 305         """
 306         find failures and create structure comprised of runs by path:
 307            result:
 308             * False (failed)
 309             * True (passed)
 310            classification: Classification
 311             * unknown (default) < 3 runs
 312             * intermittent (not enough failures) >3 runs < 0.4 failure rate
 313             * disable_recommended (enough repeated failures) >3 runs >= 0.4
 314             * disable_manifest (disable DEFAULT if no other failures)
 315             * secondary (not first failure in group)
 316             * success
 317         """
 318
 319         failures = {}
 320         manifest_paths = {}
 321         for task in tasks:
 322             try:
 323                 if len(task.results) == 0:
 324                     continue  # ignore aborted tasks
 325                 for manifest in task.failure_types:
 326                     if manifest not in failures:
 327                         failures[manifest] = {"sum_by_label": {}, "path": {}}
 328                     if manifest not in manifest_paths:
 329                         manifest_paths[manifest] = []
 330                     for path_type in task.failure_types[manifest]:
 331                         path, _type = path_type
 332                         if path == manifest:
 333                             path = "DEFAULT"
 334                         if path not in failures[manifest]["path"]:
 335                             failures[manifest]["path"][path] = {}
 336                         if path not in manifest_paths[manifest]:
 337                             manifest_paths[manifest].append(path)
 338                         if task.label not in failures[manifest]["sum_by_label"]:
 339                             failures[manifest]["sum_by_label"][task.label] = {
 340                                 Classification.UNKNOWN: 0,
 341                                 Classification.SECONDARY: 0,
 342                                 Classification.INTERMITTENT: 0,
 343                                 Classification.DISABLE_RECOMMENDED: 0,
 344                                 Classification.DISABLE_MANIFEST: 0,
 345                                 Classification.SUCCESS: 0,
 346                             }
 347                         if task.label not in failures[manifest]["path"][path]:
 348                             failures[manifest]["path"][path][task.label] = {
 349                                 "total_runs": 0,
 350                                 "failed_runs": 0,
 351                                 "classification": Classification.UNKNOWN,
 352                                 "runs": {task.id: False},
 353                             }
 354                         else:
 355                             failures[manifest]["path"][path][task.label]["runs"][
 356                                 task.id
 357                             ] = False
 358             except AttributeError as ae:
 359                 self.warning(f"unknown attribute in task: {ae}")
 360
 361         # calculate success/failure for each known path
 362         for manifest in manifest_paths:
 363             manifest_paths[manifest] = sort_paths(manifest_paths[manifest])
 364         for task in tasks:
 365             try:
 366                 if len(task.results) == 0:
 367                     continue  # ignore aborted tasks
 368                 for result in task.results:
 369                     manifest = result.group
 370                     if manifest not in failures:
 371                         self.warning(
 372                             f"result for {manifest} not in any failures, ignored"
 373                         )
 374                         continue
 375                     for path in manifest_paths[manifest]:
 376                         if task.label not in failures[manifest]["sum_by_label"]:
 377                             failures[manifest]["sum_by_label"][task.label] = {
 378                                 Classification.UNKNOWN: 0,
 379                                 Classification.SECONDARY: 0,
 380                                 Classification.INTERMITTENT: 0,
 381                                 Classification.DISABLE_RECOMMENDED: 0,
 382                                 Classification.DISABLE_MANIFEST: 0,
 383                                 Classification.SUCCESS: 0,
 384                             }
 385                         if task.label not in failures[manifest]["path"][path]:
 386                             failures[manifest]["path"][path][task.label] = {
 387                                 "total_runs": 0,
 388                                 "failed_runs": 0,
 389                                 "classification": Classification.UNKNOWN,
 390                                 "runs": {},
 391                             }
 392                         if (
 393                             task.id
 394                             not in failures[manifest]["path"][path][task.label]["runs"]
 395                         ):
 396                             ok = True
 397                             failures[manifest]["path"][path][task.label]["runs"][
 398                                 task.id
 399                             ] = ok
 400                         else:
 401                             ok = (
 402                                 result.ok
 403                                 or failures[manifest]["path"][path][task.label]["runs"][
 404                                     task.id
 405                                 ]
 406                             )
 407                         failures[manifest]["path"][path][task.label]["total_runs"] += 1
 408                         if not ok:
 409                             failures[manifest]["path"][path][task.label][
 410                                 "failed_runs"
 411                             ] += 1
 412             except AttributeError as ae:
 413                 self.warning(f"unknown attribute in task: {ae}")
 414
 415         # classify failures and roll up summary statistics
 416         for manifest in failures:
 417             for path in failures[manifest]["path"]:
 418                 for label in failures[manifest]["path"][path]:
 419                     failed_runs = failures[manifest]["path"][path][label]["failed_runs"]
 420                     total_runs = failures[manifest]["path"][path][label]["total_runs"]
 421                     classification = failures[manifest]["path"][path][label][
 422                         "classification"
 423                     ]
 424                     if total_runs >= 3:
 425                         if failed_runs / total_runs < 0.4:
 426                             if failed_runs == 0:
 427                                 classification = Classification.SUCCESS
 428                             else:
 429                                 classification = Classification.INTERMITTENT
 430                         else:
 431                             classification = Classification.SECONDARY
 432                         failures[manifest]["path"][path][label][
 433                             "classification"
 434                         ] = classification
 435                     failures[manifest]["sum_by_label"][label][classification] += 1
 436
 437         # Identify the first failure (for each test, in a manifest, by label)
 438         for manifest in failures:
 439             alpha_paths = sort_paths(failures[manifest]["path"].keys())
 440             for path in alpha_paths:
 441                 for label in failures[manifest]["path"][path]:
 442                     primary = (
 443                         failures[manifest]["sum_by_label"][label][
 444                             Classification.DISABLE_RECOMMENDED
 445                         ]
 446                         == 0
 447                     )
 448                     if path == "DEFAULT":
 449                         classification = failures[manifest]["path"][path][label][
 450                             "classification"
 451                         ]
 452                         if (
 453                             classification == Classification.SECONDARY
 454                             and failures[manifest]["sum_by_label"][label][
 455                                 classification
 456                             ]
 457                             == 1
 458                         ):
 459                             # ONLY failure in the manifest for this label => DISABLE
 460                             failures[manifest]["path"][path][label][
 461                                 "classification"
 462                             ] = Classification.DISABLE_MANIFEST
 463                             failures[manifest]["sum_by_label"][label][
 464                                 classification
 465                             ] -= 1
 466                             failures[manifest]["sum_by_label"][label][
 467                                 Classification.DISABLE_MANIFEST
 468                             ] += 1
 469
 470                     else:
 471                         if (
 472                             primary
 473                             and failures[manifest]["path"][path][label][
 474                                 "classification"
 475                             ]
 476                             == Classification.SECONDARY
 477                         ):
 478                             # FIRST failure in the manifest for this label => DISABLE
 479                             failures[manifest]["path"][path][label][
 480                                 "classification"
 481                             ] = Classification.DISABLE_RECOMMENDED
 482                             failures[manifest]["sum_by_label"][label][
 483                                 Classification.SECONDARY
 484                             ] -= 1
 485                             failures[manifest]["sum_by_label"][label][
 486                                 Classification.DISABLE_RECOMMENDED
 487                             ] += 1
 488
 489         return failures
 490
 491     def _get_os_version(self, os, platform):
 492         """Return the os_version given the label platform string"""
 493         i = platform.find(os)
 494         j = i + len(os)
 495         yy = platform[j : j + 2]
 496         mm = platform[j + 2 : j + 4]
 497         return yy + "." + mm
 498
 499     def get_bug_by_id(self, id):
 500         """Get bug by bug id"""
 501
 502         self._initialize_bzapi()
 503         bug = self._bzapi.getbug(id)
 504         return bug
 505
 506     def get_bugs_by_summary(self, summary):
 507         """Get bug by bug summary"""
 508
 509         self._initialize_bzapi()
 510         query = self._bzapi.build_query(short_desc=summary)
 511         query["include_fields"] = [
 512             "id",
 513             "product",
 514             "component",
 515             "status",
 516             "resolution",
 517             "summary",
 518             "blocks",
 519         ]
 520         bugs = self._bzapi.query(query)
 521         return bugs
 522
 523     def create_bug(
 524         self,
 525         summary="Bug short description",
 526         description="Bug description",
 527         product="Testing",
 528         component="General",
 529         version="unspecified",
 530         bugtype="task",
 531     ):
 532         """Create a bug"""
 533
 534         self._initialize_bzapi()
 535         if not self._bzapi.logged_in:
 536             self.error(
 537                 "Must create a Bugzilla API key per https://github.com/mozilla/mozci-tools/blob/main/citools/test_triage_bug_filer.py"
 538             )
 539             raise PermissionError(f"Not authenticated for Bugzilla {self.bugzilla}")
 540         createinfo = self._bzapi.build_createbug(
 541             product=product,
 542             component=component,
 543             summary=summary,
 544             version=version,
 545             description=description,
 546         )
 547         createinfo["type"] = bugtype
 548         bug = self._bzapi.createbug(createinfo)
 549         return bug
 550
 551     def add_bug_comment(self, id, comment, meta_bug_id=None):
 552         """Add a comment to an existing bug"""
 553
 554         self._initialize_bzapi()
 555         if not self._bzapi.logged_in:
 556             self.error(BUGZILLA_AUTHENTICATION_HELP)
 557             raise PermissionError("Not authenticated for Bugzilla")
 558         if meta_bug_id is not None:
 559             blocks_add = [meta_bug_id]
 560         else:
 561             blocks_add = None
 562         updateinfo = self._bzapi.build_update(comment=comment, blocks_add=blocks_add)
 563         self._bzapi.update_bugs([id], updateinfo)
 564
 565     def skip_failure(
 566         self,
 567         manifest,
 568         path,
 569         label,
 570         classification,
 571         task_id,
 572         try_url,
 573         revision,
 574         repo,
 575         meta_bug_id=None,
 576     ):
 577         """Skip a failure"""
 578
 579         self.vinfo(f"===== Skip failure in manifest: {manifest} =====")
 580         skip_if = self.task_to_skip_if(task_id)
 581         if skip_if is None:
 582             self.warning(
 583                 f"Unable to calculate skip-if condition from manifest={manifest} from failure label={label}"
 584             )
 585             return
 586         bug_reference = ""
 587         if classification == Classification.DISABLE_MANIFEST:
 588             filename = "DEFAULT"
 589             comment = "Disabled entire manifest due to crash result"
 590         else:
 591             filename = self.get_filename_in_manifest(manifest, path)
 592             comment = f'Disabled test due to failures: "{filename}"'
 593             if classification == Classification.SECONDARY:
 594                 comment += " (secondary)"
 595                 bug_reference = " (secondary)"
 596         comment += f"\nTry URL = {try_url}"
 597         comment += f"\nrevision = {revision}"
 598         comment += f"\nrepo = {repo}"
 599         comment += f"\nlabel = {label}"
 600         comment += f"\ntask_id = {task_id}"
 601         push_id = self.get_push_id(revision, repo)
 602         if push_id is not None:
 603             comment += f"\npush_id = {push_id}"
 604             job_id = self.get_job_id(push_id, task_id)
 605             if job_id is not None:
 606                 comment += f"\njob_id = {job_id}"
 607                 suggestions_url, line_number, line, log_url = self.get_bug_suggestions(
 608                     repo, job_id, path
 609                 )
 610                 if log_url is not None:
 611                     comment += f"\n\nBug suggestions: {suggestions_url}"
 612                     comment += f"\nSpecifically see at line {line_number} in the attached log: {log_url}"
 613                     comment += f'\n\n  "{line}"\n'
 614         platform, testname = self.label_to_platform_testname(label)
 615         if platform is not None:
 616             comment += "\n\nCommand line to reproduce:\n\n"
 617             comment += f"  \"mach try fuzzy -q '{platform}' {testname}\""
 618         bug_summary = f"MANIFEST {manifest}"
 619         attachments = {}
 620         bugs = self.get_bugs_by_summary(bug_summary)
 621         if len(bugs) == 0:
 622             description = (
 623                 f"This bug covers excluded failing tests in the MANIFEST {manifest}"
 624             )
 625             description += "\n(generated by mach manifest skip-fails)"
 626             product, component = self.get_file_info(path)
 627             if self.dry_run:
 628                 self.warning(
 629                     f'Dry-run NOT creating bug: {product}::{component} "{bug_summary}"'
 630                 )
 631                 bugid = "TBD"
 632             else:
 633                 bug = self.create_bug(bug_summary, description, product, component)
 634                 bugid = bug.id
 635                 self.vinfo(
 636                     f'Created Bug {bugid} {product}::{component} : "{bug_summary}"'
 637                 )
 638             bug_reference = f"Bug {bugid}" + bug_reference
 639         elif len(bugs) == 1:
 640             bugid = bugs[0].id
 641             bug_reference = f"Bug {bugid}" + bug_reference
 642             product = bugs[0].product
 643             component = bugs[0].component
 644             self.vinfo(f'Found Bug {bugid} {product}::{component} "{bug_summary}"')
 645             if meta_bug_id is not None:
 646                 if meta_bug_id in bugs[0].blocks:
 647                     self.vinfo(f"  Bug {bugid} already blocks meta bug {meta_bug_id}")
 648                     meta_bug_id = None  # no need to add again
 649             comments = bugs[0].getcomments()
 650             for i in range(len(comments)):
 651                 text = comments[i]["text"]
 652                 m = self._attach_rx.findall(text)
 653                 if len(m) == 1:
 654                     a_task_id = m[0][1]
 655                     attachments[a_task_id] = m[0][0]
 656                     if a_task_id == task_id:
 657                         self.vinfo(
 658                             f"  Bug {bugid} already has the compressed log attached for this task"
 659                         )
 660         else:
 661             self.error(f'More than one bug found for summary: "{bug_summary}"')
 662             return
 663         if self.dry_run:
 664             self.warning(f"Dry-run NOT adding comment to Bug {bugid}: {comment}")
 665             self.info(f'Dry-run NOT editing ["{filename}"] manifest: "{manifest}"')
 666             self.info(f'would add skip-if condition: "{skip_if}" # {bug_reference}')
 667             if task_id not in attachments:
 668                 self.info("would add compressed log for this task")
 669             return
 670         self.add_bug_comment(bugid, comment, meta_bug_id)
 671         self.info(f"Added comment to Bug {bugid}: {comment}")
 672         if meta_bug_id is not None:
 673             self.info(f"  Bug {bugid} blocks meta Bug: {meta_bug_id}")
 674         if task_id not in attachments:
 675             self.add_attachment_log_for_task(bugid, task_id)
 676             self.info("Added compressed log for this task")
 677         mp = ManifestParser(use_toml=True, document=True)
 678         manifest_path = os.path.join(self.topsrcdir, os.path.normpath(manifest))
 679         mp.read(manifest_path)
 680         document = mp.source_documents[manifest_path]
 681         add_skip_if(document, filename, skip_if, bug_reference)
 682         manifest_str = alphabetize_toml_str(document)
 683         fp = io.open(manifest_path, "w", encoding="utf-8", newline="\n")
 684         fp.write(manifest_str)
 685         fp.close()
 686         self.info(f'Edited ["{filename}"] in manifest: "{manifest}"')
 687         self.info(f'added skip-if condition: "{skip_if}" # {bug_reference}')
 688
 689     def get_variants(self):
 690         """Get mozinfo for each test variants"""
 691
 692         if len(self.variants) == 0:
 693             variants_file = "taskcluster/ci/test/variants.yml"
 694             variants_path = os.path.join(
 695                 self.topsrcdir, os.path.normpath(variants_file)
 696             )
 697             fp = io.open(variants_path, "r", encoding="utf-8")
 698             raw_variants = load(fp, Loader=Loader)
 699             fp.close()
 700             for k, v in raw_variants.items():
 701                 mozinfo = k
 702                 if "mozinfo" in v:
 703                     mozinfo = v["mozinfo"]
 704                 self.variants[k] = mozinfo
 705         return self.variants
 706
 707     def get_task(self, task_id):
 708         """Download details for task task_id"""
 709
 710         if task_id in self.tasks:  # if cached
 711             task = self.tasks[task_id]
 712         else:
 713             task = get_task(task_id)
 714             self.tasks[task_id] = task
 715         return task
 716
 717     def task_to_skip_if(self, task_id):
 718         """Calculate the skip-if condition for failing task task_id"""
 719
 720         self.get_variants()
 721         task = self.get_task(task_id)
 722         os = None
 723         os_version = None
 724         bits = None
 725         display = None
 726         runtimes = []
 727         build_types = []
 728         test_setting = task.get("extra", {}).get("test-setting", {})
 729         platform = test_setting.get("platform", {})
 730         platform_os = platform.get("os", {})
 731         if "name" in platform_os:
 732             os = platform_os["name"]
 733             if os == "windows":
 734                 os = "win"
 735             if os == "macosx":
 736                 os = "mac"
 737         if "version" in platform_os:
 738             os_version = platform_os["version"]
 739             if len(os_version) == 4:
 740                 os_version = os_version[0:2] + "." + os_version[2:4]
 741         if "arch" in platform:
 742             arch = platform["arch"]
 743             if arch == "x86" or arch.find("32") >= 0:
 744                 bits = "32"
 745         if "display" in platform:
 746             display = platform["display"]
 747         if "runtime" in test_setting:
 748             for k in test_setting["runtime"]:
 749                 if k in self.variants:
 750                     runtimes.append(self.variants[k])  # adds mozinfo
 751         if "build" in test_setting:
 752             tbuild = test_setting["build"]
 753             opt = False
 754             debug = False
 755             for k in tbuild:
 756                 if k == "type":
 757                     if tbuild[k] == "opt":
 758                         opt = True
 759                     elif tbuild[k] == "debug":
 760                         debug = True
 761                 else:
 762                     build_types.append(k)
 763             if len(build_types) == 0:
 764                 if opt:
 765                     build_types.append("!debug")
 766                 if debug:
 767                     build_types.append("debug")
 768         skip_if = None
 769         if os is not None:
 770             skip_if = "os == '" + os + "'"
 771             if os_version is not None:
 772                 skip_if += " && "
 773                 skip_if += "os_version == '" + os_version + "'"
 774             if bits is not None:
 775                 skip_if += " && "
 776                 skip_if += "bits == '" + bits + "'"
 777             if display is not None:
 778                 skip_if += " && "
 779                 skip_if += "display == '" + display + "'"
 780             for runtime in runtimes:
 781                 skip_if += " && "
 782                 skip_if += runtime
 783             for build_type in build_types:
 784                 skip_if += " && "
 785                 skip_if += build_type
 786         return skip_if
 787
 788     def get_file_info(self, path, product="Testing", component="General"):
 789         """
 790         Get bugzilla product and component for the path.
 791         Provide defaults (in case command_context is not defined
 792         or there isn't file info available).
 793         """
 794         if path != "DEFAULT" and self.command_context is not None:
 795             reader = self.command_context.mozbuild_reader(config_mode="empty")
 796             info = reader.files_info([path])
 797             cp = info[path]["BUG_COMPONENT"]
 798             product = cp.product
 799             component = cp.component
 800         return product, component
 801
 802     def get_filename_in_manifest(self, manifest, path):
 803         """return relative filename for path in manifest"""
 804
 805         filename = os.path.basename(path)
 806         if filename == "DEFAULT":
 807             return filename
 808         manifest_dir = os.path.dirname(manifest)
 809         i = 0
 810         j = min(len(manifest_dir), len(path))
 811         while i < j and manifest_dir[i] == path[i]:
 812             i += 1
 813         if i < len(manifest_dir):
 814             for _ in range(manifest_dir.count("/", i) + 1):
 815                 filename = "../" + filename
 816         elif i < len(path):
 817             filename = path[i + 1 :]
 818         return filename
 819
 820     def get_push_id(self, revision, repo):
 821         """Return the push_id for revision and repo (or None)"""
 822
 823         self.vinfo(f"Retrieving push_id for {repo} revision: {revision} ...")
 824         if revision in self.push_ids:  # if cached
 825             push_id = self.push_ids[revision]
 826         else:
 827             push_id = None
 828             push_url = f"https://treeherder.mozilla.org/api/project/{repo}/push/"
 829             params = {}
 830             params["full"] = "true"
 831             params["count"] = 10
 832             params["revision"] = revision
 833             r = requests.get(push_url, headers=self.headers, params=params)
 834             if r.status_code != 200:
 835                 self.warning(f"FAILED to query Treeherder = {r} for {r.url}")
 836             else:
 837                 response = r.json()
 838                 if "results" in response:
 839                     results = response["results"]
 840                     if len(results) > 0:
 841                         r0 = results[0]
 842                         if "id" in r0:
 843                             push_id = r0["id"]
 844             self.push_ids[revision] = push_id
 845         return push_id
 846
 847     def get_job_id(self, push_id, task_id):
 848         """Return the job_id for push_id, task_id (or None)"""
 849
 850         self.vinfo(f"Retrieving job_id for push_id: {push_id}, task_id: {task_id} ...")
 851         if push_id in self.job_ids:  # if cached
 852             job_id = self.job_ids[push_id]
 853         else:
 854             job_id = None
 855             params = {}
 856             params["push_id"] = push_id
 857             r = requests.get(self.jobs_url, headers=self.headers, params=params)
 858             if r.status_code != 200:
 859                 self.warning(f"FAILED to query Treeherder = {r} for {r.url}")
 860             else:
 861                 response = r.json()
 862                 if "results" in response:
 863                     results = response["results"]
 864                     if len(results) > 0:
 865                         for result in results:
 866                             if len(result) > 14:
 867                                 if result[14] == task_id:
 868                                     job_id = result[1]
 869                                     break
 870             self.job_ids[push_id] = job_id
 871         return job_id
 872
 873     def get_bug_suggestions(self, repo, job_id, path):
 874         """
 875         Return the (suggestions_url, line_number, line, log_url)
 876         for the given repo and job_id
 877         """
 878         self.vinfo(
 879             f"Retrieving bug_suggestions for {repo} job_id: {job_id}, path: {path} ..."
 880         )
 881         suggestions_url = f"https://treeherder.mozilla.org/api/project/{repo}/jobs/{job_id}/bug_suggestions/"
 882         line_number = None
 883         line = None
 884         log_url = None
 885         r = requests.get(suggestions_url, headers=self.headers)
 886         if r.status_code != 200:
 887             self.warning(f"FAILED to query Treeherder = {r} for {r.url}")
 888         else:
 889             response = r.json()
 890             if len(response) > 0:
 891                 for sugg in response:
 892                     if sugg["path_end"] == path:
 893                         line_number = sugg["line_number"] + 1
 894                         line = sugg["search"]
 895                         log_url = f"https://treeherder.mozilla.org/logviewer?repo={repo}&job_id={job_id}&lineNumber={line_number}"
 896                         break
 897         rv = (suggestions_url, line_number, line, log_url)
 898         return rv
 899
 900     def read_json(self, filename):
 901         """read data as JSON from filename"""
 902         fp = io.open(filename, "r", encoding="utf-8")
 903         data = json.load(fp)
 904         fp.close()
 905         return data
 906
 907     def write_json(self, filename, data):
 908         """saves data as JSON to filename"""
 909         fp = io.open(filename, "w", encoding="utf-8")
 910         json.dump(data, fp, indent=2, sort_keys=True)
 911         fp.close()
 912
 913     def write_tasks(self, save_tasks, tasks):
 914         """saves tasks as JSON to save_tasks"""
 915         jtasks = []
 916         for task in tasks:
 917             if not isinstance(task, TestTask):
 918                 continue
 919             jtask = {}
 920             jtask["id"] = task.id
 921             jtask["label"] = task.label
 922             jtask["duration"] = task.duration
 923             jtask["result"] = task.result
 924             jtask["state"] = task.state
 925             jtags = {}
 926             for k, v in task.tags.items():
 927                 if k == "createdForUser":
 928                     jtags[k] = "ci@mozilla.com"
 929                 else:
 930                     jtags[k] = v
 931             jtask["tags"] = jtags
 932             jtask["tier"] = task.tier
 933             jtask["results"] = [
 934                 {"group": r.group, "ok": r.ok, "duration": r.duration}
 935                 for r in task.results
 936             ]
 937             jtask["errors"] = None  # Bug with task.errors property??
 938             jft = {}
 939             for k in task.failure_types:
 940                 jft[k] = [[f[0], f[1].value] for f in task.failure_types[k]]
 941             jtask["failure_types"] = jft
 942             jtasks.append(jtask)
 943         self.write_json(save_tasks, jtasks)
 944
 945     def label_to_platform_testname(self, label):
 946         """convert from label to platform, testname for mach command line"""
 947         platform = None
 948         testname = None
 949         platform_details = label.split("/")
 950         if len(platform_details) == 2:
 951             platform, details = platform_details
 952             words = details.split("-")
 953             if len(words) > 2:
 954                 platform += "/" + words.pop(0)  # opt or debug
 955                 try:
 956                     _chunk = int(words[-1])
 957                     words.pop()
 958                 except ValueError:
 959                     pass
 960                 words.pop()  # remove test suffix
 961                 testname = "-".join(words)
 962             else:
 963                 platform = None
 964         return platform, testname
 965
 966     def add_attachment_log_for_task(self, bugid, task_id):
 967         """Adds compressed log for this task to bugid"""
 968
 969         log_url = f"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/{task_id}/artifacts/public/logs/live_backing.log"
 970         r = requests.get(log_url, headers=self.headers)
 971         if r.status_code != 200:
 972             self.error(f"Unable get log for task: {task_id}")
 973             return
 974         attach_fp = tempfile.NamedTemporaryFile()
 975         fp = gzip.open(attach_fp, "wb")
 976         fp.write(r.text.encode("utf-8"))
 977         fp.close()
 978         self._initialize_bzapi()
 979         description = ATTACHMENT_DESCRIPTION + task_id
 980         file_name = TASK_LOG + ".gz"
 981         comment = "Added compressed log"
 982         content_type = "application/gzip"
 983         try:
 984             self._bzapi.attachfile(
 985                 [bugid],
 986                 attach_fp.name,
 987                 description,
 988                 file_name=file_name,
 989                 comment=comment,
 990                 content_type=content_type,
 991                 is_private=False,
 992             )
 993         except Fault:
 994             pass  # Fault expected: Failed to fetch key 9372091 from network storage: The specified key does not exist.