testing/skipfails.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import io
   6 import json
   7 import logging
   8 import os
   9 import os.path
  10 import pprint
  11 import sys
  12 import urllib.parse
  13 from enum import Enum
  14 from pathlib import Path
  15
  16 from yaml import load
  17
  18 try:
  19     from yaml import CLoader as Loader
  20 except ImportError:
  21     from yaml import Loader
  22
  23 import bugzilla
  24 import mozci.push
  25 import requests
  26 from manifestparser import ManifestParser
  27 from manifestparser.toml import add_skip_if, alphabetize_toml_str, sort_paths
  28 from mozci.task import TestTask
  29 from mozci.util.taskcluster import get_task
  30
  31 BUGZILLA_AUTHENTICATION_HELP = "Must create a Bugzilla API key per https://github.com/mozilla/mozci-tools/blob/main/citools/test_triage_bug_filer.py"
  32
  33
  34 class MockResult(object):
  35     def __init__(self, result):
  36         self.result = result
  37
  38     @property
  39     def group(self):
  40         return self.result["group"]
  41
  42     @property
  43     def ok(self):
  44         _ok = self.result["ok"]
  45         return _ok
  46
  47
  48 class MockTask(object):
  49     def __init__(self, task):
  50         self.task = task
  51         if "results" in self.task:
  52             self.task["results"] = [
  53                 MockResult(result) for result in self.task["results"]
  54             ]
  55         else:
  56             self.task["results"] = []
  57
  58     @property
  59     def failure_types(self):
  60         if "failure_types" in self.task:
  61             return self.task["failure_types"]
  62         else:  # note no failure_types in Task object
  63             return {}
  64
  65     @property
  66     def id(self):
  67         return self.task["id"]
  68
  69     @property
  70     def label(self):
  71         return self.task["label"]
  72
  73     @property
  74     def results(self):
  75         return self.task["results"]
  76
  77
  78 class Classification(object):
  79     "Classification of the failure (not the task result)"
  80
  81     DISABLE_MANIFEST = "disable_manifest"  # crash found
  82     DISABLE_RECOMMENDED = "disable_recommended"  # disable first failing path
  83     INTERMITTENT = "intermittent"
  84     SECONDARY = "secondary"  # secondary failing path
  85     SUCCESS = "success"  # path always succeeds
  86     UNKNOWN = "unknown"
  87
  88
  89 class Run(Enum):
  90     """
  91     constant indexes for attributes of a run
  92     """
  93
  94     MANIFEST = 0
  95     TASK_ID = 1
  96     TASK_LABEL = 2
  97     RESULT = 3
  98     CLASSIFICATION = 4
  99
 100
 101 class Skipfails(object):
 102     "mach manifest skip-fails implementation: Update manifests to skip failing tests"
 103
 104     REPO = "repo"
 105     REVISION = "revision"
 106     TREEHERDER = "treeherder.mozilla.org"
 107     BUGZILLA_SERVER_DEFAULT = "bugzilla.allizom.org"
 108
 109     def __init__(
 110         self,
 111         command_context=None,
 112         try_url="",
 113         verbose=False,
 114         bugzilla=None,
 115         dry_run=False,
 116         turbo=False,
 117     ):
 118         self.command_context = command_context
 119         if self.command_context is not None:
 120             self.topsrcdir = self.command_context.topsrcdir
 121         else:
 122             self.topsrcdir = Path(__file__).parent.parent
 123         self.topsrcdir = os.path.normpath(self.topsrcdir)
 124         if isinstance(try_url, list) and len(try_url) == 1:
 125             self.try_url = try_url[0]
 126         else:
 127             self.try_url = try_url
 128         self.dry_run = dry_run
 129         self.verbose = verbose
 130         self.turbo = turbo
 131         if bugzilla is not None:
 132             self.bugzilla = bugzilla
 133         else:
 134             if "BUGZILLA" in os.environ:
 135                 self.bugzilla = os.environ["BUGZILLA"]
 136             else:
 137                 self.bugzilla = Skipfails.BUGZILLA_SERVER_DEFAULT
 138         self.component = "skip-fails"
 139         self._bzapi = None
 140         self.variants = {}
 141         self.tasks = {}
 142         self.pp = None
 143         self.headers = {}  # for Treeherder requests
 144         self.headers["Accept"] = "application/json"
 145         self.headers["User-Agent"] = "treeherder-pyclient"
 146         self.jobs_url = "https://treeherder.mozilla.org/api/jobs/"
 147         self.push_ids = {}
 148         self.job_ids = {}
 149
 150     def _initialize_bzapi(self):
 151         """Lazily initializes the Bugzilla API"""
 152         if self._bzapi is None:
 153             self._bzapi = bugzilla.Bugzilla(self.bugzilla)
 154
 155     def pprint(self, obj):
 156         if self.pp is None:
 157             self.pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)
 158         self.pp.pprint(obj)
 159         sys.stderr.flush()
 160
 161     def error(self, e):
 162         if self.command_context is not None:
 163             self.command_context.log(
 164                 logging.ERROR, self.component, {"error": str(e)}, "ERROR: {error}"
 165             )
 166         else:
 167             print(f"ERROR: {e}", file=sys.stderr, flush=True)
 168
 169     def warning(self, e):
 170         if self.command_context is not None:
 171             self.command_context.log(
 172                 logging.WARNING, self.component, {"error": str(e)}, "WARNING: {error}"
 173             )
 174         else:
 175             print(f"WARNING: {e}", file=sys.stderr, flush=True)
 176
 177     def info(self, e):
 178         if self.command_context is not None:
 179             self.command_context.log(
 180                 logging.INFO, self.component, {"error": str(e)}, "INFO: {error}"
 181             )
 182         else:
 183             print(f"INFO: {e}", file=sys.stderr, flush=True)
 184
 185     def run(
 186         self,
 187         meta_bug_id=None,
 188         save_tasks=None,
 189         use_tasks=None,
 190         save_failures=None,
 191         use_failures=None,
 192     ):
 193         "Run skip-fails on try_url, return True on success"
 194
 195         try_url = self.try_url
 196         revision, repo = self.get_revision(try_url)
 197
 198         if use_tasks is not None:
 199             if os.path.exists(use_tasks):
 200                 self.info(f"use tasks: {use_tasks}")
 201                 tasks = self.read_json(use_tasks)
 202                 tasks = [MockTask(task) for task in tasks]
 203             else:
 204                 self.error(f"uses tasks JSON file does not exist: {use_tasks}")
 205                 return False
 206         else:
 207             tasks = self.get_tasks(revision, repo)
 208
 209         if use_failures is not None:
 210             if os.path.exists(use_failures):
 211                 self.info(f"use failures: {use_failures}")
 212                 failures = self.read_json(use_failures)
 213             else:
 214                 self.error(f"use failures JSON file does not exist: {use_failures}")
 215                 return False
 216         else:
 217             failures = self.get_failures(tasks)
 218             if save_failures is not None:
 219                 self.info(f"save failures: {save_failures}")
 220                 self.write_json(save_failures, failures)
 221
 222         if save_tasks is not None:
 223             self.info(f"save tasks: {save_tasks}")
 224             self.write_tasks(save_tasks, tasks)
 225
 226         for manifest in failures:
 227             if not manifest.endswith(".toml"):
 228                 self.warning(f"cannot process skip-fails on INI manifests: {manifest}")
 229             else:
 230                 for path in failures[manifest]["path"]:
 231                     for label in failures[manifest]["path"][path]:
 232                         classification = failures[manifest]["path"][path][label][
 233                             "classification"
 234                         ]
 235                         if classification.startswith("disable_") or (
 236                             self.turbo and classification == Classification.SECONDARY
 237                         ):
 238                             for task_id in failures[manifest]["path"][path][label][
 239                                 "runs"
 240                             ].keys():
 241                                 self.skip_failure(
 242                                     manifest,
 243                                     path,
 244                                     label,
 245                                     classification,
 246                                     task_id,
 247                                     try_url,
 248                                     revision,
 249                                     repo,
 250                                     meta_bug_id,
 251                                 )
 252                                 break  # just use the first task_id
 253         return True
 254
 255     def get_revision(self, url):
 256         parsed = urllib.parse.urlparse(url)
 257         if parsed.scheme != "https":
 258             raise ValueError("try_url scheme not https")
 259         if parsed.netloc != Skipfails.TREEHERDER:
 260             raise ValueError(f"try_url server not {Skipfails.TREEHERDER}")
 261         if len(parsed.query) == 0:
 262             raise ValueError("try_url query missing")
 263         query = urllib.parse.parse_qs(parsed.query)
 264         if Skipfails.REVISION not in query:
 265             raise ValueError("try_url query missing revision")
 266         revision = query[Skipfails.REVISION][0]
 267         if Skipfails.REPO in query:
 268             repo = query[Skipfails.REPO][0]
 269         else:
 270             repo = "try"
 271         if self.verbose:
 272             self.info(f"considering {repo} revision={revision}")
 273         return revision, repo
 274
 275     def get_tasks(self, revision, repo):
 276         push = mozci.push.Push(revision, repo)
 277         return push.tasks
 278
 279     def get_failures(self, tasks):
 280         """
 281         find failures and create structure comprised of runs by path:
 282            result:
 283             * False (failed)
 284             * True (passed)
 285            classification: Classification
 286             * unknown (default) < 3 runs
 287             * intermittent (not enough failures) >3 runs < 0.5 failure rate
 288             * disable_recommended (enough repeated failures) >3 runs >= 0.5
 289             * disable_manifest (disable DEFAULT if no other failures)
 290             * secondary (not first failure in group)
 291             * success
 292         """
 293
 294         failures = {}
 295         manifest_paths = {}
 296         for task in tasks:
 297             try:
 298                 if len(task.results) == 0:
 299                     continue  # ignore aborted tasks
 300                 for manifest in task.failure_types:
 301                     if manifest not in failures:
 302                         failures[manifest] = {"sum_by_label": {}, "path": {}}
 303                     if manifest not in manifest_paths:
 304                         manifest_paths[manifest] = []
 305                     for path_type in task.failure_types[manifest]:
 306                         path, _type = path_type
 307                         if path == manifest:
 308                             path = "DEFAULT"
 309                         if path not in failures[manifest]["path"]:
 310                             failures[manifest]["path"][path] = {}
 311                         if path not in manifest_paths[manifest]:
 312                             manifest_paths[manifest].append(path)
 313                         if task.label not in failures[manifest]["sum_by_label"]:
 314                             failures[manifest]["sum_by_label"][task.label] = {
 315                                 Classification.UNKNOWN: 0,
 316                                 Classification.SECONDARY: 0,
 317                                 Classification.INTERMITTENT: 0,
 318                                 Classification.DISABLE_RECOMMENDED: 0,
 319                                 Classification.DISABLE_MANIFEST: 0,
 320                                 Classification.SUCCESS: 0,
 321                             }
 322                         if task.label not in failures[manifest]["path"][path]:
 323                             failures[manifest]["path"][path][task.label] = {
 324                                 "total_runs": 0,
 325                                 "failed_runs": 0,
 326                                 "classification": Classification.UNKNOWN,
 327                                 "runs": {task.id: False},
 328                             }
 329                         else:
 330                             failures[manifest]["path"][path][task.label]["runs"][
 331                                 task.id
 332                             ] = False
 333             except AttributeError as ae:
 334                 self.warning(f"unknown attribute in task: {ae}")
 335
 336         # calculate success/failure for each known path
 337         for manifest in manifest_paths:
 338             manifest_paths[manifest] = sort_paths(manifest_paths[manifest])
 339         for task in tasks:
 340             try:
 341                 if len(task.results) == 0:
 342                     continue  # ignore aborted tasks
 343                 for result in task.results:
 344                     manifest = result.group
 345                     if manifest not in failures:
 346                         self.warning(
 347                             f"result for {manifest} not in any failures, ignored"
 348                         )
 349                         continue
 350                     for path in manifest_paths[manifest]:
 351                         if task.label not in failures[manifest]["sum_by_label"]:
 352                             failures[manifest]["sum_by_label"][task.label] = {
 353                                 Classification.UNKNOWN: 0,
 354                                 Classification.SECONDARY: 0,
 355                                 Classification.INTERMITTENT: 0,
 356                                 Classification.DISABLE_RECOMMENDED: 0,
 357                                 Classification.DISABLE_MANIFEST: 0,
 358                                 Classification.SUCCESS: 0,
 359                             }
 360                         if task.label not in failures[manifest]["path"][path]:
 361                             failures[manifest]["path"][path][task.label] = {
 362                                 "total_runs": 0,
 363                                 "failed_runs": 0,
 364                                 "classification": Classification.UNKNOWN,
 365                                 "runs": {},
 366                             }
 367                         if (
 368                             task.id
 369                             not in failures[manifest]["path"][path][task.label]["runs"]
 370                         ):
 371                             ok = True
 372                             failures[manifest]["path"][path][task.label]["runs"][
 373                                 task.id
 374                             ] = ok
 375                         else:
 376                             ok = (
 377                                 result.ok
 378                                 or failures[manifest]["path"][path][task.label]["runs"][
 379                                     task.id
 380                                 ]
 381                             )
 382                         failures[manifest]["path"][path][task.label]["total_runs"] += 1
 383                         if not ok:
 384                             failures[manifest]["path"][path][task.label][
 385                                 "failed_runs"
 386                             ] += 1
 387             except AttributeError as ae:
 388                 self.warning(f"unknown attribute in task: {ae}")
 389
 390         # classify failures and roll up summary statistics
 391         for manifest in failures:
 392             for path in failures[manifest]["path"]:
 393                 for label in failures[manifest]["path"][path]:
 394                     failed_runs = failures[manifest]["path"][path][label]["failed_runs"]
 395                     total_runs = failures[manifest]["path"][path][label]["total_runs"]
 396                     classification = failures[manifest]["path"][path][label][
 397                         "classification"
 398                     ]
 399                     if total_runs >= 3:
 400                         if failed_runs / total_runs < 0.5:
 401                             if failed_runs == 0:
 402                                 classification = Classification.SUCCESS
 403                             else:
 404                                 classification = Classification.INTERMITTENT
 405                         else:
 406                             classification = Classification.SECONDARY
 407                         failures[manifest]["path"][path][label][
 408                             "classification"
 409                         ] = classification
 410                     failures[manifest]["sum_by_label"][label][classification] += 1
 411
 412         # Identify the first failure (for each test, in a manifest, by label)
 413         for manifest in failures:
 414             alpha_paths = sort_paths(failures[manifest]["path"].keys())
 415             for path in alpha_paths:
 416                 for label in failures[manifest]["path"][path]:
 417                     primary = (
 418                         failures[manifest]["sum_by_label"][label][
 419                             Classification.DISABLE_RECOMMENDED
 420                         ]
 421                         == 0
 422                     )
 423                     if path == "DEFAULT":
 424                         classification = failures[manifest]["path"][path][label][
 425                             "classification"
 426                         ]
 427                         if (
 428                             classification == Classification.SECONDARY
 429                             and failures[manifest]["sum_by_label"][label][
 430                                 classification
 431                             ]
 432                             == 1
 433                         ):
 434                             # ONLY failure in the manifest for this label => DISABLE
 435                             failures[manifest]["path"][path][label][
 436                                 "classification"
 437                             ] = Classification.DISABLE_MANIFEST
 438                             failures[manifest]["sum_by_label"][label][
 439                                 classification
 440                             ] -= 1
 441                             failures[manifest]["sum_by_label"][label][
 442                                 Classification.DISABLE_MANIFEST
 443                             ] += 1
 444
 445                     else:
 446                         if (
 447                             primary
 448                             and failures[manifest]["path"][path][label][
 449                                 "classification"
 450                             ]
 451                             == Classification.SECONDARY
 452                         ):
 453                             # FIRST failure in the manifest for this label => DISABLE
 454                             failures[manifest]["path"][path][label][
 455                                 "classification"
 456                             ] = Classification.DISABLE_RECOMMENDED
 457                             failures[manifest]["sum_by_label"][label][
 458                                 Classification.SECONDARY
 459                             ] -= 1
 460                             failures[manifest]["sum_by_label"][label][
 461                                 Classification.DISABLE_RECOMMENDED
 462                             ] += 1
 463
 464         return failures
 465
 466     def _get_os_version(self, os, platform):
 467         """Return the os_version given the label platform string"""
 468         i = platform.find(os)
 469         j = i + len(os)
 470         yy = platform[j : j + 2]
 471         mm = platform[j + 2 : j + 4]
 472         return yy + "." + mm
 473
 474     def get_bug_by_id(self, id):
 475         """Get bug by bug id"""
 476
 477         self._initialize_bzapi()
 478         bug = self._bzapi.getbug(id)
 479         return bug
 480
 481     def get_bugs_by_summary(self, summary):
 482         """Get bug by bug summary"""
 483
 484         self._initialize_bzapi()
 485         query = self._bzapi.build_query(short_desc=summary)
 486         query["include_fields"] = [
 487             "id",
 488             "product",
 489             "component",
 490             "status",
 491             "resolution",
 492             "summary",
 493             "blocks",
 494         ]
 495         bugs = self._bzapi.query(query)
 496         return bugs
 497
 498     def create_bug(
 499         self,
 500         summary="Bug short description",
 501         description="Bug description",
 502         product="Testing",
 503         component="General",
 504         version="unspecified",
 505         bugtype="task",
 506     ):
 507         """Create a bug"""
 508
 509         self._initialize_bzapi()
 510         if not self._bzapi.logged_in:
 511             self.error(
 512                 "Must create a Bugzilla API key per https://github.com/mozilla/mozci-tools/blob/main/citools/test_triage_bug_filer.py"
 513             )
 514             raise PermissionError(f"Not authenticated for Bugzilla {self.bugzilla}")
 515         createinfo = self._bzapi.build_createbug(
 516             product=product,
 517             component=component,
 518             summary=summary,
 519             version=version,
 520             description=description,
 521         )
 522         createinfo["type"] = bugtype
 523         bug = self._bzapi.createbug(createinfo)
 524         return bug
 525
 526     def add_bug_comment(self, id, comment, meta_bug_id=None):
 527         """Add a comment to an existing bug"""
 528
 529         self._initialize_bzapi()
 530         if not self._bzapi.logged_in:
 531             self.error(BUGZILLA_AUTHENTICATION_HELP)
 532             raise PermissionError("Not authenticated for Bugzilla")
 533         if meta_bug_id is not None:
 534             blocks_add = [meta_bug_id]
 535         else:
 536             blocks_add = None
 537         updateinfo = self._bzapi.build_update(comment=comment, blocks_add=blocks_add)
 538         self._bzapi.update_bugs([id], updateinfo)
 539
 540     def skip_failure(
 541         self,
 542         manifest,
 543         path,
 544         label,
 545         classification,
 546         task_id,
 547         try_url,
 548         revision,
 549         repo,
 550         meta_bug_id=None,
 551     ):
 552         """Skip a failure"""
 553
 554         skip_if = self.task_to_skip_if(task_id)
 555         if skip_if is None:
 556             self.warning(
 557                 f"Unable to calculate skip-if condition from manifest={manifest} from failure label={label}"
 558             )
 559             return
 560         bug_reference = ""
 561         if classification == Classification.DISABLE_MANIFEST:
 562             filename = "DEFAULT"
 563             comment = "Disabled entire manifest due to crash result"
 564         else:
 565             filename = self.get_filename_in_manifest(manifest, path)
 566             comment = f'Disabled test due to failures: "{filename}"'
 567             if classification == Classification.SECONDARY:
 568                 comment += " (secondary)"
 569                 bug_reference = " (secondary)"
 570         comment += f"\nTry URL = {try_url}"
 571         comment += f"\nrevision = {revision}"
 572         comment += f"\nrepo = {repo}"
 573         comment += f"\nlabel = {label}"
 574         comment += f"\ntask_id = {task_id}"
 575         push_id = self.get_push_id(revision, repo)
 576         if push_id is not None:
 577             comment += f"\npush_id = {push_id}"
 578             job_id = self.get_job_id(push_id, task_id)
 579             if job_id is not None:
 580                 comment += f"\njob_id = {job_id}"
 581                 suggestions_url, line_number, line, log_url = self.get_bug_suggestions(
 582                     repo, job_id, path
 583                 )
 584                 if log_url is not None:
 585                     comment += f"\n\nBug suggestions: {suggestions_url}"
 586                     comment += f"\nSpecifically see at line {line_number}:\n"
 587                     comment += f'\n  "{line}"'
 588                     comment += f"\n\nIn the log: {log_url}"
 589         bug_summary = f"MANIFEST {manifest}"
 590         bugs = self.get_bugs_by_summary(bug_summary)
 591         if len(bugs) == 0:
 592             description = (
 593                 f"This bug covers excluded failing tests in the MANIFEST {manifest}"
 594             )
 595             description += "\n(generated by mach manifest skip-fails)"
 596             product, component = self.get_file_info(path)
 597             if self.dry_run:
 598                 self.warning(
 599                     f'Dry-run NOT creating bug: {product}::{component} "{bug_summary}"'
 600                 )
 601                 bugid = "TBD"
 602             else:
 603                 bug = self.create_bug(bug_summary, description, product, component)
 604                 bugid = bug.id
 605                 self.info(
 606                     f'Created Bug {bugid} {product}::{component} : "{bug_summary}"'
 607                 )
 608             bug_reference = f"Bug {bugid}" + bug_reference
 609         elif len(bugs) == 1:
 610             bugid = bugs[0].id
 611             bug_reference = f"Bug {bugid}" + bug_reference
 612             product = bugs[0].product
 613             component = bugs[0].component
 614             self.info(f'Found Bug {bugid} {product}::{component} "{bug_summary}"')
 615             if meta_bug_id is not None:
 616                 if meta_bug_id in bugs[0].blocks:
 617                     self.info(f"  Bug {bugid} already blocks meta bug {meta_bug_id}")
 618                     meta_bug_id = None  # no need to add again
 619         else:
 620             self.error(f'More than one bug found for summary: "{bug_summary}"')
 621             return
 622         if self.dry_run:
 623             self.warning(f"Dry-run NOT adding comment to Bug {bugid}: {comment}")
 624             self.info(f'Dry-run NOT editing ["{filename}"] manifest: "{manifest}"')
 625             self.info(f'would add skip-if condition: "{skip_if}" # {bug_reference}')
 626             return
 627         self.add_bug_comment(bugid, comment, meta_bug_id)
 628         self.info(f"Added comment to Bug {bugid}: {comment}")
 629         if meta_bug_id is not None:
 630             self.info(f"  Bug {bugid} blocks meta Bug: {meta_bug_id}")
 631         mp = ManifestParser(use_toml=True, document=True)
 632         manifest_path = os.path.join(self.topsrcdir, os.path.normpath(manifest))
 633         mp.read(manifest_path)
 634         document = mp.source_documents[manifest_path]
 635         add_skip_if(document, filename, skip_if, bug_reference)
 636         manifest_str = alphabetize_toml_str(document)
 637         fp = io.open(manifest_path, "w", encoding="utf-8", newline="\n")
 638         fp.write(manifest_str)
 639         fp.close()
 640         self.info(f'Edited ["{filename}"] in manifest: "{manifest}"')
 641         self.info(f'added skip-if condition: "{skip_if}" # {bug_reference}')
 642
 643     def get_variants(self):
 644         """Get mozinfo for each test variants"""
 645
 646         if len(self.variants) == 0:
 647             variants_file = "taskcluster/ci/test/variants.yml"
 648             variants_path = os.path.join(
 649                 self.topsrcdir, os.path.normpath(variants_file)
 650             )
 651             fp = io.open(variants_path, "r", encoding="utf-8")
 652             raw_variants = load(fp, Loader=Loader)
 653             fp.close()
 654             for k, v in raw_variants.items():
 655                 mozinfo = k
 656                 if "mozinfo" in v:
 657                     mozinfo = v["mozinfo"]
 658                 self.variants[k] = mozinfo
 659         return self.variants
 660
 661     def get_task(self, task_id):
 662         """Download details for task task_id"""
 663
 664         if task_id in self.tasks:  # if cached
 665             task = self.tasks[task_id]
 666         else:
 667             task = get_task(task_id)
 668             self.tasks[task_id] = task
 669         return task
 670
 671     def task_to_skip_if(self, task_id):
 672         """Calculate the skip-if condition for failing task task_id"""
 673
 674         self.get_variants()
 675         task = self.get_task(task_id)
 676         os = None
 677         os_version = None
 678         bits = None
 679         display = None
 680         runtimes = []
 681         build_types = []
 682         test_setting = task.get("extra", {}).get("test-setting", {})
 683         platform = test_setting.get("platform", {})
 684         platform_os = platform.get("os", {})
 685         if "name" in platform_os:
 686             os = platform_os["name"]
 687             if os == "windows":
 688                 os = "win"
 689             if os == "macosx":
 690                 os = "mac"
 691         if "version" in platform_os:
 692             os_version = platform_os["version"]
 693             if len(os_version) == 4:
 694                 os_version = os_version[0:2] + "." + os_version[2:4]
 695         if "arch" in platform:
 696             arch = platform["arch"]
 697             if arch == "x86" or arch.find("32") >= 0:
 698                 bits = "32"
 699         if "display" in platform:
 700             display = platform["display"]
 701         if "runtime" in test_setting:
 702             for k in test_setting["runtime"]:
 703                 if k in self.variants:
 704                     runtimes.append(self.variants[k])  # adds mozinfo
 705         if "build" in test_setting:
 706             tbuild = test_setting["build"]
 707             opt = False
 708             debug = False
 709             for k in tbuild:
 710                 if k == "type":
 711                     if tbuild[k] == "opt":
 712                         opt = True
 713                     elif tbuild[k] == "debug":
 714                         debug = True
 715                 else:
 716                     build_types.append(k)
 717             if len(build_types) == 0:
 718                 if opt:
 719                     build_types.append("!debug")
 720                 if debug:
 721                     build_types.append("debug")
 722         skip_if = None
 723         if os is not None:
 724             skip_if = "os == '" + os + "'"
 725             if os_version is not None:
 726                 skip_if += " && "
 727                 skip_if += "os_version == '" + os_version + "'"
 728             if bits is not None:
 729                 skip_if += " && "
 730                 skip_if += "bits == '" + bits + "'"
 731             if display is not None:
 732                 skip_if += " && "
 733                 skip_if += "display == '" + display + "'"
 734             for runtime in runtimes:
 735                 skip_if += " && "
 736                 skip_if += runtime
 737             for build_type in build_types:
 738                 skip_if += " && "
 739                 skip_if += build_type
 740         return skip_if
 741
 742     def get_file_info(self, path, product="Testing", component="General"):
 743         """
 744         Get bugzilla product and component for the path.
 745         Provide defaults (in case command_context is not defined
 746         or there isn't file info available).
 747         """
 748         if self.command_context is not None:
 749             reader = self.command_context.mozbuild_reader(config_mode="empty")
 750             info = reader.files_info([path])
 751             cp = info[path]["BUG_COMPONENT"]
 752             product = cp.product
 753             component = cp.component
 754         return product, component
 755
 756     def get_filename_in_manifest(self, manifest, path):
 757         """return relative filename for path in manifest"""
 758
 759         filename = os.path.basename(path)
 760         if filename == "DEFAULT":
 761             return filename
 762         manifest_dir = os.path.dirname(manifest)
 763         i = 0
 764         j = min(len(manifest_dir), len(path))
 765         while i < j and manifest_dir[i] == path[i]:
 766             i += 1
 767         if i < len(manifest_dir):
 768             for _ in range(manifest_dir.count("/", i) + 1):
 769                 filename = "../" + filename
 770         elif i < len(path):
 771             filename = path[i + 1 :]
 772         return filename
 773
 774     def get_push_id(self, revision, repo):
 775         """Return the push_id for revision and repo (or None)"""
 776
 777         self.info(f"Retrieving push_id for {repo} revision: {revision} ...")
 778         if revision in self.push_ids:  # if cached
 779             push_id = self.push_ids[revision]
 780         else:
 781             push_id = None
 782             push_url = f"https://treeherder.mozilla.org/api/project/{repo}/push/"
 783             params = {}
 784             params["full"] = "true"
 785             params["count"] = 10
 786             params["revision"] = revision
 787             r = requests.get(push_url, headers=self.headers, params=params)
 788             if r.status_code != 200:
 789                 self.warning(f"FAILED to query Treeherder = {r} for {r.url}")
 790             else:
 791                 response = r.json()
 792                 if "results" in response:
 793                     results = response["results"]
 794                     if len(results) > 0:
 795                         r0 = results[0]
 796                         if "id" in r0:
 797                             push_id = r0["id"]
 798             self.push_ids[revision] = push_id
 799         return push_id
 800
 801     def get_job_id(self, push_id, task_id):
 802         """Return the job_id for push_id, task_id (or None)"""
 803
 804         self.info(f"Retrieving job_id for push_id: {push_id}, task_id: {task_id} ...")
 805         if push_id in self.job_ids:  # if cached
 806             job_id = self.job_ids[push_id]
 807         else:
 808             job_id = None
 809             params = {}
 810             params["push_id"] = push_id
 811             r = requests.get(self.jobs_url, headers=self.headers, params=params)
 812             if r.status_code != 200:
 813                 self.warning(f"FAILED to query Treeherder = {r} for {r.url}")
 814             else:
 815                 response = r.json()
 816                 if "results" in response:
 817                     results = response["results"]
 818                     if len(results) > 0:
 819                         for result in results:
 820                             if len(result) > 14:
 821                                 if result[14] == task_id:
 822                                     job_id = result[1]
 823                                     break
 824             self.job_ids[push_id] = job_id
 825         return job_id
 826
 827     def get_bug_suggestions(self, repo, job_id, path):
 828         """
 829         Return the (suggestions_url, line_number, line, log_url)
 830         for the given repo and job_id
 831         """
 832         self.info(
 833             f"Retrieving bug_suggestions for {repo} job_id: {job_id}, path: {path} ..."
 834         )
 835         suggestions_url = f"https://treeherder.mozilla.org/api/project/{repo}/jobs/{job_id}/bug_suggestions/"
 836         line_number = None
 837         line = None
 838         log_url = None
 839         r = requests.get(suggestions_url, headers=self.headers)
 840         if r.status_code != 200:
 841             self.warning(f"FAILED to query Treeherder = {r} for {r.url}")
 842         else:
 843             response = r.json()
 844             if len(response) > 0:
 845                 for sugg in response:
 846                     if sugg["path_end"] == path:
 847                         line_number = sugg["line_number"]
 848                         line = sugg["search"]
 849                         log_url = f"https://treeherder.mozilla.org/logviewer?repo={repo}&job_id={job_id}&lineNumber={line_number}"
 850                         break
 851         rv = (suggestions_url, line_number, line, log_url)
 852         return rv
 853
 854     def read_json(self, filename):
 855         """read data as JSON from filename"""
 856         fp = io.open(filename, "r", encoding="utf-8")
 857         data = json.load(fp)
 858         fp.close()
 859         return data
 860
 861     def write_json(self, filename, data):
 862         """saves data as JSON to filename"""
 863         fp = io.open(filename, "w", encoding="utf-8")
 864         json.dump(data, fp, indent=2, sort_keys=True)
 865         fp.close()
 866
 867     def write_tasks(self, save_tasks, tasks):
 868         """saves tasks as JSON to save_tasks"""
 869         jtasks = []
 870         for task in tasks:
 871             if not isinstance(task, TestTask):
 872                 continue
 873             jtask = {}
 874             jtask["id"] = task.id
 875             jtask["label"] = task.label
 876             jtask["duration"] = task.duration
 877             jtask["result"] = task.result
 878             jtask["state"] = task.state
 879             jtags = {}
 880             for k, v in task.tags.items():
 881                 if k == "createdForUser":
 882                     jtags[k] = "ci@mozilla.com"
 883                 else:
 884                     jtags[k] = v
 885             jtask["tags"] = jtags
 886             jtask["tier"] = task.tier
 887             jtask["results"] = [
 888                 {"group": r.group, "ok": r.ok, "duration": r.duration}
 889                 for r in task.results
 890             ]
 891             jtask["errors"] = None  # Bug with task.errors property??
 892             jft = {}
 893             for k in task.failure_types:
 894                 jft[k] = [[f[0], f[1].value] for f in task.failure_types[k]]
 895             jtask["failure_types"] = jft
 896             jtasks.append(jtask)
 897         self.write_json(save_tasks, jtasks)