python/mozperftest/mozperftest/utils.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4 import contextlib
   5 import functools
   6 import importlib
   7 import inspect
   8 import logging
   9 import os
  10 import pathlib
  11 import re
  12 import shlex
  13 import shutil
  14 import subprocess
  15 import sys
  16 import tempfile
  17 from collections import defaultdict
  18 from datetime import date, datetime, timedelta
  19 from io import StringIO
  20 from pathlib import Path
  21
  22 import requests
  23 from redo import retry
  24 from requests.packages.urllib3.util.retry import Retry
  25
  26 RETRY_SLEEP = 10
  27 API_ROOT = "https://firefox-ci-tc.services.mozilla.com/api/index/v1"
  28 MULTI_REVISION_ROOT = f"{API_ROOT}/namespaces"
  29 MULTI_TASK_ROOT = f"{API_ROOT}/tasks"
  30 ON_TRY = "MOZ_AUTOMATION" in os.environ
  31 DOWNLOAD_TIMEOUT = 30
  32 METRICS_MATCHER = re.compile(r"(perfMetrics\s.*)")
  33 PRETTY_APP_NAMES = {
  34     "org.mozilla.fenix": "fenix",
  35     "org.mozilla.firefox": "fenix",
  36     "org.mozilla.geckoview_example": "geckoview",
  37 }
  38
  39
  40 class NoPerfMetricsError(Exception):
  41     """Raised when perfMetrics were not found, or were not output
  42     during a test run."""
  43
  44     def __init__(self, flavor):
  45         super().__init__(
  46             f"No perftest results were found in the {flavor} test. Results must be "
  47             'reported using:\n info("perfMetrics", { metricName: metricValue });'
  48         )
  49
  50
  51 class LogProcessor:
  52     def __init__(self, matcher):
  53         self.buf = ""
  54         self.stdout = sys.__stdout__
  55         self.matcher = matcher
  56         self._match = []
  57
  58     @property
  59     def match(self):
  60         return self._match
  61
  62     def write(self, buf):
  63         while buf:
  64             try:
  65                 newline_index = buf.index("\n")
  66             except ValueError:
  67                 # No newline, wait for next call
  68                 self.buf += buf
  69                 break
  70
  71             # Get data up to next newline and combine with previously buffered data
  72             data = self.buf + buf[: newline_index + 1]
  73             buf = buf[newline_index + 1 :]
  74
  75             # Reset buffer then output line
  76             self.buf = ""
  77             if data.strip() == "":
  78                 continue
  79             self.stdout.write(data.strip("\n") + "\n")
  80
  81             # Check if a temporary commit wa created
  82             match = self.matcher.match(data)
  83             if match:
  84                 # Last line found is the revision we want
  85                 self._match.append(match.group(1))
  86
  87     def flush(self):
  88         pass
  89
  90
  91 @contextlib.contextmanager
  92 def silence(layer=None):
  93     if layer is None:
  94         to_patch = (MachLogger,)
  95     else:
  96         to_patch = (MachLogger, layer)
  97
  98     meths = ("info", "debug", "warning", "error", "log")
  99     patched = defaultdict(dict)
 100
 101     oldout, olderr = sys.stdout, sys.stderr
 102     sys.stdout, sys.stderr = StringIO(), StringIO()
 103
 104     def _vacuum(*args, **kw):
 105         sys.stdout.write(str(args))
 106
 107     for obj in to_patch:
 108         for meth in meths:
 109             if not hasattr(obj, meth):
 110                 continue
 111             patched[obj][meth] = getattr(obj, meth)
 112             setattr(obj, meth, _vacuum)
 113
 114     stdout = stderr = None
 115     try:
 116         sys.stdout.buffer = sys.stdout
 117         sys.stderr.buffer = sys.stderr
 118         sys.stdout.fileno = sys.stderr.fileno = lambda: -1
 119         try:
 120             yield sys.stdout, sys.stderr
 121         except Exception:
 122             sys.stdout.seek(0)
 123             stdout = sys.stdout.read()
 124             sys.stderr.seek(0)
 125             stderr = sys.stderr.read()
 126             raise
 127     finally:
 128         sys.stdout, sys.stderr = oldout, olderr
 129         for obj, meths in patched.items():
 130             for name, old_func in meths.items():
 131                 try:
 132                     setattr(obj, name, old_func)
 133                 except Exception:
 134                     pass
 135         if stdout is not None:
 136             print(stdout)
 137         if stderr is not None:
 138             print(stderr)
 139
 140
 141 def simple_platform():
 142     plat = host_platform()
 143
 144     if plat.startswith("win"):
 145         return "win"
 146     elif plat.startswith("linux"):
 147         return "linux"
 148     else:
 149         return "mac"
 150
 151
 152 def host_platform():
 153     is_64bits = sys.maxsize > 2**32
 154
 155     if sys.platform.startswith("win"):
 156         if is_64bits:
 157             return "win64"
 158     elif sys.platform.startswith("linux"):
 159         if is_64bits:
 160             return "linux64"
 161     elif sys.platform.startswith("darwin"):
 162         return "darwin"
 163
 164     raise ValueError(f"platform not yet supported: {sys.platform}")
 165
 166
 167 class MachLogger:
 168     """Wrapper around the mach logger to make logging simpler."""
 169
 170     def __init__(self, mach_cmd):
 171         self._logger = mach_cmd.log
 172
 173     @property
 174     def log(self):
 175         return self._logger
 176
 177     def info(self, msg, name="mozperftest", **kwargs):
 178         self._logger(logging.INFO, name, kwargs, msg)
 179
 180     def debug(self, msg, name="mozperftest", **kwargs):
 181         self._logger(logging.DEBUG, name, kwargs, msg)
 182
 183     def warning(self, msg, name="mozperftest", **kwargs):
 184         self._logger(logging.WARNING, name, kwargs, msg)
 185
 186     def error(self, msg, name="mozperftest", **kwargs):
 187         self._logger(logging.ERROR, name, kwargs, msg)
 188
 189
 190 def install_package(virtualenv_manager, package, ignore_failure=False):
 191     """Installs a package using the virtualenv manager.
 192
 193     Makes sure the package is really installed when the user already has it
 194     in their local installation.
 195
 196     Returns True on success, or re-raise the error. If ignore_failure
 197     is set to True, ignore the error and return False
 198     """
 199     from pip._internal.req.constructors import install_req_from_line
 200
 201     # Ensure that we are looking in the right places for packages. This
 202     # is required in CI because pip installs in an area that is not in
 203     # the search path.
 204     venv_site_lib = str(Path(virtualenv_manager.bin_path, "..", "lib").resolve())
 205     venv_site_packages = str(
 206         Path(
 207             venv_site_lib,
 208             f"python{sys.version_info.major}.{sys.version_info.minor}",
 209             "site-packages",
 210         )
 211     )
 212     if venv_site_packages not in sys.path and ON_TRY:
 213         sys.path.insert(0, venv_site_packages)
 214
 215     req = install_req_from_line(package)
 216     req.check_if_exists(use_user_site=False)
 217     # already installed, check if it's in our venv
 218     if req.satisfied_by is not None:
 219         site_packages = os.path.abspath(req.satisfied_by.location)
 220         if site_packages.startswith(venv_site_lib):
 221             # already installed in this venv, we can skip
 222             return True
 223     with silence():
 224         try:
 225             subprocess.check_call(
 226                 [virtualenv_manager.python_path, "-m", "pip", "install", package]
 227             )
 228             return True
 229         except Exception:
 230             if not ignore_failure:
 231                 raise
 232     return False
 233
 234
 235 def install_requirements_file(
 236     virtualenv_manager, requirements_file, ignore_failure=False
 237 ):
 238     """Installs a package using the virtualenv manager.
 239
 240     Makes sure the package is really installed when the user already has it
 241     in their local installation.
 242
 243     Returns True on success, or re-raise the error. If ignore_failure
 244     is set to True, ignore the error and return False
 245     """
 246
 247     # Ensure that we are looking in the right places for packages. This
 248     # is required in CI because pip installs in an area that is not in
 249     # the search path.
 250     venv_site_lib = str(Path(virtualenv_manager.bin_path, "..", "lib").resolve())
 251     venv_site_packages = Path(
 252         venv_site_lib,
 253         f"python{sys.version_info.major}.{sys.version_info.minor}",
 254         "site-packages",
 255     )
 256     if not venv_site_packages.exists():
 257         venv_site_packages = Path(
 258             venv_site_lib,
 259             "site-packages",
 260         )
 261
 262     venv_site_packages = str(venv_site_packages)
 263     if venv_site_packages not in sys.path and ON_TRY:
 264         sys.path.insert(0, venv_site_packages)
 265
 266     with silence():
 267         cwd = os.getcwd()
 268         try:
 269             os.chdir(Path(requirements_file).parent)
 270             subprocess.check_call(
 271                 [
 272                     virtualenv_manager.python_path,
 273                     "-m",
 274                     "pip",
 275                     "install",
 276                     "--no-deps",
 277                     "-r",
 278                     requirements_file,
 279                     "--no-index",
 280                     "--find-links",
 281                     "https://pypi.pub.build.mozilla.org/pub/",
 282                 ]
 283             )
 284             return True
 285         except Exception:
 286             if not ignore_failure:
 287                 raise
 288         finally:
 289             os.chdir(cwd)
 290     return False
 291
 292
 293 # on try, we create tests packages where tests, like
 294 # xpcshell tests, don't have the same path.
 295 # see - python/mozbuild/mozbuild/action/test_archive.py
 296 # this mapping will map paths when running there.
 297 # The key is the source path, and the value the ci path
 298 _TRY_MAPPING = {
 299     Path("netwerk"): Path("xpcshell", "tests", "netwerk"),
 300     Path("dom"): Path("mochitest", "tests", "dom"),
 301 }
 302
 303
 304 def build_test_list(tests):
 305     """Collects tests given a list of directories, files and URLs.
 306
 307     Returns a tuple containing the list of tests found and a temp dir for tests
 308     that were downloaded from an URL.
 309     """
 310     temp_dir = None
 311
 312     if isinstance(tests, str):
 313         tests = [tests]
 314     res = []
 315     for test in tests:
 316         if test.startswith("http"):
 317             if temp_dir is None:
 318                 temp_dir = tempfile.mkdtemp()
 319             target = Path(temp_dir, test.split("/")[-1])
 320             download_file(test, target)
 321             res.append(str(target))
 322             continue
 323
 324         p_test = Path(test)
 325         if ON_TRY and not p_test.resolve().exists():
 326             # until we have pathlib.Path.is_relative_to() (3.9)
 327             for src_path, ci_path in _TRY_MAPPING.items():
 328                 src_path, ci_path = str(src_path), str(ci_path)  # noqa
 329                 if test.startswith(src_path):
 330                     p_test = Path(test.replace(src_path, ci_path))
 331                     break
 332
 333         resolved_test = p_test.resolve()
 334
 335         if resolved_test.is_file():
 336             res.append(str(resolved_test))
 337         elif resolved_test.is_dir():
 338             for file in resolved_test.rglob("perftest_*.js"):
 339                 res.append(str(file))
 340         else:
 341             raise FileNotFoundError(str(resolved_test))
 342     res.sort()
 343     return res, temp_dir
 344
 345
 346 def download_file(url, target, retry_sleep=RETRY_SLEEP, attempts=3):
 347     """Downloads a file, given an URL in the target path.
 348
 349     The function will attempt several times on failures.
 350     """
 351
 352     def _download_file(url, target):
 353         req = requests.get(url, stream=True, timeout=30)
 354         target_dir = target.parent.resolve()
 355         if str(target_dir) != "":
 356             target_dir.mkdir(exist_ok=True)
 357
 358         with target.open("wb") as f:
 359             for chunk in req.iter_content(chunk_size=1024):
 360                 if not chunk:
 361                     continue
 362                 f.write(chunk)
 363                 f.flush()
 364         return target
 365
 366     return retry(
 367         _download_file,
 368         args=(url, target),
 369         attempts=attempts,
 370         sleeptime=retry_sleep,
 371         jitter=0,
 372     )
 373
 374
 375 @contextlib.contextmanager
 376 def temporary_env(**env):
 377     old = {}
 378     for key, value in env.items():
 379         old[key] = os.environ.get(key)
 380         if value is None and key in os.environ:
 381             del os.environ[key]
 382         elif value is not None:
 383             os.environ[key] = value
 384     try:
 385         yield
 386     finally:
 387         for key, value in old.items():
 388             if value is None and key in os.environ:
 389                 del os.environ[key]
 390             elif value is not None:
 391                 os.environ[key] = value
 392
 393
 394 def convert_day(day):
 395     if day in ("yesterday", "today"):
 396         curr = date.today()
 397         if day == "yesterday":
 398             curr = curr - timedelta(1)
 399         day = curr.strftime("%Y.%m.%d")
 400     else:
 401         # verify that the user provided string is in the expected format
 402         # if it can't parse it, it'll raise a value error
 403         datetime.strptime(day, "%Y.%m.%d")
 404
 405     return day
 406
 407
 408 def get_revision_namespace_url(route, day="yesterday"):
 409     """Builds a URL to obtain all the namespaces of a given build route for a single day."""
 410     day = convert_day(day)
 411     return f"""{MULTI_REVISION_ROOT}/{route}.{day}.revision"""
 412
 413
 414 def get_multi_tasks_url(route, revision, day="yesterday"):
 415     """Builds a URL to obtain all the tasks of a given build route for a single day.
 416
 417     If previous is true, then we get builds from the previous day,
 418     otherwise, we look at the current day.
 419     """
 420     day = convert_day(day)
 421     return f"""{MULTI_TASK_ROOT}/{route}.{day}.revision.{revision}"""
 422
 423
 424 def strtobool(val):
 425     if isinstance(val, (bool, int)):
 426         return bool(val)
 427     if not isinstance(bool, str):
 428         raise ValueError(val)
 429     val = val.lower()
 430     if val in ("y", "yes", "t", "true", "on", "1"):
 431         return 1
 432     elif val in ("n", "no", "f", "false", "off", "0"):
 433         return 0
 434     else:
 435         raise ValueError("invalid truth value %r" % (val,))
 436
 437
 438 @contextlib.contextmanager
 439 def temp_dir():
 440     tempdir = tempfile.mkdtemp()
 441     try:
 442         yield tempdir
 443     finally:
 444         shutil.rmtree(tempdir)
 445
 446
 447 def load_class(path):
 448     """Loads a class given its path and returns it.
 449
 450     The path is a string of the form `package.module:class` that points
 451     to the class to be imported.
 452
 453     If if can't find it, or if the path is malformed,
 454     an ImportError is raised.
 455     """
 456     if ":" not in path:
 457         raise ImportError(f"Malformed path '{path}'")
 458     elmts = path.split(":")
 459     if len(elmts) != 2:
 460         raise ImportError(f"Malformed path '{path}'")
 461     mod_name, klass_name = elmts
 462     try:
 463         mod = importlib.import_module(mod_name)
 464     except ModuleNotFoundError:
 465         raise ImportError(f"Can't find '{mod_name}'")
 466     try:
 467         klass = getattr(mod, klass_name)
 468     except AttributeError:
 469         raise ImportError(f"Can't find '{klass_name}' in '{mod_name}'")
 470     return klass
 471
 472
 473 def load_class_from_path(klass_name, path):
 474     """This function returns a Transformer class with the given path.
 475
 476     :param str path: The path points to the custom transformer.
 477     :param bool ret_members: If true then return inspect.getmembers().
 478     :return Transformer if not ret_members else inspect.getmembers().
 479     """
 480     file = pathlib.Path(path)
 481
 482     if not file.exists():
 483         raise ImportError(f"The class path {path} does not exist.")
 484
 485     # Importing a source file directly
 486     spec = importlib.util.spec_from_file_location(name=file.name, location=path)
 487     module = importlib.util.module_from_spec(spec)
 488     spec.loader.exec_module(module)
 489
 490     members = inspect.getmembers(
 491         module,
 492         lambda c: inspect.isclass(c) and c.__name__ == klass_name,
 493     )
 494
 495     if not members:
 496         raise ImportError(f"The path {path} was found but it was not a valid class.")
 497
 498     return members[0][-1]
 499
 500
 501 def run_script(cmd, cmd_args=None, verbose=False, display=False, label=None):
 502     """Used to run a command in a subprocess."""
 503     if isinstance(cmd, str):
 504         cmd = shlex.split(cmd)
 505     try:
 506         joiner = shlex.join
 507     except AttributeError:
 508         # Python < 3.8
 509         joiner = subprocess.list2cmdline
 510
 511     if label is None:
 512         label = joiner(cmd)
 513     sys.stdout.write(f"=> {label} ")
 514     if cmd_args is None:
 515         args = cmd
 516     else:
 517         args = cmd + list(cmd_args)
 518     sys.stdout.flush()
 519     try:
 520         if verbose:
 521             sys.stdout.write(f"\nRunning {' '.join(args)}\n")
 522             sys.stdout.flush()
 523         output = subprocess.check_output(args)
 524         if display:
 525             sys.stdout.write("\n")
 526             for line in output.split(b"\n"):
 527                 sys.stdout.write(line.decode("utf8") + "\n")
 528         sys.stdout.write("[OK]\n")
 529         sys.stdout.flush()
 530         return True, output
 531     except subprocess.CalledProcessError as e:
 532         for line in e.output.split(b"\n"):
 533             sys.stdout.write(line.decode("utf8") + "\n")
 534         sys.stdout.write("[FAILED]\n")
 535         sys.stdout.flush()
 536         return False, e
 537
 538
 539 def run_python_script(
 540     virtualenv_manager,
 541     module,
 542     module_args=None,
 543     verbose=False,
 544     display=False,
 545     label=None,
 546 ):
 547     """Used to run a Python script in isolation."""
 548     if label is None:
 549         label = module
 550     cmd = [virtualenv_manager.python_path, "-m", module]
 551     return run_script(cmd, module_args, verbose=verbose, display=display, label=label)
 552
 553
 554 def checkout_script(cmd, cmd_args=None, verbose=False, display=False, label=None):
 555     return run_script(cmd, cmd_args, verbose, display, label)[0]
 556
 557
 558 def checkout_python_script(
 559     virtualenv_manager,
 560     module,
 561     module_args=None,
 562     verbose=False,
 563     display=False,
 564     label=None,
 565 ):
 566     return run_python_script(
 567         virtualenv_manager, module, module_args, verbose, display, label
 568     )[0]
 569
 570
 571 _URL = (
 572     "{0}/secrets/v1/secret/project"
 573     "{1}releng{1}gecko{1}build{1}level-{2}{1}conditioned-profiles"
 574 )
 575 _WPT_URL = "{0}/secrets/v1/secret/project/perftest/gecko/level-{1}/perftest-login"
 576 _DEFAULT_SERVER = "https://firefox-ci-tc.services.mozilla.com"
 577
 578
 579 @functools.lru_cache()
 580 def get_tc_secret(wpt=False):
 581     """Returns the Taskcluster secret.
 582
 583     Raises an OSError when not running on try
 584     """
 585     if not ON_TRY:
 586         raise OSError("Not running in Taskcluster")
 587     session = requests.Session()
 588     retry = Retry(total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504])
 589     http_adapter = requests.adapters.HTTPAdapter(max_retries=retry)
 590     session.mount("https://", http_adapter)
 591     session.mount("http://", http_adapter)
 592     secrets_url = _URL.format(
 593         os.environ.get("TASKCLUSTER_PROXY_URL", _DEFAULT_SERVER),
 594         "%2F",
 595         os.environ.get("MOZ_SCM_LEVEL", "1"),
 596     )
 597     if wpt:
 598         secrets_url = _WPT_URL.format(
 599             os.environ.get("TASKCLUSTER_PROXY_URL", _DEFAULT_SERVER),
 600             os.environ.get("MOZ_SCM_LEVEL", "1"),
 601         )
 602     res = session.get(secrets_url, timeout=DOWNLOAD_TIMEOUT)
 603     res.raise_for_status()
 604     return res.json()["secret"]
 605
 606
 607 def get_output_dir(output, folder=None):
 608     if output is None:
 609         raise Exception("Output path was not provided.")
 610
 611     result_dir = Path(output)
 612     if folder is not None:
 613         result_dir = Path(result_dir, folder)
 614
 615     result_dir.mkdir(parents=True, exist_ok=True)
 616     result_dir = result_dir.resolve()
 617
 618     return result_dir
 619
 620
 621 def create_path(path):
 622     if path.exists():
 623         return path
 624     else:
 625         create_path(path.parent)
 626         path.mkdir(exist_ok=True)
 627         return path
 628
 629
 630 def get_pretty_app_name(app):
 631     # XXX See bug 1712337, we need a singluar point of entry
 632     # for the binary to allow us to get the version/app info
 633     # so that we can get a pretty name on desktop.
 634     return PRETTY_APP_NAMES[app]