testing/mozharness/scripts/android_hardware_unittest.py

   1 #!/usr/bin/env python
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4 # You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 import copy
   7 import datetime
   8 import json
   9 import os
  10 import subprocess
  11 import sys
  12
  13 # load modules from parent dir
  14 sys.path.insert(1, os.path.dirname(sys.path[0]))
  15
  16 from mozharness.base.log import WARNING
  17 from mozharness.base.script import BaseScript, PreScriptAction
  18 from mozharness.mozilla.automation import TBPL_RETRY
  19 from mozharness.mozilla.mozbase import MozbaseMixin
  20 from mozharness.mozilla.testing.android import AndroidMixin
  21 from mozharness.mozilla.testing.codecoverage import CodeCoverageMixin
  22 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
  23
  24 SUITE_DEFAULT_E10S = ["geckoview-junit", "mochitest", "reftest"]
  25 SUITE_NO_E10S = ["cppunittest", "gtest", "jittest"]
  26 SUITE_REPEATABLE = ["mochitest", "reftest", "xpcshell"]
  27
  28
  29 class AndroidHardwareTest(
  30     TestingMixin, BaseScript, MozbaseMixin, CodeCoverageMixin, AndroidMixin
  31 ):
  32     config_options = [
  33         [["--test-suite"], {"action": "store", "dest": "test_suite", "default": None}],
  34         [
  35             ["--adb-path"],
  36             {
  37                 "action": "store",
  38                 "dest": "adb_path",
  39                 "default": None,
  40                 "help": "Path to adb",
  41             },
  42         ],
  43         [
  44             ["--total-chunk"],
  45             {
  46                 "action": "store",
  47                 "dest": "total_chunks",
  48                 "default": None,
  49                 "help": "Number of total chunks",
  50             },
  51         ],
  52         [
  53             ["--this-chunk"],
  54             {
  55                 "action": "store",
  56                 "dest": "this_chunk",
  57                 "default": None,
  58                 "help": "Number of this chunk",
  59             },
  60         ],
  61         [
  62             ["--log-raw-level"],
  63             {
  64                 "action": "store",
  65                 "dest": "log_raw_level",
  66                 "default": "info",
  67                 "help": "Set log level (debug|info|warning|error|critical|fatal)",
  68             },
  69         ],
  70         [
  71             ["--log-tbpl-level"],
  72             {
  73                 "action": "store",
  74                 "dest": "log_tbpl_level",
  75                 "default": "info",
  76                 "help": "Set log level (debug|info|warning|error|critical|fatal)",
  77             },
  78         ],
  79         [
  80             ["--disable-e10s"],
  81             {
  82                 "action": "store_false",
  83                 "dest": "e10s",
  84                 "default": True,
  85                 "help": "Run tests without multiple processes (e10s).",
  86             },
  87         ],
  88         [
  89             ["--disable-fission"],
  90             {
  91                 "action": "store_true",
  92                 "dest": "disable_fission",
  93                 "default": False,
  94                 "help": "Run with Fission disabled.",
  95             },
  96         ],
  97         [
  98             ["--repeat"],
  99             {
 100                 "action": "store",
 101                 "type": "int",
 102                 "dest": "repeat",
 103                 "default": 0,
 104                 "help": "Repeat the tests the given number of times. Supported "
 105                 "by mochitest, reftest, crashtest, ignored otherwise.",
 106             },
 107         ],
 108         [
 109             [
 110                 "--setpref",
 111             ],
 112             {
 113                 "action": "append",
 114                 "dest": "extra_prefs",
 115                 "default": [],
 116                 "help": "Extra user prefs.",
 117             },
 118         ],
 119         [
 120             ["--jittest-flags"],
 121             {
 122                 "action": "store",
 123                 "dest": "jittest_flags",
 124                 "default": "debug",
 125                 "help": "Flags to run with jittest (all, debug, etc.).",
 126             },
 127         ],
 128     ] + copy.deepcopy(testing_config_options)
 129
 130     def __init__(self, require_config_file=False):
 131         super(AndroidHardwareTest, self).__init__(
 132             config_options=self.config_options,
 133             all_actions=[
 134                 "clobber",
 135                 "download-and-extract",
 136                 "create-virtualenv",
 137                 "verify-device",
 138                 "install",
 139                 "run-tests",
 140             ],
 141             require_config_file=require_config_file,
 142             config={
 143                 "virtualenv_modules": [],
 144                 "virtualenv_requirements": [],
 145                 "require_test_zip": True,
 146                 # IP address of the host as seen from the device.
 147                 "remote_webserver": os.environ["HOST_IP"],
 148             },
 149         )
 150
 151         # these are necessary since self.config is read only
 152         c = self.config
 153         self.installer_url = c.get("installer_url")
 154         self.installer_path = c.get("installer_path")
 155         self.test_url = c.get("test_url")
 156         self.test_packages_url = c.get("test_packages_url")
 157         self.test_manifest = c.get("test_manifest")
 158         suite = c.get("test_suite")
 159         self.test_suite = suite
 160         self.this_chunk = c.get("this_chunk")
 161         self.total_chunks = c.get("total_chunks")
 162         self.xre_path = None
 163         self.log_raw_level = c.get("log_raw_level")
 164         self.log_tbpl_level = c.get("log_tbpl_level")
 165         self.disable_e10s = c.get("disable_e10s")
 166         self.disable_fission = c.get("disable_fission")
 167         self.extra_prefs = c.get("extra_prefs")
 168         self.jittest_flags = c.get("jittest_flags")
 169
 170     def query_abs_dirs(self):
 171         if self.abs_dirs:
 172             return self.abs_dirs
 173         abs_dirs = super(AndroidHardwareTest, self).query_abs_dirs()
 174         dirs = {}
 175         dirs["abs_test_install_dir"] = os.path.join(abs_dirs["abs_work_dir"], "tests")
 176         dirs["abs_test_bin_dir"] = os.path.join(
 177             abs_dirs["abs_work_dir"], "tests", "bin"
 178         )
 179         dirs["abs_modules_dir"] = os.path.join(dirs["abs_test_install_dir"], "modules")
 180         dirs["abs_blob_upload_dir"] = os.path.join(
 181             abs_dirs["abs_work_dir"], "blobber_upload_dir"
 182         )
 183         dirs["abs_mochitest_dir"] = os.path.join(
 184             dirs["abs_test_install_dir"], "mochitest"
 185         )
 186         dirs["abs_reftest_dir"] = os.path.join(dirs["abs_test_install_dir"], "reftest")
 187         dirs["abs_xpcshell_dir"] = os.path.join(
 188             dirs["abs_test_install_dir"], "xpcshell"
 189         )
 190         work_dir = os.environ.get("MOZ_FETCHES_DIR") or abs_dirs["abs_work_dir"]
 191         dirs["abs_xre_dir"] = os.path.join(work_dir, "hostutils")
 192
 193         for key in dirs.keys():
 194             if key not in abs_dirs:
 195                 abs_dirs[key] = dirs[key]
 196         self.abs_dirs = abs_dirs
 197         return self.abs_dirs
 198
 199     def _query_tests_dir(self):
 200         dirs = self.query_abs_dirs()
 201         try:
 202             test_dir = self.config["suite_definitions"][self.test_suite]["testsdir"]
 203         except Exception:
 204             test_dir = self.test_suite
 205         return os.path.join(dirs["abs_test_install_dir"], test_dir)
 206
 207     def _build_command(self):
 208         c = self.config
 209         dirs = self.query_abs_dirs()
 210
 211         if self.test_suite not in self.config["suite_definitions"]:
 212             self.fatal("Key '%s' not defined in the config!" % self.test_suite)
 213
 214         cmd = [
 215             self.query_python_path("python"),
 216             "-u",
 217             os.path.join(
 218                 self._query_tests_dir(),
 219                 self.config["suite_definitions"][self.test_suite]["run_filename"],
 220             ),
 221         ]
 222
 223         raw_log_file, error_summary_file = self.get_indexed_logs(
 224             dirs["abs_blob_upload_dir"], self.test_suite
 225         )
 226
 227         str_format_values = {
 228             "device_serial": self.device_serial,
 229             "remote_webserver": c["remote_webserver"],
 230             "xre_path": self.xre_path,
 231             "utility_path": self.xre_path,
 232             "http_port": "8854",  # starting http port  to use for the mochitest server
 233             "ssl_port": "4454",  # starting ssl port to use for the server
 234             "certs_path": os.path.join(dirs["abs_work_dir"], "tests/certs"),
 235             # TestingMixin._download_and_extract_symbols() will set
 236             # self.symbols_path when downloading/extracting.
 237             "symbols_path": self.symbols_path,
 238             "modules_dir": dirs["abs_modules_dir"],
 239             "installer_path": self.installer_path,
 240             "raw_log_file": raw_log_file,
 241             "log_tbpl_level": self.log_tbpl_level,
 242             "log_raw_level": self.log_raw_level,
 243             "error_summary_file": error_summary_file,
 244             "xpcshell_extra": c.get("xpcshell_extra", ""),
 245             "jittest_flags": self.jittest_flags,
 246         }
 247
 248         user_paths = json.loads(os.environ.get("MOZHARNESS_TEST_PATHS", '""'))
 249         confirm_paths = json.loads(os.environ.get("MOZHARNESS_CONFIRM_PATHS", '""'))
 250
 251         for option in self.config["suite_definitions"][self.test_suite]["options"]:
 252             opt = option.split("=")[0]
 253             # override configured chunk options with script args, if specified
 254             if opt in ("--this-chunk", "--total-chunks"):
 255                 if (
 256                     user_paths
 257                     or getattr(self, opt.replace("-", "_").strip("_"), None) is not None
 258                 ):
 259                     continue
 260
 261             if "%(app)" in option:
 262                 # only query package name if requested
 263                 cmd.extend([option % {"app": self.query_package_name()}])
 264             else:
 265                 option = option % str_format_values
 266                 if option:
 267                     cmd.extend([option])
 268
 269         if not self.verify_enabled and not user_paths:
 270             if self.this_chunk is not None:
 271                 cmd.extend(["--this-chunk", self.this_chunk])
 272             if self.total_chunks is not None:
 273                 cmd.extend(["--total-chunks", self.total_chunks])
 274
 275         if "mochitest" in self.test_suite:
 276             category = "mochitest"
 277         elif "reftest" in self.test_suite or "crashtest" in self.test_suite:
 278             category = "reftest"
 279         else:
 280             category = self.test_suite
 281         if c.get("repeat"):
 282             if category in SUITE_REPEATABLE:
 283                 cmd.extend(["--repeat=%s" % c.get("repeat")])
 284             else:
 285                 self.log("--repeat not supported in {}".format(category), level=WARNING)
 286
 287         if category not in SUITE_NO_E10S:
 288             if category in SUITE_DEFAULT_E10S and not c["e10s"]:
 289                 cmd.append("--disable-e10s")
 290             elif category not in SUITE_DEFAULT_E10S and c["e10s"]:
 291                 cmd.append("--e10s")
 292
 293         if self.disable_fission and category not in SUITE_NO_E10S:
 294             cmd.append("--disable-fission")
 295
 296         cmd.extend(["--setpref={}".format(p) for p in self.extra_prefs])
 297
 298         try_options, try_tests = self.try_args(self.test_suite)
 299         if try_options:
 300             cmd.extend(try_options)
 301
 302         if user_paths:
 303             # reftest on android-hw uses a subset (reftest-qr) of tests,
 304             # but scheduling only knows about 'reftest'
 305             suite = self.test_suite
 306             if suite == "reftest-qr":
 307                 suite = "reftest"
 308
 309             if user_paths.get(suite, []):
 310                 suite_test_paths = user_paths.get(suite, [])
 311                 # NOTE: we do not want to prepend 'tests' if a single path
 312                 if confirm_paths and confirm_paths.get(suite, []):
 313                     suite_test_paths = confirm_paths.get(suite, [])
 314                 suite_test_paths = [os.path.join("tests", p) for p in suite_test_paths]
 315                 cmd.extend(suite_test_paths)
 316
 317         elif not self.verify_enabled and not self.per_test_coverage:
 318             cmd.extend(
 319                 self.query_tests_args(
 320                     self.config["suite_definitions"][self.test_suite].get("tests"),
 321                     None,
 322                     try_tests,
 323                 )
 324             )
 325
 326         if self.config.get("restartAfterFailure", False):
 327             cmd.append("--restartAfterFailure")
 328
 329         return cmd
 330
 331     def _query_suites(self):
 332         if self.test_suite:
 333             return [(self.test_suite, self.test_suite)]
 334         # per-test mode: determine test suites to run
 335         all = [
 336             (
 337                 "mochitest",
 338                 {
 339                     "mochitest-plain": "mochitest-plain",
 340                     "mochitest-plain-gpu": "mochitest-plain-gpu",
 341                 },
 342             ),
 343             ("reftest", {"reftest": "reftest", "crashtest": "crashtest"}),
 344             ("xpcshell", {"xpcshell": "xpcshell"}),
 345         ]
 346         suites = []
 347         for category, all_suites in all:
 348             cat_suites = self.query_per_test_category_suites(category, all_suites)
 349             for k in cat_suites.keys():
 350                 suites.append((k, cat_suites[k]))
 351         return suites
 352
 353     def _query_suite_categories(self):
 354         if self.test_suite:
 355             categories = [self.test_suite]
 356         else:
 357             # per-test mode
 358             categories = ["mochitest", "reftest", "xpcshell"]
 359         return categories
 360
 361     ##########################################
 362     # Actions for AndroidHardwareTest        #
 363     ##########################################
 364
 365     def preflight_install(self):
 366         # in the base class, this checks for mozinstall, but we don't use it
 367         pass
 368
 369     @PreScriptAction("create-virtualenv")
 370     def pre_create_virtualenv(self, action):
 371         dirs = self.query_abs_dirs()
 372         requirements = None
 373         suites = self._query_suites()
 374         if ("mochitest-media", "mochitest-media") in suites:
 375             # mochitest-media is the only thing that needs this
 376             requirements = os.path.join(
 377                 dirs["abs_mochitest_dir"],
 378                 "websocketprocessbridge",
 379                 "websocketprocessbridge_requirements_3.txt",
 380             )
 381         if requirements:
 382             self.register_virtualenv_module(requirements=[requirements])
 383
 384     def download_and_extract(self):
 385         """
 386         Download and extract product APK, tests.zip, and host utils.
 387         """
 388         super(AndroidHardwareTest, self).download_and_extract(
 389             suite_categories=self._query_suite_categories()
 390         )
 391         dirs = self.query_abs_dirs()
 392         self.xre_path = dirs["abs_xre_dir"]
 393
 394     def install(self):
 395         """
 396         Install APKs on the device.
 397         """
 398         install_needed = (not self.test_suite) or self.config["suite_definitions"][
 399             self.test_suite
 400         ].get("install")
 401         if install_needed is False:
 402             self.info("Skipping apk installation for %s" % self.test_suite)
 403             return
 404         assert (
 405             self.installer_path is not None
 406         ), "Either add installer_path to the config or use --installer-path."
 407         self.uninstall_android_app()
 408         self.install_android_app(self.installer_path)
 409         self.info("Finished installing apps for %s" % self.device_name)
 410
 411     def run_tests(self):
 412         """
 413         Run the tests
 414         """
 415         self.start_time = datetime.datetime.now()
 416         max_per_test_time = datetime.timedelta(minutes=60)
 417
 418         per_test_args = []
 419         suites = self._query_suites()
 420         minidump = self.query_minidump_stackwalk()
 421         for per_test_suite, suite in suites:
 422             self.test_suite = suite
 423
 424             try:
 425                 cwd = self._query_tests_dir()
 426             except Exception:
 427                 self.fatal("Don't know how to run --test-suite '%s'!" % self.test_suite)
 428             env = self.query_env()
 429             if minidump:
 430                 env["MINIDUMP_STACKWALK"] = minidump
 431             env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"]
 432             env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"]
 433             env["RUST_BACKTRACE"] = "full"
 434
 435             summary = None
 436             for per_test_args in self.query_args(per_test_suite):
 437                 if (datetime.datetime.now() - self.start_time) > max_per_test_time:
 438                     # Running tests has run out of time. That is okay! Stop running
 439                     # them so that a task timeout is not triggered, and so that
 440                     # (partial) results are made available in a timely manner.
 441                     self.info(
 442                         "TinderboxPrint: Running tests took too long: "
 443                         "Not all tests were executed.<br/>"
 444                     )
 445                     # Signal per-test time exceeded, to break out of suites and
 446                     # suite categories loops also.
 447                     return
 448
 449                 cmd = self._build_command()
 450                 final_cmd = copy.copy(cmd)
 451                 if len(per_test_args) > 0:
 452                     # in per-test mode, remove any chunk arguments from command
 453                     for arg in final_cmd:
 454                         if "total-chunk" in arg or "this-chunk" in arg:
 455                             final_cmd.remove(arg)
 456                 final_cmd.extend(per_test_args)
 457
 458                 self.info(
 459                     "Running on %s the command %s"
 460                     % (self.device_name, subprocess.list2cmdline(final_cmd))
 461                 )
 462                 self.info("##### %s log begins" % self.test_suite)
 463
 464                 suite_category = self.test_suite
 465                 parser = self.get_test_output_parser(
 466                     suite_category,
 467                     config=self.config,
 468                     log_obj=self.log_obj,
 469                     error_list=[],
 470                 )
 471                 self.run_command(final_cmd, cwd=cwd, env=env, output_parser=parser)
 472                 tbpl_status, log_level, summary = parser.evaluate_parser(0, summary)
 473                 parser.append_tinderboxprint_line(self.test_suite)
 474
 475                 self.info("##### %s log ends" % self.test_suite)
 476
 477                 if len(per_test_args) > 0:
 478                     self.record_status(tbpl_status, level=log_level)
 479                     self.log_per_test_status(per_test_args[-1], tbpl_status, log_level)
 480                     if tbpl_status == TBPL_RETRY:
 481                         self.info("Per-test run abandoned due to RETRY status")
 482                         return
 483                 else:
 484                     self.record_status(tbpl_status, level=log_level)
 485                     # report as INFO instead of log_level to avoid extra Treeherder lines
 486                     self.info(
 487                         "The %s suite: %s ran with return status: %s"
 488                         % (suite_category, suite, tbpl_status),
 489                     )
 490
 491
 492 if __name__ == "__main__":
 493     test = AndroidHardwareTest()
 494     test.run_and_exit()