testing/mozharness/scripts/android_emulator_unittest.py

   1 #!/usr/bin/env python
   2 # ***** BEGIN LICENSE BLOCK *****
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
   5 # You can obtain one at http://mozilla.org/MPL/2.0/.
   6 # ***** END LICENSE BLOCK *****
   7
   8 import copy
   9 import datetime
  10 import json
  11 import os
  12 import subprocess
  13 import sys
  14
  15 # load modules from parent dir
  16 here = os.path.abspath(os.path.dirname(__file__))
  17 sys.path.insert(1, os.path.dirname(here))
  18
  19 from mozharness.base.log import WARNING
  20 from mozharness.base.script import BaseScript, PreScriptAction
  21 from mozharness.mozilla.automation import TBPL_RETRY
  22 from mozharness.mozilla.mozbase import MozbaseMixin
  23 from mozharness.mozilla.testing.android import AndroidMixin
  24 from mozharness.mozilla.testing.codecoverage import (
  25     CodeCoverageMixin,
  26     code_coverage_config_options,
  27 )
  28 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
  29
  30 SUITE_DEFAULT_E10S = ["geckoview-junit", "mochitest", "reftest"]
  31 SUITE_NO_E10S = ["cppunittest", "gtest", "jittest", "xpcshell"]
  32 SUITE_REPEATABLE = ["mochitest", "reftest", "xpcshell"]
  33
  34
  35 class AndroidEmulatorTest(
  36     TestingMixin, BaseScript, MozbaseMixin, CodeCoverageMixin, AndroidMixin
  37 ):
  38     """
  39     A mozharness script for Android functional tests (like mochitests and reftests)
  40     run on an Android emulator. This script starts and manages an Android emulator
  41     for the duration of the required tests. This is like desktop_unittest.py, but
  42     for Android emulator test platforms.
  43     """
  44
  45     config_options = (
  46         [
  47             [
  48                 ["--test-suite"],
  49                 {"action": "store", "dest": "test_suite", "default": None},
  50             ],
  51             [
  52                 ["--total-chunk"],
  53                 {
  54                     "action": "store",
  55                     "dest": "total_chunks",
  56                     "default": None,
  57                     "help": "Number of total chunks",
  58                 },
  59             ],
  60             [
  61                 ["--this-chunk"],
  62                 {
  63                     "action": "store",
  64                     "dest": "this_chunk",
  65                     "default": None,
  66                     "help": "Number of this chunk",
  67                 },
  68             ],
  69             [
  70                 ["--enable-xorigin-tests"],
  71                 {
  72                     "action": "store_true",
  73                     "dest": "enable_xorigin_tests",
  74                     "default": False,
  75                     "help": "Run tests in a cross origin iframe.",
  76                 },
  77             ],
  78             [
  79                 ["--gpu-required"],
  80                 {
  81                     "action": "store_true",
  82                     "dest": "gpu_required",
  83                     "default": False,
  84                     "help": "Run additional verification on modified tests using gpu instances.",
  85                 },
  86             ],
  87             [
  88                 ["--log-raw-level"],
  89                 {
  90                     "action": "store",
  91                     "dest": "log_raw_level",
  92                     "default": "info",
  93                     "help": "Set log level (debug|info|warning|error|critical|fatal)",
  94                 },
  95             ],
  96             [
  97                 ["--log-tbpl-level"],
  98                 {
  99                     "action": "store",
 100                     "dest": "log_tbpl_level",
 101                     "default": "info",
 102                     "help": "Set log level (debug|info|warning|error|critical|fatal)",
 103                 },
 104             ],
 105             [
 106                 ["--disable-e10s"],
 107                 {
 108                     "action": "store_false",
 109                     "dest": "e10s",
 110                     "default": True,
 111                     "help": "Run tests without multiple processes (e10s).",
 112                 },
 113             ],
 114             [
 115                 ["--disable-fission"],
 116                 {
 117                     "action": "store_true",
 118                     "dest": "disable_fission",
 119                     "default": False,
 120                     "help": "Run without Fission enabled.",
 121                 },
 122             ],
 123             [
 124                 ["--web-content-isolation-strategy"],
 125                 {
 126                     "action": "store",
 127                     "type": "int",
 128                     "dest": "web_content_isolation_strategy",
 129                     "help": "Strategy used to determine whether or not a particular site should"
 130                     "load into a webIsolated content process, see "
 131                     "fission.webContentIsolationStrategy.",
 132                 },
 133             ],
 134             [
 135                 ["--repeat"],
 136                 {
 137                     "action": "store",
 138                     "type": "int",
 139                     "dest": "repeat",
 140                     "default": 0,
 141                     "help": "Repeat the tests the given number of times. Supported "
 142                     "by mochitest, reftest, crashtest, ignored otherwise.",
 143                 },
 144             ],
 145             [
 146                 ["--setpref"],
 147                 {
 148                     "action": "append",
 149                     "metavar": "PREF=VALUE",
 150                     "dest": "extra_prefs",
 151                     "default": [],
 152                     "help": "Extra user prefs.",
 153                 },
 154             ],
 155         ]
 156         + copy.deepcopy(testing_config_options)
 157         + copy.deepcopy(code_coverage_config_options)
 158     )
 159
 160     def __init__(self, require_config_file=False):
 161         super(AndroidEmulatorTest, self).__init__(
 162             config_options=self.config_options,
 163             all_actions=[
 164                 "clobber",
 165                 "download-and-extract",
 166                 "create-virtualenv",
 167                 "start-emulator",
 168                 "verify-device",
 169                 "install",
 170                 "run-tests",
 171             ],
 172             require_config_file=require_config_file,
 173             config={
 174                 "virtualenv_modules": [],
 175                 "virtualenv_requirements": [],
 176                 "require_test_zip": True,
 177             },
 178         )
 179
 180         # these are necessary since self.config is read only
 181         c = self.config
 182         self.installer_url = c.get("installer_url")
 183         self.installer_path = c.get("installer_path")
 184         self.test_url = c.get("test_url")
 185         self.test_packages_url = c.get("test_packages_url")
 186         self.test_manifest = c.get("test_manifest")
 187         suite = c.get("test_suite")
 188         self.test_suite = suite
 189         self.this_chunk = c.get("this_chunk")
 190         self.total_chunks = c.get("total_chunks")
 191         self.xre_path = None
 192         self.device_serial = "emulator-5554"
 193         self.log_raw_level = c.get("log_raw_level")
 194         self.log_tbpl_level = c.get("log_tbpl_level")
 195         # AndroidMixin uses this when launching the emulator. We only want
 196         # GLES3 if we're running WebRender (default)
 197         self.use_gles3 = True
 198         self.disable_e10s = c.get("disable_e10s")
 199         self.disable_fission = c.get("disable_fission")
 200         self.web_content_isolation_strategy = c.get("web_content_isolation_strategy")
 201         self.extra_prefs = c.get("extra_prefs")
 202
 203     def query_abs_dirs(self):
 204         if self.abs_dirs:
 205             return self.abs_dirs
 206         abs_dirs = super(AndroidEmulatorTest, self).query_abs_dirs()
 207         dirs = {}
 208         dirs["abs_test_install_dir"] = os.path.join(abs_dirs["abs_work_dir"], "tests")
 209         dirs["abs_test_bin_dir"] = os.path.join(
 210             abs_dirs["abs_work_dir"], "tests", "bin"
 211         )
 212         dirs["abs_xre_dir"] = os.path.join(abs_dirs["abs_work_dir"], "hostutils")
 213         dirs["abs_modules_dir"] = os.path.join(dirs["abs_test_install_dir"], "modules")
 214         dirs["abs_blob_upload_dir"] = os.path.join(
 215             abs_dirs["abs_work_dir"], "blobber_upload_dir"
 216         )
 217         dirs["abs_mochitest_dir"] = os.path.join(
 218             dirs["abs_test_install_dir"], "mochitest"
 219         )
 220         dirs["abs_reftest_dir"] = os.path.join(dirs["abs_test_install_dir"], "reftest")
 221         dirs["abs_xpcshell_dir"] = os.path.join(
 222             dirs["abs_test_install_dir"], "xpcshell"
 223         )
 224         work_dir = os.environ.get("MOZ_FETCHES_DIR") or abs_dirs["abs_work_dir"]
 225         dirs["abs_sdk_dir"] = os.path.join(work_dir, "android-sdk-linux")
 226         dirs["abs_avds_dir"] = os.path.join(work_dir, "android-device")
 227         dirs["abs_bundletool_path"] = os.path.join(work_dir, "bundletool.jar")
 228
 229         for key in dirs.keys():
 230             if key not in abs_dirs:
 231                 abs_dirs[key] = dirs[key]
 232         self.abs_dirs = abs_dirs
 233         return self.abs_dirs
 234
 235     def _query_tests_dir(self, test_suite):
 236         dirs = self.query_abs_dirs()
 237         try:
 238             test_dir = self.config["suite_definitions"][test_suite]["testsdir"]
 239         except Exception:
 240             test_dir = test_suite
 241         return os.path.join(dirs["abs_test_install_dir"], test_dir)
 242
 243     def _get_mozharness_test_paths(self, suite):
 244         test_paths = os.environ.get("MOZHARNESS_TEST_PATHS")
 245         if not test_paths:
 246             return
 247
 248         return json.loads(test_paths).get(suite)
 249
 250     def _build_command(self):
 251         c = self.config
 252         dirs = self.query_abs_dirs()
 253
 254         if self.test_suite not in self.config["suite_definitions"]:
 255             self.fatal("Key '%s' not defined in the config!" % self.test_suite)
 256
 257         cmd = [
 258             self.query_python_path("python"),
 259             "-u",
 260             os.path.join(
 261                 self._query_tests_dir(self.test_suite),
 262                 self.config["suite_definitions"][self.test_suite]["run_filename"],
 263             ),
 264         ]
 265
 266         raw_log_file, error_summary_file = self.get_indexed_logs(
 267             dirs["abs_blob_upload_dir"], self.test_suite
 268         )
 269
 270         str_format_values = {
 271             "device_serial": self.device_serial,
 272             # IP address of the host as seen from the emulator
 273             "remote_webserver": "10.0.2.2",
 274             "xre_path": self.xre_path,
 275             "utility_path": self.xre_path,
 276             "http_port": "8854",  # starting http port  to use for the mochitest server
 277             "ssl_port": "4454",  # starting ssl port to use for the server
 278             "certs_path": os.path.join(dirs["abs_work_dir"], "tests/certs"),
 279             # TestingMixin._download_and_extract_symbols() will set
 280             # self.symbols_path when downloading/extracting.
 281             "symbols_path": self.symbols_path,
 282             "modules_dir": dirs["abs_modules_dir"],
 283             "installer_path": self.installer_path,
 284             "raw_log_file": raw_log_file,
 285             "log_tbpl_level": self.log_tbpl_level,
 286             "log_raw_level": self.log_raw_level,
 287             "error_summary_file": error_summary_file,
 288             "xpcshell_extra": c.get("xpcshell_extra", ""),
 289             "gtest_dir": os.path.join(dirs["abs_test_install_dir"], "gtest"),
 290         }
 291
 292         user_paths = self._get_mozharness_test_paths(self.test_suite)
 293
 294         for option in self.config["suite_definitions"][self.test_suite]["options"]:
 295             opt = option.split("=")[0]
 296             # override configured chunk options with script args, if specified
 297             if opt in ("--this-chunk", "--total-chunks"):
 298                 if (
 299                     user_paths
 300                     or getattr(self, opt.replace("-", "_").strip("_"), None) is not None
 301                 ):
 302                     continue
 303
 304             if "%(app)" in option:
 305                 # only query package name if requested
 306                 cmd.extend([option % {"app": self.query_package_name()}])
 307             else:
 308                 option = option % str_format_values
 309                 if option:
 310                     cmd.extend([option])
 311
 312         if "mochitest" in self.test_suite:
 313             category = "mochitest"
 314         elif "reftest" in self.test_suite or "crashtest" in self.test_suite:
 315             category = "reftest"
 316         else:
 317             category = self.test_suite
 318         if c.get("repeat"):
 319             if category in SUITE_REPEATABLE:
 320                 cmd.extend(["--repeat=%s" % c.get("repeat")])
 321             else:
 322                 self.log("--repeat not supported in {}".format(category), level=WARNING)
 323
 324         # do not add --disable fission if we don't have --disable-e10s
 325         if c["disable_fission"] and category not in ["gtest", "cppunittest"]:
 326             cmd.append("--disable-fission")
 327
 328         if "web_content_isolation_strategy" in c:
 329             cmd.append(
 330                 "--web-content-isolation-strategy=%s"
 331                 % c["web_content_isolation_strategy"]
 332             )
 333         cmd.extend(["--setpref={}".format(p) for p in self.extra_prefs])
 334
 335         if not (self.verify_enabled or self.per_test_coverage):
 336             if user_paths:
 337                 cmd.extend(user_paths)
 338             elif not (self.verify_enabled or self.per_test_coverage):
 339                 if self.this_chunk is not None:
 340                     cmd.extend(["--this-chunk", self.this_chunk])
 341                 if self.total_chunks is not None:
 342                     cmd.extend(["--total-chunks", self.total_chunks])
 343
 344         if category not in SUITE_NO_E10S:
 345             if category in SUITE_DEFAULT_E10S and not c["e10s"]:
 346                 cmd.append("--disable-e10s")
 347             elif category not in SUITE_DEFAULT_E10S and c["e10s"]:
 348                 cmd.append("--e10s")
 349
 350         if c.get("enable_xorigin_tests"):
 351             cmd.extend(["--enable-xorigin-tests"])
 352
 353         try_options, try_tests = self.try_args(self.test_suite)
 354         cmd.extend(try_options)
 355         if not self.verify_enabled and not self.per_test_coverage:
 356             cmd.extend(
 357                 self.query_tests_args(
 358                     self.config["suite_definitions"][self.test_suite].get("tests"),
 359                     None,
 360                     try_tests,
 361                 )
 362             )
 363
 364         if self.java_code_coverage_enabled:
 365             cmd.extend(
 366                 [
 367                     "--enable-coverage",
 368                     "--coverage-output-dir",
 369                     self.java_coverage_output_dir,
 370                 ]
 371             )
 372
 373         return cmd
 374
 375     def _query_suites(self):
 376         if self.test_suite:
 377             return [(self.test_suite, self.test_suite)]
 378         # per-test mode: determine test suites to run
 379
 380         # For each test category, provide a list of supported sub-suites and a mapping
 381         # between the per_test_base suite name and the android suite name.
 382         all = [
 383             (
 384                 "mochitest",
 385                 {
 386                     "mochitest-plain": "mochitest-plain",
 387                     "mochitest-media": "mochitest-media",
 388                     "mochitest-plain-gpu": "mochitest-plain-gpu",
 389                 },
 390             ),
 391             (
 392                 "reftest",
 393                 {
 394                     "reftest": "reftest",
 395                     "crashtest": "crashtest",
 396                     "jsreftest": "jsreftest",
 397                 },
 398             ),
 399             ("xpcshell", {"xpcshell": "xpcshell"}),
 400         ]
 401         suites = []
 402         for category, all_suites in all:
 403             cat_suites = self.query_per_test_category_suites(category, all_suites)
 404             for k in cat_suites.keys():
 405                 suites.append((k, cat_suites[k]))
 406         return suites
 407
 408     def _query_suite_categories(self):
 409         if self.test_suite:
 410             categories = [self.test_suite]
 411         else:
 412             # per-test mode
 413             categories = ["mochitest", "reftest", "xpcshell"]
 414         return categories
 415
 416     ##########################################
 417     # Actions for AndroidEmulatorTest        #
 418     ##########################################
 419
 420     def preflight_install(self):
 421         # in the base class, this checks for mozinstall, but we don't use it
 422         pass
 423
 424     @PreScriptAction("create-virtualenv")
 425     def pre_create_virtualenv(self, action):
 426         dirs = self.query_abs_dirs()
 427         requirements = None
 428         suites = self._query_suites()
 429         if ("mochitest-media", "mochitest-media") in suites:
 430             # mochitest-media is the only thing that needs this
 431             requirements = os.path.join(
 432                 dirs["abs_mochitest_dir"],
 433                 "websocketprocessbridge",
 434                 "websocketprocessbridge_requirements_3.txt",
 435             )
 436         if requirements:
 437             self.register_virtualenv_module(requirements=[requirements], two_pass=True)
 438
 439     def download_and_extract(self):
 440         """
 441         Download and extract product APK, tests.zip, and host utils.
 442         """
 443         super(AndroidEmulatorTest, self).download_and_extract(
 444             suite_categories=self._query_suite_categories()
 445         )
 446         dirs = self.query_abs_dirs()
 447         self.xre_path = self.download_hostutils(dirs["abs_xre_dir"])
 448
 449     def install(self):
 450         """
 451         Install APKs on the device.
 452         """
 453         install_needed = (not self.test_suite) or self.config["suite_definitions"][
 454             self.test_suite
 455         ].get("install")
 456         if install_needed is False:
 457             self.info("Skipping apk installation for %s" % self.test_suite)
 458             return
 459         assert (
 460             self.installer_path is not None
 461         ), "Either add installer_path to the config or use --installer-path."
 462         self.install_android_app(self.installer_path)
 463         self.info("Finished installing apps for %s" % self.device_serial)
 464
 465     def run_tests(self):
 466         """
 467         Run the tests
 468         """
 469         self.start_time = datetime.datetime.now()
 470         max_per_test_time = datetime.timedelta(minutes=60)
 471
 472         per_test_args = []
 473         suites = self._query_suites()
 474         minidump = self.query_minidump_stackwalk()
 475         for per_test_suite, suite in suites:
 476             self.test_suite = suite
 477
 478             try:
 479                 cwd = self._query_tests_dir(self.test_suite)
 480             except Exception:
 481                 self.fatal("Don't know how to run --test-suite '%s'!" % self.test_suite)
 482
 483             env = self.query_env()
 484             if minidump:
 485                 env["MINIDUMP_STACKWALK"] = minidump
 486             env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"]
 487             env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"]
 488             env["RUST_BACKTRACE"] = "full"
 489             if self.config["nodejs_path"]:
 490                 env["MOZ_NODE_PATH"] = self.config["nodejs_path"]
 491
 492             summary = {}
 493             for per_test_args in self.query_args(per_test_suite):
 494                 if (datetime.datetime.now() - self.start_time) > max_per_test_time:
 495                     # Running tests has run out of time. That is okay! Stop running
 496                     # them so that a task timeout is not triggered, and so that
 497                     # (partial) results are made available in a timely manner.
 498                     self.info(
 499                         "TinderboxPrint: Running tests took too long: "
 500                         "Not all tests were executed.<br/>"
 501                     )
 502                     # Signal per-test time exceeded, to break out of suites and
 503                     # suite categories loops also.
 504                     return
 505
 506                 cmd = self._build_command()
 507                 final_cmd = copy.copy(cmd)
 508                 if len(per_test_args) > 0:
 509                     # in per-test mode, remove any chunk arguments from command
 510                     for arg in final_cmd:
 511                         if "total-chunk" in arg or "this-chunk" in arg:
 512                             final_cmd.remove(arg)
 513                 final_cmd.extend(per_test_args)
 514
 515                 self.info("Running the command %s" % subprocess.list2cmdline(final_cmd))
 516                 self.info("##### %s log begins" % self.test_suite)
 517
 518                 suite_category = self.test_suite
 519                 parser = self.get_test_output_parser(
 520                     suite_category,
 521                     config=self.config,
 522                     log_obj=self.log_obj,
 523                     error_list=[],
 524                 )
 525                 self.run_command(final_cmd, cwd=cwd, env=env, output_parser=parser)
 526                 tbpl_status, log_level, summary = parser.evaluate_parser(
 527                     0, previous_summary=summary
 528                 )
 529                 parser.append_tinderboxprint_line(self.test_suite)
 530
 531                 self.info("##### %s log ends" % self.test_suite)
 532
 533                 if len(per_test_args) > 0:
 534                     self.record_status(tbpl_status, level=log_level)
 535                     self.log_per_test_status(per_test_args[-1], tbpl_status, log_level)
 536                     if tbpl_status == TBPL_RETRY:
 537                         self.info("Per-test run abandoned due to RETRY status")
 538                         return
 539                 else:
 540                     self.record_status(tbpl_status, level=log_level)
 541                     # report as INFO instead of log_level to avoid extra Treeherder lines
 542                     self.info(
 543                         "The %s suite: %s ran with return status: %s"
 544                         % (suite_category, suite, tbpl_status),
 545                     )
 546
 547
 548 if __name__ == "__main__":
 549     test = AndroidEmulatorTest()
 550     test.run_and_exit()