testing/mochitest/runjunit.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, print_function
   6
   7 import argparse
   8 import os
   9 import posixpath
  10 import re
  11 import shutil
  12 import six
  13 import sys
  14 import tempfile
  15 import traceback
  16
  17 import mozcrash
  18 import mozinfo
  19 import mozlog
  20 import moznetwork
  21 from mozdevice import ADBDeviceFactory, ADBError, ADBTimeoutError
  22 from mozprofile import Profile, DEFAULT_PORTS
  23 from mozprofile.cli import parse_preferences
  24 from mozprofile.permissions import ServerLocations
  25 from runtests import MochitestDesktop, update_mozinfo
  26
  27 here = os.path.abspath(os.path.dirname(__file__))
  28
  29 try:
  30     from mozbuild.base import (
  31         MozbuildObject,
  32         MachCommandConditions as conditions,
  33     )
  34     from mach.util import UserError
  35
  36     build_obj = MozbuildObject.from_environment(cwd=here)
  37 except ImportError:
  38     build_obj = None
  39     conditions = None
  40     UserError = Exception
  41
  42
  43 class JavaTestHarnessException(Exception):
  44     pass
  45
  46
  47 class JUnitTestRunner(MochitestDesktop):
  48     """
  49     A test harness to run geckoview junit tests on a remote device.
  50     """
  51
  52     def __init__(self, log, options):
  53         self.log = log
  54         self.verbose = False
  55         if (
  56             options.log_tbpl_level == "debug"
  57             or options.log_mach_level == "debug"
  58             or options.verbose
  59         ):
  60             self.verbose = True
  61         self.device = ADBDeviceFactory(
  62             adb=options.adbPath or "adb",
  63             device=options.deviceSerial,
  64             test_root=options.remoteTestRoot,
  65             verbose=self.verbose,
  66             run_as_package=options.app,
  67         )
  68         self.options = options
  69         self.log.debug("options=%s" % vars(options))
  70         update_mozinfo()
  71         self.remote_profile = posixpath.join(self.device.test_root, "junit-profile")
  72         self.remote_filter_list = posixpath.join(
  73             self.device.test_root, "junit-filters.list"
  74         )
  75
  76         if self.options.coverage and not self.options.coverage_output_dir:
  77             raise UserError(
  78                 "--coverage-output-dir is required when using --enable-coverage"
  79             )
  80         if self.options.coverage:
  81             self.remote_coverage_output_file = posixpath.join(
  82                 self.device.test_root, "junit-coverage.ec"
  83             )
  84             self.coverage_output_file = os.path.join(
  85                 self.options.coverage_output_dir, "junit-coverage.ec"
  86             )
  87
  88         self.server_init()
  89
  90         self.cleanup()
  91         self.device.clear_logcat()
  92         self.build_profile()
  93         self.startServers(self.options, debuggerInfo=None, public=True)
  94         self.log.debug("Servers started")
  95
  96     def collectLogcatForCurrentTest(self):
  97         # These are unique start and end markers logged by GeckoSessionTestRule.java
  98         START_MARKER = "1f0befec-3ff2-40ff-89cf-b127eb38b1ec"
  99         END_MARKER = "c5ee677f-bc83-49bd-9e28-2d35f3d0f059"
 100         logcat = self.device.get_logcat()
 101         test_logcat = ""
 102         started = False
 103         for l in logcat:
 104             if START_MARKER in l and self.test_name in l:
 105                 started = True
 106             if started:
 107                 test_logcat += l + "\n"
 108             if started and END_MARKER in l:
 109                 return test_logcat
 110
 111     def needsWebsocketProcessBridge(self, options):
 112         """
 113         Overrides MochitestDesktop.needsWebsocketProcessBridge and always
 114         returns False as the junit tests do not use the websocket process
 115         bridge. This is needed to satisfy MochitestDesktop.startServers.
 116         """
 117         return False
 118
 119     def server_init(self):
 120         """
 121         Additional initialization required to satisfy MochitestDesktop.startServers
 122         """
 123         self._locations = None
 124         self.server = None
 125         self.wsserver = None
 126         self.websocketProcessBridge = None
 127         self.SERVER_STARTUP_TIMEOUT = 180 if mozinfo.info.get("debug") else 90
 128         if self.options.remoteWebServer is None:
 129             self.options.remoteWebServer = moznetwork.get_ip()
 130         self.options.webServer = self.options.remoteWebServer
 131         self.options.webSocketPort = "9988"
 132         self.options.httpdPath = None
 133         self.options.keep_open = False
 134         self.options.pidFile = ""
 135         self.options.subsuite = None
 136         self.options.xrePath = None
 137         if build_obj and "MOZ_HOST_BIN" in os.environ:
 138             self.options.xrePath = os.environ["MOZ_HOST_BIN"]
 139             if not self.options.utilityPath:
 140                 self.options.utilityPath = self.options.xrePath
 141         if not self.options.xrePath:
 142             self.options.xrePath = self.options.utilityPath
 143         if build_obj:
 144             self.options.certPath = os.path.join(
 145                 build_obj.topsrcdir, "build", "pgo", "certs"
 146             )
 147
 148     def build_profile(self):
 149         """
 150         Create a local profile with test prefs and proxy definitions and
 151         push it to the remote device.
 152         """
 153
 154         self.profile = Profile(locations=self.locations, proxy=self.proxy(self.options))
 155         self.options.profilePath = self.profile.profile
 156
 157         # Set preferences
 158         self.merge_base_profiles(self.options, "geckoview-junit")
 159         prefs = parse_preferences(self.options.extra_prefs)
 160         self.profile.set_preferences(prefs)
 161
 162         if self.fillCertificateDB(self.options):
 163             self.log.error("Certificate integration failed")
 164
 165         self.device.push(self.profile.profile, self.remote_profile)
 166         self.log.debug(
 167             "profile %s -> %s" % (str(self.profile.profile), str(self.remote_profile))
 168         )
 169
 170     def cleanup(self):
 171         try:
 172             self.stopServers()
 173             self.log.debug("Servers stopped")
 174             self.device.stop_application(self.options.app)
 175             self.device.rm(self.remote_profile, force=True, recursive=True)
 176             if hasattr(self, "profile"):
 177                 del self.profile
 178             self.device.rm(self.remote_filter_list, force=True)
 179         except Exception:
 180             traceback.print_exc()
 181             self.log.info("Caught and ignored an exception during cleanup")
 182
 183     def build_command_line(self, test_filters_file, test_filters):
 184         """
 185         Construct and return the 'am instrument' command line.
 186         """
 187         cmd = "am instrument -w -r"
 188         # profile location
 189         cmd = cmd + " -e args '-profile %s'" % self.remote_profile
 190         # chunks (shards)
 191         shards = self.options.totalChunks
 192         shard = self.options.thisChunk
 193         if shards is not None and shard is not None:
 194             shard -= 1  # shard index is 0 based
 195             cmd = cmd + " -e numShards %d -e shardIndex %d" % (shards, shard)
 196
 197         # test filters: limit run to specific test(s)
 198         # filter can be class-name or 'class-name#method-name' (single test)
 199         # Multiple filters must be specified as a line-separated text file
 200         # and then pushed to the device.
 201         filter_list_name = None
 202
 203         if test_filters_file:
 204             # We specified a pre-existing file, so use that
 205             filter_list_name = test_filters_file
 206         elif test_filters:
 207             if len(test_filters) > 1:
 208                 # Generate the list file from test_filters
 209                 with tempfile.NamedTemporaryFile(delete=False, mode="w") as filter_list:
 210                     for f in test_filters:
 211                         print(f, file=filter_list)
 212                     filter_list_name = filter_list.name
 213             else:
 214                 # A single filter may be directly appended to the command line
 215                 cmd = cmd + " -e class %s" % test_filters[0]
 216
 217         if filter_list_name:
 218             self.device.push(filter_list_name, self.remote_filter_list)
 219
 220             if test_filters:
 221                 # We only remove the filter list if we generated it as a
 222                 # temporary file.
 223                 os.remove(filter_list_name)
 224
 225             cmd = cmd + " -e testFile %s" % self.remote_filter_list
 226
 227         # enable code coverage reports
 228         if self.options.coverage:
 229             cmd = cmd + " -e coverage true"
 230             cmd = cmd + " -e coverageFile %s" % self.remote_coverage_output_file
 231         # environment
 232         env = {}
 233         env["MOZ_CRASHREPORTER"] = "1"
 234         env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
 235         env["XPCOM_DEBUG_BREAK"] = "stack"
 236         env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
 237         env["MOZ_IN_AUTOMATION"] = "1"
 238         env["R_LOG_VERBOSE"] = "1"
 239         env["R_LOG_LEVEL"] = "6"
 240         env["R_LOG_DESTINATION"] = "stderr"
 241         if self.options.enable_webrender:
 242             env["MOZ_WEBRENDER"] = "1"
 243         else:
 244             env["MOZ_WEBRENDER"] = "0"
 245         if self.options.enable_fission:
 246             env["MOZ_FORCE_ENABLE_FISSION"] = "1"
 247         # Add additional env variables
 248         for [key, value] in [p.split("=", 1) for p in self.options.add_env]:
 249             env[key] = value
 250
 251         for (env_count, (env_key, env_val)) in enumerate(six.iteritems(env)):
 252             cmd = cmd + " -e env%d %s=%s" % (env_count, env_key, env_val)
 253         # runner
 254         cmd = cmd + " %s/%s" % (self.options.app, self.options.runner)
 255         return cmd
 256
 257     @property
 258     def locations(self):
 259         if self._locations is not None:
 260             return self._locations
 261         locations_file = os.path.join(here, "server-locations.txt")
 262         self._locations = ServerLocations(locations_file)
 263         return self._locations
 264
 265     def need_more_runs(self):
 266         if self.options.run_until_failure and (self.fail_count == 0):
 267             return True
 268         if self.runs <= self.options.repeat:
 269             return True
 270         return False
 271
 272     def run_tests(self, test_filters_file=None, test_filters=None):
 273         """
 274         Run the tests.
 275         """
 276         if not self.device.is_app_installed(self.options.app):
 277             raise UserError("%s is not installed" % self.options.app)
 278         if self.device.process_exist(self.options.app):
 279             raise UserError(
 280                 "%s already running before starting tests" % self.options.app
 281             )
 282         # test_filters_file and test_filters must be mutually-exclusive
 283         if test_filters_file and test_filters:
 284             raise UserError(
 285                 "Test filters may not be specified when test-filters-file is provided"
 286             )
 287
 288         self.test_started = False
 289         self.pass_count = 0
 290         self.fail_count = 0
 291         self.todo_count = 0
 292         self.runs = 0
 293
 294         def callback(line):
 295             # Output callback: Parse the raw junit log messages, translating into
 296             # treeherder-friendly test start/pass/fail messages.
 297
 298             line = six.ensure_str(line)
 299             self.log.process_output(self.options.app, str(line))
 300             # Expect per-test info like: "INSTRUMENTATION_STATUS: class=something"
 301             match = re.match(r"INSTRUMENTATION_STATUS:\s*class=(.*)", line)
 302             if match:
 303                 self.class_name = match.group(1)
 304             # Expect per-test info like: "INSTRUMENTATION_STATUS: test=something"
 305             match = re.match(r"INSTRUMENTATION_STATUS:\s*test=(.*)", line)
 306             if match:
 307                 self.test_name = match.group(1)
 308             match = re.match(r"INSTRUMENTATION_STATUS:\s*stack=(.*)", line)
 309             if match:
 310                 self.exception_message = match.group(1)
 311             if (
 312                 "org.mozilla.geckoview.test.rule.TestHarnessException"
 313                 in self.exception_message
 314             ):
 315                 # This is actually a problem in the test harness itself
 316                 raise JavaTestHarnessException(self.exception_message)
 317
 318             # Expect per-test info like: "INSTRUMENTATION_STATUS_CODE: 0|1|..."
 319             match = re.match(r"INSTRUMENTATION_STATUS_CODE:\s*([+-]?\d+)", line)
 320             if match:
 321                 status = match.group(1)
 322                 full_name = "%s.%s" % (self.class_name, self.test_name)
 323                 if full_name == self.current_full_name:
 324                     if status == "0":
 325                         message = ""
 326                         status = "PASS"
 327                         expected = "PASS"
 328                         self.pass_count += 1
 329                         if self.verbose:
 330                             self.log.info("Printing logcat for test:")
 331                             print(self.collectLogcatForCurrentTest())
 332                     elif status == "-3":  # ignored (skipped)
 333                         message = ""
 334                         status = "SKIP"
 335                         expected = "SKIP"
 336                         self.todo_count += 1
 337                     elif status == "-4":  # known fail
 338                         message = ""
 339                         status = "FAIL"
 340                         expected = "FAIL"
 341                         self.todo_count += 1
 342                     else:
 343                         if self.exception_message:
 344                             message = self.exception_message
 345                         else:
 346                             message = "status %s" % status
 347                         status = "FAIL"
 348                         expected = "PASS"
 349                         self.fail_count += 1
 350                         self.log.info("Printing logcat for test:")
 351                         print(self.collectLogcatForCurrentTest())
 352                     self.log.test_end(full_name, status, expected, message)
 353                     self.test_started = False
 354                 else:
 355                     if self.test_started:
 356                         # next test started without reporting previous status
 357                         self.fail_count += 1
 358                         status = "FAIL"
 359                         expected = "PASS"
 360                         self.log.test_end(
 361                             self.current_full_name,
 362                             status,
 363                             expected,
 364                             "missing test completion status",
 365                         )
 366                     self.log.test_start(full_name)
 367                     self.test_started = True
 368                     self.current_full_name = full_name
 369
 370         # Ideally all test names should be reported to suite_start, but these test
 371         # names are not known in advance.
 372         self.log.suite_start(["geckoview-junit"])
 373         try:
 374             self.device.grant_runtime_permissions(self.options.app)
 375             cmd = self.build_command_line(
 376                 test_filters_file=test_filters_file, test_filters=test_filters
 377             )
 378             while self.need_more_runs():
 379                 self.class_name = ""
 380                 self.exception_message = ""
 381                 self.test_name = ""
 382                 self.current_full_name = ""
 383                 self.runs += 1
 384                 self.log.info("launching %s" % cmd)
 385                 p = self.device.shell(
 386                     cmd, timeout=self.options.max_time, stdout_callback=callback
 387                 )
 388                 if p.timedout:
 389                     self.log.error(
 390                         "TEST-UNEXPECTED-TIMEOUT | runjunit.py | "
 391                         "Timed out after %d seconds" % self.options.max_time
 392                     )
 393             self.log.info("Passed: %d" % self.pass_count)
 394             self.log.info("Failed: %d" % self.fail_count)
 395             self.log.info("Todo: %d" % self.todo_count)
 396         finally:
 397             self.log.suite_end()
 398
 399         if self.check_for_crashes():
 400             self.fail_count = 1
 401
 402         if self.options.coverage:
 403             try:
 404                 self.device.pull(
 405                     self.remote_coverage_output_file, self.coverage_output_file
 406                 )
 407             except ADBError:
 408                 # Avoid a task retry in case the code coverage file is not found.
 409                 self.log.error(
 410                     "No code coverage file (%s) found on remote device"
 411                     % self.remote_coverage_output_file
 412                 )
 413                 return -1
 414
 415         return 1 if self.fail_count else 0
 416
 417     def check_for_crashes(self):
 418         symbols_path = self.options.symbolsPath
 419         try:
 420             dump_dir = tempfile.mkdtemp()
 421             remote_dir = posixpath.join(self.remote_profile, "minidumps")
 422             if not self.device.is_dir(remote_dir):
 423                 return False
 424             self.device.pull(remote_dir, dump_dir)
 425             crashed = mozcrash.log_crashes(
 426                 self.log, dump_dir, symbols_path, test=self.current_full_name
 427             )
 428         finally:
 429             try:
 430                 shutil.rmtree(dump_dir)
 431             except Exception:
 432                 self.log.warning("unable to remove directory: %s" % dump_dir)
 433         return crashed
 434
 435
 436 class JunitArgumentParser(argparse.ArgumentParser):
 437     """
 438     An argument parser for geckoview-junit.
 439     """
 440
 441     def __init__(self, **kwargs):
 442         super(JunitArgumentParser, self).__init__(**kwargs)
 443
 444         self.add_argument(
 445             "--appname",
 446             action="store",
 447             type=str,
 448             dest="app",
 449             default="org.mozilla.geckoview.test",
 450             help="Test package name.",
 451         )
 452         self.add_argument(
 453             "--adbpath",
 454             action="store",
 455             type=str,
 456             dest="adbPath",
 457             default=None,
 458             help="Path to adb binary.",
 459         )
 460         self.add_argument(
 461             "--deviceSerial",
 462             action="store",
 463             type=str,
 464             dest="deviceSerial",
 465             help="adb serial number of remote device. This is required "
 466             "when more than one device is connected to the host. "
 467             "Use 'adb devices' to see connected devices. ",
 468         )
 469         self.add_argument(
 470             "--setenv",
 471             dest="add_env",
 472             action="append",
 473             default=[],
 474             help="Set target environment variable, like FOO=BAR",
 475         )
 476         self.add_argument(
 477             "--remoteTestRoot",
 478             action="store",
 479             type=str,
 480             dest="remoteTestRoot",
 481             help="Remote directory to use as test root "
 482             "(eg. /data/local/tmp/test_root).",
 483         )
 484         self.add_argument(
 485             "--max-time",
 486             action="store",
 487             type=int,
 488             dest="max_time",
 489             default="2400",
 490             help="Max time in seconds to wait for tests (default 2400s).",
 491         )
 492         self.add_argument(
 493             "--runner",
 494             action="store",
 495             type=str,
 496             dest="runner",
 497             default="androidx.test.runner.AndroidJUnitRunner",
 498             help="Test runner name.",
 499         )
 500         self.add_argument(
 501             "--symbols-path",
 502             action="store",
 503             type=str,
 504             dest="symbolsPath",
 505             default=None,
 506             help="Path to directory containing breakpad symbols, "
 507             "or the URL of a zip file containing symbols.",
 508         )
 509         self.add_argument(
 510             "--utility-path",
 511             action="store",
 512             type=str,
 513             dest="utilityPath",
 514             default=None,
 515             help="Path to directory containing host utility programs.",
 516         )
 517         self.add_argument(
 518             "--total-chunks",
 519             action="store",
 520             type=int,
 521             dest="totalChunks",
 522             default=None,
 523             help="Total number of chunks to split tests into.",
 524         )
 525         self.add_argument(
 526             "--this-chunk",
 527             action="store",
 528             type=int,
 529             dest="thisChunk",
 530             default=None,
 531             help="If running tests by chunks, the chunk number to run.",
 532         )
 533         self.add_argument(
 534             "--verbose",
 535             "-v",
 536             action="store_true",
 537             dest="verbose",
 538             default=False,
 539             help="Verbose output - enable debug log messages",
 540         )
 541         self.add_argument(
 542             "--enable-coverage",
 543             action="store_true",
 544             dest="coverage",
 545             default=False,
 546             help="Enable code coverage collection.",
 547         )
 548         self.add_argument(
 549             "--coverage-output-dir",
 550             action="store",
 551             type=str,
 552             dest="coverage_output_dir",
 553             default=None,
 554             help="If collecting code coverage, save the report file in this dir.",
 555         )
 556         self.add_argument(
 557             "--enable-webrender",
 558             action="store_true",
 559             dest="enable_webrender",
 560             default=False,
 561             help="Enable the WebRender compositor in Gecko.",
 562         )
 563         self.add_argument(
 564             "--enable-fission",
 565             action="store_true",
 566             dest="enable_fission",
 567             default=False,
 568             help="Run the tests with Fission (site isolation) enabled.",
 569         )
 570         self.add_argument(
 571             "--repeat",
 572             type=int,
 573             default=0,
 574             help="Repeat the tests the given number of times.",
 575         )
 576         self.add_argument(
 577             "--run-until-failure",
 578             action="store_true",
 579             dest="run_until_failure",
 580             default=False,
 581             help="Run tests repeatedly but stop the first time a test fails.",
 582         )
 583         self.add_argument(
 584             "--setpref",
 585             action="append",
 586             dest="extra_prefs",
 587             default=[],
 588             metavar="PREF=VALUE",
 589             help="Defines an extra user preference.",
 590         )
 591         # Additional options for server.
 592         self.add_argument(
 593             "--certificate-path",
 594             action="store",
 595             type=str,
 596             dest="certPath",
 597             default=None,
 598             help="Path to directory containing certificate store.",
 599         ),
 600         self.add_argument(
 601             "--http-port",
 602             action="store",
 603             type=str,
 604             dest="httpPort",
 605             default=DEFAULT_PORTS["http"],
 606             help="http port of the remote web server.",
 607         )
 608         self.add_argument(
 609             "--remote-webserver",
 610             action="store",
 611             type=str,
 612             dest="remoteWebServer",
 613             help="IP address of the remote web server.",
 614         )
 615         self.add_argument(
 616             "--ssl-port",
 617             action="store",
 618             type=str,
 619             dest="sslPort",
 620             default=DEFAULT_PORTS["https"],
 621             help="ssl port of the remote web server.",
 622         )
 623         self.add_argument(
 624             "--test-filters-file",
 625             action="store",
 626             type=str,
 627             dest="test_filters_file",
 628             default=None,
 629             help="Line-delimited file containing test filter(s)",
 630         )
 631         # Remaining arguments are test filters.
 632         self.add_argument(
 633             "test_filters",
 634             nargs="*",
 635             help="Test filter(s): class and/or method names of test(s) to run.",
 636         )
 637
 638         mozlog.commandline.add_logging_group(self)
 639
 640
 641 def run_test_harness(parser, options):
 642     if hasattr(options, "log"):
 643         log = options.log
 644     else:
 645         log = mozlog.commandline.setup_logging(
 646             "runjunit", options, {"tbpl": sys.stdout}
 647         )
 648     runner = JUnitTestRunner(log, options)
 649     result = -1
 650     try:
 651         device_exception = False
 652         result = runner.run_tests(
 653             test_filters_file=options.test_filters_file,
 654             test_filters=options.test_filters,
 655         )
 656     except KeyboardInterrupt:
 657         log.info("runjunit.py | Received keyboard interrupt")
 658         result = -1
 659     except JavaTestHarnessException as e:
 660         log.error(
 661             "TEST-UNEXPECTED-FAIL | runjunit.py | The previous test failed because "
 662             "of an error in the test harness | %s" % (str(e))
 663         )
 664     except Exception as e:
 665         traceback.print_exc()
 666         log.error("runjunit.py | Received unexpected exception while running tests")
 667         result = 1
 668         if isinstance(e, ADBTimeoutError):
 669             device_exception = True
 670     finally:
 671         if not device_exception:
 672             runner.cleanup()
 673     return result
 674
 675
 676 def main(args=sys.argv[1:]):
 677     parser = JunitArgumentParser()
 678     options = parser.parse_args()
 679     return run_test_harness(parser, options)
 680
 681
 682 if __name__ == "__main__":
 683     sys.exit(main())