testing/mochitest/runjunit.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import argparse
   6 import os
   7 import posixpath
   8 import re
   9 import shutil
  10 import sys
  11 import tempfile
  12 import traceback
  13
  14 import mozcrash
  15 import mozinfo
  16 import mozlog
  17 import moznetwork
  18 import six
  19 from mozdevice import ADBDeviceFactory, ADBError, ADBTimeoutError
  20 from mozprofile import DEFAULT_PORTS, Profile
  21 from mozprofile.cli import parse_preferences
  22 from mozprofile.permissions import ServerLocations
  23 from runtests import MochitestDesktop, update_mozinfo
  24
  25 here = os.path.abspath(os.path.dirname(__file__))
  26
  27 try:
  28     from mach.util import UserError
  29     from mozbuild.base import MachCommandConditions as conditions
  30     from mozbuild.base import MozbuildObject
  31
  32     build_obj = MozbuildObject.from_environment(cwd=here)
  33 except ImportError:
  34     build_obj = None
  35     conditions = None
  36     UserError = Exception
  37
  38
  39 class JavaTestHarnessException(Exception):
  40     pass
  41
  42
  43 class JUnitTestRunner(MochitestDesktop):
  44     """
  45     A test harness to run geckoview junit tests on a remote device.
  46     """
  47
  48     def __init__(self, log, options):
  49         self.log = log
  50         self.verbose = False
  51         self.http3Server = None
  52         self.http2Server = None
  53         self.dohServer = None
  54         if (
  55             options.log_tbpl_level == "debug"
  56             or options.log_mach_level == "debug"
  57             or options.verbose
  58         ):
  59             self.verbose = True
  60         self.device = ADBDeviceFactory(
  61             adb=options.adbPath or "adb",
  62             device=options.deviceSerial,
  63             test_root=options.remoteTestRoot,
  64             verbose=self.verbose,
  65             run_as_package=options.app,
  66         )
  67         self.options = options
  68         self.log.debug("options=%s" % vars(options))
  69         update_mozinfo()
  70         self.remote_profile = posixpath.join(self.device.test_root, "junit-profile")
  71         self.remote_filter_list = posixpath.join(
  72             self.device.test_root, "junit-filters.list"
  73         )
  74
  75         if self.options.coverage and not self.options.coverage_output_dir:
  76             raise UserError(
  77                 "--coverage-output-dir is required when using --enable-coverage"
  78             )
  79         if self.options.coverage:
  80             self.remote_coverage_output_file = posixpath.join(
  81                 self.device.test_root, "junit-coverage.ec"
  82             )
  83             self.coverage_output_file = os.path.join(
  84                 self.options.coverage_output_dir, "junit-coverage.ec"
  85             )
  86
  87         self.server_init()
  88
  89         self.cleanup()
  90         self.device.clear_logcat()
  91         self.build_profile()
  92         self.startServers(self.options, debuggerInfo=None, public=True)
  93         self.log.debug("Servers started")
  94
  95     def collectLogcatForCurrentTest(self):
  96         # These are unique start and end markers logged by GeckoSessionTestRule.java
  97         START_MARKER = "1f0befec-3ff2-40ff-89cf-b127eb38b1ec"
  98         END_MARKER = "c5ee677f-bc83-49bd-9e28-2d35f3d0f059"
  99         logcat = self.device.get_logcat()
 100         test_logcat = ""
 101         started = False
 102         for l in logcat:
 103             if START_MARKER in l and self.test_name in l:
 104                 started = True
 105             if started:
 106                 test_logcat += l + "\n"
 107             if started and END_MARKER in l:
 108                 return test_logcat
 109
 110     def needsWebsocketProcessBridge(self, options):
 111         """
 112         Overrides MochitestDesktop.needsWebsocketProcessBridge and always
 113         returns False as the junit tests do not use the websocket process
 114         bridge. This is needed to satisfy MochitestDesktop.startServers.
 115         """
 116         return False
 117
 118     def server_init(self):
 119         """
 120         Additional initialization required to satisfy MochitestDesktop.startServers
 121         """
 122         self._locations = None
 123         self.server = None
 124         self.wsserver = None
 125         self.websocketProcessBridge = None
 126         self.SERVER_STARTUP_TIMEOUT = 180 if mozinfo.info.get("debug") else 90
 127         if self.options.remoteWebServer is None:
 128             self.options.remoteWebServer = moznetwork.get_ip()
 129         self.options.webServer = self.options.remoteWebServer
 130         self.options.webSocketPort = "9988"
 131         self.options.httpdPath = None
 132         self.options.http3ServerPath = None
 133         self.options.http2ServerPath = None
 134         self.options.keep_open = False
 135         self.options.pidFile = ""
 136         self.options.subsuite = None
 137         self.options.xrePath = None
 138         self.options.useHttp3Server = False
 139         self.options.useHttp2Server = False
 140         if build_obj and "MOZ_HOST_BIN" in os.environ:
 141             self.options.xrePath = os.environ["MOZ_HOST_BIN"]
 142             if not self.options.utilityPath:
 143                 self.options.utilityPath = self.options.xrePath
 144         if not self.options.xrePath:
 145             self.options.xrePath = self.options.utilityPath
 146         if build_obj:
 147             self.options.certPath = os.path.join(
 148                 build_obj.topsrcdir, "build", "pgo", "certs"
 149             )
 150
 151     def build_profile(self):
 152         """
 153         Create a local profile with test prefs and proxy definitions and
 154         push it to the remote device.
 155         """
 156
 157         self.profile = Profile(locations=self.locations, proxy=self.proxy(self.options))
 158         self.options.profilePath = self.profile.profile
 159
 160         # Set preferences
 161         self.merge_base_profiles(self.options, "geckoview-junit")
 162
 163         if self.options.web_content_isolation_strategy is not None:
 164             self.options.extra_prefs.append(
 165                 "fission.webContentIsolationStrategy=%s"
 166                 % self.options.web_content_isolation_strategy
 167             )
 168         self.options.extra_prefs.append("fission.autostart=true")
 169         if self.options.disable_fission:
 170             self.options.extra_prefs.pop()
 171             self.options.extra_prefs.append("fission.autostart=false")
 172         prefs = parse_preferences(self.options.extra_prefs)
 173         self.profile.set_preferences(prefs)
 174
 175         if self.fillCertificateDB(self.options):
 176             self.log.error("Certificate integration failed")
 177
 178         self.device.push(self.profile.profile, self.remote_profile)
 179         self.log.debug(
 180             "profile %s -> %s" % (str(self.profile.profile), str(self.remote_profile))
 181         )
 182
 183     def cleanup(self):
 184         try:
 185             self.stopServers()
 186             self.log.debug("Servers stopped")
 187             self.device.stop_application(self.options.app)
 188             self.device.rm(self.remote_profile, force=True, recursive=True)
 189             if hasattr(self, "profile"):
 190                 del self.profile
 191             self.device.rm(self.remote_filter_list, force=True)
 192         except Exception:
 193             traceback.print_exc()
 194             self.log.info("Caught and ignored an exception during cleanup")
 195
 196     def build_command_line(self, test_filters_file, test_filters):
 197         """
 198         Construct and return the 'am instrument' command line.
 199         """
 200         cmd = "am instrument -w -r"
 201         # profile location
 202         cmd = cmd + " -e args '-profile %s'" % self.remote_profile
 203         # chunks (shards)
 204         shards = self.options.totalChunks
 205         shard = self.options.thisChunk
 206         if shards is not None and shard is not None:
 207             shard -= 1  # shard index is 0 based
 208             cmd = cmd + " -e numShards %d -e shardIndex %d" % (shards, shard)
 209
 210         # test filters: limit run to specific test(s)
 211         # filter can be class-name or 'class-name#method-name' (single test)
 212         # Multiple filters must be specified as a line-separated text file
 213         # and then pushed to the device.
 214         filter_list_name = None
 215
 216         if test_filters_file:
 217             # We specified a pre-existing file, so use that
 218             filter_list_name = test_filters_file
 219         elif test_filters:
 220             if len(test_filters) > 1:
 221                 # Generate the list file from test_filters
 222                 with tempfile.NamedTemporaryFile(delete=False, mode="w") as filter_list:
 223                     for f in test_filters:
 224                         print(f, file=filter_list)
 225                     filter_list_name = filter_list.name
 226             else:
 227                 # A single filter may be directly appended to the command line
 228                 cmd = cmd + " -e class %s" % test_filters[0]
 229
 230         if filter_list_name:
 231             self.device.push(filter_list_name, self.remote_filter_list)
 232
 233             if test_filters:
 234                 # We only remove the filter list if we generated it as a
 235                 # temporary file.
 236                 os.remove(filter_list_name)
 237
 238             cmd = cmd + " -e testFile %s" % self.remote_filter_list
 239
 240         # enable code coverage reports
 241         if self.options.coverage:
 242             cmd = cmd + " -e coverage true"
 243             cmd = cmd + " -e coverageFile %s" % self.remote_coverage_output_file
 244         # environment
 245         env = {}
 246         env["MOZ_CRASHREPORTER"] = "1"
 247         env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
 248         env["XPCOM_DEBUG_BREAK"] = "stack"
 249         env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
 250         env["MOZ_IN_AUTOMATION"] = "1"
 251         env["R_LOG_VERBOSE"] = "1"
 252         env["R_LOG_LEVEL"] = "6"
 253         env["R_LOG_DESTINATION"] = "stderr"
 254         # webrender needs gfx.webrender.all=true, gtest doesn't use prefs
 255         env["MOZ_WEBRENDER"] = "1"
 256         # FIXME: When android switches to using Fission by default,
 257         # MOZ_FORCE_DISABLE_FISSION will need to be configured correctly.
 258         if self.options.disable_fission:
 259             env["MOZ_FORCE_DISABLE_FISSION"] = "1"
 260         else:
 261             env["MOZ_FORCE_ENABLE_FISSION"] = "1"
 262
 263         # Add additional env variables
 264         for [key, value] in [p.split("=", 1) for p in self.options.add_env]:
 265             env[key] = value
 266
 267         for (env_count, (env_key, env_val)) in enumerate(six.iteritems(env)):
 268             cmd = cmd + " -e env%d %s=%s" % (env_count, env_key, env_val)
 269         # runner
 270         cmd = cmd + " %s/%s" % (self.options.app, self.options.runner)
 271         return cmd
 272
 273     @property
 274     def locations(self):
 275         if self._locations is not None:
 276             return self._locations
 277         locations_file = os.path.join(here, "server-locations.txt")
 278         self._locations = ServerLocations(locations_file)
 279         return self._locations
 280
 281     def need_more_runs(self):
 282         if self.options.run_until_failure and (self.fail_count == 0):
 283             return True
 284         if self.runs <= self.options.repeat:
 285             return True
 286         return False
 287
 288     def run_tests(self, test_filters_file=None, test_filters=None):
 289         """
 290         Run the tests.
 291         """
 292         if not self.device.is_app_installed(self.options.app):
 293             raise UserError("%s is not installed" % self.options.app)
 294         if self.device.process_exist(self.options.app):
 295             raise UserError(
 296                 "%s already running before starting tests" % self.options.app
 297             )
 298         # test_filters_file and test_filters must be mutually-exclusive
 299         if test_filters_file and test_filters:
 300             raise UserError(
 301                 "Test filters may not be specified when test-filters-file is provided"
 302             )
 303
 304         self.test_started = False
 305         self.pass_count = 0
 306         self.fail_count = 0
 307         self.todo_count = 0
 308         self.total_count = 0
 309         self.runs = 0
 310         self.seen_last_test = False
 311
 312         def callback(line):
 313             # Output callback: Parse the raw junit log messages, translating into
 314             # treeherder-friendly test start/pass/fail messages.
 315
 316             line = six.ensure_str(line)
 317             self.log.process_output(self.options.app, str(line))
 318             # Expect per-test info like: "INSTRUMENTATION_STATUS: class=something"
 319             match = re.match(r"INSTRUMENTATION_STATUS:\s*class=(.*)", line)
 320             if match:
 321                 self.class_name = match.group(1)
 322             # Expect per-test info like: "INSTRUMENTATION_STATUS: test=something"
 323             match = re.match(r"INSTRUMENTATION_STATUS:\s*test=(.*)", line)
 324             if match:
 325                 self.test_name = match.group(1)
 326             match = re.match(r"INSTRUMENTATION_STATUS:\s*numtests=(.*)", line)
 327             if match:
 328                 self.total_count = int(match.group(1))
 329             match = re.match(r"INSTRUMENTATION_STATUS:\s*current=(.*)", line)
 330             if match:
 331                 self.current_test_id = int(match.group(1))
 332             match = re.match(r"INSTRUMENTATION_STATUS:\s*stack=(.*)", line)
 333             if match:
 334                 self.exception_message = match.group(1)
 335             if (
 336                 "org.mozilla.geckoview.test.rule.TestHarnessException"
 337                 in self.exception_message
 338             ):
 339                 # This is actually a problem in the test harness itself
 340                 raise JavaTestHarnessException(self.exception_message)
 341
 342             # Expect per-test info like: "INSTRUMENTATION_STATUS_CODE: 0|1|..."
 343             match = re.match(r"INSTRUMENTATION_STATUS_CODE:\s*([+-]?\d+)", line)
 344             if match:
 345                 status = match.group(1)
 346                 full_name = "%s#%s" % (self.class_name, self.test_name)
 347                 if full_name == self.current_full_name:
 348                     # A crash in the test harness might cause us to ignore tests,
 349                     # so we double check that we've actually ran all the tests
 350                     if self.total_count == self.current_test_id:
 351                         self.seen_last_test = True
 352
 353                     if status == "0":
 354                         message = ""
 355                         status = "PASS"
 356                         expected = "PASS"
 357                         self.pass_count += 1
 358                         if self.verbose:
 359                             self.log.info("Printing logcat for test:")
 360                             print(self.collectLogcatForCurrentTest())
 361                     elif status == "-3":  # ignored (skipped)
 362                         message = ""
 363                         status = "SKIP"
 364                         expected = "SKIP"
 365                         self.todo_count += 1
 366                     elif status == "-4":  # known fail
 367                         message = ""
 368                         status = "FAIL"
 369                         expected = "FAIL"
 370                         self.todo_count += 1
 371                     else:
 372                         if self.exception_message:
 373                             message = self.exception_message
 374                         else:
 375                             message = "status %s" % status
 376                         status = "FAIL"
 377                         expected = "PASS"
 378                         self.fail_count += 1
 379                         self.log.info("Printing logcat for test:")
 380                         print(self.collectLogcatForCurrentTest())
 381                     self.log.test_end(full_name, status, expected, message)
 382                     self.test_started = False
 383                 else:
 384                     if self.test_started:
 385                         # next test started without reporting previous status
 386                         self.fail_count += 1
 387                         status = "FAIL"
 388                         expected = "PASS"
 389                         self.log.test_end(
 390                             self.current_full_name,
 391                             status,
 392                             expected,
 393                             "missing test completion status",
 394                         )
 395                     self.log.test_start(full_name)
 396                     self.test_started = True
 397                     self.current_full_name = full_name
 398
 399         # Ideally all test names should be reported to suite_start, but these test
 400         # names are not known in advance.
 401         self.log.suite_start(["geckoview-junit"])
 402         try:
 403             self.device.grant_runtime_permissions(self.options.app)
 404             self.device.add_change_device_settings(self.options.app)
 405             self.device.add_mock_location(self.options.app)
 406             cmd = self.build_command_line(
 407                 test_filters_file=test_filters_file, test_filters=test_filters
 408             )
 409             while self.need_more_runs():
 410                 self.class_name = ""
 411                 self.exception_message = ""
 412                 self.test_name = ""
 413                 self.current_full_name = ""
 414                 self.current_test_id = 0
 415                 self.runs += 1
 416                 self.log.info("launching %s" % cmd)
 417                 p = self.device.shell(
 418                     cmd, timeout=self.options.max_time, stdout_callback=callback
 419                 )
 420                 if p.timedout:
 421                     self.log.error(
 422                         "TEST-UNEXPECTED-TIMEOUT | runjunit.py | "
 423                         "Timed out after %d seconds" % self.options.max_time
 424                     )
 425             self.log.info("Passed: %d" % self.pass_count)
 426             self.log.info("Failed: %d" % self.fail_count)
 427             self.log.info("Todo: %d" % self.todo_count)
 428             if not self.seen_last_test:
 429                 self.log.error(
 430                     "TEST-UNEXPECTED-FAIL | runjunit.py | "
 431                     "Some tests did not run (probably due to a crash in the harness)"
 432                 )
 433         finally:
 434             self.log.suite_end()
 435
 436         if self.check_for_crashes():
 437             self.fail_count = 1
 438
 439         if self.options.coverage:
 440             try:
 441                 self.device.pull(
 442                     self.remote_coverage_output_file, self.coverage_output_file
 443                 )
 444             except ADBError:
 445                 # Avoid a task retry in case the code coverage file is not found.
 446                 self.log.error(
 447                     "No code coverage file (%s) found on remote device"
 448                     % self.remote_coverage_output_file
 449                 )
 450                 return -1
 451
 452         return 1 if self.fail_count else 0
 453
 454     def check_for_crashes(self):
 455         symbols_path = self.options.symbolsPath
 456         try:
 457             dump_dir = tempfile.mkdtemp()
 458             remote_dir = posixpath.join(self.remote_profile, "minidumps")
 459             if not self.device.is_dir(remote_dir):
 460                 return False
 461             self.device.pull(remote_dir, dump_dir)
 462             crashed = mozcrash.log_crashes(
 463                 self.log, dump_dir, symbols_path, test=self.current_full_name
 464             )
 465         finally:
 466             try:
 467                 shutil.rmtree(dump_dir)
 468             except Exception:
 469                 self.log.warning("unable to remove directory: %s" % dump_dir)
 470         return crashed
 471
 472
 473 class JunitArgumentParser(argparse.ArgumentParser):
 474     """
 475     An argument parser for geckoview-junit.
 476     """
 477
 478     def __init__(self, **kwargs):
 479         super(JunitArgumentParser, self).__init__(**kwargs)
 480
 481         self.add_argument(
 482             "--appname",
 483             action="store",
 484             type=str,
 485             dest="app",
 486             default="org.mozilla.geckoview.test",
 487             help="Test package name.",
 488         )
 489         self.add_argument(
 490             "--adbpath",
 491             action="store",
 492             type=str,
 493             dest="adbPath",
 494             default=None,
 495             help="Path to adb binary.",
 496         )
 497         self.add_argument(
 498             "--deviceSerial",
 499             action="store",
 500             type=str,
 501             dest="deviceSerial",
 502             help="adb serial number of remote device. This is required "
 503             "when more than one device is connected to the host. "
 504             "Use 'adb devices' to see connected devices. ",
 505         )
 506         self.add_argument(
 507             "--setenv",
 508             dest="add_env",
 509             action="append",
 510             default=[],
 511             help="Set target environment variable, like FOO=BAR",
 512         )
 513         self.add_argument(
 514             "--remoteTestRoot",
 515             action="store",
 516             type=str,
 517             dest="remoteTestRoot",
 518             help="Remote directory to use as test root "
 519             "(eg. /data/local/tmp/test_root).",
 520         )
 521         self.add_argument(
 522             "--max-time",
 523             action="store",
 524             type=int,
 525             dest="max_time",
 526             default="3000",
 527             help="Max time in seconds to wait for tests (default 3000s).",
 528         )
 529         self.add_argument(
 530             "--runner",
 531             action="store",
 532             type=str,
 533             dest="runner",
 534             default="androidx.test.runner.AndroidJUnitRunner",
 535             help="Test runner name.",
 536         )
 537         self.add_argument(
 538             "--symbols-path",
 539             action="store",
 540             type=str,
 541             dest="symbolsPath",
 542             default=None,
 543             help="Path to directory containing breakpad symbols, "
 544             "or the URL of a zip file containing symbols.",
 545         )
 546         self.add_argument(
 547             "--utility-path",
 548             action="store",
 549             type=str,
 550             dest="utilityPath",
 551             default=None,
 552             help="Path to directory containing host utility programs.",
 553         )
 554         self.add_argument(
 555             "--total-chunks",
 556             action="store",
 557             type=int,
 558             dest="totalChunks",
 559             default=None,
 560             help="Total number of chunks to split tests into.",
 561         )
 562         self.add_argument(
 563             "--this-chunk",
 564             action="store",
 565             type=int,
 566             dest="thisChunk",
 567             default=None,
 568             help="If running tests by chunks, the chunk number to run.",
 569         )
 570         self.add_argument(
 571             "--verbose",
 572             "-v",
 573             action="store_true",
 574             dest="verbose",
 575             default=False,
 576             help="Verbose output - enable debug log messages",
 577         )
 578         self.add_argument(
 579             "--enable-coverage",
 580             action="store_true",
 581             dest="coverage",
 582             default=False,
 583             help="Enable code coverage collection.",
 584         )
 585         self.add_argument(
 586             "--coverage-output-dir",
 587             action="store",
 588             type=str,
 589             dest="coverage_output_dir",
 590             default=None,
 591             help="If collecting code coverage, save the report file in this dir.",
 592         )
 593         self.add_argument(
 594             "--disable-fission",
 595             action="store_true",
 596             dest="disable_fission",
 597             default=False,
 598             help="Run the tests without Fission (site isolation) enabled.",
 599         )
 600         self.add_argument(
 601             "--web-content-isolation-strategy",
 602             type=int,
 603             dest="web_content_isolation_strategy",
 604             help="Strategy used to determine whether or not a particular site should load into "
 605             "a webIsolated content process, see fission.webContentIsolationStrategy.",
 606         )
 607         self.add_argument(
 608             "--repeat",
 609             type=int,
 610             default=0,
 611             help="Repeat the tests the given number of times.",
 612         )
 613         self.add_argument(
 614             "--run-until-failure",
 615             action="store_true",
 616             dest="run_until_failure",
 617             default=False,
 618             help="Run tests repeatedly but stop the first time a test fails.",
 619         )
 620         self.add_argument(
 621             "--setpref",
 622             action="append",
 623             dest="extra_prefs",
 624             default=[],
 625             metavar="PREF=VALUE",
 626             help="Defines an extra user preference.",
 627         )
 628         # Additional options for server.
 629         self.add_argument(
 630             "--certificate-path",
 631             action="store",
 632             type=str,
 633             dest="certPath",
 634             default=None,
 635             help="Path to directory containing certificate store.",
 636         ),
 637         self.add_argument(
 638             "--http-port",
 639             action="store",
 640             type=str,
 641             dest="httpPort",
 642             default=DEFAULT_PORTS["http"],
 643             help="http port of the remote web server.",
 644         )
 645         self.add_argument(
 646             "--remote-webserver",
 647             action="store",
 648             type=str,
 649             dest="remoteWebServer",
 650             help="IP address of the remote web server.",
 651         )
 652         self.add_argument(
 653             "--ssl-port",
 654             action="store",
 655             type=str,
 656             dest="sslPort",
 657             default=DEFAULT_PORTS["https"],
 658             help="ssl port of the remote web server.",
 659         )
 660         self.add_argument(
 661             "--test-filters-file",
 662             action="store",
 663             type=str,
 664             dest="test_filters_file",
 665             default=None,
 666             help="Line-delimited file containing test filter(s)",
 667         )
 668         # Remaining arguments are test filters.
 669         self.add_argument(
 670             "test_filters",
 671             nargs="*",
 672             help="Test filter(s): class and/or method names of test(s) to run.",
 673         )
 674
 675         mozlog.commandline.add_logging_group(self)
 676
 677
 678 def run_test_harness(parser, options):
 679     if hasattr(options, "log"):
 680         log = options.log
 681     else:
 682         log = mozlog.commandline.setup_logging(
 683             "runjunit", options, {"tbpl": sys.stdout}
 684         )
 685     runner = JUnitTestRunner(log, options)
 686     result = -1
 687     try:
 688         device_exception = False
 689         result = runner.run_tests(
 690             test_filters_file=options.test_filters_file,
 691             test_filters=options.test_filters,
 692         )
 693     except KeyboardInterrupt:
 694         log.info("runjunit.py | Received keyboard interrupt")
 695         result = -1
 696     except JavaTestHarnessException as e:
 697         log.error(
 698             "TEST-UNEXPECTED-FAIL | runjunit.py | The previous test failed because "
 699             "of an error in the test harness | %s" % (str(e))
 700         )
 701     except Exception as e:
 702         traceback.print_exc()
 703         log.error("runjunit.py | Received unexpected exception while running tests")
 704         result = 1
 705         if isinstance(e, ADBTimeoutError):
 706             device_exception = True
 707     finally:
 708         if not device_exception:
 709             runner.cleanup()
 710     return result
 711
 712
 713 def main(args=sys.argv[1:]):
 714     parser = JunitArgumentParser()
 715     options = parser.parse_args()
 716     return run_test_harness(parser, options)
 717
 718
 719 if __name__ == "__main__":
 720     sys.exit(main())