testing/gtest/remotegtests.py

   1 #!/usr/bin/env python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 import argparse
   8 import datetime
   9 import glob
  10 import os
  11 import posixpath
  12 import shutil
  13 import sys
  14 import tempfile
  15 import time
  16 import traceback
  17
  18 import mozcrash
  19 import mozdevice
  20 import mozinfo
  21 import mozlog
  22 import six
  23
  24 LOGGER_NAME = "gtest"
  25 log = mozlog.unstructured.getLogger(LOGGER_NAME)
  26
  27
  28 class RemoteGTests(object):
  29     """
  30     A test harness to run gtest on Android.
  31     """
  32
  33     def __init__(self):
  34         self.device = None
  35
  36     def build_environment(self, shuffle, test_filter):
  37         """
  38         Create and return a dictionary of all the appropriate env variables
  39         and values.
  40         """
  41         env = {}
  42         env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
  43         env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
  44         env["MOZ_CRASHREPORTER"] = "1"
  45         env["MOZ_RUN_GTEST"] = "1"
  46         # custom output parser is mandatory on Android
  47         env["MOZ_TBPL_PARSER"] = "1"
  48         env["MOZ_GTEST_LOG_PATH"] = self.remote_log
  49         env["MOZ_GTEST_CWD"] = self.remote_profile
  50         env["MOZ_GTEST_MINIDUMPS_PATH"] = self.remote_minidumps
  51         env["MOZ_IN_AUTOMATION"] = "1"
  52         env["MOZ_ANDROID_LIBDIR_OVERRIDE"] = posixpath.join(
  53             self.remote_libdir, "libxul.so"
  54         )
  55         if shuffle:
  56             env["GTEST_SHUFFLE"] = "True"
  57         if test_filter:
  58             env["GTEST_FILTER"] = test_filter
  59
  60         # webrender needs gfx.webrender.all=true, gtest doesn't use prefs
  61         env["MOZ_WEBRENDER"] = "1"
  62
  63         return env
  64
  65     def run_gtest(
  66         self,
  67         test_dir,
  68         shuffle,
  69         test_filter,
  70         package,
  71         adb_path,
  72         device_serial,
  73         remote_test_root,
  74         libxul_path,
  75         symbols_path,
  76     ):
  77         """
  78         Launch the test app, run gtest, collect test results and wait for completion.
  79         Return False if a crash or other failure is detected, else True.
  80         """
  81         update_mozinfo()
  82         self.device = mozdevice.ADBDeviceFactory(
  83             adb=adb_path,
  84             device=device_serial,
  85             test_root=remote_test_root,
  86             logger_name=LOGGER_NAME,
  87             verbose=False,
  88             run_as_package=package,
  89         )
  90         root = self.device.test_root
  91         self.remote_profile = posixpath.join(root, "gtest-profile")
  92         self.remote_minidumps = posixpath.join(root, "gtest-minidumps")
  93         self.remote_log = posixpath.join(root, "gtest.log")
  94         self.remote_libdir = posixpath.join(root, "gtest")
  95
  96         self.package = package
  97         self.cleanup()
  98         self.device.mkdir(self.remote_profile)
  99         self.device.mkdir(self.remote_minidumps)
 100         self.device.mkdir(self.remote_libdir)
 101
 102         log.info("Running Android gtest")
 103         if not self.device.is_app_installed(self.package):
 104             raise Exception("%s is not installed on this device" % self.package)
 105
 106         # Push the gtest libxul.so to the device. The harness assumes an architecture-
 107         # appropriate library is specified and pushes it to the arch-agnostic remote
 108         # directory.
 109         # TODO -- consider packaging the gtest libxul.so in an apk
 110         self.device.push(libxul_path, self.remote_libdir)
 111
 112         # Push support files to device. Avoid sub-directories so that libxul.so
 113         # is not included.
 114         for f in glob.glob(os.path.join(test_dir, "*")):
 115             if not os.path.isdir(f):
 116                 self.device.push(f, self.remote_profile)
 117
 118         if test_filter is not None:
 119             test_filter = six.ensure_text(test_filter)
 120         env = self.build_environment(shuffle, test_filter)
 121         args = [
 122             "-unittest",
 123             "--gtest_death_test_style=threadsafe",
 124             "-profile %s" % self.remote_profile,
 125         ]
 126         if "geckoview" in self.package:
 127             activity = "TestRunnerActivity"
 128             self.device.launch_activity(
 129                 self.package,
 130                 activity_name=activity,
 131                 e10s=False,  # gtest is non-e10s on desktop
 132                 moz_env=env,
 133                 extra_args=args,
 134                 wait=False,
 135             )
 136         else:
 137             self.device.launch_fennec(self.package, moz_env=env, extra_args=args)
 138         waiter = AppWaiter(self.device, self.remote_log)
 139         timed_out = waiter.wait(self.package)
 140         self.shutdown(use_kill=True if timed_out else False)
 141         if self.check_for_crashes(symbols_path):
 142             return False
 143         return True
 144
 145     def shutdown(self, use_kill):
 146         """
 147         Stop the remote application.
 148         If use_kill is specified, a multi-stage kill procedure is used,
 149         attempting to trigger ANR and minidump reports before ending
 150         the process.
 151         """
 152         if not use_kill:
 153             self.device.stop_application(self.package)
 154         else:
 155             # Trigger an ANR report with "kill -3" (SIGQUIT)
 156             try:
 157                 self.device.pkill(self.package, sig=3, attempts=1)
 158             except mozdevice.ADBTimeoutError:
 159                 raise
 160             except:  # NOQA: E722
 161                 pass
 162             time.sleep(3)
 163             # Trigger a breakpad dump with "kill -6" (SIGABRT)
 164             try:
 165                 self.device.pkill(self.package, sig=6, attempts=1)
 166             except mozdevice.ADBTimeoutError:
 167                 raise
 168             except:  # NOQA: E722
 169                 pass
 170             # Wait for process to end
 171             retries = 0
 172             while retries < 3:
 173                 if self.device.process_exist(self.package):
 174                     log.info("%s still alive after SIGABRT: waiting..." % self.package)
 175                     time.sleep(5)
 176                 else:
 177                     break
 178                 retries += 1
 179             if self.device.process_exist(self.package):
 180                 try:
 181                     self.device.pkill(self.package, sig=9, attempts=1)
 182                 except mozdevice.ADBTimeoutError:
 183                     raise
 184                 except:  # NOQA: E722
 185                     log.warning("%s still alive after SIGKILL!" % self.package)
 186             if self.device.process_exist(self.package):
 187                 self.device.stop_application(self.package)
 188         # Test harnesses use the MOZ_CRASHREPORTER environment variables to suppress
 189         # the interactive crash reporter, but that may not always be effective;
 190         # check for and cleanup errant crashreporters.
 191         crashreporter = "%s.CrashReporter" % self.package
 192         if self.device.process_exist(crashreporter):
 193             log.warning("%s unexpectedly found running. Killing..." % crashreporter)
 194             try:
 195                 self.device.pkill(crashreporter)
 196             except mozdevice.ADBTimeoutError:
 197                 raise
 198             except:  # NOQA: E722
 199                 pass
 200         if self.device.process_exist(crashreporter):
 201             log.error("%s still running!!" % crashreporter)
 202
 203     def check_for_crashes(self, symbols_path):
 204         """
 205         Pull minidumps from the remote device and generate crash reports.
 206         Returns True if a crash was detected, or suspected.
 207         """
 208         try:
 209             dump_dir = tempfile.mkdtemp()
 210             remote_dir = self.remote_minidumps
 211             if not self.device.is_dir(remote_dir):
 212                 return False
 213             self.device.pull(remote_dir, dump_dir)
 214             crashed = mozcrash.check_for_crashes(
 215                 dump_dir, symbols_path, test_name="gtest"
 216             )
 217         except Exception as e:
 218             log.error("unable to check for crashes: %s" % str(e))
 219             crashed = True
 220         finally:
 221             try:
 222                 shutil.rmtree(dump_dir)
 223             except Exception:
 224                 log.warning("unable to remove directory: %s" % dump_dir)
 225         return crashed
 226
 227     def cleanup(self):
 228         if self.device:
 229             self.device.stop_application(self.package)
 230             self.device.rm(self.remote_log, force=True)
 231             self.device.rm(self.remote_profile, recursive=True, force=True)
 232             self.device.rm(self.remote_minidumps, recursive=True, force=True)
 233             self.device.rm(self.remote_libdir, recursive=True, force=True)
 234
 235
 236 class AppWaiter(object):
 237     def __init__(
 238         self,
 239         device,
 240         remote_log,
 241         test_proc_timeout=1200,
 242         test_proc_no_output_timeout=300,
 243         test_proc_start_timeout=60,
 244         output_poll_interval=10,
 245     ):
 246         self.device = device
 247         self.remote_log = remote_log
 248         self.start_time = datetime.datetime.now()
 249         self.timeout_delta = datetime.timedelta(seconds=test_proc_timeout)
 250         self.output_timeout_delta = datetime.timedelta(
 251             seconds=test_proc_no_output_timeout
 252         )
 253         self.start_timeout_delta = datetime.timedelta(seconds=test_proc_start_timeout)
 254         self.output_poll_interval = output_poll_interval
 255         self.last_output_time = datetime.datetime.now()
 256         self.remote_log_len = 0
 257
 258     def start_timed_out(self):
 259         if datetime.datetime.now() - self.start_time > self.start_timeout_delta:
 260             return True
 261         return False
 262
 263     def timed_out(self):
 264         if datetime.datetime.now() - self.start_time > self.timeout_delta:
 265             return True
 266         return False
 267
 268     def output_timed_out(self):
 269         if datetime.datetime.now() - self.last_output_time > self.output_timeout_delta:
 270             return True
 271         return False
 272
 273     def get_top(self):
 274         top = self.device.get_top_activity(timeout=60)
 275         if top is None:
 276             log.info("Failed to get top activity, retrying, once...")
 277             top = self.device.get_top_activity(timeout=60)
 278         return top
 279
 280     def wait_for_start(self, package):
 281         top = None
 282         while top != package and not self.start_timed_out():
 283             if self.update_log():
 284                 # if log content is available, assume the app started; otherwise,
 285                 # a short run (few tests) might complete without ever being detected
 286                 # in the foreground
 287                 return package
 288             time.sleep(1)
 289             top = self.get_top()
 290         return top
 291
 292     def wait(self, package):
 293         """
 294         Wait until:
 295          - the app loses foreground, or
 296          - no new output is observed for the output timeout, or
 297          - the timeout is exceeded.
 298         While waiting, update the log every periodically: pull the gtest log from
 299         device and log any new content.
 300         """
 301         top = self.wait_for_start(package)
 302         if top != package:
 303             log.testFail("gtest | %s failed to start" % package)
 304             return
 305         while not self.timed_out():
 306             if not self.update_log():
 307                 top = self.get_top()
 308                 if top != package or self.output_timed_out():
 309                     time.sleep(self.output_poll_interval)
 310                     break
 311             time.sleep(self.output_poll_interval)
 312         self.update_log()
 313         if self.timed_out():
 314             log.testFail(
 315                 "gtest | timed out after %d seconds", self.timeout_delta.seconds
 316             )
 317         elif self.output_timed_out():
 318             log.testFail(
 319                 "gtest | timed out after %d seconds without output",
 320                 self.output_timeout_delta.seconds,
 321             )
 322         else:
 323             log.info("gtest | wait for %s complete; top activity=%s" % (package, top))
 324         return True if top == package else False
 325
 326     def update_log(self):
 327         """
 328         Pull the test log from the remote device and display new content.
 329         """
 330         if not self.device.is_file(self.remote_log):
 331             log.info("gtest | update_log %s is not a file." % self.remote_log)
 332             return False
 333         try:
 334             new_content = self.device.get_file(
 335                 self.remote_log, offset=self.remote_log_len
 336             )
 337         except mozdevice.ADBTimeoutError:
 338             raise
 339         except Exception as e:
 340             log.info("gtest | update_log : exception reading log: %s" % str(e))
 341             return False
 342         if not new_content:
 343             log.info("gtest | update_log : no new content")
 344             return False
 345         new_content = six.ensure_text(new_content)
 346         last_full_line_pos = new_content.rfind("\n")
 347         if last_full_line_pos <= 0:
 348             # wait for a full line
 349             return False
 350         # trim partial line
 351         new_content = new_content[:last_full_line_pos]
 352         self.remote_log_len += len(new_content)
 353         for line in new_content.lstrip("\n").split("\n"):
 354             print(line)
 355         self.last_output_time = datetime.datetime.now()
 356         return True
 357
 358
 359 class remoteGtestOptions(argparse.ArgumentParser):
 360     def __init__(self):
 361         super(remoteGtestOptions, self).__init__(
 362             usage="usage: %prog [options] test_filter"
 363         )
 364         self.add_argument(
 365             "--package",
 366             dest="package",
 367             default="org.mozilla.geckoview.test_runner",
 368             help="Package name of test app.",
 369         )
 370         self.add_argument(
 371             "--adbpath",
 372             action="store",
 373             type=str,
 374             dest="adb_path",
 375             default="adb",
 376             help="Path to adb binary.",
 377         )
 378         self.add_argument(
 379             "--deviceSerial",
 380             action="store",
 381             type=str,
 382             dest="device_serial",
 383             help="adb serial number of remote device. This is required "
 384             "when more than one device is connected to the host. "
 385             "Use 'adb devices' to see connected devices. ",
 386         )
 387         self.add_argument(
 388             "--remoteTestRoot",
 389             action="store",
 390             type=str,
 391             dest="remote_test_root",
 392             help="Remote directory to use as test root "
 393             "(eg. /data/local/tmp/test_root).",
 394         )
 395         self.add_argument(
 396             "--libxul",
 397             action="store",
 398             type=str,
 399             dest="libxul_path",
 400             default=None,
 401             help="Path to gtest libxul.so.",
 402         )
 403         self.add_argument(
 404             "--symbols-path",
 405             dest="symbols_path",
 406             default=None,
 407             help="absolute path to directory containing breakpad "
 408             "symbols, or the URL of a zip file containing symbols",
 409         )
 410         self.add_argument(
 411             "--shuffle",
 412             action="store_true",
 413             default=False,
 414             help="Randomize the execution order of tests.",
 415         )
 416         self.add_argument(
 417             "--tests-path",
 418             default=None,
 419             help="Path to gtest directory containing test support files.",
 420         )
 421         self.add_argument("args", nargs=argparse.REMAINDER)
 422
 423
 424 def update_mozinfo():
 425     """
 426     Walk up directories to find mozinfo.json and update the info.
 427     """
 428     path = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
 429     dirs = set()
 430     while path != os.path.expanduser("~"):
 431         if path in dirs:
 432             break
 433         dirs.add(path)
 434         path = os.path.split(path)[0]
 435     mozinfo.find_and_update_from_json(*dirs)
 436
 437
 438 def main():
 439     parser = remoteGtestOptions()
 440     options = parser.parse_args()
 441     args = options.args
 442     if not options.libxul_path:
 443         parser.error("--libxul is required")
 444         sys.exit(1)
 445     if len(args) > 1:
 446         parser.error("only one test_filter is allowed")
 447         sys.exit(1)
 448     test_filter = args[0] if args else None
 449     tester = RemoteGTests()
 450     result = False
 451     try:
 452         device_exception = False
 453         result = tester.run_gtest(
 454             options.tests_path,
 455             options.shuffle,
 456             test_filter,
 457             options.package,
 458             options.adb_path,
 459             options.device_serial,
 460             options.remote_test_root,
 461             options.libxul_path,
 462             options.symbols_path,
 463         )
 464     except KeyboardInterrupt:
 465         log.info("gtest | Received keyboard interrupt")
 466     except Exception as e:
 467         log.error(str(e))
 468         traceback.print_exc()
 469         if isinstance(e, mozdevice.ADBTimeoutError):
 470             device_exception = True
 471     finally:
 472         if not device_exception:
 473             tester.cleanup()
 474     sys.exit(0 if result else 1)
 475
 476
 477 if __name__ == "__main__":
 478     main()