testing/mochitest/runtestsremote.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import datetime
   6 import os
   7 import posixpath
   8 import shutil
   9 import sys
  10 import tempfile
  11 import traceback
  12 import uuid
  13
  14 sys.path.insert(0, os.path.abspath(os.path.realpath(os.path.dirname(__file__))))
  15
  16 import mozcrash
  17 import mozinfo
  18 from mochitest_options import MochitestArgumentParser, build_obj
  19 from mozdevice import ADBDeviceFactory, ADBTimeoutError, RemoteProcessMonitor
  20 from mozscreenshot import dump_device_screen, dump_screen
  21 from runtests import MessageLogger, MochitestDesktop
  22
  23 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
  24
  25
  26 class MochiRemote(MochitestDesktop):
  27     localProfile = None
  28     logMessages = []
  29
  30     def __init__(self, options):
  31         MochitestDesktop.__init__(self, options.flavor, vars(options))
  32
  33         verbose = False
  34         if (
  35             options.log_mach_verbose
  36             or options.log_tbpl_level == "debug"
  37             or options.log_mach_level == "debug"
  38             or options.log_raw_level == "debug"
  39         ):
  40             verbose = True
  41         if hasattr(options, "log"):
  42             delattr(options, "log")
  43
  44         self.certdbNew = True
  45         self.chromePushed = False
  46
  47         expected = options.app.split("/")[-1]
  48         self.device = ADBDeviceFactory(
  49             adb=options.adbPath or "adb",
  50             device=options.deviceSerial,
  51             test_root=options.remoteTestRoot,
  52             verbose=verbose,
  53             run_as_package=expected,
  54         )
  55
  56         if options.remoteTestRoot is None:
  57             options.remoteTestRoot = self.device.test_root
  58         options.dumpOutputDirectory = options.remoteTestRoot
  59         self.remoteLogFile = posixpath.join(
  60             options.remoteTestRoot, "logs", "mochitest.log"
  61         )
  62         logParent = posixpath.dirname(self.remoteLogFile)
  63         self.device.rm(logParent, force=True, recursive=True)
  64         self.device.mkdir(logParent, parents=True)
  65
  66         self.remoteProfile = posixpath.join(options.remoteTestRoot, "profile")
  67         self.device.rm(self.remoteProfile, force=True, recursive=True)
  68
  69         self.message_logger = MessageLogger(logger=None)
  70         self.message_logger.logger = self.log
  71
  72         # Check that Firefox is installed
  73         expected = options.app.split("/")[-1]
  74         if not self.device.is_app_installed(expected):
  75             raise Exception("%s is not installed on this device" % expected)
  76
  77         self.device.clear_logcat()
  78
  79         self.remoteModulesDir = posixpath.join(options.remoteTestRoot, "modules/")
  80
  81         self.remoteCache = posixpath.join(options.remoteTestRoot, "cache/")
  82         self.device.rm(self.remoteCache, force=True, recursive=True)
  83
  84         # move necko cache to a location that can be cleaned up
  85         options.extraPrefs += [
  86             "browser.cache.disk.parent_directory=%s" % self.remoteCache
  87         ]
  88
  89         self.remoteMozLog = posixpath.join(options.remoteTestRoot, "mozlog")
  90         self.device.rm(self.remoteMozLog, force=True, recursive=True)
  91         self.device.mkdir(self.remoteMozLog, parents=True)
  92
  93         self.remoteChromeTestDir = posixpath.join(options.remoteTestRoot, "chrome")
  94         self.device.rm(self.remoteChromeTestDir, force=True, recursive=True)
  95         self.device.mkdir(self.remoteChromeTestDir, parents=True)
  96
  97         self.appName = options.remoteappname
  98         self.device.stop_application(self.appName)
  99         if self.device.process_exist(self.appName):
 100             self.log.warning("unable to kill %s before running tests!" % self.appName)
 101
 102         # Add Android version (SDK level) to mozinfo so that manifest entries
 103         # can be conditional on android_version.
 104         self.log.info(
 105             "Android sdk version '%s'; will use this to filter manifests"
 106             % str(self.device.version)
 107         )
 108         mozinfo.info["android_version"] = str(self.device.version)
 109         mozinfo.info["is_emulator"] = self.device._device_serial.startswith("emulator-")
 110
 111     def cleanup(self, options, final=False):
 112         if final:
 113             self.device.rm(self.remoteChromeTestDir, force=True, recursive=True)
 114             self.chromePushed = False
 115             uploadDir = os.environ.get("MOZ_UPLOAD_DIR", None)
 116             if uploadDir and self.device.is_dir(self.remoteMozLog):
 117                 self.device.pull(self.remoteMozLog, uploadDir)
 118         self.device.rm(self.remoteLogFile, force=True)
 119         self.device.rm(self.remoteProfile, force=True, recursive=True)
 120         self.device.rm(self.remoteCache, force=True, recursive=True)
 121         MochitestDesktop.cleanup(self, options, final)
 122         self.localProfile = None
 123
 124     def dumpScreen(self, utilityPath):
 125         if self.haveDumpedScreen:
 126             self.log.info(
 127                 "Not taking screenshot here: see the one that was previously logged"
 128             )
 129             return
 130         self.haveDumpedScreen = True
 131         if self.device._device_serial.startswith("emulator-"):
 132             dump_screen(utilityPath, self.log)
 133         else:
 134             dump_device_screen(self.device, self.log)
 135
 136     def findPath(self, paths, filename=None):
 137         for path in paths:
 138             p = path
 139             if filename:
 140                 p = os.path.join(p, filename)
 141             if os.path.exists(self.getFullPath(p)):
 142                 return path
 143         return None
 144
 145     # This seems kludgy, but this class uses paths from the remote host in the
 146     # options, except when calling up to the base class, which doesn't
 147     # understand the distinction.  This switches out the remote values for local
 148     # ones that the base class understands.  This is necessary for the web
 149     # server, SSL tunnel and profile building functions.
 150     def switchToLocalPaths(self, options):
 151         """Set local paths in the options, return a function that will restore remote values"""
 152         remoteXrePath = options.xrePath
 153         remoteProfilePath = options.profilePath
 154         remoteUtilityPath = options.utilityPath
 155
 156         paths = [
 157             options.xrePath,
 158         ]
 159         if build_obj:
 160             paths.append(os.path.join(build_obj.topobjdir, "dist", "bin"))
 161         options.xrePath = self.findPath(paths)
 162         if options.xrePath is None:
 163             self.log.error(
 164                 "unable to find xulrunner path for %s, please specify with --xre-path"
 165                 % os.name
 166             )
 167             sys.exit(1)
 168
 169         xpcshell = "xpcshell"
 170         if os.name == "nt":
 171             xpcshell += ".exe"
 172
 173         if options.utilityPath:
 174             paths = [options.utilityPath, options.xrePath]
 175         else:
 176             paths = [options.xrePath]
 177         options.utilityPath = self.findPath(paths, xpcshell)
 178
 179         if options.utilityPath is None:
 180             self.log.error(
 181                 "unable to find utility path for %s, please specify with --utility-path"
 182                 % os.name
 183             )
 184             sys.exit(1)
 185
 186         xpcshell_path = os.path.join(options.utilityPath, xpcshell)
 187         if RemoteProcessMonitor.elf_arm(xpcshell_path):
 188             self.log.error(
 189                 "xpcshell at %s is an ARM binary; please use "
 190                 "the --utility-path argument to specify the path "
 191                 "to a desktop version." % xpcshell_path
 192             )
 193             sys.exit(1)
 194
 195         if self.localProfile:
 196             options.profilePath = self.localProfile
 197         else:
 198             options.profilePath = None
 199
 200         def fixup():
 201             options.xrePath = remoteXrePath
 202             options.utilityPath = remoteUtilityPath
 203             options.profilePath = remoteProfilePath
 204
 205         return fixup
 206
 207     def startServers(self, options, debuggerInfo, public=None):
 208         """Create the servers on the host and start them up"""
 209         restoreRemotePaths = self.switchToLocalPaths(options)
 210         MochitestDesktop.startServers(self, options, debuggerInfo, public=True)
 211         restoreRemotePaths()
 212
 213     def buildProfile(self, options):
 214         restoreRemotePaths = self.switchToLocalPaths(options)
 215         if options.testingModulesDir:
 216             try:
 217                 self.device.push(options.testingModulesDir, self.remoteModulesDir)
 218                 self.device.chmod(self.remoteModulesDir, recursive=True)
 219             except Exception:
 220                 self.log.error(
 221                     "Automation Error: Unable to copy test modules to device."
 222                 )
 223                 raise
 224             savedTestingModulesDir = options.testingModulesDir
 225             options.testingModulesDir = self.remoteModulesDir
 226         else:
 227             savedTestingModulesDir = None
 228         manifest = MochitestDesktop.buildProfile(self, options)
 229         if savedTestingModulesDir:
 230             options.testingModulesDir = savedTestingModulesDir
 231         self.localProfile = options.profilePath
 232
 233         restoreRemotePaths()
 234         options.profilePath = self.remoteProfile
 235         return manifest
 236
 237     def buildURLOptions(self, options, env):
 238         saveLogFile = options.logFile
 239         options.logFile = self.remoteLogFile
 240         options.profilePath = self.localProfile
 241         env["MOZ_HIDE_RESULTS_TABLE"] = "1"
 242         retVal = MochitestDesktop.buildURLOptions(self, options, env)
 243
 244         # we really need testConfig.js (for browser chrome)
 245         try:
 246             self.device.push(options.profilePath, self.remoteProfile)
 247             self.device.chmod(self.remoteProfile, recursive=True)
 248         except Exception:
 249             self.log.error("Automation Error: Unable to copy profile to device.")
 250             raise
 251
 252         options.profilePath = self.remoteProfile
 253         options.logFile = saveLogFile
 254         return retVal
 255
 256     def getChromeTestDir(self, options):
 257         local = super(MochiRemote, self).getChromeTestDir(options)
 258         remote = self.remoteChromeTestDir
 259         if options.flavor == "chrome" and not self.chromePushed:
 260             self.log.info("pushing %s to %s on device..." % (local, remote))
 261             local = os.path.join(local, "chrome")
 262             self.device.push(local, remote)
 263             self.chromePushed = True
 264         return remote
 265
 266     def getLogFilePath(self, logFile):
 267         return logFile
 268
 269     def getGMPPluginPath(self, options):
 270         # TODO: bug 1149374
 271         return None
 272
 273     def environment(self, env=None, crashreporter=True, **kwargs):
 274         # Since running remote, do not mimic the local env: do not copy os.environ
 275         if env is None:
 276             env = {}
 277
 278         if crashreporter:
 279             env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
 280             env["MOZ_CRASHREPORTER"] = "1"
 281             env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
 282         else:
 283             env["MOZ_CRASHREPORTER_DISABLE"] = "1"
 284
 285         # Crash on non-local network connections by default.
 286         # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily
 287         # enable non-local connections for the purposes of local testing.
 288         # Don't override the user's choice here.  See bug 1049688.
 289         env.setdefault("MOZ_DISABLE_NONLOCAL_CONNECTIONS", "1")
 290
 291         # Send an env var noting that we are in automation. Passing any
 292         # value except the empty string will declare the value to exist.
 293         #
 294         # This may be used to disabled network connections during testing, e.g.
 295         # Switchboard & telemetry uploads.
 296         env.setdefault("MOZ_IN_AUTOMATION", "1")
 297
 298         # Set WebRTC logging in case it is not set yet.
 299         env.setdefault("R_LOG_LEVEL", "6")
 300         env.setdefault("R_LOG_DESTINATION", "stderr")
 301         env.setdefault("R_LOG_VERBOSE", "1")
 302
 303         return env
 304
 305     def buildBrowserEnv(self, options, debugger=False):
 306         browserEnv = MochitestDesktop.buildBrowserEnv(self, options, debugger=debugger)
 307         # remove desktop environment not used on device
 308         if "XPCOM_MEM_BLOAT_LOG" in browserEnv:
 309             del browserEnv["XPCOM_MEM_BLOAT_LOG"]
 310         if self.mozLogs:
 311             browserEnv["MOZ_LOG_FILE"] = os.path.join(
 312                 self.remoteMozLog, "moz-pid=%PID-uid={}.log".format(str(uuid.uuid4()))
 313             )
 314         if options.dmd:
 315             browserEnv["DMD"] = "1"
 316         # Contents of remoteMozLog will be pulled from device and copied to the
 317         # host MOZ_UPLOAD_DIR, to be made available as test artifacts. Make
 318         # MOZ_UPLOAD_DIR available to the browser environment so that tests
 319         # can use it as though they were running on the host.
 320         browserEnv["MOZ_UPLOAD_DIR"] = self.remoteMozLog
 321         return browserEnv
 322
 323     def runApp(
 324         self,
 325         testUrl,
 326         env,
 327         app,
 328         profile,
 329         extraArgs,
 330         utilityPath,
 331         debuggerInfo=None,
 332         valgrindPath=None,
 333         valgrindArgs=None,
 334         valgrindSuppFiles=None,
 335         symbolsPath=None,
 336         timeout=-1,
 337         detectShutdownLeaks=False,
 338         screenshotOnFail=False,
 339         bisectChunk=None,
 340         restartAfterFailure=False,
 341         marionette_args=None,
 342         e10s=True,
 343         runFailures=False,
 344         crashAsPass=False,
 345         currentManifest=None,
 346     ):
 347         """
 348         Run the app, log the duration it took to execute, return the status code.
 349         Kill the app if it outputs nothing for |timeout| seconds.
 350         """
 351
 352         if timeout == -1:
 353             timeout = self.DEFAULT_TIMEOUT
 354
 355         rpm = RemoteProcessMonitor(
 356             self.appName,
 357             self.device,
 358             self.log,
 359             self.message_logger,
 360             self.remoteLogFile,
 361             self.remoteProfile,
 362         )
 363         startTime = datetime.datetime.now()
 364         status = 0
 365         profileDirectory = self.remoteProfile + "/"
 366         args = []
 367         args.extend(extraArgs)
 368         args.extend(("-profile", profileDirectory))
 369
 370         pid = rpm.launch(
 371             app,
 372             debuggerInfo,
 373             testUrl,
 374             args,
 375             env=self.environment(env=env, crashreporter=not debuggerInfo),
 376             e10s=e10s,
 377         )
 378
 379         # TODO: not using runFailures or crashAsPass, if we choose to use them
 380         # we need to adjust status and check_for_crashes
 381         self.log.info("runtestsremote.py | Application pid: %d" % pid)
 382         if not rpm.wait(timeout):
 383             status = 1
 384         self.log.info(
 385             "runtestsremote.py | Application ran for: %s"
 386             % str(datetime.datetime.now() - startTime)
 387         )
 388
 389         lastTestSeen = currentManifest or "Main app process exited normally"
 390
 391         crashed = self.check_for_crashes(symbolsPath, lastTestSeen)
 392         if crashed:
 393             status = 1
 394
 395         self.countpass += rpm.counts["pass"]
 396         self.countfail += rpm.counts["fail"]
 397         self.counttodo += rpm.counts["todo"]
 398
 399         return status, lastTestSeen
 400
 401     def check_for_crashes(self, symbols_path, last_test_seen):
 402         """
 403         Pull any minidumps from remote profile and log any associated crashes.
 404         """
 405         try:
 406             dump_dir = tempfile.mkdtemp()
 407             remote_crash_dir = posixpath.join(self.remoteProfile, "minidumps")
 408             if not self.device.is_dir(remote_crash_dir):
 409                 return False
 410             self.device.pull(remote_crash_dir, dump_dir)
 411             crashed = mozcrash.log_crashes(
 412                 self.log, dump_dir, symbols_path, test=last_test_seen
 413             )
 414         finally:
 415             try:
 416                 shutil.rmtree(dump_dir)
 417             except Exception as e:
 418                 self.log.warning(
 419                     "unable to remove directory %s: %s" % (dump_dir, str(e))
 420                 )
 421         return crashed
 422
 423
 424 def run_test_harness(parser, options):
 425     parser.validate(options)
 426
 427     if options is None:
 428         raise ValueError(
 429             "Invalid options specified, use --help for a list of valid options"
 430         )
 431
 432     options.runByManifest = True
 433
 434     mochitest = MochiRemote(options)
 435
 436     try:
 437         if options.verify:
 438             retVal = mochitest.verifyTests(options)
 439         else:
 440             retVal = mochitest.runTests(options)
 441     except Exception as e:
 442         mochitest.log.error("Automation Error: Exception caught while running tests")
 443         traceback.print_exc()
 444         if isinstance(e, ADBTimeoutError):
 445             mochitest.log.info("Device disconnected. Will not run mochitest.cleanup().")
 446         else:
 447             try:
 448                 mochitest.cleanup(options)
 449             except Exception:
 450                 # device error cleaning up... oh well!
 451                 traceback.print_exc()
 452         retVal = 1
 453
 454     mochitest.archiveMozLogs()
 455     mochitest.message_logger.finish()
 456
 457     return retVal
 458
 459
 460 def main(args=sys.argv[1:]):
 461     parser = MochitestArgumentParser(app="android")
 462     options = parser.parse_args(args)
 463
 464     return run_test_harness(parser, options)
 465
 466
 467 if __name__ == "__main__":
 468     sys.exit(main())