testing/xpcshell/remotexpcshelltests.py

   1 #!/usr/bin/env python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 import datetime
   8 import os
   9 import posixpath
  10 import shutil
  11 import sys
  12 import tempfile
  13 import time
  14 import uuid
  15 from argparse import Namespace
  16 from zipfile import ZipFile
  17
  18 import mozcrash
  19 import mozdevice
  20 import mozfile
  21 import mozinfo
  22 import runxpcshelltests as xpcshell
  23 import six
  24 from mozdevice import ADBDevice, ADBDeviceFactory, ADBTimeoutError
  25 from mozlog import commandline
  26 from xpcshellcommandline import parser_remote
  27
  28 here = os.path.dirname(os.path.abspath(__file__))
  29
  30
  31 class RemoteProcessMonitor(object):
  32     processStatus = []
  33
  34     def __init__(self, package, device, log, remoteLogFile):
  35         self.package = package
  36         self.device = device
  37         self.log = log
  38         self.remoteLogFile = remoteLogFile
  39         self.selectedProcess = -1
  40
  41     @classmethod
  42     def pickUnusedProcess(cls):
  43         for i in range(len(cls.processStatus)):
  44             if not cls.processStatus[i]:
  45                 cls.processStatus[i] = True
  46                 return i
  47         # No more free processes :(
  48         return -1
  49
  50     @classmethod
  51     def freeProcess(cls, processId):
  52         cls.processStatus[processId] = False
  53
  54     def kill(self):
  55         self.device.pkill(self.process_name, sig=9, attempts=1)
  56
  57     def launch_service(self, extra_args, env, selectedProcess, test_name=None):
  58         if not self.device.process_exist(self.package):
  59             # Make sure the main app is running, this should help making the
  60             # tests get foreground priority scheduling.
  61             self.device.launch_activity(
  62                 self.package,
  63                 intent="org.mozilla.geckoview.test_runner.XPCSHELL_TEST_MAIN",
  64                 activity_name="TestRunnerActivity",
  65                 e10s=True,
  66             )
  67             # Newer Androids require that background services originate from
  68             # active apps, so wait here until the test runner is the top
  69             # activity.
  70             retries = 20
  71             top = self.device.get_top_activity(timeout=60)
  72             while top != self.package and retries > 0:
  73                 self.log.info(
  74                     "%s | Checking that %s is the top activity."
  75                     % (test_name, self.package)
  76                 )
  77                 top = self.device.get_top_activity(timeout=60)
  78                 time.sleep(1)
  79                 retries -= 1
  80
  81         self.process_name = self.package + (":xpcshell%d" % selectedProcess)
  82
  83         retries = 20
  84         while retries > 0 and self.device.process_exist(self.process_name):
  85             self.log.info(
  86                 "%s | %s | Killing left-over process %s"
  87                 % (test_name, self.pid, self.process_name)
  88             )
  89             self.kill()
  90             time.sleep(1)
  91             retries -= 1
  92
  93         if self.device.process_exist(self.process_name):
  94             raise Exception(
  95                 "%s | %s | Could not kill left-over process" % (test_name, self.pid)
  96             )
  97
  98         self.device.launch_service(
  99             self.package,
 100             activity_name=("XpcshellTestRunnerService$i%d" % selectedProcess),
 101             e10s=True,
 102             moz_env=env,
 103             grant_runtime_permissions=False,
 104             extra_args=extra_args,
 105             out_file=self.remoteLogFile,
 106         )
 107         return self.pid
 108
 109     def wait(self, timeout, interval=0.1, test_name=None):
 110         timer = 0
 111         status = True
 112
 113         # wait for log creation on startup
 114         retries = 0
 115         while retries < 20 / interval and not self.device.is_file(self.remoteLogFile):
 116             retries += 1
 117             time.sleep(interval)
 118         if not self.device.is_file(self.remoteLogFile):
 119             self.log.warning(
 120                 "%s | Failed wait for remote log: %s missing?"
 121                 % (test_name, self.remoteLogFile)
 122             )
 123
 124         while self.device.process_exist(self.process_name):
 125             time.sleep(interval)
 126             timer += interval
 127             interval *= 1.5
 128             if timeout and timer > timeout:
 129                 status = False
 130                 self.log.info(
 131                     "remotexpcshelltests.py | %s | %s | Timing out"
 132                     % (test_name, str(self.pid))
 133                 )
 134                 self.kill()
 135                 break
 136         return status
 137
 138     @property
 139     def pid(self):
 140         """
 141         Determine the pid of the remote process (or the first process with
 142         the same name).
 143         """
 144         procs = self.device.get_process_list()
 145         # limit the comparison to the first 75 characters due to a
 146         # limitation in processname length in android.
 147         pids = [proc[0] for proc in procs if proc[1] == self.process_name[:75]]
 148         if pids is None or len(pids) < 1:
 149             return 0
 150         return pids[0]
 151
 152
 153 class RemoteXPCShellTestThread(xpcshell.XPCShellTestThread):
 154     def __init__(self, *args, **kwargs):
 155         xpcshell.XPCShellTestThread.__init__(self, *args, **kwargs)
 156
 157         self.shellReturnCode = None
 158         # embed the mobile params from the harness into the TestThread
 159         mobileArgs = kwargs.get("mobileArgs")
 160         for key in mobileArgs:
 161             setattr(self, key, mobileArgs[key])
 162         self.remoteLogFile = posixpath.join(
 163             mobileArgs["remoteLogFolder"], "xpcshell-%s.log" % str(uuid.uuid4())
 164         )
 165
 166     def initDir(self, path, mask="777", timeout=None):
 167         """Initialize a directory by removing it if it exists, creating it
 168         and changing the permissions."""
 169         self.device.rm(path, recursive=True, force=True, timeout=timeout)
 170         self.device.mkdir(path, parents=True, timeout=timeout)
 171
 172     def updateTestPrefsFile(self):
 173         # The base method will either be no-op (and return the existing
 174         # remote path), or return a path to a new local file.
 175         testPrefsFile = xpcshell.XPCShellTestThread.updateTestPrefsFile(self)
 176         if testPrefsFile == self.rootPrefsFile:
 177             # The pref file is the shared one, which has been already pushed on the
 178             # device, and so there is nothing more to do here.
 179             return self.rootPrefsFile
 180
 181         # Push the per-test prefs file in the remote temp dir.
 182         remoteTestPrefsFile = posixpath.join(self.remoteTmpDir, "user.js")
 183         self.device.push(testPrefsFile, remoteTestPrefsFile)
 184         self.device.chmod(remoteTestPrefsFile)
 185         os.remove(testPrefsFile)
 186         return remoteTestPrefsFile
 187
 188     def buildCmdTestFile(self, name):
 189         remoteDir = self.remoteForLocal(os.path.dirname(name))
 190         if remoteDir == self.remoteHere:
 191             remoteName = os.path.basename(name)
 192         else:
 193             remoteName = posixpath.join(remoteDir, os.path.basename(name))
 194         return [
 195             "-e",
 196             'const _TEST_CWD = "%s";' % self.remoteHere,
 197             "-e",
 198             'const _TEST_FILE = ["%s"];' % remoteName.replace("\\", "/"),
 199         ]
 200
 201     def remoteForLocal(self, local):
 202         for mapping in self.pathMapping:
 203             if os.path.abspath(mapping.local) == os.path.abspath(local):
 204                 return mapping.remote
 205         return local
 206
 207     def setupTempDir(self):
 208         self.remoteTmpDir = posixpath.join(self.remoteTmpDir, str(uuid.uuid4()))
 209         # make sure the temp dir exists
 210         self.initDir(self.remoteTmpDir)
 211         # env var is set in buildEnvironment
 212         self.env["XPCSHELL_TEST_TEMP_DIR"] = self.remoteTmpDir
 213         return self.remoteTmpDir
 214
 215     def setupProfileDir(self):
 216         profileId = str(uuid.uuid4())
 217         self.profileDir = posixpath.join(self.profileDir, profileId)
 218         self.initDir(self.profileDir)
 219         if self.interactive or self.singleFile:
 220             self.log.info("profile dir is %s" % self.profileDir)
 221         self.env["XPCSHELL_TEST_PROFILE_DIR"] = self.profileDir
 222         self.env["TMPDIR"] = self.profileDir
 223         self.remoteMinidumpDir = posixpath.join(self.remoteMinidumpRootDir, profileId)
 224         self.initDir(self.remoteMinidumpDir)
 225         self.env["XPCSHELL_MINIDUMP_DIR"] = self.remoteMinidumpDir
 226         return self.profileDir
 227
 228     def clean_temp_dirs(self, name):
 229         self.log.info("Cleaning up profile for %s folder: %s" % (name, self.profileDir))
 230         self.device.rm(self.profileDir, force=True, recursive=True)
 231         self.device.rm(self.remoteTmpDir, force=True, recursive=True)
 232         self.device.rm(self.remoteMinidumpDir, force=True, recursive=True)
 233
 234     def setupMozinfoJS(self):
 235         local = tempfile.mktemp()
 236         mozinfo.output_to_file(local)
 237         mozInfoJSPath = posixpath.join(self.profileDir, "mozinfo.json")
 238         self.device.push(local, mozInfoJSPath)
 239         self.device.chmod(mozInfoJSPath)
 240         os.remove(local)
 241         return mozInfoJSPath
 242
 243     def logCommand(self, name, completeCmd, testdir):
 244         self.log.info("%s | full command: %r" % (name, completeCmd))
 245         self.log.info("%s | current directory: %r" % (name, self.remoteHere))
 246         self.log.info("%s | environment: %s" % (name, self.env))
 247
 248     def getHeadFiles(self, test):
 249         """Override parent method to find files on remote device.
 250
 251         Obtains lists of head- files.  Returns a list of head files.
 252         """
 253
 254         def sanitize_list(s, kind):
 255             for f in s.strip().split(" "):
 256                 f = f.strip()
 257                 if len(f) < 1:
 258                     continue
 259
 260                 path = posixpath.join(self.remoteHere, f)
 261
 262                 # skip check for file existence: the convenience of discovering
 263                 # a missing file does not justify the time cost of the round trip
 264                 # to the device
 265                 yield path
 266
 267         self.remoteHere = self.remoteForLocal(test["here"])
 268
 269         headlist = test.get("head", "")
 270         return list(sanitize_list(headlist, "head"))
 271
 272     def buildXpcsCmd(self):
 273         # change base class' paths to remote paths and use base class to build command
 274         self.xpcshell = posixpath.join(self.remoteBinDir, "xpcw")
 275         self.headJSPath = posixpath.join(self.remoteScriptsDir, "head.js")
 276         self.testingModulesDir = self.remoteModulesDir
 277         self.testharnessdir = self.remoteScriptsDir
 278         xpcsCmd = xpcshell.XPCShellTestThread.buildXpcsCmd(self)
 279         # remove "-g <dir> -a <dir>" and replace with remote alternatives
 280         del xpcsCmd[1:5]
 281         if self.options["localAPK"]:
 282             xpcsCmd.insert(1, "--greomni")
 283             xpcsCmd.insert(2, self.remoteAPK)
 284         xpcsCmd.insert(1, "-g")
 285         xpcsCmd.insert(2, self.remoteBinDir)
 286
 287         if self.remoteDebugger:
 288             # for example, "/data/local/gdbserver" "localhost:12345"
 289             xpcsCmd = [self.remoteDebugger, self.remoteDebuggerArgs] + xpcsCmd
 290         return xpcsCmd
 291
 292     def killTimeout(self, proc):
 293         self.kill(proc)
 294
 295     def launchProcess(
 296         self, cmd, stdout, stderr, env, cwd, timeout=None, test_name=None
 297     ):
 298         rpm = RemoteProcessMonitor(
 299             "org.mozilla.geckoview.test_runner",
 300             self.device,
 301             self.log,
 302             self.remoteLogFile,
 303         )
 304
 305         startTime = datetime.datetime.now()
 306
 307         try:
 308             pid = rpm.launch_service(
 309                 cmd[1:], self.env, self.selectedProcess, test_name=test_name
 310             )
 311         except Exception as e:
 312             self.log.info(
 313                 "remotexpcshelltests.py | Failed to start process: %s" % str(e)
 314             )
 315             self.shellReturnCode = 1
 316             return ""
 317
 318         self.log.info(
 319             "remotexpcshelltests.py | %s | %s | Launched Test App"
 320             % (test_name, str(pid))
 321         )
 322
 323         if rpm.wait(timeout, test_name=test_name):
 324             self.shellReturnCode = 0
 325         else:
 326             self.shellReturnCode = 1
 327         self.log.info(
 328             "remotexpcshelltests.py | %s | %s | Application ran for: %s"
 329             % (test_name, str(pid), str(datetime.datetime.now() - startTime))
 330         )
 331
 332         try:
 333             return self.device.get_file(self.remoteLogFile)
 334         except mozdevice.ADBTimeoutError:
 335             raise
 336         except Exception as e:
 337             self.log.info(
 338                 "remotexpcshelltests.py | %s | %s | Could not read log file: %s"
 339                 % (test_name, str(pid), str(e))
 340             )
 341             self.shellReturnCode = 1
 342             return ""
 343
 344     def checkForCrashes(self, dump_directory, symbols_path, test_name=None):
 345         with mozfile.TemporaryDirectory() as dumpDir:
 346             self.device.pull(self.remoteMinidumpDir, dumpDir)
 347             crashed = mozcrash.log_crashes(
 348                 self.log, dumpDir, symbols_path, test=test_name
 349             )
 350         return crashed
 351
 352     def communicate(self, proc):
 353         return proc, ""
 354
 355     def poll(self, proc):
 356         if not self.device.process_exist("xpcshell"):
 357             return self.getReturnCode(proc)
 358         # Process is still running
 359         return None
 360
 361     def kill(self, proc):
 362         return self.device.pkill("xpcshell")
 363
 364     def getReturnCode(self, proc):
 365         if self.shellReturnCode is not None:
 366             return self.shellReturnCode
 367         else:
 368             return -1
 369
 370     def removeDir(self, dirname):
 371         try:
 372             self.device.rm(dirname, recursive=True)
 373         except ADBTimeoutError:
 374             raise
 375         except Exception as e:
 376             self.log.warning(str(e))
 377
 378     def createLogFile(self, test, stdout):
 379         filename = test.replace("\\", "/").split("/")[-1] + ".log"
 380         with open(filename, "wb") as f:
 381             f.write(stdout)
 382
 383
 384 # A specialization of XPCShellTests that runs tests on an Android device.
 385 class XPCShellRemote(xpcshell.XPCShellTests, object):
 386     def __init__(self, options, log):
 387         xpcshell.XPCShellTests.__init__(self, log)
 388
 389         options["threadCount"] = min(options["threadCount"] or 4, 4)
 390
 391         self.options = options
 392         verbose = False
 393         if options["log_tbpl_level"] == "debug" or options["log_mach_level"] == "debug":
 394             verbose = True
 395         self.device = ADBDeviceFactory(
 396             adb=options["adbPath"] or "adb",
 397             device=options["deviceSerial"],
 398             test_root=options["remoteTestRoot"],
 399             verbose=verbose,
 400         )
 401         self.remoteTestRoot = posixpath.join(self.device.test_root, "xpc")
 402         self.remoteLogFolder = posixpath.join(self.remoteTestRoot, "logs")
 403         # Add Android version (SDK level) to mozinfo so that manifest entries
 404         # can be conditional on android_version.
 405         mozinfo.info["android_version"] = str(self.device.version)
 406         mozinfo.info["is_emulator"] = self.device._device_serial.startswith("emulator-")
 407
 408         self.localBin = options["localBin"]
 409         self.pathMapping = []
 410         # remoteBinDir contains xpcshell and its wrapper script, both of which must
 411         # be executable. Since +x permissions cannot usually be set on /mnt/sdcard,
 412         # and the test root may be on /mnt/sdcard, remoteBinDir is set to be on
 413         # /data/local, always.
 414         self.remoteBinDir = posixpath.join(self.device.test_root, "xpcb")
 415         # Terse directory names are used here ("c" for the components directory)
 416         # to minimize the length of the command line used to execute
 417         # xpcshell on the remote device. adb has a limit to the number
 418         # of characters used in a shell command, and the xpcshell command
 419         # line can be quite complex.
 420         self.remoteTmpDir = posixpath.join(self.remoteTestRoot, "tmp")
 421         self.remoteScriptsDir = self.remoteTestRoot
 422         self.remoteComponentsDir = posixpath.join(self.remoteTestRoot, "c")
 423         self.remoteModulesDir = posixpath.join(self.remoteTestRoot, "m")
 424         self.remoteMinidumpRootDir = posixpath.join(self.remoteTestRoot, "minidumps")
 425         self.profileDir = posixpath.join(self.remoteTestRoot, "p")
 426         self.remoteDebugger = options["debugger"]
 427         self.remoteDebuggerArgs = options["debuggerArgs"]
 428         self.testingModulesDir = options["testingModulesDir"]
 429
 430         self.initDir(self.remoteTmpDir)
 431         self.initDir(self.profileDir)
 432
 433         # Make sure we get a fresh start
 434         self.device.stop_application("org.mozilla.geckoview.test_runner")
 435
 436         for i in range(options["threadCount"]):
 437             RemoteProcessMonitor.processStatus += [False]
 438
 439         self.env = {}
 440
 441         if options["objdir"]:
 442             self.xpcDir = os.path.join(options["objdir"], "_tests/xpcshell")
 443         elif os.path.isdir(os.path.join(here, "tests")):
 444             self.xpcDir = os.path.join(here, "tests")
 445         else:
 446             print("Couldn't find local xpcshell test directory", file=sys.stderr)
 447             sys.exit(1)
 448
 449         self.remoteAPK = None
 450         if options["localAPK"]:
 451             self.localAPKContents = ZipFile(options["localAPK"])
 452             self.remoteAPK = posixpath.join(
 453                 self.remoteBinDir, os.path.basename(options["localAPK"])
 454             )
 455         else:
 456             self.localAPKContents = None
 457         if options["setup"]:
 458             self.setupTestDir()
 459             self.setupUtilities()
 460             self.setupModules()
 461         self.initDir(self.remoteMinidumpRootDir)
 462         self.initDir(self.remoteLogFolder)
 463
 464         eprefs = options.get("extraPrefs") or []
 465         if options.get("disableFission"):
 466             eprefs.append("fission.autostart=false")
 467         else:
 468             # should be by default, just in case
 469             eprefs.append("fission.autostart=true")
 470         options["extraPrefs"] = eprefs
 471
 472         # data that needs to be passed to the RemoteXPCShellTestThread
 473         self.mobileArgs = {
 474             "device": self.device,
 475             "remoteBinDir": self.remoteBinDir,
 476             "remoteScriptsDir": self.remoteScriptsDir,
 477             "remoteComponentsDir": self.remoteComponentsDir,
 478             "remoteModulesDir": self.remoteModulesDir,
 479             "options": self.options,
 480             "remoteDebugger": self.remoteDebugger,
 481             "remoteDebuggerArgs": self.remoteDebuggerArgs,
 482             "pathMapping": self.pathMapping,
 483             "profileDir": self.profileDir,
 484             "remoteLogFolder": self.remoteLogFolder,
 485             "remoteTmpDir": self.remoteTmpDir,
 486             "remoteMinidumpRootDir": self.remoteMinidumpRootDir,
 487         }
 488         if self.remoteAPK:
 489             self.mobileArgs["remoteAPK"] = self.remoteAPK
 490
 491     def initDir(self, path, mask="777", timeout=None):
 492         """Initialize a directory by removing it if it exists, creating it
 493         and changing the permissions."""
 494         self.device.rm(path, recursive=True, force=True, timeout=timeout)
 495         self.device.mkdir(path, parents=True, timeout=timeout)
 496
 497     def setLD_LIBRARY_PATH(self):
 498         self.env["LD_LIBRARY_PATH"] = self.remoteBinDir
 499
 500     def pushWrapper(self):
 501         # Rather than executing xpcshell directly, this wrapper script is
 502         # used. By setting environment variables and the cwd in the script,
 503         # the length of the per-test command line is shortened. This is
 504         # often important when using ADB, as there is a limit to the length
 505         # of the ADB command line.
 506         localWrapper = tempfile.mktemp()
 507         with open(localWrapper, "w") as f:
 508             f.write("#!/system/bin/sh\n")
 509             for envkey, envval in six.iteritems(self.env):
 510                 f.write("export %s=%s\n" % (envkey, envval))
 511             f.writelines(
 512                 [
 513                     "cd $1\n",
 514                     "echo xpcw: cd $1\n",
 515                     "shift\n",
 516                     'echo xpcw: xpcshell "$@"\n',
 517                     '%s/xpcshell "$@"\n' % self.remoteBinDir,
 518                 ]
 519             )
 520         remoteWrapper = posixpath.join(self.remoteBinDir, "xpcw")
 521         self.device.push(localWrapper, remoteWrapper)
 522         self.device.chmod(remoteWrapper)
 523         os.remove(localWrapper)
 524
 525     def start_test(self, test):
 526         test.selectedProcess = RemoteProcessMonitor.pickUnusedProcess()
 527         if test.selectedProcess == -1:
 528             self.log.error(
 529                 "TEST-UNEXPECTED-FAIL | remotexpcshelltests.py | "
 530                 "no more free processes"
 531             )
 532         test.start()
 533
 534     def test_ended(self, test):
 535         RemoteProcessMonitor.freeProcess(test.selectedProcess)
 536
 537     def buildPrefsFile(self, extraPrefs):
 538         prefs = super(XPCShellRemote, self).buildPrefsFile(extraPrefs)
 539         remotePrefsFile = posixpath.join(self.remoteTestRoot, "user.js")
 540         self.device.push(self.prefsFile, remotePrefsFile)
 541         self.device.chmod(remotePrefsFile)
 542         # os.remove(self.prefsFile) is not called despite having pushed the
 543         # file to the device, because the local file is relied upon by the
 544         # updateTestPrefsFile method
 545         self.prefsFile = remotePrefsFile
 546         return prefs
 547
 548     def buildEnvironment(self):
 549         self.buildCoreEnvironment()
 550         self.setLD_LIBRARY_PATH()
 551         self.env["MOZ_LINKER_CACHE"] = self.remoteBinDir
 552         self.env["GRE_HOME"] = self.remoteBinDir
 553         self.env["XPCSHELL_TEST_PROFILE_DIR"] = self.profileDir
 554         self.env["HOME"] = self.profileDir
 555         self.env["XPCSHELL_TEST_TEMP_DIR"] = self.remoteTmpDir
 556         self.env["MOZ_ANDROID_DATA_DIR"] = self.remoteBinDir
 557         self.env["MOZ_IN_AUTOMATION"] = "1"
 558
 559         # Guard against intermittent failures to retrieve abi property;
 560         # without an abi, xpcshell cannot find greprefs.js and crashes.
 561         abilistprop = None
 562         abi = None
 563         retries = 0
 564         while not abi and retries < 3:
 565             abi = self.device.get_prop("ro.product.cpu.abi")
 566             retries += 1
 567         if not abi:
 568             raise Exception("failed to get ro.product.cpu.abi from device")
 569         self.log.info("ro.product.cpu.abi %s" % abi)
 570         if self.localAPKContents:
 571             abilist = [abi]
 572             retries = 0
 573             while not abilistprop and retries < 3:
 574                 abilistprop = self.device.get_prop("ro.product.cpu.abilist")
 575                 retries += 1
 576             self.log.info("ro.product.cpu.abilist %s" % abilistprop)
 577             abi_found = False
 578             names = [
 579                 n for n in self.localAPKContents.namelist() if n.startswith("lib/")
 580             ]
 581             self.log.debug("apk names: %s" % names)
 582             if abilistprop and len(abilistprop) > 0:
 583                 abilist.extend(abilistprop.split(","))
 584             for abicand in abilist:
 585                 abi_found = (
 586                     len([n for n in names if n.startswith("lib/%s" % abicand)]) > 0
 587                 )
 588                 if abi_found:
 589                     abi = abicand
 590                     break
 591             if not abi_found:
 592                 self.log.info("failed to get matching abi from apk.")
 593                 if len(names) > 0:
 594                     self.log.info(
 595                         "device cpu abi not found in apk. Using abi from apk."
 596                     )
 597                     abi = names[0].split("/")[1]
 598         self.log.info("Using abi %s." % abi)
 599         self.env["MOZ_ANDROID_CPU_ABI"] = abi
 600         self.log.info("Using env %r" % (self.env,))
 601
 602     def setupUtilities(self):
 603         self.initDir(self.remoteTmpDir)
 604         self.initDir(self.remoteBinDir)
 605         remotePrefDir = posixpath.join(self.remoteBinDir, "defaults", "pref")
 606         self.initDir(posixpath.join(remotePrefDir, "extra"))
 607         self.initDir(self.remoteComponentsDir)
 608
 609         local = os.path.join(os.path.dirname(os.path.abspath(__file__)), "head.js")
 610         remoteFile = posixpath.join(self.remoteScriptsDir, "head.js")
 611         self.device.push(local, remoteFile)
 612         self.device.chmod(remoteFile)
 613
 614         # Additional binaries are required for some tests. This list should be
 615         # similar to TEST_HARNESS_BINS in testing/mochitest/Makefile.in.
 616         binaries = [
 617             "ssltunnel",
 618             "certutil",
 619             "pk12util",
 620             "BadCertAndPinningServer",
 621             "DelegatedCredentialsServer",
 622             "EncryptedClientHelloServer",
 623             "FaultyServer",
 624             "OCSPStaplingServer",
 625             "GenerateOCSPResponse",
 626             "SanctionsTestServer",
 627         ]
 628         for fname in binaries:
 629             local = os.path.join(self.localBin, fname)
 630             if os.path.isfile(local):
 631                 print("Pushing %s.." % fname, file=sys.stderr)
 632                 remoteFile = posixpath.join(self.remoteBinDir, fname)
 633                 self.device.push(local, remoteFile)
 634                 self.device.chmod(remoteFile)
 635             else:
 636                 print(
 637                     "*** Expected binary %s not found in %s!" % (fname, self.localBin),
 638                     file=sys.stderr,
 639                 )
 640
 641         local = os.path.join(self.localBin, "components/httpd.sys.mjs")
 642         remoteFile = posixpath.join(self.remoteComponentsDir, "httpd.sys.mjs")
 643         self.device.push(local, remoteFile)
 644         self.device.chmod(remoteFile)
 645
 646         if self.options["localAPK"]:
 647             remoteFile = posixpath.join(
 648                 self.remoteBinDir, os.path.basename(self.options["localAPK"])
 649             )
 650             self.device.push(self.options["localAPK"], remoteFile)
 651             self.device.chmod(remoteFile)
 652
 653             self.pushLibs()
 654         else:
 655             localB2G = os.path.join(self.options["objdir"], "dist", "b2g")
 656             if os.path.exists(localB2G):
 657                 self.device.push(localB2G, self.remoteBinDir)
 658                 self.device.chmod(self.remoteBinDir)
 659             else:
 660                 raise Exception("unable to install gre: no APK and not b2g")
 661
 662     def pushLibs(self):
 663         pushed_libs_count = 0
 664         try:
 665             dir = tempfile.mkdtemp()
 666             for info in self.localAPKContents.infolist():
 667                 if info.filename.endswith(".so"):
 668                     print("Pushing %s.." % info.filename, file=sys.stderr)
 669                     remoteFile = posixpath.join(
 670                         self.remoteBinDir, os.path.basename(info.filename)
 671                     )
 672                     self.localAPKContents.extract(info, dir)
 673                     localFile = os.path.join(dir, info.filename)
 674                     self.device.push(localFile, remoteFile)
 675                     pushed_libs_count += 1
 676                     self.device.chmod(remoteFile)
 677         finally:
 678             shutil.rmtree(dir)
 679         return pushed_libs_count
 680
 681     def setupModules(self):
 682         if self.testingModulesDir:
 683             self.device.push(self.testingModulesDir, self.remoteModulesDir)
 684             self.device.chmod(self.remoteModulesDir)
 685
 686     def setupTestDir(self):
 687         print("pushing %s" % self.xpcDir)
 688         # The tests directory can be quite large: 5000 files and growing!
 689         # Sometimes - like on a low-end aws instance running an emulator - the push
 690         # may exceed the default 5 minute timeout, so we increase it here to 10 minutes.
 691         self.device.rm(self.remoteScriptsDir, recursive=True, force=True, timeout=None)
 692         self.device.push(self.xpcDir, self.remoteScriptsDir, timeout=600)
 693         self.device.chmod(self.remoteScriptsDir, recursive=True)
 694
 695     def setupSocketConnections(self):
 696         # make node host ports visible to device
 697         if "MOZHTTP2_PORT" in self.env:
 698             port = "tcp:{}".format(self.env["MOZHTTP2_PORT"])
 699             self.device.create_socket_connection(
 700                 ADBDevice.SOCKET_DIRECTION_REVERSE, port, port
 701             )
 702             self.log.info("reversed MOZHTTP2_PORT connection for port " + port)
 703         if "MOZNODE_EXEC_PORT" in self.env:
 704             port = "tcp:{}".format(self.env["MOZNODE_EXEC_PORT"])
 705             self.device.create_socket_connection(
 706                 ADBDevice.SOCKET_DIRECTION_REVERSE, port, port
 707             )
 708             self.log.info("reversed MOZNODE_EXEC_PORT connection for port " + port)
 709
 710     def buildTestList(self, test_tags=None, test_paths=None, verify=False):
 711         xpcshell.XPCShellTests.buildTestList(
 712             self, test_tags=test_tags, test_paths=test_paths, verify=verify
 713         )
 714         uniqueTestPaths = set([])
 715         for test in self.alltests:
 716             uniqueTestPaths.add(test["here"])
 717         for testdir in uniqueTestPaths:
 718             abbrevTestDir = os.path.relpath(testdir, self.xpcDir)
 719             remoteScriptDir = posixpath.join(self.remoteScriptsDir, abbrevTestDir)
 720             self.pathMapping.append(PathMapping(testdir, remoteScriptDir))
 721         # This is not related to building the test list, but since this is called late
 722         # in the test suite run, this is a convenient place to finalize preparations;
 723         # in particular, these operations cannot be executed much earlier because
 724         # self.env may not be finalized.
 725         self.setupSocketConnections()
 726         if self.options["setup"]:
 727             self.pushWrapper()
 728
 729
 730 def verifyRemoteOptions(parser, options):
 731     if isinstance(options, Namespace):
 732         options = vars(options)
 733
 734     if options["localBin"] is None:
 735         if options["objdir"]:
 736             options["localBin"] = os.path.join(options["objdir"], "dist", "bin")
 737             if not os.path.isdir(options["localBin"]):
 738                 parser.error("Couldn't find local binary dir, specify --local-bin-dir")
 739         elif os.path.isfile(os.path.join(here, "..", "bin", "xpcshell")):
 740             # assume tests are being run from a tests archive
 741             options["localBin"] = os.path.abspath(os.path.join(here, "..", "bin"))
 742         else:
 743             parser.error("Couldn't find local binary dir, specify --local-bin-dir")
 744     return options
 745
 746
 747 class PathMapping:
 748     def __init__(self, localDir, remoteDir):
 749         self.local = localDir
 750         self.remote = remoteDir
 751
 752
 753 def main():
 754     if sys.version_info < (2, 7):
 755         print(
 756             "Error: You must use python version 2.7 or newer but less than 3.0",
 757             file=sys.stderr,
 758         )
 759         sys.exit(1)
 760
 761     parser = parser_remote()
 762     options = parser.parse_args()
 763
 764     options = verifyRemoteOptions(parser, options)
 765     log = commandline.setup_logging("Remote XPCShell", options, {"tbpl": sys.stdout})
 766
 767     if options["interactive"] and not options["testPath"]:
 768         print(
 769             "Error: You must specify a test filename in interactive mode!",
 770             file=sys.stderr,
 771         )
 772         sys.exit(1)
 773
 774     if options["xpcshell"] is None:
 775         options["xpcshell"] = "xpcshell"
 776
 777     # The threadCount depends on the emulator rather than the host machine and
 778     # empirically 10 seems to yield the best performance.
 779     options["threadCount"] = min(options["threadCount"], 10)
 780
 781     xpcsh = XPCShellRemote(options, log)
 782
 783     if not xpcsh.runTests(
 784         options, testClass=RemoteXPCShellTestThread, mobileArgs=xpcsh.mobileArgs
 785     ):
 786         sys.exit(1)
 787
 788
 789 if __name__ == "__main__":
 790     main()