testing/xpcshell/remotexpcshelltests.py

   1 #!/usr/bin/env python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 from __future__ import absolute_import, print_function
   8
   9 from argparse import Namespace
  10 import datetime
  11 import os
  12 import posixpath
  13 import mozdevice
  14 import shutil
  15 import six
  16 import sys
  17 import runxpcshelltests as xpcshell
  18 import tempfile
  19 import time
  20 import uuid
  21 from zipfile import ZipFile
  22
  23 import mozcrash
  24 from mozdevice import ADBDevice, ADBDeviceFactory, ADBTimeoutError
  25 import mozfile
  26 import mozinfo
  27 from mozlog import commandline
  28
  29 from xpcshellcommandline import parser_remote
  30
  31 here = os.path.dirname(os.path.abspath(__file__))
  32
  33
  34 class RemoteProcessMonitor(object):
  35     processStatus = []
  36
  37     def __init__(self, package, device, log, remoteLogFile):
  38         self.package = package
  39         self.device = device
  40         self.log = log
  41         self.remoteLogFile = remoteLogFile
  42         self.selectedProcess = -1
  43
  44     @classmethod
  45     def pickUnusedProcess(cls):
  46         for i in range(len(cls.processStatus)):
  47             if not cls.processStatus[i]:
  48                 cls.processStatus[i] = True
  49                 return i
  50         # No more free processes :(
  51         return -1
  52
  53     @classmethod
  54     def freeProcess(cls, processId):
  55         cls.processStatus[processId] = False
  56
  57     def kill(self):
  58         self.device.pkill(self.process_name, sig=9, attempts=1)
  59
  60     def launch_service(self, extra_args, env, selectedProcess):
  61         if not self.device.process_exist(self.package):
  62             # Make sure the main app is running, this should help making the
  63             # tests get foreground priority scheduling.
  64             self.device.launch_activity(
  65                 self.package,
  66                 intent="org.mozilla.geckoview.test.XPCSHELL_TEST_MAIN",
  67                 activity_name="TestRunnerActivity",
  68                 e10s=True,
  69             )
  70             # Newer Androids require that background services originate from
  71             # active apps, so wait here until the test runner is the top
  72             # activity.
  73             retries = 20
  74             top = self.device.get_top_activity(timeout=60)
  75             while top != self.package and retries > 0:
  76                 self.log.info("Checking that %s is the top activity." % self.package)
  77                 top = self.device.get_top_activity(timeout=60)
  78                 time.sleep(1)
  79                 retries -= 1
  80
  81         self.process_name = self.package + (":xpcshell%d" % selectedProcess)
  82         self.device.launch_service(
  83             self.package,
  84             activity_name=("XpcshellTestRunnerService$i%d" % selectedProcess),
  85             e10s=True,
  86             moz_env=env,
  87             grant_runtime_permissions=False,
  88             extra_args=extra_args,
  89             out_file=self.remoteLogFile,
  90         )
  91         return self.pid
  92
  93     def wait(self, timeout, interval=0.1):
  94         timer = 0
  95         status = True
  96
  97         # wait for log creation on startup
  98         retries = 0
  99         while retries < 20 / interval and not self.device.is_file(self.remoteLogFile):
 100             retries += 1
 101             time.sleep(interval)
 102         if not self.device.is_file(self.remoteLogFile):
 103             self.log.warning(
 104                 "Failed wait for remote log: %s missing?" % self.remoteLogFile
 105             )
 106
 107         while self.device.process_exist(self.process_name):
 108             time.sleep(interval)
 109             timer += interval
 110             interval *= 1.5
 111             if timeout and timer > timeout:
 112                 status = False
 113                 self.log.info("Timing out...")
 114                 self.kill()
 115                 break
 116         return status
 117
 118     @property
 119     def pid(self):
 120         """
 121         Determine the pid of the remote process (or the first process with
 122         the same name).
 123         """
 124         procs = self.device.get_process_list()
 125         # limit the comparison to the first 75 characters due to a
 126         # limitation in processname length in android.
 127         pids = [proc[0] for proc in procs if proc[1] == self.process_name[:75]]
 128         if pids is None or len(pids) < 1:
 129             return 0
 130         return pids[0]
 131
 132
 133 class RemoteXPCShellTestThread(xpcshell.XPCShellTestThread):
 134     def __init__(self, *args, **kwargs):
 135         xpcshell.XPCShellTestThread.__init__(self, *args, **kwargs)
 136
 137         self.shellReturnCode = None
 138         # embed the mobile params from the harness into the TestThread
 139         mobileArgs = kwargs.get("mobileArgs")
 140         for key in mobileArgs:
 141             setattr(self, key, mobileArgs[key])
 142         self.remoteLogFile = posixpath.join(
 143             mobileArgs["remoteLogFolder"], "xpcshell-%s.log" % str(uuid.uuid4())
 144         )
 145
 146     def initDir(self, path, mask="777", timeout=None):
 147         """Initialize a directory by removing it if it exists, creating it
 148         and changing the permissions."""
 149         self.device.rm(path, recursive=True, force=True, timeout=timeout)
 150         self.device.mkdir(path, parents=True, timeout=timeout)
 151
 152     def updateTestPrefsFile(self):
 153         # The base method will either be no-op (and return the existing
 154         # remote path), or return a path to a new local file.
 155         testPrefsFile = xpcshell.XPCShellTestThread.updateTestPrefsFile(self)
 156         if testPrefsFile == self.rootPrefsFile:
 157             # The pref file is the shared one, which has been already pushed on the
 158             # device, and so there is nothing more to do here.
 159             return self.rootPrefsFile
 160
 161         # Push the per-test prefs file in the remote temp dir.
 162         remoteTestPrefsFile = posixpath.join(self.remoteTmpDir, "user.js")
 163         self.device.push(testPrefsFile, remoteTestPrefsFile)
 164         self.device.chmod(remoteTestPrefsFile)
 165         os.remove(testPrefsFile)
 166         return remoteTestPrefsFile
 167
 168     def buildCmdTestFile(self, name):
 169         remoteDir = self.remoteForLocal(os.path.dirname(name))
 170         if remoteDir == self.remoteHere:
 171             remoteName = os.path.basename(name)
 172         else:
 173             remoteName = posixpath.join(remoteDir, os.path.basename(name))
 174         return [
 175             "-e",
 176             'const _TEST_CWD = "%s";' % self.remoteHere,
 177             "-e",
 178             'const _TEST_FILE = ["%s"];' % remoteName.replace("\\", "/"),
 179         ]
 180
 181     def remoteForLocal(self, local):
 182         for mapping in self.pathMapping:
 183             if os.path.abspath(mapping.local) == os.path.abspath(local):
 184                 return mapping.remote
 185         return local
 186
 187     def setupTempDir(self):
 188         self.remoteTmpDir = posixpath.join(self.remoteTmpDir, str(uuid.uuid4()))
 189         # make sure the temp dir exists
 190         self.initDir(self.remoteTmpDir)
 191         # env var is set in buildEnvironment
 192         self.env["XPCSHELL_TEST_TEMP_DIR"] = self.remoteTmpDir
 193         return self.remoteTmpDir
 194
 195     def setupPluginsDir(self):
 196         if not os.path.isdir(self.pluginsPath):
 197             return None
 198
 199         # making sure tmp dir is set up
 200         self.setupTempDir()
 201
 202         pluginsDir = posixpath.join(self.remoteTmpDir, "plugins")
 203         self.device.push(self.pluginsPath, pluginsDir)
 204         self.device.chmod(pluginsDir)
 205         if self.interactive:
 206             self.log.info("plugins dir is %s" % pluginsDir)
 207         return pluginsDir
 208
 209     def setupProfileDir(self):
 210         profileId = str(uuid.uuid4())
 211         self.profileDir = posixpath.join(self.profileDir, profileId)
 212         self.initDir(self.profileDir)
 213         if self.interactive or self.singleFile:
 214             self.log.info("profile dir is %s" % self.profileDir)
 215         self.env["XPCSHELL_TEST_PROFILE_DIR"] = self.profileDir
 216         self.env["TMPDIR"] = self.profileDir
 217         self.remoteMinidumpDir = posixpath.join(self.remoteMinidumpRootDir, profileId)
 218         self.initDir(self.remoteMinidumpDir)
 219         self.env["XPCSHELL_MINIDUMP_DIR"] = self.remoteMinidumpDir
 220         return self.profileDir
 221
 222     def clean_temp_dirs(self, name):
 223         self.log.info("Cleaning up profile for %s folder: %s" % (name, self.profileDir))
 224         self.device.rm(self.profileDir, force=True, recursive=True)
 225         self.device.rm(self.remoteTmpDir, force=True, recursive=True)
 226         self.device.rm(self.remoteMinidumpDir, force=True, recursive=True)
 227
 228     def setupMozinfoJS(self):
 229         local = tempfile.mktemp()
 230         mozinfo.output_to_file(local)
 231         mozInfoJSPath = posixpath.join(self.profileDir, "mozinfo.json")
 232         self.device.push(local, mozInfoJSPath)
 233         self.device.chmod(mozInfoJSPath)
 234         os.remove(local)
 235         return mozInfoJSPath
 236
 237     def logCommand(self, name, completeCmd, testdir):
 238         self.log.info("%s | full command: %r" % (name, completeCmd))
 239         self.log.info("%s | current directory: %r" % (name, self.remoteHere))
 240         self.log.info("%s | environment: %s" % (name, self.env))
 241
 242     def getHeadFiles(self, test):
 243         """Override parent method to find files on remote device.
 244
 245         Obtains lists of head- files.  Returns a list of head files.
 246         """
 247
 248         def sanitize_list(s, kind):
 249             for f in s.strip().split(" "):
 250                 f = f.strip()
 251                 if len(f) < 1:
 252                     continue
 253
 254                 path = posixpath.join(self.remoteHere, f)
 255
 256                 # skip check for file existence: the convenience of discovering
 257                 # a missing file does not justify the time cost of the round trip
 258                 # to the device
 259                 yield path
 260
 261         self.remoteHere = self.remoteForLocal(test["here"])
 262
 263         headlist = test.get("head", "")
 264         return list(sanitize_list(headlist, "head"))
 265
 266     def buildXpcsCmd(self):
 267         # change base class' paths to remote paths and use base class to build command
 268         self.xpcshell = posixpath.join(self.remoteBinDir, "xpcw")
 269         self.headJSPath = posixpath.join(self.remoteScriptsDir, "head.js")
 270         self.httpdJSPath = posixpath.join(self.remoteComponentsDir, "httpd.js")
 271         self.testingModulesDir = self.remoteModulesDir
 272         self.testharnessdir = self.remoteScriptsDir
 273         xpcsCmd = xpcshell.XPCShellTestThread.buildXpcsCmd(self)
 274         # remove "-g <dir> -a <dir>" and replace with remote alternatives
 275         del xpcsCmd[1:5]
 276         if self.options["localAPK"]:
 277             xpcsCmd.insert(1, "--greomni")
 278             xpcsCmd.insert(2, self.remoteAPK)
 279         xpcsCmd.insert(1, "-g")
 280         xpcsCmd.insert(2, self.remoteBinDir)
 281
 282         if self.remoteDebugger:
 283             # for example, "/data/local/gdbserver" "localhost:12345"
 284             xpcsCmd = [self.remoteDebugger, self.remoteDebuggerArgs] + xpcsCmd
 285         return xpcsCmd
 286
 287     def killTimeout(self, proc):
 288         self.kill(proc)
 289
 290     def launchProcess(self, cmd, stdout, stderr, env, cwd, timeout=None):
 291         rpm = RemoteProcessMonitor(
 292             "org.mozilla.geckoview.test",
 293             self.device,
 294             self.log,
 295             self.remoteLogFile,
 296         )
 297
 298         startTime = datetime.datetime.now()
 299
 300         try:
 301             pid = rpm.launch_service(cmd[1:], self.env, self.selectedProcess)
 302         except Exception as e:
 303             self.log.info(
 304                 "remotexpcshelltests.py | Failed to start process: %s" % str(e)
 305             )
 306             self.shellReturnCode = 1
 307             return ""
 308
 309         self.log.info("remotexpcshelltests.py | Launched Test App PID=%s" % str(pid))
 310
 311         if rpm.wait(timeout):
 312             self.shellReturnCode = 0
 313         else:
 314             self.shellReturnCode = 1
 315         self.log.info(
 316             "remotexpcshelltests.py | Application ran for: %s"
 317             % str(datetime.datetime.now() - startTime)
 318         )
 319
 320         try:
 321             return self.device.get_file(self.remoteLogFile)
 322         except mozdevice.ADBTimeoutError:
 323             raise
 324         except Exception as e:
 325             self.log.info(
 326                 "remotexpcshelltests.py | Could not read log file: %s" % str(e)
 327             )
 328             self.shellReturnCode = 1
 329             return ""
 330
 331     def checkForCrashes(self, dump_directory, symbols_path, test_name=None):
 332         with mozfile.TemporaryDirectory() as dumpDir:
 333             self.device.pull(self.remoteMinidumpDir, dumpDir)
 334             crashed = mozcrash.log_crashes(
 335                 self.log, dumpDir, symbols_path, test=test_name
 336             )
 337         return crashed
 338
 339     def communicate(self, proc):
 340         return proc, ""
 341
 342     def poll(self, proc):
 343         if not self.device.process_exist("xpcshell"):
 344             return self.getReturnCode(proc)
 345         # Process is still running
 346         return None
 347
 348     def kill(self, proc):
 349         return self.device.pkill("xpcshell")
 350
 351     def getReturnCode(self, proc):
 352         if self.shellReturnCode is not None:
 353             return self.shellReturnCode
 354         else:
 355             return -1
 356
 357     def removeDir(self, dirname):
 358         try:
 359             self.device.rm(dirname, recursive=True)
 360         except ADBTimeoutError:
 361             raise
 362         except Exception as e:
 363             self.log.warning(str(e))
 364
 365     def createLogFile(self, test, stdout):
 366         filename = test.replace("\\", "/").split("/")[-1] + ".log"
 367         with open(filename, "wb") as f:
 368             f.write(stdout)
 369
 370
 371 # A specialization of XPCShellTests that runs tests on an Android device.
 372 class XPCShellRemote(xpcshell.XPCShellTests, object):
 373     def __init__(self, options, log):
 374         xpcshell.XPCShellTests.__init__(self, log)
 375
 376         options["threadCount"] = min(options["threadCount"] or 4, 4)
 377
 378         self.options = options
 379         verbose = False
 380         if options["log_tbpl_level"] == "debug" or options["log_mach_level"] == "debug":
 381             verbose = True
 382         self.device = ADBDeviceFactory(
 383             adb=options["adbPath"] or "adb",
 384             device=options["deviceSerial"],
 385             test_root=options["remoteTestRoot"],
 386             verbose=verbose,
 387         )
 388         self.remoteTestRoot = posixpath.join(self.device.test_root, "xpc")
 389         self.remoteLogFolder = posixpath.join(self.remoteTestRoot, "logs")
 390         # Add Android version (SDK level) to mozinfo so that manifest entries
 391         # can be conditional on android_version.
 392         mozinfo.info["android_version"] = str(self.device.version)
 393         mozinfo.info["is_emulator"] = self.device._device_serial.startswith("emulator-")
 394
 395         self.localBin = options["localBin"]
 396         self.pathMapping = []
 397         # remoteBinDir contains xpcshell and its wrapper script, both of which must
 398         # be executable. Since +x permissions cannot usually be set on /mnt/sdcard,
 399         # and the test root may be on /mnt/sdcard, remoteBinDir is set to be on
 400         # /data/local, always.
 401         self.remoteBinDir = posixpath.join(self.device.test_root, "xpcb")
 402         # Terse directory names are used here ("c" for the components directory)
 403         # to minimize the length of the command line used to execute
 404         # xpcshell on the remote device. adb has a limit to the number
 405         # of characters used in a shell command, and the xpcshell command
 406         # line can be quite complex.
 407         self.remoteTmpDir = posixpath.join(self.remoteTestRoot, "tmp")
 408         self.remoteScriptsDir = self.remoteTestRoot
 409         self.remoteComponentsDir = posixpath.join(self.remoteTestRoot, "c")
 410         self.remoteModulesDir = posixpath.join(self.remoteTestRoot, "m")
 411         self.remoteMinidumpRootDir = posixpath.join(self.remoteTestRoot, "minidumps")
 412         self.profileDir = posixpath.join(self.remoteTestRoot, "p")
 413         self.remoteDebugger = options["debugger"]
 414         self.remoteDebuggerArgs = options["debuggerArgs"]
 415         self.testingModulesDir = options["testingModulesDir"]
 416
 417         self.initDir(self.remoteTmpDir)
 418         self.initDir(self.profileDir)
 419
 420         # Make sure we get a fresh start
 421         self.device.stop_application("org.mozilla.geckoview.test")
 422
 423         for i in range(options["threadCount"]):
 424             RemoteProcessMonitor.processStatus += [False]
 425
 426         self.env = {}
 427
 428         if options["objdir"]:
 429             self.xpcDir = os.path.join(options["objdir"], "_tests/xpcshell")
 430         elif os.path.isdir(os.path.join(here, "tests")):
 431             self.xpcDir = os.path.join(here, "tests")
 432         else:
 433             print("Couldn't find local xpcshell test directory", file=sys.stderr)
 434             sys.exit(1)
 435
 436         self.remoteAPK = None
 437         if options["localAPK"]:
 438             self.localAPKContents = ZipFile(options["localAPK"])
 439             self.remoteAPK = posixpath.join(
 440                 self.remoteBinDir, os.path.basename(options["localAPK"])
 441             )
 442         else:
 443             self.localAPKContents = None
 444         if options["setup"]:
 445             self.setupTestDir()
 446             self.setupUtilities()
 447             self.setupModules()
 448         self.initDir(self.remoteMinidumpRootDir)
 449         self.initDir(self.remoteLogFolder)
 450
 451         # data that needs to be passed to the RemoteXPCShellTestThread
 452         self.mobileArgs = {
 453             "device": self.device,
 454             "remoteBinDir": self.remoteBinDir,
 455             "remoteScriptsDir": self.remoteScriptsDir,
 456             "remoteComponentsDir": self.remoteComponentsDir,
 457             "remoteModulesDir": self.remoteModulesDir,
 458             "options": self.options,
 459             "remoteDebugger": self.remoteDebugger,
 460             "remoteDebuggerArgs": self.remoteDebuggerArgs,
 461             "pathMapping": self.pathMapping,
 462             "profileDir": self.profileDir,
 463             "remoteLogFolder": self.remoteLogFolder,
 464             "remoteTmpDir": self.remoteTmpDir,
 465             "remoteMinidumpRootDir": self.remoteMinidumpRootDir,
 466         }
 467         if self.remoteAPK:
 468             self.mobileArgs["remoteAPK"] = self.remoteAPK
 469
 470     def initDir(self, path, mask="777", timeout=None):
 471         """Initialize a directory by removing it if it exists, creating it
 472         and changing the permissions."""
 473         self.device.rm(path, recursive=True, force=True, timeout=timeout)
 474         self.device.mkdir(path, parents=True, timeout=timeout)
 475
 476     def setLD_LIBRARY_PATH(self):
 477         self.env["LD_LIBRARY_PATH"] = self.remoteBinDir
 478
 479     def pushWrapper(self):
 480         # Rather than executing xpcshell directly, this wrapper script is
 481         # used. By setting environment variables and the cwd in the script,
 482         # the length of the per-test command line is shortened. This is
 483         # often important when using ADB, as there is a limit to the length
 484         # of the ADB command line.
 485         localWrapper = tempfile.mktemp()
 486         with open(localWrapper, "w") as f:
 487             f.write("#!/system/bin/sh\n")
 488             for envkey, envval in six.iteritems(self.env):
 489                 f.write("export %s=%s\n" % (envkey, envval))
 490             f.writelines(
 491                 [
 492                     "cd $1\n",
 493                     "echo xpcw: cd $1\n",
 494                     "shift\n",
 495                     'echo xpcw: xpcshell "$@"\n',
 496                     '%s/xpcshell "$@"\n' % self.remoteBinDir,
 497                 ]
 498             )
 499         remoteWrapper = posixpath.join(self.remoteBinDir, "xpcw")
 500         self.device.push(localWrapper, remoteWrapper)
 501         self.device.chmod(remoteWrapper)
 502         os.remove(localWrapper)
 503
 504     def start_test(self, test):
 505         test.selectedProcess = RemoteProcessMonitor.pickUnusedProcess()
 506         if test.selectedProcess == -1:
 507             self.log.error(
 508                 "TEST-UNEXPECTED-FAIL | remotexpcshelltests.py | "
 509                 "no more free processes"
 510             )
 511         test.start()
 512
 513     def test_ended(self, test):
 514         RemoteProcessMonitor.freeProcess(test.selectedProcess)
 515
 516     def buildPrefsFile(self, extraPrefs):
 517         prefs = super(XPCShellRemote, self).buildPrefsFile(extraPrefs)
 518         remotePrefsFile = posixpath.join(self.remoteTestRoot, "user.js")
 519         self.device.push(self.prefsFile, remotePrefsFile)
 520         self.device.chmod(remotePrefsFile)
 521         # os.remove(self.prefsFile) is not called despite having pushed the
 522         # file to the device, because the local file is relied upon by the
 523         # updateTestPrefsFile method
 524         self.prefsFile = remotePrefsFile
 525         return prefs
 526
 527     def buildEnvironment(self):
 528         self.buildCoreEnvironment()
 529         self.setLD_LIBRARY_PATH()
 530         self.env["MOZ_LINKER_CACHE"] = self.remoteBinDir
 531         self.env["GRE_HOME"] = self.remoteBinDir
 532         self.env["XPCSHELL_TEST_PROFILE_DIR"] = self.profileDir
 533         self.env["HOME"] = self.profileDir
 534         self.env["XPCSHELL_TEST_TEMP_DIR"] = self.remoteTmpDir
 535         self.env["MOZ_ANDROID_DATA_DIR"] = self.remoteBinDir
 536         self.env["MOZ_IN_AUTOMATION"] = "1"
 537
 538         # Guard against intermittent failures to retrieve abi property;
 539         # without an abi, xpcshell cannot find greprefs.js and crashes.
 540         abilistprop = None
 541         abi = None
 542         retries = 0
 543         while not abi and retries < 3:
 544             abi = self.device.get_prop("ro.product.cpu.abi")
 545             retries += 1
 546         if not abi:
 547             raise Exception("failed to get ro.product.cpu.abi from device")
 548         self.log.info("ro.product.cpu.abi %s" % abi)
 549         if self.localAPKContents:
 550             abilist = [abi]
 551             retries = 0
 552             while not abilistprop and retries < 3:
 553                 abilistprop = self.device.get_prop("ro.product.cpu.abilist")
 554                 retries += 1
 555             self.log.info("ro.product.cpu.abilist %s" % abilistprop)
 556             abi_found = False
 557             names = [
 558                 n for n in self.localAPKContents.namelist() if n.startswith("lib/")
 559             ]
 560             self.log.debug("apk names: %s" % names)
 561             if abilistprop and len(abilistprop) > 0:
 562                 abilist.extend(abilistprop.split(","))
 563             for abicand in abilist:
 564                 abi_found = (
 565                     len([n for n in names if n.startswith("lib/%s" % abicand)]) > 0
 566                 )
 567                 if abi_found:
 568                     abi = abicand
 569                     break
 570             if not abi_found:
 571                 self.log.info("failed to get matching abi from apk.")
 572                 if len(names) > 0:
 573                     self.log.info(
 574                         "device cpu abi not found in apk. Using abi from apk."
 575                     )
 576                     abi = names[0].split("/")[1]
 577         self.log.info("Using abi %s." % abi)
 578         self.env["MOZ_ANDROID_CPU_ABI"] = abi
 579         self.log.info("Using env %r" % (self.env,))
 580
 581     def setupUtilities(self):
 582         self.initDir(self.remoteTmpDir)
 583         self.initDir(self.remoteBinDir)
 584         remotePrefDir = posixpath.join(self.remoteBinDir, "defaults", "pref")
 585         self.initDir(posixpath.join(remotePrefDir, "extra"))
 586         self.initDir(self.remoteComponentsDir)
 587
 588         local = os.path.join(os.path.dirname(os.path.abspath(__file__)), "head.js")
 589         remoteFile = posixpath.join(self.remoteScriptsDir, "head.js")
 590         self.device.push(local, remoteFile)
 591         self.device.chmod(remoteFile)
 592
 593         # Additional binaries are required for some tests. This list should be
 594         # similar to TEST_HARNESS_BINS in testing/mochitest/Makefile.in.
 595         binaries = [
 596             "ssltunnel",
 597             "certutil",
 598             "pk12util",
 599             "BadCertAndPinningServer",
 600             "DelegatedCredentialsServer",
 601             "EncryptedClientHelloServer",
 602             "OCSPStaplingServer",
 603             "GenerateOCSPResponse",
 604             "SanctionsTestServer",
 605         ]
 606         for fname in binaries:
 607             local = os.path.join(self.localBin, fname)
 608             if os.path.isfile(local):
 609                 print("Pushing %s.." % fname, file=sys.stderr)
 610                 remoteFile = posixpath.join(self.remoteBinDir, fname)
 611                 self.device.push(local, remoteFile)
 612                 self.device.chmod(remoteFile)
 613             else:
 614                 print(
 615                     "*** Expected binary %s not found in %s!" % (fname, self.localBin),
 616                     file=sys.stderr,
 617                 )
 618
 619         local = os.path.join(self.localBin, "components/httpd.js")
 620         remoteFile = posixpath.join(self.remoteComponentsDir, "httpd.js")
 621         self.device.push(local, remoteFile)
 622         self.device.chmod(remoteFile)
 623
 624         if self.options["localAPK"]:
 625             remoteFile = posixpath.join(
 626                 self.remoteBinDir, os.path.basename(self.options["localAPK"])
 627             )
 628             self.device.push(self.options["localAPK"], remoteFile)
 629             self.device.chmod(remoteFile)
 630
 631             self.pushLibs()
 632         else:
 633             localB2G = os.path.join(self.options["objdir"], "dist", "b2g")
 634             if os.path.exists(localB2G):
 635                 self.device.push(localB2G, self.remoteBinDir)
 636                 self.device.chmod(self.remoteBinDir)
 637             else:
 638                 raise Exception("unable to install gre: no APK and not b2g")
 639
 640     def pushLibs(self):
 641         pushed_libs_count = 0
 642         try:
 643             dir = tempfile.mkdtemp()
 644             for info in self.localAPKContents.infolist():
 645                 if info.filename.endswith(".so"):
 646                     print("Pushing %s.." % info.filename, file=sys.stderr)
 647                     remoteFile = posixpath.join(
 648                         self.remoteBinDir, os.path.basename(info.filename)
 649                     )
 650                     self.localAPKContents.extract(info, dir)
 651                     localFile = os.path.join(dir, info.filename)
 652                     self.device.push(localFile, remoteFile)
 653                     pushed_libs_count += 1
 654                     self.device.chmod(remoteFile)
 655         finally:
 656             shutil.rmtree(dir)
 657         return pushed_libs_count
 658
 659     def setupModules(self):
 660         if self.testingModulesDir:
 661             self.device.push(self.testingModulesDir, self.remoteModulesDir)
 662             self.device.chmod(self.remoteModulesDir)
 663
 664     def setupTestDir(self):
 665         print("pushing %s" % self.xpcDir)
 666         # The tests directory can be quite large: 5000 files and growing!
 667         # Sometimes - like on a low-end aws instance running an emulator - the push
 668         # may exceed the default 5 minute timeout, so we increase it here to 10 minutes.
 669         self.device.rm(self.remoteScriptsDir, recursive=True, force=True, timeout=None)
 670         self.device.push(self.xpcDir, self.remoteScriptsDir, timeout=600)
 671         self.device.chmod(self.remoteScriptsDir, recursive=True)
 672
 673     def setupSocketConnections(self):
 674         # make node host ports visible to device
 675         if "MOZHTTP2_PORT" in self.env:
 676             port = "tcp:{}".format(self.env["MOZHTTP2_PORT"])
 677             self.device.create_socket_connection(
 678                 ADBDevice.SOCKET_DIRECTION_REVERSE, port, port
 679             )
 680             self.log.info("reversed MOZHTTP2_PORT connection for port " + port)
 681         if "MOZNODE_EXEC_PORT" in self.env:
 682             port = "tcp:{}".format(self.env["MOZNODE_EXEC_PORT"])
 683             self.device.create_socket_connection(
 684                 ADBDevice.SOCKET_DIRECTION_REVERSE, port, port
 685             )
 686             self.log.info("reversed MOZNODE_EXEC_PORT connection for port " + port)
 687
 688     def buildTestList(self, test_tags=None, test_paths=None, verify=False):
 689         xpcshell.XPCShellTests.buildTestList(
 690             self, test_tags=test_tags, test_paths=test_paths, verify=verify
 691         )
 692         uniqueTestPaths = set([])
 693         for test in self.alltests:
 694             uniqueTestPaths.add(test["here"])
 695         for testdir in uniqueTestPaths:
 696             abbrevTestDir = os.path.relpath(testdir, self.xpcDir)
 697             remoteScriptDir = posixpath.join(self.remoteScriptsDir, abbrevTestDir)
 698             self.pathMapping.append(PathMapping(testdir, remoteScriptDir))
 699         # This is not related to building the test list, but since this is called late
 700         # in the test suite run, this is a convenient place to finalize preparations;
 701         # in particular, these operations cannot be executed much earlier because
 702         # self.env may not be finalized.
 703         self.setupSocketConnections()
 704         if self.options["setup"]:
 705             self.pushWrapper()
 706
 707
 708 def verifyRemoteOptions(parser, options):
 709     if isinstance(options, Namespace):
 710         options = vars(options)
 711
 712     if options["localBin"] is None:
 713         if options["objdir"]:
 714             options["localBin"] = os.path.join(options["objdir"], "dist", "bin")
 715             if not os.path.isdir(options["localBin"]):
 716                 parser.error("Couldn't find local binary dir, specify --local-bin-dir")
 717         elif os.path.isfile(os.path.join(here, "..", "bin", "xpcshell")):
 718             # assume tests are being run from a tests archive
 719             options["localBin"] = os.path.abspath(os.path.join(here, "..", "bin"))
 720         else:
 721             parser.error("Couldn't find local binary dir, specify --local-bin-dir")
 722     return options
 723
 724
 725 class PathMapping:
 726     def __init__(self, localDir, remoteDir):
 727         self.local = localDir
 728         self.remote = remoteDir
 729
 730
 731 def main():
 732     if sys.version_info < (2, 7):
 733         print(
 734             "Error: You must use python version 2.7 or newer but less than 3.0",
 735             file=sys.stderr,
 736         )
 737         sys.exit(1)
 738
 739     parser = parser_remote()
 740     options = parser.parse_args()
 741
 742     options = verifyRemoteOptions(parser, options)
 743     log = commandline.setup_logging("Remote XPCShell", options, {"tbpl": sys.stdout})
 744
 745     if options["interactive"] and not options["testPath"]:
 746         print(
 747             "Error: You must specify a test filename in interactive mode!",
 748             file=sys.stderr,
 749         )
 750         sys.exit(1)
 751
 752     if options["xpcshell"] is None:
 753         options["xpcshell"] = "xpcshell"
 754
 755     # The threadCount depends on the emulator rather than the host machine and
 756     # empirically 10 seems to yield the best performance.
 757     options["threadCount"] = min(options["threadCount"], 10)
 758
 759     xpcsh = XPCShellRemote(options, log)
 760
 761     if not xpcsh.runTests(
 762         options, testClass=RemoteXPCShellTestThread, mobileArgs=xpcsh.mobileArgs
 763     ):
 764         sys.exit(1)
 765
 766
 767 if __name__ == "__main__":
 768     main()