testing/mochitest/runtestsremote.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import sys
   6 import os
   7 import time
   8 import tempfile
   9 import re
  10 import traceback
  11 import shutil
  12 import math
  13 import base64
  14
  15 sys.path.insert(0, os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]))))
  16
  17 from automation import Automation
  18 from remoteautomation import RemoteAutomation, fennecLogcatFilters
  19 from runtests import Mochitest
  20 from runtests import MochitestOptions
  21 from runtests import MochitestServer
  22
  23 import devicemanager
  24 import droid
  25 import manifestparser
  26
  27 class RemoteOptions(MochitestOptions):
  28
  29     def __init__(self, automation, scriptdir, **kwargs):
  30         defaults = {}
  31         MochitestOptions.__init__(self, automation, scriptdir)
  32
  33         self.add_option("--remote-app-path", action="store",
  34                     type = "string", dest = "remoteAppPath",
  35                     help = "Path to remote executable relative to device root using only forward slashes. Either this or app must be specified but not both")
  36         defaults["remoteAppPath"] = None
  37
  38         self.add_option("--deviceIP", action="store",
  39                     type = "string", dest = "deviceIP",
  40                     help = "ip address of remote device to test")
  41         defaults["deviceIP"] = None
  42
  43         self.add_option("--dm_trans", action="store",
  44                     type = "string", dest = "dm_trans",
  45                     help = "the transport to use to communicate with device: [adb|sut]; default=sut")
  46         defaults["dm_trans"] = "sut"
  47
  48         self.add_option("--devicePort", action="store",
  49                     type = "string", dest = "devicePort",
  50                     help = "port of remote device to test")
  51         defaults["devicePort"] = 20701
  52
  53         self.add_option("--remote-product-name", action="store",
  54                     type = "string", dest = "remoteProductName",
  55                     help = "The executable's name of remote product to test - either fennec or firefox, defaults to fennec")
  56         defaults["remoteProductName"] = "fennec"
  57
  58         self.add_option("--remote-logfile", action="store",
  59                     type = "string", dest = "remoteLogFile",
  60                     help = "Name of log file on the device relative to the device root.  PLEASE ONLY USE A FILENAME.")
  61         defaults["remoteLogFile"] = None
  62
  63         self.add_option("--remote-webserver", action = "store",
  64                     type = "string", dest = "remoteWebServer",
  65                     help = "ip address where the remote web server is hosted at")
  66         defaults["remoteWebServer"] = None
  67
  68         self.add_option("--http-port", action = "store",
  69                     type = "string", dest = "httpPort",
  70                     help = "http port of the remote web server")
  71         defaults["httpPort"] = automation.DEFAULT_HTTP_PORT
  72
  73         self.add_option("--ssl-port", action = "store",
  74                     type = "string", dest = "sslPort",
  75                     help = "ssl port of the remote web server")
  76         defaults["sslPort"] = automation.DEFAULT_SSL_PORT
  77
  78         self.add_option("--pidfile", action = "store",
  79                     type = "string", dest = "pidFile",
  80                     help = "name of the pidfile to generate")
  81         defaults["pidFile"] = ""
  82
  83         self.add_option("--robocop-ini", action = "store",
  84                     type = "string", dest = "robocopIni",
  85                     help = "name of the .ini file containing the list of tests to run")
  86         defaults["robocopIni"] = ""
  87
  88         self.add_option("--robocop", action = "store",
  89                     type = "string", dest = "robocop",
  90                     help = "name of the .ini file containing the list of tests to run. [DEPRECATED- please use --robocop-ini")
  91         defaults["robocop"] = ""
  92
  93         self.add_option("--robocop-apk", action = "store",
  94                     type = "string", dest = "robocopApk",
  95                     help = "name of the Robocop APK to use for ADB test running")
  96         defaults["robocopApk"] = ""
  97
  98         self.add_option("--robocop-path", action = "store",
  99                     type = "string", dest = "robocopPath",
 100                     help = "Path to the folder where robocop.apk is located at.  Primarily used for ADB test running. [DEPRECATED- please use --robocop-apk]")
 101         defaults["robocopPath"] = ""
 102
 103         self.add_option("--robocop-ids", action = "store",
 104                     type = "string", dest = "robocopIds",
 105                     help = "name of the file containing the view ID map (fennec_ids.txt)")
 106         defaults["robocopIds"] = ""
 107
 108         self.add_option("--remoteTestRoot", action = "store",
 109                     type = "string", dest = "remoteTestRoot",
 110                     help = "remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)")
 111         defaults["remoteTestRoot"] = None
 112
 113         defaults["logFile"] = "mochitest.log"
 114         defaults["autorun"] = True
 115         defaults["closeWhenDone"] = True
 116         defaults["testPath"] = ""
 117         defaults["app"] = None
 118
 119         self.set_defaults(**defaults)
 120
 121     def verifyRemoteOptions(self, options, automation):
 122         if not options.remoteTestRoot:
 123             options.remoteTestRoot = automation._devicemanager.getDeviceRoot()
 124         productRoot = options.remoteTestRoot + "/" + automation._product
 125
 126         if (options.utilityPath == self._automation.DIST_BIN):
 127             options.utilityPath = productRoot + "/bin"
 128
 129         if options.remoteWebServer == None:
 130             if os.name != "nt":
 131                 options.remoteWebServer = automation.getLanIp()
 132             else:
 133                 print "ERROR: you must specify a --remote-webserver=<ip address>\n"
 134                 return None
 135
 136         options.webServer = options.remoteWebServer
 137
 138         if (options.deviceIP == None):
 139             print "ERROR: you must provide a device IP"
 140             return None
 141
 142         if (options.remoteLogFile == None):
 143             options.remoteLogFile = options.remoteTestRoot + '/logs/mochitest.log'
 144
 145         if (options.remoteLogFile.count('/') < 1):
 146             options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile
 147
 148         # remoteAppPath or app must be specified to find the product to launch
 149         if (options.remoteAppPath and options.app):
 150             print "ERROR: You cannot specify both the remoteAppPath and the app setting"
 151             return None
 152         elif (options.remoteAppPath):
 153             options.app = options.remoteTestRoot + "/" + options.remoteAppPath
 154         elif (options.app == None):
 155             # Neither remoteAppPath nor app are set -- error
 156             print "ERROR: You must specify either appPath or app"
 157             return None
 158
 159         # Only reset the xrePath if it wasn't provided
 160         if (options.xrePath == None):
 161             if (automation._product == "fennec"):
 162                 options.xrePath = productRoot + "/xulrunner"
 163             else:
 164                 options.xrePath = options.utilityPath
 165
 166         if (options.pidFile != ""):
 167             f = open(options.pidFile, 'w')
 168             f.write("%s" % os.getpid())
 169             f.close()
 170
 171         # Robocop specific deprecated options.
 172         if options.robocop:
 173             if options.robocopIni:
 174                 print "ERROR: can not use deprecated --robocop and replacement --robocop-ini together"
 175                 return None
 176             options.robocopIni = options.robocop
 177             del options.robocop
 178
 179         if options.robocopPath:
 180             if options.robocopApk:
 181                 print "ERROR: can not use deprecated --robocop-path and replacement --robocop-apk together"
 182                 return None
 183             options.robocopApk = os.path.join(options.robocopPath, 'robocop.apk')
 184             del options.robocopPath
 185
 186         # Robocop specific options
 187         if options.robocopIni != "":
 188             if not os.path.exists(options.robocopIni):
 189                 print "ERROR: Unable to find specified robocop .ini manifest '%s'" % options.robocopIni
 190                 return None
 191             options.robocopIni = os.path.abspath(options.robocopIni)
 192
 193         if options.robocopApk != "":
 194             if not os.path.exists(options.robocopApk):
 195                 print "ERROR: Unable to find robocop APK '%s'" % options.robocopApk
 196                 return None
 197             options.robocopApk = os.path.abspath(options.robocopApk)
 198
 199         if options.robocopIds != "":
 200             if not os.path.exists(options.robocopIds):
 201                 print "ERROR: Unable to find specified robocop IDs file '%s'" % options.robocopIds
 202                 return None
 203             options.robocopIds = os.path.abspath(options.robocopIds)
 204
 205         # allow us to keep original application around for cleanup while running robocop via 'am'
 206         options.remoteappname = options.app
 207         return options
 208
 209     def verifyOptions(self, options, mochitest):
 210         # since we are reusing verifyOptions, it will exit if App is not found
 211         temp = options.app
 212         options.app = sys.argv[0]
 213         tempPort = options.httpPort
 214         tempSSL = options.sslPort
 215         tempIP = options.webServer
 216         options = MochitestOptions.verifyOptions(self, options, mochitest)
 217         options.webServer = tempIP
 218         options.app = temp
 219         options.sslPort = tempSSL
 220         options.httpPort = tempPort
 221
 222         return options
 223
 224 class MochiRemote(Mochitest):
 225
 226     _automation = None
 227     _dm = None
 228     localProfile = None
 229     logLines = []
 230
 231     def __init__(self, automation, devmgr, options):
 232         self._automation = automation
 233         Mochitest.__init__(self, self._automation)
 234         self._dm = devmgr
 235         self.runSSLTunnel = False
 236         self.remoteProfile = options.remoteTestRoot + "/profile"
 237         self._automation.setRemoteProfile(self.remoteProfile)
 238         self.remoteLog = options.remoteLogFile
 239         self.localLog = options.logFile
 240         self._automation.deleteANRs()
 241
 242     def cleanup(self, manifest, options):
 243         if self._dm.fileExists(self.remoteLog):
 244             self._dm.getFile(self.remoteLog, self.localLog)
 245             self._dm.removeFile(self.remoteLog)
 246         else:
 247             print "WARNING: Unable to retrieve log file (%s) from remote " \
 248                 "device" % self.remoteLog
 249         self._dm.removeDir(self.remoteProfile)
 250
 251         if (options.pidFile != ""):
 252             try:
 253                 os.remove(options.pidFile)
 254                 os.remove(options.pidFile + ".xpcshell.pid")
 255             except:
 256                 print "Warning: cleaning up pidfile '%s' was unsuccessful from the test harness" % options.pidFile
 257
 258     def findPath(self, paths, filename = None):
 259         for path in paths:
 260             p = path
 261             if filename:
 262                 p = os.path.join(p, filename)
 263             if os.path.exists(self.getFullPath(p)):
 264                 return path
 265         return None
 266
 267     def startWebServer(self, options):
 268         """ Create the webserver on the host and start it up """
 269         remoteXrePath = options.xrePath
 270         remoteProfilePath = options.profilePath
 271         remoteUtilityPath = options.utilityPath
 272         localAutomation = Automation()
 273         localAutomation.IS_WIN32 = False
 274         localAutomation.IS_LINUX = False
 275         localAutomation.IS_MAC = False
 276         localAutomation.UNIXISH = False
 277         hostos = sys.platform
 278         if (hostos == 'mac' or  hostos == 'darwin'):
 279           localAutomation.IS_MAC = True
 280         elif (hostos == 'linux' or hostos == 'linux2'):
 281           localAutomation.IS_LINUX = True
 282           localAutomation.UNIXISH = True
 283         elif (hostos == 'win32' or hostos == 'win64'):
 284           localAutomation.BIN_SUFFIX = ".exe"
 285           localAutomation.IS_WIN32 = True
 286
 287         paths = [options.xrePath, localAutomation.DIST_BIN, self._automation._product, os.path.join('..', self._automation._product)]
 288         options.xrePath = self.findPath(paths)
 289         if options.xrePath == None:
 290             print "ERROR: unable to find xulrunner path for %s, please specify with --xre-path" % (os.name)
 291             sys.exit(1)
 292         paths.append("bin")
 293         paths.append(os.path.join("..", "bin"))
 294
 295         xpcshell = "xpcshell"
 296         if (os.name == "nt"):
 297             xpcshell += ".exe"
 298
 299         if (options.utilityPath):
 300             paths.insert(0, options.utilityPath)
 301         options.utilityPath = self.findPath(paths, xpcshell)
 302         if options.utilityPath == None:
 303             print "ERROR: unable to find utility path for %s, please specify with --utility-path" % (os.name)
 304             sys.exit(1)
 305         # httpd-path is specified by standard makefile targets and may be specified
 306         # on the command line to select a particular version of httpd.js. If not
 307         # specified, try to select the one from hostutils.zip, as required in bug 882932.
 308         if not options.httpdPath:
 309             options.httpdPath = os.path.join(options.utilityPath, "components")
 310
 311         xpcshell_path = os.path.join(options.utilityPath, xpcshell)
 312         if localAutomation.elf_arm(xpcshell_path):
 313             self.error('xpcshell at %s is an ARM binary; please use '
 314                        'the --utility-path argument to specify the path '
 315                        'to a desktop version.' % xpcshell)
 316
 317         options.profilePath = tempfile.mkdtemp()
 318         self.server = MochitestServer(localAutomation, options)
 319         self.server.start()
 320
 321         if (options.pidFile != ""):
 322             f = open(options.pidFile + ".xpcshell.pid", 'w')
 323             f.write("%s" % self.server._process.pid)
 324             f.close()
 325         self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT)
 326
 327         options.xrePath = remoteXrePath
 328         options.utilityPath = remoteUtilityPath
 329         options.profilePath = remoteProfilePath
 330
 331     def stopWebServer(self, options):
 332         self.server.stop()
 333
 334     def buildProfile(self, options):
 335         if self.localProfile:
 336             options.profilePath = self.localProfile
 337         manifest = Mochitest.buildProfile(self, options)
 338         self.localProfile = options.profilePath
 339         self._dm.removeDir(self.remoteProfile)
 340
 341         # we do not need this for robotium based tests, lets save a LOT of time
 342         if options.robocopIni:
 343             shutil.rmtree(os.path.join(options.profilePath, 'webapps'))
 344             shutil.rmtree(os.path.join(options.profilePath, 'extensions', 'staged', 'mochikit@mozilla.org'))
 345             shutil.rmtree(os.path.join(options.profilePath, 'extensions', 'staged', 'worker-test@mozilla.org'))
 346             shutil.rmtree(os.path.join(options.profilePath, 'extensions', 'staged', 'workerbootstrap-test@mozilla.org'))
 347             os.remove(os.path.join(options.profilePath, 'userChrome.css'))
 348             if os.path.exists(os.path.join(options.profilePath, 'tests.jar')):
 349                 os.remove(os.path.join(options.profilePath, 'tests.jar'))
 350             if os.path.exists(os.path.join(options.profilePath, 'tests.manifest')):
 351                 os.remove(os.path.join(options.profilePath, 'tests.manifest'))
 352
 353         try:
 354             self._dm.pushDir(options.profilePath, self.remoteProfile)
 355         except devicemanager.DMError:
 356             print "Automation Error: Unable to copy profile to device."
 357             raise
 358
 359         options.profilePath = self.remoteProfile
 360         return manifest
 361
 362     def buildURLOptions(self, options, env):
 363         self.localLog = options.logFile
 364         options.logFile = self.remoteLog
 365         options.profilePath = self.localProfile
 366         env["MOZ_HIDE_RESULTS_TABLE"] = "1"
 367         retVal = Mochitest.buildURLOptions(self, options, env)
 368
 369         if not options.robocopIni:
 370             #we really need testConfig.js (for browser chrome)
 371             try:
 372                 self._dm.pushDir(options.profilePath, self.remoteProfile)
 373             except devicemanager.DMError:
 374                 print "Automation Error: Unable to copy profile to device."
 375                 raise
 376
 377         options.profilePath = self.remoteProfile
 378         options.logFile = self.localLog
 379         return retVal
 380
 381     def installChromeFile(self, filename, options):
 382         parts = options.app.split('/')
 383         if (parts[0] == options.app):
 384           return "NO_CHROME_ON_DROID"
 385         path = '/'.join(parts[:-1])
 386         manifest = path + "/chrome/" + os.path.basename(filename)
 387         try:
 388             self._dm.pushFile(filename, manifest)
 389         except devicemanager.DMError:
 390             print "Automation Error: Unable to install Chrome files on device."
 391             raise
 392
 393         return manifest
 394
 395     def getLogFilePath(self, logFile):
 396         return logFile
 397
 398     # In the future we could use LogParser: http://hg.mozilla.org/automation/logparser/
 399     def addLogData(self):
 400         with open(self.localLog) as currentLog:
 401             data = currentLog.readlines()
 402
 403         restart = re.compile('0 INFO SimpleTest START.*')
 404         reend = re.compile('([0-9]+) INFO TEST-START . Shutdown.*')
 405         refail = re.compile('([0-9]+) INFO TEST-UNEXPECTED-FAIL.*')
 406         start_found = False
 407         end_found = False
 408         fail_found = False
 409         for line in data:
 410             if reend.match(line):
 411                 end_found = True
 412                 start_found = False
 413                 break
 414
 415             if start_found and not end_found:
 416                 # Append the line without the number to increment
 417                 self.logLines.append(' '.join(line.split(' ')[1:]))
 418
 419             if restart.match(line):
 420                 start_found = True
 421             if refail.match(line):
 422                 fail_found = True
 423         result = 0
 424         if fail_found:
 425             result = 1
 426         if not end_found:
 427             print "ERROR: missing end of test marker (process crashed?)"
 428             result = 1
 429         return result
 430
 431     def printLog(self):
 432         passed = 0
 433         failed = 0
 434         todo = 0
 435         incr = 1
 436         logFile = []
 437         logFile.append("0 INFO SimpleTest START")
 438         for line in self.logLines:
 439             if line.startswith("INFO TEST-PASS"):
 440                 passed += 1
 441             elif line.startswith("INFO TEST-UNEXPECTED"):
 442                 failed += 1
 443             elif line.startswith("INFO TEST-KNOWN"):
 444                 todo += 1
 445             incr += 1
 446
 447         logFile.append("%s INFO TEST-START | Shutdown" % incr)
 448         incr += 1
 449         logFile.append("%s INFO Passed: %s" % (incr, passed))
 450         incr += 1
 451         logFile.append("%s INFO Failed: %s" % (incr, failed))
 452         incr += 1
 453         logFile.append("%s INFO Todo: %s" % (incr, todo))
 454         incr += 1
 455         logFile.append("%s INFO SimpleTest FINISHED" % incr)
 456
 457         # TODO: Consider not printing to stdout because we might be duplicating output
 458         print '\n'.join(logFile)
 459         with open(self.localLog, 'w') as localLog:
 460             localLog.write('\n'.join(logFile))
 461
 462         if failed > 0:
 463             return 1
 464         return 0
 465
 466     def printScreenshot(self):
 467         try:
 468             image = self._dm.pullFile("/mnt/sdcard/Robotium-Screenshots/robocop-screenshot.jpg")
 469             encoded = base64.b64encode(image)
 470             print "SCREENSHOT: data:image/jpg;base64,%s" % encoded
 471         except:
 472             # If the test passes, no screenshot will be generated and
 473             # pullFile will fail -- continue silently.
 474             pass
 475
 476     def printDeviceInfo(self, printLogcat=False):
 477         try:
 478             if printLogcat:
 479                 logcat = self._dm.getLogcat(filterOutRegexps=fennecLogcatFilters)
 480                 print ''.join(logcat)
 481             print "Device info: %s" % self._dm.getInfo()
 482             print "Test root: %s" % self._dm.getDeviceRoot()
 483         except devicemanager.DMError:
 484             print "WARNING: Error getting device information"
 485
 486     def buildRobotiumConfig(self, options, browserEnv):
 487         deviceRoot = self._dm.getDeviceRoot()
 488         fHandle = tempfile.NamedTemporaryFile(suffix='.config',
 489                                               prefix='robotium-',
 490                                               dir=os.getcwd(),
 491                                               delete=False)
 492         fHandle.write("profile=%s\n" % (self.remoteProfile))
 493         fHandle.write("logfile=%s\n" % (options.remoteLogFile))
 494         fHandle.write("host=http://mochi.test:8888/tests\n")
 495         fHandle.write("rawhost=http://%s:%s/tests\n" % (options.remoteWebServer, options.httpPort))
 496
 497         if browserEnv:
 498             envstr = ""
 499             delim = ""
 500             for key, value in browserEnv.items():
 501                 try:
 502                     value.index(',')
 503                     print "Found: Error an ',' in our value, unable to process value."
 504                 except ValueError, e:
 505                     envstr += "%s%s=%s" % (delim, key, value)
 506                     delim = ","
 507
 508             fHandle.write("envvars=%s\n" % envstr)
 509         fHandle.close()
 510
 511         self._dm.removeFile(os.path.join(deviceRoot, "robotium.config"))
 512         self._dm.pushFile(fHandle.name, os.path.join(deviceRoot, "robotium.config"))
 513         os.unlink(fHandle.name)
 514
 515     def buildBrowserEnv(self, options):
 516         browserEnv = Mochitest.buildBrowserEnv(self, options)
 517         self.buildRobotiumConfig(options, browserEnv)
 518         return browserEnv
 519
 520
 521 def main():
 522     scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
 523     auto = RemoteAutomation(None, "fennec")
 524     parser = RemoteOptions(auto, scriptdir)
 525     options, args = parser.parse_args()
 526
 527     if (options.dm_trans == "adb"):
 528         if (options.deviceIP):
 529             dm = droid.DroidADB(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot)
 530         else:
 531             dm = droid.DroidADB(deviceRoot=options.remoteTestRoot)
 532     else:
 533          dm = droid.DroidSUT(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot)
 534     auto.setDeviceManager(dm)
 535     options = parser.verifyRemoteOptions(options, auto)
 536     if (options == None):
 537         print "ERROR: Invalid options specified, use --help for a list of valid options"
 538         sys.exit(1)
 539
 540     productPieces = options.remoteProductName.split('.')
 541     if (productPieces != None):
 542         auto.setProduct(productPieces[0])
 543     else:
 544         auto.setProduct(options.remoteProductName)
 545
 546     mochitest = MochiRemote(auto, dm, options)
 547
 548     options = parser.verifyOptions(options, mochitest)
 549     if (options == None):
 550         sys.exit(1)
 551
 552     logParent = os.path.dirname(options.remoteLogFile)
 553     dm.mkDir(logParent);
 554     auto.setRemoteLog(options.remoteLogFile)
 555     auto.setServerInfo(options.webServer, options.httpPort, options.sslPort)
 556
 557     mochitest.printDeviceInfo()
 558
 559     procName = options.app.split('/')[-1]
 560     if (dm.processExist(procName)):
 561         dm.killProcess(procName)
 562
 563     if options.robocopIni != "":
 564         # sut may wait up to 300 s for a robocop am process before returning
 565         dm.default_timeout = 320
 566         mp = manifestparser.TestManifest(strict=False)
 567         # TODO: pull this in dynamically
 568         mp.read(options.robocopIni)
 569         robocop_tests = mp.active_tests(exists=False)
 570         tests = []
 571         my_tests = tests
 572         for test in robocop_tests:
 573             tests.append(test['name'])
 574
 575         if options.totalChunks:
 576             tests_per_chunk = math.ceil(len(tests) / (options.totalChunks * 1.0))
 577             start = int(round((options.thisChunk-1) * tests_per_chunk))
 578             end = int(round(options.thisChunk * tests_per_chunk))
 579             if end > len(tests):
 580                 end = len(tests)
 581             my_tests = tests[start:end]
 582             print "Running tests %d-%d/%d" % ((start+1), end, len(tests))
 583
 584         deviceRoot = dm.getDeviceRoot()
 585         dm.removeFile(os.path.join(deviceRoot, "fennec_ids.txt"))
 586         fennec_ids = os.path.abspath("fennec_ids.txt")
 587         if not os.path.exists(fennec_ids) and options.robocopIds:
 588             fennec_ids = options.robocopIds
 589         dm.pushFile(fennec_ids, os.path.join(deviceRoot, "fennec_ids.txt"))
 590         options.extraPrefs.append('robocop.logfile="%s/robocop.log"' % deviceRoot)
 591         options.extraPrefs.append('browser.search.suggest.enabled=true')
 592         options.extraPrefs.append('browser.search.suggest.prompted=true')
 593         options.extraPrefs.append('browser.viewport.scaleRatio=100')
 594         options.extraPrefs.append('browser.chrome.dynamictoolbar=false')
 595
 596         if (options.dm_trans == 'adb' and options.robocopApk):
 597           dm._checkCmd(["install", "-r", options.robocopApk])
 598
 599         retVal = None
 600         for test in robocop_tests:
 601             if options.testPath and options.testPath != test['name']:
 602                 continue
 603
 604             if not test['name'] in my_tests:
 605                 continue
 606
 607             options.app = "am"
 608             options.browserArgs = ["instrument", "-w", "-e", "deviceroot", deviceRoot, "-e", "class"]
 609             options.browserArgs.append("%s.tests.%s" % (options.remoteappname, test['name']))
 610             options.browserArgs.append("org.mozilla.roboexample.test/%s.FennecInstrumentationTestRunner" % options.remoteappname)
 611
 612             # If the test is for checking the import from bookmarks then make sure there is data to import
 613             if test['name'] == "testImportFromAndroid":
 614
 615                 # Get the OS so we can run the insert in the apropriate database and following the correct table schema
 616                 osInfo = dm.getInfo("os")
 617                 devOS = " ".join(osInfo['os'])
 618
 619                 if ("pandaboard" in devOS):
 620                     delete = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"]
 621                 else:
 622                     delete = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"]
 623                 if (options.dm_trans == "sut"):
 624                     dm._runCmds([{"cmd": " ".join(delete)}])
 625
 626                 # Insert the bookmarks
 627                 print "Insert bookmarks in the default android browser database"
 628                 for i in range(20):
 629                    if ("pandaboard" in devOS):
 630                        cmd = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db 'insert or replace into bookmarks(_id,title,url,folder,parent,position) values (" + str(30 + i) + ",\"Bookmark"+ str(i) + "\",\"http://www.bookmark" + str(i) + ".com\",0,1," + str(100 + i) + ");'"]
 631                    else:
 632                        cmd = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db 'insert into bookmarks(title,url,bookmark) values (\"Bookmark"+ str(i) + "\",\"http://www.bookmark" + str(i) + ".com\",1);'"]
 633                    if (options.dm_trans == "sut"):
 634                        dm._runCmds([{"cmd": " ".join(cmd)}])
 635             try:
 636                 dm.removeDir("/mnt/sdcard/Robotium-Screenshots")
 637                 dm.recordLogcat()
 638                 result = mochitest.runTests(options)
 639                 if result != 0:
 640                     print "ERROR: runTests() exited with code %s" % result
 641                 log_result = mochitest.addLogData()
 642                 if result != 0 or log_result != 0:
 643                     mochitest.printDeviceInfo(printLogcat=True)
 644                     mochitest.printScreenshot()
 645                 # Ensure earlier failures aren't overwritten by success on this run
 646                 if retVal is None or retVal == 0:
 647                     retVal = result
 648             except:
 649                 print "Automation Error: Exception caught while running tests"
 650                 traceback.print_exc()
 651                 mochitest.stopWebServer(options)
 652                 mochitest.stopWebSocketServer(options)
 653                 try:
 654                     mochitest.cleanup(None, options)
 655                 except devicemanager.DMError:
 656                     # device error cleaning up... oh well!
 657                     pass
 658                 retVal = 1
 659                 break
 660             finally:
 661                 # Clean-up added bookmarks
 662                 if test['name'] == "testImportFromAndroid":
 663                     if ("pandaboard" in devOS):
 664                         cmd_del = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"]
 665                     else:
 666                         cmd_del = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"]
 667                     if (options.dm_trans == "sut"):
 668                         dm._runCmds([{"cmd": " ".join(cmd_del)}])
 669         if retVal is None:
 670             print "No tests run. Did you pass an invalid TEST_PATH?"
 671             retVal = 1
 672         else:
 673             # if we didn't have some kind of error running the tests, make
 674             # sure the tests actually passed
 675             print "INFO | runtests.py | Test summary: start."
 676             overallResult = mochitest.printLog()
 677             print "INFO | runtests.py | Test summary: end."
 678             if retVal == 0:
 679                 retVal = overallResult
 680     else:
 681         try:
 682             dm.recordLogcat()
 683             retVal = mochitest.runTests(options)
 684         except:
 685             print "Automation Error: Exception caught while running tests"
 686             traceback.print_exc()
 687             mochitest.stopWebServer(options)
 688             mochitest.stopWebSocketServer(options)
 689             try:
 690                 mochitest.cleanup(None, options)
 691             except devicemanager.DMError:
 692                 # device error cleaning up... oh well!
 693                 pass
 694             retVal = 1
 695
 696     mochitest.printDeviceInfo(printLogcat=True)
 697
 698     sys.exit(retVal)
 699
 700 if __name__ == "__main__":
 701     main()