testing/mochitest/runrobocop.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import json
   6 import os
   7 import posixpath
   8 import sys
   9 import tempfile
  10 import traceback
  11 from collections import defaultdict
  12
  13 sys.path.insert(
  14     0, os.path.abspath(
  15         os.path.realpath(
  16             os.path.dirname(__file__))))
  17
  18 from automation import Automation
  19 from remoteautomation import RemoteAutomation, fennecLogcatFilters
  20 from runtests import KeyValueParseError, MochitestDesktop, MessageLogger
  21 from mochitest_options import MochitestArgumentParser
  22
  23 from manifestparser import TestManifest
  24 from manifestparser.filters import chunk_by_slice
  25 from mozdevice import ADBAndroid, ADBTimeoutError
  26 from mozprofile.cli import parse_key_value, parse_preferences
  27 import mozfile
  28 import mozinfo
  29
  30 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
  31
  32
  33 class RobocopTestRunner(MochitestDesktop):
  34     """
  35        A test harness for Robocop. Robocop tests are UI tests for Firefox for Android,
  36        based on the Robotium test framework. This harness leverages some functionality
  37        from mochitest, for convenience.
  38     """
  39     # Some robocop tests run for >60 seconds without generating any output.
  40     NO_OUTPUT_TIMEOUT = 180
  41
  42     def __init__(self, options, message_logger):
  43         """
  44            Simple one-time initialization.
  45         """
  46         MochitestDesktop.__init__(self, options.flavor, vars(options))
  47
  48         verbose = False
  49         if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug':
  50             verbose = True
  51         self.device = ADBAndroid(adb=options.adbPath or 'adb',
  52                                  device=options.deviceSerial,
  53                                  test_root=options.remoteTestRoot,
  54                                  verbose=verbose)
  55
  56         # Check that Firefox is installed
  57         expected = options.app.split('/')[-1]
  58         if not self.device.is_app_installed(expected):
  59             raise Exception("%s is not installed on this device" % expected)
  60
  61         options.logFile = "robocop.log"
  62         if options.remoteTestRoot is None:
  63             options.remoteTestRoot = self.device.test_root
  64         self.remoteProfile = posixpath.join(options.remoteTestRoot, "profile")
  65         self.remoteProfileCopy = posixpath.join(options.remoteTestRoot, "profile-copy")
  66
  67         self.remoteModulesDir = posixpath.join(options.remoteTestRoot, "modules/")
  68         self.remoteConfigFile = posixpath.join(options.remoteTestRoot, "robotium.config")
  69         self.remoteLogFile = posixpath.join(options.remoteTestRoot, "logs", "robocop.log")
  70
  71         self.options = options
  72
  73         process_args = {'messageLogger': message_logger}
  74         self.auto = RemoteAutomation(self.device, options.remoteappname, self.remoteProfile,
  75                                      self.remoteLogFile, processArgs=process_args)
  76         self.environment = self.auto.environment
  77
  78         self.remoteScreenshots = "/mnt/sdcard/Robotium-Screenshots"
  79         self.remoteMozLog = posixpath.join(options.remoteTestRoot, "mozlog")
  80
  81         self.localLog = options.logFile
  82         self.localProfile = None
  83         self.certdbNew = True
  84         self.passed = 0
  85         self.failed = 0
  86         self.todo = 0
  87
  88     def startup(self):
  89         """
  90            Second-stage initialization: One-time initialization which may require cleanup.
  91         """
  92         # Despite our efforts to clean up servers started by this script, in practice
  93         # we still see infrequent cases where a process is orphaned and interferes
  94         # with future tests, typically because the old server is keeping the port in use.
  95         # Try to avoid those failures by checking for and killing servers before
  96         # trying to start new ones.
  97         self.killNamedProc('ssltunnel')
  98         self.killNamedProc('xpcshell')
  99         self.auto.deleteANRs()
 100         self.auto.deleteTombstones()
 101         procName = self.options.app.split('/')[-1]
 102         self.device.stop_application(procName)
 103         if self.device.process_exist(procName):
 104             self.log.warning("unable to kill %s before running tests!" % procName)
 105         self.device.rm(self.remoteScreenshots, force=True, recursive=True)
 106         self.device.rm(self.remoteMozLog, force=True, recursive=True)
 107         self.device.mkdir(self.remoteMozLog)
 108         logParent = posixpath.dirname(self.remoteLogFile)
 109         self.device.rm(logParent, force=True, recursive=True)
 110         self.device.mkdir(logParent)
 111         # Add Android version (SDK level) to mozinfo so that manifest entries
 112         # can be conditional on android_version.
 113         self.log.info(
 114             "Android sdk version '%s'; will use this to filter manifests" %
 115             str(self.device.version))
 116         mozinfo.info['android_version'] = str(self.device.version)
 117         if self.options.robocopApk:
 118             self.device.install_app(self.options.robocopApk, replace=True)
 119             self.log.debug("Robocop APK %s installed" %
 120                            self.options.robocopApk)
 121         # Display remote diagnostics; if running in mach, keep output terse.
 122         if self.options.log_mach is None:
 123             self.printDeviceInfo()
 124         self.setupLocalPaths()
 125         self.buildProfile()
 126         # ignoreSSLTunnelExts is a workaround for bug 1109310
 127         self.startServers(
 128             self.options,
 129             debuggerInfo=None,
 130             ignoreSSLTunnelExts=True)
 131         self.log.debug("Servers started")
 132
 133     def cleanup(self):
 134         """
 135            Cleanup at end of job run.
 136         """
 137         self.log.debug("Cleaning up...")
 138         self.stopServers()
 139         self.device.stop_application(self.options.app.split('/')[-1])
 140         uploadDir = os.environ.get('MOZ_UPLOAD_DIR', None)
 141         if uploadDir:
 142             self.log.debug("Pulling any remote moz logs and screenshots to %s." %
 143                            uploadDir)
 144             if self.device.is_dir(self.remoteMozLog):
 145                 self.device.pull(self.remoteMozLog, uploadDir)
 146             if self.device.is_dir(self.remoteScreenshots):
 147                 self.device.pull(self.remoteScreenshots, uploadDir)
 148         MochitestDesktop.cleanup(self, self.options)
 149         if self.localProfile:
 150             mozfile.remove(self.localProfile)
 151         self.device.rm(self.remoteProfile, force=True, recursive=True)
 152         self.device.rm(self.remoteProfileCopy, force=True, recursive=True)
 153         self.device.rm(self.remoteScreenshots, force=True, recursive=True)
 154         self.device.rm(self.remoteMozLog, force=True, recursive=True)
 155         self.device.rm(self.remoteConfigFile, force=True)
 156         self.device.rm(self.remoteLogFile, force=True)
 157         self.log.debug("Cleanup complete.")
 158
 159     def findPath(self, paths, filename=None):
 160         for path in paths:
 161             p = path
 162             if filename:
 163                 p = os.path.join(p, filename)
 164             if os.path.exists(self.getFullPath(p)):
 165                 return path
 166         return None
 167
 168     def makeLocalAutomation(self):
 169         localAutomation = Automation()
 170         localAutomation.IS_WIN32 = False
 171         localAutomation.IS_LINUX = False
 172         localAutomation.IS_MAC = False
 173         localAutomation.UNIXISH = False
 174         hostos = sys.platform
 175         if (hostos == 'mac' or hostos == 'darwin'):
 176             localAutomation.IS_MAC = True
 177         elif (hostos == 'linux' or hostos == 'linux2'):
 178             localAutomation.IS_LINUX = True
 179             localAutomation.UNIXISH = True
 180         elif (hostos == 'win32' or hostos == 'win64'):
 181             localAutomation.BIN_SUFFIX = ".exe"
 182             localAutomation.IS_WIN32 = True
 183         return localAutomation
 184
 185     def setupLocalPaths(self):
 186         """
 187            Setup xrePath and utilityPath and verify xpcshell.
 188
 189            This is similar to switchToLocalPaths in runtestsremote.py.
 190         """
 191         localAutomation = self.makeLocalAutomation()
 192         paths = [
 193             self.options.xrePath,
 194             localAutomation.DIST_BIN
 195         ]
 196         self.options.xrePath = self.findPath(paths)
 197         if self.options.xrePath is None:
 198             self.log.error(
 199                 "unable to find xulrunner path for %s, please specify with --xre-path" %
 200                 os.name)
 201             sys.exit(1)
 202         self.log.debug("using xre path %s" % self.options.xrePath)
 203         xpcshell = "xpcshell"
 204         if (os.name == "nt"):
 205             xpcshell += ".exe"
 206         if self.options.utilityPath:
 207             paths = [self.options.utilityPath, self.options.xrePath]
 208         else:
 209             paths = [self.options.xrePath]
 210         self.options.utilityPath = self.findPath(paths, xpcshell)
 211         if self.options.utilityPath is None:
 212             self.log.error(
 213                 "unable to find utility path for %s, please specify with --utility-path" %
 214                 os.name)
 215             sys.exit(1)
 216         self.log.debug("using utility path %s" % self.options.utilityPath)
 217         xpcshell_path = os.path.join(self.options.utilityPath, xpcshell)
 218         if localAutomation.elf_arm(xpcshell_path):
 219             self.log.error('xpcshell at %s is an ARM binary; please use '
 220                            'the --utility-path argument to specify the path '
 221                            'to a desktop version.' % xpcshell_path)
 222             sys.exit(1)
 223         self.log.debug("xpcshell found at %s" % xpcshell_path)
 224
 225     def buildProfile(self):
 226         """
 227            Build a profile locally, keep it locally for use by servers and
 228            push a copy to the remote profile-copy directory.
 229
 230            This is similar to buildProfile in runtestsremote.py.
 231         """
 232         self.options.extraPrefs.append('browser.search.suggest.enabled=true')
 233         self.options.extraPrefs.append('browser.search.suggest.prompted=true')
 234         self.options.extraPrefs.append('layout.css.devPixelsPerPx=1.0')
 235         self.options.extraPrefs.append('browser.chrome.dynamictoolbar=false')
 236         self.options.extraPrefs.append('browser.snippets.enabled=false')
 237         self.options.extraPrefs.append('extensions.autoupdate.enabled=false')
 238
 239         # Override the telemetry init delay for integration testing.
 240         self.options.extraPrefs.append('toolkit.telemetry.initDelay=1')
 241
 242         self.options.extensionsToExclude.extend([
 243             'mochikit@mozilla.org',
 244         ])
 245
 246         self.extraPrefs = parse_preferences(self.options.extraPrefs)
 247         if self.options.testingModulesDir:
 248             try:
 249                 self.device.push(self.options.testingModulesDir, self.remoteModulesDir)
 250                 self.device.chmod(self.remoteModulesDir, recursive=True, root=True)
 251             except Exception:
 252                 self.log.error(
 253                     "Automation Error: Unable to copy test modules to device.")
 254                 raise
 255             savedTestingModulesDir = self.options.testingModulesDir
 256             self.options.testingModulesDir = self.remoteModulesDir
 257         else:
 258             savedTestingModulesDir = None
 259         manifest = MochitestDesktop.buildProfile(self, self.options)
 260         if savedTestingModulesDir:
 261             self.options.testingModulesDir = savedTestingModulesDir
 262         self.localProfile = self.options.profilePath
 263         self.log.debug("Profile created at %s" % self.localProfile)
 264         # some files are not needed for robocop; save time by not pushing
 265         os.remove(os.path.join(self.localProfile, 'userChrome.css'))
 266         try:
 267             self.device.push(self.localProfile, self.remoteProfileCopy)
 268         except Exception:
 269             self.log.error(
 270                 "Automation Error: Unable to copy profile to device.")
 271             raise
 272
 273         return manifest
 274
 275     def setupRemoteProfile(self):
 276         """
 277            Remove any remote profile and re-create it.
 278         """
 279         self.log.debug("Updating remote profile at %s" % self.remoteProfile)
 280         self.device.rm(self.remoteProfile, force=True, recursive=True)
 281         self.device.cp(self.remoteProfileCopy, self.remoteProfile, recursive=True)
 282
 283     def parseLocalLog(self):
 284         """
 285            Read and parse the local log file, noting any failures.
 286         """
 287         with open(self.localLog) as currentLog:
 288             data = currentLog.readlines()
 289         os.unlink(self.localLog)
 290         start_found = False
 291         end_found = False
 292         fail_found = False
 293         for line in data:
 294             try:
 295                 message = json.loads(line)
 296                 if not isinstance(message, dict) or 'action' not in message:
 297                     continue
 298             except ValueError:
 299                 continue
 300             if message['action'] == 'test_end':
 301                 end_found = True
 302                 start_found = False
 303                 break
 304             if start_found and not end_found:
 305                 if 'status' in message:
 306                     if 'expected' in message:
 307                         self.failed += 1
 308                     elif message['status'] == 'PASS':
 309                         self.passed += 1
 310                     elif message['status'] == 'FAIL':
 311                         self.todo += 1
 312             if message['action'] == 'test_start':
 313                 start_found = True
 314             if 'expected' in message:
 315                 fail_found = True
 316         result = 0
 317         if fail_found:
 318             result = 1
 319         if not end_found:
 320             self.log.info(
 321                 "PROCESS-CRASH | Automation Error: Missing end of test marker (process crashed?)")
 322             result = 1
 323         return result
 324
 325     def logTestSummary(self):
 326         """
 327            Print a summary of all tests run to stdout, for treeherder parsing
 328            (logging via self.log does not work here).
 329         """
 330         print("0 INFO TEST-START | Shutdown")
 331         print("1 INFO Passed: %s" % (self.passed))
 332         print("2 INFO Failed: %s" % (self.failed))
 333         print("3 INFO Todo: %s" % (self.todo))
 334         print("4 INFO SimpleTest FINISHED")
 335         if self.failed > 0:
 336             return 1
 337         return 0
 338
 339     def printDeviceInfo(self, printLogcat=False):
 340         """
 341            Log remote device information and logcat (if requested).
 342
 343            This is similar to printDeviceInfo in runtestsremote.py
 344         """
 345         try:
 346             if printLogcat:
 347                 logcat = self.device.get_logcat(
 348                     filter_out_regexps=fennecLogcatFilters)
 349                 for l in logcat:
 350                     self.log.info(l.decode('utf-8', 'replace'))
 351             self.log.info("Device info:")
 352             devinfo = self.device.get_info()
 353             for category in devinfo:
 354                 if type(devinfo[category]) is list:
 355                     self.log.info("  %s:" % category)
 356                     for item in devinfo[category]:
 357                         self.log.info("     %s" % item)
 358                 else:
 359                     self.log.info("  %s: %s" % (category, devinfo[category]))
 360             self.log.info("Test root: %s" % self.device.test_root)
 361         except ADBTimeoutError:
 362             raise
 363         except Exception as e:
 364             self.log.warning("Error getting device information: %s" % str(e))
 365
 366     def setupRobotiumConfig(self, browserEnv):
 367         """
 368            Create robotium.config and push it to the device.
 369         """
 370         fHandle = tempfile.NamedTemporaryFile(suffix='.config',
 371                                               prefix='robotium-',
 372                                               dir=os.getcwd(),
 373                                               delete=False)
 374         fHandle.write("profile=%s\n" % self.remoteProfile)
 375         fHandle.write("logfile=%s\n" % self.remoteLogFile)
 376         fHandle.write("host=http://mochi.test:8888/tests\n")
 377         fHandle.write(
 378             "rawhost=http://%s:%s/tests\n" %
 379             (self.options.remoteWebServer, self.options.httpPort))
 380         if browserEnv:
 381             envstr = ""
 382             delim = ""
 383             for key, value in browserEnv.items():
 384                 try:
 385                     value.index(',')
 386                     self.log.error("setupRobotiumConfig: browserEnv - Found a ',' "
 387                                    "in our value, unable to process value. key=%s,value=%s" %
 388                                    (key, value))
 389                     self.log.error("browserEnv=%s" % browserEnv)
 390                 except ValueError:
 391                     envstr += "%s%s=%s" % (delim, key, value)
 392                     delim = ","
 393             fHandle.write("envvars=%s\n" % envstr)
 394         fHandle.close()
 395         self.device.rm(self.remoteConfigFile, force=True)
 396         self.device.push(fHandle.name, self.remoteConfigFile)
 397         os.unlink(fHandle.name)
 398
 399     def buildBrowserEnv(self):
 400         """
 401            Return an environment dictionary suitable for remote use.
 402
 403            This is similar to buildBrowserEnv in runtestsremote.py.
 404         """
 405         browserEnv = self.environment(
 406             xrePath=None,
 407             debugger=None)
 408         # remove desktop environment not used on device
 409         if "XPCOM_MEM_BLOAT_LOG" in browserEnv:
 410             del browserEnv["XPCOM_MEM_BLOAT_LOG"]
 411         browserEnv["MOZ_LOG_FILE"] = os.path.join(
 412             self.remoteMozLog,
 413             self.mozLogName)
 414
 415         try:
 416             browserEnv.update(
 417                 dict(
 418                     parse_key_value(
 419                         self.options.environment,
 420                         context='--setenv')))
 421         except KeyValueParseError as e:
 422             self.log.error(str(e))
 423             return None
 424
 425         return browserEnv
 426
 427     def runSingleTest(self, test):
 428         """
 429            Run the specified test.
 430         """
 431         self.log.debug("Running test %s" % test['name'])
 432         self.mozLogName = "moz-%s.log" % test['name']
 433         browserEnv = self.buildBrowserEnv()
 434         self.setupRobotiumConfig(browserEnv)
 435         self.setupRemoteProfile()
 436         self.options.app = "am"
 437         timeout = None
 438
 439         testName = test['name'].split('/')[-1].split('.java')[0]
 440         if self.options.enable_coverage:
 441             remoteCoverageFile = posixpath.join(self.options.remoteTestRoot,
 442                                                 'robocop-coverage-%s.ec' % testName)
 443             coverageFile = os.path.join(self.options.coverage_output_dir,
 444                                         'robocop-coverage-%s.ec' % testName)
 445         if self.options.autorun:
 446             # This launches a test (using "am instrument") and instructs
 447             # Fennec to /quit/ the browser (using Robocop:Quit) and to
 448             # /finish/ all opened activities.
 449             browserArgs = [
 450                 "instrument",
 451             ]
 452
 453             if self.options.enable_coverage:
 454                 browserArgs += [
 455                     "-e", "coverage", "true",
 456                     "-e", "coverageFile", remoteCoverageFile,
 457                 ]
 458
 459             browserArgs += [
 460                 "-e", "quit_and_finish", "1",
 461                 "-e", "deviceroot", self.device.test_root,
 462                 "-e", "class",
 463                 "org.mozilla.gecko.tests.%s" % testName,
 464                 "org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner",
 465             ]
 466         else:
 467             # This does not launch a test at all. It launches an activity
 468             # that starts Fennec and then waits indefinitely, since cat
 469             # never returns.
 470             browserArgs = ["start", "-n",
 471                            "org.mozilla.roboexample.test/org.mozilla."
 472                            "gecko.LaunchFennecWithConfigurationActivity", "&&", "cat"]
 473             timeout = sys.maxint  # Forever.
 474
 475             self.log.info("")
 476             self.log.info("Serving mochi.test Robocop root at http://%s:%s/tests/robocop/" %
 477                           (self.options.remoteWebServer, self.options.httpPort))
 478             self.log.info("")
 479         result = -1
 480         log_result = -1
 481         try:
 482             self.device.clear_logcat()
 483             if not timeout:
 484                 timeout = self.options.timeout
 485             if not timeout:
 486                 timeout = self.NO_OUTPUT_TIMEOUT
 487             result, _ = self.auto.runApp(
 488                 None, browserEnv, "am", self.localProfile, browserArgs,
 489                 timeout=timeout, symbolsPath=self.options.symbolsPath)
 490             self.log.debug("runApp completes with status %d" % result)
 491             if result != 0:
 492                 self.log.error("runApp() exited with code %s" % result)
 493             if self.device.is_file(self.remoteLogFile):
 494                 self.device.pull(self.remoteLogFile, self.localLog)
 495                 self.device.rm(self.remoteLogFile)
 496             log_result = self.parseLocalLog()
 497             if result != 0 or log_result != 0:
 498                 # Display remote diagnostics; if running in mach, keep output
 499                 # terse.
 500                 if self.options.log_mach is None:
 501                     self.printDeviceInfo(printLogcat=True)
 502             if self.options.enable_coverage:
 503                 if self.device.is_file(remoteCoverageFile):
 504                     self.device.pull(remoteCoverageFile, coverageFile)
 505                     self.device.rm(remoteCoverageFile)
 506                 else:
 507                     self.log.warning("Code coverage output not found on remote device: %s" %
 508                                      remoteCoverageFile)
 509
 510         except Exception:
 511             self.log.error(
 512                 "Automation Error: Exception caught while running tests")
 513             traceback.print_exc()
 514             result = 1
 515         self.log.debug("Test %s completes with status %d (log status %d)" %
 516                        (test['name'], int(result), int(log_result)))
 517         return result
 518
 519     def runTests(self):
 520         self.startup()
 521         if isinstance(self.options.manifestFile, TestManifest):
 522             mp = self.options.manifestFile
 523         else:
 524             mp = TestManifest(strict=False)
 525             mp.read("robocop.ini")
 526         filters = []
 527         if self.options.totalChunks:
 528             filters.append(
 529                 chunk_by_slice(self.options.thisChunk, self.options.totalChunks))
 530         robocop_tests = mp.active_tests(
 531             exists=False, filters=filters, **mozinfo.info)
 532         if not self.options.autorun:
 533             # Force a single loop iteration. The iteration will start Fennec and
 534             # the httpd server, but not actually run a test.
 535             self.options.test_paths = [robocop_tests[0]['name']]
 536         active_tests = []
 537         for test in robocop_tests:
 538             if self.options.test_paths and test['name'] not in self.options.test_paths:
 539                 continue
 540             if 'disabled' in test:
 541                 self.log.info('TEST-INFO | skipping %s | %s' %
 542                               (test['name'], test['disabled']))
 543                 continue
 544             active_tests.append(test)
 545
 546         tests_by_manifest = defaultdict(list)
 547         for test in active_tests:
 548             tests_by_manifest[test['manifest']].append(test['name'])
 549         self.log.suite_start(tests_by_manifest)
 550
 551         worstTestResult = None
 552         for test in active_tests:
 553             result = self.runSingleTest(test)
 554             if worstTestResult is None or worstTestResult == 0:
 555                 worstTestResult = result
 556         if worstTestResult is None:
 557             self.log.warning(
 558                 "No tests run. Did you pass an invalid TEST_PATH?")
 559             worstTestResult = 1
 560         else:
 561             print "INFO | runtests.py | Test summary: start."
 562             logResult = self.logTestSummary()
 563             print "INFO | runtests.py | Test summary: end."
 564             if worstTestResult == 0:
 565                 worstTestResult = logResult
 566         return worstTestResult
 567
 568
 569 def run_test_harness(parser, options):
 570     parser.validate(options)
 571
 572     if options is None:
 573         raise ValueError(
 574             "Invalid options specified, use --help for a list of valid options")
 575     message_logger = MessageLogger(logger=None)
 576     runResult = -1
 577     robocop = RobocopTestRunner(options, message_logger)
 578
 579     try:
 580         message_logger.logger = robocop.log
 581         message_logger.buffering = False
 582         robocop.message_logger = message_logger
 583         robocop.log.debug("options=%s" % vars(options))
 584         runResult = robocop.runTests()
 585     except KeyboardInterrupt:
 586         robocop.log.info("runrobocop.py | Received keyboard interrupt")
 587         runResult = -1
 588     except Exception:
 589         traceback.print_exc()
 590         robocop.log.error(
 591             "runrobocop.py | Received unexpected exception while running tests")
 592         runResult = 1
 593     finally:
 594         try:
 595             robocop.cleanup()
 596         except Exception:
 597             # ignore device error while cleaning up
 598             traceback.print_exc()
 599         message_logger.finish()
 600     return runResult
 601
 602
 603 def main(args=sys.argv[1:]):
 604     parser = MochitestArgumentParser(app='android')
 605     options = parser.parse_args(args)
 606     return run_test_harness(parser, options)
 607
 608
 609 if __name__ == "__main__":
 610     sys.exit(main())