build/mobile/remoteautomation.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import glob
   6 import time
   7 import re
   8 import os
   9 import tempfile
  10 import shutil
  11 import subprocess
  12 import sys
  13
  14 from automation import Automation
  15 from devicemanager import DMError
  16 from mozlog.structured import get_default_logger
  17 import mozcrash
  18
  19 # signatures for logcat messages that we don't care about much
  20 fennecLogcatFilters = [ "The character encoding of the HTML document was not declared",
  21                         "Use of Mutation Events is deprecated. Use MutationObserver instead.",
  22                         "Unexpected value from nativeGetEnabledTags: 0" ]
  23
  24 class RemoteAutomation(Automation):
  25     _devicemanager = None
  26
  27     def __init__(self, deviceManager, appName = '', remoteLog = None,
  28                  processArgs=None):
  29         self._devicemanager = deviceManager
  30         self._appName = appName
  31         self._remoteProfile = None
  32         self._remoteLog = remoteLog
  33         self._processArgs = processArgs or {};
  34
  35         # Default our product to fennec
  36         self._product = "fennec"
  37         self.lastTestSeen = "remoteautomation.py"
  38         Automation.__init__(self)
  39
  40     def setDeviceManager(self, deviceManager):
  41         self._devicemanager = deviceManager
  42
  43     def setAppName(self, appName):
  44         self._appName = appName
  45
  46     def setRemoteProfile(self, remoteProfile):
  47         self._remoteProfile = remoteProfile
  48
  49     def setProduct(self, product):
  50         self._product = product
  51
  52     def setRemoteLog(self, logfile):
  53         self._remoteLog = logfile
  54
  55     # Set up what we need for the remote environment
  56     def environment(self, env=None, xrePath=None, crashreporter=True, debugger=False, dmdPath=None, lsanPath=None):
  57         # Because we are running remote, we don't want to mimic the local env
  58         # so no copying of os.environ
  59         if env is None:
  60             env = {}
  61
  62         if dmdPath:
  63             env['MOZ_REPLACE_MALLOC_LIB'] = os.path.join(dmdPath, 'libdmd.so')
  64
  65         # Except for the mochitest results table hiding option, which isn't
  66         # passed to runtestsremote.py as an actual option, but through the
  67         # MOZ_HIDE_RESULTS_TABLE environment variable.
  68         if 'MOZ_HIDE_RESULTS_TABLE' in os.environ:
  69             env['MOZ_HIDE_RESULTS_TABLE'] = os.environ['MOZ_HIDE_RESULTS_TABLE']
  70
  71         if crashreporter and not debugger:
  72             env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
  73             env['MOZ_CRASHREPORTER'] = '1'
  74         else:
  75             env['MOZ_CRASHREPORTER_DISABLE'] = '1'
  76
  77         # Crash on non-local network connections by default.
  78         # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily
  79         # enable non-local connections for the purposes of local testing.
  80         # Don't override the user's choice here.  See bug 1049688.
  81         env.setdefault('MOZ_DISABLE_NONLOCAL_CONNECTIONS', '1')
  82
  83         return env
  84
  85     def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath):
  86         """ Wait for tests to finish.
  87             If maxTime seconds elapse or no output is detected for timeout
  88             seconds, kill the process and fail the test.
  89         """
  90         # maxTime is used to override the default timeout, we should honor that
  91         status = proc.wait(timeout = maxTime, noOutputTimeout = timeout)
  92         self.lastTestSeen = proc.getLastTestSeen
  93
  94         topActivity = self._devicemanager.getTopActivity()
  95         if topActivity == proc.procName:
  96             proc.kill(True)
  97         if status == 1:
  98             if maxTime:
  99                 print "TEST-UNEXPECTED-FAIL | %s | application ran for longer than " \
 100                       "allowed maximum time of %s seconds" % (self.lastTestSeen, maxTime)
 101             else:
 102                 print "TEST-UNEXPECTED-FAIL | %s | application ran for longer than " \
 103                       "allowed maximum time" % (self.lastTestSeen)
 104         if status == 2:
 105             print "TEST-UNEXPECTED-FAIL | %s | application timed out after %d seconds with no output" \
 106                 % (self.lastTestSeen, int(timeout))
 107
 108         return status
 109
 110     def deleteANRs(self):
 111         # empty ANR traces.txt file; usually need root permissions
 112         # we make it empty and writable so we can test the ANR reporter later
 113         traces = "/data/anr/traces.txt"
 114         try:
 115             self._devicemanager.shellCheckOutput(['echo', '', '>', traces], root=True)
 116             self._devicemanager.shellCheckOutput(['chmod', '666', traces], root=True)
 117         except DMError:
 118             print "Error deleting %s" % traces
 119             pass
 120
 121     def checkForANRs(self):
 122         traces = "/data/anr/traces.txt"
 123         if self._devicemanager.fileExists(traces):
 124             try:
 125                 t = self._devicemanager.pullFile(traces)
 126                 print "Contents of %s:" % traces
 127                 print t
 128                 # Once reported, delete traces
 129                 self.deleteANRs()
 130             except DMError:
 131                 print "Error pulling %s" % traces
 132             except IOError:
 133                 print "Error pulling %s" % traces
 134         else:
 135             print "%s not found" % traces
 136
 137     def deleteTombstones(self):
 138         # delete any existing tombstone files from device
 139         remoteDir = "/data/tombstones"
 140         try:
 141             self._devicemanager.shellCheckOutput(['rm', '-r', remoteDir], root=True)
 142         except DMError:
 143             # This may just indicate that the tombstone directory is missing
 144             pass
 145
 146     def checkForTombstones(self):
 147         # pull any tombstones from device and move to MOZ_UPLOAD_DIR
 148         remoteDir = "/data/tombstones"
 149         blobberUploadDir = os.environ.get('MOZ_UPLOAD_DIR', None)
 150         if blobberUploadDir:
 151             if not os.path.exists(blobberUploadDir):
 152                 os.mkdir(blobberUploadDir)
 153             if self._devicemanager.dirExists(remoteDir):
 154                 # copy tombstone files from device to local blobber upload directory
 155                 try:
 156                     self._devicemanager.shellCheckOutput(['chmod', '777', remoteDir], root=True)
 157                     self._devicemanager.shellCheckOutput(['chmod', '666', os.path.join(remoteDir, '*')], root=True)
 158                     self._devicemanager.getDirectory(remoteDir, blobberUploadDir, False)
 159                 except DMError:
 160                     # This may just indicate that no tombstone files are present
 161                     pass
 162                 self.deleteTombstones()
 163                 # add a .txt file extension to each tombstone file name, so
 164                 # that blobber will upload it
 165                 for f in glob.glob(os.path.join(blobberUploadDir, "tombstone_??")):
 166                     # add a unique integer to the file name, in case there are
 167                     # multiple tombstones generated with the same name, for
 168                     # instance, after multiple robocop tests
 169                     for i in xrange(1, sys.maxint):
 170                         newname = "%s.%d.txt" % (f, i)
 171                         if not os.path.exists(newname):
 172                             os.rename(f, newname)
 173                             break
 174             else:
 175                 print "%s does not exist; tombstone check skipped" % remoteDir
 176         else:
 177             print "MOZ_UPLOAD_DIR not defined; tombstone check skipped"
 178
 179     def checkForCrashes(self, directory, symbolsPath):
 180         self.checkForANRs()
 181         self.checkForTombstones()
 182
 183         logcat = self._devicemanager.getLogcat(filterOutRegexps=fennecLogcatFilters)
 184         javaException = mozcrash.check_for_java_exception(logcat)
 185         if javaException:
 186             return True
 187
 188         # If crash reporting is disabled (MOZ_CRASHREPORTER!=1), we can't say
 189         # anything.
 190         if not self.CRASHREPORTER:
 191             return False
 192
 193         try:
 194             dumpDir = tempfile.mkdtemp()
 195             remoteCrashDir = self._remoteProfile + '/minidumps/'
 196             if not self._devicemanager.dirExists(remoteCrashDir):
 197                 # If crash reporting is enabled (MOZ_CRASHREPORTER=1), the
 198                 # minidumps directory is automatically created when Fennec
 199                 # (first) starts, so its lack of presence is a hint that
 200                 # something went wrong.
 201                 print "Automation Error: No crash directory (%s) found on remote device" % remoteCrashDir
 202                 # Whilst no crash was found, the run should still display as a failure
 203                 return True
 204             self._devicemanager.getDirectory(remoteCrashDir, dumpDir)
 205
 206             logger = get_default_logger()
 207             if logger is not None:
 208                 crashed = mozcrash.log_crashes(logger, dumpDir, symbolsPath, test=self.lastTestSeen)
 209             else:
 210                 crashed = Automation.checkForCrashes(self, dumpDir, symbolsPath)
 211
 212         finally:
 213             try:
 214                 shutil.rmtree(dumpDir)
 215             except:
 216                 print "WARNING: unable to remove directory: %s" % dumpDir
 217         return crashed
 218
 219     def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs):
 220         # If remote profile is specified, use that instead
 221         if (self._remoteProfile):
 222             profileDir = self._remoteProfile
 223
 224         # Hack for robocop, if app & testURL == None and extraArgs contains the rest of the stuff, lets
 225         # assume extraArgs is all we need
 226         if app == "am" and extraArgs[0] == "instrument":
 227             return app, extraArgs
 228
 229         cmd, args = Automation.buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs)
 230         # Remove -foreground if it exists, if it doesn't this just returns
 231         try:
 232             args.remove('-foreground')
 233         except:
 234             pass
 235 #TODO: figure out which platform require NO_EM_RESTART
 236 #        return app, ['--environ:NO_EM_RESTART=1'] + args
 237         return app, args
 238
 239     def Process(self, cmd, stdout = None, stderr = None, env = None, cwd = None):
 240         if stdout == None or stdout == -1 or stdout == subprocess.PIPE:
 241             stdout = self._remoteLog
 242
 243         return self.RProcess(self._devicemanager, cmd, stdout, stderr, env, cwd, self._appName,
 244                              **self._processArgs)
 245
 246     # be careful here as this inner class doesn't have access to outer class members
 247     class RProcess(object):
 248         # device manager process
 249         dm = None
 250         def __init__(self, dm, cmd, stdout=None, stderr=None, env=None, cwd=None, app=None,
 251                      messageLogger=None):
 252             self.dm = dm
 253             self.stdoutlen = 0
 254             self.lastTestSeen = "remoteautomation.py"
 255             self.proc = dm.launchProcess(cmd, stdout, cwd, env, True)
 256             self.messageLogger = messageLogger
 257
 258             if (self.proc is None):
 259                 if cmd[0] == 'am':
 260                     self.proc = stdout
 261                 else:
 262                     raise Exception("unable to launch process")
 263             self.procName = cmd[0].split('/')[-1]
 264             if cmd[0] == 'am' and cmd[1] == "instrument":
 265                 self.procName = app
 266                 print "Robocop process name: "+self.procName
 267
 268             # Setting timeout at 1 hour since on a remote device this takes much longer
 269             self.timeout = 3600
 270             # The benefit of the following sleep is unclear; it was formerly 15 seconds
 271             time.sleep(1)
 272
 273             # Used to buffer log messages until we meet a line break
 274             self.logBuffer = ""
 275
 276         @property
 277         def pid(self):
 278             pid = self.dm.processExist(self.procName)
 279             # HACK: we should probably be more sophisticated about monitoring
 280             # running processes for the remote case, but for now we'll assume
 281             # that this method can be called when nothing exists and it is not
 282             # an error
 283             if pid is None:
 284                 return 0
 285             return pid
 286
 287         def read_stdout(self):
 288             """ Fetch the full remote log file using devicemanager and return just
 289                 the new log entries since the last call (as a list of messages or lines).
 290             """
 291             if not self.dm.fileExists(self.proc):
 292                 return []
 293             try:
 294                 newLogContent = self.dm.pullFile(self.proc, self.stdoutlen)
 295             except DMError:
 296                 # we currently don't retry properly in the pullFile
 297                 # function in dmSUT, so an error here is not necessarily
 298                 # the end of the world
 299                 return []
 300             if not newLogContent:
 301                 return []
 302
 303             self.stdoutlen += len(newLogContent)
 304
 305             if self.messageLogger is None:
 306                 testStartFilenames = re.findall(r"TEST-START \| ([^\s]*)", newLogContent)
 307                 if testStartFilenames:
 308                     self.lastTestSeen = testStartFilenames[-1]
 309                 print newLogContent
 310                 return [newLogContent]
 311
 312             self.logBuffer += newLogContent
 313             lines = self.logBuffer.split('\n')
 314             if not lines:
 315                 return
 316
 317             # We only keep the last (unfinished) line in the buffer
 318             self.logBuffer = lines[-1]
 319             del lines[-1]
 320             messages = []
 321             for line in lines:
 322                 # This passes the line to the logger (to be logged or buffered)
 323                 # and returns a list of structured messages (dict)
 324                 parsed_messages = self.messageLogger.write(line)
 325                 for message in parsed_messages:
 326                     if message['action'] == 'test_start':
 327                         self.lastTestSeen = message['test']
 328                 messages += parsed_messages
 329             return messages
 330
 331         @property
 332         def getLastTestSeen(self):
 333             return self.lastTestSeen
 334
 335         # Wait for the remote process to end (or for its activity to go to background).
 336         # While waiting, periodically retrieve the process output and print it.
 337         # If the process is still running after *timeout* seconds, return 1;
 338         # If the process is still running but no output is received in *noOutputTimeout*
 339         # seconds, return 2;
 340         # Else, once the process exits/goes to background, return 0.
 341         def wait(self, timeout = None, noOutputTimeout = None):
 342             timer = 0
 343             noOutputTimer = 0
 344             interval = 20
 345
 346             if timeout == None:
 347                 timeout = self.timeout
 348
 349             status = 0
 350             while (self.dm.getTopActivity() == self.procName):
 351                 # retrieve log updates every 60 seconds
 352                 if timer % 60 == 0:
 353                     messages = self.read_stdout()
 354                     if messages:
 355                         noOutputTimer = 0
 356
 357                 time.sleep(interval)
 358                 timer += interval
 359                 noOutputTimer += interval
 360                 if (timer > timeout):
 361                     status = 1
 362                     break
 363                 if (noOutputTimeout and noOutputTimer > noOutputTimeout):
 364                     status = 2
 365                     break
 366
 367             # Flush anything added to stdout during the sleep
 368             self.read_stdout()
 369
 370             return status
 371
 372         def kill(self, stagedShutdown = False):
 373             if stagedShutdown:
 374                 # Trigger an ANR report with "kill -3" (SIGQUIT)
 375                 self.dm.killProcess(self.procName, 3)
 376                 time.sleep(3)
 377                 # Trigger a breakpad dump with "kill -6" (SIGABRT)
 378                 self.dm.killProcess(self.procName, 6)
 379                 # Wait for process to end
 380                 retries = 0
 381                 while retries < 3:
 382                     pid = self.dm.processExist(self.procName)
 383                     if pid and pid > 0:
 384                         print "%s still alive after SIGABRT: waiting..." % self.procName
 385                         time.sleep(5)
 386                     else:
 387                         return
 388                     retries += 1
 389                 self.dm.killProcess(self.procName, 9)
 390                 pid = self.dm.processExist(self.procName)
 391                 if pid and pid > 0:
 392                     self.dm.killProcess(self.procName)
 393             else:
 394                 self.dm.killProcess(self.procName)