build/mobile/devicemanager.py

   1 # ***** BEGIN LICENSE BLOCK *****
   2 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   3 #
   4 # The contents of this file are subject to the Mozilla Public License Version
   5 # 1.1 (the "License"); you may not use this file except in compliance with
   6 # the License. You may obtain a copy of the License at
   7 # http://www.mozilla.org/MPL/
   8 #
   9 # Software distributed under the License is distributed on an "AS IS" basis,
  10 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  11 # for the specific language governing rights and limitations under the
  12 # License.
  13 #
  14 # The Original Code is Test Automation Framework.
  15 #
  16 # The Initial Developer of the Original Code is Joel Maher.
  17 #
  18 # Portions created by the Initial Developer are Copyright (C) 2009
  19 # the Initial Developer. All Rights Reserved.
  20 #
  21 # Contributor(s):
  22 # Joel Maher <joel.maher@gmail.com> (Original Developer)
  23 #
  24 # Alternatively, the contents of this file may be used under the terms of
  25 # either the GNU General Public License Version 2 or later (the "GPL"), or
  26 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  27 # in which case the provisions of the GPL or the LGPL are applicable instead
  28 # of those above. If you wish to allow use of your version of this file only
  29 # under the terms of either the GPL or the LGPL, and not to allow others to
  30 # use your version of this file under the terms of the MPL, indicate your
  31 # decision by deleting the provisions above and replace them with the notice
  32 # and other provisions required by the GPL or the LGPL. If you do not delete
  33 # the provisions above, a recipient may use your version of this file under
  34 # the terms of any one of the MPL, the GPL or the LGPL.
  35 #
  36 # ***** END LICENSE BLOCK *****
  37
  38 import socket
  39 import time, datetime
  40 import os
  41 import re
  42 import hashlib
  43 import subprocess
  44 from threading import Thread
  45 import traceback
  46 import sys
  47
  48 class FileError(Exception):
  49   " Signifies an error which occurs while doing a file operation."
  50
  51   def __init__(self, msg = ''):
  52     self.msg = msg
  53
  54   def __str__(self):
  55     return self.msg
  56
  57 class DeviceManager:
  58   host = ''
  59   port = 0
  60   debug = 3
  61   _redo = False
  62   deviceRoot = None
  63   tempRoot = os.getcwd()
  64   base_prompt = '\$\>'
  65   prompt_sep = '\x00'
  66   prompt_regex = '.*' + base_prompt + prompt_sep
  67
  68   def __init__(self, host, port = 27020):
  69     self.host = host
  70     self.port = port
  71     self._sock = None
  72     self.getDeviceRoot()
  73
  74   def sendCMD(self, cmdline, newline = True, sleep = 0):
  75     promptre = re.compile(self.prompt_regex + '$')
  76
  77     # TODO: any commands that don't output anything and quit need to match this RE
  78     pushre = re.compile('^push .*$')
  79     data = ""
  80     noQuit = False
  81
  82     if (self._sock == None):
  83       try:
  84         if (self.debug >= 1):
  85           print "reconnecting socket"
  86         self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  87       except:
  88         self._redo = True
  89         self._sock = None
  90         if (self.debug >= 2):
  91           print "unable to create socket"
  92         return None
  93
  94       try:
  95         self._sock.connect((self.host, int(self.port)))
  96         self._sock.recv(1024)
  97       except:
  98         self._redo = True
  99         self._sock.close()
 100         self._sock = None
 101         if (self.debug >= 2):
 102           print "unable to connect socket"
 103         return None
 104
 105     for cmd in cmdline:
 106       if (cmd == 'quit'): break
 107       if newline: cmd += '\r\n'
 108
 109       try:
 110         self._sock.send(cmd)
 111         if (self.debug >= 4): print "send cmd: " + str(cmd)
 112       except:
 113         self._redo = True
 114         self._sock.close()
 115         self._sock = None
 116         return None
 117
 118       if (pushre.match(cmd) or cmd == 'rebt'):
 119         noQuit = True
 120       elif noQuit == False:
 121         time.sleep(int(sleep))
 122         found = False
 123         while (found == False):
 124           if (self.debug >= 4): print "recv'ing..."
 125
 126           try:
 127             temp = self._sock.recv(1024)
 128           except:
 129             self._redo = True
 130             self._sock.close()
 131             self._sock = None
 132             return None
 133           lines = temp.split('\n')
 134           for line in lines:
 135             if (promptre.match(line)):
 136               found = True
 137           data += temp
 138
 139     time.sleep(int(sleep))
 140     if (noQuit == True):
 141       try:
 142         self._sock.close()
 143         self._sock = None
 144       except:
 145         self._redo = True
 146         self._sock = None
 147         return None
 148
 149     return data
 150
 151
 152   # take a data blob and strip instances of the prompt '$>\x00'
 153   def stripPrompt(self, data):
 154     promptre = re.compile(self.prompt_regex + '.*')
 155     retVal = []
 156     lines = data.split('\n')
 157     for line in lines:
 158       try:
 159         while (promptre.match(line)):
 160           pieces = line.split(self.prompt_sep)
 161           index = pieces.index('$>')
 162           pieces.pop(index)
 163           line = self.prompt_sep.join(pieces)
 164       except(ValueError):
 165         pass
 166       retVal.append(line)
 167
 168     return '\n'.join(retVal)
 169
 170
 171   def pushFile(self, localname, destname):
 172     if (self.debug >= 2): print "in push file with: " + localname + ", and: " + destname
 173     if (self.validateFile(destname, localname) == True):
 174       if (self.debug >= 2): print "files are validated"
 175       return ''
 176
 177     if self.mkDirs(destname) == None:
 178       print "unable to make dirs: " + destname
 179       return None
 180
 181     if (self.debug >= 2): print "sending: push " + destname
 182
 183     # sleep 5 seconds / MB
 184     filesize = os.path.getsize(localname)
 185     sleepsize = 1024 * 1024
 186     sleepTime = (int(filesize / sleepsize) * 5) + 2
 187     f = open(localname, 'rb')
 188     data = f.read()
 189     f.close()
 190     retVal = self.sendCMD(['push ' + destname + '\r\n', data], newline = False, sleep = sleepTime)
 191     if (retVal == None):
 192       if (self.debug >= 2): print "Error in sendCMD, not validating push"
 193       return None
 194
 195     if (self.validateFile(destname, localname) == False):
 196       if (self.debug >= 2): print "file did not copy as expected"
 197       return None
 198
 199     return retVal
 200
 201   def mkDir(self, name):
 202     return self.sendCMD(['mkdr ' + name])
 203
 204
 205   # make directory structure on the device
 206   def mkDirs(self, filename):
 207     parts = filename.split('/')
 208     name = ""
 209     for part in parts:
 210       if (part == parts[-1]): break
 211       if (part != ""):
 212         name += '/' + part
 213         if (self.mkDir(name) == None):
 214           print "failed making directory: " + str(name)
 215           return None
 216     return ''
 217
 218   # push localDir from host to remoteDir on the device
 219   def pushDir(self, localDir, remoteDir):
 220     if (self.debug >= 2): print "pushing directory: " + localDir + " to " + remoteDir
 221     for root, dirs, files in os.walk(localDir):
 222       parts = root.split(localDir)
 223       for file in files:
 224         remoteRoot = remoteDir + '/' + parts[1]
 225         remoteName = remoteRoot + '/' + file
 226         if (parts[1] == ""): remoteRoot = remoteDir
 227         if (self.pushFile(os.path.join(root, file), remoteName) == None):
 228           time.sleep(5)
 229           self.removeFile(remoteName)
 230           time.sleep(5)
 231           if (self.pushFile(os.path.join(root, file), remoteName) == None):
 232             return None
 233     return True
 234
 235
 236   def dirExists(self, dirname):
 237     match = ".*" + dirname + "$"
 238     dirre = re.compile(match)
 239     data = self.sendCMD(['cd ' + dirname, 'cwd', 'quit'], sleep = 1)
 240     if (data == None):
 241       return None
 242     retVal = self.stripPrompt(data)
 243     data = retVal.split('\n')
 244     found = False
 245     for d in data:
 246       if (dirre.match(d)):
 247         found = True
 248
 249     return found
 250
 251   # Because we always have / style paths we make this a lot easier with some
 252   # assumptions
 253   def fileExists(self, filepath):
 254     s = filepath.split('/')
 255     containingpath = '/'.join(s[:-1])
 256     listfiles = self.listFiles(containingpath)
 257     for f in listfiles:
 258       if (f == s[-1]):
 259         return True
 260     return False
 261
 262   # list files on the device, requires cd to directory first
 263   def listFiles(self, rootdir):
 264     if (self.dirExists(rootdir) == False):
 265       return []
 266     data = self.sendCMD(['cd ' + rootdir, 'ls', 'quit'], sleep=1)
 267     if (data == None):
 268       return None
 269     retVal = self.stripPrompt(data)
 270     return retVal.split('\n')
 271
 272   def removeFile(self, filename):
 273     if (self.debug>= 2): print "removing file: " + filename
 274     return self.sendCMD(['rm ' + filename, 'quit'])
 275
 276
 277   # does a recursive delete of directory on the device: rm -Rf remoteDir
 278   def removeDir(self, remoteDir):
 279     self.sendCMD(['rmdr ' + remoteDir], sleep = 5)
 280
 281
 282   def getProcessList(self):
 283     data = self.sendCMD(['ps'], sleep = 3)
 284     if (data == None):
 285       return None
 286
 287     retVal = self.stripPrompt(data)
 288     lines = retVal.split('\n')
 289     files = []
 290     for line in lines:
 291       if (line.strip() != ''):
 292         pidproc = line.strip().split()
 293         if (len(pidproc) == 2):
 294           files += [[pidproc[0], pidproc[1]]]
 295         elif (len(pidproc) == 3):
 296           #android returns <userID> <procID> <procName>
 297           files += [[pidproc[1], pidproc[2], pidproc[0]]]
 298     return files
 299
 300   def getMemInfo(self):
 301     data = self.sendCMD(['mems', 'quit'])
 302     if (data == None):
 303       return None
 304     retVal = self.stripPrompt(data)
 305     # TODO: this is hardcoded for now
 306     fhandle = open("memlog.txt", 'a')
 307     fhandle.write("\n")
 308     fhandle.write(retVal)
 309     fhandle.close()
 310
 311   def fireProcess(self, appname):
 312     if (self.debug >= 2): print "FIRE PROC: '" + appname + "'"
 313
 314     if (self.processExist(appname) != ''):
 315       print "WARNING: process %s appears to be running already\n" % appname
 316
 317     self.sendCMD(['exec ' + appname])
 318
 319     #NOTE: we sleep for 30 seconds to allow the application to startup
 320     time.sleep(30)
 321
 322     self.process = self.processExist(appname)
 323     if (self.debug >= 4): print "got pid: " + str(self.process) + " for process: " + str(appname)
 324
 325   def launchProcess(self, cmd, outputFile = "process.txt", cwd = ''):
 326     if (outputFile == "process.txt"):
 327       outputFile = self.getDeviceRoot() + '/' + "process.txt"
 328
 329     cmdline = subprocess.list2cmdline(cmd)
 330     self.fireProcess(cmdline + " > " + outputFile)
 331     handle = outputFile
 332
 333     return handle
 334
 335   #hardcoded: sleep interval of 5 seconds, timeout of 10 minutes
 336   def communicate(self, process, timeout = 600):
 337     interval = 5
 338     timed_out = True
 339     if (timeout > 0):
 340       total_time = 0
 341       while total_time < timeout:
 342         time.sleep(interval)
 343         if (not self.poll(process)):
 344           timed_out = False
 345           break
 346         total_time += interval
 347
 348     if (timed_out == True):
 349       return None
 350
 351     return [self.getFile(process, "temp.txt"), None]
 352
 353
 354   def poll(self, process):
 355     try:
 356       if (self.processExist(process) == None):
 357         return None
 358       return 1
 359     except:
 360       return None
 361     return 1
 362
 363
 364
 365   # iterates process list and returns pid if exists, otherwise ''
 366   def processExist(self, appname):
 367     pid = ''
 368
 369     pieces = appname.split(' ')
 370     parts = pieces[0].split('/')
 371     app = parts[-1]
 372     procre = re.compile('.*' + app + '.*')
 373
 374     procList = self.getProcessList()
 375     if (procList == None):
 376       return None
 377
 378     for proc in procList:
 379       if (procre.match(proc[1])):
 380         pid = proc[0]
 381         break
 382     return pid
 383
 384
 385   def killProcess(self, appname):
 386     if (self.sendCMD(['kill ' + appname]) == None):
 387       return None
 388
 389     return True
 390
 391   def getTempDir(self):
 392     retVal = ''
 393     data = self.sendCMD(['tmpd', 'quit'])
 394     if (data == None):
 395       return None
 396     return self.stripPrompt(data).strip('\n')
 397
 398
 399   # copy file from device (remoteFile) to host (localFile)
 400   def getFile(self, remoteFile, localFile = ''):
 401     if localFile == '':
 402         localFile = os.path.join(self.tempRoot, "temp.txt")
 403
 404     promptre = re.compile(self.prompt_regex + '.*')
 405     data = self.sendCMD(['cat ' + remoteFile, 'quit'], sleep = 5)
 406     if (data == None):
 407       return None
 408     retVal = self.stripPrompt(data)
 409     fhandle = open(localFile, 'wb')
 410     fhandle.write(retVal)
 411     fhandle.close()
 412     return retVal
 413
 414
 415   # copy directory structure from device (remoteDir) to host (localDir)
 416   def getDirectory(self, remoteDir, localDir):
 417     if (self.debug >= 2): print "getting files in '" + remoteDir + "'"
 418     filelist = self.listFiles(remoteDir)
 419     if (filelist == None):
 420       return None
 421     if (self.debug >= 3): print filelist
 422     if not os.path.exists(localDir):
 423       os.makedirs(localDir)
 424
 425     # TODO: is this a comprehensive file regex?
 426     isFile = re.compile('^([a-zA-Z0-9_\-\. ]+)\.([a-zA-Z0-9]+)$')
 427     for f in filelist:
 428       if (isFile.match(f)):
 429         if (self.getFile(remoteDir + '/' + f, os.path.join(localDir, f)) == None):
 430           return None
 431       else:
 432         if (self.getDirectory(remoteDir + '/' + f, os.path.join(localDir, f)) == None):
 433           return None
 434
 435
 436   # true/false check if the two files have the same md5 sum
 437   def validateFile(self, remoteFile, localFile):
 438     remoteHash = self.getRemoteHash(remoteFile)
 439     localHash = self.getLocalHash(localFile)
 440
 441     if (remoteHash == localHash):
 442         return True
 443
 444     return False
 445
 446
 447   # return the md5 sum of a remote file
 448   def getRemoteHash(self, filename):
 449       data = self.sendCMD(['hash ' + filename, 'quit'], sleep = 1)
 450       if (data == None):
 451           return ''
 452       retVal = self.stripPrompt(data)
 453       if (retVal != None):
 454         retVal = retVal.strip('\n')
 455       if (self.debug >= 3): print "remote hash returned: '" + retVal + "'"
 456       return retVal
 457
 458
 459   # return the md5 sum of a file on the host
 460   def getLocalHash(self, filename):
 461       file = open(filename, 'rb')
 462       if (file == None):
 463           return None
 464
 465       try:
 466         mdsum = hashlib.md5()
 467       except:
 468         return None
 469
 470       while 1:
 471           data = file.read(1024)
 472           if not data:
 473               break
 474           mdsum.update(data)
 475
 476       file.close()
 477       hexval = mdsum.hexdigest()
 478       if (self.debug >= 3): print "local hash returned: '" + hexval + "'"
 479       return hexval
 480
 481   # Gets the device root for the testing area on the device
 482   # For all devices we will use / type slashes and depend on the device-agent
 483   # to sort those out.  The agent will return us the device location where we
 484   # should store things, we will then create our /tests structure relative to
 485   # that returned path.
 486   # Structure on the device is as follows:
 487   # /tests
 488   #       /<fennec>|<firefox>  --> approot
 489   #       /profile
 490   #       /xpcshell
 491   #       /reftest
 492   #       /mochitest
 493   def getDeviceRoot(self):
 494     if (not self.deviceRoot):
 495       data = self.sendCMD(['testroot'], sleep = 1)
 496       if (data == None):
 497         return '/tests'
 498       self.deviceRoot = self.stripPrompt(data).strip('\n') + '/tests'
 499
 500     if (not self.dirExists(self.deviceRoot)):
 501       self.mkDir(self.deviceRoot)
 502
 503     return self.deviceRoot
 504
 505   # Either we will have /tests/fennec or /tests/firefox but we will never have
 506   # both.  Return the one that exists
 507   def getAppRoot(self):
 508     if (self.dirExists(self.getDeviceRoot() + '/fennec')):
 509       return self.getDeviceRoot() + '/fennec'
 510     elif (self.dirExists(self.getDeviceRoot() + '/firefox')):
 511       return self.getDeviceRoot() + '/firefox'
 512     else:
 513       return 'org.mozilla.fennec'
 514
 515   # Gets the directory location on the device for a specific test type
 516   # Type is one of: xpcshell|reftest|mochitest
 517   def getTestRoot(self, type):
 518     if (re.search('xpcshell', type, re.I)):
 519       self.testRoot = self.getDeviceRoot() + '/xpcshell'
 520     elif (re.search('?(i)reftest', type)):
 521       self.testRoot = self.getDeviceRoot() + '/reftest'
 522     elif (re.search('?(i)mochitest', type)):
 523       self.testRoot = self.getDeviceRoot() + '/mochitest'
 524     return self.testRoot
 525
 526   # Sends a specific process ID a signal code and action.
 527   # For Example: SIGINT and SIGDFL to process x
 528   def signal(self, processID, signalType, signalAction):
 529     # currently not implemented in device agent - todo
 530     pass
 531
 532   # Get a return code from process ending -- needs support on device-agent
 533   # this is a todo
 534   def getReturnCode(self, processID):
 535     # todo make this real
 536     return 0
 537
 538   def unpackFile(self, filename):
 539     dir = ''
 540     parts = filename.split('/')
 541     if (len(parts) > 1):
 542       if self.fileExists(filename):
 543         dir = '/'.join(parts[:-1])
 544     elif self.fileExists('/' + filename):
 545       dir = '/' + filename
 546     elif self.fileExists(self.getDeviceRoot() + '/' + filename):
 547       dir = self.getDeviceRoot() + '/' + filename
 548     else:
 549       return None
 550
 551     return self.sendCMD(['cd ' + dir, 'unzp ' + filename])
 552
 553
 554   def reboot(self, wait = False):
 555     self.sendCMD(['rebt'])
 556
 557     if wait == True:
 558       time.sleep(30)
 559       timeout = 270
 560       done = False
 561       while (not done):
 562         if self.listFiles('/') != None:
 563           return ''
 564         print "sleeping another 10 seconds"
 565         time.sleep(10)
 566         timeout = timeout - 10
 567         if (timeout <= 0):
 568           return None
 569     return ''
 570
 571   # validate localDir from host to remoteDir on the device
 572   def validateDir(self, localDir, remoteDir):
 573     if (self.debug >= 2): print "validating directory: " + localDir + " to " + remoteDir
 574     for root, dirs, files in os.walk(localDir):
 575       parts = root.split(localDir)
 576       for file in files:
 577         remoteRoot = remoteDir + '/' + parts[1]
 578         remoteRoot = remoteRoot.replace('/', '/')
 579         if (parts[1] == ""): remoteRoot = remoteDir
 580         remoteName = remoteRoot + '/' + file
 581         if (self.validateFile(remoteName, os.path.join(root, file)) <> True):
 582             return None
 583     return True
 584
 585   #TODO: make this simpler by doing a single directive at a time
 586   # Returns information about the device:
 587   # Directive indicates the information you want to get, your choices are:
 588   # os - name of the os
 589   # id - unique id of the device
 590   # uptime - uptime of the device
 591   # systime - system time of the device
 592   # screen - screen resolution
 593   # memory - memory stats
 594   # process - list of running processes (same as ps)
 595   # disk - total, free, available bytes on disk
 596   # power - power status (charge, battery temp)
 597   # all - all of them
 598   def getInfo(self, directive):
 599     data = None
 600     if (directive in ('os','id','uptime','systime','screen','memory','process',
 601                       'disk','power')):
 602       data = self.sendCMD(['info ' + directive, 'quit'], sleep = 1)
 603     else:
 604       directive = None
 605       data = self.sendCMD(['info', 'quit'], sleep = 1)
 606
 607     if (data is None):
 608       return None
 609
 610     data = self.stripPrompt(data)
 611     result = {}
 612
 613     if directive:
 614       result[directive] = data.split('\n')
 615       for i in range(len(result[directive])):
 616         if (len(result[directive][i]) != 0):
 617           result[directive][i] = result[directive][i].strip()
 618
 619       # Get rid of any empty attributes
 620       result[directive].remove('')
 621
 622     else:
 623       lines = data.split('\n')
 624       result['id'] = lines[0]
 625       result['os'] = lines[1]
 626       result['systime'] = lines[2]
 627       result['uptime'] = lines[3]
 628       result['screen'] = lines[4]
 629       result['memory'] = lines[5]
 630       if (lines[6] == 'Power status'):
 631         tmp = []
 632         for i in range(4):
 633           tmp.append(line[7 + i])
 634         result['power'] = tmp
 635       tmp = []
 636
 637       # Linenum is the line where the process list begins
 638       linenum = 11
 639       for j in range(len(lines) - linenum):
 640         if (lines[j + linenum].strip() != ''):
 641           procline = lines[j + linenum].split('\t')
 642
 643           if len(procline) == 2:
 644             tmp.append([procline[0], procline[1]])
 645           elif len(procline) == 3:
 646             # Android has <userid> <procid> <procname>
 647             # We put the userid to achieve a common format
 648             tmp.append([procline[1], procline[2], procline[0]])
 649       result['process'] = tmp
 650     return result
 651