testing/mochitest/runtestsvmware.py

   1 #
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 import sys
   7 import os
   8 import re
   9 import types
  10 from optparse import OptionValueError
  11 from subprocess import PIPE
  12 from time import sleep
  13 from tempfile import mkstemp
  14
  15 sys.path.insert(0, os.path.abspath(os.path.realpath(
  16   os.path.dirname(sys.argv[0]))))
  17
  18 from automation import Automation
  19 from runtests import Mochitest, MochitestOptions
  20
  21 class VMwareOptions(MochitestOptions):
  22   def __init__(self, automation, mochitest, **kwargs):
  23     defaults = {}
  24     MochitestOptions.__init__(self, automation, mochitest.SCRIPT_DIRECTORY)
  25
  26     def checkPathCallback(option, opt_str, value, parser):
  27       path = mochitest.getFullPath(value)
  28       if not os.path.exists(path):
  29         raise OptionValueError("Path %s does not exist for %s option"
  30                                % (path, opt_str))
  31       setattr(parser.values, option.dest, path)
  32
  33     self.add_option("--with-vmware-vm",
  34                     action = "callback", type = "string", dest = "vmx",
  35                     callback = checkPathCallback,
  36                     help = "launches the given VM and runs mochitests inside")
  37     defaults["vmx"] = None
  38
  39     self.add_option("--with-vmrun-executable",
  40                     action = "callback", type = "string", dest = "vmrun",
  41                     callback = checkPathCallback,
  42                     help = "specifies the vmrun.exe to use for VMware control")
  43     defaults["vmrun"] = None
  44
  45     self.add_option("--shutdown-vm-when-done",
  46                     action = "store_true", dest = "shutdownVM",
  47                     help = "shuts down the VM when mochitests complete")
  48     defaults["shutdownVM"] = False
  49
  50     self.add_option("--repeat-until-failure",
  51                     action = "store_true", dest = "repeatUntilFailure",
  52                     help = "Runs tests continuously until failure")
  53     defaults["repeatUntilFailure"] = False
  54
  55     self.set_defaults(**defaults)
  56
  57 class VMwareMochitest(Mochitest):
  58   _pathFixRegEx = re.compile(r'^[cC](\:[\\\/]+)')
  59
  60   def convertHostPathsToGuestPaths(self, string):
  61     """ converts a path on the host machine to a path on the guest machine """
  62     # XXXbent Lame!
  63     return self._pathFixRegEx.sub(r'z\1', string)
  64
  65   def prepareGuestArguments(self, parser, options):
  66     """ returns an array of command line arguments needed to replicate the
  67         current set of options in the guest """
  68     args = []
  69     for key in options.__dict__.keys():
  70       # Don't send these args to the vm test runner!
  71       if key == "vmrun" or key == "vmx" or key == "repeatUntilFailure":
  72         continue
  73
  74       value = options.__dict__[key]
  75       valueType = type(value)
  76
  77       # Find the option in the parser's list.
  78       option = None
  79       for index in range(len(parser.option_list)):
  80         if str(parser.option_list[index].dest) == key:
  81           option = parser.option_list[index]
  82           break
  83       if not option:
  84         continue
  85
  86       # No need to pass args on the command line if they're just going to set
  87       # default values. The exception is list values... For some reason the
  88       # option parser modifies the defaults as well as the values when using the
  89       # "append" action.
  90       if value == parser.defaults[option.dest]:
  91         if valueType == types.StringType and \
  92            value == self.convertHostPathsToGuestPaths(value):
  93           continue
  94         if valueType != types.ListType:
  95           continue
  96
  97       def getArgString(arg, option):
  98         if option.action == "store_true" or option.action == "store_false":
  99           return str(option)
 100         return "%s=%s" % (str(option),
 101                           self.convertHostPathsToGuestPaths(str(arg)))
 102
 103       if valueType == types.ListType:
 104         # Expand lists into separate args.
 105         for item in value:
 106           args.append(getArgString(item, option))
 107       else:
 108         args.append(getArgString(value, option))
 109
 110     return tuple(args)
 111
 112   def launchVM(self, options):
 113     """ launches the VM and enables shared folders """
 114     # Launch VM first.
 115     self.automation.log.info("INFO | runtests.py | Launching the VM.")
 116     (result, stdout) = self.runVMCommand(self.vmrunargs + ("start", self.vmx))
 117     if result:
 118       return result
 119
 120     # Make sure that shared folders are enabled.
 121     self.automation.log.info("INFO | runtests.py | Enabling shared folders in "
 122                              "the VM.")
 123     (result, stdout) = self.runVMCommand(self.vmrunargs + \
 124                                          ("enableSharedFolders", self.vmx))
 125     if result:
 126       return result
 127
 128   def shutdownVM(self):
 129     """ shuts down the VM """
 130     self.automation.log.info("INFO | runtests.py | Shutting down the VM.")
 131     command = self.vmrunargs + ("runProgramInGuest", self.vmx,
 132               "c:\\windows\\system32\\shutdown.exe", "/s", "/t", "1")
 133     (result, stdout) = self.runVMCommand(command)
 134     return result
 135
 136   def runVMCommand(self, command, expectedErrors=[], silent=False):
 137     """ runs a command in the VM using the vmrun.exe helper """
 138     commandString = ""
 139     for part in command:
 140       commandString += str(part) + " "
 141     if not silent:
 142       self.automation.log.info("INFO | runtests.py | Running command: %s"
 143                                % commandString)
 144
 145     commonErrors = ["Error: Invalid user name or password for the guest OS",
 146                     "Unable to connect to host."]
 147     expectedErrors.extend(commonErrors)
 148
 149     # VMware can't run commands until the VM has fully loaded so keep running
 150     # this command in a loop until it succeeds or we try 100 times.
 151     errorString = ""
 152     for i in range(100):
 153       process = Automation.Process(command, stdout=PIPE)
 154       result = process.wait()
 155       if result == 0:
 156         break
 157
 158       for line in process.stdout.readlines():
 159         line = line.strip()
 160         if not line:
 161           continue
 162         errorString = line
 163         break
 164
 165       expected = False
 166       for error in expectedErrors:
 167         if errorString.startswith(error):
 168           expected = True
 169
 170       if not expected:
 171         self.automation.log.warning("WARNING | runtests.py | Command \"%s\" "
 172                                     "failed with result %d, : %s"
 173                                     % (commandString, result, errorString))
 174         break
 175
 176       if not silent:
 177         self.automation.log.info("INFO | runtests.py | Running command again.")
 178
 179     return (result, process.stdout.readlines())
 180
 181   def monitorVMExecution(self, appname, logfilepath):
 182     """ monitors test execution in the VM. Waits for the test process to start,
 183         then watches the log file for test failures and checks the status of the
 184         process to catch crashes. Returns True if mochitests ran successfully.
 185     """
 186     success = True
 187
 188     self.automation.log.info("INFO | runtests.py | Waiting for test process to "
 189                              "start.")
 190
 191     listProcessesCommand = self.vmrunargs + ("listProcessesInGuest", self.vmx)
 192     expectedErrors = [ "Error: The virtual machine is not powered on" ]
 193
 194     running  = False
 195     for i in range(100):
 196       (result, stdout) = self.runVMCommand(listProcessesCommand, expectedErrors,
 197                                            silent=True)
 198       if result:
 199         self.automation.log.warning("WARNING | runtests.py | Failed to get "
 200                                     "list of processes in VM!")
 201         return False
 202       for line in stdout:
 203         line = line.strip()
 204         if line.find(appname) != -1:
 205           running = True
 206           break
 207       if running:
 208         break
 209       sleep(1)
 210
 211     self.automation.log.info("INFO | runtests.py | Found test process, "
 212                              "monitoring log.")
 213
 214     completed = False
 215     nextLine = 0
 216     while running:
 217       log = open(logfilepath, "rb")
 218       lines = log.readlines()
 219       if len(lines) > nextLine:
 220         linesToPrint = lines[nextLine:]
 221         for line in linesToPrint:
 222           line = line.strip()
 223           if line.find("INFO SimpleTest FINISHED") != -1:
 224             completed = True
 225             continue
 226           if line.find("ERROR TEST-UNEXPECTED-FAIL") != -1:
 227             self.automation.log.info("INFO | runtests.py | Detected test "
 228                                      "failure: \"%s\"" % line)
 229             success = False
 230         nextLine = len(lines)
 231       log.close()
 232
 233       (result, stdout) = self.runVMCommand(listProcessesCommand, expectedErrors,
 234                                            silent=True)
 235       if result:
 236         self.automation.log.warning("WARNING | runtests.py | Failed to get "
 237                                     "list of processes in VM!")
 238         return False
 239
 240       stillRunning = False
 241       for line in stdout:
 242         line = line.strip()
 243         if line.find(appname) != -1:
 244           stillRunning = True
 245           break
 246       if stillRunning:
 247         sleep(5)
 248       else:
 249         if not completed:
 250           self.automation.log.info("INFO | runtests.py | Test process exited "
 251                                    "without finishing tests, maybe crashed.")
 252           success = False
 253         running = stillRunning
 254
 255     return success
 256
 257   def getCurentSnapshotList(self):
 258     """ gets a list of snapshots from the VM """
 259     (result, stdout) = self.runVMCommand(self.vmrunargs + ("listSnapshots",
 260                                                            self.vmx))
 261     snapshots = []
 262     if result != 0:
 263       self.automation.log.warning("WARNING | runtests.py | Failed to get list "
 264                                   "of snapshots in VM!")
 265       return snapshots
 266     for line in stdout:
 267       if line.startswith("Total snapshots:"):
 268         continue
 269       snapshots.append(line.strip())
 270     return snapshots
 271
 272   def runTests(self, parser, options):
 273     """ runs mochitests in the VM """
 274     # Base args that must always be passed to vmrun.
 275     self.vmrunargs = (options.vmrun, "-T", "ws", "-gu", "Replay", "-gp",
 276                       "mozilla")
 277     self.vmrun = options.vmrun
 278     self.vmx = options.vmx
 279
 280     result = self.launchVM(options)
 281     if result:
 282       return result
 283
 284     if options.vmwareRecording:
 285       snapshots = self.getCurentSnapshotList()
 286
 287     def innerRun():
 288       """ subset of the function that must run every time if we're running until
 289           failure """
 290       # Make a new shared file for the log file.
 291       (logfile, logfilepath) = mkstemp(suffix=".log")
 292       os.close(logfile)
 293       # Get args to pass to VM process. Make sure we autorun and autoclose.
 294       options.autorun = True
 295       options.closeWhenDone = True
 296       options.logFile = logfilepath
 297       self.automation.log.info("INFO | runtests.py | Determining guest "
 298                                "arguments.")
 299       runtestsArgs = self.prepareGuestArguments(parser, options)
 300       runtestsPath = self.convertHostPathsToGuestPaths(self.SCRIPT_DIRECTORY)
 301       runtestsPath = os.path.join(runtestsPath, "runtests.py")
 302       runtestsCommand = self.vmrunargs + ("runProgramInGuest", self.vmx,
 303                         "-activeWindow", "-interactive", "-noWait",
 304                         "c:\\mozilla-build\\python25\\python.exe",
 305                         runtestsPath) + runtestsArgs
 306       expectedErrors = [ "Unable to connect to host.",
 307                          "Error: The virtual machine is not powered on" ]
 308       self.automation.log.info("INFO | runtests.py | Launching guest test "
 309                                "runner.")
 310       (result, stdout) = self.runVMCommand(runtestsCommand, expectedErrors)
 311       if result:
 312         return (result, False)
 313       self.automation.log.info("INFO | runtests.py | Waiting for guest test "
 314                                "runner to complete.")
 315       mochitestsSucceeded = self.monitorVMExecution(
 316         os.path.basename(options.app), logfilepath)
 317       if mochitestsSucceeded:
 318         self.automation.log.info("INFO | runtests.py | Guest tests passed!")
 319       else:
 320         self.automation.log.info("INFO | runtests.py | Guest tests failed.")
 321       if mochitestsSucceeded and options.vmwareRecording:
 322         newSnapshots = self.getCurentSnapshotList()
 323         if len(newSnapshots) > len(snapshots):
 324           self.automation.log.info("INFO | runtests.py | Removing last "
 325                                    "recording.")
 326           (result, stdout) = self.runVMCommand(self.vmrunargs + \
 327                                                ("deleteSnapshot", self.vmx,
 328                                                 newSnapshots[-1]))
 329       self.automation.log.info("INFO | runtests.py | Removing guest log file.")
 330       for i in range(30):
 331         try:
 332           os.remove(logfilepath)
 333           break
 334         except:
 335           sleep(1)
 336           self.automation.log.warning("WARNING | runtests.py | Couldn't remove "
 337                                       "guest log file, trying again.")
 338       return (result, mochitestsSucceeded)
 339
 340     if options.repeatUntilFailure:
 341       succeeded = True
 342       result = 0
 343       count = 1
 344       while result == 0 and succeeded:
 345         self.automation.log.info("INFO | runtests.py | Beginning mochitest run "
 346                                  "(%d)." % count)
 347         count += 1
 348         (result, succeeded) = innerRun()
 349     else:
 350       self.automation.log.info("INFO | runtests.py | Beginning mochitest run.")
 351       (result, succeeded) = innerRun()
 352
 353     if not succeeded and options.vmwareRecording:
 354       newSnapshots = self.getCurentSnapshotList()
 355       if len(newSnapshots) > len(snapshots):
 356         self.automation.log.info("INFO | runtests.py | Failed recording saved "
 357                                  "as '%s'." % newSnapshots[-1])
 358
 359     if result:
 360       return result
 361
 362     if options.shutdownVM:
 363       result = self.shutdownVM()
 364       if result:
 365         return result
 366
 367     return 0
 368
 369 def main():
 370   automation = Automation()
 371   mochitest = VMwareMochitest(automation)
 372
 373   parser = VMwareOptions(automation, mochitest)
 374   options, args = parser.parse_args()
 375   options = parser.verifyOptions(options, mochitest)
 376   if (options == None):
 377     sys.exit(1)
 378
 379   if options.vmx is None:
 380     parser.error("A virtual machine must be specified with " +
 381                  "--with-vmware-vm")
 382
 383   if options.vmrun is None:
 384     options.vmrun = os.path.join("c:\\", "Program Files", "VMware",
 385                                  "VMware VIX", "vmrun.exe")
 386     if not os.path.exists(options.vmrun):
 387       options.vmrun = os.path.join("c:\\", "Program Files (x86)", "VMware",
 388                                    "VMware VIX", "vmrun.exe")
 389       if not os.path.exists(options.vmrun):
 390         parser.error("Could not locate vmrun.exe, use --with-vmrun-executable" +
 391                      " to identify its location")
 392
 393   sys.exit(mochitest.runTests(parser, options))
 394
 395 if __name__ == "__main__":
 396   main()