client/virt/virt_env_process.py

   1 import os, time, commands, re, logging, glob, threading, shutil
   2 from autotest_lib.client.bin import utils
   3 from autotest_lib.client.common_lib import error
   4 import aexpect, virt_utils, kvm_monitor, ppm_utils, virt_test_setup
   5 import virt_vm, kvm_vm
   6 try:
   7     import PIL.Image
   8 except ImportError:
   9     logging.warning('No python imaging library installed. PPM image '
  10                     'conversion to JPEG disabled. In order to enable it, '
  11                     'please install python-imaging or the equivalent for your '
  12                     'distro.')
  13
  14
  15 _screendump_thread = None
  16 _screendump_thread_termination_event = None
  17
  18
  19 def preprocess_image(test, params):
  20     """
  21     Preprocess a single QEMU image according to the instructions in params.
  22
  23     @param test: Autotest test object.
  24     @param params: A dict containing image preprocessing parameters.
  25     @note: Currently this function just creates an image if requested.
  26     """
  27     image_filename = virt_vm.get_image_filename(params, test.bindir)
  28
  29     create_image = False
  30
  31     if params.get("force_create_image") == "yes":
  32         logging.debug("'force_create_image' specified; creating image...")
  33         create_image = True
  34     elif (params.get("create_image") == "yes" and not
  35           os.path.exists(image_filename)):
  36         logging.debug("Creating image...")
  37         create_image = True
  38
  39     if create_image and not virt_vm.create_image(params, test.bindir):
  40         raise error.TestError("Could not create image")
  41
  42
  43 def preprocess_vm(test, params, env, name):
  44     """
  45     Preprocess a single VM object according to the instructions in params.
  46     Start the VM if requested and get a screendump.
  47
  48     @param test: An Autotest test object.
  49     @param params: A dict containing VM preprocessing parameters.
  50     @param env: The environment (a dict-like object).
  51     @param name: The name of the VM object.
  52     """
  53     logging.debug("Preprocessing VM '%s'..." % name)
  54     vm = env.get_vm(name)
  55     if not vm:
  56         logging.debug("VM object does not exist; creating it")
  57         vm_type = params.get('vm_type')
  58         if vm_type == 'kvm':
  59             vm = kvm_vm.VM(name, params, test.bindir, env.get("address_cache"))
  60         env.register_vm(name, vm)
  61
  62     start_vm = False
  63
  64     if params.get("restart_vm") == "yes":
  65         logging.debug("'restart_vm' specified; (re)starting VM...")
  66         start_vm = True
  67     elif params.get("migration_mode"):
  68         logging.debug("Starting VM in incoming migration mode...")
  69         start_vm = True
  70     elif params.get("start_vm") == "yes":
  71         if not vm.is_alive():
  72             logging.debug("VM is not alive; starting it...")
  73             start_vm = True
  74         if vm.needs_restart(name=name, params=params, basedir=test.bindir):
  75             logging.debug("Current VM specs differ from requested one; "
  76                           "restarting it...")
  77             start_vm = True
  78
  79     if start_vm:
  80         # Start the VM (or restart it if it's already up)
  81         vm.create(name, params, test.bindir,
  82                   migration_mode=params.get("migration_mode"))
  83     else:
  84         # Don't start the VM, just update its params
  85         vm.params = params
  86
  87     scrdump_filename = os.path.join(test.debugdir, "pre_%s.ppm" % name)
  88     try:
  89         if vm.monitor:
  90             vm.monitor.screendump(scrdump_filename, debug=False)
  91     except kvm_monitor.MonitorError, e:
  92         logging.warn(e)
  93
  94
  95 def postprocess_image(test, params):
  96     """
  97     Postprocess a single QEMU image according to the instructions in params.
  98
  99     @param test: An Autotest test object.
 100     @param params: A dict containing image postprocessing parameters.
 101     """
 102     if params.get("check_image") == "yes":
 103         virt_vm.check_image(params, test.bindir)
 104     if params.get("remove_image") == "yes":
 105         virt_vm.remove_image(params, test.bindir)
 106
 107
 108 def postprocess_vm(test, params, env, name):
 109     """
 110     Postprocess a single VM object according to the instructions in params.
 111     Kill the VM if requested and get a screendump.
 112
 113     @param test: An Autotest test object.
 114     @param params: A dict containing VM postprocessing parameters.
 115     @param env: The environment (a dict-like object).
 116     @param name: The name of the VM object.
 117     """
 118     logging.debug("Postprocessing VM '%s'..." % name)
 119     vm = env.get_vm(name)
 120     if not vm:
 121         return
 122
 123     scrdump_filename = os.path.join(test.debugdir, "post_%s.ppm" % name)
 124     try:
 125         if vm.monitor:
 126             vm.monitor.screendump(scrdump_filename, debug=False)
 127     except kvm_monitor.MonitorError, e:
 128         logging.warn(e)
 129
 130     if params.get("kill_vm") == "yes":
 131         kill_vm_timeout = float(params.get("kill_vm_timeout", 0))
 132         if kill_vm_timeout:
 133             logging.debug("'kill_vm' specified; waiting for VM to shut down "
 134                           "before killing it...")
 135             virt_utils.wait_for(vm.is_dead, kill_vm_timeout, 0, 1)
 136         else:
 137             logging.debug("'kill_vm' specified; killing VM...")
 138         vm.destroy(gracefully = params.get("kill_vm_gracefully") == "yes")
 139
 140
 141 def process_command(test, params, env, command, command_timeout,
 142                     command_noncritical):
 143     """
 144     Pre- or post- custom commands to be executed before/after a test is run
 145
 146     @param test: An Autotest test object.
 147     @param params: A dict containing all VM and image parameters.
 148     @param env: The environment (a dict-like object).
 149     @param command: Command to be run.
 150     @param command_timeout: Timeout for command execution.
 151     @param command_noncritical: If True test will not fail if command fails.
 152     """
 153     # Export environment vars
 154     for k in params:
 155         os.putenv("KVM_TEST_%s" % k, str(params[k]))
 156     # Execute commands
 157     try:
 158         utils.system("cd %s; %s" % (test.bindir, command))
 159     except error.CmdError, e:
 160         if command_noncritical:
 161             logging.warn(e)
 162         else:
 163             raise
 164
 165 def process(test, params, env, image_func, vm_func):
 166     """
 167     Pre- or post-process VMs and images according to the instructions in params.
 168     Call image_func for each image listed in params and vm_func for each VM.
 169
 170     @param test: An Autotest test object.
 171     @param params: A dict containing all VM and image parameters.
 172     @param env: The environment (a dict-like object).
 173     @param image_func: A function to call for each image.
 174     @param vm_func: A function to call for each VM.
 175     """
 176     # Get list of VMs specified for this test
 177     for vm_name in params.objects("vms"):
 178         vm_params = params.object_params(vm_name)
 179         # Get list of images specified for this VM
 180         for image_name in vm_params.objects("images"):
 181             image_params = vm_params.object_params(image_name)
 182             # Call image_func for each image
 183             image_func(test, image_params)
 184         # Call vm_func for each vm
 185         vm_func(test, vm_params, env, vm_name)
 186
 187
 188 @error.context_aware
 189 def preprocess(test, params, env):
 190     """
 191     Preprocess all VMs and images according to the instructions in params.
 192     Also, collect some host information, such as the KVM version.
 193
 194     @param test: An Autotest test object.
 195     @param params: A dict containing all VM and image parameters.
 196     @param env: The environment (a dict-like object).
 197     """
 198     error.context("preprocessing")
 199
 200     if params.get("bridge") == "private":
 201         brcfg = virt_test_setup.PrivateBridgeConfig(params)
 202         brcfg.setup()
 203
 204     # Start tcpdump if it isn't already running
 205     if "address_cache" not in env:
 206         env["address_cache"] = {}
 207     if "tcpdump" in env and not env["tcpdump"].is_alive():
 208         env["tcpdump"].close()
 209         del env["tcpdump"]
 210     if "tcpdump" not in env and params.get("run_tcpdump", "yes") == "yes":
 211         cmd = "%s -npvi any 'dst port 68'" % virt_utils.find_command("tcpdump")
 212         logging.debug("Starting tcpdump (%s)...", cmd)
 213         env["tcpdump"] = aexpect.Tail(
 214             command=cmd,
 215             output_func=_update_address_cache,
 216             output_params=(env["address_cache"],))
 217         if virt_utils.wait_for(lambda: not env["tcpdump"].is_alive(),
 218                               0.1, 0.1, 1.0):
 219             logging.warn("Could not start tcpdump")
 220             logging.warn("Status: %s" % env["tcpdump"].get_status())
 221             logging.warn("Output:" + virt_utils.format_str_for_message(
 222                 env["tcpdump"].get_output()))
 223
 224     # Destroy and remove VMs that are no longer needed in the environment
 225     requested_vms = params.objects("vms")
 226     for key in env.keys():
 227         vm = env[key]
 228         if not virt_utils.is_vm(vm):
 229             continue
 230         if not vm.name in requested_vms:
 231             logging.debug("VM '%s' found in environment but not required for "
 232                           "test; removing it..." % vm.name)
 233             vm.destroy()
 234             del env[key]
 235
 236     # Get the KVM kernel module version and write it as a keyval
 237     logging.debug("Fetching KVM module version...")
 238     if os.path.exists("/dev/kvm"):
 239         try:
 240             kvm_version = open("/sys/module/kvm/version").read().strip()
 241         except:
 242             kvm_version = os.uname()[2]
 243     else:
 244         kvm_version = "Unknown"
 245         logging.debug("KVM module not loaded")
 246     logging.debug("KVM version: %s" % kvm_version)
 247     test.write_test_keyval({"kvm_version": kvm_version})
 248
 249     # Get the KVM userspace version and write it as a keyval
 250     logging.debug("Fetching KVM userspace version...")
 251     qemu_path = virt_utils.get_path(test.bindir, params.get("qemu_binary",
 252                                                            "qemu"))
 253     version_line = commands.getoutput("%s -help | head -n 1" % qemu_path)
 254     matches = re.findall("[Vv]ersion .*?,", version_line)
 255     if matches:
 256         kvm_userspace_version = " ".join(matches[0].split()[1:]).strip(",")
 257     else:
 258         kvm_userspace_version = "Unknown"
 259         logging.debug("Could not fetch KVM userspace version")
 260     logging.debug("KVM userspace version: %s" % kvm_userspace_version)
 261     test.write_test_keyval({"kvm_userspace_version": kvm_userspace_version})
 262
 263     if params.get("setup_hugepages") == "yes":
 264         h = virt_test_setup.HugePageConfig(params)
 265         h.setup()
 266
 267     # Execute any pre_commands
 268     if params.get("pre_command"):
 269         process_command(test, params, env, params.get("pre_command"),
 270                         int(params.get("pre_command_timeout", "600")),
 271                         params.get("pre_command_noncritical") == "yes")
 272
 273     # Preprocess all VMs and images
 274     process(test, params, env, preprocess_image, preprocess_vm)
 275
 276     # Start the screendump thread
 277     if params.get("take_regular_screendumps") == "yes":
 278         logging.debug("Starting screendump thread")
 279         global _screendump_thread, _screendump_thread_termination_event
 280         _screendump_thread_termination_event = threading.Event()
 281         _screendump_thread = threading.Thread(target=_take_screendumps,
 282                                               args=(test, params, env))
 283         _screendump_thread.start()
 284
 285
 286 @error.context_aware
 287 def postprocess(test, params, env):
 288     """
 289     Postprocess all VMs and images according to the instructions in params.
 290
 291     @param test: An Autotest test object.
 292     @param params: Dict containing all VM and image parameters.
 293     @param env: The environment (a dict-like object).
 294     """
 295     error.context("postprocessing")
 296
 297     # Postprocess all VMs and images
 298     process(test, params, env, postprocess_image, postprocess_vm)
 299
 300     # Terminate the screendump thread
 301     global _screendump_thread, _screendump_thread_termination_event
 302     if _screendump_thread:
 303         logging.debug("Terminating screendump thread...")
 304         _screendump_thread_termination_event.set()
 305         _screendump_thread.join(10)
 306         _screendump_thread = None
 307
 308     # Warn about corrupt PPM files
 309     for f in glob.glob(os.path.join(test.debugdir, "*.ppm")):
 310         if not ppm_utils.image_verify_ppm_file(f):
 311             logging.warn("Found corrupt PPM file: %s", f)
 312
 313     # Should we convert PPM files to PNG format?
 314     if params.get("convert_ppm_files_to_png") == "yes":
 315         logging.debug("'convert_ppm_files_to_png' specified; converting PPM "
 316                       "files to PNG format...")
 317         try:
 318             for f in glob.glob(os.path.join(test.debugdir, "*.ppm")):
 319                 if ppm_utils.image_verify_ppm_file(f):
 320                     new_path = f.replace(".ppm", ".png")
 321                     image = PIL.Image.open(f)
 322                     image.save(new_path, format='PNG')
 323         except NameError:
 324             pass
 325
 326     # Should we keep the PPM files?
 327     if params.get("keep_ppm_files") != "yes":
 328         logging.debug("'keep_ppm_files' not specified; removing all PPM files "
 329                       "from debug dir...")
 330         for f in glob.glob(os.path.join(test.debugdir, '*.ppm')):
 331             os.unlink(f)
 332
 333     # Should we keep the screendump dirs?
 334     if params.get("keep_screendumps") != "yes":
 335         logging.debug("'keep_screendumps' not specified; removing screendump "
 336                       "dirs...")
 337         for d in glob.glob(os.path.join(test.debugdir, "screendumps_*")):
 338             if os.path.isdir(d) and not os.path.islink(d):
 339                 shutil.rmtree(d, ignore_errors=True)
 340
 341     # Kill all unresponsive VMs
 342     if params.get("kill_unresponsive_vms") == "yes":
 343         logging.debug("'kill_unresponsive_vms' specified; killing all VMs "
 344                       "that fail to respond to a remote login request...")
 345         for vm in env.get_all_vms():
 346             if vm.is_alive():
 347                 try:
 348                     session = vm.login()
 349                     session.close()
 350                 except (virt_utils.LoginError, virt_vm.VMError), e:
 351                     logging.warn(e)
 352                     vm.destroy(gracefully=False)
 353
 354     # Kill all aexpect tail threads
 355     aexpect.kill_tail_threads()
 356
 357     # Terminate tcpdump if no VMs are alive
 358     living_vms = [vm for vm in env.get_all_vms() if vm.is_alive()]
 359     if not living_vms and "tcpdump" in env:
 360         env["tcpdump"].close()
 361         del env["tcpdump"]
 362
 363     if params.get("setup_hugepages") == "yes":
 364         h = virt_test_setup.HugePageConfig(params)
 365         h.cleanup()
 366
 367     # Execute any post_commands
 368     if params.get("post_command"):
 369         process_command(test, params, env, params.get("post_command"),
 370                         int(params.get("post_command_timeout", "600")),
 371                         params.get("post_command_noncritical") == "yes")
 372
 373     if params.get("bridge") == "private":
 374         brcfg = virt_test_setup.PrivateBridgeConfig()
 375         brcfg.cleanup()
 376
 377
 378 def postprocess_on_error(test, params, env):
 379     """
 380     Perform postprocessing operations required only if the test failed.
 381
 382     @param test: An Autotest test object.
 383     @param params: A dict containing all VM and image parameters.
 384     @param env: The environment (a dict-like object).
 385     """
 386     params.update(params.object_params("on_error"))
 387
 388
 389 def _update_address_cache(address_cache, line):
 390     if re.search("Your.IP", line, re.IGNORECASE):
 391         matches = re.findall(r"\d*\.\d*\.\d*\.\d*", line)
 392         if matches:
 393             address_cache["last_seen"] = matches[0]
 394     if re.search("Client.Ethernet.Address", line, re.IGNORECASE):
 395         matches = re.findall(r"\w*:\w*:\w*:\w*:\w*:\w*", line)
 396         if matches and address_cache.get("last_seen"):
 397             mac_address = matches[0].lower()
 398             if time.time() - address_cache.get("time_%s" % mac_address, 0) > 5:
 399                 logging.debug("(address cache) Adding cache entry: %s ---> %s",
 400                               mac_address, address_cache.get("last_seen"))
 401             address_cache[mac_address] = address_cache.get("last_seen")
 402             address_cache["time_%s" % mac_address] = time.time()
 403             del address_cache["last_seen"]
 404
 405
 406 def _take_screendumps(test, params, env):
 407     global _screendump_thread_termination_event
 408     temp_dir = test.debugdir
 409     if params.get("screendump_temp_dir"):
 410         temp_dir = virt_utils.get_path(test.bindir,
 411                                       params.get("screendump_temp_dir"))
 412         try:
 413             os.makedirs(temp_dir)
 414         except OSError:
 415             pass
 416     temp_filename = os.path.join(temp_dir, "scrdump-%s.ppm" %
 417                                  virt_utils.generate_random_string(6))
 418     delay = float(params.get("screendump_delay", 5))
 419     quality = int(params.get("screendump_quality", 30))
 420
 421     cache = {}
 422
 423     while True:
 424         for vm in env.get_all_vms():
 425             if not vm.is_alive():
 426                 continue
 427             try:
 428                 vm.monitor.screendump(filename=temp_filename, debug=False)
 429             except kvm_monitor.MonitorError, e:
 430                 logging.warn(e)
 431                 continue
 432             if not os.path.exists(temp_filename):
 433                 logging.warn("VM '%s' failed to produce a screendump", vm.name)
 434                 continue
 435             if not ppm_utils.image_verify_ppm_file(temp_filename):
 436                 logging.warn("VM '%s' produced an invalid screendump", vm.name)
 437                 os.unlink(temp_filename)
 438                 continue
 439             screendump_dir = os.path.join(test.debugdir,
 440                                           "screendumps_%s" % vm.name)
 441             try:
 442                 os.makedirs(screendump_dir)
 443             except OSError:
 444                 pass
 445             screendump_filename = os.path.join(screendump_dir,
 446                     "%s_%s.jpg" % (vm.name,
 447                                    time.strftime("%Y-%m-%d_%H-%M-%S")))
 448             hash = utils.hash_file(temp_filename)
 449             if hash in cache:
 450                 try:
 451                     os.link(cache[hash], screendump_filename)
 452                 except OSError:
 453                     pass
 454             else:
 455                 try:
 456                     image = PIL.Image.open(temp_filename)
 457                     image.save(screendump_filename, format="JPEG", quality=quality)
 458                     cache[hash] = screendump_filename
 459                 except NameError:
 460                     pass
 461             os.unlink(temp_filename)
 462         if _screendump_thread_termination_event.isSet():
 463             _screendump_thread_termination_event = None
 464             break
 465         _screendump_thread_termination_event.wait(delay)