From b2313be38868d287d318ef269924fc2388376e07 Mon Sep 17 00:00:00 2001 From: lmr Date: Wed, 27 Apr 2011 20:39:18 +0000 Subject: [PATCH] KVM test: Removing the old libraries and programs As they've been replaced by the new virt_namespace, as well as the new program versions moved to tools. Signed-off-by: Lucas Meneghel Rodrigues git-svn-id: svn://test.kernel.org/autotest/trunk@5332 592f7852-d20e-0410-864c-8624ca9c26a4 --- client/tests/kvm/cd_hash.py | 48 - client/tests/kvm/html_report.py | 1727 ------------------------ client/tests/kvm/installer.py | 838 ------------ client/tests/kvm/kvm_config.py | 698 ---------- client/tests/kvm/kvm_monitor.py | 763 ----------- client/tests/kvm/kvm_preprocessing.py | 454 ------- client/tests/kvm/kvm_scheduler.py | 229 ---- client/tests/kvm/kvm_subprocess.py | 1351 ------------------- client/tests/kvm/kvm_test_utils.py | 753 ----------- client/tests/kvm/kvm_utils.py | 2317 --------------------------------- client/tests/kvm/kvm_vm.py | 1860 -------------------------- client/tests/kvm/ppm_utils.py | 237 ---- client/tests/kvm/rss_file_transfer.py | 519 -------- client/tests/kvm/scan_results.py | 97 -- client/tests/kvm/stepeditor.py | 1401 -------------------- client/tests/kvm/test_setup.py | 107 -- 16 files changed, 13399 deletions(-) delete mode 100755 client/tests/kvm/cd_hash.py delete mode 100755 client/tests/kvm/html_report.py delete mode 100644 client/tests/kvm/installer.py delete mode 100755 client/tests/kvm/kvm_config.py delete mode 100644 client/tests/kvm/kvm_monitor.py delete mode 100644 client/tests/kvm/kvm_preprocessing.py delete mode 100644 client/tests/kvm/kvm_scheduler.py delete mode 100755 client/tests/kvm/kvm_subprocess.py delete mode 100644 client/tests/kvm/kvm_test_utils.py delete mode 100644 client/tests/kvm/kvm_utils.py delete mode 100755 client/tests/kvm/kvm_vm.py delete mode 100644 client/tests/kvm/ppm_utils.py delete mode 100755 client/tests/kvm/rss_file_transfer.py delete mode 100755 client/tests/kvm/scan_results.py delete mode 100755 client/tests/kvm/stepeditor.py delete mode 100644 client/tests/kvm/test_setup.py diff --git a/client/tests/kvm/cd_hash.py b/client/tests/kvm/cd_hash.py deleted file mode 100755 index 04f8cbee..00000000 --- a/client/tests/kvm/cd_hash.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/python -""" -Program that calculates several hashes for a given CD image. - -@copyright: Red Hat 2008-2009 -""" - -import os, sys, optparse, logging -import common -import kvm_utils -from autotest_lib.client.common_lib import logging_manager -from autotest_lib.client.bin import utils - - -if __name__ == "__main__": - parser = optparse.OptionParser("usage: %prog [options] [filenames]") - options, args = parser.parse_args() - - logging_manager.configure_logging(kvm_utils.KvmLoggingConfig()) - - if args: - filenames = args - else: - parser.print_help() - sys.exit(1) - - for filename in filenames: - filename = os.path.abspath(filename) - - file_exists = os.path.isfile(filename) - can_read_file = os.access(filename, os.R_OK) - if not file_exists: - logging.critical("File %s does not exist!", filename) - continue - if not can_read_file: - logging.critical("File %s does not have read permissions!", - filename) - continue - - logging.info("Hash values for file %s", os.path.basename(filename)) - logging.info("md5 (1m): %s", utils.hash_file(filename, 1024*1024, - method="md5")) - logging.info("sha1 (1m): %s", utils.hash_file(filename, 1024*1024, - method="sha1")) - logging.info("md5 (full): %s", utils.hash_file(filename, method="md5")) - logging.info("sha1 (full): %s", utils.hash_file(filename, - method="sha1")) - logging.info("") diff --git a/client/tests/kvm/html_report.py b/client/tests/kvm/html_report.py deleted file mode 100755 index 8b4b109d..00000000 --- a/client/tests/kvm/html_report.py +++ /dev/null @@ -1,1727 +0,0 @@ -#!/usr/bin/python -""" -Script used to parse the test results and generate an HTML report. - -@copyright: (c)2005-2007 Matt Kruse (javascripttoolbox.com) -@copyright: Red Hat 2008-2009 -@author: Dror Russo (drusso@redhat.com) -""" - -import os, sys, re, getopt, time, datetime, commands -import common - - -format_css = """ -html,body { - padding:0; - color:#222; - background:#FFFFFF; -} - -body { - padding:0px; - font:76%/150% "Lucida Grande", "Lucida Sans Unicode", Lucida, Verdana, Geneva, Arial, Helvetica, sans-serif; -} - -#page_title{ - text-decoration:none; - font:bold 2em/2em Arial, Helvetica, sans-serif; - text-transform:none; - text-shadow: 2px 2px 2px #555; - text-align: left; - color:#555555; - border-bottom: 1px solid #555555; -} - -#page_sub_title{ - text-decoration:none; - font:bold 16px Arial, Helvetica, sans-serif; - text-transform:uppercase; - text-shadow: 2px 2px 2px #555; - text-align: left; - color:#555555; - margin-bottom:0; -} - -#comment{ - text-decoration:none; - font:bold 10px Arial, Helvetica, sans-serif; - text-transform:none; - text-align: left; - color:#999999; - margin-top:0; -} - - -#meta_headline{ - text-decoration:none; - font-family: Verdana, Geneva, Arial, Helvetica, sans-serif ; - text-align: left; - color:black; - font-weight: bold; - font-size: 14px; - } - - -table.meta_table -{text-align: center; -font-family: Verdana, Geneva, Arial, Helvetica, sans-serif ; -width: 90%; -background-color: #FFFFFF; -border: 0px; -border-top: 1px #003377 solid; -border-bottom: 1px #003377 solid; -border-right: 1px #003377 solid; -border-left: 1px #003377 solid; -border-collapse: collapse; -border-spacing: 0px;} - -table.meta_table td -{background-color: #FFFFFF; -color: #000; -padding: 4px; -border-top: 1px #BBBBBB solid; -border-bottom: 1px #BBBBBB solid; -font-weight: normal; -font-size: 13px;} - - -table.stats -{text-align: center; -font-family: Verdana, Geneva, Arial, Helvetica, sans-serif ; -width: 100%; -background-color: #FFFFFF; -border: 0px; -border-top: 1px #003377 solid; -border-bottom: 1px #003377 solid; -border-right: 1px #003377 solid; -border-left: 1px #003377 solid; -border-collapse: collapse; -border-spacing: 0px;} - -table.stats td{ -background-color: #FFFFFF; -color: #000; -padding: 4px; -border-top: 1px #BBBBBB solid; -border-bottom: 1px #BBBBBB solid; -font-weight: normal; -font-size: 11px;} - -table.stats th{ -background: #dcdcdc; -color: #000; -padding: 6px; -font-size: 12px; -border-bottom: 1px #003377 solid; -font-weight: bold;} - -table.stats td.top{ -background-color: #dcdcdc; -color: #000; -padding: 6px; -text-align: center; -border: 0px; -border-bottom: 1px #003377 solid; -font-size: 10px; -font-weight: bold;} - -table.stats th.table-sorted-asc{ - background-image: url(ascending.gif); - background-position: top left ; - background-repeat: no-repeat; -} - -table.stats th.table-sorted-desc{ - background-image: url(descending.gif); - background-position: top left; - background-repeat: no-repeat; -} - -table.stats2 -{text-align: left; -font-family: Verdana, Geneva, Arial, Helvetica, sans-serif ; -width: 100%; -background-color: #FFFFFF; -border: 0px; -} - -table.stats2 td{ -background-color: #FFFFFF; -color: #000; -padding: 0px; -font-weight: bold; -font-size: 13px;} - - - -/* Put this inside a @media qualifier so Netscape 4 ignores it */ -@media screen, print { - /* Turn off list bullets */ - ul.mktree li { list-style: none; } - /* Control how "spaced out" the tree is */ - ul.mktree, ul.mktree ul , ul.mktree li { margin-left:10px; padding:0px; } - /* Provide space for our own "bullet" inside the LI */ - ul.mktree li .bullet { padding-left: 15px; } - /* Show "bullets" in the links, depending on the class of the LI that the link's in */ - ul.mktree li.liOpen .bullet { cursor: pointer; } - ul.mktree li.liClosed .bullet { cursor: pointer; } - ul.mktree li.liBullet .bullet { cursor: default; } - /* Sublists are visible or not based on class of parent LI */ - ul.mktree li.liOpen ul { display: block; } - ul.mktree li.liClosed ul { display: none; } - - /* Format menu items differently depending on what level of the tree they are in */ - /* Uncomment this if you want your fonts to decrease in size the deeper they are in the tree */ -/* - ul.mktree li ul li { font-size: 90% } -*/ -} -""" - - -table_js = """ -/** - * Copyright (c)2005-2007 Matt Kruse (javascripttoolbox.com) - * - * Dual licensed under the MIT and GPL licenses. - * This basically means you can use this code however you want for - * free, but don't claim to have written it yourself! - * Donations always accepted: http://www.JavascriptToolbox.com/donate/ - * - * Please do not link to the .js files on javascripttoolbox.com from - * your site. Copy the files locally to your server instead. - * - */ -/** - * Table.js - * Functions for interactive Tables - * - * Copyright (c) 2007 Matt Kruse (javascripttoolbox.com) - * Dual licensed under the MIT and GPL licenses. - * - * @version 0.981 - * - * @history 0.981 2007-03-19 Added Sort.numeric_comma, additional date parsing formats - * @history 0.980 2007-03-18 Release new BETA release pending some testing. Todo: Additional docs, examples, plus jQuery plugin. - * @history 0.959 2007-03-05 Added more "auto" functionality, couple bug fixes - * @history 0.958 2007-02-28 Added auto functionality based on class names - * @history 0.957 2007-02-21 Speed increases, more code cleanup, added Auto Sort functionality - * @history 0.956 2007-02-16 Cleaned up the code and added Auto Filter functionality. - * @history 0.950 2006-11-15 First BETA release. - * - * @todo Add more date format parsers - * @todo Add style classes to colgroup tags after sorting/filtering in case the user wants to highlight the whole column - * @todo Correct for colspans in data rows (this may slow it down) - * @todo Fix for IE losing form control values after sort? - */ - -/** - * Sort Functions - */ -var Sort = (function(){ - var sort = {}; - // Default alpha-numeric sort - // -------------------------- - sort.alphanumeric = function(a,b) { - return (a==b)?0:(asort.numeric.convert(b)) { - return (-1); - } - if (sort.numeric.convert(a)==sort.numeric.convert(b)) { - return 0; - } - if (sort.numeric.convert(a)0) { - var rows = section.rows; - for (var j=0,L2=rows.length; j0) { - var cells = row.cells; - for (var k=0,L3=cells.length; k1 && cells[cells.length-1].cellIndex>0) { - // Define the new function, overwrite the one we're running now, and then run the new one - (this.getCellIndex = function(td) { - return td.cellIndex; - })(td); - } - // Safari will always go through this slower block every time. Oh well. - for (var i=0,L=cells.length; i=0 && node.options) { - // Sort select elements by the visible text - return node.options[node.selectedIndex].text; - } - return ""; - }, - 'IMG':function(node) { - return node.name || ""; - } - }; - - /** - * Get the text value of a cell. Only use innerText if explicitly told to, because - * otherwise we want to be able to handle sorting on inputs and other types - */ - table.getCellValue = function(td,useInnerText) { - if (useInnerText && def(td.innerText)) { - return td.innerText; - } - if (!td.childNodes) { - return ""; - } - var childNodes=td.childNodes; - var ret = ""; - for (var i=0,L=childNodes.length; i-1) { - filters={ 'filter':filters.options[filters.selectedIndex].value }; - } - // Also allow for a regular input - if (filters.nodeName=="INPUT" && filters.type=="text") { - filters={ 'filter':"/"+filters.value+"/" }; - } - // Force filters to be an array - if (typeof(filters)=="object" && !filters.length) { - filters = [filters]; - } - - // Convert regular expression strings to RegExp objects and function strings to function objects - for (var i=0,L=filters.length; ipageend) { - hideRow = true; - } - } - } - - row.style.display = hideRow?"none":""; - } - } - - if (def(page)) { - // Check to see if filtering has put us past the requested page index. If it has, - // then go back to the last page and show it. - if (pagestart>=unfilteredrowcount) { - pagestart = unfilteredrowcount-(unfilteredrowcount%pagesize); - tdata.page = page = pagestart/pagesize; - for (var i=pagestart,L=unfilteredrows.length; i0) { - if (typeof(args.insert)=="function") { - func.insert(cell,colValues); - } - else { - var sel = ''; - cell.innerHTML += "
"+sel; - } - } - } - }); - if (val = classValue(t,table.FilteredRowcountPrefix)) { - tdata.container_filtered_count = document.getElementById(val); - } - if (val = classValue(t,table.RowcountPrefix)) { - tdata.container_all_count = document.getElementById(val); - } - }; - - /** - * Attach the auto event so it happens on load. - * use jQuery's ready() function if available - */ - if (typeof(jQuery)!="undefined") { - jQuery(table.auto); - } - else if (window.addEventListener) { - window.addEventListener( "load", table.auto, false ); - } - else if (window.attachEvent) { - window.attachEvent( "onload", table.auto ); - } - - return table; -})(); -""" - - -maketree_js = """/** - * Copyright (c)2005-2007 Matt Kruse (javascripttoolbox.com) - * - * Dual licensed under the MIT and GPL licenses. - * This basically means you can use this code however you want for - * free, but don't claim to have written it yourself! - * Donations always accepted: http://www.JavascriptToolbox.com/donate/ - * - * Please do not link to the .js files on javascripttoolbox.com from - * your site. Copy the files locally to your server instead. - * - */ -/* -This code is inspired by and extended from Stuart Langridge's aqlist code: - http://www.kryogenix.org/code/browser/aqlists/ - Stuart Langridge, November 2002 - sil@kryogenix.org - Inspired by Aaron's labels.js (http://youngpup.net/demos/labels/) - and Dave Lindquist's menuDropDown.js (http://www.gazingus.org/dhtml/?id=109) -*/ - -// Automatically attach a listener to the window onload, to convert the trees -addEvent(window,"load",convertTrees); - -// Utility function to add an event listener -function addEvent(o,e,f){ - if (o.addEventListener){ o.addEventListener(e,f,false); return true; } - else if (o.attachEvent){ return o.attachEvent("on"+e,f); } - else { return false; } -} - -// utility function to set a global variable if it is not already set -function setDefault(name,val) { - if (typeof(window[name])=="undefined" || window[name]==null) { - window[name]=val; - } -} - -// Full expands a tree with a given ID -function expandTree(treeId) { - var ul = document.getElementById(treeId); - if (ul == null) { return false; } - expandCollapseList(ul,nodeOpenClass); -} - -// Fully collapses a tree with a given ID -function collapseTree(treeId) { - var ul = document.getElementById(treeId); - if (ul == null) { return false; } - expandCollapseList(ul,nodeClosedClass); -} - -// Expands enough nodes to expose an LI with a given ID -function expandToItem(treeId,itemId) { - var ul = document.getElementById(treeId); - if (ul == null) { return false; } - var ret = expandCollapseList(ul,nodeOpenClass,itemId); - if (ret) { - var o = document.getElementById(itemId); - if (o.scrollIntoView) { - o.scrollIntoView(false); - } - } -} - -// Performs 3 functions: -// a) Expand all nodes -// b) Collapse all nodes -// c) Expand all nodes to reach a certain ID -function expandCollapseList(ul,cName,itemId) { - if (!ul.childNodes || ul.childNodes.length==0) { return false; } - // Iterate LIs - for (var itemi=0;itemi - - -KVM Autotest Results - - - - -""" % (format_css, table_js, maketree_js) - - - if output_file_name: - output = open(output_file_name, "w") - else: #if no output file defined, print html file to console - output = sys.stdout - # create html page - print >> output, html_prefix - print >> output, '

KVM Autotest Execution Report

' - - # formating date and time to print - t = datetime.datetime.now() - - epoch_sec = time.mktime(t.timetuple()) - now = datetime.datetime.fromtimestamp(epoch_sec) - - # basic statistics - total_executed = 0 - total_failed = 0 - total_passed = 0 - for res in results: - total_executed += 1 - if res['status'] == 'GOOD': - total_passed += 1 - else: - total_failed += 1 - stat_str = 'No test cases executed' - if total_executed > 0: - failed_perct = int(float(total_failed)/float(total_executed)*100) - stat_str = ('From %d tests executed, %d have passed (%d%% failures)' % - (total_executed, total_passed, failed_perct)) - - kvm_ver_str = metadata['kvmver'] - - print >> output, '' - print >> output, '' % host - print >> output, '' % tag - print >> output, '' % now.ctime() - print >> output, ''% stat_str - print >> output, '' - print >> output, '' % kvm_ver_str - print >> output, '
HOST:%s
RESULTS DIR:%s
DATE:%s
STATS:%s
KVM VERSION:%s
' - - - ## print test results - print >> output, '
' - print >> output, '

Test Results

' - print >> output, '

click on table headers to asc/desc sort

' - result_table_prefix = """ - - - - - - - - - - -""" - print >> output, result_table_prefix - for res in results: - print >> output, '' - print >> output, '' % res['time'] - print >> output, '' % res['testcase'] - if res['status'] == 'GOOD': - print >> output, '' - elif res['status'] == 'FAIL': - print >> output, '' - elif res['status'] == 'ERROR': - print >> output, '' - else: - print >> output, '' % res['status'] - # print exec time (seconds) - print >> output, '' % res['exec_time_sec'] - # print log only if test failed.. - if res['log']: - #chop all '\n' from log text (to prevent html errors) - rx1 = re.compile('(\s+)') - log_text = rx1.sub(' ', res['log']) - - # allow only a-zA-Z0-9_ in html title name - # (due to bug in MS-explorer) - rx2 = re.compile('([^a-zA-Z_0-9])') - updated_tag = rx2.sub('_', res['title']) - - html_body_text = '%s%s' % (str(updated_tag), log_text) - print >> output, '' % (str(updated_tag), str(html_body_text)) - else: - print >> output, '' - # print execution time - print >> output, '' % os.path.join(dirname, res['title'], "debug") - - print >> output, '' - print >> output, "
Date/TimeTest Case
StatusTime (sec)InfoDebug
%s%sPASSFAILERROR!%s%sInfoDebug
" - - - print >> output, '

Host Info

' - print >> output, '

click on each item to expend/collapse

' - ## Meta list comes here.. - print >> output, '

' - print >> output, 'Expand All' - print >> output, '   ' - print >> output, 'Collapse All' - print >> output, '

' - - print >> output, '
    ' - counter = 0 - keys = metadata.keys() - keys.sort() - for key in keys: - val = metadata[key] - print >> output, '
  • %s' % key - print >> output, '
      %s
    • ' % val - print >> output, '
' - - print >> output, "" - if output_file_name: - output.close() - - -def parse_result(dirname, line): - parts = line.split() - if len(parts) < 4: - return None - global stimelist - if parts[0] == 'START': - pair = parts[3].split('=') - stime = int(pair[1]) - stimelist.append(stime) - - elif (parts[0] == 'END'): - result = {} - exec_time = '' - # fetch time stamp - if len(parts) > 7: - temp = parts[5].split('=') - exec_time = temp[1] + ' ' + parts[6] + ' ' + parts[7] - # assign default values - result['time'] = exec_time - result['testcase'] = 'na' - result['status'] = 'na' - result['log'] = None - result['exec_time_sec'] = 'na' - tag = parts[3] - - # assign actual values - rx = re.compile('^(\w+)\.(.*)$') - m1 = rx.findall(parts[3]) - result['testcase'] = m1[0][1] - result['title'] = str(tag) - result['status'] = parts[1] - if result['status'] != 'GOOD': - result['log'] = get_exec_log(dirname, tag) - if len(stimelist)>0: - pair = parts[4].split('=') - etime = int(pair[1]) - stime = stimelist.pop() - total_exec_time_sec = etime - stime - result['exec_time_sec'] = total_exec_time_sec - return result - return None - - -def get_exec_log(resdir, tag): - stdout_file = os.path.join(resdir, tag) + '/debug/stdout' - stderr_file = os.path.join(resdir, tag) + '/debug/stderr' - status_file = os.path.join(resdir, tag) + '/status' - dmesg_file = os.path.join(resdir, tag) + '/sysinfo/dmesg' - log = '' - log += '
STDERR:
' - log += get_info_file(stderr_file) - log += '
STDOUT:
' - log += get_info_file(stdout_file) - log += '
STATUS:
' - log += get_info_file(status_file) - log += '
DMESG:
' - log += get_info_file(dmesg_file) - return log - - -def get_info_file(filename): - data = '' - errors = re.compile(r"\b(error|fail|failed)\b", re.IGNORECASE) - if os.path.isfile(filename): - f = open('%s' % filename, "r") - lines = f.readlines() - f.close() - rx = re.compile('(\'|\")') - for line in lines: - new_line = rx.sub('', line) - errors_found = errors.findall(new_line) - if len(errors_found) > 0: - data += '%s
' % str(new_line) - else: - data += '%s
' % str(new_line) - if not data: - data = 'No Information Found.
' - else: - data = 'File not found.
' - return data - - - -def usage(): - print 'usage:', - print 'make_html_report.py -r [-f output_file] [-R]' - print '(e.g. make_html_reporter.py -r '\ - '/usr/local/autotest/client/results/default -f /tmp/myreport.html)' - print 'add "-R" for an html report with relative-paths (relative '\ - 'to results directory)' - print '' - sys.exit(1) - - -def get_keyval_value(result_dir, key): - """ - Return the value of the first appearance of key in any keyval file in - result_dir. If no appropriate line is found, return 'Unknown'. - """ - keyval_pattern = os.path.join(result_dir, "kvm.*", "keyval") - keyval_lines = commands.getoutput(r"grep -h '\b%s\b.*=' %s" - % (key, keyval_pattern)) - if not keyval_lines: - return "Unknown" - keyval_line = keyval_lines.splitlines()[0] - if key in keyval_line and "=" in keyval_line: - return keyval_line.split("=")[1].strip() - else: - return "Unknown" - - -def get_kvm_version(result_dir): - """ - Return an HTML string describing the KVM version. - - @param result_dir: An Autotest job result dir - """ - kvm_version = get_keyval_value(result_dir, "kvm_version") - kvm_userspace_version = get_keyval_value(result_dir, - "kvm_userspace_version") - return "Kernel: %s
Userspace: %s" % (kvm_version, kvm_userspace_version) - - -def main(argv): - dirname = None - output_file_name = None - relative_path = False - try: - opts, args = getopt.getopt(argv, "r:f:h:R", ['help']) - except getopt.GetoptError: - usage() - sys.exit(2) - for opt, arg in opts: - if opt in ("-h", "--help"): - usage() - sys.exit() - elif opt == '-r': - dirname = arg - elif opt == '-f': - output_file_name = arg - elif opt == '-R': - relative_path = True - else: - usage() - sys.exit(1) - - html_path = dirname - # don't use absolute path in html output if relative flag passed - if relative_path: - html_path = '' - - if dirname: - if os.path.isdir(dirname): # TBD: replace it with a validation of - # autotest result dir - res_dir = os.path.abspath(dirname) - tag = res_dir - status_file_name = dirname + '/status' - sysinfo_dir = dirname + '/sysinfo' - host = get_info_file('%s/hostname' % sysinfo_dir) - rx = re.compile('^\s+[END|START].*$') - # create the results set dict - results_data = [] - if os.path.exists(status_file_name): - f = open(status_file_name, "r") - lines = f.readlines() - f.close() - for line in lines: - if rx.match(line): - result_dict = parse_result(dirname, line) - if result_dict: - results_data.append(result_dict) - # create the meta info dict - metalist = { - 'uname': get_info_file('%s/uname' % sysinfo_dir), - 'cpuinfo':get_info_file('%s/cpuinfo' % sysinfo_dir), - 'meminfo':get_info_file('%s/meminfo' % sysinfo_dir), - 'df':get_info_file('%s/df' % sysinfo_dir), - 'modules':get_info_file('%s/modules' % sysinfo_dir), - 'gcc':get_info_file('%s/gcc_--version' % sysinfo_dir), - 'dmidecode':get_info_file('%s/dmidecode' % sysinfo_dir), - 'dmesg':get_info_file('%s/dmesg' % sysinfo_dir), - 'kvmver':get_kvm_version(dirname) - } - - make_html_file(metalist, results_data, tag, host, output_file_name, - html_path) - sys.exit(0) - else: - print 'Invalid result directory <%s>' % dirname - sys.exit(1) - else: - usage() - sys.exit(1) - - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/client/tests/kvm/installer.py b/client/tests/kvm/installer.py deleted file mode 100644 index 8026d6dd..00000000 --- a/client/tests/kvm/installer.py +++ /dev/null @@ -1,838 +0,0 @@ -import os, logging, datetime, glob -import shutil -from autotest_lib.client.bin import utils, os_dep -from autotest_lib.client.common_lib import error -import kvm_utils - - -def check_configure_options(script_path): - """ - Return the list of available options (flags) of a given kvm configure build - script. - - @param script: Path to the configure script - """ - abspath = os.path.abspath(script_path) - help_raw = utils.system_output('%s --help' % abspath, ignore_status=True) - help_output = help_raw.split("\n") - option_list = [] - for line in help_output: - cleaned_line = line.lstrip() - if cleaned_line.startswith("--"): - option = cleaned_line.split()[0] - option = option.split("=")[0] - option_list.append(option) - - return option_list - - -def kill_qemu_processes(): - """ - Kills all qemu processes, also kills all processes holding /dev/kvm down. - """ - logging.debug("Killing any qemu processes that might be left behind") - utils.system("pkill qemu", ignore_status=True) - # Let's double check to see if some other process is holding /dev/kvm - if os.path.isfile("/dev/kvm"): - utils.system("fuser -k /dev/kvm", ignore_status=True) - - -def cpu_vendor(): - vendor = "intel" - if os.system("grep vmx /proc/cpuinfo 1>/dev/null") != 0: - vendor = "amd" - logging.debug("Detected CPU vendor as '%s'", vendor) - return vendor - - -def _unload_kvm_modules(mod_list): - logging.info("Unloading previously loaded KVM modules") - for module in reversed(mod_list): - utils.unload_module(module) - - -def _load_kvm_modules(mod_list, module_dir=None, load_stock=False): - """ - Just load the KVM modules, without killing Qemu or unloading previous - modules. - - Load modules present on any sub directory of module_dir. Function will walk - through module_dir until it finds the modules. - - @param module_dir: Directory where the KVM modules are located. - @param load_stock: Whether we are going to load system kernel modules. - @param extra_modules: List of extra modules to load. - """ - if module_dir: - logging.info("Loading the built KVM modules...") - kvm_module_path = None - kvm_vendor_module_path = None - abort = False - - list_modules = ['%s.ko' % (m) for m in mod_list] - - list_module_paths = [] - for folder, subdirs, files in os.walk(module_dir): - for module in list_modules: - if module in files: - module_path = os.path.join(folder, module) - list_module_paths.append(module_path) - - # We might need to arrange the modules in the correct order - # to avoid module load problems - list_modules_load = [] - for module in list_modules: - for module_path in list_module_paths: - if os.path.basename(module_path) == module: - list_modules_load.append(module_path) - - if len(list_module_paths) != len(list_modules): - logging.error("KVM modules not found. If you don't want to use the " - "modules built by this test, make sure the option " - "load_modules: 'no' is marked on the test control " - "file.") - raise error.TestError("The modules %s were requested to be loaded, " - "but the only modules found were %s" % - (list_modules, list_module_paths)) - - for module_path in list_modules_load: - try: - utils.system("insmod %s" % module_path) - except Exception, e: - raise error.TestFail("Failed to load KVM modules: %s" % e) - - if load_stock: - logging.info("Loading current system KVM modules...") - for module in mod_list: - utils.system("modprobe %s" % module) - - -def create_symlinks(test_bindir, prefix=None, bin_list=None, unittest=None): - """ - Create symbolic links for the appropriate qemu and qemu-img commands on - the kvm test bindir. - - @param test_bindir: KVM test bindir - @param prefix: KVM prefix path - @param bin_list: List of qemu binaries to link - @param unittest: Path to configuration file unittests.cfg - """ - qemu_path = os.path.join(test_bindir, "qemu") - qemu_img_path = os.path.join(test_bindir, "qemu-img") - qemu_unittest_path = os.path.join(test_bindir, "unittests") - if os.path.lexists(qemu_path): - os.unlink(qemu_path) - if os.path.lexists(qemu_img_path): - os.unlink(qemu_img_path) - if unittest and os.path.lexists(qemu_unittest_path): - os.unlink(qemu_unittest_path) - - logging.debug("Linking qemu binaries") - - if bin_list: - for bin in bin_list: - if os.path.basename(bin) == 'qemu-kvm': - os.symlink(bin, qemu_path) - elif os.path.basename(bin) == 'qemu-img': - os.symlink(bin, qemu_img_path) - - elif prefix: - kvm_qemu = os.path.join(prefix, "bin", "qemu-system-x86_64") - if not os.path.isfile(kvm_qemu): - raise error.TestError('Invalid qemu path') - kvm_qemu_img = os.path.join(prefix, "bin", "qemu-img") - if not os.path.isfile(kvm_qemu_img): - raise error.TestError('Invalid qemu-img path') - os.symlink(kvm_qemu, qemu_path) - os.symlink(kvm_qemu_img, qemu_img_path) - - if unittest: - logging.debug("Linking unittest dir") - os.symlink(unittest, qemu_unittest_path) - - -def install_roms(rom_dir, prefix): - logging.debug("Path to roms specified. Copying roms to install prefix") - rom_dst_dir = os.path.join(prefix, 'share', 'qemu') - for rom_src in glob.glob('%s/*.bin' % rom_dir): - rom_dst = os.path.join(rom_dst_dir, os.path.basename(rom_src)) - logging.debug("Copying rom file %s to %s", rom_src, rom_dst) - shutil.copy(rom_src, rom_dst) - - -def save_build(build_dir, dest_dir): - logging.debug('Saving the result of the build on %s', dest_dir) - base_name = os.path.basename(build_dir) - tarball_name = base_name + '.tar.bz2' - os.chdir(os.path.dirname(build_dir)) - utils.system('tar -cjf %s %s' % (tarball_name, base_name)) - shutil.move(tarball_name, os.path.join(dest_dir, tarball_name)) - - -class KvmInstallException(Exception): - pass - - -class FailedKvmInstall(KvmInstallException): - pass - - -class KvmNotInstalled(KvmInstallException): - pass - - -class BaseInstaller(object): - # default value for load_stock argument - load_stock_modules = True - def __init__(self, mode=None): - self.install_mode = mode - self._full_module_list = None - - def set_install_params(self, test, params): - self.params = params - - load_modules = params.get('load_modules', 'no') - if not load_modules or load_modules == 'yes': - self.should_load_modules = True - elif load_modules == 'no': - self.should_load_modules = False - default_extra_modules = str(None) - self.extra_modules = eval(params.get("extra_modules", - default_extra_modules)) - - self.cpu_vendor = cpu_vendor() - - self.srcdir = test.srcdir - if not os.path.isdir(self.srcdir): - os.makedirs(self.srcdir) - - self.test_bindir = test.bindir - self.results_dir = test.resultsdir - - # KVM build prefix, for the modes that do need it - prefix = os.path.join(test.bindir, 'build') - self.prefix = os.path.abspath(prefix) - - # Current host kernel directory - default_host_kernel_source = '/lib/modules/%s/build' % os.uname()[2] - self.host_kernel_srcdir = params.get('host_kernel_source', - default_host_kernel_source) - - # Extra parameters that can be passed to the configure script - self.extra_configure_options = params.get('extra_configure_options', - None) - - # Do we want to save the result of the build on test.resultsdir? - self.save_results = True - save_results = params.get('save_results', 'no') - if save_results == 'no': - self.save_results = False - - self._full_module_list = list(self._module_list()) - - - def install_unittests(self): - userspace_srcdir = os.path.join(self.srcdir, "kvm_userspace") - test_repo = self.params.get("test_git_repo") - test_branch = self.params.get("test_branch", "master") - test_commit = self.params.get("test_commit", None) - test_lbranch = self.params.get("test_lbranch", "master") - - if test_repo: - test_srcdir = os.path.join(self.srcdir, "kvm-unit-tests") - kvm_utils.get_git_branch(test_repo, test_branch, test_srcdir, - test_commit, test_lbranch) - unittest_cfg = os.path.join(test_srcdir, 'x86', - 'unittests.cfg') - self.test_srcdir = test_srcdir - else: - unittest_cfg = os.path.join(userspace_srcdir, 'kvm', 'test', 'x86', - 'unittests.cfg') - self.unittest_cfg = None - if os.path.isfile(unittest_cfg): - self.unittest_cfg = unittest_cfg - else: - if test_repo: - logging.error("No unittest config file %s found, skipping " - "unittest build", self.unittest_cfg) - - self.unittest_prefix = None - if self.unittest_cfg: - logging.info("Building and installing unittests") - os.chdir(os.path.dirname(os.path.dirname(self.unittest_cfg))) - utils.system('./configure --prefix=%s' % self.prefix) - utils.system('make') - utils.system('make install') - self.unittest_prefix = os.path.join(self.prefix, 'share', 'qemu', - 'tests') - - - def full_module_list(self): - """Return the module list used by the installer - - Used by the module_probe test, to avoid using utils.unload_module(). - """ - if self._full_module_list is None: - raise KvmNotInstalled("KVM modules not installed yet (installer: %s)" % (type(self))) - return self._full_module_list - - - def _module_list(self): - """Generate the list of modules that need to be loaded - """ - yield 'kvm' - yield 'kvm-%s' % (self.cpu_vendor) - if self.extra_modules: - for module in self.extra_modules: - yield module - - - def _load_modules(self, mod_list): - """ - Load the KVM modules - - May be overridden by subclasses. - """ - _load_kvm_modules(mod_list, load_stock=self.load_stock_modules) - - - def load_modules(self, mod_list=None): - if mod_list is None: - mod_list = self.full_module_list() - self._load_modules(mod_list) - - - def _unload_modules(self, mod_list=None): - """ - Just unload the KVM modules, without trying to kill Qemu - """ - if mod_list is None: - mod_list = self.full_module_list() - _unload_kvm_modules(mod_list) - - - def unload_modules(self, mod_list=None): - """ - Kill Qemu and unload the KVM modules - """ - kill_qemu_processes() - self._unload_modules(mod_list) - - - def reload_modules(self): - """ - Reload the KVM modules after killing Qemu and unloading the current modules - """ - self.unload_modules() - self.load_modules() - - - def reload_modules_if_needed(self): - if self.should_load_modules: - self.reload_modules() - - -class YumInstaller(BaseInstaller): - """ - Class that uses yum to install and remove packages. - """ - load_stock_modules = True - def set_install_params(self, test, params): - super(YumInstaller, self).set_install_params(test, params) - # Checking if all required dependencies are available - os_dep.command("rpm") - os_dep.command("yum") - - default_pkg_list = str(['qemu-kvm', 'qemu-kvm-tools']) - default_qemu_bin_paths = str(['/usr/bin/qemu-kvm', '/usr/bin/qemu-img']) - default_pkg_path_list = str(None) - self.pkg_list = eval(params.get("pkg_list", default_pkg_list)) - self.pkg_path_list = eval(params.get("pkg_path_list", - default_pkg_path_list)) - self.qemu_bin_paths = eval(params.get("qemu_bin_paths", - default_qemu_bin_paths)) - - - def _clean_previous_installs(self): - kill_qemu_processes() - removable_packages = "" - for pkg in self.pkg_list: - removable_packages += " %s" % pkg - - utils.system("yum remove -y %s" % removable_packages) - - - def _get_packages(self): - for pkg in self.pkg_path_list: - utils.get_file(pkg, os.path.join(self.srcdir, - os.path.basename(pkg))) - - - def _install_packages(self): - """ - Install all downloaded packages. - """ - os.chdir(self.srcdir) - utils.system("yum install --nogpgcheck -y *.rpm") - - - def install(self): - self.install_unittests() - self._clean_previous_installs() - self._get_packages() - self._install_packages() - create_symlinks(test_bindir=self.test_bindir, - bin_list=self.qemu_bin_paths, - unittest=self.unittest_prefix) - self.reload_modules_if_needed() - if self.save_results: - save_build(self.srcdir, self.results_dir) - - -class KojiInstaller(YumInstaller): - """ - Class that handles installing KVM from the fedora build service, koji. - - It uses yum to install and remove packages. Packages are specified - according to the syntax defined in the PkgSpec class. - """ - load_stock_modules = True - def set_install_params(self, test, params): - """ - Gets parameters and initializes the package downloader. - - @param test: kvm test object - @param params: Dictionary with test arguments - """ - super(KojiInstaller, self).set_install_params(test, params) - self.tag = params.get("koji_tag", None) - self.koji_cmd = params.get("koji_cmd", None) - if self.tag is not None: - kvm_utils.set_default_koji_tag(self.tag) - self.koji_pkgs = eval(params.get("koji_pkgs", "[]")) - - - def _get_packages(self): - """ - Downloads the specific arch RPMs for the specific build name. - """ - koji_client = kvm_utils.KojiClient(cmd=self.koji_cmd) - for pkg_text in self.koji_pkgs: - pkg = kvm_utils.KojiPkgSpec(pkg_text) - if pkg.is_valid(): - koji_client.get_pkgs(pkg, dst_dir=self.srcdir) - else: - logging.error('Package specification (%s) is invalid: %s', pkg, - pkg.describe_invalid()) - - - def _clean_previous_installs(self): - kill_qemu_processes() - removable_packages = " ".join(self._get_rpm_names()) - utils.system("yum -y remove %s" % removable_packages) - - - def install(self): - self._clean_previous_installs() - self._get_packages() - self._install_packages() - self.install_unittests() - create_symlinks(test_bindir=self.test_bindir, - bin_list=self.qemu_bin_paths, - unittest=self.unittest_prefix) - self.reload_modules_if_needed() - if self.save_results: - save_build(self.srcdir, self.results_dir) - - - def _get_rpm_names(self): - all_rpm_names = [] - koji_client = kvm_utils.KojiClient(cmd=self.koji_cmd) - for pkg_text in self.koji_pkgs: - pkg = kvm_utils.KojiPkgSpec(pkg_text) - rpm_names = koji_client.get_pkg_rpm_names(pkg) - all_rpm_names += rpm_names - return all_rpm_names - - - def _get_rpm_file_names(self): - all_rpm_file_names = [] - koji_client = kvm_utils.KojiClient(cmd=self.koji_cmd) - for pkg_text in self.koji_pkgs: - pkg = kvm_utils.KojiPkgSpec(pkg_text) - rpm_file_names = koji_client.get_pkg_rpm_file_names(pkg) - all_rpm_file_names += rpm_file_names - return all_rpm_file_names - - - def _install_packages(self): - """ - Install all downloaded packages. - """ - os.chdir(self.srcdir) - rpm_file_names = " ".join(self._get_rpm_file_names()) - utils.system("yum --nogpgcheck -y localinstall %s" % rpm_file_names) - - -class SourceDirInstaller(BaseInstaller): - """ - Class that handles building/installing KVM directly from a tarball or - a single source code dir. - """ - def set_install_params(self, test, params): - """ - Initializes class attributes, and retrieves KVM code. - - @param test: kvm test object - @param params: Dictionary with test arguments - """ - super(SourceDirInstaller, self).set_install_params(test, params) - - self.mod_install_dir = os.path.join(self.prefix, 'modules') - self.installed_kmods = False # it will be set to True in case we - # installed our own modules - - srcdir = params.get("srcdir", None) - self.path_to_roms = params.get("path_to_rom_images", None) - - if self.install_mode == 'localsrc': - if srcdir is None: - raise error.TestError("Install from source directory specified" - "but no source directory provided on the" - "control file.") - else: - shutil.copytree(srcdir, self.srcdir) - - if self.install_mode == 'release': - release_tag = params.get("release_tag") - release_dir = params.get("release_dir") - release_listing = params.get("release_listing") - logging.info("Installing KVM from release tarball") - if not release_tag: - release_tag = kvm_utils.get_latest_kvm_release_tag( - release_listing) - tarball = os.path.join(release_dir, 'kvm', release_tag, - "kvm-%s.tar.gz" % release_tag) - logging.info("Retrieving release kvm-%s" % release_tag) - tarball = utils.unmap_url("/", tarball, "/tmp") - - elif self.install_mode == 'snapshot': - logging.info("Installing KVM from snapshot") - snapshot_dir = params.get("snapshot_dir") - if not snapshot_dir: - raise error.TestError("Snapshot dir not provided") - snapshot_date = params.get("snapshot_date") - if not snapshot_date: - # Take yesterday's snapshot - d = (datetime.date.today() - - datetime.timedelta(1)).strftime("%Y%m%d") - else: - d = snapshot_date - tarball = os.path.join(snapshot_dir, "kvm-snapshot-%s.tar.gz" % d) - logging.info("Retrieving kvm-snapshot-%s" % d) - tarball = utils.unmap_url("/", tarball, "/tmp") - - elif self.install_mode == 'localtar': - tarball = params.get("tarball") - if not tarball: - raise error.TestError("KVM Tarball install specified but no" - " tarball provided on control file.") - logging.info("Installing KVM from a local tarball") - logging.info("Using tarball %s") - tarball = utils.unmap_url("/", params.get("tarball"), "/tmp") - - if self.install_mode in ['release', 'snapshot', 'localtar']: - utils.extract_tarball_to_dir(tarball, self.srcdir) - - if self.install_mode in ['release', 'snapshot', 'localtar', 'srcdir']: - self.repo_type = kvm_utils.check_kvm_source_dir(self.srcdir) - configure_script = os.path.join(self.srcdir, 'configure') - self.configure_options = check_configure_options(configure_script) - - - def _build(self): - make_jobs = utils.count_cpus() - os.chdir(self.srcdir) - # For testing purposes, it's better to build qemu binaries with - # debugging symbols, so we can extract more meaningful stack traces. - cfg = "./configure --prefix=%s" % self.prefix - if "--disable-strip" in self.configure_options: - cfg += " --disable-strip" - steps = [cfg, "make clean", "make -j %s" % make_jobs] - logging.info("Building KVM") - for step in steps: - utils.system(step) - - - def _install_kmods_old_userspace(self, userspace_path): - """ - Run the module install command. - - This is for the "old userspace" code, that contained a 'kernel' subdirectory - with the kmod build code. - - The code would be much simpler if we could specify the module install - path as parameter to the toplevel Makefile. As we can't do that and - the module install code doesn't use --prefix, we have to call - 'make -C kernel install' directly, setting the module directory - parameters. - - If the userspace tree doens't have a 'kernel' subdirectory, the - module install step will be skipped. - - @param userspace_path: the path the kvm-userspace directory - """ - kdir = os.path.join(userspace_path, 'kernel') - if os.path.isdir(kdir): - os.chdir(kdir) - # INSTALLDIR is the target dir for the modules - # ORIGMODDIR is the dir where the old modules will be removed. we - # don't want to mess with the system modules, so set it - # to a non-existing directory - utils.system('make install INSTALLDIR=%s ORIGMODDIR=/tmp/no-old-modules' % (self.mod_install_dir)) - self.installed_kmods = True - - - def _install_kmods(self, kmod_path): - """Run the module install command for the kmod-kvm repository - - @param kmod_path: the path to the kmod-kvm.git working copy - """ - os.chdir(kmod_path) - utils.system('make modules_install DESTDIR=%s' % (self.mod_install_dir)) - self.installed_kmods = True - - - def _install(self): - os.chdir(self.srcdir) - logging.info("Installing KVM userspace") - if self.repo_type == 1: - utils.system("make -C qemu install") - self._install_kmods_old_userspace(self.srcdir) - elif self.repo_type == 2: - utils.system("make install") - if self.path_to_roms: - install_roms(self.path_to_roms, self.prefix) - self.install_unittests() - create_symlinks(test_bindir=self.test_bindir, - prefix=self.prefix, - unittest=self.unittest_prefix) - - - def _load_modules(self, mod_list): - # load the installed KVM modules in case we installed them - # ourselves. Otherwise, just load the system modules. - if self.installed_kmods: - logging.info("Loading installed KVM modules") - _load_kvm_modules(mod_list, module_dir=self.mod_install_dir) - else: - logging.info("Loading stock KVM modules") - _load_kvm_modules(mod_list, load_stock=True) - - - def install(self): - self._build() - self._install() - self.reload_modules_if_needed() - if self.save_results: - save_build(self.srcdir, self.results_dir) - - -class GitInstaller(SourceDirInstaller): - def _pull_code(self): - """ - Retrieves code from git repositories. - """ - params = self.params - - kernel_repo = params.get("git_repo") - user_repo = params.get("user_git_repo") - kmod_repo = params.get("kmod_repo") - - kernel_branch = params.get("kernel_branch", "master") - user_branch = params.get("user_branch", "master") - kmod_branch = params.get("kmod_branch", "master") - - kernel_lbranch = params.get("kernel_lbranch", "master") - user_lbranch = params.get("user_lbranch", "master") - kmod_lbranch = params.get("kmod_lbranch", "master") - - kernel_commit = params.get("kernel_commit", None) - user_commit = params.get("user_commit", None) - kmod_commit = params.get("kmod_commit", None) - - kernel_patches = eval(params.get("kernel_patches", "[]")) - user_patches = eval(params.get("user_patches", "[]")) - kmod_patches = eval(params.get("user_patches", "[]")) - - if not user_repo: - message = "KVM user git repository path not specified" - logging.error(message) - raise error.TestError(message) - - userspace_srcdir = os.path.join(self.srcdir, "kvm_userspace") - kvm_utils.get_git_branch(user_repo, user_branch, userspace_srcdir, - user_commit, user_lbranch) - self.userspace_srcdir = userspace_srcdir - - if user_patches: - os.chdir(self.userspace_srcdir) - for patch in user_patches: - utils.get_file(patch, os.path.join(self.userspace_srcdir, - os.path.basename(patch))) - utils.system('patch -p1 < %s' % os.path.basename(patch)) - - if kernel_repo: - kernel_srcdir = os.path.join(self.srcdir, "kvm") - kvm_utils.get_git_branch(kernel_repo, kernel_branch, kernel_srcdir, - kernel_commit, kernel_lbranch) - self.kernel_srcdir = kernel_srcdir - if kernel_patches: - os.chdir(self.kernel_srcdir) - for patch in kernel_patches: - utils.get_file(patch, os.path.join(self.userspace_srcdir, - os.path.basename(patch))) - utils.system('patch -p1 < %s' % os.path.basename(patch)) - else: - self.kernel_srcdir = None - - if kmod_repo: - kmod_srcdir = os.path.join (self.srcdir, "kvm_kmod") - kvm_utils.get_git_branch(kmod_repo, kmod_branch, kmod_srcdir, - kmod_commit, kmod_lbranch) - self.kmod_srcdir = kmod_srcdir - if kmod_patches: - os.chdir(self.kmod_srcdir) - for patch in kmod_patches: - utils.get_file(patch, os.path.join(self.userspace_srcdir, - os.path.basename(patch))) - utils.system('patch -p1 < %s' % os.path.basename(patch)) - else: - self.kmod_srcdir = None - - configure_script = os.path.join(self.userspace_srcdir, 'configure') - self.configure_options = check_configure_options(configure_script) - - - def _build(self): - make_jobs = utils.count_cpus() - cfg = './configure' - if self.kmod_srcdir: - logging.info('Building KVM modules') - os.chdir(self.kmod_srcdir) - module_build_steps = [cfg, - 'make clean', - 'make sync LINUX=%s' % self.kernel_srcdir, - 'make'] - elif self.kernel_srcdir: - logging.info('Building KVM modules') - os.chdir(self.userspace_srcdir) - cfg += ' --kerneldir=%s' % self.host_kernel_srcdir - module_build_steps = [cfg, - 'make clean', - 'make -C kernel LINUX=%s sync' % self.kernel_srcdir] - else: - module_build_steps = [] - - for step in module_build_steps: - utils.run(step) - - logging.info('Building KVM userspace code') - os.chdir(self.userspace_srcdir) - cfg += ' --prefix=%s' % self.prefix - if "--disable-strip" in self.configure_options: - cfg += ' --disable-strip' - if self.extra_configure_options: - cfg += ' %s' % self.extra_configure_options - utils.system(cfg) - utils.system('make clean') - utils.system('make -j %s' % make_jobs) - - - def _install(self): - if self.kernel_srcdir: - os.chdir(self.userspace_srcdir) - # the kernel module install with --prefix doesn't work, and DESTDIR - # wouldn't work for the userspace stuff, so we clear WANT_MODULE: - utils.system('make install WANT_MODULE=') - # and install the old-style-kmod modules manually: - self._install_kmods_old_userspace(self.userspace_srcdir) - elif self.kmod_srcdir: - # if we have a kmod repository, it is easier: - # 1) install userspace: - os.chdir(self.userspace_srcdir) - utils.system('make install') - # 2) install kmod: - self._install_kmods(self.kmod_srcdir) - else: - # if we don't have kmod sources, we just install - # userspace: - os.chdir(self.userspace_srcdir) - utils.system('make install') - - if self.path_to_roms: - install_roms(self.path_to_roms, self.prefix) - self.install_unittests() - create_symlinks(test_bindir=self.test_bindir, prefix=self.prefix, - bin_list=None, - unittest=self.unittest_prefix) - - - def install(self): - self._pull_code() - self._build() - self._install() - self.reload_modules_if_needed() - if self.save_results: - save_build(self.srcdir, self.results_dir) - - -class PreInstalledKvm(BaseInstaller): - # load_modules() will use the stock modules: - load_stock_modules = True - def install(self): - logging.info("Expecting KVM to be already installed. Doing nothing") - - -class FailedInstaller: - """ - Class used to be returned instead of the installer if a installation fails - - Useful to make sure no installer object is used if KVM installation fails. - """ - def __init__(self, msg="KVM install failed"): - self._msg = msg - - - def load_modules(self): - """Will refuse to load the KVM modules as install failed""" - raise FailedKvmInstall("KVM modules not available. reason: %s" % (self._msg)) - - -installer_classes = { - 'localsrc': SourceDirInstaller, - 'localtar': SourceDirInstaller, - 'release': SourceDirInstaller, - 'snapshot': SourceDirInstaller, - 'git': GitInstaller, - 'yum': YumInstaller, - 'koji': KojiInstaller, - 'preinstalled': PreInstalledKvm, -} - - -def _installer_class(install_mode): - c = installer_classes.get(install_mode) - if c is None: - raise error.TestError('Invalid or unsupported' - ' install mode: %s' % install_mode) - return c - - -def make_installer(params): - # priority: - # - 'install_mode' param - # - 'mode' param - mode = params.get("install_mode", params.get("mode")) - klass = _installer_class(mode) - return klass(mode) diff --git a/client/tests/kvm/kvm_config.py b/client/tests/kvm/kvm_config.py deleted file mode 100755 index 4dbb1d49..00000000 --- a/client/tests/kvm/kvm_config.py +++ /dev/null @@ -1,698 +0,0 @@ -#!/usr/bin/python -""" -KVM test configuration file parser - -@copyright: Red Hat 2008-2011 -""" - -import re, os, sys, optparse, collections - - -# Filter syntax: -# , means OR -# .. means AND -# . means IMMEDIATELY-FOLLOWED-BY - -# Example: -# qcow2..Fedora.14, RHEL.6..raw..boot, smp2..qcow2..migrate..ide -# means match all dicts whose names have: -# (qcow2 AND (Fedora IMMEDIATELY-FOLLOWED-BY 14)) OR -# ((RHEL IMMEDIATELY-FOLLOWED-BY 6) AND raw AND boot) OR -# (smp2 AND qcow2 AND migrate AND ide) - -# Note: -# 'qcow2..Fedora.14' is equivalent to 'Fedora.14..qcow2'. -# 'qcow2..Fedora.14' is not equivalent to 'qcow2..14.Fedora'. -# 'ide, scsi' is equivalent to 'scsi, ide'. - -# Filters can be used in 3 ways: -# only -# no -# : -# The last one starts a conditional block. - - -class ParserError: - def __init__(self, msg, line=None, filename=None, linenum=None): - self.msg = msg - self.line = line - self.filename = filename - self.linenum = linenum - - def __str__(self): - if self.line: - return "%s: %r (%s:%s)" % (self.msg, self.line, - self.filename, self.linenum) - else: - return "%s (%s:%s)" % (self.msg, self.filename, self.linenum) - - -num_failed_cases = 5 - - -class Node(object): - def __init__(self): - self.name = [] - self.dep = [] - self.content = [] - self.children = [] - self.labels = set() - self.append_to_shortname = False - self.failed_cases = collections.deque() - - -def _match_adjacent(block, ctx, ctx_set): - # TODO: explain what this function does - if block[0] not in ctx_set: - return 0 - if len(block) == 1: - return 1 - if block[1] not in ctx_set: - return int(ctx[-1] == block[0]) - k = 0 - i = ctx.index(block[0]) - while i < len(ctx): - if k > 0 and ctx[i] != block[k]: - i -= k - 1 - k = 0 - if ctx[i] == block[k]: - k += 1 - if k >= len(block): - break - if block[k] not in ctx_set: - break - i += 1 - return k - - -def _might_match_adjacent(block, ctx, ctx_set, descendant_labels): - matched = _match_adjacent(block, ctx, ctx_set) - for elem in block[matched:]: - if elem not in descendant_labels: - return False - return True - - -# Filter must inherit from object (otherwise type() won't work) -class Filter(object): - def __init__(self, s): - self.filter = [] - for char in s: - if not (char.isalnum() or char.isspace() or char in ".,_-"): - raise ParserError("Illegal characters in filter") - for word in s.replace(",", " ").split(): - word = [block.split(".") for block in word.split("..")] - for block in word: - for elem in block: - if not elem: - raise ParserError("Syntax error") - self.filter += [word] - - - def match(self, ctx, ctx_set): - for word in self.filter: - for block in word: - if _match_adjacent(block, ctx, ctx_set) != len(block): - break - else: - return True - return False - - - def might_match(self, ctx, ctx_set, descendant_labels): - for word in self.filter: - for block in word: - if not _might_match_adjacent(block, ctx, ctx_set, - descendant_labels): - break - else: - return True - return False - - -class NoOnlyFilter(Filter): - def __init__(self, line): - Filter.__init__(self, line.split(None, 1)[1]) - self.line = line - - -class OnlyFilter(NoOnlyFilter): - def is_irrelevant(self, ctx, ctx_set, descendant_labels): - return self.match(ctx, ctx_set) - - - def requires_action(self, ctx, ctx_set, descendant_labels): - return not self.might_match(ctx, ctx_set, descendant_labels) - - - def might_pass(self, failed_ctx, failed_ctx_set, ctx, ctx_set, - descendant_labels): - for word in self.filter: - for block in word: - if (_match_adjacent(block, ctx, ctx_set) > - _match_adjacent(block, failed_ctx, failed_ctx_set)): - return self.might_match(ctx, ctx_set, descendant_labels) - return False - - -class NoFilter(NoOnlyFilter): - def is_irrelevant(self, ctx, ctx_set, descendant_labels): - return not self.might_match(ctx, ctx_set, descendant_labels) - - - def requires_action(self, ctx, ctx_set, descendant_labels): - return self.match(ctx, ctx_set) - - - def might_pass(self, failed_ctx, failed_ctx_set, ctx, ctx_set, - descendant_labels): - for word in self.filter: - for block in word: - if (_match_adjacent(block, ctx, ctx_set) < - _match_adjacent(block, failed_ctx, failed_ctx_set)): - return not self.match(ctx, ctx_set) - return False - - -class Condition(NoFilter): - def __init__(self, line): - Filter.__init__(self, line.rstrip(":")) - self.line = line - self.content = [] - - -class NegativeCondition(OnlyFilter): - def __init__(self, line): - Filter.__init__(self, line.lstrip("!").rstrip(":")) - self.line = line - self.content = [] - - -class Parser(object): - """ - Parse an input file or string that follows the KVM Test Config File format - and generate a list of dicts that will be later used as configuration - parameters by the KVM tests. - - @see: http://www.linux-kvm.org/page/KVM-Autotest/Test_Config_File - """ - - def __init__(self, filename=None, debug=False): - """ - Initialize the parser and optionally parse a file. - - @param filename: Path of the file to parse. - @param debug: Whether to turn on debugging output. - """ - self.node = Node() - self.debug = debug - if filename: - self.parse_file(filename) - - - def parse_file(self, filename): - """ - Parse a file. - - @param filename: Path of the configuration file. - """ - self.node = self._parse(FileReader(filename), self.node) - - - def parse_string(self, s): - """ - Parse a string. - - @param s: String to parse. - """ - self.node = self._parse(StrReader(s), self.node) - - - def get_dicts(self, node=None, ctx=[], content=[], shortname=[], dep=[]): - """ - Generate dictionaries from the code parsed so far. This should - be called after parsing something. - - @return: A dict generator. - """ - def process_content(content, failed_filters): - # 1. Check that the filters in content are OK with the current - # context (ctx). - # 2. Move the parts of content that are still relevant into - # new_content and unpack conditional blocks if appropriate. - # For example, if an 'only' statement fully matches ctx, it - # becomes irrelevant and is not appended to new_content. - # If a conditional block fully matches, its contents are - # unpacked into new_content. - # 3. Move failed filters into failed_filters, so that next time we - # reach this node or one of its ancestors, we'll check those - # filters first. - for t in content: - filename, linenum, obj = t - if type(obj) is Op: - new_content.append(t) - continue - # obj is an OnlyFilter/NoFilter/Condition/NegativeCondition - if obj.requires_action(ctx, ctx_set, labels): - # This filter requires action now - if type(obj) is OnlyFilter or type(obj) is NoFilter: - self._debug(" filter did not pass: %r (%s:%s)", - obj.line, filename, linenum) - failed_filters.append(t) - return False - else: - self._debug(" conditional block matches: %r (%s:%s)", - obj.line, filename, linenum) - # Check and unpack the content inside this Condition - # object (note: the failed filters should go into - # new_internal_filters because we don't expect them to - # come from outside this node, even if the Condition - # itself was external) - if not process_content(obj.content, - new_internal_filters): - failed_filters.append(t) - return False - continue - elif obj.is_irrelevant(ctx, ctx_set, labels): - # This filter is no longer relevant and can be removed - continue - else: - # Keep the filter and check it again later - new_content.append(t) - return True - - def might_pass(failed_ctx, - failed_ctx_set, - failed_external_filters, - failed_internal_filters): - for t in failed_external_filters: - if t not in content: - return True - filename, linenum, filter = t - if filter.might_pass(failed_ctx, failed_ctx_set, ctx, ctx_set, - labels): - return True - for t in failed_internal_filters: - filename, linenum, filter = t - if filter.might_pass(failed_ctx, failed_ctx_set, ctx, ctx_set, - labels): - return True - return False - - def add_failed_case(): - node.failed_cases.appendleft((ctx, ctx_set, - new_external_filters, - new_internal_filters)) - if len(node.failed_cases) > num_failed_cases: - node.failed_cases.pop() - - node = node or self.node - # Update dep - for d in node.dep: - dep = dep + [".".join(ctx + [d])] - # Update ctx - ctx = ctx + node.name - ctx_set = set(ctx) - labels = node.labels - # Get the current name - name = ".".join(ctx) - if node.name: - self._debug("checking out %r", name) - # Check previously failed filters - for i, failed_case in enumerate(node.failed_cases): - if not might_pass(*failed_case): - self._debug(" this subtree has failed before") - del node.failed_cases[i] - node.failed_cases.appendleft(failed_case) - return - # Check content and unpack it into new_content - new_content = [] - new_external_filters = [] - new_internal_filters = [] - if (not process_content(node.content, new_internal_filters) or - not process_content(content, new_external_filters)): - add_failed_case() - return - # Update shortname - if node.append_to_shortname: - shortname = shortname + node.name - # Recurse into children - count = 0 - for n in node.children: - for d in self.get_dicts(n, ctx, new_content, shortname, dep): - count += 1 - yield d - # Reached leaf? - if not node.children: - self._debug(" reached leaf, returning it") - d = {"name": name, "dep": dep, "shortname": ".".join(shortname)} - for filename, linenum, op in new_content: - op.apply_to_dict(d) - yield d - # If this node did not produce any dicts, remember the failed filters - # of its descendants - elif not count: - new_external_filters = [] - new_internal_filters = [] - for n in node.children: - (failed_ctx, - failed_ctx_set, - failed_external_filters, - failed_internal_filters) = n.failed_cases[0] - for obj in failed_internal_filters: - if obj not in new_internal_filters: - new_internal_filters.append(obj) - for obj in failed_external_filters: - if obj in content: - if obj not in new_external_filters: - new_external_filters.append(obj) - else: - if obj not in new_internal_filters: - new_internal_filters.append(obj) - add_failed_case() - - - def _debug(self, s, *args): - if self.debug: - s = "DEBUG: %s" % s - print s % args - - - def _warn(self, s, *args): - s = "WARNING: %s" % s - print s % args - - - def _parse_variants(self, cr, node, prev_indent=-1): - """ - Read and parse lines from a FileReader object until a line with an - indent level lower than or equal to prev_indent is encountered. - - @param cr: A FileReader/StrReader object. - @param node: A node to operate on. - @param prev_indent: The indent level of the "parent" block. - @return: A node object. - """ - node4 = Node() - - while True: - line, indent, linenum = cr.get_next_line(prev_indent) - if not line: - break - - name, dep = map(str.strip, line.lstrip("- ").split(":", 1)) - for char in name: - if not (char.isalnum() or char in "@._-"): - raise ParserError("Illegal characters in variant name", - line, cr.filename, linenum) - for char in dep: - if not (char.isalnum() or char.isspace() or char in ".,_-"): - raise ParserError("Illegal characters in dependencies", - line, cr.filename, linenum) - - node2 = Node() - node2.children = [node] - node2.labels = node.labels - - node3 = self._parse(cr, node2, prev_indent=indent) - node3.name = name.lstrip("@").split(".") - node3.dep = dep.replace(",", " ").split() - node3.append_to_shortname = not name.startswith("@") - - node4.children += [node3] - node4.labels.update(node3.labels) - node4.labels.update(node3.name) - - return node4 - - - def _parse(self, cr, node, prev_indent=-1): - """ - Read and parse lines from a StrReader object until a line with an - indent level lower than or equal to prev_indent is encountered. - - @param cr: A FileReader/StrReader object. - @param node: A Node or a Condition object to operate on. - @param prev_indent: The indent level of the "parent" block. - @return: A node object. - """ - while True: - line, indent, linenum = cr.get_next_line(prev_indent) - if not line: - break - - words = line.split(None, 1) - - # Parse 'variants' - if line == "variants:": - # 'variants' is not allowed inside a conditional block - if (isinstance(node, Condition) or - isinstance(node, NegativeCondition)): - raise ParserError("'variants' is not allowed inside a " - "conditional block", - None, cr.filename, linenum) - node = self._parse_variants(cr, node, prev_indent=indent) - continue - - # Parse 'include' statements - if words[0] == "include": - if len(words) < 2: - raise ParserError("Syntax error: missing parameter", - line, cr.filename, linenum) - filename = os.path.expanduser(words[1]) - if isinstance(cr, FileReader) and not os.path.isabs(filename): - filename = os.path.join(os.path.dirname(cr.filename), - filename) - if not os.path.isfile(filename): - self._warn("%r (%s:%s): file doesn't exist or is not a " - "regular file", line, cr.filename, linenum) - continue - node = self._parse(FileReader(filename), node) - continue - - # Parse 'only' and 'no' filters - if words[0] in ("only", "no"): - if len(words) < 2: - raise ParserError("Syntax error: missing parameter", - line, cr.filename, linenum) - try: - if words[0] == "only": - f = OnlyFilter(line) - elif words[0] == "no": - f = NoFilter(line) - except ParserError, e: - e.line = line - e.filename = cr.filename - e.linenum = linenum - raise - node.content += [(cr.filename, linenum, f)] - continue - - # Look for operators - op_match = _ops_exp.search(line) - - # Parse conditional blocks - if ":" in line: - index = line.index(":") - if not op_match or index < op_match.start(): - index += 1 - cr.set_next_line(line[index:], indent, linenum) - line = line[:index] - try: - if line.startswith("!"): - cond = NegativeCondition(line) - else: - cond = Condition(line) - except ParserError, e: - e.line = line - e.filename = cr.filename - e.linenum = linenum - raise - self._parse(cr, cond, prev_indent=indent) - node.content += [(cr.filename, linenum, cond)] - continue - - # Parse regular operators - if not op_match: - raise ParserError("Syntax error", line, cr.filename, linenum) - node.content += [(cr.filename, linenum, Op(line, op_match))] - - return node - - -# Assignment operators - -_reserved_keys = set(("name", "shortname", "dep")) - - -def _op_set(d, key, value): - if key not in _reserved_keys: - d[key] = value - - -def _op_append(d, key, value): - if key not in _reserved_keys: - d[key] = d.get(key, "") + value - - -def _op_prepend(d, key, value): - if key not in _reserved_keys: - d[key] = value + d.get(key, "") - - -def _op_regex_set(d, exp, value): - exp = re.compile("%s$" % exp) - for key in d: - if key not in _reserved_keys and exp.match(key): - d[key] = value - - -def _op_regex_append(d, exp, value): - exp = re.compile("%s$" % exp) - for key in d: - if key not in _reserved_keys and exp.match(key): - d[key] += value - - -def _op_regex_prepend(d, exp, value): - exp = re.compile("%s$" % exp) - for key in d: - if key not in _reserved_keys and exp.match(key): - d[key] = value + d[key] - - -def _op_regex_del(d, empty, exp): - exp = re.compile("%s$" % exp) - for key in d.keys(): - if key not in _reserved_keys and exp.match(key): - del d[key] - - -_ops = {"=": (r"\=", _op_set), - "+=": (r"\+\=", _op_append), - "<=": (r"\<\=", _op_prepend), - "?=": (r"\?\=", _op_regex_set), - "?+=": (r"\?\+\=", _op_regex_append), - "?<=": (r"\?\<\=", _op_regex_prepend), - "del": (r"^del\b", _op_regex_del)} - -_ops_exp = re.compile("|".join([op[0] for op in _ops.values()])) - - -class Op(object): - def __init__(self, line, m): - self.func = _ops[m.group()][1] - self.key = line[:m.start()].strip() - value = line[m.end():].strip() - if value and (value[0] == value[-1] == '"' or - value[0] == value[-1] == "'"): - value = value[1:-1] - self.value = value - - - def apply_to_dict(self, d): - self.func(d, self.key, self.value) - - -# StrReader and FileReader - -class StrReader(object): - """ - Preprocess an input string for easy reading. - """ - def __init__(self, s): - """ - Initialize the reader. - - @param s: The string to parse. - """ - self.filename = "" - self._lines = [] - self._line_index = 0 - self._stored_line = None - for linenum, line in enumerate(s.splitlines()): - line = line.rstrip().expandtabs() - stripped_line = line.lstrip() - indent = len(line) - len(stripped_line) - if (not stripped_line - or stripped_line.startswith("#") - or stripped_line.startswith("//")): - continue - self._lines.append((stripped_line, indent, linenum + 1)) - - - def get_next_line(self, prev_indent): - """ - Get the next line in the current block. - - @param prev_indent: The indentation level of the previous block. - @return: (line, indent, linenum), where indent is the line's - indentation level. If no line is available, (None, -1, -1) is - returned. - """ - if self._stored_line: - ret = self._stored_line - self._stored_line = None - return ret - if self._line_index >= len(self._lines): - return None, -1, -1 - line, indent, linenum = self._lines[self._line_index] - if indent <= prev_indent: - return None, -1, -1 - self._line_index += 1 - return line, indent, linenum - - - def set_next_line(self, line, indent, linenum): - """ - Make the next call to get_next_line() return the given line instead of - the real next line. - """ - line = line.strip() - if line: - self._stored_line = line, indent, linenum - - -class FileReader(StrReader): - """ - Preprocess an input file for easy reading. - """ - def __init__(self, filename): - """ - Initialize the reader. - - @parse filename: The name of the input file. - """ - StrReader.__init__(self, open(filename).read()) - self.filename = filename - - -if __name__ == "__main__": - parser = optparse.OptionParser('usage: %prog [options] filename ' - '[extra code] ...\n\nExample:\n\n ' - '%prog tests.cfg "only my_set" "no qcow2"') - parser.add_option("-v", "--verbose", dest="debug", action="store_true", - help="include debug messages in console output") - parser.add_option("-f", "--fullname", dest="fullname", action="store_true", - help="show full dict names instead of short names") - parser.add_option("-c", "--contents", dest="contents", action="store_true", - help="show dict contents") - - options, args = parser.parse_args() - if not args: - parser.error("filename required") - - c = Parser(args[0], debug=options.debug) - for s in args[1:]: - c.parse_string(s) - - for i, d in enumerate(c.get_dicts()): - if options.fullname: - print "dict %4d: %s" % (i + 1, d["name"]) - else: - print "dict %4d: %s" % (i + 1, d["shortname"]) - if options.contents: - keys = d.keys() - keys.sort() - for key in keys: - print " %s = %s" % (key, d[key]) diff --git a/client/tests/kvm/kvm_monitor.py b/client/tests/kvm/kvm_monitor.py deleted file mode 100644 index 159c575a..00000000 --- a/client/tests/kvm/kvm_monitor.py +++ /dev/null @@ -1,763 +0,0 @@ -""" -Interfaces to the QEMU monitor. - -@copyright: 2008-2010 Red Hat Inc. -""" - -import socket, time, threading, logging, select -import kvm_utils -try: - import json -except ImportError: - logging.warning("Could not import json module. " - "QMP monitor functionality disabled.") - - -class MonitorError(Exception): - pass - - -class MonitorConnectError(MonitorError): - pass - - -class MonitorSocketError(MonitorError): - def __init__(self, msg, e): - Exception.__init__(self, msg, e) - self.msg = msg - self.e = e - - def __str__(self): - return "%s (%s)" % (self.msg, self.e) - - -class MonitorLockError(MonitorError): - pass - - -class MonitorProtocolError(MonitorError): - pass - - -class MonitorNotSupportedError(MonitorError): - pass - - -class QMPCmdError(MonitorError): - def __init__(self, cmd, qmp_args, data): - MonitorError.__init__(self, cmd, qmp_args, data) - self.cmd = cmd - self.qmp_args = qmp_args - self.data = data - - def __str__(self): - return ("QMP command %r failed (arguments: %r, " - "error message: %r)" % (self.cmd, self.qmp_args, self.data)) - - -class Monitor: - """ - Common code for monitor classes. - """ - - def __init__(self, name, filename): - """ - Initialize the instance. - - @param name: Monitor identifier (a string) - @param filename: Monitor socket filename - @raise MonitorConnectError: Raised if the connection fails - """ - self.name = name - self.filename = filename - self._lock = threading.RLock() - self._socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - - try: - self._socket.connect(filename) - except socket.error: - raise MonitorConnectError("Could not connect to monitor socket") - - - def __del__(self): - # Automatically close the connection when the instance is garbage - # collected - try: - self._socket.shutdown(socket.SHUT_RDWR) - except socket.error: - pass - self._socket.close() - - - # The following two functions are defined to make sure the state is set - # exclusively by the constructor call as specified in __getinitargs__(). - - def __getstate__(self): - pass - - - def __setstate__(self, state): - pass - - - def __getinitargs__(self): - # Save some information when pickling -- will be passed to the - # constructor upon unpickling - return self.name, self.filename, True - - - def _acquire_lock(self, timeout=20): - end_time = time.time() + timeout - while time.time() < end_time: - if self._lock.acquire(False): - return True - time.sleep(0.05) - return False - - - def _data_available(self, timeout=0): - timeout = max(0, timeout) - return bool(select.select([self._socket], [], [], timeout)[0]) - - - def _recvall(self): - s = "" - while self._data_available(): - try: - data = self._socket.recv(1024) - except socket.error, e: - raise MonitorSocketError("Could not receive data from monitor", - e) - if not data: - break - s += data - return s - - - def is_responsive(self): - """ - Return True iff the monitor is responsive. - """ - try: - self.verify_responsive() - return True - except MonitorError: - return False - - -class HumanMonitor(Monitor): - """ - Wraps "human monitor" commands. - """ - - def __init__(self, name, filename, suppress_exceptions=False): - """ - Connect to the monitor socket and find the (qemu) prompt. - - @param name: Monitor identifier (a string) - @param filename: Monitor socket filename - @raise MonitorConnectError: Raised if the connection fails and - suppress_exceptions is False - @raise MonitorProtocolError: Raised if the initial (qemu) prompt isn't - found and suppress_exceptions is False - @note: Other exceptions may be raised. See cmd()'s - docstring. - """ - try: - Monitor.__init__(self, name, filename) - - self.protocol = "human" - - # Find the initial (qemu) prompt - s, o = self._read_up_to_qemu_prompt(20) - if not s: - raise MonitorProtocolError("Could not find (qemu) prompt " - "after connecting to monitor. " - "Output so far: %r" % o) - - # Save the output of 'help' for future use - self._help_str = self.cmd("help", debug=False) - - except MonitorError, e: - if suppress_exceptions: - logging.warn(e) - else: - raise - - - # Private methods - - def _read_up_to_qemu_prompt(self, timeout=20): - s = "" - end_time = time.time() + timeout - while self._data_available(end_time - time.time()): - data = self._recvall() - if not data: - break - s += data - try: - if s.splitlines()[-1].split()[-1] == "(qemu)": - return True, "\n".join(s.splitlines()[:-1]) - except IndexError: - continue - return False, "\n".join(s.splitlines()) - - - def _send(self, cmd): - """ - Send a command without waiting for output. - - @param cmd: Command to send - @raise MonitorLockError: Raised if the lock cannot be acquired - @raise MonitorSocketError: Raised if a socket error occurs - """ - if not self._acquire_lock(20): - raise MonitorLockError("Could not acquire exclusive lock to send " - "monitor command '%s'" % cmd) - - try: - try: - self._socket.sendall(cmd + "\n") - except socket.error, e: - raise MonitorSocketError("Could not send monitor command %r" % - cmd, e) - - finally: - self._lock.release() - - - # Public methods - - def cmd(self, command, timeout=20, debug=True): - """ - Send command to the monitor. - - @param command: Command to send to the monitor - @param timeout: Time duration to wait for the (qemu) prompt to return - @param debug: Whether to print the commands being sent and responses - @return: Output received from the monitor - @raise MonitorLockError: Raised if the lock cannot be acquired - @raise MonitorSocketError: Raised if a socket error occurs - @raise MonitorProtocolError: Raised if the (qemu) prompt cannot be - found after sending the command - """ - if debug: - logging.debug("(monitor %s) Sending command '%s'", - self.name, command) - if not self._acquire_lock(20): - raise MonitorLockError("Could not acquire exclusive lock to send " - "monitor command '%s'" % command) - - try: - # Read any data that might be available - self._recvall() - # Send command - self._send(command) - # Read output - s, o = self._read_up_to_qemu_prompt(timeout) - # Remove command echo from output - o = "\n".join(o.splitlines()[1:]) - # Report success/failure - if s: - if debug and o: - logging.debug("(monitor %s) " - "Response to '%s'", self.name, - command) - for l in o.splitlines(): - logging.debug("(monitor %s) %s", self.name, l) - return o - else: - msg = ("Could not find (qemu) prompt after command '%s'. " - "Output so far: %r" % (command, o)) - raise MonitorProtocolError(msg) - - finally: - self._lock.release() - - - def verify_responsive(self): - """ - Make sure the monitor is responsive by sending a command. - """ - self.cmd("info status", debug=False) - - - # Command wrappers - # Notes: - # - All of the following commands raise exceptions in a similar manner to - # cmd(). - # - A command wrapper should use self._help_str if it requires information - # about the monitor's capabilities. - - def quit(self): - """ - Send "quit" without waiting for output. - """ - self._send("quit") - - - def info(self, what): - """ - Request info about something and return the output. - """ - return self.cmd("info %s" % what) - - - def query(self, what): - """ - Alias for info. - """ - return self.info(what) - - - def screendump(self, filename, debug=True): - """ - Request a screendump. - - @param filename: Location for the screendump - @return: The command's output - """ - return self.cmd(command="screendump %s" % filename, debug=debug) - - - def migrate(self, uri, full_copy=False, incremental_copy=False, wait=False): - """ - Migrate. - - @param uri: destination URI - @param full_copy: If true, migrate with full disk copy - @param incremental_copy: If true, migrate with incremental disk copy - @param wait: If true, wait for completion - @return: The command's output - """ - cmd = "migrate" - if not wait: - cmd += " -d" - if full_copy: - cmd += " -b" - if incremental_copy: - cmd += " -i" - cmd += " %s" % uri - return self.cmd(cmd) - - - def migrate_set_speed(self, value): - """ - Set maximum speed (in bytes/sec) for migrations. - - @param value: Speed in bytes/sec - @return: The command's output - """ - return self.cmd("migrate_set_speed %s" % value) - - - def sendkey(self, keystr, hold_time=1): - """ - Send key combination to VM. - - @param keystr: Key combination string - @param hold_time: Hold time in ms (should normally stay 1 ms) - @return: The command's output - """ - return self.cmd("sendkey %s %s" % (keystr, hold_time)) - - - def mouse_move(self, dx, dy): - """ - Move mouse. - - @param dx: X amount - @param dy: Y amount - @return: The command's output - """ - return self.cmd("mouse_move %d %d" % (dx, dy)) - - - def mouse_button(self, state): - """ - Set mouse button state. - - @param state: Button state (1=L, 2=M, 4=R) - @return: The command's output - """ - return self.cmd("mouse_button %d" % state) - - -class QMPMonitor(Monitor): - """ - Wraps QMP monitor commands. - """ - - def __init__(self, name, filename, suppress_exceptions=False): - """ - Connect to the monitor socket, read the greeting message and issue the - qmp_capabilities command. Also make sure the json module is available. - - @param name: Monitor identifier (a string) - @param filename: Monitor socket filename - @raise MonitorConnectError: Raised if the connection fails and - suppress_exceptions is False - @raise MonitorProtocolError: Raised if the no QMP greeting message is - received and suppress_exceptions is False - @raise MonitorNotSupportedError: Raised if json isn't available and - suppress_exceptions is False - @note: Other exceptions may be raised if the qmp_capabilities command - fails. See cmd()'s docstring. - """ - try: - Monitor.__init__(self, name, filename) - - self.protocol = "qmp" - self._greeting = None - self._events = [] - - # Make sure json is available - try: - json - except NameError: - raise MonitorNotSupportedError("QMP requires the json module " - "(Python 2.6 and up)") - - # Read greeting message - end_time = time.time() + 20 - while time.time() < end_time: - for obj in self._read_objects(): - if "QMP" in obj: - self._greeting = obj - break - if self._greeting: - break - time.sleep(0.1) - else: - raise MonitorProtocolError("No QMP greeting message received") - - # Issue qmp_capabilities - self.cmd("qmp_capabilities") - - except MonitorError, e: - if suppress_exceptions: - logging.warn(e) - else: - raise - - - # Private methods - - def _build_cmd(self, cmd, args=None, id=None): - obj = {"execute": cmd} - if args is not None: - obj["arguments"] = args - if id is not None: - obj["id"] = id - return obj - - - def _read_objects(self, timeout=5): - """ - Read lines from the monitor and try to decode them. - Stop when all available lines have been successfully decoded, or when - timeout expires. If any decoded objects are asynchronous events, store - them in self._events. Return all decoded objects. - - @param timeout: Time to wait for all lines to decode successfully - @return: A list of objects - """ - if not self._data_available(): - return [] - s = "" - end_time = time.time() + timeout - while self._data_available(end_time - time.time()): - s += self._recvall() - # Make sure all lines are decodable - for line in s.splitlines(): - if line: - try: - json.loads(line) - except: - # Found an incomplete or broken line -- keep reading - break - else: - # All lines are OK -- stop reading - break - # Decode all decodable lines - objs = [] - for line in s.splitlines(): - try: - objs += [json.loads(line)] - except: - pass - # Keep track of asynchronous events - self._events += [obj for obj in objs if "event" in obj] - return objs - - - def _send(self, data): - """ - Send raw data without waiting for response. - - @param data: Data to send - @raise MonitorSocketError: Raised if a socket error occurs - """ - try: - self._socket.sendall(data) - except socket.error, e: - raise MonitorSocketError("Could not send data: %r" % data, e) - - - def _get_response(self, id=None, timeout=20): - """ - Read a response from the QMP monitor. - - @param id: If not None, look for a response with this id - @param timeout: Time duration to wait for response - @return: The response dict, or None if none was found - """ - end_time = time.time() + timeout - while self._data_available(end_time - time.time()): - for obj in self._read_objects(): - if isinstance(obj, dict): - if id is not None and obj.get("id") != id: - continue - if "return" in obj or "error" in obj: - return obj - - - # Public methods - - def cmd(self, cmd, args=None, timeout=20, debug=True): - """ - Send a QMP monitor command and return the response. - - Note: an id is automatically assigned to the command and the response - is checked for the presence of the same id. - - @param cmd: Command to send - @param args: A dict containing command arguments, or None - @param timeout: Time duration to wait for response - @return: The response received - @raise MonitorLockError: Raised if the lock cannot be acquired - @raise MonitorSocketError: Raised if a socket error occurs - @raise MonitorProtocolError: Raised if no response is received - @raise QMPCmdError: Raised if the response is an error message - (the exception's args are (cmd, args, data) where data is the - error data) - """ - if debug: - logging.debug("(monitor %s) Sending command '%s'", - self.name, cmd) - if not self._acquire_lock(20): - raise MonitorLockError("Could not acquire exclusive lock to send " - "QMP command '%s'" % cmd) - - try: - # Read any data that might be available - self._read_objects() - # Send command - id = kvm_utils.generate_random_string(8) - self._send(json.dumps(self._build_cmd(cmd, args, id)) + "\n") - # Read response - r = self._get_response(id, timeout) - if r is None: - raise MonitorProtocolError("Received no response to QMP " - "command '%s', or received a " - "response with an incorrect id" - % cmd) - if "return" in r: - if debug and r["return"]: - logging.debug("(monitor %s) " - "Response to '%s'", self.name, cmd) - o = str(r["return"]) - for l in o.splitlines(): - logging.debug("(monitor %s) %s", self.name, l) - return r["return"] - if "error" in r: - raise QMPCmdError(cmd, args, r["error"]) - - finally: - self._lock.release() - - - def cmd_raw(self, data, timeout=20): - """ - Send a raw string to the QMP monitor and return the response. - Unlike cmd(), return the raw response dict without performing any - checks on it. - - @param data: The data to send - @param timeout: Time duration to wait for response - @return: The response received - @raise MonitorLockError: Raised if the lock cannot be acquired - @raise MonitorSocketError: Raised if a socket error occurs - @raise MonitorProtocolError: Raised if no response is received - """ - if not self._acquire_lock(20): - raise MonitorLockError("Could not acquire exclusive lock to send " - "data: %r" % data) - - try: - self._read_objects() - self._send(data) - r = self._get_response(None, timeout) - if r is None: - raise MonitorProtocolError("Received no response to data: %r" % - data) - return r - - finally: - self._lock.release() - - - def cmd_obj(self, obj, timeout=20): - """ - Transform a Python object to JSON, send the resulting string to the QMP - monitor, and return the response. - Unlike cmd(), return the raw response dict without performing any - checks on it. - - @param obj: The object to send - @param timeout: Time duration to wait for response - @return: The response received - @raise MonitorLockError: Raised if the lock cannot be acquired - @raise MonitorSocketError: Raised if a socket error occurs - @raise MonitorProtocolError: Raised if no response is received - """ - return self.cmd_raw(json.dumps(obj) + "\n") - - - def cmd_qmp(self, cmd, args=None, id=None, timeout=20): - """ - Build a QMP command from the passed arguments, send it to the monitor - and return the response. - Unlike cmd(), return the raw response dict without performing any - checks on it. - - @param cmd: Command to send - @param args: A dict containing command arguments, or None - @param id: An id for the command, or None - @param timeout: Time duration to wait for response - @return: The response received - @raise MonitorLockError: Raised if the lock cannot be acquired - @raise MonitorSocketError: Raised if a socket error occurs - @raise MonitorProtocolError: Raised if no response is received - """ - return self.cmd_obj(self._build_cmd(cmd, args, id), timeout) - - - def verify_responsive(self): - """ - Make sure the monitor is responsive by sending a command. - """ - self.cmd(cmd="query-status", debug=False) - - - def get_events(self): - """ - Return a list of the asynchronous events received since the last - clear_events() call. - - @return: A list of events (the objects returned have an "event" key) - @raise MonitorLockError: Raised if the lock cannot be acquired - """ - if not self._acquire_lock(20): - raise MonitorLockError("Could not acquire exclusive lock to read " - "QMP events") - try: - self._read_objects() - return self._events[:] - finally: - self._lock.release() - - - def get_event(self, name): - """ - Look for an event with the given name in the list of events. - - @param name: The name of the event to look for (e.g. 'RESET') - @return: An event object or None if none is found - """ - for e in self.get_events(): - if e.get("event") == name: - return e - - - def clear_events(self): - """ - Clear the list of asynchronous events. - - @raise MonitorLockError: Raised if the lock cannot be acquired - """ - if not self._acquire_lock(20): - raise MonitorLockError("Could not acquire exclusive lock to clear " - "QMP event list") - self._events = [] - self._lock.release() - - - def get_greeting(self): - """ - Return QMP greeting message. - """ - return self._greeting - - - # Command wrappers - # Note: all of the following functions raise exceptions in a similar manner - # to cmd(). - - def quit(self): - """ - Send "quit" and return the response. - """ - return self.cmd("quit") - - - def info(self, what): - """ - Request info about something and return the response. - """ - return self.cmd("query-%s" % what) - - - def query(self, what): - """ - Alias for info. - """ - return self.info(what) - - - def screendump(self, filename, debug=True): - """ - Request a screendump. - - @param filename: Location for the screendump - @return: The response to the command - """ - args = {"filename": filename} - return self.cmd(cmd="screendump", args=args, debug=debug) - - - def migrate(self, uri, full_copy=False, incremental_copy=False, wait=False): - """ - Migrate. - - @param uri: destination URI - @param full_copy: If true, migrate with full disk copy - @param incremental_copy: If true, migrate with incremental disk copy - @param wait: If true, wait for completion - @return: The response to the command - """ - args = {"uri": uri, - "blk": full_copy, - "inc": incremental_copy} - return self.cmd("migrate", args) - - - def migrate_set_speed(self, value): - """ - Set maximum speed (in bytes/sec) for migrations. - - @param value: Speed in bytes/sec - @return: The response to the command - """ - args = {"value": value} - return self.cmd("migrate_set_speed", args) diff --git a/client/tests/kvm/kvm_preprocessing.py b/client/tests/kvm/kvm_preprocessing.py deleted file mode 100644 index f679be04..00000000 --- a/client/tests/kvm/kvm_preprocessing.py +++ /dev/null @@ -1,454 +0,0 @@ -import os, time, commands, re, logging, glob, threading, shutil -from autotest_lib.client.bin import utils -from autotest_lib.client.common_lib import error -import kvm_vm, kvm_utils, kvm_subprocess, kvm_monitor, ppm_utils, test_setup -try: - import PIL.Image -except ImportError: - logging.warning('No python imaging library installed. PPM image ' - 'conversion to JPEG disabled. In order to enable it, ' - 'please install python-imaging or the equivalent for your ' - 'distro.') - - -_screendump_thread = None -_screendump_thread_termination_event = None - - -def preprocess_image(test, params): - """ - Preprocess a single QEMU image according to the instructions in params. - - @param test: Autotest test object. - @param params: A dict containing image preprocessing parameters. - @note: Currently this function just creates an image if requested. - """ - image_filename = kvm_vm.get_image_filename(params, test.bindir) - - create_image = False - - if params.get("force_create_image") == "yes": - logging.debug("'force_create_image' specified; creating image...") - create_image = True - elif (params.get("create_image") == "yes" and not - os.path.exists(image_filename)): - logging.debug("Creating image...") - create_image = True - - if create_image and not kvm_vm.create_image(params, test.bindir): - raise error.TestError("Could not create image") - - -def preprocess_vm(test, params, env, name): - """ - Preprocess a single VM object according to the instructions in params. - Start the VM if requested and get a screendump. - - @param test: An Autotest test object. - @param params: A dict containing VM preprocessing parameters. - @param env: The environment (a dict-like object). - @param name: The name of the VM object. - """ - logging.debug("Preprocessing VM '%s'..." % name) - vm = env.get_vm(name) - if not vm: - logging.debug("VM object does not exist; creating it") - vm = kvm_vm.VM(name, params, test.bindir, env.get("address_cache")) - env.register_vm(name, vm) - - start_vm = False - - if params.get("restart_vm") == "yes": - logging.debug("'restart_vm' specified; (re)starting VM...") - start_vm = True - elif params.get("migration_mode"): - logging.debug("Starting VM in incoming migration mode...") - start_vm = True - elif params.get("start_vm") == "yes": - if not vm.is_alive(): - logging.debug("VM is not alive; starting it...") - start_vm = True - elif vm.make_qemu_command() != vm.make_qemu_command(name, params, - test.bindir): - logging.debug("VM's qemu command differs from requested one; " - "restarting it...") - start_vm = True - - if start_vm: - # Start the VM (or restart it if it's already up) - vm.create(name, params, test.bindir, - migration_mode=params.get("migration_mode")) - else: - # Don't start the VM, just update its params - vm.params = params - - scrdump_filename = os.path.join(test.debugdir, "pre_%s.ppm" % name) - try: - if vm.monitor: - vm.monitor.screendump(scrdump_filename, debug=False) - except kvm_monitor.MonitorError, e: - logging.warn(e) - - -def postprocess_image(test, params): - """ - Postprocess a single QEMU image according to the instructions in params. - - @param test: An Autotest test object. - @param params: A dict containing image postprocessing parameters. - """ - if params.get("check_image") == "yes": - kvm_vm.check_image(params, test.bindir) - if params.get("remove_image") == "yes": - kvm_vm.remove_image(params, test.bindir) - - -def postprocess_vm(test, params, env, name): - """ - Postprocess a single VM object according to the instructions in params. - Kill the VM if requested and get a screendump. - - @param test: An Autotest test object. - @param params: A dict containing VM postprocessing parameters. - @param env: The environment (a dict-like object). - @param name: The name of the VM object. - """ - logging.debug("Postprocessing VM '%s'..." % name) - vm = env.get_vm(name) - if not vm: - return - - scrdump_filename = os.path.join(test.debugdir, "post_%s.ppm" % name) - try: - if vm.monitor: - vm.monitor.screendump(scrdump_filename, debug=False) - except kvm_monitor.MonitorError, e: - logging.warn(e) - - if params.get("kill_vm") == "yes": - kill_vm_timeout = float(params.get("kill_vm_timeout", 0)) - if kill_vm_timeout: - logging.debug("'kill_vm' specified; waiting for VM to shut down " - "before killing it...") - kvm_utils.wait_for(vm.is_dead, kill_vm_timeout, 0, 1) - else: - logging.debug("'kill_vm' specified; killing VM...") - vm.destroy(gracefully = params.get("kill_vm_gracefully") == "yes") - - -def process_command(test, params, env, command, command_timeout, - command_noncritical): - """ - Pre- or post- custom commands to be executed before/after a test is run - - @param test: An Autotest test object. - @param params: A dict containing all VM and image parameters. - @param env: The environment (a dict-like object). - @param command: Command to be run. - @param command_timeout: Timeout for command execution. - @param command_noncritical: If True test will not fail if command fails. - """ - # Export environment vars - for k in params: - os.putenv("KVM_TEST_%s" % k, str(params[k])) - # Execute commands - try: - utils.system("cd %s; %s" % (test.bindir, command)) - except error.CmdError, e: - if command_noncritical: - logging.warn(e) - else: - raise - -def process(test, params, env, image_func, vm_func): - """ - Pre- or post-process VMs and images according to the instructions in params. - Call image_func for each image listed in params and vm_func for each VM. - - @param test: An Autotest test object. - @param params: A dict containing all VM and image parameters. - @param env: The environment (a dict-like object). - @param image_func: A function to call for each image. - @param vm_func: A function to call for each VM. - """ - # Get list of VMs specified for this test - for vm_name in params.objects("vms"): - vm_params = params.object_params(vm_name) - # Get list of images specified for this VM - for image_name in vm_params.objects("images"): - image_params = vm_params.object_params(image_name) - # Call image_func for each image - image_func(test, image_params) - # Call vm_func for each vm - vm_func(test, vm_params, env, vm_name) - - -@error.context_aware -def preprocess(test, params, env): - """ - Preprocess all VMs and images according to the instructions in params. - Also, collect some host information, such as the KVM version. - - @param test: An Autotest test object. - @param params: A dict containing all VM and image parameters. - @param env: The environment (a dict-like object). - """ - error.context("preprocessing") - # Start tcpdump if it isn't already running - if "address_cache" not in env: - env["address_cache"] = {} - if "tcpdump" in env and not env["tcpdump"].is_alive(): - env["tcpdump"].close() - del env["tcpdump"] - if "tcpdump" not in env and params.get("run_tcpdump", "yes") == "yes": - cmd = "%s -npvi any 'dst port 68'" % kvm_utils.find_command("tcpdump") - logging.debug("Starting tcpdump (%s)...", cmd) - env["tcpdump"] = kvm_subprocess.Tail( - command=cmd, - output_func=_update_address_cache, - output_params=(env["address_cache"],)) - if kvm_utils.wait_for(lambda: not env["tcpdump"].is_alive(), - 0.1, 0.1, 1.0): - logging.warn("Could not start tcpdump") - logging.warn("Status: %s" % env["tcpdump"].get_status()) - logging.warn("Output:" + kvm_utils.format_str_for_message( - env["tcpdump"].get_output())) - - # Destroy and remove VMs that are no longer needed in the environment - requested_vms = params.objects("vms") - for key in env.keys(): - vm = env[key] - if not kvm_utils.is_vm(vm): - continue - if not vm.name in requested_vms: - logging.debug("VM '%s' found in environment but not required for " - "test; removing it..." % vm.name) - vm.destroy() - del env[key] - - # Get the KVM kernel module version and write it as a keyval - logging.debug("Fetching KVM module version...") - if os.path.exists("/dev/kvm"): - try: - kvm_version = open("/sys/module/kvm/version").read().strip() - except: - kvm_version = os.uname()[2] - else: - kvm_version = "Unknown" - logging.debug("KVM module not loaded") - logging.debug("KVM version: %s" % kvm_version) - test.write_test_keyval({"kvm_version": kvm_version}) - - # Get the KVM userspace version and write it as a keyval - logging.debug("Fetching KVM userspace version...") - qemu_path = kvm_utils.get_path(test.bindir, params.get("qemu_binary", - "qemu")) - version_line = commands.getoutput("%s -help | head -n 1" % qemu_path) - matches = re.findall("[Vv]ersion .*?,", version_line) - if matches: - kvm_userspace_version = " ".join(matches[0].split()[1:]).strip(",") - else: - kvm_userspace_version = "Unknown" - logging.debug("Could not fetch KVM userspace version") - logging.debug("KVM userspace version: %s" % kvm_userspace_version) - test.write_test_keyval({"kvm_userspace_version": kvm_userspace_version}) - - if params.get("setup_hugepages") == "yes": - h = test_setup.HugePageConfig(params) - h.setup() - - # Execute any pre_commands - if params.get("pre_command"): - process_command(test, params, env, params.get("pre_command"), - int(params.get("pre_command_timeout", "600")), - params.get("pre_command_noncritical") == "yes") - - # Preprocess all VMs and images - process(test, params, env, preprocess_image, preprocess_vm) - - # Start the screendump thread - if params.get("take_regular_screendumps") == "yes": - logging.debug("Starting screendump thread") - global _screendump_thread, _screendump_thread_termination_event - _screendump_thread_termination_event = threading.Event() - _screendump_thread = threading.Thread(target=_take_screendumps, - args=(test, params, env)) - _screendump_thread.start() - - -@error.context_aware -def postprocess(test, params, env): - """ - Postprocess all VMs and images according to the instructions in params. - - @param test: An Autotest test object. - @param params: Dict containing all VM and image parameters. - @param env: The environment (a dict-like object). - """ - error.context("postprocessing") - - # Postprocess all VMs and images - process(test, params, env, postprocess_image, postprocess_vm) - - # Terminate the screendump thread - global _screendump_thread, _screendump_thread_termination_event - if _screendump_thread: - logging.debug("Terminating screendump thread...") - _screendump_thread_termination_event.set() - _screendump_thread.join(10) - _screendump_thread = None - - # Warn about corrupt PPM files - for f in glob.glob(os.path.join(test.debugdir, "*.ppm")): - if not ppm_utils.image_verify_ppm_file(f): - logging.warn("Found corrupt PPM file: %s", f) - - # Should we convert PPM files to PNG format? - if params.get("convert_ppm_files_to_png") == "yes": - logging.debug("'convert_ppm_files_to_png' specified; converting PPM " - "files to PNG format...") - try: - for f in glob.glob(os.path.join(test.debugdir, "*.ppm")): - if ppm_utils.image_verify_ppm_file(f): - new_path = f.replace(".ppm", ".png") - image = PIL.Image.open(f) - image.save(new_path, format='PNG') - except NameError: - pass - - # Should we keep the PPM files? - if params.get("keep_ppm_files") != "yes": - logging.debug("'keep_ppm_files' not specified; removing all PPM files " - "from debug dir...") - for f in glob.glob(os.path.join(test.debugdir, '*.ppm')): - os.unlink(f) - - # Should we keep the screendump dirs? - if params.get("keep_screendumps") != "yes": - logging.debug("'keep_screendumps' not specified; removing screendump " - "dirs...") - for d in glob.glob(os.path.join(test.debugdir, "screendumps_*")): - if os.path.isdir(d) and not os.path.islink(d): - shutil.rmtree(d, ignore_errors=True) - - # Kill all unresponsive VMs - if params.get("kill_unresponsive_vms") == "yes": - logging.debug("'kill_unresponsive_vms' specified; killing all VMs " - "that fail to respond to a remote login request...") - for vm in env.get_all_vms(): - if vm.is_alive(): - try: - session = vm.login() - session.close() - except (kvm_utils.LoginError, kvm_vm.VMError), e: - logging.warn(e) - vm.destroy(gracefully=False) - - # Kill all kvm_subprocess tail threads - kvm_subprocess.kill_tail_threads() - - # Terminate tcpdump if no VMs are alive - living_vms = [vm for vm in env.get_all_vms() if vm.is_alive()] - if not living_vms and "tcpdump" in env: - env["tcpdump"].close() - del env["tcpdump"] - - if params.get("setup_hugepages") == "yes": - h = test_setup.HugePageConfig(params) - h.cleanup() - - # Execute any post_commands - if params.get("post_command"): - process_command(test, params, env, params.get("post_command"), - int(params.get("post_command_timeout", "600")), - params.get("post_command_noncritical") == "yes") - - -def postprocess_on_error(test, params, env): - """ - Perform postprocessing operations required only if the test failed. - - @param test: An Autotest test object. - @param params: A dict containing all VM and image parameters. - @param env: The environment (a dict-like object). - """ - params.update(params.object_params("on_error")) - - -def _update_address_cache(address_cache, line): - if re.search("Your.IP", line, re.IGNORECASE): - matches = re.findall(r"\d*\.\d*\.\d*\.\d*", line) - if matches: - address_cache["last_seen"] = matches[0] - if re.search("Client.Ethernet.Address", line, re.IGNORECASE): - matches = re.findall(r"\w*:\w*:\w*:\w*:\w*:\w*", line) - if matches and address_cache.get("last_seen"): - mac_address = matches[0].lower() - if time.time() - address_cache.get("time_%s" % mac_address, 0) > 5: - logging.debug("(address cache) Adding cache entry: %s ---> %s", - mac_address, address_cache.get("last_seen")) - address_cache[mac_address] = address_cache.get("last_seen") - address_cache["time_%s" % mac_address] = time.time() - del address_cache["last_seen"] - - -def _take_screendumps(test, params, env): - global _screendump_thread_termination_event - temp_dir = test.debugdir - if params.get("screendump_temp_dir"): - temp_dir = kvm_utils.get_path(test.bindir, - params.get("screendump_temp_dir")) - try: - os.makedirs(temp_dir) - except OSError: - pass - temp_filename = os.path.join(temp_dir, "scrdump-%s.ppm" % - kvm_utils.generate_random_string(6)) - delay = float(params.get("screendump_delay", 5)) - quality = int(params.get("screendump_quality", 30)) - - cache = {} - - while True: - for vm in env.get_all_vms(): - if not vm.is_alive(): - continue - try: - vm.monitor.screendump(filename=temp_filename, debug=False) - except kvm_monitor.MonitorError, e: - logging.warn(e) - continue - if not os.path.exists(temp_filename): - logging.warn("VM '%s' failed to produce a screendump", vm.name) - continue - if not ppm_utils.image_verify_ppm_file(temp_filename): - logging.warn("VM '%s' produced an invalid screendump", vm.name) - os.unlink(temp_filename) - continue - screendump_dir = os.path.join(test.debugdir, - "screendumps_%s" % vm.name) - try: - os.makedirs(screendump_dir) - except OSError: - pass - screendump_filename = os.path.join(screendump_dir, - "%s_%s.jpg" % (vm.name, - time.strftime("%Y-%m-%d_%H-%M-%S"))) - hash = utils.hash_file(temp_filename) - if hash in cache: - try: - os.link(cache[hash], screendump_filename) - except OSError: - pass - else: - try: - image = PIL.Image.open(temp_filename) - image.save(screendump_filename, format="JPEG", quality=quality) - cache[hash] = screendump_filename - except NameError: - pass - os.unlink(temp_filename) - if _screendump_thread_termination_event.isSet(): - _screendump_thread_termination_event = None - break - _screendump_thread_termination_event.wait(delay) diff --git a/client/tests/kvm/kvm_scheduler.py b/client/tests/kvm/kvm_scheduler.py deleted file mode 100644 index b96bb327..00000000 --- a/client/tests/kvm/kvm_scheduler.py +++ /dev/null @@ -1,229 +0,0 @@ -import os, select -import kvm_utils, kvm_vm, kvm_subprocess - - -class scheduler: - """ - A scheduler that manages several parallel test execution pipelines on a - single host. - """ - - def __init__(self, tests, num_workers, total_cpus, total_mem, bindir): - """ - Initialize the class. - - @param tests: A list of test dictionaries. - @param num_workers: The number of workers (pipelines). - @param total_cpus: The total number of CPUs to dedicate to tests. - @param total_mem: The total amount of memory to dedicate to tests. - @param bindir: The directory where environment files reside. - """ - self.tests = tests - self.num_workers = num_workers - self.total_cpus = total_cpus - self.total_mem = total_mem - self.bindir = bindir - # Pipes -- s stands for scheduler, w stands for worker - self.s2w = [os.pipe() for i in range(num_workers)] - self.w2s = [os.pipe() for i in range(num_workers)] - self.s2w_r = [os.fdopen(r, "r", 0) for r, w in self.s2w] - self.s2w_w = [os.fdopen(w, "w", 0) for r, w in self.s2w] - self.w2s_r = [os.fdopen(r, "r", 0) for r, w in self.w2s] - self.w2s_w = [os.fdopen(w, "w", 0) for r, w in self.w2s] - # "Personal" worker dicts contain modifications that are applied - # specifically to each worker. For example, each worker must use a - # different environment file and a different MAC address pool. - self.worker_dicts = [{"env": "env%d" % i} for i in range(num_workers)] - - - def worker(self, index, run_test_func): - """ - The worker function. - - Waits for commands from the scheduler and processes them. - - @param index: The index of this worker (in the range 0..num_workers-1). - @param run_test_func: A function to be called to run a test - (e.g. job.run_test). - """ - r = self.s2w_r[index] - w = self.w2s_w[index] - self_dict = self.worker_dicts[index] - - # Inform the scheduler this worker is ready - w.write("ready\n") - - while True: - cmd = r.readline().split() - if not cmd: - continue - - # The scheduler wants this worker to run a test - if cmd[0] == "run": - test_index = int(cmd[1]) - test = self.tests[test_index].copy() - test.update(self_dict) - test_iterations = int(test.get("iterations", 1)) - status = run_test_func("kvm", params=test, - tag=test.get("shortname"), - iterations=test_iterations) - w.write("done %s %s\n" % (test_index, status)) - w.write("ready\n") - - # The scheduler wants this worker to free its used resources - elif cmd[0] == "cleanup": - env_filename = os.path.join(self.bindir, self_dict["env"]) - env = kvm_utils.Env(env_filename) - for obj in env.values(): - if isinstance(obj, kvm_vm.VM): - obj.destroy() - elif isinstance(obj, kvm_subprocess.Spawn): - obj.close() - env.save() - w.write("cleanup_done\n") - w.write("ready\n") - - # There's no more work for this worker - elif cmd[0] == "terminate": - break - - - def scheduler(self): - """ - The scheduler function. - - Sends commands to workers, telling them to run tests, clean up or - terminate execution. - """ - idle_workers = [] - closing_workers = [] - test_status = ["waiting"] * len(self.tests) - test_worker = [None] * len(self.tests) - used_cpus = [0] * self.num_workers - used_mem = [0] * self.num_workers - - while True: - # Wait for a message from a worker - r, w, x = select.select(self.w2s_r, [], []) - - someone_is_ready = False - - for pipe in r: - worker_index = self.w2s_r.index(pipe) - msg = pipe.readline().split() - if not msg: - continue - - # A worker is ready -- add it to the idle_workers list - if msg[0] == "ready": - idle_workers.append(worker_index) - someone_is_ready = True - - # A worker completed a test - elif msg[0] == "done": - test_index = int(msg[1]) - test = self.tests[test_index] - status = int(eval(msg[2])) - test_status[test_index] = ("fail", "pass")[status] - # If the test failed, mark all dependent tests as "failed" too - if not status: - for i, other_test in enumerate(self.tests): - for dep in other_test.get("dep", []): - if dep in test["name"]: - test_status[i] = "fail" - - # A worker is done shutting down its VMs and other processes - elif msg[0] == "cleanup_done": - used_cpus[worker_index] = 0 - used_mem[worker_index] = 0 - closing_workers.remove(worker_index) - - if not someone_is_ready: - continue - - for worker in idle_workers[:]: - # Find a test for this worker - test_found = False - for i, test in enumerate(self.tests): - # We only want "waiting" tests - if test_status[i] != "waiting": - continue - # Make sure the test isn't assigned to another worker - if test_worker[i] is not None and test_worker[i] != worker: - continue - # Make sure the test's dependencies are satisfied - dependencies_satisfied = True - for dep in test["dep"]: - dependencies = [j for j, t in enumerate(self.tests) - if dep in t["name"]] - bad_status_deps = [j for j in dependencies - if test_status[j] != "pass"] - if bad_status_deps: - dependencies_satisfied = False - break - if not dependencies_satisfied: - continue - # Make sure we have enough resources to run the test - test_used_cpus = int(test.get("used_cpus", 1)) - test_used_mem = int(test.get("used_mem", 128)) - # First make sure the other workers aren't using too many - # CPUs (not including the workers currently shutting down) - uc = (sum(used_cpus) - used_cpus[worker] - - sum(used_cpus[i] for i in closing_workers)) - if uc and uc + test_used_cpus > self.total_cpus: - continue - # ... or too much memory - um = (sum(used_mem) - used_mem[worker] - - sum(used_mem[i] for i in closing_workers)) - if um and um + test_used_mem > self.total_mem: - continue - # If we reached this point it means there are, or will - # soon be, enough resources to run the test - test_found = True - # Now check if the test can be run right now, i.e. if the - # other workers, including the ones currently shutting - # down, aren't using too many CPUs - uc = (sum(used_cpus) - used_cpus[worker]) - if uc and uc + test_used_cpus > self.total_cpus: - continue - # ... or too much memory - um = (sum(used_mem) - used_mem[worker]) - if um and um + test_used_mem > self.total_mem: - continue - # Everything is OK -- run the test - test_status[i] = "running" - test_worker[i] = worker - idle_workers.remove(worker) - # Update used_cpus and used_mem - used_cpus[worker] = test_used_cpus - used_mem[worker] = test_used_mem - # Assign all related tests to this worker - for j, other_test in enumerate(self.tests): - for other_dep in other_test["dep"]: - # All tests that depend on this test - if other_dep in test["name"]: - test_worker[j] = worker - break - # ... and all tests that share a dependency - # with this test - for dep in test["dep"]: - if dep in other_dep or other_dep in dep: - test_worker[j] = worker - break - # Tell the worker to run the test - self.s2w_w[worker].write("run %s\n" % i) - break - - # If there won't be any tests for this worker to run soon, tell - # the worker to free its used resources - if not test_found and (used_cpus[worker] or used_mem[worker]): - self.s2w_w[worker].write("cleanup\n") - idle_workers.remove(worker) - closing_workers.append(worker) - - # If there are no more new tests to run, terminate the workers and - # the scheduler - if len(idle_workers) == self.num_workers: - for worker in idle_workers: - self.s2w_w[worker].write("terminate\n") - break diff --git a/client/tests/kvm/kvm_subprocess.py b/client/tests/kvm/kvm_subprocess.py deleted file mode 100755 index 0b8734f7..00000000 --- a/client/tests/kvm/kvm_subprocess.py +++ /dev/null @@ -1,1351 +0,0 @@ -#!/usr/bin/python -""" -A class and functions used for running and controlling child processes. - -@copyright: 2008-2009 Red Hat Inc. -""" - -import os, sys, pty, select, termios, fcntl - - -# The following helper functions are shared by the server and the client. - -def _lock(filename): - if not os.path.exists(filename): - open(filename, "w").close() - fd = os.open(filename, os.O_RDWR) - fcntl.lockf(fd, fcntl.LOCK_EX) - return fd - - -def _unlock(fd): - fcntl.lockf(fd, fcntl.LOCK_UN) - os.close(fd) - - -def _locked(filename): - try: - fd = os.open(filename, os.O_RDWR) - except: - return False - try: - fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except: - os.close(fd) - return True - fcntl.lockf(fd, fcntl.LOCK_UN) - os.close(fd) - return False - - -def _wait(filename): - fd = _lock(filename) - _unlock(fd) - - -def _get_filenames(base_dir, id): - return [os.path.join(base_dir, s + id) for s in - "shell-pid-", "status-", "output-", "inpipe-", - "lock-server-running-", "lock-client-starting-"] - - -def _get_reader_filename(base_dir, id, reader): - return os.path.join(base_dir, "outpipe-%s-%s" % (reader, id)) - - -# The following is the server part of the module. - -if __name__ == "__main__": - id = sys.stdin.readline().strip() - echo = sys.stdin.readline().strip() == "True" - readers = sys.stdin.readline().strip().split(",") - command = sys.stdin.readline().strip() + " && echo %s > /dev/null" % id - - # Define filenames to be used for communication - base_dir = "/tmp/kvm_spawn" - (shell_pid_filename, - status_filename, - output_filename, - inpipe_filename, - lock_server_running_filename, - lock_client_starting_filename) = _get_filenames(base_dir, id) - - # Populate the reader filenames list - reader_filenames = [_get_reader_filename(base_dir, id, reader) - for reader in readers] - - # Set $TERM = dumb - os.putenv("TERM", "dumb") - - (shell_pid, shell_fd) = pty.fork() - if shell_pid == 0: - # Child process: run the command in a subshell - os.execv("/bin/sh", ["/bin/sh", "-c", command]) - else: - # Parent process - lock_server_running = _lock(lock_server_running_filename) - - # Set terminal echo on/off and disable pre- and post-processing - attr = termios.tcgetattr(shell_fd) - attr[0] &= ~termios.INLCR - attr[0] &= ~termios.ICRNL - attr[0] &= ~termios.IGNCR - attr[1] &= ~termios.OPOST - if echo: - attr[3] |= termios.ECHO - else: - attr[3] &= ~termios.ECHO - termios.tcsetattr(shell_fd, termios.TCSANOW, attr) - - # Open output file - output_file = open(output_filename, "w") - # Open input pipe - os.mkfifo(inpipe_filename) - inpipe_fd = os.open(inpipe_filename, os.O_RDWR) - # Open output pipes (readers) - reader_fds = [] - for filename in reader_filenames: - os.mkfifo(filename) - reader_fds.append(os.open(filename, os.O_RDWR)) - - # Write shell PID to file - file = open(shell_pid_filename, "w") - file.write(str(shell_pid)) - file.close() - - # Print something to stdout so the client can start working - print "Server %s ready" % id - sys.stdout.flush() - - # Initialize buffers - buffers = ["" for reader in readers] - - # Read from child and write to files/pipes - while True: - check_termination = False - # Make a list of reader pipes whose buffers are not empty - fds = [fd for (i, fd) in enumerate(reader_fds) if buffers[i]] - # Wait until there's something to do - r, w, x = select.select([shell_fd, inpipe_fd], fds, [], 0.5) - # If a reader pipe is ready for writing -- - for (i, fd) in enumerate(reader_fds): - if fd in w: - bytes_written = os.write(fd, buffers[i]) - buffers[i] = buffers[i][bytes_written:] - # If there's data to read from the child process -- - if shell_fd in r: - try: - data = os.read(shell_fd, 16384) - except OSError: - data = "" - if not data: - check_termination = True - # Remove carriage returns from the data -- they often cause - # trouble and are normally not needed - data = data.replace("\r", "") - output_file.write(data) - output_file.flush() - for i in range(len(readers)): - buffers[i] += data - # If os.read() raised an exception or there was nothing to read -- - if check_termination or shell_fd not in r: - pid, status = os.waitpid(shell_pid, os.WNOHANG) - if pid: - status = os.WEXITSTATUS(status) - break - # If there's data to read from the client -- - if inpipe_fd in r: - data = os.read(inpipe_fd, 1024) - os.write(shell_fd, data) - - # Write the exit status to a file - file = open(status_filename, "w") - file.write(str(status)) - file.close() - - # Wait for the client to finish initializing - _wait(lock_client_starting_filename) - - # Delete FIFOs - for filename in reader_filenames + [inpipe_filename]: - try: - os.unlink(filename) - except OSError: - pass - - # Close all files and pipes - output_file.close() - os.close(inpipe_fd) - for fd in reader_fds: - os.close(fd) - - _unlock(lock_server_running) - exit(0) - - -# The following is the client part of the module. - -import subprocess, time, signal, re, threading, logging -import common, kvm_utils - - -class ExpectError(Exception): - def __init__(self, patterns, output): - Exception.__init__(self, patterns, output) - self.patterns = patterns - self.output = output - - def _pattern_str(self): - if len(self.patterns) == 1: - return "pattern %r" % self.patterns[0] - else: - return "patterns %r" % self.patterns - - def __str__(self): - return ("Unknown error occurred while looking for %s (output: %r)" % - (self._pattern_str(), self.output)) - - -class ExpectTimeoutError(ExpectError): - def __str__(self): - return ("Timeout expired while looking for %s (output: %r)" % - (self._pattern_str(), self.output)) - - -class ExpectProcessTerminatedError(ExpectError): - def __init__(self, patterns, status, output): - ExpectError.__init__(self, patterns, output) - self.status = status - - def __str__(self): - return ("Process terminated while looking for %s " - "(status: %s, output: %r)" % (self._pattern_str(), - self.status, self.output)) - - -class ShellError(Exception): - def __init__(self, cmd, output): - Exception.__init__(self, cmd, output) - self.cmd = cmd - self.output = output - - def __str__(self): - return ("Could not execute shell command %r (output: %r)" % - (self.cmd, self.output)) - - -class ShellTimeoutError(ShellError): - def __str__(self): - return ("Timeout expired while waiting for shell command to " - "complete: %r (output: %r)" % (self.cmd, self.output)) - - -class ShellProcessTerminatedError(ShellError): - # Raised when the shell process itself (e.g. ssh, netcat, telnet) - # terminates unexpectedly - def __init__(self, cmd, status, output): - ShellError.__init__(self, cmd, output) - self.status = status - - def __str__(self): - return ("Shell process terminated while waiting for command to " - "complete: %r (status: %s, output: %r)" % - (self.cmd, self.status, self.output)) - - -class ShellCmdError(ShellError): - # Raised when a command executed in a shell terminates with a nonzero - # exit code (status) - def __init__(self, cmd, status, output): - ShellError.__init__(self, cmd, output) - self.status = status - - def __str__(self): - return ("Shell command failed: %r (status: %s, output: %r)" % - (self.cmd, self.status, self.output)) - - -class ShellStatusError(ShellError): - # Raised when the command's exit status cannot be obtained - def __str__(self): - return ("Could not get exit status of command: %r (output: %r)" % - (self.cmd, self.output)) - - -def run_bg(command, termination_func=None, output_func=None, output_prefix="", - timeout=1.0): - """ - Run command as a subprocess. Call output_func with each line of output - from the subprocess (prefixed by output_prefix). Call termination_func - when the subprocess terminates. Return when timeout expires or when the - subprocess exits -- whichever occurs first. - - @brief: Run a subprocess in the background and collect its output and - exit status. - - @param command: The shell command to execute - @param termination_func: A function to call when the process terminates - (should take an integer exit status parameter) - @param output_func: A function to call with each line of output from - the subprocess (should take a string parameter) - @param output_prefix: A string to pre-pend to each line of the output, - before passing it to stdout_func - @param timeout: Time duration (in seconds) to wait for the subprocess to - terminate before returning - - @return: A Tail object. - """ - process = Tail(command=command, - termination_func=termination_func, - output_func=output_func, - output_prefix=output_prefix) - - end_time = time.time() + timeout - while time.time() < end_time and process.is_alive(): - time.sleep(0.1) - - return process - - -def run_fg(command, output_func=None, output_prefix="", timeout=1.0): - """ - Run command as a subprocess. Call output_func with each line of output - from the subprocess (prefixed by prefix). Return when timeout expires or - when the subprocess exits -- whichever occurs first. If timeout expires - and the subprocess is still running, kill it before returning. - - @brief: Run a subprocess in the foreground and collect its output and - exit status. - - @param command: The shell command to execute - @param output_func: A function to call with each line of output from - the subprocess (should take a string parameter) - @param output_prefix: A string to pre-pend to each line of the output, - before passing it to stdout_func - @param timeout: Time duration (in seconds) to wait for the subprocess to - terminate before killing it and returning - - @return: A 2-tuple containing the exit status of the process and its - STDOUT/STDERR output. If timeout expires before the process - terminates, the returned status is None. - """ - process = run_bg(command, None, output_func, output_prefix, timeout) - output = process.get_output() - if process.is_alive(): - status = None - else: - status = process.get_status() - process.close() - return (status, output) - - -class Spawn: - """ - This class is used for spawning and controlling a child process. - - A new instance of this class can either run a new server (a small Python - program that reads output from the child process and reports it to the - client and to a text file) or attach to an already running server. - When a server is started it runs the child process. - The server writes output from the child's STDOUT and STDERR to a text file. - The text file can be accessed at any time using get_output(). - In addition, the server opens as many pipes as requested by the client and - writes the output to them. - The pipes are requested and accessed by classes derived from Spawn. - These pipes are referred to as "readers". - The server also receives input from the client and sends it to the child - process. - An instance of this class can be pickled. Every derived class is - responsible for restoring its own state by properly defining - __getinitargs__(). - - The first named pipe is used by _tail(), a function that runs in the - background and reports new output from the child as it is produced. - The second named pipe is used by a set of functions that read and parse - output as requested by the user in an interactive manner, similar to - pexpect. - When unpickled it automatically - resumes _tail() if needed. - """ - - def __init__(self, command=None, id=None, auto_close=False, echo=False, - linesep="\n"): - """ - Initialize the class and run command as a child process. - - @param command: Command to run, or None if accessing an already running - server. - @param id: ID of an already running server, if accessing a running - server, or None if starting a new one. - @param auto_close: If True, close() the instance automatically when its - reference count drops to zero (default False). - @param echo: Boolean indicating whether echo should be initially - enabled for the pseudo terminal running the subprocess. This - parameter has an effect only when starting a new server. - @param linesep: Line separator to be appended to strings sent to the - child process by sendline(). - """ - self.id = id or kvm_utils.generate_random_string(8) - - # Define filenames for communication with server - base_dir = "/tmp/kvm_spawn" - try: - os.makedirs(base_dir) - except: - pass - (self.shell_pid_filename, - self.status_filename, - self.output_filename, - self.inpipe_filename, - self.lock_server_running_filename, - self.lock_client_starting_filename) = _get_filenames(base_dir, - self.id) - - # Remember some attributes - self.auto_close = auto_close - self.echo = echo - self.linesep = linesep - - # Make sure the 'readers' and 'close_hooks' attributes exist - if not hasattr(self, "readers"): - self.readers = [] - if not hasattr(self, "close_hooks"): - self.close_hooks = [] - - # Define the reader filenames - self.reader_filenames = dict( - (reader, _get_reader_filename(base_dir, self.id, reader)) - for reader in self.readers) - - # Let the server know a client intends to open some pipes; - # if the executed command terminates quickly, the server will wait for - # the client to release the lock before exiting - lock_client_starting = _lock(self.lock_client_starting_filename) - - # Start the server (which runs the command) - if command: - sub = subprocess.Popen("%s %s" % (sys.executable, __file__), - shell=True, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - # Send parameters to the server - sub.stdin.write("%s\n" % self.id) - sub.stdin.write("%s\n" % echo) - sub.stdin.write("%s\n" % ",".join(self.readers)) - sub.stdin.write("%s\n" % command) - # Wait for the server to complete its initialization - while not "Server %s ready" % self.id in sub.stdout.readline(): - pass - - # Open the reading pipes - self.reader_fds = {} - try: - assert(_locked(self.lock_server_running_filename)) - for reader, filename in self.reader_filenames.items(): - self.reader_fds[reader] = os.open(filename, os.O_RDONLY) - except: - pass - - # Allow the server to continue - _unlock(lock_client_starting) - - - # The following two functions are defined to make sure the state is set - # exclusively by the constructor call as specified in __getinitargs__(). - - def __getstate__(self): - pass - - - def __setstate__(self, state): - pass - - - def __getinitargs__(self): - # Save some information when pickling -- will be passed to the - # constructor upon unpickling - return (None, self.id, self.auto_close, self.echo, self.linesep) - - - def __del__(self): - if self.auto_close: - self.close() - - - def _add_reader(self, reader): - """ - Add a reader whose file descriptor can be obtained with _get_fd(). - Should be called before __init__(). Intended for use by derived - classes. - - @param reader: The name of the reader. - """ - if not hasattr(self, "readers"): - self.readers = [] - self.readers.append(reader) - - - def _add_close_hook(self, hook): - """ - Add a close hook function to be called when close() is called. - The function will be called after the process terminates but before - final cleanup. Intended for use by derived classes. - - @param hook: The hook function. - """ - if not hasattr(self, "close_hooks"): - self.close_hooks = [] - self.close_hooks.append(hook) - - - def _get_fd(self, reader): - """ - Return an open file descriptor corresponding to the specified reader - pipe. If no such reader exists, or the pipe could not be opened, - return None. Intended for use by derived classes. - - @param reader: The name of the reader. - """ - return self.reader_fds.get(reader) - - - def get_id(self): - """ - Return the instance's id attribute, which may be used to access the - process in the future. - """ - return self.id - - - def get_pid(self): - """ - Return the PID of the process. - - Note: this may be the PID of the shell process running the user given - command. - """ - try: - file = open(self.shell_pid_filename, "r") - pid = int(file.read()) - file.close() - return pid - except: - return None - - - def get_status(self): - """ - Wait for the process to exit and return its exit status, or None - if the exit status is not available. - """ - _wait(self.lock_server_running_filename) - try: - file = open(self.status_filename, "r") - status = int(file.read()) - file.close() - return status - except: - return None - - - def get_output(self): - """ - Return the STDOUT and STDERR output of the process so far. - """ - try: - file = open(self.output_filename, "r") - output = file.read() - file.close() - return output - except: - return "" - - - def is_alive(self): - """ - Return True if the process is running. - """ - return _locked(self.lock_server_running_filename) - - - def close(self, sig=signal.SIGKILL): - """ - Kill the child process if it's alive and remove temporary files. - - @param sig: The signal to send the process when attempting to kill it. - """ - # Kill it if it's alive - if self.is_alive(): - kvm_utils.kill_process_tree(self.get_pid(), sig) - # Wait for the server to exit - _wait(self.lock_server_running_filename) - # Call all cleanup routines - for hook in self.close_hooks: - hook(self) - # Close reader file descriptors - for fd in self.reader_fds.values(): - try: - os.close(fd) - except: - pass - self.reader_fds = {} - # Remove all used files - for filename in (_get_filenames("/tmp/kvm_spawn", self.id) + - self.reader_filenames.values()): - try: - os.unlink(filename) - except OSError: - pass - - - def set_linesep(self, linesep): - """ - Sets the line separator string (usually "\\n"). - - @param linesep: Line separator string. - """ - self.linesep = linesep - - - def send(self, str=""): - """ - Send a string to the child process. - - @param str: String to send to the child process. - """ - try: - fd = os.open(self.inpipe_filename, os.O_RDWR) - os.write(fd, str) - os.close(fd) - except: - pass - - - def sendline(self, str=""): - """ - Send a string followed by a line separator to the child process. - - @param str: String to send to the child process. - """ - self.send(str + self.linesep) - - -_thread_kill_requested = False - -def kill_tail_threads(): - """ - Kill all Tail threads. - - After calling this function no new threads should be started. - """ - global _thread_kill_requested - _thread_kill_requested = True - for t in threading.enumerate(): - if hasattr(t, "name") and t.name.startswith("tail_thread"): - t.join(10) - _thread_kill_requested = False - - -class Tail(Spawn): - """ - This class runs a child process in the background and sends its output in - real time, line-by-line, to a callback function. - - See Spawn's docstring. - - This class uses a single pipe reader to read data in real time from the - child process and report it to a given callback function. - When the child process exits, its exit status is reported to an additional - callback function. - - When this class is unpickled, it automatically resumes reporting output. - """ - - def __init__(self, command=None, id=None, auto_close=False, echo=False, - linesep="\n", termination_func=None, termination_params=(), - output_func=None, output_params=(), output_prefix=""): - """ - Initialize the class and run command as a child process. - - @param command: Command to run, or None if accessing an already running - server. - @param id: ID of an already running server, if accessing a running - server, or None if starting a new one. - @param auto_close: If True, close() the instance automatically when its - reference count drops to zero (default False). - @param echo: Boolean indicating whether echo should be initially - enabled for the pseudo terminal running the subprocess. This - parameter has an effect only when starting a new server. - @param linesep: Line separator to be appended to strings sent to the - child process by sendline(). - @param termination_func: Function to call when the process exits. The - function must accept a single exit status parameter. - @param termination_params: Parameters to send to termination_func - before the exit status. - @param output_func: Function to call whenever a line of output is - available from the STDOUT or STDERR streams of the process. - The function must accept a single string parameter. The string - does not include the final newline. - @param output_params: Parameters to send to output_func before the - output line. - @param output_prefix: String to prepend to lines sent to output_func. - """ - # Add a reader and a close hook - self._add_reader("tail") - self._add_close_hook(Tail._join_thread) - - # Init the superclass - Spawn.__init__(self, command, id, auto_close, echo, linesep) - - # Remember some attributes - self.termination_func = termination_func - self.termination_params = termination_params - self.output_func = output_func - self.output_params = output_params - self.output_prefix = output_prefix - - # Start the thread in the background - self.tail_thread = None - if termination_func or output_func: - self._start_thread() - - - def __getinitargs__(self): - return Spawn.__getinitargs__(self) + (self.termination_func, - self.termination_params, - self.output_func, - self.output_params, - self.output_prefix) - - - def set_termination_func(self, termination_func): - """ - Set the termination_func attribute. See __init__() for details. - - @param termination_func: Function to call when the process terminates. - Must take a single parameter -- the exit status. - """ - self.termination_func = termination_func - if termination_func and not self.tail_thread: - self._start_thread() - - - def set_termination_params(self, termination_params): - """ - Set the termination_params attribute. See __init__() for details. - - @param termination_params: Parameters to send to termination_func - before the exit status. - """ - self.termination_params = termination_params - - - def set_output_func(self, output_func): - """ - Set the output_func attribute. See __init__() for details. - - @param output_func: Function to call for each line of STDOUT/STDERR - output from the process. Must take a single string parameter. - """ - self.output_func = output_func - if output_func and not self.tail_thread: - self._start_thread() - - - def set_output_params(self, output_params): - """ - Set the output_params attribute. See __init__() for details. - - @param output_params: Parameters to send to output_func before the - output line. - """ - self.output_params = output_params - - - def set_output_prefix(self, output_prefix): - """ - Set the output_prefix attribute. See __init__() for details. - - @param output_prefix: String to pre-pend to each line sent to - output_func (see set_output_callback()). - """ - self.output_prefix = output_prefix - - - def _tail(self): - def print_line(text): - # Pre-pend prefix and remove trailing whitespace - text = self.output_prefix + text.rstrip() - # Pass text to output_func - try: - params = self.output_params + (text,) - self.output_func(*params) - except TypeError: - pass - - try: - fd = self._get_fd("tail") - buffer = "" - while True: - global _thread_kill_requested - if _thread_kill_requested: - return - try: - # See if there's any data to read from the pipe - r, w, x = select.select([fd], [], [], 0.05) - except: - break - if fd in r: - # Some data is available; read it - new_data = os.read(fd, 1024) - if not new_data: - break - buffer += new_data - # Send the output to output_func line by line - # (except for the last line) - if self.output_func: - lines = buffer.split("\n") - for line in lines[:-1]: - print_line(line) - # Leave only the last line - last_newline_index = buffer.rfind("\n") - buffer = buffer[last_newline_index+1:] - else: - # No output is available right now; flush the buffer - if buffer: - print_line(buffer) - buffer = "" - # The process terminated; print any remaining output - if buffer: - print_line(buffer) - # Get the exit status, print it and send it to termination_func - status = self.get_status() - if status is None: - return - print_line("(Process terminated with status %s)" % status) - try: - params = self.termination_params + (status,) - self.termination_func(*params) - except TypeError: - pass - finally: - self.tail_thread = None - - - def _start_thread(self): - self.tail_thread = threading.Thread(target=self._tail, - name="tail_thread_%s" % self.id) - self.tail_thread.start() - - - def _join_thread(self): - # Wait for the tail thread to exit - # (it's done this way because self.tail_thread may become None at any - # time) - t = self.tail_thread - if t: - t.join() - - -class Expect(Tail): - """ - This class runs a child process in the background and provides expect-like - services. - - It also provides all of Tail's functionality. - """ - - def __init__(self, command=None, id=None, auto_close=True, echo=False, - linesep="\n", termination_func=None, termination_params=(), - output_func=None, output_params=(), output_prefix=""): - """ - Initialize the class and run command as a child process. - - @param command: Command to run, or None if accessing an already running - server. - @param id: ID of an already running server, if accessing a running - server, or None if starting a new one. - @param auto_close: If True, close() the instance automatically when its - reference count drops to zero (default False). - @param echo: Boolean indicating whether echo should be initially - enabled for the pseudo terminal running the subprocess. This - parameter has an effect only when starting a new server. - @param linesep: Line separator to be appended to strings sent to the - child process by sendline(). - @param termination_func: Function to call when the process exits. The - function must accept a single exit status parameter. - @param termination_params: Parameters to send to termination_func - before the exit status. - @param output_func: Function to call whenever a line of output is - available from the STDOUT or STDERR streams of the process. - The function must accept a single string parameter. The string - does not include the final newline. - @param output_params: Parameters to send to output_func before the - output line. - @param output_prefix: String to prepend to lines sent to output_func. - """ - # Add a reader - self._add_reader("expect") - - # Init the superclass - Tail.__init__(self, command, id, auto_close, echo, linesep, - termination_func, termination_params, - output_func, output_params, output_prefix) - - - def __getinitargs__(self): - return Tail.__getinitargs__(self) - - - def read_nonblocking(self, timeout=None): - """ - Read from child until there is nothing to read for timeout seconds. - - @param timeout: Time (seconds) to wait before we give up reading from - the child process, or None to use the default value. - """ - if timeout is None: - timeout = 0.1 - fd = self._get_fd("expect") - data = "" - while True: - try: - r, w, x = select.select([fd], [], [], timeout) - except: - return data - if fd in r: - new_data = os.read(fd, 1024) - if not new_data: - return data - data += new_data - else: - return data - - - def match_patterns(self, str, patterns): - """ - Match str against a list of patterns. - - Return the index of the first pattern that matches a substring of str. - None and empty strings in patterns are ignored. - If no match is found, return None. - - @param patterns: List of strings (regular expression patterns). - """ - for i in range(len(patterns)): - if not patterns[i]: - continue - if re.search(patterns[i], str): - return i - - - def read_until_output_matches(self, patterns, filter=lambda x: x, - timeout=60, internal_timeout=None, - print_func=None): - """ - Read using read_nonblocking until a match is found using match_patterns, - or until timeout expires. Before attempting to search for a match, the - data is filtered using the filter function provided. - - @brief: Read from child using read_nonblocking until a pattern - matches. - @param patterns: List of strings (regular expression patterns) - @param filter: Function to apply to the data read from the child before - attempting to match it against the patterns (should take and - return a string) - @param timeout: The duration (in seconds) to wait until a match is - found - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - @return: Tuple containing the match index and the data read so far - @raise ExpectTimeoutError: Raised if timeout expires - @raise ExpectProcessTerminatedError: Raised if the child process - terminates while waiting for output - @raise ExpectError: Raised if an unknown error occurs - """ - fd = self._get_fd("expect") - o = "" - end_time = time.time() + timeout - while True: - try: - r, w, x = select.select([fd], [], [], - max(0, end_time - time.time())) - except (select.error, TypeError): - break - if not r: - raise ExpectTimeoutError(patterns, o) - # Read data from child - data = self.read_nonblocking(internal_timeout) - if not data: - break - # Print it if necessary - if print_func: - for line in data.splitlines(): - print_func(line) - # Look for patterns - o += data - match = self.match_patterns(filter(o), patterns) - if match is not None: - return match, o - - # Check if the child has terminated - if kvm_utils.wait_for(lambda: not self.is_alive(), 5, 0, 0.1): - raise ExpectProcessTerminatedError(patterns, self.get_status(), o) - else: - # This shouldn't happen - raise ExpectError(patterns, o) - - - def read_until_last_word_matches(self, patterns, timeout=60, - internal_timeout=None, print_func=None): - """ - Read using read_nonblocking until the last word of the output matches - one of the patterns (using match_patterns), or until timeout expires. - - @param patterns: A list of strings (regular expression patterns) - @param timeout: The duration (in seconds) to wait until a match is - found - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - @return: A tuple containing the match index and the data read so far - @raise ExpectTimeoutError: Raised if timeout expires - @raise ExpectProcessTerminatedError: Raised if the child process - terminates while waiting for output - @raise ExpectError: Raised if an unknown error occurs - """ - def get_last_word(str): - if str: - return str.split()[-1] - else: - return "" - - return self.read_until_output_matches(patterns, get_last_word, - timeout, internal_timeout, - print_func) - - - def read_until_last_line_matches(self, patterns, timeout=60, - internal_timeout=None, print_func=None): - """ - Read using read_nonblocking until the last non-empty line of the output - matches one of the patterns (using match_patterns), or until timeout - expires. Return a tuple containing the match index (or None if no match - was found) and the data read so far. - - @brief: Read using read_nonblocking until the last non-empty line - matches a pattern. - - @param patterns: A list of strings (regular expression patterns) - @param timeout: The duration (in seconds) to wait until a match is - found - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - @return: A tuple containing the match index and the data read so far - @raise ExpectTimeoutError: Raised if timeout expires - @raise ExpectProcessTerminatedError: Raised if the child process - terminates while waiting for output - @raise ExpectError: Raised if an unknown error occurs - """ - def get_last_nonempty_line(str): - nonempty_lines = [l for l in str.splitlines() if l.strip()] - if nonempty_lines: - return nonempty_lines[-1] - else: - return "" - - return self.read_until_output_matches(patterns, get_last_nonempty_line, - timeout, internal_timeout, - print_func) - - -class ShellSession(Expect): - """ - This class runs a child process in the background. It it suited for - processes that provide an interactive shell, such as SSH and Telnet. - - It provides all services of Expect and Tail. In addition, it - provides command running services, and a utility function to test the - process for responsiveness. - """ - - def __init__(self, command=None, id=None, auto_close=True, echo=False, - linesep="\n", termination_func=None, termination_params=(), - output_func=None, output_params=(), output_prefix="", - prompt=r"[\#\$]\s*$", status_test_command="echo $?"): - """ - Initialize the class and run command as a child process. - - @param command: Command to run, or None if accessing an already running - server. - @param id: ID of an already running server, if accessing a running - server, or None if starting a new one. - @param auto_close: If True, close() the instance automatically when its - reference count drops to zero (default True). - @param echo: Boolean indicating whether echo should be initially - enabled for the pseudo terminal running the subprocess. This - parameter has an effect only when starting a new server. - @param linesep: Line separator to be appended to strings sent to the - child process by sendline(). - @param termination_func: Function to call when the process exits. The - function must accept a single exit status parameter. - @param termination_params: Parameters to send to termination_func - before the exit status. - @param output_func: Function to call whenever a line of output is - available from the STDOUT or STDERR streams of the process. - The function must accept a single string parameter. The string - does not include the final newline. - @param output_params: Parameters to send to output_func before the - output line. - @param output_prefix: String to prepend to lines sent to output_func. - @param prompt: Regular expression describing the shell's prompt line. - @param status_test_command: Command to be used for getting the last - exit status of commands run inside the shell (used by - cmd_status_output() and friends). - """ - # Init the superclass - Expect.__init__(self, command, id, auto_close, echo, linesep, - termination_func, termination_params, - output_func, output_params, output_prefix) - - # Remember some attributes - self.prompt = prompt - self.status_test_command = status_test_command - - - def __getinitargs__(self): - return Expect.__getinitargs__(self) + (self.prompt, - self.status_test_command) - - - def set_prompt(self, prompt): - """ - Set the prompt attribute for later use by read_up_to_prompt. - - @param: String that describes the prompt contents. - """ - self.prompt = prompt - - - def set_status_test_command(self, status_test_command): - """ - Set the command to be sent in order to get the last exit status. - - @param status_test_command: Command that will be sent to get the last - exit status. - """ - self.status_test_command = status_test_command - - - def is_responsive(self, timeout=5.0): - """ - Return True if the process responds to STDIN/terminal input. - - Send a newline to the child process (e.g. SSH or Telnet) and read some - output using read_nonblocking(). - If all is OK, some output should be available (e.g. the shell prompt). - In that case return True. Otherwise return False. - - @param timeout: Time duration to wait before the process is considered - unresponsive. - """ - # Read all output that's waiting to be read, to make sure the output - # we read next is in response to the newline sent - self.read_nonblocking(timeout=0) - # Send a newline - self.sendline() - # Wait up to timeout seconds for some output from the child - end_time = time.time() + timeout - while time.time() < end_time: - time.sleep(0.5) - if self.read_nonblocking(timeout=0).strip(): - return True - # No output -- report unresponsive - return False - - - def read_up_to_prompt(self, timeout=60, internal_timeout=None, - print_func=None): - """ - Read using read_nonblocking until the last non-empty line of the output - matches the prompt regular expression set by set_prompt, or until - timeout expires. - - @brief: Read using read_nonblocking until the last non-empty line - matches the prompt. - - @param timeout: The duration (in seconds) to wait until a match is - found - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being - read (should take a string parameter) - - @return: The data read so far - @raise ExpectTimeoutError: Raised if timeout expires - @raise ExpectProcessTerminatedError: Raised if the shell process - terminates while waiting for output - @raise ExpectError: Raised if an unknown error occurs - """ - m, o = self.read_until_last_line_matches([self.prompt], timeout, - internal_timeout, print_func) - return o - - - def cmd_output(self, cmd, timeout=60, internal_timeout=None, - print_func=None): - """ - Send a command and return its output. - - @param cmd: Command to send (must not contain newline characters) - @param timeout: The duration (in seconds) to wait for the prompt to - return - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - - @return: The output of cmd - @raise ShellTimeoutError: Raised if timeout expires - @raise ShellProcessTerminatedError: Raised if the shell process - terminates while waiting for output - @raise ShellError: Raised if an unknown error occurs - """ - def remove_command_echo(str, cmd): - if str and str.splitlines()[0] == cmd: - str = "".join(str.splitlines(True)[1:]) - return str - - def remove_last_nonempty_line(str): - return "".join(str.rstrip().splitlines(True)[:-1]) - - logging.debug("Sending command: %s" % cmd) - self.read_nonblocking(timeout=0) - self.sendline(cmd) - try: - o = self.read_up_to_prompt(timeout, internal_timeout, print_func) - except ExpectError, e: - o = remove_command_echo(e.output, cmd) - if isinstance(e, ExpectTimeoutError): - raise ShellTimeoutError(cmd, o) - elif isinstance(e, ExpectProcessTerminatedError): - raise ShellProcessTerminatedError(cmd, e.status, o) - else: - raise ShellError(cmd, o) - - # Remove the echoed command and the final shell prompt - return remove_last_nonempty_line(remove_command_echo(o, cmd)) - - - def cmd_status_output(self, cmd, timeout=60, internal_timeout=None, - print_func=None): - """ - Send a command and return its exit status and output. - - @param cmd: Command to send (must not contain newline characters) - @param timeout: The duration (in seconds) to wait for the prompt to - return - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - - @return: A tuple (status, output) where status is the exit status and - output is the output of cmd - @raise ShellTimeoutError: Raised if timeout expires - @raise ShellProcessTerminatedError: Raised if the shell process - terminates while waiting for output - @raise ShellStatusError: Raised if the exit status cannot be obtained - @raise ShellError: Raised if an unknown error occurs - """ - o = self.cmd_output(cmd, timeout, internal_timeout, print_func) - try: - # Send the 'echo $?' (or equivalent) command to get the exit status - s = self.cmd_output(self.status_test_command, 10, internal_timeout) - except ShellError: - raise ShellStatusError(cmd, o) - - # Get the first line consisting of digits only - digit_lines = [l for l in s.splitlines() if l.strip().isdigit()] - if digit_lines: - return int(digit_lines[0].strip()), o - else: - raise ShellStatusError(cmd, o) - - - def cmd_status(self, cmd, timeout=60, internal_timeout=None, - print_func=None): - """ - Send a command and return its exit status. - - @param cmd: Command to send (must not contain newline characters) - @param timeout: The duration (in seconds) to wait for the prompt to - return - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - - @return: The exit status of cmd - @raise ShellTimeoutError: Raised if timeout expires - @raise ShellProcessTerminatedError: Raised if the shell process - terminates while waiting for output - @raise ShellStatusError: Raised if the exit status cannot be obtained - @raise ShellError: Raised if an unknown error occurs - """ - s, o = self.cmd_status_output(cmd, timeout, internal_timeout, - print_func) - return s - - - def cmd(self, cmd, timeout=60, internal_timeout=None, print_func=None): - """ - Send a command and return its output. If the command's exit status is - nonzero, raise an exception. - - @param cmd: Command to send (must not contain newline characters) - @param timeout: The duration (in seconds) to wait for the prompt to - return - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - - @return: The output of cmd - @raise ShellTimeoutError: Raised if timeout expires - @raise ShellProcessTerminatedError: Raised if the shell process - terminates while waiting for output - @raise ShellError: Raised if the exit status cannot be obtained or if - an unknown error occurs - @raise ShellStatusError: Raised if the exit status cannot be obtained - @raise ShellError: Raised if an unknown error occurs - @raise ShellCmdError: Raised if the exit status is nonzero - """ - s, o = self.cmd_status_output(cmd, timeout, internal_timeout, - print_func) - if s != 0: - raise ShellCmdError(cmd, s, o) - return o - - - def get_command_output(self, cmd, timeout=60, internal_timeout=None, - print_func=None): - """ - Alias for cmd_output() for backward compatibility. - """ - return self.cmd_output(cmd, timeout, internal_timeout, print_func) - - - def get_command_status_output(self, cmd, timeout=60, internal_timeout=None, - print_func=None): - """ - Alias for cmd_status_output() for backward compatibility. - """ - return self.cmd_status_output(cmd, timeout, internal_timeout, - print_func) - - - def get_command_status(self, cmd, timeout=60, internal_timeout=None, - print_func=None): - """ - Alias for cmd_status() for backward compatibility. - """ - return self.cmd_status(cmd, timeout, internal_timeout, print_func) diff --git a/client/tests/kvm/kvm_test_utils.py b/client/tests/kvm/kvm_test_utils.py deleted file mode 100644 index b5c4a24e..00000000 --- a/client/tests/kvm/kvm_test_utils.py +++ /dev/null @@ -1,753 +0,0 @@ -""" -High-level KVM test utility functions. - -This module is meant to reduce code size by performing common test procedures. -Generally, code here should look like test code. -More specifically: - - Functions in this module should raise exceptions if things go wrong - (unlike functions in kvm_utils.py and kvm_vm.py which report failure via - their returned values). - - Functions in this module may use logging.info(), in addition to - logging.debug() and logging.error(), to log messages the user may be - interested in (unlike kvm_utils.py and kvm_vm.py which use - logging.debug() for anything that isn't an error). - - Functions in this module typically use functions and classes from - lower-level modules (e.g. kvm_utils.py, kvm_vm.py, kvm_subprocess.py). - - Functions in this module should not be used by lower-level modules. - - Functions in this module should be used in the right context. - For example, a function should not be used where it may display - misleading or inaccurate info or debug messages. - -@copyright: 2008-2009 Red Hat Inc. -""" - -import time, os, logging, re, signal -from autotest_lib.client.common_lib import error -from autotest_lib.client.bin import utils -import kvm_utils, kvm_vm, kvm_subprocess, scan_results - - -def get_living_vm(env, vm_name): - """ - Get a VM object from the environment and make sure it's alive. - - @param env: Dictionary with test environment. - @param vm_name: Name of the desired VM object. - @return: A VM object. - """ - vm = env.get_vm(vm_name) - if not vm: - raise error.TestError("VM '%s' not found in environment" % vm_name) - if not vm.is_alive(): - raise error.TestError("VM '%s' seems to be dead; test requires a " - "living VM" % vm_name) - return vm - - -def wait_for_login(vm, nic_index=0, timeout=240, start=0, step=2, serial=None): - """ - Try logging into a VM repeatedly. Stop on success or when timeout expires. - - @param vm: VM object. - @param nic_index: Index of NIC to access in the VM. - @param timeout: Time to wait before giving up. - @param serial: Whether to use a serial connection instead of a remote - (ssh, rss) one. - @return: A shell session object. - """ - end_time = time.time() + timeout - session = None - if serial: - type = 'serial' - logging.info("Trying to log into guest %s using serial connection," - " timeout %ds", vm.name, timeout) - time.sleep(start) - while time.time() < end_time: - try: - session = vm.serial_login() - break - except kvm_utils.LoginError, e: - logging.debug(e) - time.sleep(step) - else: - type = 'remote' - logging.info("Trying to log into guest %s using remote connection," - " timeout %ds", vm.name, timeout) - time.sleep(start) - while time.time() < end_time: - try: - session = vm.login(nic_index=nic_index) - break - except (kvm_utils.LoginError, kvm_vm.VMError), e: - logging.debug(e) - time.sleep(step) - if not session: - raise error.TestFail("Could not log into guest %s using %s connection" % - (vm.name, type)) - logging.info("Logged into guest %s using %s connection", vm.name, type) - return session - - -def reboot(vm, session, method="shell", sleep_before_reset=10, nic_index=0, - timeout=240): - """ - Reboot the VM and wait for it to come back up by trying to log in until - timeout expires. - - @param vm: VM object. - @param session: A shell session object. - @param method: Reboot method. Can be "shell" (send a shell reboot - command) or "system_reset" (send a system_reset monitor command). - @param nic_index: Index of NIC to access in the VM, when logging in after - rebooting. - @param timeout: Time to wait before giving up (after rebooting). - @return: A new shell session object. - """ - if method == "shell": - # Send a reboot command to the guest's shell - session.sendline(vm.get_params().get("reboot_command")) - logging.info("Reboot command sent. Waiting for guest to go down...") - elif method == "system_reset": - # Sleep for a while before sending the command - time.sleep(sleep_before_reset) - # Clear the event list of all QMP monitors - monitors = [m for m in vm.monitors if m.protocol == "qmp"] - for m in monitors: - m.clear_events() - # Send a system_reset monitor command - vm.monitor.cmd("system_reset") - logging.info("Monitor command system_reset sent. Waiting for guest to " - "go down...") - # Look for RESET QMP events - time.sleep(1) - for m in monitors: - if not m.get_event("RESET"): - raise error.TestFail("RESET QMP event not received after " - "system_reset (monitor '%s')" % m.name) - else: - logging.info("RESET QMP event received") - else: - logging.error("Unknown reboot method: %s", method) - - # Wait for the session to become unresponsive and close it - if not kvm_utils.wait_for(lambda: not session.is_responsive(timeout=30), - 120, 0, 1): - raise error.TestFail("Guest refuses to go down") - session.close() - - # Try logging into the guest until timeout expires - logging.info("Guest is down. Waiting for it to go up again, timeout %ds", - timeout) - session = vm.wait_for_login(nic_index, timeout=timeout) - logging.info("Guest is up again") - return session - - -def migrate(vm, env=None, mig_timeout=3600, mig_protocol="tcp", - mig_cancel=False, offline=False, stable_check=False, - clean=False, save_path=None, dest_host='localhost', mig_port=None): - """ - Migrate a VM locally and re-register it in the environment. - - @param vm: The VM to migrate. - @param env: The environment dictionary. If omitted, the migrated VM will - not be registered. - @param mig_timeout: timeout value for migration. - @param mig_protocol: migration protocol - @param mig_cancel: Test migrate_cancel or not when protocol is tcp. - @param dest_host: Destination host (defaults to 'localhost'). - @param mig_port: Port that will be used for migration. - @return: The post-migration VM, in case of same host migration, True in - case of multi-host migration. - """ - def mig_finished(): - o = vm.monitor.info("migrate") - if isinstance(o, str): - return "status: active" not in o - else: - return o.get("status") != "active" - - def mig_succeeded(): - o = vm.monitor.info("migrate") - if isinstance(o, str): - return "status: completed" in o - else: - return o.get("status") == "completed" - - def mig_failed(): - o = vm.monitor.info("migrate") - if isinstance(o, str): - return "status: failed" in o - else: - return o.get("status") == "failed" - - def mig_cancelled(): - o = vm.monitor.info("migrate") - if isinstance(o, str): - return ("Migration status: cancelled" in o or - "Migration status: canceled" in o) - else: - return (o.get("status") == "cancelled" or - o.get("status") == "canceled") - - def wait_for_migration(): - if not kvm_utils.wait_for(mig_finished, mig_timeout, 2, 2, - "Waiting for migration to finish..."): - raise error.TestFail("Timeout expired while waiting for migration " - "to finish") - - if dest_host == 'localhost': - dest_vm = vm.clone() - - if (dest_host == 'localhost') and stable_check: - # Pause the dest vm after creation - dest_vm.params['extra_params'] = (dest_vm.params.get('extra_params','') - + ' -S') - - if dest_host == 'localhost': - dest_vm.create(migration_mode=mig_protocol, mac_source=vm) - - try: - try: - if mig_protocol == "tcp": - if dest_host == 'localhost': - uri = "tcp:localhost:%d" % dest_vm.migration_port - else: - uri = 'tcp:%s:%d' % (dest_host, mig_port) - elif mig_protocol == "unix": - uri = "unix:%s" % dest_vm.migration_file - elif mig_protocol == "exec": - uri = '"exec:nc localhost %s"' % dest_vm.migration_port - - if offline: - vm.monitor.cmd("stop") - vm.monitor.migrate(uri) - - if mig_cancel: - time.sleep(2) - vm.monitor.cmd("migrate_cancel") - if not kvm_utils.wait_for(mig_cancelled, 60, 2, 2, - "Waiting for migration " - "cancellation"): - raise error.TestFail("Failed to cancel migration") - if offline: - vm.monitor.cmd("cont") - if dest_host == 'localhost': - dest_vm.destroy(gracefully=False) - return vm - else: - wait_for_migration() - if (dest_host == 'localhost') and stable_check: - save_path = None or "/tmp" - save1 = os.path.join(save_path, "src") - save2 = os.path.join(save_path, "dst") - - vm.save_to_file(save1) - dest_vm.save_to_file(save2) - - # Fail if we see deltas - md5_save1 = utils.hash_file(save1) - md5_save2 = utils.hash_file(save2) - if md5_save1 != md5_save2: - raise error.TestFail("Mismatch of VM state before " - "and after migration") - - if (dest_host == 'localhost') and offline: - dest_vm.monitor.cmd("cont") - except: - if dest_host == 'localhost': - dest_vm.destroy() - raise - - finally: - if (dest_host == 'localhost') and stable_check and clean: - logging.debug("Cleaning the state files") - if os.path.isfile(save1): - os.remove(save1) - if os.path.isfile(save2): - os.remove(save2) - - # Report migration status - if mig_succeeded(): - logging.info("Migration finished successfully") - elif mig_failed(): - raise error.TestFail("Migration failed") - else: - raise error.TestFail("Migration ended with unknown status") - - if dest_host == 'localhost': - if "paused" in dest_vm.monitor.info("status"): - logging.debug("Destination VM is paused, resuming it...") - dest_vm.monitor.cmd("cont") - - # Kill the source VM - vm.destroy(gracefully=False) - - # Replace the source VM with the new cloned VM - if (dest_host == 'localhost') and (env is not None): - env.register_vm(vm.name, dest_vm) - - # Return the new cloned VM - if dest_host == 'localhost': - return dest_vm - else: - return vm - - -def stop_windows_service(session, service, timeout=120): - """ - Stop a Windows service using sc. - If the service is already stopped or is not installed, do nothing. - - @param service: The name of the service - @param timeout: Time duration to wait for service to stop - @raise error.TestError: Raised if the service can't be stopped - """ - end_time = time.time() + timeout - while time.time() < end_time: - o = session.cmd_output("sc stop %s" % service, timeout=60) - # FAILED 1060 means the service isn't installed. - # FAILED 1062 means the service hasn't been started. - if re.search(r"\bFAILED (1060|1062)\b", o, re.I): - break - time.sleep(1) - else: - raise error.TestError("Could not stop service '%s'" % service) - - -def start_windows_service(session, service, timeout=120): - """ - Start a Windows service using sc. - If the service is already running, do nothing. - If the service isn't installed, fail. - - @param service: The name of the service - @param timeout: Time duration to wait for service to start - @raise error.TestError: Raised if the service can't be started - """ - end_time = time.time() + timeout - while time.time() < end_time: - o = session.cmd_output("sc start %s" % service, timeout=60) - # FAILED 1060 means the service isn't installed. - if re.search(r"\bFAILED 1060\b", o, re.I): - raise error.TestError("Could not start service '%s' " - "(service not installed)" % service) - # FAILED 1056 means the service is already running. - if re.search(r"\bFAILED 1056\b", o, re.I): - break - time.sleep(1) - else: - raise error.TestError("Could not start service '%s'" % service) - - -def get_time(session, time_command, time_filter_re, time_format): - """ - Return the host time and guest time. If the guest time cannot be fetched - a TestError exception is raised. - - Note that the shell session should be ready to receive commands - (i.e. should "display" a command prompt and should be done with all - previous commands). - - @param session: A shell session. - @param time_command: Command to issue to get the current guest time. - @param time_filter_re: Regex filter to apply on the output of - time_command in order to get the current time. - @param time_format: Format string to pass to time.strptime() with the - result of the regex filter. - @return: A tuple containing the host time and guest time. - """ - if len(re.findall("ntpdate|w32tm", time_command)) == 0: - host_time = time.time() - s = session.cmd_output(time_command) - - try: - s = re.findall(time_filter_re, s)[0] - except IndexError: - logging.debug("The time string from guest is:\n%s", s) - raise error.TestError("The time string from guest is unexpected.") - except Exception, e: - logging.debug("(time_filter_re, time_string): (%s, %s)", - time_filter_re, s) - raise e - - guest_time = time.mktime(time.strptime(s, time_format)) - else: - o = session.cmd(time_command) - if re.match('ntpdate', time_command): - offset = re.findall('offset (.*) sec', o)[0] - host_main, host_mantissa = re.findall(time_filter_re, o)[0] - host_time = (time.mktime(time.strptime(host_main, time_format)) + - float("0.%s" % host_mantissa)) - guest_time = host_time + float(offset) - else: - guest_time = re.findall(time_filter_re, o)[0] - offset = re.findall("o:(.*)s", o)[0] - if re.match('PM', guest_time): - hour = re.findall('\d+ (\d+):', guest_time)[0] - hour = str(int(hour) + 12) - guest_time = re.sub('\d+\s\d+:', "\d+\s%s:" % hour, - guest_time)[:-3] - else: - guest_time = guest_time[:-3] - guest_time = time.mktime(time.strptime(guest_time, time_format)) - host_time = guest_time - float(offset) - - return (host_time, guest_time) - - -def get_memory_info(lvms): - """ - Get memory information from host and guests in format: - Host: memfree = XXXM; Guests memsh = {XXX,XXX,...} - - @params lvms: List of VM objects - @return: String with memory info report - """ - if not isinstance(lvms, list): - raise error.TestError("Invalid list passed to get_stat: %s " % lvms) - - try: - meminfo = "Host: memfree = " - meminfo += str(int(utils.freememtotal()) / 1024) + "M; " - meminfo += "swapfree = " - mf = int(utils.read_from_meminfo("SwapFree")) / 1024 - meminfo += str(mf) + "M; " - except Exception, e: - raise error.TestFail("Could not fetch host free memory info, " - "reason: %s" % e) - - meminfo += "Guests memsh = {" - for vm in lvms: - shm = vm.get_shared_meminfo() - if shm is None: - raise error.TestError("Could not get shared meminfo from " - "VM %s" % vm) - meminfo += "%dM; " % shm - meminfo = meminfo[0:-2] + "}" - - return meminfo - - -def run_autotest(vm, session, control_path, timeout, outputdir, params): - """ - Run an autotest control file inside a guest (linux only utility). - - @param vm: VM object. - @param session: A shell session on the VM provided. - @param control_path: A path to an autotest control file. - @param timeout: Timeout under which the autotest control file must complete. - @param outputdir: Path on host where we should copy the guest autotest - results to. - - The following params is used by the migration - @param params: Test params used in the migration test - """ - def copy_if_hash_differs(vm, local_path, remote_path): - """ - Copy a file to a guest if it doesn't exist or if its MD5sum differs. - - @param vm: VM object. - @param local_path: Local path. - @param remote_path: Remote path. - """ - local_hash = utils.hash_file(local_path) - basename = os.path.basename(local_path) - output = session.cmd_output("md5sum %s" % remote_path) - if "such file" in output: - remote_hash = "0" - elif output: - remote_hash = output.split()[0] - else: - logging.warning("MD5 check for remote path %s did not return.", - remote_path) - # Let's be a little more lenient here and see if it wasn't a - # temporary problem - remote_hash = "0" - if remote_hash != local_hash: - logging.debug("Copying %s to guest", basename) - vm.copy_files_to(local_path, remote_path) - - - def extract(vm, remote_path, dest_dir="."): - """ - Extract a .tar.bz2 file on the guest. - - @param vm: VM object - @param remote_path: Remote file path - @param dest_dir: Destination dir for the contents - """ - basename = os.path.basename(remote_path) - logging.info("Extracting %s...", basename) - e_cmd = "tar xjvf %s -C %s" % (remote_path, dest_dir) - session.cmd(e_cmd, timeout=120) - - - def get_results(): - """ - Copy autotest results present on the guest back to the host. - """ - logging.info("Trying to copy autotest results from guest") - guest_results_dir = os.path.join(outputdir, "guest_autotest_results") - if not os.path.exists(guest_results_dir): - os.mkdir(guest_results_dir) - vm.copy_files_from("%s/results/default/*" % autotest_path, - guest_results_dir) - - - def get_results_summary(): - """ - Get the status of the tests that were executed on the host and close - the session where autotest was being executed. - """ - output = session.cmd_output("cat results/*/status") - try: - results = scan_results.parse_results(output) - # Report test results - logging.info("Results (test, status, duration, info):") - for result in results: - logging.info(str(result)) - session.close() - return results - except Exception, e: - logging.error("Error processing guest autotest results: %s", e) - return None - - - if not os.path.isfile(control_path): - raise error.TestError("Invalid path to autotest control file: %s" % - control_path) - - migrate_background = params.get("migrate_background") == "yes" - if migrate_background: - mig_timeout = float(params.get("mig_timeout", "3600")) - mig_protocol = params.get("migration_protocol", "tcp") - - compressed_autotest_path = "/tmp/autotest.tar.bz2" - - # To avoid problems, let's make the test use the current AUTODIR - # (autotest client path) location - autotest_path = os.environ['AUTODIR'] - - # tar the contents of bindir/autotest - cmd = "tar cvjf %s %s/*" % (compressed_autotest_path, autotest_path) - # Until we have nested virtualization, we don't need the kvm test :) - cmd += " --exclude=%s/tests/kvm" % autotest_path - cmd += " --exclude=%s/results" % autotest_path - cmd += " --exclude=%s/tmp" % autotest_path - cmd += " --exclude=%s/control*" % autotest_path - cmd += " --exclude=*.pyc" - cmd += " --exclude=*.svn" - cmd += " --exclude=*.git" - utils.run(cmd) - - # Copy autotest.tar.bz2 - copy_if_hash_differs(vm, compressed_autotest_path, compressed_autotest_path) - - # Extract autotest.tar.bz2 - extract(vm, compressed_autotest_path, "/") - - vm.copy_files_to(control_path, os.path.join(autotest_path, 'control')) - - # Run the test - logging.info("Running autotest control file %s on guest, timeout %ss", - os.path.basename(control_path), timeout) - session.cmd("cd %s" % autotest_path) - try: - session.cmd("rm -f control.state") - session.cmd("rm -rf results/*") - except kvm_subprocess.ShellError: - pass - try: - bg = None - try: - logging.info("---------------- Test output ----------------") - if migrate_background: - mig_timeout = float(params.get("mig_timeout", "3600")) - mig_protocol = params.get("migration_protocol", "tcp") - - bg = kvm_utils.Thread(session.cmd_output, - kwargs={'cmd': "bin/autotest control", - 'timeout': timeout, - 'print_func': logging.info}) - - bg.start() - - while bg.is_alive(): - logging.info("Tests is not ended, start a round of" - "migration ...") - vm.migrate(timeout=mig_timeout, protocol=mig_protocol) - else: - session.cmd_output("bin/autotest control", timeout=timeout, - print_func=logging.info) - finally: - logging.info("------------- End of test output ------------") - if migrate_background and bg: - bg.join() - except kvm_subprocess.ShellTimeoutError: - if vm.is_alive(): - get_results() - get_results_summary() - raise error.TestError("Timeout elapsed while waiting for job to " - "complete") - else: - raise error.TestError("Autotest job on guest failed " - "(VM terminated during job)") - except kvm_subprocess.ShellProcessTerminatedError: - get_results() - raise error.TestError("Autotest job on guest failed " - "(Remote session terminated during job)") - - results = get_results_summary() - get_results() - - # Make a list of FAIL/ERROR/ABORT results (make sure FAIL results appear - # before ERROR results, and ERROR results appear before ABORT results) - bad_results = [r[0] for r in results if r[1] == "FAIL"] - bad_results += [r[0] for r in results if r[1] == "ERROR"] - bad_results += [r[0] for r in results if r[1] == "ABORT"] - - # Fail the test if necessary - if not results: - raise error.TestFail("Autotest control file run did not produce any " - "recognizable results") - if bad_results: - if len(bad_results) == 1: - e_msg = ("Test %s failed during control file execution" % - bad_results[0]) - else: - e_msg = ("Tests %s failed during control file execution" % - " ".join(bad_results)) - raise error.TestFail(e_msg) - - -def get_loss_ratio(output): - """ - Get the packet loss ratio from the output of ping -. - @param output: Ping output. - """ - try: - return int(re.findall('(\d+)% packet loss', output)[0]) - except IndexError: - logging.debug(output) - return -1 - - -def raw_ping(command, timeout, session, output_func): - """ - Low-level ping command execution. - - @param command: Ping command. - @param timeout: Timeout of the ping command. - @param session: Local executon hint or session to execute the ping command. - """ - if session is None: - process = kvm_subprocess.run_bg(command, output_func=output_func, - timeout=timeout) - - # Send SIGINT signal to notify the timeout of running ping process, - # Because ping have the ability to catch the SIGINT signal so we can - # always get the packet loss ratio even if timeout. - if process.is_alive(): - kvm_utils.kill_process_tree(process.get_pid(), signal.SIGINT) - - status = process.get_status() - output = process.get_output() - - process.close() - return status, output - else: - output = "" - try: - output = session.cmd_output(command, timeout=timeout, - print_func=output_func) - except kvm_subprocess.ShellTimeoutError: - # Send ctrl+c (SIGINT) through ssh session - session.send("\003") - try: - output2 = session.read_up_to_prompt(print_func=output_func) - output += output2 - except kvm_subprocess.ExpectTimeoutError, e: - output += e.output - # We also need to use this session to query the return value - session.send("\003") - - session.sendline(session.status_test_command) - try: - o2 = session.read_up_to_prompt() - except kvm_subprocess.ExpectError: - status = -1 - else: - try: - status = int(re.findall("\d+", o2)[0]) - except: - status = -1 - - return status, output - - -def ping(dest=None, count=None, interval=None, interface=None, - packetsize=None, ttl=None, hint=None, adaptive=False, - broadcast=False, flood=False, timeout=0, - output_func=logging.debug, session=None): - """ - Wrapper of ping. - - @param dest: Destination address. - @param count: Count of icmp packet. - @param interval: Interval of two icmp echo request. - @param interface: Specified interface of the source address. - @param packetsize: Packet size of icmp. - @param ttl: IP time to live. - @param hint: Path mtu discovery hint. - @param adaptive: Adaptive ping flag. - @param broadcast: Broadcast ping flag. - @param flood: Flood ping flag. - @param timeout: Timeout for the ping command. - @param output_func: Function used to log the result of ping. - @param session: Local executon hint or session to execute the ping command. - """ - if dest is not None: - command = "ping %s " % dest - else: - command = "ping localhost " - if count is not None: - command += " -c %s" % count - if interval is not None: - command += " -i %s" % interval - if interface is not None: - command += " -I %s" % interface - if packetsize is not None: - command += " -s %s" % packetsize - if ttl is not None: - command += " -t %s" % ttl - if hint is not None: - command += " -M %s" % hint - if adaptive: - command += " -A" - if broadcast: - command += " -b" - if flood: - command += " -f -q" - output_func = None - - return raw_ping(command, timeout, session, output_func) - - -def get_linux_ifname(session, mac_address): - """ - Get the interface name through the mac address. - - @param session: session to the virtual machine - @mac_address: the macaddress of nic - """ - - output = session.cmd_output("ifconfig -a") - - try: - ethname = re.findall("(\w+)\s+Link.*%s" % mac_address, output, - re.IGNORECASE)[0] - return ethname - except: - return None diff --git a/client/tests/kvm/kvm_utils.py b/client/tests/kvm/kvm_utils.py deleted file mode 100644 index 38c4afe3..00000000 --- a/client/tests/kvm/kvm_utils.py +++ /dev/null @@ -1,2317 +0,0 @@ -""" -KVM test utility functions. - -@copyright: 2008-2009 Red Hat Inc. -""" - -import time, string, random, socket, os, signal, re, logging, commands, cPickle -import fcntl, shelve, ConfigParser, rss_file_transfer, threading, sys, UserDict -import inspect -from autotest_lib.client.bin import utils, os_dep -from autotest_lib.client.common_lib import error, logging_config -import kvm_subprocess -try: - import koji - KOJI_INSTALLED = True -except ImportError: - KOJI_INSTALLED = False - - -def _lock_file(filename): - f = open(filename, "w") - fcntl.lockf(f, fcntl.LOCK_EX) - return f - - -def _unlock_file(f): - fcntl.lockf(f, fcntl.LOCK_UN) - f.close() - - -def is_vm(obj): - """ - Tests whether a given object is a VM object. - - @param obj: Python object. - """ - return obj.__class__.__name__ == "VM" - - -class Env(UserDict.IterableUserDict): - """ - A dict-like object containing global objects used by tests. - """ - def __init__(self, filename=None, version=0): - """ - Create an empty Env object or load an existing one from a file. - - If the version recorded in the file is lower than version, or if some - error occurs during unpickling, or if filename is not supplied, - create an empty Env object. - - @param filename: Path to an env file. - @param version: Required env version (int). - """ - UserDict.IterableUserDict.__init__(self) - empty = {"version": version} - if filename: - self._filename = filename - try: - f = open(filename, "r") - env = cPickle.load(f) - f.close() - if env.get("version", 0) >= version: - self.data = env - else: - logging.warn("Incompatible env file found. Not using it.") - self.data = empty - # Almost any exception can be raised during unpickling, so let's - # catch them all - except Exception, e: - logging.warn(e) - self.data = empty - else: - self.data = empty - - - def save(self, filename=None): - """ - Pickle the contents of the Env object into a file. - - @param filename: Filename to pickle the dict into. If not supplied, - use the filename from which the dict was loaded. - """ - filename = filename or self._filename - f = open(filename, "w") - cPickle.dump(self.data, f) - f.close() - - - def get_all_vms(self): - """ - Return a list of all VM objects in this Env object. - """ - return [o for o in self.values() if is_vm(o)] - - - def get_vm(self, name): - """ - Return a VM object by its name. - - @param name: VM name. - """ - return self.get("vm__%s" % name) - - - def register_vm(self, name, vm): - """ - Register a VM in this Env object. - - @param name: VM name. - @param vm: VM object. - """ - self["vm__%s" % name] = vm - - - def unregister_vm(self, name): - """ - Remove a given VM. - - @param name: VM name. - """ - del self["vm__%s" % name] - - - def register_installer(self, installer): - """ - Register a installer that was just run - - The installer will be available for other tests, so that - information about the installed KVM modules and qemu-kvm can be used by - them. - """ - self['last_installer'] = installer - - - def previous_installer(self): - """ - Return the last installer that was registered - """ - return self.get('last_installer') - - -class Params(UserDict.IterableUserDict): - """ - A dict-like object passed to every test. - """ - def objects(self, key): - """ - Return the names of objects defined using a given key. - - @param key: The name of the key whose value lists the objects - (e.g. 'nics'). - """ - return self.get(key, "").split() - - - def object_params(self, obj_name): - """ - Return a dict-like object containing the parameters of an individual - object. - - This method behaves as follows: the suffix '_' + obj_name is removed - from all key names that have it. Other key names are left unchanged. - The values of keys with the suffix overwrite the values of their - suffixless versions. - - @param obj_name: The name of the object (objects are listed by the - objects() method). - """ - suffix = "_" + obj_name - new_dict = self.copy() - for key in self: - if key.endswith(suffix): - new_key = key.split(suffix)[0] - new_dict[new_key] = self[key] - return new_dict - - -# Functions related to MAC/IP addresses - -def _open_mac_pool(lock_mode): - lock_file = open("/tmp/mac_lock", "w+") - fcntl.lockf(lock_file, lock_mode) - pool = shelve.open("/tmp/address_pool") - return pool, lock_file - - -def _close_mac_pool(pool, lock_file): - pool.close() - fcntl.lockf(lock_file, fcntl.LOCK_UN) - lock_file.close() - - -def _generate_mac_address_prefix(mac_pool): - """ - Generate a random MAC address prefix and add it to the MAC pool dictionary. - If there's a MAC prefix there already, do not update the MAC pool and just - return what's in there. By convention we will set KVM autotest MAC - addresses to start with 0x9a. - - @param mac_pool: The MAC address pool object. - @return: The MAC address prefix. - """ - if "prefix" in mac_pool: - prefix = mac_pool["prefix"] - logging.debug("Used previously generated MAC address prefix for this " - "host: %s", prefix) - else: - r = random.SystemRandom() - prefix = "9a:%02x:%02x:%02x:" % (r.randint(0x00, 0xff), - r.randint(0x00, 0xff), - r.randint(0x00, 0xff)) - mac_pool["prefix"] = prefix - logging.debug("Generated MAC address prefix for this host: %s", prefix) - return prefix - - -def generate_mac_address(vm_instance, nic_index): - """ - Randomly generate a MAC address and add it to the MAC address pool. - - Try to generate a MAC address based on a randomly generated MAC address - prefix and add it to a persistent dictionary. - key = VM instance + NIC index, value = MAC address - e.g. {'20100310-165222-Wt7l:0': '9a:5d:94:6a:9b:f9'} - - @param vm_instance: The instance attribute of a VM. - @param nic_index: The index of the NIC. - @return: MAC address string. - """ - mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_EX) - key = "%s:%s" % (vm_instance, nic_index) - if key in mac_pool: - mac = mac_pool[key] - else: - prefix = _generate_mac_address_prefix(mac_pool) - r = random.SystemRandom() - while key not in mac_pool: - mac = prefix + "%02x:%02x" % (r.randint(0x00, 0xff), - r.randint(0x00, 0xff)) - if mac in mac_pool.values(): - continue - mac_pool[key] = mac - logging.debug("Generated MAC address for NIC %s: %s", key, mac) - _close_mac_pool(mac_pool, lock_file) - return mac - - -def free_mac_address(vm_instance, nic_index): - """ - Remove a MAC address from the address pool. - - @param vm_instance: The instance attribute of a VM. - @param nic_index: The index of the NIC. - """ - mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_EX) - key = "%s:%s" % (vm_instance, nic_index) - if key in mac_pool: - logging.debug("Freeing MAC address for NIC %s: %s", key, mac_pool[key]) - del mac_pool[key] - _close_mac_pool(mac_pool, lock_file) - - -def set_mac_address(vm_instance, nic_index, mac): - """ - Set a MAC address in the pool. - - @param vm_instance: The instance attribute of a VM. - @param nic_index: The index of the NIC. - """ - mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_EX) - mac_pool["%s:%s" % (vm_instance, nic_index)] = mac - _close_mac_pool(mac_pool, lock_file) - - -def get_mac_address(vm_instance, nic_index): - """ - Return a MAC address from the pool. - - @param vm_instance: The instance attribute of a VM. - @param nic_index: The index of the NIC. - @return: MAC address string. - """ - mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_SH) - mac = mac_pool.get("%s:%s" % (vm_instance, nic_index)) - _close_mac_pool(mac_pool, lock_file) - return mac - - -def verify_ip_address_ownership(ip, macs, timeout=10.0): - """ - Use arping and the ARP cache to make sure a given IP address belongs to one - of the given MAC addresses. - - @param ip: An IP address. - @param macs: A list or tuple of MAC addresses. - @return: True iff ip is assigned to a MAC address in macs. - """ - # Compile a regex that matches the given IP address and any of the given - # MAC addresses - mac_regex = "|".join("(%s)" % mac for mac in macs) - regex = re.compile(r"\b%s\b.*\b(%s)\b" % (ip, mac_regex), re.IGNORECASE) - - # Check the ARP cache - o = commands.getoutput("%s -n" % find_command("arp")) - if regex.search(o): - return True - - # Get the name of the bridge device for arping - o = commands.getoutput("%s route get %s" % (find_command("ip"), ip)) - dev = re.findall("dev\s+\S+", o, re.IGNORECASE) - if not dev: - return False - dev = dev[0].split()[-1] - - # Send an ARP request - o = commands.getoutput("%s -f -c 3 -I %s %s" % - (find_command("arping"), dev, ip)) - return bool(regex.search(o)) - - -# Utility functions for dealing with external processes - -def find_command(cmd): - for dir in ["/usr/local/sbin", "/usr/local/bin", - "/usr/sbin", "/usr/bin", "/sbin", "/bin"]: - file = os.path.join(dir, cmd) - if os.path.exists(file): - return file - raise ValueError('Missing command: %s' % cmd) - - -def pid_exists(pid): - """ - Return True if a given PID exists. - - @param pid: Process ID number. - """ - try: - os.kill(pid, 0) - return True - except: - return False - - -def safe_kill(pid, signal): - """ - Attempt to send a signal to a given process that may or may not exist. - - @param signal: Signal number. - """ - try: - os.kill(pid, signal) - return True - except: - return False - - -def kill_process_tree(pid, sig=signal.SIGKILL): - """Signal a process and all of its children. - - If the process does not exist -- return. - - @param pid: The pid of the process to signal. - @param sig: The signal to send to the processes. - """ - if not safe_kill(pid, signal.SIGSTOP): - return - children = commands.getoutput("ps --ppid=%d -o pid=" % pid).split() - for child in children: - kill_process_tree(int(child), sig) - safe_kill(pid, sig) - safe_kill(pid, signal.SIGCONT) - - -def get_latest_kvm_release_tag(release_listing): - """ - Fetches the latest release tag for KVM. - - @param release_listing: URL that contains a list of the Source Forge - KVM project files. - """ - try: - release_page = utils.urlopen(release_listing) - data = release_page.read() - release_page.close() - rx = re.compile("kvm-(\d+).tar.gz", re.IGNORECASE) - matches = rx.findall(data) - # In all regexp matches to something that looks like a release tag, - # get the largest integer. That will be our latest release tag. - latest_tag = max(int(x) for x in matches) - return str(latest_tag) - except Exception, e: - message = "Could not fetch latest KVM release tag: %s" % str(e) - logging.error(message) - raise error.TestError(message) - - -def get_git_branch(repository, branch, srcdir, commit=None, lbranch=None): - """ - Retrieves a given git code repository. - - @param repository: Git repository URL - """ - logging.info("Fetching git [REP '%s' BRANCH '%s' COMMIT '%s'] -> %s", - repository, branch, commit, srcdir) - if not os.path.exists(srcdir): - os.makedirs(srcdir) - os.chdir(srcdir) - - if os.path.exists(".git"): - utils.system("git reset --hard") - else: - utils.system("git init") - - if not lbranch: - lbranch = branch - - utils.system("git fetch -q -f -u -t %s %s:%s" % - (repository, branch, lbranch)) - utils.system("git checkout %s" % lbranch) - if commit: - utils.system("git checkout %s" % commit) - - h = utils.system_output('git log --pretty=format:"%H" -1') - try: - desc = "tag %s" % utils.system_output("git describe") - except error.CmdError: - desc = "no tag found" - - logging.info("Commit hash for %s is %s (%s)", repository, h.strip(), desc) - return srcdir - - -def check_kvm_source_dir(source_dir): - """ - Inspects the kvm source directory and verifies its disposition. In some - occasions build may be dependant on the source directory disposition. - The reason why the return codes are numbers is that we might have more - changes on the source directory layout, so it's not scalable to just use - strings like 'old_repo', 'new_repo' and such. - - @param source_dir: Source code path that will be inspected. - """ - os.chdir(source_dir) - has_qemu_dir = os.path.isdir('qemu') - has_kvm_dir = os.path.isdir('kvm') - if has_qemu_dir: - logging.debug("qemu directory detected, source dir layout 1") - return 1 - if has_kvm_dir and not has_qemu_dir: - logging.debug("kvm directory detected, source dir layout 2") - return 2 - else: - raise error.TestError("Unknown source dir layout, cannot proceed.") - - -# Functions and classes used for logging into guests and transferring files - -class LoginError(Exception): - def __init__(self, msg, output): - Exception.__init__(self, msg, output) - self.msg = msg - self.output = output - - def __str__(self): - return "%s (output: %r)" % (self.msg, self.output) - - -class LoginAuthenticationError(LoginError): - pass - - -class LoginTimeoutError(LoginError): - def __init__(self, output): - LoginError.__init__(self, "Login timeout expired", output) - - -class LoginProcessTerminatedError(LoginError): - def __init__(self, status, output): - LoginError.__init__(self, None, output) - self.status = status - - def __str__(self): - return ("Client process terminated (status: %s, output: %r)" % - (self.status, self.output)) - - -class LoginBadClientError(LoginError): - def __init__(self, client): - LoginError.__init__(self, None, None) - self.client = client - - def __str__(self): - return "Unknown remote shell client: %r" % self.client - - -class SCPError(Exception): - def __init__(self, msg, output): - Exception.__init__(self, msg, output) - self.msg = msg - self.output = output - - def __str__(self): - return "%s (output: %r)" % (self.msg, self.output) - - -class SCPAuthenticationError(SCPError): - pass - - -class SCPAuthenticationTimeoutError(SCPAuthenticationError): - def __init__(self, output): - SCPAuthenticationError.__init__(self, "Authentication timeout expired", - output) - - -class SCPTransferTimeoutError(SCPError): - def __init__(self, output): - SCPError.__init__(self, "Transfer timeout expired", output) - - -class SCPTransferFailedError(SCPError): - def __init__(self, status, output): - SCPError.__init__(self, None, output) - self.status = status - - def __str__(self): - return ("SCP transfer failed (status: %s, output: %r)" % - (self.status, self.output)) - - -def _remote_login(session, username, password, prompt, timeout=10): - """ - Log into a remote host (guest) using SSH or Telnet. Wait for questions - and provide answers. If timeout expires while waiting for output from the - child (e.g. a password prompt or a shell prompt) -- fail. - - @brief: Log into a remote host (guest) using SSH or Telnet. - - @param session: An Expect or ShellSession instance to operate on - @param username: The username to send in reply to a login prompt - @param password: The password to send in reply to a password prompt - @param prompt: The shell prompt that indicates a successful login - @param timeout: The maximal time duration (in seconds) to wait for each - step of the login procedure (i.e. the "Are you sure" prompt, the - password prompt, the shell prompt, etc) - @raise LoginTimeoutError: If timeout expires - @raise LoginAuthenticationError: If authentication fails - @raise LoginProcessTerminatedError: If the client terminates during login - @raise LoginError: If some other error occurs - """ - password_prompt_count = 0 - login_prompt_count = 0 - - while True: - try: - match, text = session.read_until_last_line_matches( - [r"[Aa]re you sure", r"[Pp]assword:\s*$", r"[Ll]ogin:\s*$", - r"[Cc]onnection.*closed", r"[Cc]onnection.*refused", - r"[Pp]lease wait", prompt], - timeout=timeout, internal_timeout=0.5) - if match == 0: # "Are you sure you want to continue connecting" - logging.debug("Got 'Are you sure...'; sending 'yes'") - session.sendline("yes") - continue - elif match == 1: # "password:" - if password_prompt_count == 0: - logging.debug("Got password prompt; sending '%s'", password) - session.sendline(password) - password_prompt_count += 1 - continue - else: - raise LoginAuthenticationError("Got password prompt twice", - text) - elif match == 2: # "login:" - if login_prompt_count == 0 and password_prompt_count == 0: - logging.debug("Got username prompt; sending '%s'", username) - session.sendline(username) - login_prompt_count += 1 - continue - else: - if login_prompt_count > 0: - msg = "Got username prompt twice" - else: - msg = "Got username prompt after password prompt" - raise LoginAuthenticationError(msg, text) - elif match == 3: # "Connection closed" - raise LoginError("Client said 'connection closed'", text) - elif match == 4: # "Connection refused" - raise LoginError("Client said 'connection refused'", text) - elif match == 5: # "Please wait" - logging.debug("Got 'Please wait'") - timeout = 30 - continue - elif match == 6: # prompt - logging.debug("Got shell prompt -- logged in") - break - except kvm_subprocess.ExpectTimeoutError, e: - raise LoginTimeoutError(e.output) - except kvm_subprocess.ExpectProcessTerminatedError, e: - raise LoginProcessTerminatedError(e.status, e.output) - - -def remote_login(client, host, port, username, password, prompt, linesep="\n", - log_filename=None, timeout=10): - """ - Log into a remote host (guest) using SSH/Telnet/Netcat. - - @param client: The client to use ('ssh', 'telnet' or 'nc') - @param host: Hostname or IP address - @param port: Port to connect to - @param username: Username (if required) - @param password: Password (if required) - @param prompt: Shell prompt (regular expression) - @param linesep: The line separator to use when sending lines - (e.g. '\\n' or '\\r\\n') - @param log_filename: If specified, log all output to this file - @param timeout: The maximal time duration (in seconds) to wait for - each step of the login procedure (i.e. the "Are you sure" prompt - or the password prompt) - @raise LoginBadClientError: If an unknown client is requested - @raise: Whatever _remote_login() raises - @return: A ShellSession object. - """ - if client == "ssh": - cmd = ("ssh -o UserKnownHostsFile=/dev/null " - "-o PreferredAuthentications=password -p %s %s@%s" % - (port, username, host)) - elif client == "telnet": - cmd = "telnet -l %s %s %s" % (username, host, port) - elif client == "nc": - cmd = "nc %s %s" % (host, port) - else: - raise LoginBadClientError(client) - - logging.debug("Trying to login with command '%s'", cmd) - session = kvm_subprocess.ShellSession(cmd, linesep=linesep, prompt=prompt) - try: - _remote_login(session, username, password, prompt, timeout) - except: - session.close() - raise - if log_filename: - session.set_output_func(log_line) - session.set_output_params((log_filename,)) - return session - - -def wait_for_login(client, host, port, username, password, prompt, linesep="\n", - log_filename=None, timeout=240, internal_timeout=10): - """ - Make multiple attempts to log into a remote host (guest) until one succeeds - or timeout expires. - - @param timeout: Total time duration to wait for a successful login - @param internal_timeout: The maximal time duration (in seconds) to wait for - each step of the login procedure (e.g. the "Are you sure" prompt - or the password prompt) - @see: remote_login() - @raise: Whatever remote_login() raises - @return: A ShellSession object. - """ - logging.debug("Attempting to log into %s:%s using %s (timeout %ds)", - host, port, client, timeout) - end_time = time.time() + timeout - while time.time() < end_time: - try: - return remote_login(client, host, port, username, password, prompt, - linesep, log_filename, internal_timeout) - except LoginError, e: - logging.debug(e) - time.sleep(2) - # Timeout expired; try one more time but don't catch exceptions - return remote_login(client, host, port, username, password, prompt, - linesep, log_filename, internal_timeout) - - -def _remote_scp(session, password_list, transfer_timeout=600, login_timeout=10): - """ - Transfer file(s) to a remote host (guest) using SCP. Wait for questions - and provide answers. If login_timeout expires while waiting for output - from the child (e.g. a password prompt), fail. If transfer_timeout expires - while waiting for the transfer to complete, fail. - - @brief: Transfer files using SCP, given a command line. - - @param session: An Expect or ShellSession instance to operate on - @param password_list: Password list to send in reply to the password prompt - @param transfer_timeout: The time duration (in seconds) to wait for the - transfer to complete. - @param login_timeout: The maximal time duration (in seconds) to wait for - each step of the login procedure (i.e. the "Are you sure" prompt or - the password prompt) - @raise SCPAuthenticationError: If authentication fails - @raise SCPTransferTimeoutError: If the transfer fails to complete in time - @raise SCPTransferFailedError: If the process terminates with a nonzero - exit code - @raise SCPError: If some other error occurs - """ - password_prompt_count = 0 - timeout = login_timeout - authentication_done = False - - scp_type = len(password_list) - - while True: - try: - match, text = session.read_until_last_line_matches( - [r"[Aa]re you sure", r"[Pp]assword:\s*$", r"lost connection"], - timeout=timeout, internal_timeout=0.5) - if match == 0: # "Are you sure you want to continue connecting" - logging.debug("Got 'Are you sure...'; sending 'yes'") - session.sendline("yes") - continue - elif match == 1: # "password:" - if password_prompt_count == 0: - logging.debug("Got password prompt; sending '%s'" % - password_list[password_prompt_count]) - session.sendline(password_list[password_prompt_count]) - password_prompt_count += 1 - timeout = transfer_timeout - if scp_type == 1: - authentication_done = True - continue - elif password_prompt_count == 1 and scp_type == 2: - logging.debug("Got password prompt; sending '%s'" % - password_list[password_prompt_count]) - session.sendline(password_list[password_prompt_count]) - password_prompt_count += 1 - timeout = transfer_timeout - authentication_done = True - continue - else: - raise SCPAuthenticationError("Got password prompt twice", - text) - elif match == 2: # "lost connection" - raise SCPError("SCP client said 'lost connection'", text) - except kvm_subprocess.ExpectTimeoutError, e: - if authentication_done: - raise SCPTransferTimeoutError(e.output) - else: - raise SCPAuthenticationTimeoutError(e.output) - except kvm_subprocess.ExpectProcessTerminatedError, e: - if e.status == 0: - logging.debug("SCP process terminated with status 0") - break - else: - raise SCPTransferFailedError(e.status, e.output) - - -def remote_scp(command, password_list, log_filename=None, transfer_timeout=600, - login_timeout=10): - """ - Transfer file(s) to a remote host (guest) using SCP. - - @brief: Transfer files using SCP, given a command line. - - @param command: The command to execute - (e.g. "scp -r foobar root@localhost:/tmp/"). - @param password_list: Password list to send in reply to a password prompt. - @param log_filename: If specified, log all output to this file - @param transfer_timeout: The time duration (in seconds) to wait for the - transfer to complete. - @param login_timeout: The maximal time duration (in seconds) to wait for - each step of the login procedure (i.e. the "Are you sure" prompt - or the password prompt) - @raise: Whatever _remote_scp() raises - """ - logging.debug("Trying to SCP with command '%s', timeout %ss", - command, transfer_timeout) - if log_filename: - output_func = log_line - output_params = (log_filename,) - else: - output_func = None - output_params = () - session = kvm_subprocess.Expect(command, - output_func=output_func, - output_params=output_params) - try: - _remote_scp(session, password_list, transfer_timeout, login_timeout) - finally: - session.close() - - -def scp_to_remote(host, port, username, password, local_path, remote_path, - log_filename=None, timeout=600): - """ - Copy files to a remote host (guest) through scp. - - @param host: Hostname or IP address - @param username: Username (if required) - @param password: Password (if required) - @param local_path: Path on the local machine where we are copying from - @param remote_path: Path on the remote machine where we are copying to - @param log_filename: If specified, log all output to this file - @param timeout: The time duration (in seconds) to wait for the transfer - to complete. - @raise: Whatever remote_scp() raises - """ - command = ("scp -v -o UserKnownHostsFile=/dev/null " - "-o PreferredAuthentications=password -r -P %s %s %s@%s:%s" % - (port, local_path, username, host, remote_path)) - password_list = [] - password_list.append(password) - return remote_scp(command, password_list, log_filename, timeout) - - - -def scp_from_remote(host, port, username, password, remote_path, local_path, - log_filename=None, timeout=600): - """ - Copy files from a remote host (guest). - - @param host: Hostname or IP address - @param username: Username (if required) - @param password: Password (if required) - @param local_path: Path on the local machine where we are copying from - @param remote_path: Path on the remote machine where we are copying to - @param log_filename: If specified, log all output to this file - @param timeout: The time duration (in seconds) to wait for the transfer - to complete. - @raise: Whatever remote_scp() raises - """ - command = ("scp -v -o UserKnownHostsFile=/dev/null " - "-o PreferredAuthentications=password -r -P %s %s@%s:%s %s" % - (port, username, host, remote_path, local_path)) - password_list = [] - password_list.append(password) - remote_scp(command, password_list, log_filename, timeout) - - -def scp_between_remotes(src, dst, port, s_passwd, d_passwd, s_name, d_name, - s_path, d_path, log_filename=None, timeout=600): - """ - Copy files from a remote host (guest) to another remote host (guest). - - @param src/dst: Hostname or IP address of src and dst - @param s_name/d_name: Username (if required) - @param s_passwd/d_passwd: Password (if required) - @param s_path/d_path: Path on the remote machine where we are copying - from/to - @param log_filename: If specified, log all output to this file - @param timeout: The time duration (in seconds) to wait for the transfer - to complete. - - @return: True on success and False on failure. - """ - command = ("scp -v -o UserKnownHostsFile=/dev/null -o " - "PreferredAuthentications=password -r -P %s %s@%s:%s %s@%s:%s" % - (port, s_name, src, s_path, d_name, dst, d_path)) - password_list = [] - password_list.append(s_passwd) - password_list.append(d_passwd) - return remote_scp(command, password_list, log_filename, timeout) - - -def copy_files_to(address, client, username, password, port, local_path, - remote_path, log_filename=None, verbose=False, timeout=600): - """ - Copy files to a remote host (guest) using the selected client. - - @param client: Type of transfer client - @param username: Username (if required) - @param password: Password (if requried) - @param local_path: Path on the local machine where we are copying from - @param remote_path: Path on the remote machine where we are copying to - @param address: Address of remote host(guest) - @param log_filename: If specified, log all output to this file (SCP only) - @param verbose: If True, log some stats using logging.debug (RSS only) - @param timeout: The time duration (in seconds) to wait for the transfer to - complete. - @raise: Whatever remote_scp() raises - """ - if client == "scp": - scp_to_remote(address, port, username, password, local_path, - remote_path, log_filename, timeout) - elif client == "rss": - log_func = None - if verbose: - log_func = logging.debug - c = rss_file_transfer.FileUploadClient(address, port, log_func) - c.upload(local_path, remote_path, timeout) - c.close() - - -def copy_files_from(address, client, username, password, port, remote_path, - local_path, log_filename=None, verbose=False, timeout=600): - """ - Copy files from a remote host (guest) using the selected client. - - @param client: Type of transfer client - @param username: Username (if required) - @param password: Password (if requried) - @param remote_path: Path on the remote machine where we are copying from - @param local_path: Path on the local machine where we are copying to - @param address: Address of remote host(guest) - @param log_filename: If specified, log all output to this file (SCP only) - @param verbose: If True, log some stats using logging.debug (RSS only) - @param timeout: The time duration (in seconds) to wait for the transfer to - complete. - @raise: Whatever remote_scp() raises - """ - if client == "scp": - scp_from_remote(address, port, username, password, remote_path, - local_path, log_filename, timeout) - elif client == "rss": - log_func = None - if verbose: - log_func = logging.debug - c = rss_file_transfer.FileDownloadClient(address, port, log_func) - c.download(remote_path, local_path, timeout) - c.close() - - -# The following are utility functions related to ports. - -def is_port_free(port, address): - """ - Return True if the given port is available for use. - - @param port: Port number - """ - try: - s = socket.socket() - #s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if address == "localhost": - s.bind(("localhost", port)) - free = True - else: - s.connect((address, port)) - free = False - except socket.error: - if address == "localhost": - free = False - else: - free = True - s.close() - return free - - -def find_free_port(start_port, end_port, address="localhost"): - """ - Return a host free port in the range [start_port, end_port]. - - @param start_port: First port that will be checked. - @param end_port: Port immediately after the last one that will be checked. - """ - for i in range(start_port, end_port): - if is_port_free(i, address): - return i - return None - - -def find_free_ports(start_port, end_port, count, address="localhost"): - """ - Return count of host free ports in the range [start_port, end_port]. - - @count: Initial number of ports known to be free in the range. - @param start_port: First port that will be checked. - @param end_port: Port immediately after the last one that will be checked. - """ - ports = [] - i = start_port - while i < end_port and count > 0: - if is_port_free(i, address): - ports.append(i) - count -= 1 - i += 1 - return ports - - -# An easy way to log lines to files when the logging system can't be used - -_open_log_files = {} -_log_file_dir = "/tmp" - - -def log_line(filename, line): - """ - Write a line to a file. '\n' is appended to the line. - - @param filename: Path of file to write to, either absolute or relative to - the dir set by set_log_file_dir(). - @param line: Line to write. - """ - global _open_log_files, _log_file_dir - if filename not in _open_log_files: - path = get_path(_log_file_dir, filename) - try: - os.makedirs(os.path.dirname(path)) - except OSError: - pass - _open_log_files[filename] = open(path, "w") - timestr = time.strftime("%Y-%m-%d %H:%M:%S") - _open_log_files[filename].write("%s: %s\n" % (timestr, line)) - _open_log_files[filename].flush() - - -def set_log_file_dir(dir): - """ - Set the base directory for log files created by log_line(). - - @param dir: Directory for log files. - """ - global _log_file_dir - _log_file_dir = dir - - -# The following are miscellaneous utility functions. - -def get_path(base_path, user_path): - """ - Translate a user specified path to a real path. - If user_path is relative, append it to base_path. - If user_path is absolute, return it as is. - - @param base_path: The base path of relative user specified paths. - @param user_path: The user specified path. - """ - if os.path.isabs(user_path): - return user_path - else: - return os.path.join(base_path, user_path) - - -def generate_random_string(length): - """ - Return a random string using alphanumeric characters. - - @length: length of the string that will be generated. - """ - r = random.SystemRandom() - str = "" - chars = string.letters + string.digits - while length > 0: - str += r.choice(chars) - length -= 1 - return str - -def generate_random_id(): - """ - Return a random string suitable for use as a qemu id. - """ - return "id" + generate_random_string(6) - - -def generate_tmp_file_name(file, ext=None, dir='/tmp/'): - """ - Returns a temporary file name. The file is not created. - """ - while True: - file_name = (file + '-' + time.strftime("%Y%m%d-%H%M%S-") + - generate_random_string(4)) - if ext: - file_name += '.' + ext - file_name = os.path.join(dir, file_name) - if not os.path.exists(file_name): - break - - return file_name - - -def format_str_for_message(str): - """ - Format str so that it can be appended to a message. - If str consists of one line, prefix it with a space. - If str consists of multiple lines, prefix it with a newline. - - @param str: string that will be formatted. - """ - lines = str.splitlines() - num_lines = len(lines) - str = "\n".join(lines) - if num_lines == 0: - return "" - elif num_lines == 1: - return " " + str - else: - return "\n" + str - - -def wait_for(func, timeout, first=0.0, step=1.0, text=None): - """ - If func() evaluates to True before timeout expires, return the - value of func(). Otherwise return None. - - @brief: Wait until func() evaluates to True. - - @param timeout: Timeout in seconds - @param first: Time to sleep before first attempt - @param steps: Time to sleep between attempts in seconds - @param text: Text to print while waiting, for debug purposes - """ - start_time = time.time() - end_time = time.time() + timeout - - time.sleep(first) - - while time.time() < end_time: - if text: - logging.debug("%s (%f secs)", text, (time.time() - start_time)) - - output = func() - if output: - return output - - time.sleep(step) - - logging.debug("Timeout elapsed") - return None - - -def get_hash_from_file(hash_path, dvd_basename): - """ - Get the a hash from a given DVD image from a hash file - (Hash files are usually named MD5SUM or SHA1SUM and are located inside the - download directories of the DVDs) - - @param hash_path: Local path to a hash file. - @param cd_image: Basename of a CD image - """ - hash_file = open(hash_path, 'r') - for line in hash_file.readlines(): - if dvd_basename in line: - return line.split()[0] - - -def run_tests(parser, job): - """ - Runs the sequence of KVM tests based on the list of dictionaries - generated by the configuration system, handling dependencies. - - @param parser: Config parser object. - @param job: Autotest job object. - - @return: True, if all tests ran passed, False if any of them failed. - """ - for i, d in enumerate(parser.get_dicts()): - logging.info("Test %4d: %s" % (i + 1, d["shortname"])) - - status_dict = {} - failed = False - - for dict in parser.get_dicts(): - if dict.get("skip") == "yes": - continue - dependencies_satisfied = True - for dep in dict.get("dep"): - for test_name in status_dict.keys(): - if not dep in test_name: - continue - # So the only really non-fatal state is WARN, - # All the others make it not safe to proceed with dependency - # execution - if status_dict[test_name] not in ['GOOD', 'WARN']: - dependencies_satisfied = False - break - test_iterations = int(dict.get("iterations", 1)) - test_tag = dict.get("shortname") - - if dependencies_satisfied: - # Setting up profilers during test execution. - profilers = dict.get("profilers", "").split() - for profiler in profilers: - job.profilers.add(profiler) - # We need only one execution, profiled, hence we're passing - # the profile_only parameter to job.run_test(). - profile_only = bool(profilers) or None - current_status = job.run_test_detail("kvm", params=dict, - tag=test_tag, - iterations=test_iterations, - profile_only=profile_only) - for profiler in profilers: - job.profilers.delete(profiler) - else: - # We will force the test to fail as TestNA during preprocessing - dict['dependency_failed'] = 'yes' - current_status = job.run_test_detail("kvm", params=dict, - tag=test_tag, - iterations=test_iterations) - - if not current_status: - failed = True - status_dict[dict.get("name")] = current_status - - return not failed - - -def create_report(report_dir, results_dir): - """ - Creates a neatly arranged HTML results report in the results dir. - - @param report_dir: Directory where the report script is located. - @param results_dir: Directory where the results will be output. - """ - reporter = os.path.join(report_dir, 'html_report.py') - html_file = os.path.join(results_dir, 'results.html') - os.system('%s -r %s -f %s -R' % (reporter, results_dir, html_file)) - - -def display_attributes(instance): - """ - Inspects a given class instance attributes and displays them, convenient - for debugging. - """ - logging.debug("Attributes set:") - for member in inspect.getmembers(instance): - name, value = member - attribute = getattr(instance, name) - if not (name.startswith("__") or callable(attribute) or not value): - logging.debug(" %s: %s", name, value) - - -def get_full_pci_id(pci_id): - """ - Get full PCI ID of pci_id. - - @param pci_id: PCI ID of a device. - """ - cmd = "lspci -D | awk '/%s/ {print $1}'" % pci_id - status, full_id = commands.getstatusoutput(cmd) - if status != 0: - return None - return full_id - - -def get_vendor_from_pci_id(pci_id): - """ - Check out the device vendor ID according to pci_id. - - @param pci_id: PCI ID of a device. - """ - cmd = "lspci -n | awk '/%s/ {print $3}'" % pci_id - return re.sub(":", " ", commands.getoutput(cmd)) - - -class Thread(threading.Thread): - """ - Run a function in a background thread. - """ - def __init__(self, target, args=(), kwargs={}): - """ - Initialize the instance. - - @param target: Function to run in the thread. - @param args: Arguments to pass to target. - @param kwargs: Keyword arguments to pass to target. - """ - threading.Thread.__init__(self) - self._target = target - self._args = args - self._kwargs = kwargs - - - def run(self): - """ - Run target (passed to the constructor). No point in calling this - function directly. Call start() to make this function run in a new - thread. - """ - self._e = None - self._retval = None - try: - try: - self._retval = self._target(*self._args, **self._kwargs) - except: - self._e = sys.exc_info() - raise - finally: - # Avoid circular references (start() may be called only once so - # it's OK to delete these) - del self._target, self._args, self._kwargs - - - def join(self, timeout=None, suppress_exception=False): - """ - Join the thread. If target raised an exception, re-raise it. - Otherwise, return the value returned by target. - - @param timeout: Timeout value to pass to threading.Thread.join(). - @param suppress_exception: If True, don't re-raise the exception. - """ - threading.Thread.join(self, timeout) - try: - if self._e: - if not suppress_exception: - # Because the exception was raised in another thread, we - # need to explicitly insert the current context into it - s = error.exception_context(self._e[1]) - s = error.join_contexts(error.get_context(), s) - error.set_exception_context(self._e[1], s) - raise self._e[0], self._e[1], self._e[2] - else: - return self._retval - finally: - # Avoid circular references (join() may be called multiple times - # so we can't delete these) - self._e = None - self._retval = None - - -def parallel(targets): - """ - Run multiple functions in parallel. - - @param targets: A sequence of tuples or functions. If it's a sequence of - tuples, each tuple will be interpreted as (target, args, kwargs) or - (target, args) or (target,) depending on its length. If it's a - sequence of functions, the functions will be called without - arguments. - @return: A list of the values returned by the functions called. - """ - threads = [] - for target in targets: - if isinstance(target, tuple) or isinstance(target, list): - t = Thread(*target) - else: - t = Thread(target) - threads.append(t) - t.start() - return [t.join() for t in threads] - - -class KvmLoggingConfig(logging_config.LoggingConfig): - """ - Used with the sole purpose of providing convenient logging setup - for the KVM test auxiliary programs. - """ - def configure_logging(self, results_dir=None, verbose=False): - super(KvmLoggingConfig, self).configure_logging(use_console=True, - verbose=verbose) - - -class PciAssignable(object): - """ - Request PCI assignable devices on host. It will check whether to request - PF (physical Functions) or VF (Virtual Functions). - """ - def __init__(self, type="vf", driver=None, driver_option=None, - names=None, devices_requested=None): - """ - Initialize parameter 'type' which could be: - vf: Virtual Functions - pf: Physical Function (actual hardware) - mixed: Both includes VFs and PFs - - If pass through Physical NIC cards, we need to specify which devices - to be assigned, e.g. 'eth1 eth2'. - - If pass through Virtual Functions, we need to specify how many vfs - are going to be assigned, e.g. passthrough_count = 8 and max_vfs in - config file. - - @param type: PCI device type. - @param driver: Kernel module for the PCI assignable device. - @param driver_option: Module option to specify the maximum number of - VFs (eg 'max_vfs=7') - @param names: Physical NIC cards correspondent network interfaces, - e.g.'eth1 eth2 ...' - @param devices_requested: Number of devices being requested. - """ - self.type = type - self.driver = driver - self.driver_option = driver_option - if names: - self.name_list = names.split() - if devices_requested: - self.devices_requested = int(devices_requested) - else: - self.devices_requested = None - - - def _get_pf_pci_id(self, name, search_str): - """ - Get the PF PCI ID according to name. - - @param name: Name of the PCI device. - @param search_str: Search string to be used on lspci. - """ - cmd = "ethtool -i %s | awk '/bus-info/ {print $2}'" % name - s, pci_id = commands.getstatusoutput(cmd) - if not (s or "Cannot get driver information" in pci_id): - return pci_id[5:] - cmd = "lspci | awk '/%s/ {print $1}'" % search_str - pci_ids = [id for id in commands.getoutput(cmd).splitlines()] - nic_id = int(re.search('[0-9]+', name).group(0)) - if (len(pci_ids) - 1) < nic_id: - return None - return pci_ids[nic_id] - - - def _release_dev(self, pci_id): - """ - Release a single PCI device. - - @param pci_id: PCI ID of a given PCI device. - """ - base_dir = "/sys/bus/pci" - full_id = get_full_pci_id(pci_id) - vendor_id = get_vendor_from_pci_id(pci_id) - drv_path = os.path.join(base_dir, "devices/%s/driver" % full_id) - if 'pci-stub' in os.readlink(drv_path): - cmd = "echo '%s' > %s/new_id" % (vendor_id, drv_path) - if os.system(cmd): - return False - - stub_path = os.path.join(base_dir, "drivers/pci-stub") - cmd = "echo '%s' > %s/unbind" % (full_id, stub_path) - if os.system(cmd): - return False - - driver = self.dev_drivers[pci_id] - cmd = "echo '%s' > %s/bind" % (full_id, driver) - if os.system(cmd): - return False - - return True - - - def get_vf_devs(self): - """ - Catch all VFs PCI IDs. - - @return: List with all PCI IDs for the Virtual Functions avaliable - """ - if not self.sr_iov_setup(): - return [] - - cmd = "lspci | awk '/Virtual Function/ {print $1}'" - return commands.getoutput(cmd).split() - - - def get_pf_devs(self): - """ - Catch all PFs PCI IDs. - - @return: List with all PCI IDs for the physical hardware requested - """ - pf_ids = [] - for name in self.name_list: - pf_id = self._get_pf_pci_id(name, "Ethernet") - if not pf_id: - continue - pf_ids.append(pf_id) - return pf_ids - - - def get_devs(self, count): - """ - Check out all devices' PCI IDs according to their name. - - @param count: count number of PCI devices needed for pass through - @return: a list of all devices' PCI IDs - """ - if self.type == "vf": - vf_ids = self.get_vf_devs() - elif self.type == "pf": - vf_ids = self.get_pf_devs() - elif self.type == "mixed": - vf_ids = self.get_vf_devs() - vf_ids.extend(self.get_pf_devs()) - return vf_ids[0:count] - - - def get_vfs_count(self): - """ - Get VFs count number according to lspci. - """ - # FIXME: Need to think out a method of identify which - # 'virtual function' belongs to which physical card considering - # that if the host has more than one 82576 card. PCI_ID? - cmd = "lspci | grep 'Virtual Function' | wc -l" - return int(commands.getoutput(cmd)) - - - def check_vfs_count(self): - """ - Check VFs count number according to the parameter driver_options. - """ - # Network card 82576 has two network interfaces and each can be - # virtualized up to 7 virtual functions, therefore we multiply - # two for the value of driver_option 'max_vfs'. - expected_count = int((re.findall("(\d)", self.driver_option)[0])) * 2 - return (self.get_vfs_count == expected_count) - - - def is_binded_to_stub(self, full_id): - """ - Verify whether the device with full_id is already binded to pci-stub. - - @param full_id: Full ID for the given PCI device - """ - base_dir = "/sys/bus/pci" - stub_path = os.path.join(base_dir, "drivers/pci-stub") - if os.path.exists(os.path.join(stub_path, full_id)): - return True - return False - - - def sr_iov_setup(self): - """ - Ensure the PCI device is working in sr_iov mode. - - Check if the PCI hardware device drive is loaded with the appropriate, - parameters (number of VFs), and if it's not, perform setup. - - @return: True, if the setup was completed successfuly, False otherwise. - """ - re_probe = False - s, o = commands.getstatusoutput('lsmod | grep %s' % self.driver) - if s: - re_probe = True - elif not self.check_vfs_count(): - os.system("modprobe -r %s" % self.driver) - re_probe = True - else: - return True - - # Re-probe driver with proper number of VFs - if re_probe: - cmd = "modprobe %s %s" % (self.driver, self.driver_option) - logging.info("Loading the driver '%s' with option '%s'", - self.driver, self.driver_option) - s, o = commands.getstatusoutput(cmd) - if s: - return False - return True - - - def request_devs(self): - """ - Implement setup process: unbind the PCI device and then bind it - to the pci-stub driver. - - @return: a list of successfully requested devices' PCI IDs. - """ - base_dir = "/sys/bus/pci" - stub_path = os.path.join(base_dir, "drivers/pci-stub") - - self.pci_ids = self.get_devs(self.devices_requested) - logging.debug("The following pci_ids were found: %s", self.pci_ids) - requested_pci_ids = [] - self.dev_drivers = {} - - # Setup all devices specified for assignment to guest - for pci_id in self.pci_ids: - full_id = get_full_pci_id(pci_id) - if not full_id: - continue - drv_path = os.path.join(base_dir, "devices/%s/driver" % full_id) - dev_prev_driver = os.path.realpath(os.path.join(drv_path, - os.readlink(drv_path))) - self.dev_drivers[pci_id] = dev_prev_driver - - # Judge whether the device driver has been binded to stub - if not self.is_binded_to_stub(full_id): - logging.debug("Binding device %s to stub", full_id) - vendor_id = get_vendor_from_pci_id(pci_id) - stub_new_id = os.path.join(stub_path, 'new_id') - unbind_dev = os.path.join(drv_path, 'unbind') - stub_bind = os.path.join(stub_path, 'bind') - - info_write_to_files = [(vendor_id, stub_new_id), - (full_id, unbind_dev), - (full_id, stub_bind)] - - for content, file in info_write_to_files: - try: - utils.open_write_close(file, content) - except IOError: - logging.debug("Failed to write %s to file %s", content, - file) - continue - - if not self.is_binded_to_stub(full_id): - logging.error("Binding device %s to stub failed", pci_id) - continue - else: - logging.debug("Device %s already binded to stub", pci_id) - requested_pci_ids.append(pci_id) - self.pci_ids = requested_pci_ids - return self.pci_ids - - - def release_devs(self): - """ - Release all PCI devices currently assigned to VMs back to the - virtualization host. - """ - try: - for pci_id in self.dev_drivers: - if not self._release_dev(pci_id): - logging.error("Failed to release device %s to host", pci_id) - else: - logging.info("Released device %s successfully", pci_id) - except: - return - - -class KojiClient(object): - """ - Stablishes a connection with the build system, either koji or brew. - - This class provides convenience methods to retrieve information on packages - and the packages themselves hosted on the build system. Packages should be - specified in the KojiPgkSpec syntax. - """ - - CMD_LOOKUP_ORDER = ['/usr/bin/brew', '/usr/bin/koji' ] - - CONFIG_MAP = {'/usr/bin/brew': '/etc/brewkoji.conf', - '/usr/bin/koji': '/etc/koji.conf'} - - - def __init__(self, cmd=None): - """ - Verifies whether the system has koji or brew installed, then loads - the configuration file that will be used to download the files. - - @type cmd: string - @param cmd: Optional command name, either 'brew' or 'koji'. If not - set, get_default_command() is used and to look for - one of them. - @raise: ValueError - """ - if not KOJI_INSTALLED: - raise ValueError('No koji/brew installed on the machine') - - # Instance variables used by many methods - self.command = None - self.config = None - self.config_options = {} - self.session = None - - # Set koji command or get default - if cmd is None: - self.command = self.get_default_command() - else: - self.command = cmd - - # Check koji command - if not self.is_command_valid(): - raise ValueError('Koji command "%s" is not valid' % self.command) - - # Assuming command is valid, set configuration file and read it - self.config = self.CONFIG_MAP[self.command] - self.read_config() - - # Setup koji session - server_url = self.config_options['server'] - session_options = self.get_session_options() - self.session = koji.ClientSession(server_url, - session_options) - - - def read_config(self, check_is_valid=True): - ''' - Reads options from the Koji configuration file - - By default it checks if the koji configuration is valid - - @type check_valid: boolean - @param check_valid: whether to include a check on the configuration - @raises: ValueError - @returns: None - ''' - if check_is_valid: - if not self.is_config_valid(): - raise ValueError('Koji config "%s" is not valid' % self.config) - - config = ConfigParser.ConfigParser() - config.read(self.config) - - basename = os.path.basename(self.command) - for name, value in config.items(basename): - self.config_options[name] = value - - - def get_session_options(self): - ''' - Filter only options necessary for setting up a cobbler client session - - @returns: only the options used for session setup - ''' - session_options = {} - for name, value in self.config_options.items(): - if name in ('user', 'password', 'debug_xmlrpc', 'debug'): - session_options[name] = value - return session_options - - - def is_command_valid(self): - ''' - Checks if the currently set koji command is valid - - @returns: True or False - ''' - koji_command_ok = True - - if not os.path.isfile(self.command): - logging.error('Koji command "%s" is not a regular file', - self.command) - koji_command_ok = False - - if not os.access(self.command, os.X_OK): - logging.warn('Koji command "%s" is not executable: this is ' - 'not fatal but indicates an unexpected situation', - self.command) - - if not self.command in self.CONFIG_MAP.keys(): - logging.error('Koji command "%s" does not have a configuration ' - 'file associated to it', self.command) - koji_command_ok = False - - return koji_command_ok - - - def is_config_valid(self): - ''' - Checks if the currently set koji configuration is valid - - @returns: True or False - ''' - koji_config_ok = True - - if not os.path.isfile(self.config): - logging.error('Koji config "%s" is not a regular file', self.config) - koji_config_ok = False - - if not os.access(self.config, os.R_OK): - logging.error('Koji config "%s" is not readable', self.config) - koji_config_ok = False - - config = ConfigParser.ConfigParser() - config.read(self.config) - basename = os.path.basename(self.command) - if not config.has_section(basename): - logging.error('Koji configuration file "%s" does not have a ' - 'section "%s", named after the base name of the ' - 'currently set koji command "%s"', self.config, - basename, self.command) - koji_config_ok = False - - return koji_config_ok - - - def get_default_command(self): - ''' - Looks up for koji or brew "binaries" on the system - - Systems with plain koji usually don't have a brew cmd, while systems - with koji, have *both* koji and brew utilities. So we look for brew - first, and if found, we consider that the system is configured for - brew. If not, we consider this is a system with plain koji. - - @returns: either koji or brew command line executable path, or None - ''' - koji_command = None - for command in self.CMD_LOOKUP_ORDER: - if os.path.isfile(command): - koji_command = command - break - else: - koji_command_basename = os.path.basename(koji_command) - try: - koji_command = os_dep.command(koji_command_basename) - break - except ValueError: - pass - return koji_command - - - def get_pkg_info(self, pkg): - ''' - Returns information from Koji on the package - - @type pkg: KojiPkgSpec - @param pkg: information about the package, as a KojiPkgSpec instance - - @returns: information from Koji about the specified package - ''' - info = {} - if pkg.build is not None: - info = self.session.getBuild(int(pkg.build)) - elif pkg.tag is not None and pkg.package is not None: - builds = self.session.listTagged(pkg.tag, - latest=True, - inherit=True, - package=pkg.package) - if builds: - info = builds[0] - return info - - - def is_pkg_valid(self, pkg): - ''' - Checks if this package is altogether valid on Koji - - This verifies if the build or tag specified in the package - specification actually exist on the Koji server - - @returns: True or False - ''' - valid = True - if not self.is_pkg_spec_build_valid(pkg): - valid = False - if not self.is_pkg_spec_tag_valid(pkg): - valid = False - return valid - - - def is_pkg_spec_build_valid(self, pkg): - ''' - Checks if build is valid on Koji - - @param pkg: a Pkg instance - ''' - if pkg.build is not None: - info = self.session.getBuild(int(pkg.build)) - if info: - return True - return False - - - def is_pkg_spec_tag_valid(self, pkg): - ''' - Checks if tag is valid on Koji - - @type pkg: KojiPkgSpec - @param pkg: a package specification - ''' - if pkg.tag is not None: - tag = self.session.getTag(pkg.tag) - if tag: - return True - return False - - - def get_pkg_rpm_info(self, pkg, arch=None): - ''' - Returns a list of infomation on the RPM packages found on koji - - @type pkg: KojiPkgSpec - @param pkg: a package specification - @type arch: string - @param arch: packages built for this architecture, but also including - architecture independent (noarch) packages - ''' - if arch is None: - arch = utils.get_arch() - rpms = [] - info = self.get_pkg_info(pkg) - if info: - rpms = self.session.listRPMs(buildID=info['id'], - arches=[arch, 'noarch']) - if pkg.subpackages: - rpms = [d for d in rpms if d['name'] in pkg.subpackages] - return rpms - - - def get_pkg_rpm_names(self, pkg, arch=None): - ''' - Gets the names for the RPM packages specified in pkg - - @type pkg: KojiPkgSpec - @param pkg: a package specification - @type arch: string - @param arch: packages built for this architecture, but also including - architecture independent (noarch) packages - ''' - if arch is None: - arch = utils.get_arch() - rpms = self.get_pkg_rpm_info(pkg, arch) - return [rpm['name'] for rpm in rpms] - - - def get_pkg_rpm_file_names(self, pkg, arch=None): - ''' - Gets the file names for the RPM packages specified in pkg - - @type pkg: KojiPkgSpec - @param pkg: a package specification - @type arch: string - @param arch: packages built for this architecture, but also including - architecture independent (noarch) packages - ''' - if arch is None: - arch = utils.get_arch() - rpm_names = [] - rpms = self.get_pkg_rpm_info(pkg, arch) - for rpm in rpms: - arch_rpm_name = koji.pathinfo.rpm(rpm) - rpm_name = os.path.basename(arch_rpm_name) - rpm_names.append(rpm_name) - return rpm_names - - - def get_pkg_urls(self, pkg, arch=None): - ''' - Gets the urls for the packages specified in pkg - - @type pkg: KojiPkgSpec - @param pkg: a package specification - @type arch: string - @param arch: packages built for this architecture, but also including - architecture independent (noarch) packages - ''' - info = self.get_pkg_info(pkg) - rpms = self.get_pkg_rpm_info(pkg, arch) - rpm_urls = [] - for rpm in rpms: - rpm_name = koji.pathinfo.rpm(rpm) - url = ("%s/%s/%s/%s/%s" % (self.config_options['pkgurl'], - info['package_name'], - info['version'], info['release'], - rpm_name)) - rpm_urls.append(url) - return rpm_urls - - - def get_pkgs(self, pkg, dst_dir, arch=None): - ''' - Download the packages - - @type pkg: KojiPkgSpec - @param pkg: a package specification - @type dst_dir: string - @param dst_dir: the destination directory, where the downloaded - packages will be saved on - @type arch: string - @param arch: packages built for this architecture, but also including - architecture independent (noarch) packages - ''' - rpm_urls = self.get_pkg_urls(pkg, arch) - for url in rpm_urls: - utils.get_file(url, - os.path.join(dst_dir, os.path.basename(url))) - - -DEFAULT_KOJI_TAG = None -def set_default_koji_tag(tag): - ''' - Sets the default tag that will be used - ''' - global DEFAULT_KOJI_TAG - DEFAULT_KOJI_TAG = tag - - -def get_default_koji_tag(): - return DEFAULT_KOJI_TAG - - -class KojiPkgSpec: - ''' - A package specification syntax parser for Koji - - This holds information on either tag or build, and packages to be fetched - from koji and possibly installed (features external do this class). - - New objects can be created either by providing information in the textual - format or by using the actual parameters for tag, build, package and sub- - packages. The textual format is useful for command line interfaces and - configuration files, while using parameters is better for using this in - a programatic fashion. - - The following sets of examples are interchangeable. Specifying all packages - part of build number 1000: - - >>> from kvm_utils import KojiPkgSpec - >>> pkg = KojiPkgSpec('1000') - - >>> pkg = KojiPkgSpec(build=1000) - - Specifying only a subset of packages of build number 1000: - - >>> pkg = KojiPkgSpec('1000:kernel,kernel-devel') - - >>> pkg = KojiPkgSpec(build=1000, - subpackages=['kernel', 'kernel-devel']) - - Specifying the latest build for the 'kernel' package tagged with 'dist-f14': - - >>> pkg = KojiPkgSpec('dist-f14:kernel') - - >>> pkg = KojiPkgSpec(tag='dist-f14', package='kernel') - - Specifying the 'kernel' package using the default tag: - - >>> kvm_utils.set_default_koji_tag('dist-f14') - >>> pkg = KojiPkgSpec('kernel') - - >>> pkg = KojiPkgSpec(package='kernel') - - Specifying the 'kernel' package using the default tag: - - >>> kvm_utils.set_default_koji_tag('dist-f14') - >>> pkg = KojiPkgSpec('kernel') - - >>> pkg = KojiPkgSpec(package='kernel') - - If you do not specify a default tag, and give a package name without an - explicit tag, your package specification is considered invalid: - - >>> print kvm_utils.get_default_koji_tag() - None - >>> print kvm_utils.KojiPkgSpec('kernel').is_valid() - False - - >>> print kvm_utils.KojiPkgSpec(package='kernel').is_valid() - False - ''' - - SEP = ':' - - def __init__(self, text='', tag=None, build=None, - package=None, subpackages=[]): - ''' - Instantiates a new KojiPkgSpec object - - @type text: string - @param text: a textual representation of a package on Koji that - will be parsed - @type tag: string - @param tag: a koji tag, example: Fedora-14-RELEASE - (see U{http://fedoraproject.org/wiki/Koji#Tags_and_Targets}) - @type build: number - @param build: a koji build, example: 1001 - (see U{http://fedoraproject.org/wiki/Koji#Koji_Architecture}) - @type package: string - @param package: a koji package, example: python - (see U{http://fedoraproject.org/wiki/Koji#Koji_Architecture}) - @type subpackages: list of strings - @param subpackages: a list of package names, usually a subset of - the RPM packages generated by a given build - ''' - - # Set to None to indicate 'not set' (and be able to use 'is') - self.tag = None - self.build = None - self.package = None - self.subpackages = [] - - self.default_tag = None - - # Textual representation takes precedence (most common use case) - if text: - self.parse(text) - else: - self.tag = tag - self.build = build - self.package = package - self.subpackages = subpackages - - # Set the default tag, if set, as a fallback - if not self.build and not self.tag: - default_tag = get_default_koji_tag() - if default_tag is not None: - self.tag = default_tag - - - def parse(self, text): - ''' - Parses a textual representation of a package specification - - @type text: string - @param text: textual representation of a package in koji - ''' - parts = text.count(self.SEP) + 1 - if parts == 1: - if text.isdigit(): - self.build = text - else: - self.package = text - elif parts == 2: - part1, part2 = text.split(self.SEP) - if part1.isdigit(): - self.build = part1 - self.subpackages = part2.split(',') - else: - self.tag = part1 - self.package = part2 - elif parts >= 3: - # Instead of erroring on more arguments, we simply ignore them - # This makes the parser suitable for future syntax additions, such - # as specifying the package architecture - part1, part2, part3 = text.split(self.SEP)[0:3] - self.tag = part1 - self.package = part2 - self.subpackages = part3.split(',') - - - def _is_invalid_neither_tag_or_build(self): - ''' - Checks if this package is invalid due to not having either a valid - tag or build set, that is, both are empty. - - @returns: True if this is invalid and False if it's valid - ''' - return (self.tag is None and self.build is None) - - - def _is_invalid_package_but_no_tag(self): - ''' - Checks if this package is invalid due to having a package name set - but tag or build set, that is, both are empty. - - @returns: True if this is invalid and False if it's valid - ''' - return (self.package and not self.tag) - - - def _is_invalid_subpackages_but_no_main_package(self): - ''' - Checks if this package is invalid due to having a tag set (this is Ok) - but specifying subpackage names without specifying the main package - name. - - Specifying subpackages without a main package name is only valid when - a build is used instead of a tag. - - @returns: True if this is invalid and False if it's valid - ''' - return (self.tag and self.subpackages and not self.package) - - - def is_valid(self): - ''' - Checks if this package specification is valid. - - Being valid means that it has enough and not conflicting information. - It does not validate that the packages specified actually existe on - the Koji server. - - @returns: True or False - ''' - if self._is_invalid_neither_tag_or_build(): - return False - elif self._is_invalid_package_but_no_tag(): - return False - elif self._is_invalid_subpackages_but_no_main_package(): - return False - - return True - - - def describe_invalid(self): - ''' - Describes why this is not valid, in a human friendly way - ''' - if self._is_invalid_neither_tag_or_build(): - return 'neither a tag or build are set, and of them should be set' - elif self._is_invalid_package_but_no_tag(): - return 'package name specified but no tag is set' - elif self._is_invalid_subpackages_but_no_main_package(): - return 'subpackages specified but no main package is set' - - return 'unkwown reason, seems to be valid' - - - def describe(self): - ''' - Describe this package specification, in a human friendly way - - @returns: package specification description - ''' - if self.is_valid(): - description = '' - if not self.subpackages: - description += 'all subpackages from %s ' % self.package - else: - description += ('only subpackage(s) %s from package %s ' % - (', '.join(self.subpackages), self.package)) - - if self.build: - description += 'from build %s' % self.build - elif self.tag: - description += 'tagged with %s' % self.tag - else: - raise ValueError, 'neither build or tag is set' - - return description - else: - return ('Invalid package specification: %s' % - self.describe_invalid()) - - - def __repr__(self): - return ("" % - (self.tag, self.build, self.package, - ", ".join(self.subpackages))) - - -def umount(src, mount_point, type): - """ - Umount the src mounted in mount_point. - - @src: mount source - @mount_point: mount point - @type: file system type - """ - - mount_string = "%s %s %s" % (src, mount_point, type) - if mount_string in file("/etc/mtab").read(): - umount_cmd = "umount %s" % mount_point - try: - utils.system(umount_cmd) - return True - except error.CmdError: - return False - else: - logging.debug("%s is not mounted under %s", src, mount_point) - return True - - -def mount(src, mount_point, type, perm="rw"): - """ - Mount the src into mount_point of the host. - - @src: mount source - @mount_point: mount point - @type: file system type - @perm: mount premission - """ - umount(src, mount_point, type) - mount_string = "%s %s %s %s" % (src, mount_point, type, perm) - - if mount_string in file("/etc/mtab").read(): - logging.debug("%s is already mounted in %s with %s", - src, mount_point, perm) - return True - - mount_cmd = "mount -t %s %s %s -o %s" % (type, src, mount_point, perm) - try: - utils.system(mount_cmd) - except error.CmdError: - return False - - logging.debug("Verify the mount through /etc/mtab") - if mount_string in file("/etc/mtab").read(): - logging.debug("%s is successfully mounted", src) - return True - else: - logging.error("Can't find mounted NFS share - /etc/mtab contents \n%s", - file("/etc/mtab").read()) - return False - - -def install_host_kernel(job, params): - """ - Install a host kernel, given the appropriate params. - - @param job: Job object. - @param params: Dict with host kernel install params. - """ - install_type = params.get('host_kernel_install_type') - - rpm_url = params.get('host_kernel_rpm_url') - - koji_cmd = params.get('host_kernel_koji_cmd') - koji_build = params.get('host_kernel_koji_build') - koji_tag = params.get('host_kernel_koji_tag') - - git_repo = params.get('host_kernel_git_repo') - git_branch = params.get('host_kernel_git_branch') - git_commit = params.get('host_kernel_git_commit') - patch_list = params.get('host_kernel_patch_list') - if patch_list: - patch_list = patch_list.split() - kernel_config = params.get('host_kernel_config') - - if install_type == 'rpm': - logging.info('Installing host kernel through rpm') - dst = os.path.join("/tmp", os.path.basename(rpm_url)) - k = utils.get_file(rpm_url, dst) - host_kernel = job.kernel(k) - host_kernel.install(install_vmlinux=False) - host_kernel.boot() - - elif install_type in ['koji', 'brew']: - k_deps = KojiPkgSpec(tag=koji_tag, package='kernel', - subpackages=['kernel-devel', 'kernel-firmware']) - k = KojiPkgSpec(tag=koji_tag, package='kernel', - subpackages=['kernel']) - - c = KojiClient(koji_cmd) - logging.info('Fetching kernel dependencies (-devel, -firmware)') - c.get_pkgs(k_deps, job.tmpdir) - logging.info('Installing kernel dependencies (-devel, -firmware) ' - 'through %s', install_type) - k_deps_rpm_file_names = [os.path.join(job.tmpdir, rpm_file_name) for - rpm_file_name in c.get_pkg_rpm_file_names(k_deps)] - utils.run('rpm -U --force %s' % " ".join(k_deps_rpm_file_names)) - - c.get_pkgs(k, job.tmpdir) - k_rpm = os.path.join(job.tmpdir, - c.get_pkg_rpm_file_names(k)[0]) - host_kernel = job.kernel(k_rpm) - host_kernel.install(install_vmlinux=False) - host_kernel.boot() - - elif install_type == 'git': - logging.info('Chose to install host kernel through git, proceeding') - repodir = os.path.join("/tmp", 'kernel_src') - r = get_git_branch(git_repo, git_branch, repodir, git_commit) - host_kernel = job.kernel(r) - if patch_list: - host_kernel.patch(patch_list) - host_kernel.config(kernel_config) - host_kernel.build() - host_kernel.install() - host_kernel.boot() - - else: - logging.info('Chose %s, using the current kernel for the host', - install_type) diff --git a/client/tests/kvm/kvm_vm.py b/client/tests/kvm/kvm_vm.py deleted file mode 100755 index f0b81528..00000000 --- a/client/tests/kvm/kvm_vm.py +++ /dev/null @@ -1,1860 +0,0 @@ -#!/usr/bin/python -""" -Utility classes and functions to handle Virtual Machine creation using qemu. - -@copyright: 2008-2009 Red Hat Inc. -""" - -import time, os, logging, fcntl, re, commands, glob -import kvm_utils, kvm_subprocess, kvm_monitor -from autotest_lib.client.common_lib import error -from autotest_lib.client.bin import utils - - -class VMError(Exception): - pass - - -class VMCreateError(VMError): - def __init__(self, cmd, status, output): - VMError.__init__(self, cmd, status, output) - self.cmd = cmd - self.status = status - self.output = output - - def __str__(self): - return ("VM creation command failed: %r (status: %s, " - "output: %r)" % (self.cmd, self.status, self.output)) - - -class VMHashMismatchError(VMError): - def __init__(self, actual, expected): - VMError.__init__(self, actual, expected) - self.actual_hash = actual - self.expected_hash = expected - - def __str__(self): - return ("CD image hash (%s) differs from expected one (%s)" % - (self.actual_hash, self.expected_hash)) - - -class VMImageMissingError(VMError): - def __init__(self, filename): - VMError.__init__(self, filename) - self.filename = filename - - def __str__(self): - return "CD image file not found: %r" % self.filename - - -class VMImageCheckError(VMError): - def __init__(self, filename): - VMError.__init__(self, filename) - self.filename = filename - - def __str__(self): - return "Errors found on image: %r" % self.filename - - -class VMBadPATypeError(VMError): - def __init__(self, pa_type): - VMError.__init__(self, pa_type) - self.pa_type = pa_type - - def __str__(self): - return "Unsupported PCI assignable type: %r" % self.pa_type - - -class VMPAError(VMError): - def __init__(self, pa_type): - VMError.__init__(self, pa_type) - self.pa_type = pa_type - - def __str__(self): - return ("No PCI assignable devices could be assigned " - "(pci_assignable=%r)" % self.pa_type) - - -class VMPostCreateError(VMError): - def __init__(self, cmd, output): - VMError.__init__(self, cmd, output) - self.cmd = cmd - self.output = output - - -class VMHugePageError(VMPostCreateError): - def __str__(self): - return ("Cannot allocate hugepage memory (command: %r, " - "output: %r)" % (self.cmd, self.output)) - - -class VMKVMInitError(VMPostCreateError): - def __str__(self): - return ("Cannot initialize KVM (command: %r, output: %r)" % - (self.cmd, self.output)) - - -class VMDeadError(VMError): - def __init__(self, status, output): - VMError.__init__(self, status, output) - self.status = status - self.output = output - - def __str__(self): - return ("VM process is dead (status: %s, output: %r)" % - (self.status, self.output)) - - -class VMDeadKernelCrashError(VMError): - def __init__(self, kernel_crash): - VMError.__init__(self, kernel_crash) - self.kernel_crash = kernel_crash - - def __str__(self): - return ("VM is dead due to a kernel crash:\n%s" % self.kernel_crash) - - -class VMAddressError(VMError): - pass - - -class VMPortNotRedirectedError(VMAddressError): - def __init__(self, port): - VMAddressError.__init__(self, port) - self.port = port - - def __str__(self): - return "Port not redirected: %s" % self.port - - -class VMAddressVerificationError(VMAddressError): - def __init__(self, mac, ip): - VMAddressError.__init__(self, mac, ip) - self.mac = mac - self.ip = ip - - def __str__(self): - return ("Cannot verify MAC-IP address mapping using arping: " - "%s ---> %s" % (self.mac, self.ip)) - - -class VMMACAddressMissingError(VMAddressError): - def __init__(self, nic_index): - VMAddressError.__init__(self, nic_index) - self.nic_index = nic_index - - def __str__(self): - return "No MAC address defined for NIC #%s" % self.nic_index - - -class VMIPAddressMissingError(VMAddressError): - def __init__(self, mac): - VMAddressError.__init__(self, mac) - self.mac = mac - - def __str__(self): - return "Cannot find IP address for MAC address %s" % self.mac - - -class VMMigrateError(VMError): - pass - - -class VMMigrateTimeoutError(VMMigrateError): - pass - - -class VMMigrateCancelError(VMMigrateError): - pass - - -class VMMigrateFailedError(VMMigrateError): - pass - - -class VMMigrateStateMismatchError(VMMigrateError): - def __init__(self, src_hash, dst_hash): - VMMigrateError.__init__(self, src_hash, dst_hash) - self.src_hash = src_hash - self.dst_hash = dst_hash - - def __str__(self): - return ("Mismatch of VM state before and after migration (%s != %s)" % - (self.src_hash, self.dst_hash)) - - -class VMRebootError(VMError): - pass - - -def get_image_filename(params, root_dir): - """ - Generate an image path from params and root_dir. - - @param params: Dictionary containing the test parameters. - @param root_dir: Base directory for relative filenames. - - @note: params should contain: - image_name -- the name of the image file, without extension - image_format -- the format of the image (qcow2, raw etc) - """ - image_name = params.get("image_name", "image") - image_format = params.get("image_format", "qcow2") - if params.get("image_raw_device") == "yes": - return image_name - image_filename = "%s.%s" % (image_name, image_format) - image_filename = kvm_utils.get_path(root_dir, image_filename) - return image_filename - - -def create_image(params, root_dir): - """ - Create an image using qemu_image. - - @param params: Dictionary containing the test parameters. - @param root_dir: Base directory for relative filenames. - - @note: params should contain: - image_name -- the name of the image file, without extension - image_format -- the format of the image (qcow2, raw etc) - image_size -- the requested size of the image (a string - qemu-img can understand, such as '10G') - """ - qemu_img_cmd = kvm_utils.get_path(root_dir, params.get("qemu_img_binary", - "qemu-img")) - qemu_img_cmd += " create" - - format = params.get("image_format", "qcow2") - qemu_img_cmd += " -f %s" % format - - image_filename = get_image_filename(params, root_dir) - qemu_img_cmd += " %s" % image_filename - - size = params.get("image_size", "10G") - qemu_img_cmd += " %s" % size - - utils.system(qemu_img_cmd) - logging.info("Image created in %r", image_filename) - return image_filename - - -def remove_image(params, root_dir): - """ - Remove an image file. - - @param params: A dict - @param root_dir: Base directory for relative filenames. - - @note: params should contain: - image_name -- the name of the image file, without extension - image_format -- the format of the image (qcow2, raw etc) - """ - image_filename = get_image_filename(params, root_dir) - logging.debug("Removing image file %s...", image_filename) - if os.path.exists(image_filename): - os.unlink(image_filename) - else: - logging.debug("Image file %s not found") - - -def check_image(params, root_dir): - """ - Check an image using qemu-img. - - @param params: Dictionary containing the test parameters. - @param root_dir: Base directory for relative filenames. - - @note: params should contain: - image_name -- the name of the image file, without extension - image_format -- the format of the image (qcow2, raw etc) - - @raise VMImageCheckError: In case qemu-img check fails on the image. - """ - image_filename = get_image_filename(params, root_dir) - logging.debug("Checking image file %s...", image_filename) - qemu_img_cmd = kvm_utils.get_path(root_dir, - params.get("qemu_img_binary", "qemu-img")) - image_is_qcow2 = params.get("image_format") == 'qcow2' - if os.path.exists(image_filename) and image_is_qcow2: - # Verifying if qemu-img supports 'check' - q_result = utils.run(qemu_img_cmd, ignore_status=True) - q_output = q_result.stdout - check_img = True - if not "check" in q_output: - logging.error("qemu-img does not support 'check', " - "skipping check...") - check_img = False - if not "info" in q_output: - logging.error("qemu-img does not support 'info', " - "skipping check...") - check_img = False - if check_img: - try: - utils.system("%s info %s" % (qemu_img_cmd, image_filename)) - except error.CmdError: - logging.error("Error getting info from image %s", - image_filename) - - cmd_result = utils.run("%s check %s" % - (qemu_img_cmd, image_filename), - ignore_status=True) - # Error check, large chances of a non-fatal problem. - # There are chances that bad data was skipped though - if cmd_result.exit_status == 1: - for e_line in cmd_result.stdout.splitlines(): - logging.error("[stdout] %s", e_line) - for e_line in cmd_result.stderr.splitlines(): - logging.error("[stderr] %s", e_line) - raise error.TestWarn("qemu-img check error. Some bad data in " - "the image may have gone unnoticed") - # Exit status 2 is data corruption for sure, so fail the test - elif cmd_result.exit_status == 2: - for e_line in cmd_result.stdout.splitlines(): - logging.error("[stdout] %s", e_line) - for e_line in cmd_result.stderr.splitlines(): - logging.error("[stderr] %s", e_line) - raise VMImageCheckError(image_filename) - # Leaked clusters, they are known to be harmless to data integrity - elif cmd_result.exit_status == 3: - raise error.TestWarn("Leaked clusters were noticed during " - "image check. No data integrity problem " - "was found though.") - - else: - if not os.path.exists(image_filename): - logging.debug("Image file %s not found, skipping check...", - image_filename) - elif not image_is_qcow2: - logging.debug("Image file %s not qcow2, skipping check...", - image_filename) - - -class VM: - """ - This class handles all basic VM operations. - """ - - def __init__(self, name, params, root_dir, address_cache, state=None): - """ - Initialize the object and set a few attributes. - - @param name: The name of the object - @param params: A dict containing VM params - (see method make_qemu_command for a full description) - @param root_dir: Base directory for relative filenames - @param address_cache: A dict that maps MAC addresses to IP addresses - @param state: If provided, use this as self.__dict__ - """ - if state: - self.__dict__ = state - else: - self.process = None - self.serial_console = None - self.redirs = {} - self.vnc_port = 5900 - self.monitors = [] - self.pci_assignable = None - self.netdev_id = [] - self.device_id = [] - self.uuid = None - - # Find a unique identifier for this VM - while True: - self.instance = (time.strftime("%Y%m%d-%H%M%S-") + - kvm_utils.generate_random_string(4)) - if not glob.glob("/tmp/*%s" % self.instance): - break - - self.spice_port = 8000 - self.name = name - self.params = params - self.root_dir = root_dir - self.address_cache = address_cache - - - def clone(self, name=None, params=None, root_dir=None, address_cache=None, - copy_state=False): - """ - Return a clone of the VM object with optionally modified parameters. - The clone is initially not alive and needs to be started using create(). - Any parameters not passed to this function are copied from the source - VM. - - @param name: Optional new VM name - @param params: Optional new VM creation parameters - @param root_dir: Optional new base directory for relative filenames - @param address_cache: A dict that maps MAC addresses to IP addresses - @param copy_state: If True, copy the original VM's state to the clone. - Mainly useful for make_qemu_command(). - """ - if name is None: - name = self.name - if params is None: - params = self.params.copy() - if root_dir is None: - root_dir = self.root_dir - if address_cache is None: - address_cache = self.address_cache - if copy_state: - state = self.__dict__.copy() - else: - state = None - return VM(name, params, root_dir, address_cache, state) - - - def make_qemu_command(self, name=None, params=None, root_dir=None): - """ - Generate a qemu command line. All parameters are optional. If a - parameter is not supplied, the corresponding value stored in the - class attributes is used. - - @param name: The name of the object - @param params: A dict containing VM params - @param root_dir: Base directory for relative filenames - - @note: The params dict should contain: - mem -- memory size in MBs - cdrom -- ISO filename to use with the qemu -cdrom parameter - extra_params -- a string to append to the qemu command - shell_port -- port of the remote shell daemon on the guest - (SSH, Telnet or the home-made Remote Shell Server) - shell_client -- client program to use for connecting to the - remote shell daemon on the guest (ssh, telnet or nc) - x11_display -- if specified, the DISPLAY environment variable - will be be set to this value for the qemu process (useful for - SDL rendering) - images -- a list of image object names, separated by spaces - nics -- a list of NIC object names, separated by spaces - - For each image in images: - drive_format -- string to pass as 'if' parameter for this - image (e.g. ide, scsi) - image_snapshot -- if yes, pass 'snapshot=on' to qemu for - this image - image_boot -- if yes, pass 'boot=on' to qemu for this image - In addition, all parameters required by get_image_filename. - - For each NIC in nics: - nic_model -- string to pass as 'model' parameter for this - NIC (e.g. e1000) - """ - # Helper function for command line option wrappers - def has_option(help, option): - return bool(re.search(r"^-%s(\s|$)" % option, help, re.MULTILINE)) - - # Wrappers for all supported qemu command line parameters. - # This is meant to allow support for multiple qemu versions. - # Each of these functions receives the output of 'qemu -help' as a - # parameter, and should add the requested command line option - # accordingly. - - def add_name(help, name): - return " -name '%s'" % name - - def add_human_monitor(help, filename): - return " -monitor unix:'%s',server,nowait" % filename - - def add_qmp_monitor(help, filename): - return " -qmp unix:'%s',server,nowait" % filename - - def add_serial(help, filename): - return " -serial unix:'%s',server,nowait" % filename - - def add_mem(help, mem): - return " -m %s" % mem - - def add_smp(help, smp): - return " -smp %s" % smp - - def add_cdrom(help, filename, index=None): - if has_option(help, "drive"): - cmd = " -drive file='%s',media=cdrom" % filename - if index is not None: cmd += ",index=%s" % index - return cmd - else: - return " -cdrom '%s'" % filename - - def add_drive(help, filename, index=None, format=None, cache=None, - werror=None, serial=None, snapshot=False, boot=False): - cmd = " -drive file='%s'" % filename - if index is not None: - cmd += ",index=%s" % index - if format: - cmd += ",if=%s" % format - if cache: - cmd += ",cache=%s" % cache - if werror: - cmd += ",werror=%s" % werror - if serial: - cmd += ",serial='%s'" % serial - if snapshot: - cmd += ",snapshot=on" - if boot: - cmd += ",boot=on" - return cmd - - def add_nic(help, vlan, model=None, mac=None, device_id=None, netdev_id=None, - nic_extra_params=None): - if has_option(help, "netdev"): - netdev_vlan_str = ",netdev=%s" % netdev_id - else: - netdev_vlan_str = ",vlan=%d" % vlan - if has_option(help, "device"): - if not model: - model = "rtl8139" - elif model == "virtio": - model = "virtio-net-pci" - cmd = " -device %s" % model + netdev_vlan_str - if mac: - cmd += ",mac='%s'" % mac - if nic_extra_params: - cmd += ",%s" % nic_extra_params - else: - cmd = " -net nic" + netdev_vlan_str - if model: - cmd += ",model=%s" % model - if mac: - cmd += ",macaddr='%s'" % mac - if device_id: - cmd += ",id='%s'" % device_id - return cmd - - def add_net(help, vlan, mode, ifname=None, script=None, - downscript=None, tftp=None, bootfile=None, hostfwd=[], - netdev_id=None, netdev_extra_params=None): - if has_option(help, "netdev"): - cmd = " -netdev %s,id=%s" % (mode, netdev_id) - if netdev_extra_params: - cmd += ",%s" % netdev_extra_params - else: - cmd = " -net %s,vlan=%d" % (mode, vlan) - if mode == "tap": - if ifname: cmd += ",ifname='%s'" % ifname - if script: cmd += ",script='%s'" % script - cmd += ",downscript='%s'" % (downscript or "no") - elif mode == "user": - if tftp and "[,tftp=" in help: - cmd += ",tftp='%s'" % tftp - if bootfile and "[,bootfile=" in help: - cmd += ",bootfile='%s'" % bootfile - if "[,hostfwd=" in help: - for host_port, guest_port in hostfwd: - cmd += ",hostfwd=tcp::%s-:%s" % (host_port, guest_port) - return cmd - - def add_floppy(help, filename): - return " -fda '%s'" % filename - - def add_tftp(help, filename): - # If the new syntax is supported, don't add -tftp - if "[,tftp=" in help: - return "" - else: - return " -tftp '%s'" % filename - - def add_bootp(help, filename): - # If the new syntax is supported, don't add -bootp - if "[,bootfile=" in help: - return "" - else: - return " -bootp '%s'" % filename - - def add_tcp_redir(help, host_port, guest_port): - # If the new syntax is supported, don't add -redir - if "[,hostfwd=" in help: - return "" - else: - return " -redir tcp:%s::%s" % (host_port, guest_port) - - def add_vnc(help, vnc_port): - return " -vnc :%d" % (vnc_port - 5900) - - def add_sdl(help): - if has_option(help, "sdl"): - return " -sdl" - else: - return "" - - def add_nographic(help): - return " -nographic" - - def add_uuid(help, uuid): - return " -uuid '%s'" % uuid - - def add_pcidevice(help, host): - return " -pcidevice host='%s'" % host - - def add_spice(help, port, param): - if has_option(help,"spice"): - return " -spice port=%s,%s" % (port, param) - else: - return "" - - def add_qxl_vga(help, qxl, vga, qxl_dev_nr=None): - str = "" - if has_option(help, "qxl"): - if qxl and qxl_dev_nr is not None: - str += " -qxl %s" % qxl_dev_nr - if has_option(help, "vga") and vga and vga != "qxl": - str += " -vga %s" % vga - elif has_option(help, "vga"): - if qxl: - str += " -vga qxl" - elif vga: - str += " -vga %s" % vga - return str - - def add_kernel(help, filename): - return " -kernel '%s'" % filename - - def add_initrd(help, filename): - return " -initrd '%s'" % filename - - def add_kernel_cmdline(help, cmdline): - return " -append %s" % cmdline - - def add_testdev(help, filename): - return (" -chardev file,id=testlog,path=%s" - " -device testdev,chardev=testlog" % filename) - - def add_no_hpet(help): - if has_option(help, "no-hpet"): - return " -no-hpet" - else: - return "" - - # End of command line option wrappers - - if name is None: - name = self.name - if params is None: - params = self.params - if root_dir is None: - root_dir = self.root_dir - - # Clone this VM using the new params - vm = self.clone(name, params, root_dir, copy_state=True) - - qemu_binary = kvm_utils.get_path(root_dir, params.get("qemu_binary", - "qemu")) - # Get the output of 'qemu -help' (log a message in case this call never - # returns or causes some other kind of trouble) - logging.debug("Getting output of 'qemu -help'") - help = commands.getoutput("%s -help" % qemu_binary) - - # Start constructing the qemu command - qemu_cmd = "" - # Set the X11 display parameter if requested - if params.get("x11_display"): - qemu_cmd += "DISPLAY=%s " % params.get("x11_display") - # Add the qemu binary - qemu_cmd += qemu_binary - # Add the VM's name - qemu_cmd += add_name(help, name) - # Add monitors - for monitor_name in params.objects("monitors"): - monitor_params = params.object_params(monitor_name) - monitor_filename = vm.get_monitor_filename(monitor_name) - if monitor_params.get("monitor_type") == "qmp": - qemu_cmd += add_qmp_monitor(help, monitor_filename) - else: - qemu_cmd += add_human_monitor(help, monitor_filename) - - # Add serial console redirection - qemu_cmd += add_serial(help, vm.get_serial_console_filename()) - - for image_name in params.objects("images"): - image_params = params.object_params(image_name) - if image_params.get("boot_drive") == "no": - continue - qemu_cmd += add_drive(help, - get_image_filename(image_params, root_dir), - image_params.get("drive_index"), - image_params.get("drive_format"), - image_params.get("drive_cache"), - image_params.get("drive_werror"), - image_params.get("drive_serial"), - image_params.get("image_snapshot") == "yes", - image_params.get("image_boot") == "yes") - - redirs = [] - for redir_name in params.objects("redirs"): - redir_params = params.object_params(redir_name) - guest_port = int(redir_params.get("guest_port")) - host_port = vm.redirs.get(guest_port) - redirs += [(host_port, guest_port)] - - vlan = 0 - for nic_name in params.objects("nics"): - nic_params = params.object_params(nic_name) - try: - netdev_id = vm.netdev_id[vlan] - device_id = vm.device_id[vlan] - except IndexError: - netdev_id = None - # Handle the '-net nic' part - try: - mac = vm.get_mac_address(vlan) - except VMAddressError: - mac = None - qemu_cmd += add_nic(help, vlan, nic_params.get("nic_model"), mac, - device_id, netdev_id, nic_params.get("nic_extra_params")) - # Handle the '-net tap' or '-net user' or '-netdev' part - script = nic_params.get("nic_script") - downscript = nic_params.get("nic_downscript") - tftp = nic_params.get("tftp") - if script: - script = kvm_utils.get_path(root_dir, script) - if downscript: - downscript = kvm_utils.get_path(root_dir, downscript) - if tftp: - tftp = kvm_utils.get_path(root_dir, tftp) - qemu_cmd += add_net(help, vlan, nic_params.get("nic_mode", "user"), - vm.get_ifname(vlan), - script, downscript, tftp, - nic_params.get("bootp"), redirs, netdev_id, - nic_params.get("netdev_extra_params")) - # Proceed to next NIC - vlan += 1 - - mem = params.get("mem") - if mem: - qemu_cmd += add_mem(help, mem) - - smp = params.get("smp") - if smp: - qemu_cmd += add_smp(help, smp) - - for cdrom in params.objects("cdroms"): - cdrom_params = params.object_params(cdrom) - iso = cdrom_params.get("cdrom") - if iso: - qemu_cmd += add_cdrom(help, kvm_utils.get_path(root_dir, iso), - cdrom_params.get("drive_index")) - - # We may want to add {floppy_otps} parameter for -fda - # {fat:floppy:}/path/. However vvfat is not usually recommended. - floppy = params.get("floppy") - if floppy: - floppy = kvm_utils.get_path(root_dir, floppy) - qemu_cmd += add_floppy(help, floppy) - - tftp = params.get("tftp") - if tftp: - tftp = kvm_utils.get_path(root_dir, tftp) - qemu_cmd += add_tftp(help, tftp) - - bootp = params.get("bootp") - if bootp: - qemu_cmd += add_bootp(help, bootp) - - kernel = params.get("kernel") - if kernel: - kernel = kvm_utils.get_path(root_dir, kernel) - qemu_cmd += add_kernel(help, kernel) - - kernel_cmdline = params.get("kernel_cmdline") - if kernel_cmdline: - qemu_cmd += add_kernel_cmdline(help, kernel_cmdline) - - initrd = params.get("initrd") - if initrd: - initrd = kvm_utils.get_path(root_dir, initrd) - qemu_cmd += add_initrd(help, initrd) - - for host_port, guest_port in redirs: - qemu_cmd += add_tcp_redir(help, host_port, guest_port) - - if params.get("display") == "vnc": - qemu_cmd += add_vnc(help, vm.vnc_port) - elif params.get("display") == "sdl": - qemu_cmd += add_sdl(help) - elif params.get("display") == "nographic": - qemu_cmd += add_nographic(help) - elif params.get("display") == "spice": - qemu_cmd += add_spice(help, self.spice_port, params.get("spice")) - - qxl = "" - vga = "" - if params.get("qxl"): - qxl = params.get("qxl") - if params.get("vga"): - vga = params.get("vga") - if qxl or vga: - if params.get("display") == "spice": - qxl_dev_nr = params.get("qxl_dev_nr", None) - qemu_cmd += add_qxl_vga(help, qxl, vga, qxl_dev_nr) - - if params.get("uuid") == "random": - qemu_cmd += add_uuid(help, vm.uuid) - elif params.get("uuid"): - qemu_cmd += add_uuid(help, params.get("uuid")) - - if params.get("testdev") == "yes": - qemu_cmd += add_testdev(help, vm.get_testlog_filename()) - - if params.get("disable_hpet") == "yes": - qemu_cmd += add_no_hpet(help) - - # If the PCI assignment step went OK, add each one of the PCI assigned - # devices to the qemu command line. - if vm.pci_assignable: - for pci_id in vm.pa_pci_ids: - qemu_cmd += add_pcidevice(help, pci_id) - - extra_params = params.get("extra_params") - if extra_params: - qemu_cmd += " %s" % extra_params - - return qemu_cmd - - - @error.context_aware - def create(self, name=None, params=None, root_dir=None, timeout=5.0, - migration_mode=None, mac_source=None): - """ - Start the VM by running a qemu command. - All parameters are optional. If name, params or root_dir are not - supplied, the respective values stored as class attributes are used. - - @param name: The name of the object - @param params: A dict containing VM params - @param root_dir: Base directory for relative filenames - @param migration_mode: If supplied, start VM for incoming migration - using this protocol (either 'tcp', 'unix' or 'exec') - @param migration_exec_cmd: Command to embed in '-incoming "exec: ..."' - (e.g. 'gzip -c -d filename') if migration_mode is 'exec' - @param mac_source: A VM object from which to copy MAC addresses. If not - specified, new addresses will be generated. - - @raise VMCreateError: If qemu terminates unexpectedly - @raise VMKVMInitError: If KVM initialization fails - @raise VMHugePageError: If hugepage initialization fails - @raise VMImageMissingError: If a CD image is missing - @raise VMHashMismatchError: If a CD image hash has doesn't match the - expected hash - @raise VMBadPATypeError: If an unsupported PCI assignment type is - requested - @raise VMPAError: If no PCI assignable devices could be assigned - """ - error.context("creating '%s'" % self.name) - self.destroy(free_mac_addresses=False) - - if name is not None: - self.name = name - if params is not None: - self.params = params - if root_dir is not None: - self.root_dir = root_dir - name = self.name - params = self.params - root_dir = self.root_dir - - # Verify the md5sum of the ISO images - for cdrom in params.objects("cdroms"): - cdrom_params = params.object_params(cdrom) - iso = cdrom_params.get("cdrom") - if iso: - iso = kvm_utils.get_path(root_dir, iso) - if not os.path.exists(iso): - raise VMImageMissingError(iso) - compare = False - if cdrom_params.get("md5sum_1m"): - logging.debug("Comparing expected MD5 sum with MD5 sum of " - "first MB of ISO file...") - actual_hash = utils.hash_file(iso, 1048576, method="md5") - expected_hash = cdrom_params.get("md5sum_1m") - compare = True - elif cdrom_params.get("md5sum"): - logging.debug("Comparing expected MD5 sum with MD5 sum of " - "ISO file...") - actual_hash = utils.hash_file(iso, method="md5") - expected_hash = cdrom_params.get("md5sum") - compare = True - elif cdrom_params.get("sha1sum"): - logging.debug("Comparing expected SHA1 sum with SHA1 sum " - "of ISO file...") - actual_hash = utils.hash_file(iso, method="sha1") - expected_hash = cdrom_params.get("sha1sum") - compare = True - if compare: - if actual_hash == expected_hash: - logging.debug("Hashes match") - else: - raise VMHashMismatchError(actual_hash, expected_hash) - - # Make sure the following code is not executed by more than one thread - # at the same time - lockfile = open("/tmp/kvm-autotest-vm-create.lock", "w+") - fcntl.lockf(lockfile, fcntl.LOCK_EX) - - try: - # Handle port redirections - redir_names = params.objects("redirs") - host_ports = kvm_utils.find_free_ports(5000, 6000, len(redir_names)) - self.redirs = {} - for i in range(len(redir_names)): - redir_params = params.object_params(redir_names[i]) - guest_port = int(redir_params.get("guest_port")) - self.redirs[guest_port] = host_ports[i] - - # Generate netdev/device IDs for all NICs - self.netdev_id = [] - self.device_id = [] - for nic in params.objects("nics"): - self.netdev_id.append(kvm_utils.generate_random_id()) - self.device_id.append(kvm_utils.generate_random_id()) - - # Find available VNC port, if needed - if params.get("display") == "vnc": - self.vnc_port = kvm_utils.find_free_port(5900, 6100) - - # Find available spice port - if params.get("spice"): - self.spice_port = kvm_utils.find_free_port(8000, 8100) - - # Find random UUID if specified 'uuid = random' in config file - if params.get("uuid") == "random": - f = open("/proc/sys/kernel/random/uuid") - self.uuid = f.read().strip() - f.close() - - # Generate or copy MAC addresses for all NICs - num_nics = len(params.objects("nics")) - for vlan in range(num_nics): - nic_name = params.objects("nics")[vlan] - nic_params = params.object_params(nic_name) - mac = (nic_params.get("nic_mac") or - mac_source and mac_source.get_mac_address(vlan)) - if mac: - kvm_utils.set_mac_address(self.instance, vlan, mac) - else: - kvm_utils.generate_mac_address(self.instance, vlan) - - # Assign a PCI assignable device - self.pci_assignable = None - pa_type = params.get("pci_assignable") - if pa_type and pa_type != "no": - pa_devices_requested = params.get("devices_requested") - - # Virtual Functions (VF) assignable devices - if pa_type == "vf": - self.pci_assignable = kvm_utils.PciAssignable( - type=pa_type, - driver=params.get("driver"), - driver_option=params.get("driver_option"), - devices_requested=pa_devices_requested) - # Physical NIC (PF) assignable devices - elif pa_type == "pf": - self.pci_assignable = kvm_utils.PciAssignable( - type=pa_type, - names=params.get("device_names"), - devices_requested=pa_devices_requested) - # Working with both VF and PF - elif pa_type == "mixed": - self.pci_assignable = kvm_utils.PciAssignable( - type=pa_type, - driver=params.get("driver"), - driver_option=params.get("driver_option"), - names=params.get("device_names"), - devices_requested=pa_devices_requested) - else: - raise VMBadPATypeError(pa_type) - - self.pa_pci_ids = self.pci_assignable.request_devs() - - if self.pa_pci_ids: - logging.debug("Successfuly assigned devices: %s", - self.pa_pci_ids) - else: - raise VMPAError(pa_type) - - # Make qemu command - qemu_command = self.make_qemu_command() - - # Add migration parameters if required - if migration_mode == "tcp": - self.migration_port = kvm_utils.find_free_port(5200, 6000) - qemu_command += " -incoming tcp:0:%d" % self.migration_port - elif migration_mode == "unix": - self.migration_file = "/tmp/migration-unix-%s" % self.instance - qemu_command += " -incoming unix:%s" % self.migration_file - elif migration_mode == "exec": - self.migration_port = kvm_utils.find_free_port(5200, 6000) - qemu_command += (' -incoming "exec:nc -l %s"' % - self.migration_port) - - logging.info("Running qemu command:\n%s", qemu_command) - self.process = kvm_subprocess.run_bg(qemu_command, None, - logging.info, "(qemu) ") - - # Make sure the process was started successfully - if not self.process.is_alive(): - e = VMCreateError(qemu_command, - self.process.get_status(), - self.process.get_output()) - self.destroy() - raise e - - # Establish monitor connections - self.monitors = [] - for monitor_name in params.objects("monitors"): - monitor_params = params.object_params(monitor_name) - # Wait for monitor connection to succeed - end_time = time.time() + timeout - while time.time() < end_time: - try: - if monitor_params.get("monitor_type") == "qmp": - # Add a QMP monitor - monitor = kvm_monitor.QMPMonitor( - monitor_name, - self.get_monitor_filename(monitor_name)) - else: - # Add a "human" monitor - monitor = kvm_monitor.HumanMonitor( - monitor_name, - self.get_monitor_filename(monitor_name)) - monitor.verify_responsive() - break - except kvm_monitor.MonitorError, e: - logging.warn(e) - time.sleep(1) - else: - self.destroy() - raise e - # Add this monitor to the list - self.monitors += [monitor] - - # Get the output so far, to see if we have any problems with - # KVM modules or with hugepage setup. - output = self.process.get_output() - - if re.search("Could not initialize KVM", output, re.IGNORECASE): - e = VMKVMInitError(qemu_command, self.process.get_output()) - self.destroy() - raise e - - if "alloc_mem_area" in output: - e = VMHugePageError(qemu_command, self.process.get_output()) - self.destroy() - raise e - - logging.debug("VM appears to be alive with PID %s", self.get_pid()) - - # Establish a session with the serial console -- requires a version - # of netcat that supports -U - self.serial_console = kvm_subprocess.ShellSession( - "nc -U %s" % self.get_serial_console_filename(), - auto_close=False, - output_func=kvm_utils.log_line, - output_params=("serial-%s.log" % name,)) - - finally: - fcntl.lockf(lockfile, fcntl.LOCK_UN) - lockfile.close() - - - def destroy(self, gracefully=True, free_mac_addresses=True): - """ - Destroy the VM. - - If gracefully is True, first attempt to shutdown the VM with a shell - command. Then, attempt to destroy the VM via the monitor with a 'quit' - command. If that fails, send SIGKILL to the qemu process. - - @param gracefully: If True, an attempt will be made to end the VM - using a shell command before trying to end the qemu process - with a 'quit' or a kill signal. - @param free_mac_addresses: If True, the MAC addresses used by the VM - will be freed. - """ - try: - # Is it already dead? - if self.is_dead(): - return - - logging.debug("Destroying VM with PID %s...", self.get_pid()) - - if gracefully and self.params.get("shutdown_command"): - # Try to destroy with shell command - logging.debug("Trying to shutdown VM with shell command...") - try: - session = self.login() - except (kvm_utils.LoginError, VMError), e: - logging.debug(e) - else: - try: - # Send the shutdown command - session.sendline(self.params.get("shutdown_command")) - logging.debug("Shutdown command sent; waiting for VM " - "to go down...") - if kvm_utils.wait_for(self.is_dead, 60, 1, 1): - logging.debug("VM is down") - return - finally: - session.close() - - if self.monitor: - # Try to destroy with a monitor command - logging.debug("Trying to kill VM with monitor command...") - try: - self.monitor.quit() - except kvm_monitor.MonitorError, e: - logging.warn(e) - else: - # Wait for the VM to be really dead - if kvm_utils.wait_for(self.is_dead, 5, 0.5, 0.5): - logging.debug("VM is down") - return - - # If the VM isn't dead yet... - logging.debug("Cannot quit normally; sending a kill to close the " - "deal...") - kvm_utils.kill_process_tree(self.process.get_pid(), 9) - # Wait for the VM to be really dead - if kvm_utils.wait_for(self.is_dead, 5, 0.5, 0.5): - logging.debug("VM is down") - return - - logging.error("Process %s is a zombie!", self.process.get_pid()) - - finally: - self.monitors = [] - if self.pci_assignable: - self.pci_assignable.release_devs() - if self.process: - self.process.close() - if self.serial_console: - self.serial_console.close() - for f in ([self.get_testlog_filename(), - self.get_serial_console_filename()] + - self.get_monitor_filenames()): - try: - os.unlink(f) - except OSError: - pass - if hasattr(self, "migration_file"): - try: - os.unlink(self.migration_file) - except OSError: - pass - if free_mac_addresses: - num_nics = len(self.params.objects("nics")) - for vlan in range(num_nics): - self.free_mac_address(vlan) - - - @property - def monitor(self): - """ - Return the main monitor object, selected by the parameter main_monitor. - If main_monitor isn't defined, return the first monitor. - If no monitors exist, or if main_monitor refers to a nonexistent - monitor, return None. - """ - for m in self.monitors: - if m.name == self.params.get("main_monitor"): - return m - if self.monitors and not self.params.get("main_monitor"): - return self.monitors[0] - - - def verify_alive(self): - """ - Make sure the VM is alive and that the main monitor is responsive. - - @raise VMDeadError: If the VM is dead - @raise: Various monitor exceptions if the monitor is unresponsive - """ - if self.is_dead(): - raise VMDeadError(self.process.get_status(), - self.process.get_output()) - if self.monitors: - self.monitor.verify_responsive() - - - def is_alive(self): - """ - Return True if the VM is alive and its monitor is responsive. - """ - return not self.is_dead() and (not self.monitors or - self.monitor.is_responsive()) - - - def is_dead(self): - """ - Return True if the qemu process is dead. - """ - return not self.process or not self.process.is_alive() - - - def verify_kernel_crash(self): - """ - Find kernel crash message on the VM serial console. - - @raise: VMDeadKernelCrashError, in case a kernel crash message was - found. - """ - data = self.serial_console.get_output() - match = re.search(r"BUG:.*---\[ end trace .* \]---", data, - re.DOTALL|re.MULTILINE) - if match is not None: - raise VMDeadKernelCrashError(match.group(0)) - - - def get_params(self): - """ - Return the VM's params dict. Most modified params take effect only - upon VM.create(). - """ - return self.params - - - def get_monitor_filename(self, monitor_name): - """ - Return the filename corresponding to a given monitor name. - """ - return "/tmp/monitor-%s-%s" % (monitor_name, self.instance) - - - def get_monitor_filenames(self): - """ - Return a list of all monitor filenames (as specified in the VM's - params). - """ - return [self.get_monitor_filename(m) for m in - self.params.objects("monitors")] - - - def get_serial_console_filename(self): - """ - Return the serial console filename. - """ - return "/tmp/serial-%s" % self.instance - - - def get_testlog_filename(self): - """ - Return the testlog filename. - """ - return "/tmp/testlog-%s" % self.instance - - - def get_address(self, index=0): - """ - Return the address of a NIC of the guest, in host space. - - If port redirection is used, return 'localhost' (the NIC has no IP - address of its own). Otherwise return the NIC's IP address. - - @param index: Index of the NIC whose address is requested. - @raise VMMACAddressMissingError: If no MAC address is defined for the - requested NIC - @raise VMIPAddressMissingError: If no IP address is found for the the - NIC's MAC address - @raise VMAddressVerificationError: If the MAC-IP address mapping cannot - be verified (using arping) - """ - nics = self.params.objects("nics") - nic_name = nics[index] - nic_params = self.params.object_params(nic_name) - if nic_params.get("nic_mode") == "tap": - mac = self.get_mac_address(index).lower() - # Get the IP address from the cache - ip = self.address_cache.get(mac) - if not ip: - raise VMIPAddressMissingError(mac) - # Make sure the IP address is assigned to this guest - macs = [self.get_mac_address(i) for i in range(len(nics))] - if not kvm_utils.verify_ip_address_ownership(ip, macs): - raise VMAddressVerificationError(mac, ip) - return ip - else: - return "localhost" - - - def get_port(self, port, nic_index=0): - """ - Return the port in host space corresponding to port in guest space. - - @param port: Port number in host space. - @param nic_index: Index of the NIC. - @return: If port redirection is used, return the host port redirected - to guest port port. Otherwise return port. - @raise VMPortNotRedirectedError: If an unredirected port is requested - in user mode - """ - nic_name = self.params.objects("nics")[nic_index] - nic_params = self.params.object_params(nic_name) - if nic_params.get("nic_mode") == "tap": - return port - else: - try: - return self.redirs[port] - except KeyError: - raise VMPortNotRedirectedError(port) - - - def get_peer(self, netid): - """ - Return the peer of netdev or network deivce. - - @param netid: id of netdev or device - @return: id of the peer device otherwise None - """ - network_info = self.monitor.info("network") - try: - return re.findall("%s:.*peer=(.*)" % netid, network_info)[0] - except IndexError: - return None - - - def get_ifname(self, nic_index=0): - """ - Return the ifname of a tap device associated with a NIC. - - @param nic_index: Index of the NIC - """ - nics = self.params.objects("nics") - nic_name = nics[nic_index] - nic_params = self.params.object_params(nic_name) - if nic_params.get("nic_ifname"): - return nic_params.get("nic_ifname") - else: - return "t%d-%s" % (nic_index, self.instance[-11:]) - - - def get_mac_address(self, nic_index=0): - """ - Return the MAC address of a NIC. - - @param nic_index: Index of the NIC - @raise VMMACAddressMissingError: If no MAC address is defined for the - requested NIC - """ - nic_name = self.params.objects("nics")[nic_index] - nic_params = self.params.object_params(nic_name) - mac = (nic_params.get("nic_mac") or - kvm_utils.get_mac_address(self.instance, nic_index)) - if not mac: - raise VMMACAddressMissingError(nic_index) - return mac - - - def free_mac_address(self, nic_index=0): - """ - Free a NIC's MAC address. - - @param nic_index: Index of the NIC - """ - kvm_utils.free_mac_address(self.instance, nic_index) - - - def get_pid(self): - """ - Return the VM's PID. If the VM is dead return None. - - @note: This works under the assumption that self.process.get_pid() - returns the PID of the parent shell process. - """ - try: - children = commands.getoutput("ps --ppid=%d -o pid=" % - self.process.get_pid()).split() - return int(children[0]) - except (TypeError, IndexError, ValueError): - return None - - - def get_shell_pid(self): - """ - Return the PID of the parent shell process. - - @note: This works under the assumption that self.process.get_pid() - returns the PID of the parent shell process. - """ - return self.process.get_pid() - - - def get_shared_meminfo(self): - """ - Returns the VM's shared memory information. - - @return: Shared memory used by VM (MB) - """ - if self.is_dead(): - logging.error("Could not get shared memory info from dead VM.") - return None - - filename = "/proc/%d/statm" % self.get_pid() - shm = int(open(filename).read().split()[2]) - # statm stores informations in pages, translate it to MB - return shm * 4.0 / 1024 - - - @error.context_aware - def login(self, nic_index=0, timeout=10): - """ - Log into the guest via SSH/Telnet/Netcat. - If timeout expires while waiting for output from the guest (e.g. a - password prompt or a shell prompt) -- fail. - - @param nic_index: The index of the NIC to connect to. - @param timeout: Time (seconds) before giving up logging into the - guest. - @return: A ShellSession object. - """ - error.context("logging into '%s'" % self.name) - username = self.params.get("username", "") - password = self.params.get("password", "") - prompt = self.params.get("shell_prompt", "[\#\$]") - linesep = eval("'%s'" % self.params.get("shell_linesep", r"\n")) - client = self.params.get("shell_client") - address = self.get_address(nic_index) - port = self.get_port(int(self.params.get("shell_port"))) - log_filename = ("session-%s-%s.log" % - (self.name, kvm_utils.generate_random_string(4))) - session = kvm_utils.remote_login(client, address, port, username, - password, prompt, linesep, - log_filename, timeout) - session.set_status_test_command(self.params.get("status_test_command", - "")) - return session - - - def remote_login(self, nic_index=0, timeout=10): - """ - Alias for login() for backward compatibility. - """ - return self.login(nic_index, timeout) - - - def wait_for_login(self, nic_index=0, timeout=240, internal_timeout=10): - """ - Make multiple attempts to log into the guest via SSH/Telnet/Netcat. - - @param nic_index: The index of the NIC to connect to. - @param timeout: Time (seconds) to keep trying to log in. - @param internal_timeout: Timeout to pass to login(). - @return: A ShellSession object. - """ - logging.debug("Attempting to log into '%s' (timeout %ds)", self.name, - timeout) - end_time = time.time() + timeout - while time.time() < end_time: - try: - return self.login(nic_index, internal_timeout) - except (kvm_utils.LoginError, VMError), e: - logging.debug(e) - time.sleep(2) - # Timeout expired; try one more time but don't catch exceptions - return self.login(nic_index, internal_timeout) - - - @error.context_aware - def copy_files_to(self, host_path, guest_path, nic_index=0, verbose=False, - timeout=600): - """ - Transfer files to the remote host(guest). - - @param host_path: Host path - @param guest_path: Guest path - @param nic_index: The index of the NIC to connect to. - @param verbose: If True, log some stats using logging.debug (RSS only) - @param timeout: Time (seconds) before giving up on doing the remote - copy. - """ - error.context("sending file(s) to '%s'" % self.name) - username = self.params.get("username", "") - password = self.params.get("password", "") - client = self.params.get("file_transfer_client") - address = self.get_address(nic_index) - port = self.get_port(int(self.params.get("file_transfer_port"))) - log_filename = ("transfer-%s-to-%s-%s.log" % - (self.name, address, - kvm_utils.generate_random_string(4))) - kvm_utils.copy_files_to(address, client, username, password, port, - host_path, guest_path, log_filename, verbose, - timeout) - - - @error.context_aware - def copy_files_from(self, guest_path, host_path, nic_index=0, - verbose=False, timeout=600): - """ - Transfer files from the guest. - - @param host_path: Guest path - @param guest_path: Host path - @param nic_index: The index of the NIC to connect to. - @param verbose: If True, log some stats using logging.debug (RSS only) - @param timeout: Time (seconds) before giving up on doing the remote - copy. - """ - error.context("receiving file(s) from '%s'" % self.name) - username = self.params.get("username", "") - password = self.params.get("password", "") - client = self.params.get("file_transfer_client") - address = self.get_address(nic_index) - port = self.get_port(int(self.params.get("file_transfer_port"))) - log_filename = ("transfer-%s-from-%s-%s.log" % - (self.name, address, - kvm_utils.generate_random_string(4))) - kvm_utils.copy_files_from(address, client, username, password, port, - guest_path, host_path, log_filename, - verbose, timeout) - - - @error.context_aware - def serial_login(self, timeout=10): - """ - Log into the guest via the serial console. - If timeout expires while waiting for output from the guest (e.g. a - password prompt or a shell prompt) -- fail. - - @param timeout: Time (seconds) before giving up logging into the guest. - @return: ShellSession object on success and None on failure. - """ - error.context("logging into '%s' via serial console" % self.name) - username = self.params.get("username", "") - password = self.params.get("password", "") - prompt = self.params.get("shell_prompt", "[\#\$]") - linesep = eval("'%s'" % self.params.get("shell_linesep", r"\n")) - status_test_command = self.params.get("status_test_command", "") - - self.serial_console.set_linesep(linesep) - self.serial_console.set_status_test_command(status_test_command) - - # Try to get a login prompt - self.serial_console.sendline() - - kvm_utils._remote_login(self.serial_console, username, password, - prompt, timeout) - return self.serial_console - - - def wait_for_serial_login(self, timeout=240, internal_timeout=10): - """ - Make multiple attempts to log into the guest via serial console. - - @param timeout: Time (seconds) to keep trying to log in. - @param internal_timeout: Timeout to pass to serial_login(). - @return: A ShellSession object. - """ - logging.debug("Attempting to log into '%s' via serial console " - "(timeout %ds)", self.name, timeout) - end_time = time.time() + timeout - while time.time() < end_time: - try: - return self.serial_login(internal_timeout) - except kvm_utils.LoginError, e: - logging.debug(e) - time.sleep(2) - # Timeout expired; try one more time but don't catch exceptions - return self.serial_login(internal_timeout) - - - @error.context_aware - def migrate(self, timeout=3600, protocol="tcp", cancel_delay=None, - offline=False, stable_check=False, clean=True, - save_path="/tmp", dest_host="localhost", remote_port=None): - """ - Migrate the VM. - - If the migration is local, the VM object's state is switched with that - of the destination VM. Otherwise, the state is switched with that of - a dead VM (returned by self.clone()). - - @param timeout: Time to wait for migration to complete. - @param protocol: Migration protocol ('tcp', 'unix' or 'exec'). - @param cancel_delay: If provided, specifies a time duration after which - migration will be canceled. Used for testing migrate_cancel. - @param offline: If True, pause the source VM before migration. - @param stable_check: If True, compare the VM's state after migration to - its state before migration and raise an exception if they - differ. - @param clean: If True, delete the saved state files (relevant only if - stable_check is also True). - @save_path: The path for state files. - @param dest_host: Destination host (defaults to 'localhost'). - @param remote_port: Port to use for remote migration. - """ - error.base_context("migrating '%s'" % self.name) - - def mig_finished(): - o = self.monitor.info("migrate") - if isinstance(o, str): - return "status: active" not in o - else: - return o.get("status") != "active" - - def mig_succeeded(): - o = self.monitor.info("migrate") - if isinstance(o, str): - return "status: completed" in o - else: - return o.get("status") == "completed" - - def mig_failed(): - o = self.monitor.info("migrate") - if isinstance(o, str): - return "status: failed" in o - else: - return o.get("status") == "failed" - - def mig_cancelled(): - o = self.monitor.info("migrate") - if isinstance(o, str): - return ("Migration status: cancelled" in o or - "Migration status: canceled" in o) - else: - return (o.get("status") == "cancelled" or - o.get("status") == "canceled") - - def wait_for_migration(): - if not kvm_utils.wait_for(mig_finished, timeout, 2, 2, - "Waiting for migration to complete"): - raise VMMigrateTimeoutError("Timeout expired while waiting " - "for migration to finish") - - local = dest_host == "localhost" - - clone = self.clone() - if local: - error.context("creating destination VM") - if stable_check: - # Pause the dest vm after creation - extra_params = clone.params.get("extra_params", "") + " -S" - clone.params["extra_params"] = extra_params - clone.create(migration_mode=protocol, mac_source=self) - error.context() - - try: - if protocol == "tcp": - if local: - uri = "tcp:localhost:%d" % clone.migration_port - else: - uri = "tcp:%s:%d" % (dest_host, remote_port) - elif protocol == "unix": - uri = "unix:%s" % clone.migration_file - elif protocol == "exec": - uri = '"exec:nc localhost %s"' % clone.migration_port - - if offline: - self.monitor.cmd("stop") - - logging.info("Migrating to %s", uri) - self.monitor.migrate(uri) - - if cancel_delay: - time.sleep(cancel_delay) - self.monitor.cmd("migrate_cancel") - if not kvm_utils.wait_for(mig_cancelled, 60, 2, 2, - "Waiting for migration " - "cancellation"): - raise VMMigrateCancelError("Cannot cancel migration") - return - - wait_for_migration() - - # Report migration status - if mig_succeeded(): - logging.info("Migration completed successfully") - elif mig_failed(): - raise VMMigrateFailedError("Migration failed") - else: - raise VMMigrateFailedError("Migration ended with unknown " - "status") - - # Switch self <-> clone - temp = self.clone(copy_state=True) - self.__dict__ = clone.__dict__ - clone = temp - - # From now on, clone is the source VM that will soon be destroyed - # and self is the destination VM that will remain alive. If this - # is remote migration, self is a dead VM object. - - error.context("after migration") - if local: - time.sleep(1) - self.verify_alive() - self.verify_kernel_crash() - - if local and stable_check: - try: - save1 = os.path.join(save_path, "src-" + clone.instance) - save2 = os.path.join(save_path, "dst-" + self.instance) - clone.save_to_file(save1) - self.save_to_file(save2) - # Fail if we see deltas - md5_save1 = utils.hash_file(save1) - md5_save2 = utils.hash_file(save2) - if md5_save1 != md5_save2: - raise VMMigrateStateMismatchError(md5_save1, md5_save2) - finally: - if clean: - if os.path.isfile(save1): - os.remove(save1) - if os.path.isfile(save2): - os.remove(save2) - - finally: - # If we're doing remote migration and it's completed successfully, - # self points to a dead VM object - if self.is_alive(): - self.monitor.cmd("cont") - clone.destroy(gracefully=False) - - - @error.context_aware - def reboot(self, session=None, method="shell", nic_index=0, timeout=240): - """ - Reboot the VM and wait for it to come back up by trying to log in until - timeout expires. - - @param session: A shell session object or None. - @param method: Reboot method. Can be "shell" (send a shell reboot - command) or "system_reset" (send a system_reset monitor command). - @param nic_index: Index of NIC to access in the VM, when logging in - after rebooting. - @param timeout: Time to wait for login to succeed (after rebooting). - @return: A new shell session object. - """ - error.base_context("rebooting '%s'" % self.name, logging.info) - error.context("before reboot") - session = session or self.login() - error.context() - - if method == "shell": - session.sendline(self.params.get("reboot_command")) - elif method == "system_reset": - # Clear the event list of all QMP monitors - qmp_monitors = [m for m in self.monitors if m.protocol == "qmp"] - for m in qmp_monitors: - m.clear_events() - # Send a system_reset monitor command - self.monitor.cmd("system_reset") - # Look for RESET QMP events - time.sleep(1) - for m in qmp_monitors: - if m.get_event("RESET"): - logging.info("RESET QMP event received") - else: - raise VMRebootError("RESET QMP event not received after " - "system_reset (monitor '%s')" % m.name) - else: - raise VMRebootError("Unknown reboot method: %s" % method) - - error.context("waiting for guest to go down", logging.info) - if not kvm_utils.wait_for(lambda: - not session.is_responsive(timeout=30), - 120, 0, 1): - raise VMRebootError("Guest refuses to go down") - session.close() - - error.context("logging in after reboot", logging.info) - return self.wait_for_login(nic_index, timeout=timeout) - - - def send_key(self, keystr): - """ - Send a key event to the VM. - - @param: keystr: A key event string (e.g. "ctrl-alt-delete") - """ - # For compatibility with versions of QEMU that do not recognize all - # key names: replace keyname with the hex value from the dict, which - # QEMU will definitely accept - dict = {"comma": "0x33", - "dot": "0x34", - "slash": "0x35"} - for key, value in dict.items(): - keystr = keystr.replace(key, value) - self.monitor.sendkey(keystr) - time.sleep(0.2) - - - def send_string(self, str): - """ - Send a string to the VM. - - @param str: String, that must consist of alphanumeric characters only. - Capital letters are allowed. - """ - for char in str: - if char.isupper(): - self.send_key("shift-%s" % char.lower()) - else: - self.send_key(char) - - - def get_uuid(self): - """ - Catch UUID of the VM. - - @return: None,if not specified in config file - """ - if self.params.get("uuid") == "random": - return self.uuid - else: - return self.params.get("uuid", None) - - - def get_cpu_count(self): - """ - Get the cpu count of the VM. - """ - session = self.login() - try: - return int(session.cmd(self.params.get("cpu_chk_cmd"))) - finally: - session.close() - - - def get_memory_size(self, cmd=None): - """ - Get bootup memory size of the VM. - - @param check_cmd: Command used to check memory. If not provided, - self.params.get("mem_chk_cmd") will be used. - """ - session = self.login() - try: - if not cmd: - cmd = self.params.get("mem_chk_cmd") - mem_str = session.cmd(cmd) - mem = re.findall("([0-9]+)", mem_str) - mem_size = 0 - for m in mem: - mem_size += int(m) - if "GB" in mem_str: - mem_size *= 1024 - elif "MB" in mem_str: - pass - else: - mem_size /= 1024 - return int(mem_size) - finally: - session.close() - - - def get_current_memory_size(self): - """ - Get current memory size of the VM, rather than bootup memory. - """ - cmd = self.params.get("mem_chk_cur_cmd") - return self.get_memory_size(cmd) - - - def save_to_file(self, path): - """ - Save the state of virtual machine to a file through migrate to - exec - """ - # Make sure we only get one iteration - self.monitor.cmd("migrate_set_speed 1000g") - self.monitor.cmd("migrate_set_downtime 100000000") - self.monitor.migrate('"exec:cat>%s"' % path) - # Restore the speed and downtime of migration - self.monitor.cmd("migrate_set_speed %d" % (32<<20)) - self.monitor.cmd("migrate_set_downtime 0.03") diff --git a/client/tests/kvm/ppm_utils.py b/client/tests/kvm/ppm_utils.py deleted file mode 100644 index 90ff46de..00000000 --- a/client/tests/kvm/ppm_utils.py +++ /dev/null @@ -1,237 +0,0 @@ -""" -Utility functions to deal with ppm (qemu screendump format) files. - -@copyright: Red Hat 2008-2009 -""" - -import os, struct, time, re -from autotest_lib.client.bin import utils - -# Some directory/filename utils, for consistency - -def find_id_for_screendump(md5sum, dir): - """ - Search dir for a PPM file whose name ends with md5sum. - - @param md5sum: md5 sum string - @param dir: Directory that holds the PPM files. - @return: The file's basename without any preceding path, e.g. - '20080101_120000_d41d8cd98f00b204e9800998ecf8427e.ppm'. - """ - try: - files = os.listdir(dir) - except OSError: - files = [] - for file in files: - exp = re.compile(r"(.*_)?" + md5sum + r"\.ppm", re.IGNORECASE) - if exp.match(file): - return file - - -def generate_id_for_screendump(md5sum, dir): - """ - Generate a unique filename using the given MD5 sum. - - @return: Only the file basename, without any preceding path. The - filename consists of the current date and time, the MD5 sum and a .ppm - extension, e.g. '20080101_120000_d41d8cd98f00b204e9800998ecf8427e.ppm'. - """ - filename = time.strftime("%Y%m%d_%H%M%S") + "_" + md5sum + ".ppm" - return filename - - -def get_data_dir(steps_filename): - """ - Return the data dir of the given steps filename. - """ - filename = os.path.basename(steps_filename) - return os.path.join(os.path.dirname(steps_filename), "..", "steps_data", - filename + "_data") - - -# Functions for working with PPM files - -def image_read_from_ppm_file(filename): - """ - Read a PPM image. - - @return: A 3 element tuple containing the width, height and data of the - image. - """ - fin = open(filename,"rb") - l1 = fin.readline() - l2 = fin.readline() - l3 = fin.readline() - data = fin.read() - fin.close() - - (w, h) = map(int, l2.split()) - return (w, h, data) - - -def image_write_to_ppm_file(filename, width, height, data): - """ - Write a PPM image with the given width, height and data. - - @param filename: PPM file path - @param width: PPM file width (pixels) - @param height: PPM file height (pixels) - """ - fout = open(filename,"wb") - fout.write("P6\n") - fout.write("%d %d\n" % (width, height)) - fout.write("255\n") - fout.write(data) - fout.close() - - -def image_crop(width, height, data, x1, y1, dx, dy): - """ - Crop an image. - - @param width: Original image width - @param height: Original image height - @param data: Image data - @param x1: Desired x coordinate of the cropped region - @param y1: Desired y coordinate of the cropped region - @param dx: Desired width of the cropped region - @param dy: Desired height of the cropped region - @return: A 3-tuple containing the width, height and data of the - cropped image. - """ - if x1 > width - 1: x1 = width - 1 - if y1 > height - 1: y1 = height - 1 - if dx > width - x1: dx = width - x1 - if dy > height - y1: dy = height - y1 - newdata = "" - index = (x1 + y1*width) * 3 - for i in range(dy): - newdata += data[index:(index+dx*3)] - index += width*3 - return (dx, dy, newdata) - - -def image_md5sum(width, height, data): - """ - Return the md5sum of an image. - - @param width: PPM file width - @param height: PPM file height - @data: PPM file data - """ - header = "P6\n%d %d\n255\n" % (width, height) - hash = utils.hash('md5', header) - hash.update(data) - return hash.hexdigest() - - -def get_region_md5sum(width, height, data, x1, y1, dx, dy, - cropped_image_filename=None): - """ - Return the md5sum of a cropped region. - - @param width: Original image width - @param height: Original image height - @param data: Image data - @param x1: Desired x coord of the cropped region - @param y1: Desired y coord of the cropped region - @param dx: Desired width of the cropped region - @param dy: Desired height of the cropped region - @param cropped_image_filename: if not None, write the resulting cropped - image to a file with this name - """ - (cw, ch, cdata) = image_crop(width, height, data, x1, y1, dx, dy) - # Write cropped image for debugging - if cropped_image_filename: - image_write_to_ppm_file(cropped_image_filename, cw, ch, cdata) - return image_md5sum(cw, ch, cdata) - - -def image_verify_ppm_file(filename): - """ - Verify the validity of a PPM file. - - @param filename: Path of the file being verified. - @return: True if filename is a valid PPM image file. This function - reads only the first few bytes of the file so it should be rather fast. - """ - try: - size = os.path.getsize(filename) - fin = open(filename, "rb") - assert(fin.readline().strip() == "P6") - (width, height) = map(int, fin.readline().split()) - assert(width > 0 and height > 0) - assert(fin.readline().strip() == "255") - size_read = fin.tell() - fin.close() - assert(size - size_read == width*height*3) - return True - except: - return False - - -def image_comparison(width, height, data1, data2): - """ - Generate a green-red comparison image from two given images. - - @param width: Width of both images - @param height: Height of both images - @param data1: Data of first image - @param data2: Data of second image - @return: A 3-element tuple containing the width, height and data of the - generated comparison image. - - @note: Input images must be the same size. - """ - newdata = "" - i = 0 - while i < width*height*3: - # Compute monochromatic value of current pixel in data1 - pixel1_str = data1[i:i+3] - temp = struct.unpack("BBB", pixel1_str) - value1 = int((temp[0] + temp[1] + temp[2]) / 3) - # Compute monochromatic value of current pixel in data2 - pixel2_str = data2[i:i+3] - temp = struct.unpack("BBB", pixel2_str) - value2 = int((temp[0] + temp[1] + temp[2]) / 3) - # Compute average of the two values - value = int((value1 + value2) / 2) - # Scale value to the upper half of the range [0, 255] - value = 128 + value / 2 - # Compare pixels - if pixel1_str == pixel2_str: - # Equal -- give the pixel a greenish hue - newpixel = [0, value, 0] - else: - # Not equal -- give the pixel a reddish hue - newpixel = [value, 0, 0] - newdata += struct.pack("BBB", newpixel[0], newpixel[1], newpixel[2]) - i += 3 - return (width, height, newdata) - - -def image_fuzzy_compare(width, height, data1, data2): - """ - Return the degree of equality of two given images. - - @param width: Width of both images - @param height: Height of both images - @param data1: Data of first image - @param data2: Data of second image - @return: Ratio equal_pixel_count / total_pixel_count. - - @note: Input images must be the same size. - """ - equal = 0.0 - different = 0.0 - i = 0 - while i < width*height*3: - pixel1_str = data1[i:i+3] - pixel2_str = data2[i:i+3] - # Compare pixels - if pixel1_str == pixel2_str: - equal += 1.0 - else: - different += 1.0 - i += 3 - return equal / (equal + different) diff --git a/client/tests/kvm/rss_file_transfer.py b/client/tests/kvm/rss_file_transfer.py deleted file mode 100755 index 4d00d174..00000000 --- a/client/tests/kvm/rss_file_transfer.py +++ /dev/null @@ -1,519 +0,0 @@ -#!/usr/bin/python -""" -Client for file transfer services offered by RSS (Remote Shell Server). - -@author: Michael Goldish (mgoldish@redhat.com) -@copyright: 2008-2010 Red Hat Inc. -""" - -import socket, struct, time, sys, os, glob - -# Globals -CHUNKSIZE = 65536 - -# Protocol message constants -RSS_MAGIC = 0x525353 -RSS_OK = 1 -RSS_ERROR = 2 -RSS_UPLOAD = 3 -RSS_DOWNLOAD = 4 -RSS_SET_PATH = 5 -RSS_CREATE_FILE = 6 -RSS_CREATE_DIR = 7 -RSS_LEAVE_DIR = 8 -RSS_DONE = 9 - -# See rss.cpp for protocol details. - - -class FileTransferError(Exception): - def __init__(self, msg, e=None, filename=None): - Exception.__init__(self, msg, e, filename) - self.msg = msg - self.e = e - self.filename = filename - - def __str__(self): - s = self.msg - if self.e and self.filename: - s += " (error: %s, filename: %s)" % (self.e, self.filename) - elif self.e: - s += " (%s)" % self.e - elif self.filename: - s += " (filename: %s)" % self.filename - return s - - -class FileTransferConnectError(FileTransferError): - pass - - -class FileTransferTimeoutError(FileTransferError): - pass - - -class FileTransferProtocolError(FileTransferError): - pass - - -class FileTransferSocketError(FileTransferError): - pass - - -class FileTransferServerError(FileTransferError): - def __init__(self, errmsg): - FileTransferError.__init__(self, None, errmsg) - - def __str__(self): - s = "Server said: %r" % self.e - if self.filename: - s += " (filename: %s)" % self.filename - return s - - -class FileTransferNotFoundError(FileTransferError): - pass - - -class FileTransferClient(object): - """ - Connect to a RSS (remote shell server) and transfer files. - """ - - def __init__(self, address, port, log_func=None, timeout=20): - """ - Connect to a server. - - @param address: The server's address - @param port: The server's port - @param log_func: If provided, transfer stats will be passed to this - function during the transfer - @param timeout: Time duration to wait for connection to succeed - @raise FileTransferConnectError: Raised if the connection fails - """ - self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self._socket.settimeout(timeout) - try: - self._socket.connect((address, port)) - except socket.error, e: - raise FileTransferConnectError("Cannot connect to server at " - "%s:%s" % (address, port), e) - try: - if self._receive_msg(timeout) != RSS_MAGIC: - raise FileTransferConnectError("Received wrong magic number") - except FileTransferTimeoutError: - raise FileTransferConnectError("Timeout expired while waiting to " - "receive magic number") - self._send(struct.pack("=i", CHUNKSIZE)) - self._log_func = log_func - self._last_time = time.time() - self._last_transferred = 0 - self.transferred = 0 - - - def __del__(self): - self.close() - - - def close(self): - """ - Close the connection. - """ - self._socket.close() - - - def _send(self, str, timeout=60): - try: - if timeout <= 0: - raise socket.timeout - self._socket.settimeout(timeout) - self._socket.sendall(str) - except socket.timeout: - raise FileTransferTimeoutError("Timeout expired while sending " - "data to server") - except socket.error, e: - raise FileTransferSocketError("Could not send data to server", e) - - - def _receive(self, size, timeout=60): - strs = [] - end_time = time.time() + timeout - try: - while size > 0: - timeout = end_time - time.time() - if timeout <= 0: - raise socket.timeout - self._socket.settimeout(timeout) - data = self._socket.recv(size) - if not data: - raise FileTransferProtocolError("Connection closed " - "unexpectedly while " - "receiving data from " - "server") - strs.append(data) - size -= len(data) - except socket.timeout: - raise FileTransferTimeoutError("Timeout expired while receiving " - "data from server") - except socket.error, e: - raise FileTransferSocketError("Error receiving data from server", - e) - return "".join(strs) - - - def _report_stats(self, str): - if self._log_func: - dt = time.time() - self._last_time - if dt >= 1: - transferred = self.transferred / 1048576. - speed = (self.transferred - self._last_transferred) / dt - speed /= 1048576. - self._log_func("%s %.3f MB (%.3f MB/sec)" % - (str, transferred, speed)) - self._last_time = time.time() - self._last_transferred = self.transferred - - - def _send_packet(self, str, timeout=60): - self._send(struct.pack("=I", len(str))) - self._send(str, timeout) - self.transferred += len(str) + 4 - self._report_stats("Sent") - - - def _receive_packet(self, timeout=60): - size = struct.unpack("=I", self._receive(4))[0] - str = self._receive(size, timeout) - self.transferred += len(str) + 4 - self._report_stats("Received") - return str - - - def _send_file_chunks(self, filename, timeout=60): - if self._log_func: - self._log_func("Sending file %s" % filename) - f = open(filename, "rb") - try: - try: - end_time = time.time() + timeout - while True: - data = f.read(CHUNKSIZE) - self._send_packet(data, end_time - time.time()) - if len(data) < CHUNKSIZE: - break - except FileTransferError, e: - e.filename = filename - raise - finally: - f.close() - - - def _receive_file_chunks(self, filename, timeout=60): - if self._log_func: - self._log_func("Receiving file %s" % filename) - f = open(filename, "wb") - try: - try: - end_time = time.time() + timeout - while True: - data = self._receive_packet(end_time - time.time()) - f.write(data) - if len(data) < CHUNKSIZE: - break - except FileTransferError, e: - e.filename = filename - raise - finally: - f.close() - - - def _send_msg(self, msg, timeout=60): - self._send(struct.pack("=I", msg)) - - - def _receive_msg(self, timeout=60): - s = self._receive(4, timeout) - return struct.unpack("=I", s)[0] - - - def _handle_transfer_error(self): - # Save original exception - e = sys.exc_info() - try: - # See if we can get an error message - msg = self._receive_msg() - except FileTransferError: - # No error message -- re-raise original exception - raise e[0], e[1], e[2] - if msg == RSS_ERROR: - errmsg = self._receive_packet() - raise FileTransferServerError(errmsg) - raise e[0], e[1], e[2] - - -class FileUploadClient(FileTransferClient): - """ - Connect to a RSS (remote shell server) and upload files or directory trees. - """ - - def __init__(self, address, port, log_func=None, timeout=20): - """ - Connect to a server. - - @param address: The server's address - @param port: The server's port - @param log_func: If provided, transfer stats will be passed to this - function during the transfer - @param timeout: Time duration to wait for connection to succeed - @raise FileTransferConnectError: Raised if the connection fails - @raise FileTransferProtocolError: Raised if an incorrect magic number - is received - @raise FileTransferSocketError: Raised if the RSS_UPLOAD message cannot - be sent to the server - """ - super(FileUploadClient, self).__init__(address, port, log_func, timeout) - self._send_msg(RSS_UPLOAD) - - - def _upload_file(self, path, end_time): - if os.path.isfile(path): - self._send_msg(RSS_CREATE_FILE) - self._send_packet(os.path.basename(path)) - self._send_file_chunks(path, end_time - time.time()) - elif os.path.isdir(path): - self._send_msg(RSS_CREATE_DIR) - self._send_packet(os.path.basename(path)) - for filename in os.listdir(path): - self._upload_file(os.path.join(path, filename), end_time) - self._send_msg(RSS_LEAVE_DIR) - - - def upload(self, src_pattern, dst_path, timeout=600): - """ - Send files or directory trees to the server. - The semantics of src_pattern and dst_path are similar to those of scp. - For example, the following are OK: - src_pattern='/tmp/foo.txt', dst_path='C:\\' - (uploads a single file) - src_pattern='/usr/', dst_path='C:\\Windows\\' - (uploads a directory tree recursively) - src_pattern='/usr/*', dst_path='C:\\Windows\\' - (uploads all files and directory trees under /usr/) - The following is not OK: - src_pattern='/tmp/foo.txt', dst_path='C:\\Windows\\*' - (wildcards are only allowed in src_pattern) - - @param src_pattern: A path or wildcard pattern specifying the files or - directories to send to the server - @param dst_path: A path in the server's filesystem where the files will - be saved - @param timeout: Time duration in seconds to wait for the transfer to - complete - @raise FileTransferTimeoutError: Raised if timeout expires - @raise FileTransferServerError: Raised if something goes wrong and the - server sends an informative error message to the client - @note: Other exceptions can be raised. - """ - end_time = time.time() + timeout - try: - try: - self._send_msg(RSS_SET_PATH) - self._send_packet(dst_path) - matches = glob.glob(src_pattern) - for filename in matches: - self._upload_file(os.path.abspath(filename), end_time) - self._send_msg(RSS_DONE) - except FileTransferTimeoutError: - raise - except FileTransferError: - self._handle_transfer_error() - else: - # If nothing was transferred, raise an exception - if not matches: - raise FileTransferNotFoundError("Pattern %s does not " - "match any files or " - "directories" % - src_pattern) - # Look for RSS_OK or RSS_ERROR - msg = self._receive_msg(end_time - time.time()) - if msg == RSS_OK: - return - elif msg == RSS_ERROR: - errmsg = self._receive_packet() - raise FileTransferServerError(errmsg) - else: - # Neither RSS_OK nor RSS_ERROR found - raise FileTransferProtocolError("Received unexpected msg") - except: - # In any case, if the transfer failed, close the connection - self.close() - raise - - -class FileDownloadClient(FileTransferClient): - """ - Connect to a RSS (remote shell server) and download files or directory trees. - """ - - def __init__(self, address, port, log_func=None, timeout=20): - """ - Connect to a server. - - @param address: The server's address - @param port: The server's port - @param log_func: If provided, transfer stats will be passed to this - function during the transfer - @param timeout: Time duration to wait for connection to succeed - @raise FileTransferConnectError: Raised if the connection fails - @raise FileTransferProtocolError: Raised if an incorrect magic number - is received - @raise FileTransferSendError: Raised if the RSS_UPLOAD message cannot - be sent to the server - """ - super(FileDownloadClient, self).__init__(address, port, log_func, timeout) - self._send_msg(RSS_DOWNLOAD) - - - def download(self, src_pattern, dst_path, timeout=600): - """ - Receive files or directory trees from the server. - The semantics of src_pattern and dst_path are similar to those of scp. - For example, the following are OK: - src_pattern='C:\\foo.txt', dst_path='/tmp' - (downloads a single file) - src_pattern='C:\\Windows', dst_path='/tmp' - (downloads a directory tree recursively) - src_pattern='C:\\Windows\\*', dst_path='/tmp' - (downloads all files and directory trees under C:\\Windows) - The following is not OK: - src_pattern='C:\\Windows', dst_path='/tmp/*' - (wildcards are only allowed in src_pattern) - - @param src_pattern: A path or wildcard pattern specifying the files or - directories, in the server's filesystem, that will be sent to - the client - @param dst_path: A path in the local filesystem where the files will - be saved - @param timeout: Time duration in seconds to wait for the transfer to - complete - @raise FileTransferTimeoutError: Raised if timeout expires - @raise FileTransferServerError: Raised if something goes wrong and the - server sends an informative error message to the client - @note: Other exceptions can be raised. - """ - dst_path = os.path.abspath(dst_path) - end_time = time.time() + timeout - file_count = 0 - dir_count = 0 - try: - try: - self._send_msg(RSS_SET_PATH) - self._send_packet(src_pattern) - except FileTransferError: - self._handle_transfer_error() - while True: - msg = self._receive_msg() - if msg == RSS_CREATE_FILE: - # Receive filename and file contents - filename = self._receive_packet() - if os.path.isdir(dst_path): - dst_path = os.path.join(dst_path, filename) - self._receive_file_chunks(dst_path, end_time - time.time()) - dst_path = os.path.dirname(dst_path) - file_count += 1 - elif msg == RSS_CREATE_DIR: - # Receive dirname and create the directory - dirname = self._receive_packet() - if os.path.isdir(dst_path): - dst_path = os.path.join(dst_path, dirname) - if not os.path.isdir(dst_path): - os.mkdir(dst_path) - dir_count += 1 - elif msg == RSS_LEAVE_DIR: - # Return to parent dir - dst_path = os.path.dirname(dst_path) - elif msg == RSS_DONE: - # Transfer complete - if not file_count and not dir_count: - raise FileTransferNotFoundError("Pattern %s does not " - "match any files or " - "directories that " - "could be downloaded" % - src_pattern) - break - elif msg == RSS_ERROR: - # Receive error message and abort - errmsg = self._receive_packet() - raise FileTransferServerError(errmsg) - else: - # Unexpected msg - raise FileTransferProtocolError("Received unexpected msg") - except: - # In any case, if the transfer failed, close the connection - self.close() - raise - - -def upload(address, port, src_pattern, dst_path, log_func=None, timeout=60, - connect_timeout=20): - """ - Connect to server and upload files. - - @see: FileUploadClient - """ - client = FileUploadClient(address, port, log_func, connect_timeout) - client.upload(src_pattern, dst_path, timeout) - client.close() - - -def download(address, port, src_pattern, dst_path, log_func=None, timeout=60, - connect_timeout=20): - """ - Connect to server and upload files. - - @see: FileDownloadClient - """ - client = FileDownloadClient(address, port, log_func, connect_timeout) - client.download(src_pattern, dst_path, timeout) - client.close() - - -def main(): - import optparse - - usage = "usage: %prog [options] address port src_pattern dst_path" - parser = optparse.OptionParser(usage=usage) - parser.add_option("-d", "--download", - action="store_true", dest="download", - help="download files from server") - parser.add_option("-u", "--upload", - action="store_true", dest="upload", - help="upload files to server") - parser.add_option("-v", "--verbose", - action="store_true", dest="verbose", - help="be verbose") - parser.add_option("-t", "--timeout", - type="int", dest="timeout", default=3600, - help="transfer timeout") - options, args = parser.parse_args() - if options.download == options.upload: - parser.error("you must specify either -d or -u") - if len(args) != 4: - parser.error("incorrect number of arguments") - address, port, src_pattern, dst_path = args - port = int(port) - - logger = None - if options.verbose: - def p(s): - print s - logger = p - - if options.download: - download(address, port, src_pattern, dst_path, logger, options.timeout) - elif options.upload: - upload(address, port, src_pattern, dst_path, logger, options.timeout) - - -if __name__ == "__main__": - main() diff --git a/client/tests/kvm/scan_results.py b/client/tests/kvm/scan_results.py deleted file mode 100755 index be825f6a..00000000 --- a/client/tests/kvm/scan_results.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/python -""" -Program that parses the autotest results and return a nicely printed final test -result. - -@copyright: Red Hat 2008-2009 -""" - -def parse_results(text): - """ - Parse text containing Autotest results. - - @return: A list of result 4-tuples. - """ - result_list = [] - start_time_list = [] - info_list = [] - - lines = text.splitlines() - for line in lines: - line = line.strip() - parts = line.split("\t") - - # Found a START line -- get start time - if (line.startswith("START") and len(parts) >= 5 and - parts[3].startswith("timestamp")): - start_time = float(parts[3].split("=")[1]) - start_time_list.append(start_time) - info_list.append("") - - # Found an END line -- get end time, name and status - elif (line.startswith("END") and len(parts) >= 5 and - parts[3].startswith("timestamp")): - end_time = float(parts[3].split("=")[1]) - start_time = start_time_list.pop() - info = info_list.pop() - test_name = parts[2] - test_status = parts[0].split()[1] - # Remove "kvm." prefix - if test_name.startswith("kvm."): - test_name = test_name[4:] - result_list.append((test_name, test_status, - int(end_time - start_time), info)) - - # Found a FAIL/ERROR/GOOD line -- get failure/success info - elif (len(parts) >= 6 and parts[3].startswith("timestamp") and - parts[4].startswith("localtime")): - info_list[-1] = parts[5] - - return result_list - - -def print_result(result, name_width): - """ - Nicely print a single Autotest result. - - @param result: a 4-tuple - @param name_width: test name maximum width - """ - if result: - format = "%%-%ds %%-10s %%-8s %%s" % name_width - print format % result - - -def main(resfiles): - result_lists = [] - name_width = 40 - - for resfile in resfiles: - try: - text = open(resfile).read() - except IOError: - print "Bad result file: %s" % resfile - continue - results = parse_results(text) - result_lists.append((resfile, results)) - name_width = max([name_width] + [len(r[0]) for r in results]) - - print_result(("Test", "Status", "Seconds", "Info"), name_width) - print_result(("----", "------", "-------", "----"), name_width) - - for resfile, results in result_lists: - print " (Result file: %s)" % resfile - for r in results: - print_result(r, name_width) - - -if __name__ == "__main__": - import sys, glob - - resfiles = glob.glob("../../results/default/status*") - if len(sys.argv) > 1: - if sys.argv[1] == "-h" or sys.argv[1] == "--help": - print "Usage: %s [result files]" % sys.argv[0] - sys.exit(0) - resfiles = sys.argv[1:] - main(resfiles) diff --git a/client/tests/kvm/stepeditor.py b/client/tests/kvm/stepeditor.py deleted file mode 100755 index bcdf572a..00000000 --- a/client/tests/kvm/stepeditor.py +++ /dev/null @@ -1,1401 +0,0 @@ -#!/usr/bin/python -""" -Step file creator/editor. - -@copyright: Red Hat Inc 2009 -@author: mgoldish@redhat.com (Michael Goldish) -@version: "20090401" -""" - -import pygtk, gtk, os, glob, shutil, sys, logging -import common, ppm_utils -pygtk.require('2.0') - - -# General utilities - -def corner_and_size_clipped(startpoint, endpoint, limits): - c0 = startpoint[:] - c1 = endpoint[:] - if c0[0] < 0: - c0[0] = 0 - if c0[1] < 0: - c0[1] = 0 - if c1[0] < 0: - c1[0] = 0 - if c1[1] < 0: - c1[1] = 0 - if c0[0] > limits[0] - 1: - c0[0] = limits[0] - 1 - if c0[1] > limits[1] - 1: - c0[1] = limits[1] - 1 - if c1[0] > limits[0] - 1: - c1[0] = limits[0] - 1 - if c1[1] > limits[1] - 1: - c1[1] = limits[1] - 1 - return ([min(c0[0], c1[0]), - min(c0[1], c1[1])], - [abs(c1[0] - c0[0]) + 1, - abs(c1[1] - c0[1]) + 1]) - - -def key_event_to_qemu_string(event): - keymap = gtk.gdk.keymap_get_default() - keyvals = keymap.get_entries_for_keycode(event.hardware_keycode) - keyval = keyvals[0][0] - keyname = gtk.gdk.keyval_name(keyval) - - dict = { "Return": "ret", - "Tab": "tab", - "space": "spc", - "Left": "left", - "Right": "right", - "Up": "up", - "Down": "down", - "F1": "f1", - "F2": "f2", - "F3": "f3", - "F4": "f4", - "F5": "f5", - "F6": "f6", - "F7": "f7", - "F8": "f8", - "F9": "f9", - "F10": "f10", - "F11": "f11", - "F12": "f12", - "Escape": "esc", - "minus": "minus", - "equal": "equal", - "BackSpace": "backspace", - "comma": "comma", - "period": "dot", - "slash": "slash", - "Insert": "insert", - "Delete": "delete", - "Home": "home", - "End": "end", - "Page_Up": "pgup", - "Page_Down": "pgdn", - "Menu": "menu", - "semicolon": "0x27", - "backslash": "0x2b", - "apostrophe": "0x28", - "grave": "0x29", - "less": "0x2b", - "bracketleft": "0x1a", - "bracketright": "0x1b", - "Super_L": "0xdc", - "Super_R": "0xdb", - } - - if ord('a') <= keyval <= ord('z') or ord('0') <= keyval <= ord('9'): - str = keyname - elif keyname in dict.keys(): - str = dict[keyname] - else: - return "" - - if event.state & gtk.gdk.CONTROL_MASK: - str = "ctrl-" + str - if event.state & gtk.gdk.MOD1_MASK: - str = "alt-" + str - if event.state & gtk.gdk.SHIFT_MASK: - str = "shift-" + str - - return str - - -class StepMakerWindow: - - # Constructor - - def __init__(self): - # Window - self.window = gtk.Window(gtk.WINDOW_TOPLEVEL) - self.window.set_title("Step Maker Window") - self.window.connect("delete-event", self.delete_event) - self.window.connect("destroy", self.destroy) - self.window.set_default_size(600, 800) - - # Main box (inside a frame which is inside a VBox) - self.menu_vbox = gtk.VBox() - self.window.add(self.menu_vbox) - self.menu_vbox.show() - - frame = gtk.Frame() - frame.set_border_width(10) - frame.set_shadow_type(gtk.SHADOW_NONE) - self.menu_vbox.pack_end(frame) - frame.show() - - self.main_vbox = gtk.VBox(spacing=10) - frame.add(self.main_vbox) - self.main_vbox.show() - - # EventBox - self.scrolledwindow = gtk.ScrolledWindow() - self.scrolledwindow.set_policy(gtk.POLICY_AUTOMATIC, - gtk.POLICY_AUTOMATIC) - self.scrolledwindow.set_shadow_type(gtk.SHADOW_NONE) - self.main_vbox.pack_start(self.scrolledwindow) - self.scrolledwindow.show() - - table = gtk.Table(1, 1) - self.scrolledwindow.add_with_viewport(table) - table.show() - table.realize() - - self.event_box = gtk.EventBox() - table.attach(self.event_box, 0, 1, 0, 1, gtk.EXPAND, gtk.EXPAND) - self.event_box.show() - self.event_box.realize() - - # Image - self.image = gtk.Image() - self.event_box.add(self.image) - self.image.show() - - # Data VBox - self.data_vbox = gtk.VBox(spacing=10) - self.main_vbox.pack_start(self.data_vbox, expand=False) - self.data_vbox.show() - - # User VBox - self.user_vbox = gtk.VBox(spacing=10) - self.main_vbox.pack_start(self.user_vbox, expand=False) - self.user_vbox.show() - - # Screendump ID HBox - box = gtk.HBox(spacing=10) - self.data_vbox.pack_start(box) - box.show() - - label = gtk.Label("Screendump ID:") - box.pack_start(label, False) - label.show() - - self.entry_screendump = gtk.Entry() - self.entry_screendump.set_editable(False) - box.pack_start(self.entry_screendump) - self.entry_screendump.show() - - label = gtk.Label("Time:") - box.pack_start(label, False) - label.show() - - self.entry_time = gtk.Entry() - self.entry_time.set_editable(False) - self.entry_time.set_width_chars(10) - box.pack_start(self.entry_time, False) - self.entry_time.show() - - # Comment HBox - box = gtk.HBox(spacing=10) - self.data_vbox.pack_start(box) - box.show() - - label = gtk.Label("Comment:") - box.pack_start(label, False) - label.show() - - self.entry_comment = gtk.Entry() - box.pack_start(self.entry_comment) - self.entry_comment.show() - - # Sleep HBox - box = gtk.HBox(spacing=10) - self.data_vbox.pack_start(box) - box.show() - - self.check_sleep = gtk.CheckButton("Sleep:") - self.check_sleep.connect("toggled", self.event_check_sleep_toggled) - box.pack_start(self.check_sleep, False) - self.check_sleep.show() - - self.spin_sleep = gtk.SpinButton(gtk.Adjustment(0, 0, 50000, 1, 10, 0), - climb_rate=0.0) - box.pack_start(self.spin_sleep, False) - self.spin_sleep.show() - - # Barrier HBox - box = gtk.HBox(spacing=10) - self.data_vbox.pack_start(box) - box.show() - - self.check_barrier = gtk.CheckButton("Barrier:") - self.check_barrier.connect("toggled", self.event_check_barrier_toggled) - box.pack_start(self.check_barrier, False) - self.check_barrier.show() - - vbox = gtk.VBox() - box.pack_start(vbox) - vbox.show() - - self.label_barrier_region = gtk.Label("Region:") - self.label_barrier_region.set_alignment(0, 0.5) - vbox.pack_start(self.label_barrier_region) - self.label_barrier_region.show() - - self.label_barrier_md5sum = gtk.Label("MD5:") - self.label_barrier_md5sum.set_alignment(0, 0.5) - vbox.pack_start(self.label_barrier_md5sum) - self.label_barrier_md5sum.show() - - self.label_barrier_timeout = gtk.Label("Timeout:") - box.pack_start(self.label_barrier_timeout, False) - self.label_barrier_timeout.show() - - self.spin_barrier_timeout = gtk.SpinButton(gtk.Adjustment(0, 0, 50000, - 1, 10, 0), - climb_rate=0.0) - box.pack_start(self.spin_barrier_timeout, False) - self.spin_barrier_timeout.show() - - self.check_barrier_optional = gtk.CheckButton("Optional") - box.pack_start(self.check_barrier_optional, False) - self.check_barrier_optional.show() - - # Keystrokes HBox - box = gtk.HBox(spacing=10) - self.data_vbox.pack_start(box) - box.show() - - label = gtk.Label("Keystrokes:") - box.pack_start(label, False) - label.show() - - frame = gtk.Frame() - frame.set_shadow_type(gtk.SHADOW_IN) - box.pack_start(frame) - frame.show() - - self.text_buffer = gtk.TextBuffer() - self.entry_keys = gtk.TextView(self.text_buffer) - self.entry_keys.set_wrap_mode(gtk.WRAP_WORD) - self.entry_keys.connect("key-press-event", self.event_key_press) - frame.add(self.entry_keys) - self.entry_keys.show() - - self.check_manual = gtk.CheckButton("Manual") - self.check_manual.connect("toggled", self.event_manual_toggled) - box.pack_start(self.check_manual, False) - self.check_manual.show() - - button = gtk.Button("Clear") - button.connect("clicked", self.event_clear_clicked) - box.pack_start(button, False) - button.show() - - # Mouse click HBox - box = gtk.HBox(spacing=10) - self.data_vbox.pack_start(box) - box.show() - - label = gtk.Label("Mouse action:") - box.pack_start(label, False) - label.show() - - self.button_capture = gtk.Button("Capture") - box.pack_start(self.button_capture, False) - self.button_capture.show() - - self.check_mousemove = gtk.CheckButton("Move: ...") - box.pack_start(self.check_mousemove, False) - self.check_mousemove.show() - - self.check_mouseclick = gtk.CheckButton("Click: ...") - box.pack_start(self.check_mouseclick, False) - self.check_mouseclick.show() - - self.spin_sensitivity = gtk.SpinButton(gtk.Adjustment(1, 1, 100, 1, 10, - 0), - climb_rate=0.0) - box.pack_end(self.spin_sensitivity, False) - self.spin_sensitivity.show() - - label = gtk.Label("Sensitivity:") - box.pack_end(label, False) - label.show() - - self.spin_latency = gtk.SpinButton(gtk.Adjustment(10, 1, 500, 1, 10, 0), - climb_rate=0.0) - box.pack_end(self.spin_latency, False) - self.spin_latency.show() - - label = gtk.Label("Latency:") - box.pack_end(label, False) - label.show() - - self.handler_event_box_press = None - self.handler_event_box_release = None - self.handler_event_box_scroll = None - self.handler_event_box_motion = None - self.handler_event_box_expose = None - - self.window.realize() - self.window.show() - - self.clear_state() - - # Utilities - - def message(self, text, title): - dlg = gtk.MessageDialog(self.window, - gtk.DIALOG_MODAL | gtk.DIALOG_DESTROY_WITH_PARENT, - gtk.MESSAGE_INFO, - gtk.BUTTONS_CLOSE, - title) - dlg.set_title(title) - dlg.format_secondary_text(text) - response = dlg.run() - dlg.destroy() - - - def question_yes_no(self, text, title): - dlg = gtk.MessageDialog(self.window, - gtk.DIALOG_MODAL | gtk.DIALOG_DESTROY_WITH_PARENT, - gtk.MESSAGE_QUESTION, - gtk.BUTTONS_YES_NO, - title) - dlg.set_title(title) - dlg.format_secondary_text(text) - response = dlg.run() - dlg.destroy() - if response == gtk.RESPONSE_YES: - return True - return False - - - def inputdialog(self, text, title, default_response=""): - # Define a little helper function - def inputdialog_entry_activated(entry): - dlg.response(gtk.RESPONSE_OK) - - # Create the dialog - dlg = gtk.MessageDialog(self.window, - gtk.DIALOG_MODAL | gtk.DIALOG_DESTROY_WITH_PARENT, - gtk.MESSAGE_QUESTION, - gtk.BUTTONS_OK_CANCEL, - title) - dlg.set_title(title) - dlg.format_secondary_text(text) - - # Create an entry widget - entry = gtk.Entry() - entry.set_text(default_response) - entry.connect("activate", inputdialog_entry_activated) - dlg.vbox.pack_start(entry) - entry.show() - - # Run the dialog - response = dlg.run() - dlg.destroy() - if response == gtk.RESPONSE_OK: - return entry.get_text() - return None - - - def filedialog(self, title=None, default_filename=None): - chooser = gtk.FileChooserDialog(title=title, parent=self.window, - action=gtk.FILE_CHOOSER_ACTION_OPEN, - buttons=(gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL, gtk.STOCK_OPEN, - gtk.RESPONSE_OK)) - chooser.resize(700, 500) - if default_filename: - chooser.set_filename(os.path.abspath(default_filename)) - filename = None - response = chooser.run() - if response == gtk.RESPONSE_OK: - filename = chooser.get_filename() - chooser.destroy() - return filename - - - def redirect_event_box_input(self, press=None, release=None, scroll=None, - motion=None, expose=None): - if self.handler_event_box_press != None: \ - self.event_box.disconnect(self.handler_event_box_press) - if self.handler_event_box_release != None: \ - self.event_box.disconnect(self.handler_event_box_release) - if self.handler_event_box_scroll != None: \ - self.event_box.disconnect(self.handler_event_box_scroll) - if self.handler_event_box_motion != None: \ - self.event_box.disconnect(self.handler_event_box_motion) - if self.handler_event_box_expose != None: \ - self.event_box.disconnect(self.handler_event_box_expose) - self.handler_event_box_press = None - self.handler_event_box_release = None - self.handler_event_box_scroll = None - self.handler_event_box_motion = None - self.handler_event_box_expose = None - if press != None: self.handler_event_box_press = \ - self.event_box.connect("button-press-event", press) - if release != None: self.handler_event_box_release = \ - self.event_box.connect("button-release-event", release) - if scroll != None: self.handler_event_box_scroll = \ - self.event_box.connect("scroll-event", scroll) - if motion != None: self.handler_event_box_motion = \ - self.event_box.connect("motion-notify-event", motion) - if expose != None: self.handler_event_box_expose = \ - self.event_box.connect_after("expose-event", expose) - - - def get_keys(self): - return self.text_buffer.get_text( - self.text_buffer.get_start_iter(), - self.text_buffer.get_end_iter()) - - - def add_key(self, key): - text = self.get_keys() - if len(text) > 0 and text[-1] != ' ': - text += " " - text += key - self.text_buffer.set_text(text) - - - def clear_keys(self): - self.text_buffer.set_text("") - - - def update_barrier_info(self): - if self.barrier_selected: - self.label_barrier_region.set_text("Selected region: Corner: " + \ - str(tuple(self.barrier_corner)) + \ - " Size: " + \ - str(tuple(self.barrier_size))) - else: - self.label_barrier_region.set_text("No region selected.") - self.label_barrier_md5sum.set_text("MD5: " + self.barrier_md5sum) - - - def update_mouse_click_info(self): - if self.mouse_click_captured: - self.check_mousemove.set_label("Move: " + \ - str(tuple(self.mouse_click_coords))) - self.check_mouseclick.set_label("Click: button %d" % - self.mouse_click_button) - else: - self.check_mousemove.set_label("Move: ...") - self.check_mouseclick.set_label("Click: ...") - - - def clear_state(self, clear_screendump=True): - # Recording time - self.entry_time.set_text("unknown") - if clear_screendump: - # Screendump - self.clear_image() - # Screendump ID - self.entry_screendump.set_text("") - # Comment - self.entry_comment.set_text("") - # Sleep - self.check_sleep.set_active(True) - self.check_sleep.set_active(False) - self.spin_sleep.set_value(10) - # Barrier - self.clear_barrier_state() - # Keystrokes - self.check_manual.set_active(False) - self.clear_keys() - # Mouse actions - self.check_mousemove.set_sensitive(False) - self.check_mouseclick.set_sensitive(False) - self.check_mousemove.set_active(False) - self.check_mouseclick.set_active(False) - self.mouse_click_captured = False - self.mouse_click_coords = [0, 0] - self.mouse_click_button = 0 - self.update_mouse_click_info() - - - def clear_barrier_state(self): - self.check_barrier.set_active(True) - self.check_barrier.set_active(False) - self.check_barrier_optional.set_active(False) - self.spin_barrier_timeout.set_value(10) - self.barrier_selection_started = False - self.barrier_selected = False - self.barrier_corner0 = [0, 0] - self.barrier_corner1 = [0, 0] - self.barrier_corner = [0, 0] - self.barrier_size = [0, 0] - self.barrier_md5sum = "" - self.update_barrier_info() - - - def set_image(self, w, h, data): - (self.image_width, self.image_height, self.image_data) = (w, h, data) - self.image.set_from_pixbuf(gtk.gdk.pixbuf_new_from_data( - data, gtk.gdk.COLORSPACE_RGB, False, 8, - w, h, w*3)) - hscrollbar = self.scrolledwindow.get_hscrollbar() - hscrollbar.set_range(0, w) - vscrollbar = self.scrolledwindow.get_vscrollbar() - vscrollbar.set_range(0, h) - - - def set_image_from_file(self, filename): - if not ppm_utils.image_verify_ppm_file(filename): - logging.warning("set_image_from_file: Warning: received invalid" - "screendump file") - return self.clear_image() - (w, h, data) = ppm_utils.image_read_from_ppm_file(filename) - self.set_image(w, h, data) - - - def clear_image(self): - self.image.clear() - self.image_width = 0 - self.image_height = 0 - self.image_data = "" - - - def update_screendump_id(self, data_dir): - if not self.image_data: - return - # Find a proper ID for the screendump - scrdump_md5sum = ppm_utils.image_md5sum(self.image_width, - self.image_height, - self.image_data) - scrdump_id = ppm_utils.find_id_for_screendump(scrdump_md5sum, data_dir) - if not scrdump_id: - # Not found; generate one - scrdump_id = ppm_utils.generate_id_for_screendump(scrdump_md5sum, - data_dir) - self.entry_screendump.set_text(scrdump_id) - - - def get_step_lines(self, data_dir=None): - if self.check_barrier.get_active() and not self.barrier_selected: - self.message("No barrier region selected.", "Error") - return - - str = "step" - - # Add step recording time - if self.entry_time.get_text(): - str += " " + self.entry_time.get_text() - - str += "\n" - - # Add screendump line - if self.image_data: - str += "screendump %s\n" % self.entry_screendump.get_text() - - # Add comment - if self.entry_comment.get_text(): - str += "# %s\n" % self.entry_comment.get_text() - - # Add sleep line - if self.check_sleep.get_active(): - str += "sleep %d\n" % self.spin_sleep.get_value() - - # Add barrier_2 line - if self.check_barrier.get_active(): - str += "barrier_2 %d %d %d %d %s %d" % ( - self.barrier_size[0], self.barrier_size[1], - self.barrier_corner[0], self.barrier_corner[1], - self.barrier_md5sum, self.spin_barrier_timeout.get_value()) - if self.check_barrier_optional.get_active(): - str += " optional" - str += "\n" - - # Add "Sending keys" comment - keys_to_send = self.get_keys().split() - if keys_to_send: - str += "# Sending keys: %s\n" % self.get_keys() - - # Add key and var lines - for key in keys_to_send: - if key.startswith("$"): - varname = key[1:] - str += "var %s\n" % varname - else: - str += "key %s\n" % key - - # Add mousemove line - if self.check_mousemove.get_active(): - str += "mousemove %d %d\n" % (self.mouse_click_coords[0], - self.mouse_click_coords[1]) - - # Add mouseclick line - if self.check_mouseclick.get_active(): - dict = { 1 : 1, - 2 : 2, - 3 : 4 } - str += "mouseclick %d\n" % dict[self.mouse_click_button] - - # Write screendump and cropped screendump image files - if data_dir and self.image_data: - # Create the data dir if it doesn't exist - if not os.path.exists(data_dir): - os.makedirs(data_dir) - # Get the full screendump filename - scrdump_filename = os.path.join(data_dir, - self.entry_screendump.get_text()) - # Write screendump file if it doesn't exist - if not os.path.exists(scrdump_filename): - try: - ppm_utils.image_write_to_ppm_file(scrdump_filename, - self.image_width, - self.image_height, - self.image_data) - except IOError: - self.message("Could not write screendump file.", "Error") - - #if self.check_barrier.get_active(): - # # Crop image to get the cropped screendump - # (cw, ch, cdata) = ppm_utils.image_crop( - # self.image_width, self.image_height, self.image_data, - # self.barrier_corner[0], self.barrier_corner[1], - # self.barrier_size[0], self.barrier_size[1]) - # cropped_scrdump_md5sum = ppm_utils.image_md5sum(cw, ch, cdata) - # cropped_scrdump_filename = \ - # ppm_utils.get_cropped_screendump_filename(scrdump_filename, - # cropped_scrdump_md5sum) - # # Write cropped screendump file - # try: - # ppm_utils.image_write_to_ppm_file(cropped_scrdump_filename, - # cw, ch, cdata) - # except IOError: - # self.message("Could not write cropped screendump file.", - # "Error") - - return str - - def set_state_from_step_lines(self, str, data_dir, warn=True): - self.clear_state() - - for line in str.splitlines(): - words = line.split() - if not words: - continue - - if line.startswith("#") \ - and not self.entry_comment.get_text() \ - and not line.startswith("# Sending keys:") \ - and not line.startswith("# ----"): - self.entry_comment.set_text(line.strip("#").strip()) - - elif words[0] == "step": - if len(words) >= 2: - self.entry_time.set_text(words[1]) - - elif words[0] == "screendump": - self.entry_screendump.set_text(words[1]) - self.set_image_from_file(os.path.join(data_dir, words[1])) - - elif words[0] == "sleep": - self.spin_sleep.set_value(int(words[1])) - self.check_sleep.set_active(True) - - elif words[0] == "key": - self.add_key(words[1]) - - elif words[0] == "var": - self.add_key("$%s" % words[1]) - - elif words[0] == "mousemove": - self.mouse_click_captured = True - self.mouse_click_coords = [int(words[1]), int(words[2])] - self.update_mouse_click_info() - - elif words[0] == "mouseclick": - self.mouse_click_captured = True - self.mouse_click_button = int(words[1]) - self.update_mouse_click_info() - - elif words[0] == "barrier_2": - # Get region corner and size from step lines - self.barrier_corner = [int(words[3]), int(words[4])] - self.barrier_size = [int(words[1]), int(words[2])] - # Get corner0 and corner1 from step lines - self.barrier_corner0 = self.barrier_corner - self.barrier_corner1 = [self.barrier_corner[0] + - self.barrier_size[0] - 1, - self.barrier_corner[1] + - self.barrier_size[1] - 1] - # Get the md5sum - self.barrier_md5sum = words[5] - # Pretend the user selected the region with the mouse - self.barrier_selection_started = True - self.barrier_selected = True - # Update label widgets according to region information - self.update_barrier_info() - # Check the barrier checkbutton - self.check_barrier.set_active(True) - # Set timeout value - self.spin_barrier_timeout.set_value(int(words[6])) - # Set 'optional' checkbutton state - self.check_barrier_optional.set_active(words[-1] == "optional") - # Update the image widget - self.event_box.queue_draw() - - if warn: - # See if the computed md5sum matches the one recorded in - # the file - computed_md5sum = ppm_utils.get_region_md5sum( - self.image_width, self.image_height, - self.image_data, self.barrier_corner[0], - self.barrier_corner[1], self.barrier_size[0], - self.barrier_size[1]) - if computed_md5sum != self.barrier_md5sum: - self.message("Computed MD5 sum (%s) differs from MD5" - " sum recorded in steps file (%s)" % - (computed_md5sum, self.barrier_md5sum), - "Warning") - - # Events - - def delete_event(self, widget, event): - pass - - def destroy(self, widget): - gtk.main_quit() - - def event_check_barrier_toggled(self, widget): - if self.check_barrier.get_active(): - self.redirect_event_box_input( - self.event_button_press, - self.event_button_release, - None, - None, - self.event_expose) - self.event_box.queue_draw() - self.event_box.window.set_cursor(gtk.gdk.Cursor(gtk.gdk.CROSSHAIR)) - self.label_barrier_region.set_sensitive(True) - self.label_barrier_md5sum.set_sensitive(True) - self.label_barrier_timeout.set_sensitive(True) - self.spin_barrier_timeout.set_sensitive(True) - self.check_barrier_optional.set_sensitive(True) - else: - self.redirect_event_box_input() - self.event_box.queue_draw() - self.event_box.window.set_cursor(None) - self.label_barrier_region.set_sensitive(False) - self.label_barrier_md5sum.set_sensitive(False) - self.label_barrier_timeout.set_sensitive(False) - self.spin_barrier_timeout.set_sensitive(False) - self.check_barrier_optional.set_sensitive(False) - - def event_check_sleep_toggled(self, widget): - if self.check_sleep.get_active(): - self.spin_sleep.set_sensitive(True) - else: - self.spin_sleep.set_sensitive(False) - - def event_manual_toggled(self, widget): - self.entry_keys.grab_focus() - - def event_clear_clicked(self, widget): - self.clear_keys() - self.entry_keys.grab_focus() - - def event_expose(self, widget, event): - if not self.barrier_selection_started: - return - (corner, size) = corner_and_size_clipped(self.barrier_corner0, - self.barrier_corner1, - self.event_box.size_request()) - gc = self.event_box.window.new_gc(line_style=gtk.gdk.LINE_DOUBLE_DASH, - line_width=1) - gc.set_foreground(gc.get_colormap().alloc_color("red")) - gc.set_background(gc.get_colormap().alloc_color("dark red")) - gc.set_dashes(0, (4, 4)) - self.event_box.window.draw_rectangle( - gc, False, - corner[0], corner[1], - size[0]-1, size[1]-1) - - def event_drag_motion(self, widget, event): - old_corner1 = self.barrier_corner1 - self.barrier_corner1 = [int(event.x), int(event.y)] - (corner, size) = corner_and_size_clipped(self.barrier_corner0, - self.barrier_corner1, - self.event_box.size_request()) - (old_corner, old_size) = corner_and_size_clipped(self.barrier_corner0, - old_corner1, - self.event_box.size_request()) - corner0 = [min(corner[0], old_corner[0]), min(corner[1], old_corner[1])] - corner1 = [max(corner[0] + size[0], old_corner[0] + old_size[0]), - max(corner[1] + size[1], old_corner[1] + old_size[1])] - size = [corner1[0] - corner0[0] + 1, - corner1[1] - corner0[1] + 1] - self.event_box.queue_draw_area(corner0[0], corner0[1], size[0], size[1]) - - def event_button_press(self, widget, event): - (corner, size) = corner_and_size_clipped(self.barrier_corner0, - self.barrier_corner1, - self.event_box.size_request()) - self.event_box.queue_draw_area(corner[0], corner[1], size[0], size[1]) - self.barrier_corner0 = [int(event.x), int(event.y)] - self.barrier_corner1 = [int(event.x), int(event.y)] - self.redirect_event_box_input( - self.event_button_press, - self.event_button_release, - None, - self.event_drag_motion, - self.event_expose) - self.barrier_selection_started = True - - def event_button_release(self, widget, event): - self.redirect_event_box_input( - self.event_button_press, - self.event_button_release, - None, - None, - self.event_expose) - (self.barrier_corner, self.barrier_size) = \ - corner_and_size_clipped(self.barrier_corner0, self.barrier_corner1, - self.event_box.size_request()) - self.barrier_md5sum = ppm_utils.get_region_md5sum( - self.image_width, self.image_height, self.image_data, - self.barrier_corner[0], self.barrier_corner[1], - self.barrier_size[0], self.barrier_size[1]) - self.barrier_selected = True - self.update_barrier_info() - - def event_key_press(self, widget, event): - if self.check_manual.get_active(): - return False - str = key_event_to_qemu_string(event) - self.add_key(str) - return True - - -class StepEditor(StepMakerWindow): - ui = ''' - - - - - - - - - - - - - - - - - - - - - -''' - - # Constructor - - def __init__(self, filename=None): - StepMakerWindow.__init__(self) - - self.steps_filename = None - self.steps = [] - - # Create a UIManager instance - uimanager = gtk.UIManager() - - # Add the accelerator group to the toplevel window - accelgroup = uimanager.get_accel_group() - self.window.add_accel_group(accelgroup) - - # Create an ActionGroup - actiongroup = gtk.ActionGroup('StepEditor') - - # Create actions - actiongroup.add_actions([ - ('Quit', gtk.STOCK_QUIT, '_Quit', None, 'Quit the Program', - self.quit), - ('Open', gtk.STOCK_OPEN, '_Open', None, 'Open steps file', - self.open_steps_file), - ('CopyStep', gtk.STOCK_COPY, '_Copy current step...', "", - 'Copy current step to user specified position', self.copy_step), - ('DeleteStep', gtk.STOCK_DELETE, '_Delete current step', "", - 'Delete current step', self.event_remove_clicked), - ('InsertNewBefore', gtk.STOCK_ADD, '_New step before current', "", - 'Insert new step before current step', self.insert_before), - ('InsertNewAfter', gtk.STOCK_ADD, 'N_ew step after current', "", - 'Insert new step after current step', self.insert_after), - ('InsertStepsBefore', gtk.STOCK_ADD, '_Steps before current...', - "", 'Insert steps (from file) before current step', - self.insert_steps_before), - ('InsertStepsAfter', gtk.STOCK_ADD, 'Steps _after current...', "", - 'Insert steps (from file) after current step', - self.insert_steps_after), - ('CleanUp', gtk.STOCK_DELETE, '_Clean up data directory', "", - 'Move unused PPM files to a backup directory', self.cleanup), - ('File', None, '_File'), - ('Edit', None, '_Edit'), - ('Insert', None, '_Insert'), - ('Tools', None, '_Tools') - ]) - - def create_shortcut(name, callback, keyname): - # Create an action - action = gtk.Action(name, None, None, None) - # Connect a callback to the action - action.connect("activate", callback) - actiongroup.add_action_with_accel(action, keyname) - # Have the action use accelgroup - action.set_accel_group(accelgroup) - # Connect the accelerator to the action - action.connect_accelerator() - - create_shortcut("Next", self.event_next_clicked, "Page_Down") - create_shortcut("Previous", self.event_prev_clicked, "Page_Up") - - # Add the actiongroup to the uimanager - uimanager.insert_action_group(actiongroup, 0) - - # Add a UI description - uimanager.add_ui_from_string(self.ui) - - # Create a MenuBar - menubar = uimanager.get_widget('/MenuBar') - self.menu_vbox.pack_start(menubar, False) - - # Remember the Edit menu bar for future reference - self.menu_edit = uimanager.get_widget('/MenuBar/Edit') - self.menu_edit.set_sensitive(False) - - # Remember the Insert menu bar for future reference - self.menu_insert = uimanager.get_widget('/MenuBar/Insert') - self.menu_insert.set_sensitive(False) - - # Remember the Tools menu bar for future reference - self.menu_tools = uimanager.get_widget('/MenuBar/Tools') - self.menu_tools.set_sensitive(False) - - # Next/Previous HBox - hbox = gtk.HBox(spacing=10) - self.user_vbox.pack_start(hbox) - hbox.show() - - self.button_first = gtk.Button(stock=gtk.STOCK_GOTO_FIRST) - self.button_first.connect("clicked", self.event_first_clicked) - hbox.pack_start(self.button_first) - self.button_first.show() - - #self.button_prev = gtk.Button("<< Previous") - self.button_prev = gtk.Button(stock=gtk.STOCK_GO_BACK) - self.button_prev.connect("clicked", self.event_prev_clicked) - hbox.pack_start(self.button_prev) - self.button_prev.show() - - self.label_step = gtk.Label("Step:") - hbox.pack_start(self.label_step, False) - self.label_step.show() - - self.entry_step_num = gtk.Entry() - self.entry_step_num.connect("activate", self.event_entry_step_activated) - self.entry_step_num.set_width_chars(3) - hbox.pack_start(self.entry_step_num, False) - self.entry_step_num.show() - - #self.button_next = gtk.Button("Next >>") - self.button_next = gtk.Button(stock=gtk.STOCK_GO_FORWARD) - self.button_next.connect("clicked", self.event_next_clicked) - hbox.pack_start(self.button_next) - self.button_next.show() - - self.button_last = gtk.Button(stock=gtk.STOCK_GOTO_LAST) - self.button_last.connect("clicked", self.event_last_clicked) - hbox.pack_start(self.button_last) - self.button_last.show() - - # Save HBox - hbox = gtk.HBox(spacing=10) - self.user_vbox.pack_start(hbox) - hbox.show() - - self.button_save = gtk.Button("_Save current step") - self.button_save.connect("clicked", self.event_save_clicked) - hbox.pack_start(self.button_save) - self.button_save.show() - - self.button_remove = gtk.Button("_Delete current step") - self.button_remove.connect("clicked", self.event_remove_clicked) - hbox.pack_start(self.button_remove) - self.button_remove.show() - - self.button_replace = gtk.Button("_Replace screendump") - self.button_replace.connect("clicked", self.event_replace_clicked) - hbox.pack_start(self.button_replace) - self.button_replace.show() - - # Disable unused widgets - self.button_capture.set_sensitive(False) - self.spin_latency.set_sensitive(False) - self.spin_sensitivity.set_sensitive(False) - - # Disable main vbox because no steps file is loaded - self.main_vbox.set_sensitive(False) - - # Set title - self.window.set_title("Step Editor") - - # Events - - def delete_event(self, widget, event): - # Make sure the step is saved (if the user wants it to be) - self.verify_save() - - def event_first_clicked(self, widget): - if not self.steps: - return - # Make sure the step is saved (if the user wants it to be) - self.verify_save() - # Go to first step - self.set_step(0) - - def event_last_clicked(self, widget): - if not self.steps: - return - # Make sure the step is saved (if the user wants it to be) - self.verify_save() - # Go to last step - self.set_step(len(self.steps) - 1) - - def event_prev_clicked(self, widget): - if not self.steps: - return - # Make sure the step is saved (if the user wants it to be) - self.verify_save() - # Go to previous step - index = self.current_step_index - 1 - if self.steps: - index = index % len(self.steps) - self.set_step(index) - - def event_next_clicked(self, widget): - if not self.steps: - return - # Make sure the step is saved (if the user wants it to be) - self.verify_save() - # Go to next step - index = self.current_step_index + 1 - if self.steps: - index = index % len(self.steps) - self.set_step(index) - - def event_entry_step_activated(self, widget): - if not self.steps: - return - step_index = self.entry_step_num.get_text() - if not step_index.isdigit(): - return - step_index = int(step_index) - 1 - if step_index == self.current_step_index: - return - self.verify_save() - self.set_step(step_index) - - def event_save_clicked(self, widget): - if not self.steps: - return - self.save_step() - - def event_remove_clicked(self, widget): - if not self.steps: - return - if not self.question_yes_no("This will modify the steps file." - " Are you sure?", "Remove step?"): - return - # Remove step - del self.steps[self.current_step_index] - # Write changes to file - self.write_steps_file(self.steps_filename) - # Move to previous step - self.set_step(self.current_step_index) - - def event_replace_clicked(self, widget): - if not self.steps: - return - # Let the user choose a screendump file - current_filename = os.path.join(self.steps_data_dir, - self.entry_screendump.get_text()) - filename = self.filedialog("Choose PPM image file", - default_filename=current_filename) - if not filename: - return - if not ppm_utils.image_verify_ppm_file(filename): - self.message("Not a valid PPM image file.", "Error") - return - self.clear_image() - self.clear_barrier_state() - self.set_image_from_file(filename) - self.update_screendump_id(self.steps_data_dir) - - # Menu actions - - def open_steps_file(self, action): - # Make sure the step is saved (if the user wants it to be) - self.verify_save() - # Let the user choose a steps file - current_filename = self.steps_filename - filename = self.filedialog("Open steps file", - default_filename=current_filename) - if not filename: - return - self.set_steps_file(filename) - - def quit(self, action): - # Make sure the step is saved (if the user wants it to be) - self.verify_save() - # Quit - gtk.main_quit() - - def copy_step(self, action): - if not self.steps: - return - self.verify_save() - self.set_step(self.current_step_index) - # Get the desired position - step_index = self.inputdialog("Copy step to position:", - "Copy step", - str(self.current_step_index + 2)) - if not step_index: - return - step_index = int(step_index) - 1 - # Get the lines of the current step - step = self.steps[self.current_step_index] - # Insert new step at position step_index - self.steps.insert(step_index, step) - # Go to new step - self.set_step(step_index) - # Write changes to disk - self.write_steps_file(self.steps_filename) - - def insert_before(self, action): - if not self.steps_filename: - return - if not self.question_yes_no("This will modify the steps file." - " Are you sure?", "Insert new step?"): - return - self.verify_save() - step_index = self.current_step_index - # Get the lines of a blank step - self.clear_state() - step = self.get_step_lines() - # Insert new step at position step_index - self.steps.insert(step_index, step) - # Go to new step - self.set_step(step_index) - # Write changes to disk - self.write_steps_file(self.steps_filename) - - def insert_after(self, action): - if not self.steps_filename: - return - if not self.question_yes_no("This will modify the steps file." - " Are you sure?", "Insert new step?"): - return - self.verify_save() - step_index = self.current_step_index + 1 - # Get the lines of a blank step - self.clear_state() - step = self.get_step_lines() - # Insert new step at position step_index - self.steps.insert(step_index, step) - # Go to new step - self.set_step(step_index) - # Write changes to disk - self.write_steps_file(self.steps_filename) - - def insert_steps(self, filename, index): - # Read the steps file - (steps, header) = self.read_steps_file(filename) - - data_dir = ppm_utils.get_data_dir(filename) - for step in steps: - self.set_state_from_step_lines(step, data_dir, warn=False) - step = self.get_step_lines(self.steps_data_dir) - - # Insert steps into self.steps - self.steps[index:index] = steps - # Write changes to disk - self.write_steps_file(self.steps_filename) - - def insert_steps_before(self, action): - if not self.steps_filename: - return - # Let the user choose a steps file - current_filename = self.steps_filename - filename = self.filedialog("Choose steps file", - default_filename=current_filename) - if not filename: - return - self.verify_save() - - step_index = self.current_step_index - # Insert steps at position step_index - self.insert_steps(filename, step_index) - # Go to new steps - self.set_step(step_index) - - def insert_steps_after(self, action): - if not self.steps_filename: - return - # Let the user choose a steps file - current_filename = self.steps_filename - filename = self.filedialog("Choose steps file", - default_filename=current_filename) - if not filename: - return - self.verify_save() - - step_index = self.current_step_index + 1 - # Insert new steps at position step_index - self.insert_steps(filename, step_index) - # Go to new steps - self.set_step(step_index) - - def cleanup(self, action): - if not self.steps_filename: - return - if not self.question_yes_no("All unused PPM files will be moved to a" - " backup directory. Are you sure?", - "Clean up data directory?"): - return - # Remember the current step index - current_step_index = self.current_step_index - # Get the backup dir - backup_dir = os.path.join(self.steps_data_dir, "backup") - # Create it if it doesn't exist - if not os.path.exists(backup_dir): - os.makedirs(backup_dir) - # Move all files to the backup dir - for filename in glob.glob(os.path.join(self.steps_data_dir, - "*.[Pp][Pp][Mm]")): - shutil.move(filename, backup_dir) - # Get the used files back - for step in self.steps: - self.set_state_from_step_lines(step, backup_dir, warn=False) - self.get_step_lines(self.steps_data_dir) - # Remove the used files from the backup dir - used_files = os.listdir(self.steps_data_dir) - for filename in os.listdir(backup_dir): - if filename in used_files: - os.unlink(os.path.join(backup_dir, filename)) - # Restore step index - self.set_step(current_step_index) - # Inform the user - self.message("All unused PPM files may be found at %s." % - os.path.abspath(backup_dir), - "Clean up data directory") - - # Methods - - def read_steps_file(self, filename): - steps = [] - header = "" - - file = open(filename, "r") - for line in file.readlines(): - words = line.split() - if not words: - continue - if line.startswith("# ----"): - continue - if words[0] == "step": - steps.append("") - if steps: - steps[-1] += line - else: - header += line - file.close() - - return (steps, header) - - def set_steps_file(self, filename): - try: - (self.steps, self.header) = self.read_steps_file(filename) - except (TypeError, IOError): - self.message("Cannot read file %s." % filename, "Error") - return - - self.steps_filename = filename - self.steps_data_dir = ppm_utils.get_data_dir(filename) - # Go to step 0 - self.set_step(0) - - def set_step(self, index): - # Limit index to legal boundaries - if index < 0: - index = 0 - if index > len(self.steps) - 1: - index = len(self.steps) - 1 - - # Enable the menus - self.menu_edit.set_sensitive(True) - self.menu_insert.set_sensitive(True) - self.menu_tools.set_sensitive(True) - - # If no steps exist... - if self.steps == []: - self.current_step_index = index - self.current_step = None - # Set window title - self.window.set_title("Step Editor -- %s" % - os.path.basename(self.steps_filename)) - # Set step entry widget text - self.entry_step_num.set_text("") - # Clear the state of all widgets - self.clear_state() - # Disable the main vbox - self.main_vbox.set_sensitive(False) - return - - self.current_step_index = index - self.current_step = self.steps[index] - # Set window title - self.window.set_title("Step Editor -- %s -- step %d" % - (os.path.basename(self.steps_filename), - index + 1)) - # Set step entry widget text - self.entry_step_num.set_text(str(self.current_step_index + 1)) - # Load the state from the step lines - self.set_state_from_step_lines(self.current_step, self.steps_data_dir) - # Enable the main vbox - self.main_vbox.set_sensitive(True) - # Make sure the step lines in self.current_step are identical to the - # output of self.get_step_lines - self.current_step = self.get_step_lines() - - def verify_save(self): - if not self.steps: - return - # See if the user changed anything - if self.get_step_lines() != self.current_step: - if self.question_yes_no("Step contents have been modified." - " Save step?", "Save changes?"): - self.save_step() - - def save_step(self): - lines = self.get_step_lines(self.steps_data_dir) - if lines != None: - self.steps[self.current_step_index] = lines - self.current_step = lines - self.write_steps_file(self.steps_filename) - - def write_steps_file(self, filename): - file = open(filename, "w") - file.write(self.header) - for step in self.steps: - file.write("# " + "-" * 32 + "\n") - file.write(step) - file.close() - - -if __name__ == "__main__": - se = StepEditor() - if len(sys.argv) > 1: - se.set_steps_file(sys.argv[1]) - gtk.main() diff --git a/client/tests/kvm/test_setup.py b/client/tests/kvm/test_setup.py deleted file mode 100644 index f915c1b7..00000000 --- a/client/tests/kvm/test_setup.py +++ /dev/null @@ -1,107 +0,0 @@ -""" -Library to perform pre/post test setup for KVM autotest. -""" -import os, logging -from autotest_lib.client.common_lib import error -from autotest_lib.client.bin import utils - - -class HugePageConfig(object): - def __init__(self, params): - """ - Gets environment variable values and calculates the target number - of huge memory pages. - - @param params: Dict like object containing parameters for the test. - """ - self.vms = len(params.objects("vms")) - self.mem = int(params.get("mem")) - self.max_vms = int(params.get("max_vms", 0)) - self.hugepage_path = '/mnt/kvm_hugepage' - self.hugepage_size = self.get_hugepage_size() - self.target_hugepages = self.get_target_hugepages() - self.kernel_hp_file = '/proc/sys/vm/nr_hugepages' - - - def get_hugepage_size(self): - """ - Get the current system setting for huge memory page size. - """ - meminfo = open('/proc/meminfo', 'r').readlines() - huge_line_list = [h for h in meminfo if h.startswith("Hugepagesize")] - try: - return int(huge_line_list[0].split()[1]) - except ValueError, e: - raise ValueError("Could not get huge page size setting from " - "/proc/meminfo: %s" % e) - - - def get_target_hugepages(self): - """ - Calculate the target number of hugepages for testing purposes. - """ - if self.vms < self.max_vms: - self.vms = self.max_vms - # memory of all VMs plus qemu overhead of 64MB per guest - vmsm = (self.vms * self.mem) + (self.vms * 64) - return int(vmsm * 1024 / self.hugepage_size) - - - @error.context_aware - def set_hugepages(self): - """ - Sets the hugepage limit to the target hugepage value calculated. - """ - error.context("setting hugepages limit to %s" % self.target_hugepages) - hugepage_cfg = open(self.kernel_hp_file, "r+") - hp = hugepage_cfg.readline() - while int(hp) < self.target_hugepages: - loop_hp = hp - hugepage_cfg.write(str(self.target_hugepages)) - hugepage_cfg.flush() - hugepage_cfg.seek(0) - hp = int(hugepage_cfg.readline()) - if loop_hp == hp: - raise ValueError("Cannot set the kernel hugepage setting " - "to the target value of %d hugepages." % - self.target_hugepages) - hugepage_cfg.close() - logging.debug("Successfuly set %s large memory pages on host ", - self.target_hugepages) - - - @error.context_aware - def mount_hugepage_fs(self): - """ - Verify if there's a hugetlbfs mount set. If there's none, will set up - a hugetlbfs mount using the class attribute that defines the mount - point. - """ - error.context("mounting hugepages path") - if not os.path.ismount(self.hugepage_path): - if not os.path.isdir(self.hugepage_path): - os.makedirs(self.hugepage_path) - cmd = "mount -t hugetlbfs none %s" % self.hugepage_path - utils.system(cmd) - - - def setup(self): - logging.debug("Number of VMs this test will use: %d", self.vms) - logging.debug("Amount of memory used by each vm: %s", self.mem) - logging.debug("System setting for large memory page size: %s", - self.hugepage_size) - logging.debug("Number of large memory pages needed for this test: %s", - self.target_hugepages) - self.set_hugepages() - self.mount_hugepage_fs() - - - @error.context_aware - def cleanup(self): - error.context("trying to dealocate hugepage memory") - try: - utils.system("umount %s" % self.hugepage_path) - except error.CmdError: - return - utils.system("echo 0 > %s" % self.kernel_hp_file) - logging.debug("Hugepage memory successfuly dealocated") -- 2.11.4.GIT