cli/topic_common.py

   1 #
   2 # Copyright 2008 Google Inc. All Rights Reserved.
   3 #
   4 """
   5 This module contains the generic CLI object
   6
   7 High Level Design:
   8
   9 The atest class contains attributes & method generic to all the CLI
  10 operations.
  11
  12 The class inheritance is shown here using the command
  13 'atest host create ...' as an example:
  14
  15 atest <-- host <-- host_create <-- site_host_create
  16
  17 Note: The site_<topic>.py and its classes are only needed if you need
  18 to override the common <topic>.py methods with your site specific ones.
  19
  20
  21 High Level Algorithm:
  22
  23 1. atest figures out the topic and action from the 2 first arguments
  24    on the command line and imports the <topic> (or site_<topic>)
  25    module.
  26
  27 1. Init
  28    The main atest module creates a <topic>_<action> object.  The
  29    __init__() function is used to setup the parser options, if this
  30    <action> has some specific options to add to its <topic>.
  31
  32    If it exists, the child __init__() method must call its parent
  33    class __init__() before adding its own parser arguments.
  34
  35 2. Parsing
  36    If the child wants to validate the parsing (e.g. make sure that
  37    there are hosts in the arguments), or if it wants to check the
  38    options it added in its __init__(), it should implement a parse()
  39    method.
  40
  41    The child parser must call its parent parser and gets back the
  42    options dictionary and the rest of the command line arguments
  43    (leftover). Each level gets to see all the options, but the
  44    leftovers can be deleted as they can be consumed by only one
  45    object.
  46
  47 3. Execution
  48    This execute() method is specific to the child and should use the
  49    self.execute_rpc() to send commands to the Autotest Front-End.  It
  50    should return results.
  51
  52 4. Output
  53    The child output() method is called with the execute() resutls as a
  54    parameter.  This is child-specific, but should leverage the
  55    atest.print_*() methods.
  56 """
  57
  58 import getpass, optparse, os, pwd, re, socket, sys, textwrap, traceback
  59 import socket, string, urllib2
  60 from autotest_lib.cli import rpc
  61 from autotest_lib.frontend.afe.json_rpc import proxy
  62 from autotest_lib.client.common_lib.test_utils import mock
  63
  64
  65 # Maps the AFE keys to printable names.
  66 KEYS_TO_NAMES_EN = {'hostname': 'Host',
  67                     'platform': 'Platform',
  68                     'status': 'Status',
  69                     'locked': 'Locked',
  70                     'locked_by': 'Locked by',
  71                     'lock_time': 'Locked time',
  72                     'labels': 'Labels',
  73                     'description': 'Description',
  74                     'hosts': 'Hosts',
  75                     'users': 'Users',
  76                     'id': 'Id',
  77                     'name': 'Name',
  78                     'invalid': 'Valid',
  79                     'login': 'Login',
  80                     'access_level': 'Access Level',
  81                     'job_id': 'Job Id',
  82                     'job_owner': 'Job Owner',
  83                     'job_name': 'Job Name',
  84                     'test_type': 'Test Type',
  85                     'test_class': 'Test Class',
  86                     'path': 'Path',
  87                     'owner': 'Owner',
  88                     'status_counts': 'Status Counts',
  89                     'hosts_status': 'Host Status',
  90                     'hosts_selected_status': 'Hosts filtered by Status',
  91                     'priority': 'Priority',
  92                     'control_type': 'Control Type',
  93                     'created_on': 'Created On',
  94                     'synch_type': 'Synch Type',
  95                     'control_file': 'Control File',
  96                     'only_if_needed': 'Use only if needed',
  97                     'protection': 'Protection',
  98                     'run_verify': 'Run verify',
  99                     'reboot_before': 'Pre-job reboot',
 100                     'reboot_after': 'Post-job reboot',
 101                     'experimental': 'Experimental',
 102                     'synch_count': 'Sync Count',
 103                     'max_number_of_machines': 'Max. hosts to use',
 104                     'parse_failed_repair': 'Include failed repair results',
 105                     'atomic_group.name': 'Atomic Group Name',
 106                     }
 107
 108 # In the failure, tag that will replace the item.
 109 FAIL_TAG = '<XYZ>'
 110
 111 # Global socket timeout: uploading kernels can take much,
 112 # much longer than the default
 113 UPLOAD_SOCKET_TIMEOUT = 60*30
 114
 115
 116 # Convertion functions to be called for printing,
 117 # e.g. to print True/False for booleans.
 118 def __convert_platform(field):
 119     if field is None:
 120         return ""
 121     elif isinstance(field, int):
 122         # Can be 0/1 for False/True
 123         return str(bool(field))
 124     else:
 125         # Can be a platform name
 126         return field
 127
 128
 129 def _int_2_bool_string(value):
 130     return str(bool(value))
 131
 132 KEYS_CONVERT = {'locked': _int_2_bool_string,
 133                 'invalid': lambda flag: str(bool(not flag)),
 134                 'only_if_needed': _int_2_bool_string,
 135                 'platform': __convert_platform,
 136                 'labels': lambda labels: ', '.join(labels)}
 137
 138
 139 def _get_item_key(item, key):
 140     """Allow for lookups in nested dictionaries using '.'s within a key."""
 141     if key in item:
 142         return item[key]
 143     nested_item = item
 144     for subkey in key.split('.'):
 145         if not subkey:
 146             raise ValueError('empty subkey in %r' % key)
 147         try:
 148             nested_item = nested_item[subkey]
 149         except KeyError, e:
 150             raise KeyError('%r - looking up key %r in %r' %
 151                            (e, key, nested_item))
 152     else:
 153         return nested_item
 154
 155
 156 class CliError(Exception):
 157     pass
 158
 159
 160 class item_parse_info(object):
 161     def __init__(self, attribute_name, inline_option='',
 162                  filename_option='', use_leftover=False):
 163         """Object keeping track of the parsing options that will
 164         make up the content of the atest attribute:
 165         atttribute_name: the atest attribute name to populate    (label)
 166         inline_option: the option containing the items           (--label)
 167         filename_option: the option containing the filename      (--blist)
 168         use_leftover: whether to add the leftover arguments or not."""
 169         self.attribute_name = attribute_name
 170         self.filename_option = filename_option
 171         self.inline_option = inline_option
 172         self.use_leftover = use_leftover
 173
 174
 175     def get_values(self, options, leftover=[]):
 176         """Returns the value for that attribute by accumualting all
 177         the values found through the inline option, the parsing of the
 178         file and the leftover"""
 179
 180         def __get_items(input, split_spaces=True):
 181             """Splits a string of comma separated items. Escaped commas will not
 182             be split. I.e. Splitting 'a, b\,c, d' will yield ['a', 'b,c', 'd'].
 183             If split_spaces is set to False spaces will not be split. I.e.
 184             Splitting 'a b, c\,d, e' will yield ['a b', 'c,d', 'e']"""
 185
 186             # Replace escaped slashes with null characters so we don't misparse
 187             # proceeding commas.
 188             input = input.replace(r'\\', '\0')
 189
 190             # Split on commas which are not preceded by a slash.
 191             if not split_spaces:
 192                 split = re.split(r'(?<!\\),', input)
 193             else:
 194                 split = re.split(r'(?<!\\),|\s', input)
 195
 196             # Convert null characters to single slashes and escaped commas to
 197             # just plain commas.
 198             return (item.strip().replace('\0', '\\').replace(r'\,', ',') for
 199                     item in split if item.strip())
 200
 201         if self.use_leftover:
 202             add_on = leftover
 203             leftover = []
 204         else:
 205             add_on = []
 206
 207         # Start with the add_on
 208         result = set()
 209         for items in add_on:
 210             # Don't split on space here because the add-on
 211             # may have some spaces (like the job name)
 212             result.update(__get_items(items, split_spaces=False))
 213
 214         # Process the inline_option, if any
 215         try:
 216             items = getattr(options, self.inline_option)
 217             result.update(__get_items(items))
 218         except (AttributeError, TypeError):
 219             pass
 220
 221         # Process the file list, if any and not empty
 222         # The file can contain space and/or comma separated items
 223         try:
 224             flist = getattr(options, self.filename_option)
 225             file_content = []
 226             for line in open(flist).readlines():
 227                 file_content += __get_items(line)
 228             if len(file_content) == 0:
 229                 raise CliError("Empty file %s" % flist)
 230             result.update(file_content)
 231         except (AttributeError, TypeError):
 232             pass
 233         except IOError:
 234             raise CliError("Could not open file %s" % flist)
 235
 236         return list(result), leftover
 237
 238
 239 class atest(object):
 240     """Common class for generic processing
 241     Should only be instantiated by itself for usage
 242     references, otherwise, the <topic> objects should
 243     be used."""
 244     msg_topic = "[acl|host|job|label|atomicgroup|test|user]"
 245     usage_action = "[action]"
 246     msg_items = ''
 247
 248     def invalid_arg(self, header, follow_up=''):
 249         twrap = textwrap.TextWrapper(initial_indent='        ',
 250                                      subsequent_indent='       ')
 251         rest = twrap.fill(follow_up)
 252
 253         if self.kill_on_failure:
 254             self.invalid_syntax(header + rest)
 255         else:
 256             print >> sys.stderr, header + rest
 257
 258
 259     def invalid_syntax(self, msg):
 260         print
 261         print >> sys.stderr, msg
 262         print
 263         print "usage:",
 264         print self._get_usage()
 265         print
 266         sys.exit(1)
 267
 268
 269     def generic_error(self, msg):
 270         if self.debug:
 271             traceback.print_exc()
 272         print >> sys.stderr, msg
 273         sys.exit(1)
 274
 275
 276     def parse_json_exception(self, full_error):
 277         """Parses the JSON exception to extract the bad
 278         items and returns them
 279         This is very kludgy for the moment, but we would need
 280         to refactor the exceptions sent from the front end
 281         to make this better"""
 282         errmsg = str(full_error).split('Traceback')[0].rstrip('\n')
 283         parts = errmsg.split(':')
 284         # Kludge: If there are 2 colons the last parts contains
 285         # the items that failed.
 286         if len(parts) != 3:
 287             return []
 288         return [item.strip() for item in parts[2].split(',') if item.strip()]
 289
 290
 291     def failure(self, full_error, item=None, what_failed='', fatal=False):
 292         """If kill_on_failure, print this error and die,
 293         otherwise, queue the error and accumulate all the items
 294         that triggered the same error."""
 295
 296         if self.debug:
 297             errmsg = str(full_error)
 298         else:
 299             errmsg = str(full_error).split('Traceback')[0].rstrip('\n')
 300
 301         if self.kill_on_failure or fatal:
 302             print >> sys.stderr, "%s\n    %s" % (what_failed, errmsg)
 303             sys.exit(1)
 304
 305         # Build a dictionary with the 'what_failed' as keys.  The
 306         # values are dictionaries with the errmsg as keys and a set
 307         # of items as values.
 308         # self.failed =
 309         # {'Operation delete_host_failed': {'AclAccessViolation:
 310         #                                        set('host0', 'host1')}}
 311         # Try to gather all the same error messages together,
 312         # even if they contain the 'item'
 313         if item and item in errmsg:
 314             errmsg = errmsg.replace(item, FAIL_TAG)
 315         if self.failed.has_key(what_failed):
 316             self.failed[what_failed].setdefault(errmsg, set()).add(item)
 317         else:
 318             self.failed[what_failed] = {errmsg: set([item])}
 319
 320
 321     def show_all_failures(self):
 322         if not self.failed:
 323             return 0
 324         for what_failed in self.failed.keys():
 325             print >> sys.stderr, what_failed + ':'
 326             for (errmsg, items) in self.failed[what_failed].iteritems():
 327                 if len(items) == 0:
 328                     print >> sys.stderr, errmsg
 329                 elif items == set(['']):
 330                     print >> sys.stderr, '    ' + errmsg
 331                 elif len(items) == 1:
 332                     # Restore the only item
 333                     if FAIL_TAG in errmsg:
 334                         errmsg = errmsg.replace(FAIL_TAG, items.pop())
 335                     else:
 336                         errmsg = '%s (%s)' % (errmsg, items.pop())
 337                     print >> sys.stderr, '    ' + errmsg
 338                 else:
 339                     print >> sys.stderr, '    ' + errmsg + ' with <XYZ> in:'
 340                     twrap = textwrap.TextWrapper(initial_indent='        ',
 341                                                  subsequent_indent='        ')
 342                     items = list(items)
 343                     items.sort()
 344                     print >> sys.stderr, twrap.fill(', '.join(items))
 345         return 1
 346
 347
 348     def __init__(self):
 349         """Setup the parser common options"""
 350         # Initialized for unit tests.
 351         self.afe = None
 352         self.failed = {}
 353         self.data = {}
 354         self.debug = False
 355         self.parse_delim = '|'
 356         self.kill_on_failure = False
 357         self.web_server = ''
 358         self.verbose = False
 359         self.topic_parse_info = item_parse_info(attribute_name='not_used')
 360
 361         self.parser = optparse.OptionParser(self._get_usage())
 362         self.parser.add_option('-g', '--debug',
 363                                help='Print debugging information',
 364                                action='store_true', default=False)
 365         self.parser.add_option('--kill-on-failure',
 366                                help='Stop at the first failure',
 367                                action='store_true', default=False)
 368         self.parser.add_option('--parse',
 369                                help='Print the output using | '
 370                                'separated key=value fields',
 371                                action='store_true', default=False)
 372         self.parser.add_option('--parse-delim',
 373                                help='Delimiter to use to separate the '
 374                                'key=value fields', default='|')
 375         self.parser.add_option('-v', '--verbose',
 376                                action='store_true', default=False)
 377         self.parser.add_option('-w', '--web',
 378                                help='Specify the autotest server '
 379                                'to talk to',
 380                                action='store', type='string',
 381                                dest='web_server', default=None)
 382
 383
 384     def _get_usage(self):
 385         return "atest %s %s [options] %s" % (self.msg_topic.lower(),
 386                                              self.usage_action,
 387                                              self.msg_items)
 388
 389
 390     def backward_compatibility(self, action, argv):
 391         """To be overidden by subclass if their syntax changed"""
 392         return action
 393
 394
 395     def parse(self, parse_info=[], req_items=None):
 396         """parse_info is a list of item_parse_info objects
 397
 398         There should only be one use_leftover set to True in the list.
 399
 400         Also check that the req_items is not empty after parsing."""
 401         (options, leftover) = self.parse_global()
 402
 403         all_parse_info = parse_info[:]
 404         all_parse_info.append(self.topic_parse_info)
 405
 406         try:
 407             for item_parse_info in all_parse_info:
 408                 values, leftover = item_parse_info.get_values(options,
 409                                                               leftover)
 410                 setattr(self, item_parse_info.attribute_name, values)
 411         except CliError, s:
 412             self.invalid_syntax(s)
 413
 414         if (req_items and not getattr(self, req_items, None)):
 415             self.invalid_syntax('%s %s requires at least one %s' %
 416                                 (self.msg_topic,
 417                                  self.usage_action,
 418                                  self.msg_topic))
 419
 420         return (options, leftover)
 421
 422
 423     def parse_global(self):
 424         """Parse the global arguments.
 425
 426         It consumes what the common object needs to know, and
 427         let the children look at all the options.  We could
 428         remove the options that we have used, but there is no
 429         harm in leaving them, and the children may need them
 430         in the future.
 431
 432         Must be called from its children parse()"""
 433         (options, leftover) = self.parser.parse_args()
 434         # Handle our own options setup in __init__()
 435         self.debug = options.debug
 436         self.kill_on_failure = options.kill_on_failure
 437
 438         if options.parse:
 439             suffix = '_parse'
 440         else:
 441             suffix = '_std'
 442         for func in ['print_fields', 'print_table',
 443                      'print_by_ids', 'print_list']:
 444             setattr(self, func, getattr(self, func + suffix))
 445
 446         self.parse_delim = options.parse_delim
 447
 448         self.verbose = options.verbose
 449         self.web_server = options.web_server
 450         try:
 451             self.afe = rpc.afe_comm(self.web_server)
 452         except rpc.AuthError, s:
 453             self.failure(str(s), fatal=True)
 454
 455         return (options, leftover)
 456
 457
 458     def check_and_create_items(self, op_get, op_create,
 459                                 items, **data_create):
 460         """Create the items if they don't exist already"""
 461         for item in items:
 462             ret = self.execute_rpc(op_get, name=item)
 463
 464             if len(ret) == 0:
 465                 try:
 466                     data_create['name'] = item
 467                     self.execute_rpc(op_create, **data_create)
 468                 except CliError:
 469                     continue
 470
 471
 472     def execute_rpc(self, op, item='', **data):
 473         retry = 2
 474         while retry:
 475             try:
 476                 return self.afe.run(op, **data)
 477             except urllib2.URLError, err:
 478                 if hasattr(err, 'reason'):
 479                     if 'timed out' not in err.reason:
 480                         self.invalid_syntax('Invalid server name %s: %s' %
 481                                             (self.afe.web_server, err))
 482                 if hasattr(err, 'code'):
 483                     error_parts = [str(err)]
 484                     if self.debug:
 485                         error_parts.append(err.read()) # read the response body
 486                     self.failure('\n\n'.join(error_parts), item=item,
 487                                  what_failed=("Error received from web server"))
 488                     raise CliError("Error from web server")
 489                 if self.debug:
 490                     print 'retrying: %r %d' % (data, retry)
 491                 retry -= 1
 492                 if retry == 0:
 493                     if item:
 494                         myerr = '%s timed out for %s' % (op, item)
 495                     else:
 496                         myerr = '%s timed out' % op
 497                     self.failure(myerr, item=item,
 498                                  what_failed=("Timed-out contacting "
 499                                               "the Autotest server"))
 500                     raise CliError("Timed-out contacting the Autotest server")
 501             except mock.CheckPlaybackError:
 502                 raise
 503             except Exception, full_error:
 504                 # There are various exceptions throwns by JSON,
 505                 # urllib & httplib, so catch them all.
 506                 self.failure(full_error, item=item,
 507                              what_failed='Operation %s failed' % op)
 508                 raise CliError(str(full_error))
 509
 510
 511     # There is no output() method in the atest object (yet?)
 512     # but here are some helper functions to be used by its
 513     # children
 514     def print_wrapped(self, msg, values):
 515         if len(values) == 0:
 516             return
 517         elif len(values) == 1:
 518             print msg + ': '
 519         elif len(values) > 1:
 520             if msg.endswith('s'):
 521                 print msg + ': '
 522             else:
 523                 print msg + 's: '
 524
 525         values.sort()
 526
 527         if 'AUTOTEST_CLI_NO_WRAP' in os.environ:
 528             print '\n'.join(values)
 529             return
 530
 531         twrap = textwrap.TextWrapper(initial_indent='\t',
 532                                      subsequent_indent='\t')
 533         print twrap.fill(', '.join(values))
 534
 535
 536     def __conv_value(self, type, value):
 537         return KEYS_CONVERT.get(type, str)(value)
 538
 539
 540     def print_fields_std(self, items, keys, title=None):
 541         """Print the keys in each item, one on each line"""
 542         if not items:
 543             return
 544         if title:
 545             print title
 546         for item in items:
 547             for key in keys:
 548                 print '%s: %s' % (KEYS_TO_NAMES_EN[key],
 549                                   self.__conv_value(key,
 550                                                     _get_item_key(item, key)))
 551
 552
 553     def print_fields_parse(self, items, keys, title=None):
 554         """Print the keys in each item as comma
 555         separated name=value"""
 556         for item in items:
 557             values = ['%s=%s' % (KEYS_TO_NAMES_EN[key],
 558                                   self.__conv_value(key,
 559                                                     _get_item_key(item, key)))
 560                       for key in keys
 561                       if self.__conv_value(key,
 562                                            _get_item_key(item, key)) != '']
 563             print self.parse_delim.join(values)
 564
 565
 566     def __find_justified_fmt(self, items, keys):
 567         """Find the max length for each field."""
 568         lens = {}
 569         # Don't justify the last field, otherwise we have blank
 570         # lines when the max is overlaps but the current values
 571         # are smaller
 572         if not items:
 573             print "No results"
 574             return
 575         for key in keys[:-1]:
 576             lens[key] = max(len(self.__conv_value(key,
 577                                                   _get_item_key(item, key)))
 578                             for item in items)
 579             lens[key] = max(lens[key], len(KEYS_TO_NAMES_EN[key]))
 580         lens[keys[-1]] = 0
 581
 582         return '  '.join(["%%-%ds" % lens[key] for key in keys])
 583
 584
 585     def print_table_std(self, items, keys_header, sublist_keys=()):
 586         """Print a mix of header and lists in a user readable
 587         format
 588         The headers are justified, the sublist_keys are wrapped."""
 589         if not items:
 590             return
 591         fmt = self.__find_justified_fmt(items, keys_header)
 592         header = tuple(KEYS_TO_NAMES_EN[key] for key in keys_header)
 593         print fmt % header
 594         for item in items:
 595             values = tuple(self.__conv_value(key,
 596                                              _get_item_key(item, key))
 597                            for key in keys_header)
 598             print fmt % values
 599             if sublist_keys:
 600                 for key in sublist_keys:
 601                     self.print_wrapped(KEYS_TO_NAMES_EN[key],
 602                                        _get_item_key(item, key))
 603                 print '\n'
 604
 605
 606     def print_table_parse(self, items, keys_header, sublist_keys=()):
 607         """Print a mix of header and lists in a user readable
 608         format"""
 609         for item in items:
 610             values = ['%s=%s' % (KEYS_TO_NAMES_EN[key],
 611                                  self.__conv_value(key, _get_item_key(item, key)))
 612                       for key in keys_header
 613                       if self.__conv_value(key,
 614                                            _get_item_key(item, key)) != '']
 615
 616             if sublist_keys:
 617                 [values.append('%s=%s'% (KEYS_TO_NAMES_EN[key],
 618                                          ','.join(_get_item_key(item, key))))
 619                  for key in sublist_keys
 620                  if len(_get_item_key(item, key))]
 621
 622             print self.parse_delim.join(values)
 623
 624
 625     def print_by_ids_std(self, items, title=None, line_before=False):
 626         """Prints ID & names of items in a user readable form"""
 627         if not items:
 628             return
 629         if line_before:
 630             print
 631         if title:
 632             print title + ':'
 633         self.print_table_std(items, keys_header=['id', 'name'])
 634
 635
 636     def print_by_ids_parse(self, items, title=None, line_before=False):
 637         """Prints ID & names of items in a parseable format"""
 638         if not items:
 639             return
 640         if title:
 641             print title + '=',
 642         values = []
 643         for item in items:
 644             values += ['%s=%s' % (KEYS_TO_NAMES_EN[key],
 645                                   self.__conv_value(key,
 646                                                     _get_item_key(item, key)))
 647                        for key in ['id', 'name']
 648                        if self.__conv_value(key,
 649                                             _get_item_key(item, key)) != '']
 650         print self.parse_delim.join(values)
 651
 652
 653     def print_list_std(self, items, key):
 654         """Print a wrapped list of results"""
 655         if not items:
 656             return
 657         print ' '.join(_get_item_key(item, key) for item in items)
 658
 659
 660     def print_list_parse(self, items, key):
 661         """Print a wrapped list of results"""
 662         if not items:
 663             return
 664         print '%s=%s' % (KEYS_TO_NAMES_EN[key],
 665                          ','.join(_get_item_key(item, key) for item in items))