scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2015 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 import re
  15 from ordereddict import OrderedDict
  16 import errno
  17 import getopt
  18 import os
  19 import sys
  20 import string
  21
  22 builtin_types = {
  23     'str':      'QTYPE_QSTRING',
  24     'int':      'QTYPE_QINT',
  25     'number':   'QTYPE_QFLOAT',
  26     'bool':     'QTYPE_QBOOL',
  27     'int8':     'QTYPE_QINT',
  28     'int16':    'QTYPE_QINT',
  29     'int32':    'QTYPE_QINT',
  30     'int64':    'QTYPE_QINT',
  31     'uint8':    'QTYPE_QINT',
  32     'uint16':   'QTYPE_QINT',
  33     'uint32':   'QTYPE_QINT',
  34     'uint64':   'QTYPE_QINT',
  35     'size':     'QTYPE_QINT',
  36     'any':      None,           # any qtype_code possible, actually
  37 }
  38
  39 # Whitelist of commands allowed to return a non-dictionary
  40 returns_whitelist = [
  41     # From QMP:
  42     'human-monitor-command',
  43     'qom-get',
  44     'query-migrate-cache-size',
  45     'query-tpm-models',
  46     'query-tpm-types',
  47     'ringbuf-read',
  48
  49     # From QGA:
  50     'guest-file-open',
  51     'guest-fsfreeze-freeze',
  52     'guest-fsfreeze-freeze-list',
  53     'guest-fsfreeze-status',
  54     'guest-fsfreeze-thaw',
  55     'guest-get-time',
  56     'guest-set-vcpus',
  57     'guest-sync',
  58     'guest-sync-delimited',
  59
  60     # From qapi-schema-test:
  61     'user_def_cmd3',
  62 ]
  63
  64 enum_types = []
  65 struct_types = []
  66 union_types = []
  67 events = []
  68 all_names = {}
  69
  70 #
  71 # Parsing the schema into expressions
  72 #
  73
  74 def error_path(parent):
  75     res = ""
  76     while parent:
  77         res = ("In file included from %s:%d:\n" % (parent['file'],
  78                                                    parent['line'])) + res
  79         parent = parent['parent']
  80     return res
  81
  82 class QAPISchemaError(Exception):
  83     def __init__(self, schema, msg):
  84         self.fname = schema.fname
  85         self.msg = msg
  86         self.col = 1
  87         self.line = schema.line
  88         for ch in schema.src[schema.line_pos:schema.pos]:
  89             if ch == '\t':
  90                 self.col = (self.col + 7) % 8 + 1
  91             else:
  92                 self.col += 1
  93         self.info = schema.incl_info
  94
  95     def __str__(self):
  96         return error_path(self.info) + \
  97             "%s:%d:%d: %s" % (self.fname, self.line, self.col, self.msg)
  98
  99 class QAPIExprError(Exception):
 100     def __init__(self, expr_info, msg):
 101         self.info = expr_info
 102         self.msg = msg
 103
 104     def __str__(self):
 105         return error_path(self.info['parent']) + \
 106             "%s:%d: %s" % (self.info['file'], self.info['line'], self.msg)
 107
 108 class QAPISchemaParser(object):
 109
 110     def __init__(self, fp, previously_included = [], incl_info = None):
 111         abs_fname = os.path.abspath(fp.name)
 112         fname = fp.name
 113         self.fname = fname
 114         previously_included.append(abs_fname)
 115         self.incl_info = incl_info
 116         self.src = fp.read()
 117         if self.src == '' or self.src[-1] != '\n':
 118             self.src += '\n'
 119         self.cursor = 0
 120         self.line = 1
 121         self.line_pos = 0
 122         self.exprs = []
 123         self.accept()
 124
 125         while self.tok != None:
 126             expr_info = {'file': fname, 'line': self.line,
 127                          'parent': self.incl_info}
 128             expr = self.get_expr(False)
 129             if isinstance(expr, dict) and "include" in expr:
 130                 if len(expr) != 1:
 131                     raise QAPIExprError(expr_info, "Invalid 'include' directive")
 132                 include = expr["include"]
 133                 if not isinstance(include, str):
 134                     raise QAPIExprError(expr_info,
 135                                         'Expected a file name (string), got: %s'
 136                                         % include)
 137                 incl_abs_fname = os.path.join(os.path.dirname(abs_fname),
 138                                               include)
 139                 # catch inclusion cycle
 140                 inf = expr_info
 141                 while inf:
 142                     if incl_abs_fname == os.path.abspath(inf['file']):
 143                         raise QAPIExprError(expr_info, "Inclusion loop for %s"
 144                                             % include)
 145                     inf = inf['parent']
 146                 # skip multiple include of the same file
 147                 if incl_abs_fname in previously_included:
 148                     continue
 149                 try:
 150                     fobj = open(incl_abs_fname, 'r')
 151                 except IOError, e:
 152                     raise QAPIExprError(expr_info,
 153                                         '%s: %s' % (e.strerror, include))
 154                 exprs_include = QAPISchemaParser(fobj, previously_included,
 155                                                  expr_info)
 156                 self.exprs.extend(exprs_include.exprs)
 157             else:
 158                 expr_elem = {'expr': expr,
 159                              'info': expr_info}
 160                 self.exprs.append(expr_elem)
 161
 162     def accept(self):
 163         while True:
 164             self.tok = self.src[self.cursor]
 165             self.pos = self.cursor
 166             self.cursor += 1
 167             self.val = None
 168
 169             if self.tok == '#':
 170                 self.cursor = self.src.find('\n', self.cursor)
 171             elif self.tok in ['{', '}', ':', ',', '[', ']']:
 172                 return
 173             elif self.tok == "'":
 174                 string = ''
 175                 esc = False
 176                 while True:
 177                     ch = self.src[self.cursor]
 178                     self.cursor += 1
 179                     if ch == '\n':
 180                         raise QAPISchemaError(self,
 181                                               'Missing terminating "\'"')
 182                     if esc:
 183                         if ch == 'b':
 184                             string += '\b'
 185                         elif ch == 'f':
 186                             string += '\f'
 187                         elif ch == 'n':
 188                             string += '\n'
 189                         elif ch == 'r':
 190                             string += '\r'
 191                         elif ch == 't':
 192                             string += '\t'
 193                         elif ch == 'u':
 194                             value = 0
 195                             for x in range(0, 4):
 196                                 ch = self.src[self.cursor]
 197                                 self.cursor += 1
 198                                 if ch not in "0123456789abcdefABCDEF":
 199                                     raise QAPISchemaError(self,
 200                                                           '\\u escape needs 4 '
 201                                                           'hex digits')
 202                                 value = (value << 4) + int(ch, 16)
 203                             # If Python 2 and 3 didn't disagree so much on
 204                             # how to handle Unicode, then we could allow
 205                             # Unicode string defaults.  But most of QAPI is
 206                             # ASCII-only, so we aren't losing much for now.
 207                             if not value or value > 0x7f:
 208                                 raise QAPISchemaError(self,
 209                                                       'For now, \\u escape '
 210                                                       'only supports non-zero '
 211                                                       'values up to \\u007f')
 212                             string += chr(value)
 213                         elif ch in "\\/'\"":
 214                             string += ch
 215                         else:
 216                             raise QAPISchemaError(self,
 217                                                   "Unknown escape \\%s" %ch)
 218                         esc = False
 219                     elif ch == "\\":
 220                         esc = True
 221                     elif ch == "'":
 222                         self.val = string
 223                         return
 224                     else:
 225                         string += ch
 226             elif self.src.startswith("true", self.pos):
 227                 self.val = True
 228                 self.cursor += 3
 229                 return
 230             elif self.src.startswith("false", self.pos):
 231                 self.val = False
 232                 self.cursor += 4
 233                 return
 234             elif self.src.startswith("null", self.pos):
 235                 self.val = None
 236                 self.cursor += 3
 237                 return
 238             elif self.tok == '\n':
 239                 if self.cursor == len(self.src):
 240                     self.tok = None
 241                     return
 242                 self.line += 1
 243                 self.line_pos = self.cursor
 244             elif not self.tok.isspace():
 245                 raise QAPISchemaError(self, 'Stray "%s"' % self.tok)
 246
 247     def get_members(self):
 248         expr = OrderedDict()
 249         if self.tok == '}':
 250             self.accept()
 251             return expr
 252         if self.tok != "'":
 253             raise QAPISchemaError(self, 'Expected string or "}"')
 254         while True:
 255             key = self.val
 256             self.accept()
 257             if self.tok != ':':
 258                 raise QAPISchemaError(self, 'Expected ":"')
 259             self.accept()
 260             if key in expr:
 261                 raise QAPISchemaError(self, 'Duplicate key "%s"' % key)
 262             expr[key] = self.get_expr(True)
 263             if self.tok == '}':
 264                 self.accept()
 265                 return expr
 266             if self.tok != ',':
 267                 raise QAPISchemaError(self, 'Expected "," or "}"')
 268             self.accept()
 269             if self.tok != "'":
 270                 raise QAPISchemaError(self, 'Expected string')
 271
 272     def get_values(self):
 273         expr = []
 274         if self.tok == ']':
 275             self.accept()
 276             return expr
 277         if not self.tok in "{['tfn":
 278             raise QAPISchemaError(self, 'Expected "{", "[", "]", string, '
 279                                   'boolean or "null"')
 280         while True:
 281             expr.append(self.get_expr(True))
 282             if self.tok == ']':
 283                 self.accept()
 284                 return expr
 285             if self.tok != ',':
 286                 raise QAPISchemaError(self, 'Expected "," or "]"')
 287             self.accept()
 288
 289     def get_expr(self, nested):
 290         if self.tok != '{' and not nested:
 291             raise QAPISchemaError(self, 'Expected "{"')
 292         if self.tok == '{':
 293             self.accept()
 294             expr = self.get_members()
 295         elif self.tok == '[':
 296             self.accept()
 297             expr = self.get_values()
 298         elif self.tok in "'tfn":
 299             expr = self.val
 300             self.accept()
 301         else:
 302             raise QAPISchemaError(self, 'Expected "{", "[" or string')
 303         return expr
 304
 305 #
 306 # Semantic analysis of schema expressions
 307 # TODO fold into QAPISchema
 308 # TODO catching name collisions in generated code would be nice
 309 #
 310
 311 def find_base_fields(base):
 312     base_struct_define = find_struct(base)
 313     if not base_struct_define:
 314         return None
 315     return base_struct_define['data']
 316
 317 # Return the qtype of an alternate branch, or None on error.
 318 def find_alternate_member_qtype(qapi_type):
 319     if builtin_types.has_key(qapi_type):
 320         return builtin_types[qapi_type]
 321     elif find_struct(qapi_type):
 322         return "QTYPE_QDICT"
 323     elif find_enum(qapi_type):
 324         return "QTYPE_QSTRING"
 325     elif find_union(qapi_type):
 326         return "QTYPE_QDICT"
 327     return None
 328
 329 # Return the discriminator enum define if discriminator is specified as an
 330 # enum type, otherwise return None.
 331 def discriminator_find_enum_define(expr):
 332     base = expr.get('base')
 333     discriminator = expr.get('discriminator')
 334
 335     if not (discriminator and base):
 336         return None
 337
 338     base_fields = find_base_fields(base)
 339     if not base_fields:
 340         return None
 341
 342     discriminator_type = base_fields.get(discriminator)
 343     if not discriminator_type:
 344         return None
 345
 346     return find_enum(discriminator_type)
 347
 348 # FIXME should enforce "other than downstream extensions [...], all
 349 # names should begin with a letter".
 350 valid_name = re.compile('^[a-zA-Z_][a-zA-Z0-9_.-]*$')
 351 def check_name(expr_info, source, name, allow_optional = False,
 352                enum_member = False):
 353     global valid_name
 354     membername = name
 355
 356     if not isinstance(name, str):
 357         raise QAPIExprError(expr_info,
 358                             "%s requires a string name" % source)
 359     if name.startswith('*'):
 360         membername = name[1:]
 361         if not allow_optional:
 362             raise QAPIExprError(expr_info,
 363                                 "%s does not allow optional name '%s'"
 364                                 % (source, name))
 365     # Enum members can start with a digit, because the generated C
 366     # code always prefixes it with the enum name
 367     if enum_member:
 368         membername = '_' + membername
 369     if not valid_name.match(membername):
 370         raise QAPIExprError(expr_info,
 371                             "%s uses invalid name '%s'" % (source, name))
 372
 373 def add_name(name, info, meta, implicit = False):
 374     global all_names
 375     check_name(info, "'%s'" % meta, name)
 376     # FIXME should reject names that differ only in '_' vs. '.'
 377     # vs. '-', because they're liable to clash in generated C.
 378     if name in all_names:
 379         raise QAPIExprError(info,
 380                             "%s '%s' is already defined"
 381                             % (all_names[name], name))
 382     if not implicit and name[-4:] == 'Kind':
 383         raise QAPIExprError(info,
 384                             "%s '%s' should not end in 'Kind'"
 385                             % (meta, name))
 386     all_names[name] = meta
 387
 388 def add_struct(definition, info):
 389     global struct_types
 390     name = definition['struct']
 391     add_name(name, info, 'struct')
 392     struct_types.append(definition)
 393
 394 def find_struct(name):
 395     global struct_types
 396     for struct in struct_types:
 397         if struct['struct'] == name:
 398             return struct
 399     return None
 400
 401 def add_union(definition, info):
 402     global union_types
 403     name = definition['union']
 404     add_name(name, info, 'union')
 405     union_types.append(definition)
 406
 407 def find_union(name):
 408     global union_types
 409     for union in union_types:
 410         if union['union'] == name:
 411             return union
 412     return None
 413
 414 def add_enum(name, info, enum_values = None, implicit = False):
 415     global enum_types
 416     add_name(name, info, 'enum', implicit)
 417     enum_types.append({"enum_name": name, "enum_values": enum_values})
 418
 419 def find_enum(name):
 420     global enum_types
 421     for enum in enum_types:
 422         if enum['enum_name'] == name:
 423             return enum
 424     return None
 425
 426 def is_enum(name):
 427     return find_enum(name) != None
 428
 429 def check_type(expr_info, source, value, allow_array = False,
 430                allow_dict = False, allow_optional = False,
 431                allow_star = False, allow_metas = []):
 432     global all_names
 433
 434     if value is None:
 435         return
 436
 437     if allow_star and value == '**':
 438         return
 439
 440     # Check if array type for value is okay
 441     if isinstance(value, list):
 442         if not allow_array:
 443             raise QAPIExprError(expr_info,
 444                                 "%s cannot be an array" % source)
 445         if len(value) != 1 or not isinstance(value[0], str):
 446             raise QAPIExprError(expr_info,
 447                                 "%s: array type must contain single type name"
 448                                 % source)
 449         value = value[0]
 450
 451     # Check if type name for value is okay
 452     if isinstance(value, str):
 453         if value == '**':
 454             raise QAPIExprError(expr_info,
 455                                 "%s uses '**' but did not request 'gen':false"
 456                                 % source)
 457         if not value in all_names:
 458             raise QAPIExprError(expr_info,
 459                                 "%s uses unknown type '%s'"
 460                                 % (source, value))
 461         if not all_names[value] in allow_metas:
 462             raise QAPIExprError(expr_info,
 463                                 "%s cannot use %s type '%s'"
 464                                 % (source, all_names[value], value))
 465         return
 466
 467     if not allow_dict:
 468         raise QAPIExprError(expr_info,
 469                             "%s should be a type name" % source)
 470
 471     if not isinstance(value, OrderedDict):
 472         raise QAPIExprError(expr_info,
 473                             "%s should be a dictionary or type name" % source)
 474
 475     # value is a dictionary, check that each member is okay
 476     for (key, arg) in value.items():
 477         check_name(expr_info, "Member of %s" % source, key,
 478                    allow_optional=allow_optional)
 479         # Todo: allow dictionaries to represent default values of
 480         # an optional argument.
 481         check_type(expr_info, "Member '%s' of %s" % (key, source), arg,
 482                    allow_array=True, allow_star=allow_star,
 483                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 484                                 'enum'])
 485
 486 def check_member_clash(expr_info, base_name, data, source = ""):
 487     base = find_struct(base_name)
 488     assert base
 489     base_members = base['data']
 490     for key in data.keys():
 491         if key.startswith('*'):
 492             key = key[1:]
 493         if key in base_members or "*" + key in base_members:
 494             raise QAPIExprError(expr_info,
 495                                 "Member name '%s'%s clashes with base '%s'"
 496                                 % (key, source, base_name))
 497     if base.get('base'):
 498         check_member_clash(expr_info, base['base'], data, source)
 499
 500 def check_command(expr, expr_info):
 501     name = expr['command']
 502     allow_star = expr.has_key('gen')
 503
 504     check_type(expr_info, "'data' for command '%s'" % name,
 505                expr.get('data'), allow_dict=True, allow_optional=True,
 506                allow_metas=['struct'], allow_star=allow_star)
 507     returns_meta = ['union', 'struct']
 508     if name in returns_whitelist:
 509         returns_meta += ['built-in', 'alternate', 'enum']
 510     check_type(expr_info, "'returns' for command '%s'" % name,
 511                expr.get('returns'), allow_array=True,
 512                allow_optional=True, allow_metas=returns_meta,
 513                allow_star=allow_star)
 514
 515 def check_event(expr, expr_info):
 516     global events
 517     name = expr['event']
 518
 519     if name.upper() == 'MAX':
 520         raise QAPIExprError(expr_info, "Event name 'MAX' cannot be created")
 521     events.append(name)
 522     check_type(expr_info, "'data' for event '%s'" % name,
 523                expr.get('data'), allow_dict=True, allow_optional=True,
 524                allow_metas=['struct'])
 525
 526 def check_union(expr, expr_info):
 527     name = expr['union']
 528     base = expr.get('base')
 529     discriminator = expr.get('discriminator')
 530     members = expr['data']
 531     values = { 'MAX': '(automatic)' }
 532
 533     # Two types of unions, determined by discriminator.
 534
 535     # With no discriminator it is a simple union.
 536     if discriminator is None:
 537         enum_define = None
 538         allow_metas=['built-in', 'union', 'alternate', 'struct', 'enum']
 539         if base is not None:
 540             raise QAPIExprError(expr_info,
 541                                 "Simple union '%s' must not have a base"
 542                                 % name)
 543
 544     # Else, it's a flat union.
 545     else:
 546         # The object must have a string member 'base'.
 547         if not isinstance(base, str):
 548             raise QAPIExprError(expr_info,
 549                                 "Flat union '%s' must have a string base field"
 550                                 % name)
 551         base_fields = find_base_fields(base)
 552         if not base_fields:
 553             raise QAPIExprError(expr_info,
 554                                 "Base '%s' is not a valid struct"
 555                                 % base)
 556
 557         # The value of member 'discriminator' must name a non-optional
 558         # member of the base struct.
 559         check_name(expr_info, "Discriminator of flat union '%s'" % name,
 560                    discriminator)
 561         discriminator_type = base_fields.get(discriminator)
 562         if not discriminator_type:
 563             raise QAPIExprError(expr_info,
 564                                 "Discriminator '%s' is not a member of base "
 565                                 "struct '%s'"
 566                                 % (discriminator, base))
 567         enum_define = find_enum(discriminator_type)
 568         allow_metas=['struct']
 569         # Do not allow string discriminator
 570         if not enum_define:
 571             raise QAPIExprError(expr_info,
 572                                 "Discriminator '%s' must be of enumeration "
 573                                 "type" % discriminator)
 574
 575     # Check every branch
 576     for (key, value) in members.items():
 577         check_name(expr_info, "Member of union '%s'" % name, key)
 578
 579         # Each value must name a known type; furthermore, in flat unions,
 580         # branches must be a struct with no overlapping member names
 581         check_type(expr_info, "Member '%s' of union '%s'" % (key, name),
 582                    value, allow_array=not base, allow_metas=allow_metas)
 583         if base:
 584             branch_struct = find_struct(value)
 585             assert branch_struct
 586             check_member_clash(expr_info, base, branch_struct['data'],
 587                                " of branch '%s'" % key)
 588
 589         # If the discriminator names an enum type, then all members
 590         # of 'data' must also be members of the enum type.
 591         if enum_define:
 592             if not key in enum_define['enum_values']:
 593                 raise QAPIExprError(expr_info,
 594                                     "Discriminator value '%s' is not found in "
 595                                     "enum '%s'" %
 596                                     (key, enum_define["enum_name"]))
 597
 598         # Otherwise, check for conflicts in the generated enum
 599         else:
 600             c_key = camel_to_upper(key)
 601             if c_key in values:
 602                 raise QAPIExprError(expr_info,
 603                                     "Union '%s' member '%s' clashes with '%s'"
 604                                     % (name, key, values[c_key]))
 605             values[c_key] = key
 606
 607 def check_alternate(expr, expr_info):
 608     name = expr['alternate']
 609     members = expr['data']
 610     values = { 'MAX': '(automatic)' }
 611     types_seen = {}
 612
 613     # Check every branch
 614     for (key, value) in members.items():
 615         check_name(expr_info, "Member of alternate '%s'" % name, key)
 616
 617         # Check for conflicts in the generated enum
 618         c_key = camel_to_upper(key)
 619         if c_key in values:
 620             raise QAPIExprError(expr_info,
 621                                 "Alternate '%s' member '%s' clashes with '%s'"
 622                                 % (name, key, values[c_key]))
 623         values[c_key] = key
 624
 625         # Ensure alternates have no type conflicts.
 626         check_type(expr_info, "Member '%s' of alternate '%s'" % (key, name),
 627                    value,
 628                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 629         qtype = find_alternate_member_qtype(value)
 630         assert qtype
 631         if qtype in types_seen:
 632             raise QAPIExprError(expr_info,
 633                                 "Alternate '%s' member '%s' can't "
 634                                 "be distinguished from member '%s'"
 635                                 % (name, key, types_seen[qtype]))
 636         types_seen[qtype] = key
 637
 638 def check_enum(expr, expr_info):
 639     name = expr['enum']
 640     members = expr.get('data')
 641     prefix = expr.get('prefix')
 642     values = { 'MAX': '(automatic)' }
 643
 644     if not isinstance(members, list):
 645         raise QAPIExprError(expr_info,
 646                             "Enum '%s' requires an array for 'data'" % name)
 647     if prefix is not None and not isinstance(prefix, str):
 648         raise QAPIExprError(expr_info,
 649                             "Enum '%s' requires a string for 'prefix'" % name)
 650     for member in members:
 651         check_name(expr_info, "Member of enum '%s'" %name, member,
 652                    enum_member=True)
 653         key = camel_to_upper(member)
 654         if key in values:
 655             raise QAPIExprError(expr_info,
 656                                 "Enum '%s' member '%s' clashes with '%s'"
 657                                 % (name, member, values[key]))
 658         values[key] = member
 659
 660 def check_struct(expr, expr_info):
 661     name = expr['struct']
 662     members = expr['data']
 663
 664     check_type(expr_info, "'data' for struct '%s'" % name, members,
 665                allow_dict=True, allow_optional=True)
 666     check_type(expr_info, "'base' for struct '%s'" % name, expr.get('base'),
 667                allow_metas=['struct'])
 668     if expr.get('base'):
 669         check_member_clash(expr_info, expr['base'], expr['data'])
 670
 671 def check_keys(expr_elem, meta, required, optional=[]):
 672     expr = expr_elem['expr']
 673     info = expr_elem['info']
 674     name = expr[meta]
 675     if not isinstance(name, str):
 676         raise QAPIExprError(info,
 677                             "'%s' key must have a string value" % meta)
 678     required = required + [ meta ]
 679     for (key, value) in expr.items():
 680         if not key in required and not key in optional:
 681             raise QAPIExprError(info,
 682                                 "Unknown key '%s' in %s '%s'"
 683                                 % (key, meta, name))
 684         if (key == 'gen' or key == 'success-response') and value != False:
 685             raise QAPIExprError(info,
 686                                 "'%s' of %s '%s' should only use false value"
 687                                 % (key, meta, name))
 688     for key in required:
 689         if not expr.has_key(key):
 690             raise QAPIExprError(info,
 691                                 "Key '%s' is missing from %s '%s'"
 692                                 % (key, meta, name))
 693
 694 def check_exprs(exprs):
 695     global all_names
 696
 697     # Learn the types and check for valid expression keys
 698     for builtin in builtin_types.keys():
 699         all_names[builtin] = 'built-in'
 700     for expr_elem in exprs:
 701         expr = expr_elem['expr']
 702         info = expr_elem['info']
 703         if expr.has_key('enum'):
 704             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 705             add_enum(expr['enum'], info, expr['data'])
 706         elif expr.has_key('union'):
 707             check_keys(expr_elem, 'union', ['data'],
 708                        ['base', 'discriminator'])
 709             add_union(expr, info)
 710         elif expr.has_key('alternate'):
 711             check_keys(expr_elem, 'alternate', ['data'])
 712             add_name(expr['alternate'], info, 'alternate')
 713         elif expr.has_key('struct'):
 714             check_keys(expr_elem, 'struct', ['data'], ['base'])
 715             add_struct(expr, info)
 716         elif expr.has_key('command'):
 717             check_keys(expr_elem, 'command', [],
 718                        ['data', 'returns', 'gen', 'success-response'])
 719             add_name(expr['command'], info, 'command')
 720         elif expr.has_key('event'):
 721             check_keys(expr_elem, 'event', [], ['data'])
 722             add_name(expr['event'], info, 'event')
 723         else:
 724             raise QAPIExprError(expr_elem['info'],
 725                                 "Expression is missing metatype")
 726
 727     # Try again for hidden UnionKind enum
 728     for expr_elem in exprs:
 729         expr = expr_elem['expr']
 730         if expr.has_key('union'):
 731             if not discriminator_find_enum_define(expr):
 732                 add_enum('%sKind' % expr['union'], expr_elem['info'],
 733                          implicit=True)
 734         elif expr.has_key('alternate'):
 735             add_enum('%sKind' % expr['alternate'], expr_elem['info'],
 736                      implicit=True)
 737
 738     # Validate that exprs make sense
 739     for expr_elem in exprs:
 740         expr = expr_elem['expr']
 741         info = expr_elem['info']
 742
 743         if expr.has_key('enum'):
 744             check_enum(expr, info)
 745         elif expr.has_key('union'):
 746             check_union(expr, info)
 747         elif expr.has_key('alternate'):
 748             check_alternate(expr, info)
 749         elif expr.has_key('struct'):
 750             check_struct(expr, info)
 751         elif expr.has_key('command'):
 752             check_command(expr, info)
 753         elif expr.has_key('event'):
 754             check_event(expr, info)
 755         else:
 756             assert False, 'unexpected meta type'
 757
 758     return exprs
 759
 760
 761 #
 762 # Schema compiler frontend
 763 #
 764
 765 class QAPISchemaEntity(object):
 766     def __init__(self, name, info):
 767         assert isinstance(name, str)
 768         self.name = name
 769         self.info = info
 770
 771     def c_name(self):
 772         return c_name(self.name)
 773
 774     def check(self, schema):
 775         pass
 776
 777     def visit(self, visitor):
 778         pass
 779
 780
 781 class QAPISchemaVisitor(object):
 782     def visit_begin(self, schema):
 783         pass
 784
 785     def visit_end(self):
 786         pass
 787
 788     def visit_builtin_type(self, name, info, json_type):
 789         pass
 790
 791     def visit_enum_type(self, name, info, values, prefix):
 792         pass
 793
 794     def visit_array_type(self, name, info, element_type):
 795         pass
 796
 797     def visit_object_type(self, name, info, base, members, variants):
 798         pass
 799
 800     def visit_alternate_type(self, name, info, variants):
 801         pass
 802
 803     def visit_command(self, name, info, arg_type, ret_type,
 804                       gen, success_response):
 805         pass
 806
 807     def visit_event(self, name, info, arg_type):
 808         pass
 809
 810
 811 class QAPISchemaType(QAPISchemaEntity):
 812     def c_type(self, is_param=False):
 813         return c_name(self.name) + pointer_suffix
 814
 815     def c_null(self):
 816         return 'NULL'
 817
 818     def json_type(self):
 819         pass
 820
 821     def alternate_qtype(self):
 822         json2qtype = {
 823             'string':  'QTYPE_QSTRING',
 824             'number':  'QTYPE_QFLOAT',
 825             'int':     'QTYPE_QINT',
 826             'boolean': 'QTYPE_QBOOL',
 827             'object':  'QTYPE_QDICT'
 828         }
 829         return json2qtype.get(self.json_type())
 830
 831
 832 class QAPISchemaBuiltinType(QAPISchemaType):
 833     def __init__(self, name, json_type, c_type, c_null):
 834         QAPISchemaType.__init__(self, name, None)
 835         assert not c_type or isinstance(c_type, str)
 836         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
 837                              'value')
 838         self._json_type_name = json_type
 839         self._c_type_name = c_type
 840         self._c_null_val = c_null
 841
 842     def c_name(self):
 843         return self.name
 844
 845     def c_type(self, is_param=False):
 846         if is_param and self.name == 'str':
 847             return 'const ' + self._c_type_name
 848         return self._c_type_name
 849
 850     def c_null(self):
 851         return self._c_null_val
 852
 853     def json_type(self):
 854         return self._json_type_name
 855
 856     def visit(self, visitor):
 857         visitor.visit_builtin_type(self.name, self.info, self.json_type())
 858
 859
 860 class QAPISchemaEnumType(QAPISchemaType):
 861     def __init__(self, name, info, values, prefix):
 862         QAPISchemaType.__init__(self, name, info)
 863         for v in values:
 864             assert isinstance(v, str)
 865         assert prefix is None or isinstance(prefix, str)
 866         self.values = values
 867         self.prefix = prefix
 868
 869     def check(self, schema):
 870         assert len(set(self.values)) == len(self.values)
 871
 872     def c_type(self, is_param=False):
 873         return c_name(self.name)
 874
 875     def c_null(self):
 876         return c_enum_const(self.name, (self.values + ['MAX'])[0],
 877                             self.prefix)
 878
 879     def json_type(self):
 880         return 'string'
 881
 882     def visit(self, visitor):
 883         visitor.visit_enum_type(self.name, self.info,
 884                                 self.values, self.prefix)
 885
 886
 887 class QAPISchemaArrayType(QAPISchemaType):
 888     def __init__(self, name, info, element_type):
 889         QAPISchemaType.__init__(self, name, info)
 890         assert isinstance(element_type, str)
 891         self._element_type_name = element_type
 892         self.element_type = None
 893
 894     def check(self, schema):
 895         self.element_type = schema.lookup_type(self._element_type_name)
 896         assert self.element_type
 897
 898     def json_type(self):
 899         return 'array'
 900
 901     def visit(self, visitor):
 902         visitor.visit_array_type(self.name, self.info, self.element_type)
 903
 904
 905 class QAPISchemaObjectType(QAPISchemaType):
 906     def __init__(self, name, info, base, local_members, variants):
 907         QAPISchemaType.__init__(self, name, info)
 908         assert base is None or isinstance(base, str)
 909         for m in local_members:
 910             assert isinstance(m, QAPISchemaObjectTypeMember)
 911         assert (variants is None or
 912                 isinstance(variants, QAPISchemaObjectTypeVariants))
 913         self._base_name = base
 914         self.base = None
 915         self.local_members = local_members
 916         self.variants = variants
 917         self.members = None
 918
 919     def check(self, schema):
 920         assert self.members is not False        # not running in cycles
 921         if self.members:
 922             return
 923         self.members = False                    # mark as being checked
 924         if self._base_name:
 925             self.base = schema.lookup_type(self._base_name)
 926             assert isinstance(self.base, QAPISchemaObjectType)
 927             assert not self.base.variants       # not implemented
 928             self.base.check(schema)
 929             members = list(self.base.members)
 930         else:
 931             members = []
 932         seen = {}
 933         for m in members:
 934             seen[m.name] = m
 935         for m in self.local_members:
 936             m.check(schema, members, seen)
 937         if self.variants:
 938             self.variants.check(schema, members, seen)
 939         self.members = members
 940
 941     def c_name(self):
 942         assert self.info
 943         return QAPISchemaType.c_name(self)
 944
 945     def c_type(self, is_param=False):
 946         assert self.info
 947         return QAPISchemaType.c_type(self)
 948
 949     def json_type(self):
 950         return 'object'
 951
 952     def visit(self, visitor):
 953         visitor.visit_object_type(self.name, self.info,
 954                                   self.base, self.local_members, self.variants)
 955
 956
 957 class QAPISchemaObjectTypeMember(object):
 958     def __init__(self, name, typ, optional):
 959         assert isinstance(name, str)
 960         assert isinstance(typ, str)
 961         assert isinstance(optional, bool)
 962         self.name = name
 963         self._type_name = typ
 964         self.type = None
 965         self.optional = optional
 966
 967     def check(self, schema, all_members, seen):
 968         assert self.name not in seen
 969         self.type = schema.lookup_type(self._type_name)
 970         assert self.type
 971         all_members.append(self)
 972         seen[self.name] = self
 973
 974
 975 class QAPISchemaObjectTypeVariants(object):
 976     def __init__(self, tag_name, tag_enum, variants):
 977         assert tag_name is None or isinstance(tag_name, str)
 978         assert tag_enum is None or isinstance(tag_enum, str)
 979         for v in variants:
 980             assert isinstance(v, QAPISchemaObjectTypeVariant)
 981         self.tag_name = tag_name
 982         if tag_name:
 983             assert not tag_enum
 984             self.tag_member = None
 985         else:
 986             self.tag_member = QAPISchemaObjectTypeMember('type', tag_enum,
 987                                                          False)
 988         self.variants = variants
 989
 990     def check(self, schema, members, seen):
 991         if self.tag_name:
 992             self.tag_member = seen[self.tag_name]
 993         else:
 994             self.tag_member.check(schema, members, seen)
 995         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
 996         for v in self.variants:
 997             vseen = dict(seen)
 998             v.check(schema, self.tag_member.type, vseen)
 999
1000 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1001     def __init__(self, name, typ):
1002         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1003
1004     def check(self, schema, tag_type, seen):
1005         QAPISchemaObjectTypeMember.check(self, schema, [], seen)
1006         assert self.name in tag_type.values
1007
1008     # This function exists to support ugly simple union special cases
1009     # TODO get rid of them, and drop the function
1010     def simple_union_type(self):
1011         if isinstance(self.type, QAPISchemaObjectType) and not self.type.info:
1012             assert len(self.type.members) == 1
1013             assert not self.type.variants
1014             return self.type.members[0].type
1015         return None
1016
1017
1018 class QAPISchemaAlternateType(QAPISchemaType):
1019     def __init__(self, name, info, variants):
1020         QAPISchemaType.__init__(self, name, info)
1021         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1022         assert not variants.tag_name
1023         self.variants = variants
1024
1025     def check(self, schema):
1026         self.variants.check(schema, [], {})
1027
1028     def json_type(self):
1029         return 'value'
1030
1031     def visit(self, visitor):
1032         visitor.visit_alternate_type(self.name, self.info, self.variants)
1033
1034
1035 class QAPISchemaCommand(QAPISchemaEntity):
1036     def __init__(self, name, info, arg_type, ret_type, gen, success_response):
1037         QAPISchemaEntity.__init__(self, name, info)
1038         assert not arg_type or isinstance(arg_type, str)
1039         assert not ret_type or isinstance(ret_type, str)
1040         self._arg_type_name = arg_type
1041         self.arg_type = None
1042         self._ret_type_name = ret_type
1043         self.ret_type = None
1044         self.gen = gen
1045         self.success_response = success_response
1046
1047     def check(self, schema):
1048         if self._arg_type_name:
1049             self.arg_type = schema.lookup_type(self._arg_type_name)
1050             assert isinstance(self.arg_type, QAPISchemaObjectType)
1051             assert not self.arg_type.variants   # not implemented
1052         if self._ret_type_name:
1053             self.ret_type = schema.lookup_type(self._ret_type_name)
1054             assert isinstance(self.ret_type, QAPISchemaType)
1055
1056     def visit(self, visitor):
1057         visitor.visit_command(self.name, self.info,
1058                               self.arg_type, self.ret_type,
1059                               self.gen, self.success_response)
1060
1061
1062 class QAPISchemaEvent(QAPISchemaEntity):
1063     def __init__(self, name, info, arg_type):
1064         QAPISchemaEntity.__init__(self, name, info)
1065         assert not arg_type or isinstance(arg_type, str)
1066         self._arg_type_name = arg_type
1067         self.arg_type = None
1068
1069     def check(self, schema):
1070         if self._arg_type_name:
1071             self.arg_type = schema.lookup_type(self._arg_type_name)
1072             assert isinstance(self.arg_type, QAPISchemaObjectType)
1073             assert not self.arg_type.variants   # not implemented
1074
1075     def visit(self, visitor):
1076         visitor.visit_event(self.name, self.info, self.arg_type)
1077
1078
1079 class QAPISchema(object):
1080     def __init__(self, fname):
1081         try:
1082             self.exprs = check_exprs(QAPISchemaParser(open(fname, "r")).exprs)
1083         except (QAPISchemaError, QAPIExprError), err:
1084             print >>sys.stderr, err
1085             exit(1)
1086         self._entity_dict = {}
1087         self._def_predefineds()
1088         self._def_exprs()
1089         self.check()
1090
1091     def _def_entity(self, ent):
1092         assert ent.name not in self._entity_dict
1093         self._entity_dict[ent.name] = ent
1094
1095     def lookup_entity(self, name, typ=None):
1096         ent = self._entity_dict.get(name)
1097         if typ and not isinstance(ent, typ):
1098             return None
1099         return ent
1100
1101     def lookup_type(self, name):
1102         return self.lookup_entity(name, QAPISchemaType)
1103
1104     def _def_builtin_type(self, name, json_type, c_type, c_null):
1105         self._def_entity(QAPISchemaBuiltinType(name, json_type,
1106                                                c_type, c_null))
1107         self._make_array_type(name)     # TODO really needed?
1108
1109     def _def_predefineds(self):
1110         for t in [('str',    'string',  'char' + pointer_suffix, 'NULL'),
1111                   ('number', 'number',  'double',   '0'),
1112                   ('int',    'int',     'int64_t',  '0'),
1113                   ('int8',   'int',     'int8_t',   '0'),
1114                   ('int16',  'int',     'int16_t',  '0'),
1115                   ('int32',  'int',     'int32_t',  '0'),
1116                   ('int64',  'int',     'int64_t',  '0'),
1117                   ('uint8',  'int',     'uint8_t',  '0'),
1118                   ('uint16', 'int',     'uint16_t', '0'),
1119                   ('uint32', 'int',     'uint32_t', '0'),
1120                   ('uint64', 'int',     'uint64_t', '0'),
1121                   ('size',   'int',     'uint64_t', '0'),
1122                   ('bool',   'boolean', 'bool',     'false'),
1123                   ('any',    'value',   'QObject' + pointer_suffix, 'NULL')]:
1124             self._def_builtin_type(*t)
1125         self._entity_dict['**'] = self.lookup_type('any') # TODO drop this alias
1126
1127     def _make_implicit_enum_type(self, name, values):
1128         name = name + 'Kind'
1129         self._def_entity(QAPISchemaEnumType(name, None, values, None))
1130         return name
1131
1132     def _make_array_type(self, element_type):
1133         name = element_type + 'List'
1134         if not self.lookup_type(name):
1135             self._def_entity(QAPISchemaArrayType(name, None, element_type))
1136         return name
1137
1138     def _make_implicit_object_type(self, name, role, members):
1139         if not members:
1140             return None
1141         name = ':obj-%s-%s' % (name, role)
1142         if not self.lookup_entity(name, QAPISchemaObjectType):
1143             self._def_entity(QAPISchemaObjectType(name, None, None,
1144                                                   members, None))
1145         return name
1146
1147     def _def_enum_type(self, expr, info):
1148         name = expr['enum']
1149         data = expr['data']
1150         prefix = expr.get('prefix')
1151         self._def_entity(QAPISchemaEnumType(name, info, data, prefix))
1152         self._make_array_type(name)     # TODO really needed?
1153
1154     def _make_member(self, name, typ):
1155         optional = False
1156         if name.startswith('*'):
1157             name = name[1:]
1158             optional = True
1159         if isinstance(typ, list):
1160             assert len(typ) == 1
1161             typ = self._make_array_type(typ[0])
1162         return QAPISchemaObjectTypeMember(name, typ, optional)
1163
1164     def _make_members(self, data):
1165         return [self._make_member(key, value)
1166                 for (key, value) in data.iteritems()]
1167
1168     def _def_struct_type(self, expr, info):
1169         name = expr['struct']
1170         base = expr.get('base')
1171         data = expr['data']
1172         self._def_entity(QAPISchemaObjectType(name, info, base,
1173                                               self._make_members(data),
1174                                               None))
1175         self._make_array_type(name)     # TODO really needed?
1176
1177     def _make_variant(self, case, typ):
1178         return QAPISchemaObjectTypeVariant(case, typ)
1179
1180     def _make_simple_variant(self, case, typ):
1181         if isinstance(typ, list):
1182             assert len(typ) == 1
1183             typ = self._make_array_type(typ[0])
1184         typ = self._make_implicit_object_type(typ, 'wrapper',
1185                                               [self._make_member('data', typ)])
1186         return QAPISchemaObjectTypeVariant(case, typ)
1187
1188     def _make_tag_enum(self, type_name, variants):
1189         return self._make_implicit_enum_type(type_name,
1190                                              [v.name for v in variants])
1191
1192     def _def_union_type(self, expr, info):
1193         name = expr['union']
1194         data = expr['data']
1195         base = expr.get('base')
1196         tag_name = expr.get('discriminator')
1197         tag_enum = None
1198         if tag_name:
1199             variants = [self._make_variant(key, value)
1200                         for (key, value) in data.iteritems()]
1201         else:
1202             variants = [self._make_simple_variant(key, value)
1203                         for (key, value) in data.iteritems()]
1204             tag_enum = self._make_tag_enum(name, variants)
1205         self._def_entity(
1206             QAPISchemaObjectType(name, info, base,
1207                                  self._make_members(OrderedDict()),
1208                                  QAPISchemaObjectTypeVariants(tag_name,
1209                                                               tag_enum,
1210                                                               variants)))
1211         self._make_array_type(name)     # TODO really needed?
1212
1213     def _def_alternate_type(self, expr, info):
1214         name = expr['alternate']
1215         data = expr['data']
1216         variants = [self._make_variant(key, value)
1217                     for (key, value) in data.iteritems()]
1218         tag_enum = self._make_tag_enum(name, variants)
1219         self._def_entity(
1220             QAPISchemaAlternateType(name, info,
1221                                     QAPISchemaObjectTypeVariants(None,
1222                                                                  tag_enum,
1223                                                                  variants)))
1224         self._make_array_type(name)     # TODO really needed?
1225
1226     def _def_command(self, expr, info):
1227         name = expr['command']
1228         data = expr.get('data')
1229         rets = expr.get('returns')
1230         gen = expr.get('gen', True)
1231         success_response = expr.get('success-response', True)
1232         if isinstance(data, OrderedDict):
1233             data = self._make_implicit_object_type(name, 'arg',
1234                                                    self._make_members(data))
1235         if isinstance(rets, list):
1236             assert len(rets) == 1
1237             rets = self._make_array_type(rets[0])
1238         self._def_entity(QAPISchemaCommand(name, info, data, rets, gen,
1239                                            success_response))
1240
1241     def _def_event(self, expr, info):
1242         name = expr['event']
1243         data = expr.get('data')
1244         if isinstance(data, OrderedDict):
1245             data = self._make_implicit_object_type(name, 'arg',
1246                                                    self._make_members(data))
1247         self._def_entity(QAPISchemaEvent(name, info, data))
1248
1249     def _def_exprs(self):
1250         for expr_elem in self.exprs:
1251             expr = expr_elem['expr']
1252             info = expr_elem['info']
1253             if 'enum' in expr:
1254                 self._def_enum_type(expr, info)
1255             elif 'struct' in expr:
1256                 self._def_struct_type(expr, info)
1257             elif 'union' in expr:
1258                 self._def_union_type(expr, info)
1259             elif 'alternate' in expr:
1260                 self._def_alternate_type(expr, info)
1261             elif 'command' in expr:
1262                 self._def_command(expr, info)
1263             elif 'event' in expr:
1264                 self._def_event(expr, info)
1265             else:
1266                 assert False
1267
1268     def check(self):
1269         for ent in self._entity_dict.values():
1270             ent.check(self)
1271
1272     def visit(self, visitor):
1273         visitor.visit_begin(self)
1274         for name in sorted(self._entity_dict.keys()):
1275             if self._entity_dict[name].name != name:
1276                 continue        # ignore alias TODO drop alias and remove
1277             self._entity_dict[name].visit(visitor)
1278         visitor.visit_end()
1279
1280
1281 #
1282 # Code generation helpers
1283 #
1284
1285 def camel_case(name):
1286     new_name = ''
1287     first = True
1288     for ch in name:
1289         if ch in ['_', '-']:
1290             first = True
1291         elif first:
1292             new_name += ch.upper()
1293             first = False
1294         else:
1295             new_name += ch.lower()
1296     return new_name
1297
1298 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1299 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1300 # ENUM24_Name -> ENUM24_NAME
1301 def camel_to_upper(value):
1302     c_fun_str = c_name(value, False)
1303     if value.isupper():
1304         return c_fun_str
1305
1306     new_name = ''
1307     l = len(c_fun_str)
1308     for i in range(l):
1309         c = c_fun_str[i]
1310         # When c is upper and no "_" appears before, do more checks
1311         if c.isupper() and (i > 0) and c_fun_str[i - 1] != "_":
1312             # Case 1: next string is lower
1313             # Case 2: previous string is digit
1314             if (i < (l - 1) and c_fun_str[i + 1].islower()) or \
1315             c_fun_str[i - 1].isdigit():
1316                 new_name += '_'
1317         new_name += c
1318     return new_name.lstrip('_').upper()
1319
1320 def c_enum_const(type_name, const_name, prefix=None):
1321     if prefix is not None:
1322         type_name = prefix
1323     return camel_to_upper(type_name + '_' + const_name)
1324
1325 c_name_trans = string.maketrans('.-', '__')
1326
1327 # Map @name to a valid C identifier.
1328 # If @protect, avoid returning certain ticklish identifiers (like
1329 # C keywords) by prepending "q_".
1330 #
1331 # Used for converting 'name' from a 'name':'type' qapi definition
1332 # into a generated struct member, as well as converting type names
1333 # into substrings of a generated C function name.
1334 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1335 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1336 def c_name(name, protect=True):
1337     # ANSI X3J11/88-090, 3.1.1
1338     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1339                      'default', 'do', 'double', 'else', 'enum', 'extern', 'float',
1340                      'for', 'goto', 'if', 'int', 'long', 'register', 'return',
1341                      'short', 'signed', 'sizeof', 'static', 'struct', 'switch',
1342                      'typedef', 'union', 'unsigned', 'void', 'volatile', 'while'])
1343     # ISO/IEC 9899:1999, 6.4.1
1344     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1345     # ISO/IEC 9899:2011, 6.4.1
1346     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic', '_Noreturn',
1347                      '_Static_assert', '_Thread_local'])
1348     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1349     # excluding _.*
1350     gcc_words = set(['asm', 'typeof'])
1351     # C++ ISO/IEC 14882:2003 2.11
1352     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1353                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1354                      'namespace', 'new', 'operator', 'private', 'protected',
1355                      'public', 'reinterpret_cast', 'static_cast', 'template',
1356                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1357                      'using', 'virtual', 'wchar_t',
1358                      # alternative representations
1359                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1360                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1361     # namespace pollution:
1362     polluted_words = set(['unix', 'errno'])
1363     if protect and (name in c89_words | c99_words | c11_words | gcc_words | cpp_words | polluted_words):
1364         return "q_" + name
1365     return name.translate(c_name_trans)
1366
1367 eatspace = '\033EATSPACE.'
1368 pointer_suffix = ' *' + eatspace
1369
1370 def genindent(count):
1371     ret = ""
1372     for i in range(count):
1373         ret += " "
1374     return ret
1375
1376 indent_level = 0
1377
1378 def push_indent(indent_amount=4):
1379     global indent_level
1380     indent_level += indent_amount
1381
1382 def pop_indent(indent_amount=4):
1383     global indent_level
1384     indent_level -= indent_amount
1385
1386 # Generate @code with @kwds interpolated.
1387 # Obey indent_level, and strip eatspace.
1388 def cgen(code, **kwds):
1389     raw = code % kwds
1390     if indent_level:
1391         indent = genindent(indent_level)
1392         # re.subn() lacks flags support before Python 2.7, use re.compile()
1393         raw = re.subn(re.compile("^.", re.MULTILINE),
1394                       indent + r'\g<0>', raw)
1395         raw = raw[0]
1396     return re.sub(re.escape(eatspace) + ' *', '', raw)
1397
1398 def mcgen(code, **kwds):
1399     if code[0] == '\n':
1400         code = code[1:]
1401     return cgen(code, **kwds)
1402
1403
1404 def guardname(filename):
1405     return c_name(filename, protect=False).upper()
1406
1407 def guardstart(name):
1408     return mcgen('''
1409
1410 #ifndef %(name)s
1411 #define %(name)s
1412
1413 ''',
1414                  name=guardname(name))
1415
1416 def guardend(name):
1417     return mcgen('''
1418
1419 #endif /* %(name)s */
1420
1421 ''',
1422                  name=guardname(name))
1423
1424 def gen_enum_lookup(name, values, prefix=None):
1425     ret = mcgen('''
1426
1427 const char *const %(c_name)s_lookup[] = {
1428 ''',
1429                 c_name=c_name(name))
1430     for value in values:
1431         index = c_enum_const(name, value, prefix)
1432         ret += mcgen('''
1433     [%(index)s] = "%(value)s",
1434 ''',
1435                      index=index, value=value)
1436
1437     max_index = c_enum_const(name, 'MAX', prefix)
1438     ret += mcgen('''
1439     [%(max_index)s] = NULL,
1440 };
1441 ''',
1442                  max_index=max_index)
1443     return ret
1444
1445 def gen_enum(name, values, prefix=None):
1446     # append automatically generated _MAX value
1447     enum_values = values + ['MAX']
1448
1449     ret = mcgen('''
1450
1451 typedef enum %(c_name)s {
1452 ''',
1453                 c_name=c_name(name))
1454
1455     i = 0
1456     for value in enum_values:
1457         ret += mcgen('''
1458     %(c_enum)s = %(i)d,
1459 ''',
1460                      c_enum=c_enum_const(name, value, prefix),
1461                      i=i)
1462         i += 1
1463
1464     ret += mcgen('''
1465 } %(c_name)s;
1466 ''',
1467                  c_name=c_name(name))
1468
1469     ret += mcgen('''
1470
1471 extern const char *const %(c_name)s_lookup[];
1472 ''',
1473                  c_name=c_name(name))
1474     return ret
1475
1476 def gen_params(arg_type, extra):
1477     if not arg_type:
1478         return extra
1479     assert not arg_type.variants
1480     ret = ''
1481     sep = ''
1482     for memb in arg_type.members:
1483         ret += sep
1484         sep = ', '
1485         if memb.optional:
1486             ret += 'bool has_%s, ' % c_name(memb.name)
1487         ret += '%s %s' % (memb.type.c_type(is_param=True), c_name(memb.name))
1488     if extra:
1489         ret += sep + extra
1490     return ret
1491
1492 #
1493 # Common command line parsing
1494 #
1495
1496 def parse_command_line(extra_options = "", extra_long_options = []):
1497
1498     try:
1499         opts, args = getopt.gnu_getopt(sys.argv[1:],
1500                                        "chp:o:" + extra_options,
1501                                        ["source", "header", "prefix=",
1502                                         "output-dir="] + extra_long_options)
1503     except getopt.GetoptError, err:
1504         print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
1505         sys.exit(1)
1506
1507     output_dir = ""
1508     prefix = ""
1509     do_c = False
1510     do_h = False
1511     extra_opts = []
1512
1513     for oa in opts:
1514         o, a = oa
1515         if o in ("-p", "--prefix"):
1516             match = re.match('([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1517             if match.end() != len(a):
1518                 print >>sys.stderr, \
1519                     "%s: 'funny character '%s' in argument of --prefix" \
1520                     % (sys.argv[0], a[match.end()])
1521                 sys.exit(1)
1522             prefix = a
1523         elif o in ("-o", "--output-dir"):
1524             output_dir = a + "/"
1525         elif o in ("-c", "--source"):
1526             do_c = True
1527         elif o in ("-h", "--header"):
1528             do_h = True
1529         else:
1530             extra_opts.append(oa)
1531
1532     if not do_c and not do_h:
1533         do_c = True
1534         do_h = True
1535
1536     if len(args) != 1:
1537         print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
1538         sys.exit(1)
1539     fname = args[0]
1540
1541     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
1542
1543 #
1544 # Generate output files with boilerplate
1545 #
1546
1547 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
1548                 c_comment, h_comment):
1549     guard = guardname(prefix + h_file)
1550     c_file = output_dir + prefix + c_file
1551     h_file = output_dir + prefix + h_file
1552
1553     if output_dir:
1554         try:
1555             os.makedirs(output_dir)
1556         except os.error, e:
1557             if e.errno != errno.EEXIST:
1558                 raise
1559
1560     def maybe_open(really, name, opt):
1561         if really:
1562             return open(name, opt)
1563         else:
1564             import StringIO
1565             return StringIO.StringIO()
1566
1567     fdef = maybe_open(do_c, c_file, 'w')
1568     fdecl = maybe_open(do_h, h_file, 'w')
1569
1570     fdef.write(mcgen('''
1571 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
1572 %(comment)s
1573 ''',
1574                      comment = c_comment))
1575
1576     fdecl.write(mcgen('''
1577 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
1578 %(comment)s
1579 #ifndef %(guard)s
1580 #define %(guard)s
1581
1582 ''',
1583                       comment = h_comment, guard = guard))
1584
1585     return (fdef, fdecl)
1586
1587 def close_output(fdef, fdecl):
1588     fdecl.write('''
1589 #endif
1590 ''')
1591     fdecl.close()
1592     fdef.close()