scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 import errno
  15 import getopt
  16 import os
  17 import re
  18 import string
  19 import sys
  20 from ordereddict import OrderedDict
  21
  22 builtin_types = {
  23     'null':     'QTYPE_QNULL',
  24     'str':      'QTYPE_QSTRING',
  25     'int':      'QTYPE_QNUM',
  26     'number':   'QTYPE_QNUM',
  27     'bool':     'QTYPE_QBOOL',
  28     'int8':     'QTYPE_QNUM',
  29     'int16':    'QTYPE_QNUM',
  30     'int32':    'QTYPE_QNUM',
  31     'int64':    'QTYPE_QNUM',
  32     'uint8':    'QTYPE_QNUM',
  33     'uint16':   'QTYPE_QNUM',
  34     'uint32':   'QTYPE_QNUM',
  35     'uint64':   'QTYPE_QNUM',
  36     'size':     'QTYPE_QNUM',
  37     'any':      None,           # any QType possible, actually
  38     'QType':    'QTYPE_QSTRING',
  39 }
  40
  41 # Are documentation comments required?
  42 doc_required = False
  43
  44 # Whitelist of commands allowed to return a non-dictionary
  45 returns_whitelist = []
  46
  47 # Whitelist of entities allowed to violate case conventions
  48 name_case_whitelist = []
  49
  50 enum_types = {}
  51 struct_types = {}
  52 union_types = {}
  53 all_names = {}
  54
  55 #
  56 # Parsing the schema into expressions
  57 #
  58
  59
  60 def error_path(parent):
  61     res = ''
  62     while parent:
  63         res = ('In file included from %s:%d:\n' % (parent['file'],
  64                                                    parent['line'])) + res
  65         parent = parent['parent']
  66     return res
  67
  68
  69 class QAPIError(Exception):
  70     def __init__(self, fname, line, col, incl_info, msg):
  71         Exception.__init__(self)
  72         self.fname = fname
  73         self.line = line
  74         self.col = col
  75         self.info = incl_info
  76         self.msg = msg
  77
  78     def __str__(self):
  79         loc = '%s:%d' % (self.fname, self.line)
  80         if self.col is not None:
  81             loc += ':%s' % self.col
  82         return error_path(self.info) + '%s: %s' % (loc, self.msg)
  83
  84
  85 class QAPIParseError(QAPIError):
  86     def __init__(self, parser, msg):
  87         col = 1
  88         for ch in parser.src[parser.line_pos:parser.pos]:
  89             if ch == '\t':
  90                 col = (col + 7) % 8 + 1
  91             else:
  92                 col += 1
  93         QAPIError.__init__(self, parser.fname, parser.line, col,
  94                            parser.incl_info, msg)
  95
  96
  97 class QAPISemError(QAPIError):
  98     def __init__(self, info, msg):
  99         QAPIError.__init__(self, info['file'], info['line'], None,
 100                            info['parent'], msg)
 101
 102
 103 class QAPIDoc(object):
 104     class Section(object):
 105         def __init__(self, name=None):
 106             # optional section name (argument/member or section name)
 107             self.name = name
 108             # the list of lines for this section
 109             self.text = ''
 110
 111         def append(self, line):
 112             self.text += line.rstrip() + '\n'
 113
 114     class ArgSection(Section):
 115         def __init__(self, name):
 116             QAPIDoc.Section.__init__(self, name)
 117             self.member = None
 118
 119         def connect(self, member):
 120             self.member = member
 121
 122     def __init__(self, parser, info):
 123         # self._parser is used to report errors with QAPIParseError.  The
 124         # resulting error position depends on the state of the parser.
 125         # It happens to be the beginning of the comment.  More or less
 126         # servicable, but action at a distance.
 127         self._parser = parser
 128         self.info = info
 129         self.symbol = None
 130         self.body = QAPIDoc.Section()
 131         # dict mapping parameter name to ArgSection
 132         self.args = OrderedDict()
 133         # a list of Section
 134         self.sections = []
 135         # the current section
 136         self._section = self.body
 137
 138     def has_section(self, name):
 139         """Return True if we have a section with this name."""
 140         for i in self.sections:
 141             if i.name == name:
 142                 return True
 143         return False
 144
 145     def append(self, line):
 146         """Parse a comment line and add it to the documentation."""
 147         line = line[1:]
 148         if not line:
 149             self._append_freeform(line)
 150             return
 151
 152         if line[0] != ' ':
 153             raise QAPIParseError(self._parser, "Missing space after #")
 154         line = line[1:]
 155
 156         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 157         # recognized, and get silently treated as ordinary text
 158         if self.symbol:
 159             self._append_symbol_line(line)
 160         elif not self.body.text and line.startswith('@'):
 161             if not line.endswith(':'):
 162                 raise QAPIParseError(self._parser, "Line should end with :")
 163             self.symbol = line[1:-1]
 164             # FIXME invalid names other than the empty string aren't flagged
 165             if not self.symbol:
 166                 raise QAPIParseError(self._parser, "Invalid name")
 167         else:
 168             self._append_freeform(line)
 169
 170     def end_comment(self):
 171         self._end_section()
 172
 173     def _append_symbol_line(self, line):
 174         name = line.split(' ', 1)[0]
 175
 176         if name.startswith('@') and name.endswith(':'):
 177             line = line[len(name)+1:]
 178             self._start_args_section(name[1:-1])
 179         elif name in ('Returns:', 'Since:',
 180                       # those are often singular or plural
 181                       'Note:', 'Notes:',
 182                       'Example:', 'Examples:',
 183                       'TODO:'):
 184             line = line[len(name)+1:]
 185             self._start_section(name[:-1])
 186
 187         self._append_freeform(line)
 188
 189     def _start_args_section(self, name):
 190         # FIXME invalid names other than the empty string aren't flagged
 191         if not name:
 192             raise QAPIParseError(self._parser, "Invalid parameter name")
 193         if name in self.args:
 194             raise QAPIParseError(self._parser,
 195                                  "'%s' parameter name duplicated" % name)
 196         if self.sections:
 197             raise QAPIParseError(self._parser,
 198                                  "'@%s:' can't follow '%s' section"
 199                                  % (name, self.sections[0].name))
 200         self._end_section()
 201         self._section = QAPIDoc.ArgSection(name)
 202         self.args[name] = self._section
 203
 204     def _start_section(self, name=None):
 205         if name in ('Returns', 'Since') and self.has_section(name):
 206             raise QAPIParseError(self._parser,
 207                                  "Duplicated '%s' section" % name)
 208         self._end_section()
 209         self._section = QAPIDoc.Section(name)
 210         self.sections.append(self._section)
 211
 212     def _end_section(self):
 213         if self._section:
 214             text = self._section.text = self._section.text.strip()
 215             if self._section.name and (not text or text.isspace()):
 216                 raise QAPIParseError(self._parser, "Empty doc section '%s'"
 217                                      % self._section.name)
 218             self._section = None
 219
 220     def _append_freeform(self, line):
 221         in_arg = isinstance(self._section, QAPIDoc.ArgSection)
 222         if (in_arg and self._section.text.endswith('\n\n')
 223                 and line and not line[0].isspace()):
 224             self._start_section()
 225         if (in_arg or not self._section.name
 226                 or not self._section.name.startswith('Example')):
 227             line = line.strip()
 228         match = re.match(r'(@\S+:)', line)
 229         if match:
 230             raise QAPIParseError(self._parser,
 231                                  "'%s' not allowed in free-form documentation"
 232                                  % match.group(1))
 233         self._section.append(line)
 234
 235     def connect_member(self, member):
 236         if member.name not in self.args:
 237             # Undocumented TODO outlaw
 238             self.args[member.name] = QAPIDoc.ArgSection(member.name)
 239         self.args[member.name].connect(member)
 240
 241     def check_expr(self, expr):
 242         if self.has_section('Returns') and 'command' not in expr:
 243             raise QAPISemError(self.info,
 244                                "'Returns:' is only valid for commands")
 245
 246     def check(self):
 247         bogus = [name for name, section in self.args.iteritems()
 248                  if not section.member]
 249         if bogus:
 250             raise QAPISemError(
 251                 self.info,
 252                 "The following documented members are not in "
 253                 "the declaration: %s" % ", ".join(bogus))
 254
 255
 256 class QAPISchemaParser(object):
 257
 258     def __init__(self, fp, previously_included=[], incl_info=None):
 259         abs_fname = os.path.abspath(fp.name)
 260         self.fname = fp.name
 261         previously_included.append(abs_fname)
 262         self.incl_info = incl_info
 263         self.src = fp.read()
 264         if self.src == '' or self.src[-1] != '\n':
 265             self.src += '\n'
 266         self.cursor = 0
 267         self.line = 1
 268         self.line_pos = 0
 269         self.exprs = []
 270         self.docs = []
 271         self.accept()
 272         cur_doc = None
 273
 274         while self.tok is not None:
 275             info = {'file': self.fname, 'line': self.line,
 276                     'parent': self.incl_info}
 277             if self.tok == '#':
 278                 self.reject_expr_doc(cur_doc)
 279                 cur_doc = self.get_doc(info)
 280                 self.docs.append(cur_doc)
 281                 continue
 282
 283             expr = self.get_expr(False)
 284             if 'include' in expr:
 285                 self.reject_expr_doc(cur_doc)
 286                 if len(expr) != 1:
 287                     raise QAPISemError(info, "Invalid 'include' directive")
 288                 include = expr['include']
 289                 if not isinstance(include, str):
 290                     raise QAPISemError(info,
 291                                        "Value of 'include' must be a string")
 292                 self._include(include, info, os.path.dirname(abs_fname),
 293                               previously_included)
 294             elif "pragma" in expr:
 295                 self.reject_expr_doc(cur_doc)
 296                 if len(expr) != 1:
 297                     raise QAPISemError(info, "Invalid 'pragma' directive")
 298                 pragma = expr['pragma']
 299                 if not isinstance(pragma, dict):
 300                     raise QAPISemError(
 301                         info, "Value of 'pragma' must be a dictionary")
 302                 for name, value in pragma.iteritems():
 303                     self._pragma(name, value, info)
 304             else:
 305                 expr_elem = {'expr': expr,
 306                              'info': info}
 307                 if cur_doc:
 308                     if not cur_doc.symbol:
 309                         raise QAPISemError(
 310                             cur_doc.info, "Expression documentation required")
 311                     expr_elem['doc'] = cur_doc
 312                 self.exprs.append(expr_elem)
 313             cur_doc = None
 314         self.reject_expr_doc(cur_doc)
 315
 316     @staticmethod
 317     def reject_expr_doc(doc):
 318         if doc and doc.symbol:
 319             raise QAPISemError(
 320                 doc.info,
 321                 "Documentation for '%s' is not followed by the definition"
 322                 % doc.symbol)
 323
 324     def _include(self, include, info, base_dir, previously_included):
 325         incl_abs_fname = os.path.join(base_dir, include)
 326         # catch inclusion cycle
 327         inf = info
 328         while inf:
 329             if incl_abs_fname == os.path.abspath(inf['file']):
 330                 raise QAPISemError(info, "Inclusion loop for %s" % include)
 331             inf = inf['parent']
 332
 333         # skip multiple include of the same file
 334         if incl_abs_fname in previously_included:
 335             return
 336         try:
 337             fobj = open(incl_abs_fname, 'r')
 338         except IOError as e:
 339             raise QAPISemError(info, '%s: %s' % (e.strerror, include))
 340         exprs_include = QAPISchemaParser(fobj, previously_included, info)
 341         self.exprs.extend(exprs_include.exprs)
 342         self.docs.extend(exprs_include.docs)
 343
 344     def _pragma(self, name, value, info):
 345         global doc_required, returns_whitelist, name_case_whitelist
 346         if name == 'doc-required':
 347             if not isinstance(value, bool):
 348                 raise QAPISemError(info,
 349                                    "Pragma 'doc-required' must be boolean")
 350             doc_required = value
 351         elif name == 'returns-whitelist':
 352             if (not isinstance(value, list)
 353                     or any([not isinstance(elt, str) for elt in value])):
 354                 raise QAPISemError(info,
 355                                    "Pragma returns-whitelist must be"
 356                                    " a list of strings")
 357             returns_whitelist = value
 358         elif name == 'name-case-whitelist':
 359             if (not isinstance(value, list)
 360                     or any([not isinstance(elt, str) for elt in value])):
 361                 raise QAPISemError(info,
 362                                    "Pragma name-case-whitelist must be"
 363                                    " a list of strings")
 364             name_case_whitelist = value
 365         else:
 366             raise QAPISemError(info, "Unknown pragma '%s'" % name)
 367
 368     def accept(self, skip_comment=True):
 369         while True:
 370             self.tok = self.src[self.cursor]
 371             self.pos = self.cursor
 372             self.cursor += 1
 373             self.val = None
 374
 375             if self.tok == '#':
 376                 if self.src[self.cursor] == '#':
 377                     # Start of doc comment
 378                     skip_comment = False
 379                 self.cursor = self.src.find('\n', self.cursor)
 380                 if not skip_comment:
 381                     self.val = self.src[self.pos:self.cursor]
 382                     return
 383             elif self.tok in '{}:,[]':
 384                 return
 385             elif self.tok == "'":
 386                 string = ''
 387                 esc = False
 388                 while True:
 389                     ch = self.src[self.cursor]
 390                     self.cursor += 1
 391                     if ch == '\n':
 392                         raise QAPIParseError(self, 'Missing terminating "\'"')
 393                     if esc:
 394                         if ch == 'b':
 395                             string += '\b'
 396                         elif ch == 'f':
 397                             string += '\f'
 398                         elif ch == 'n':
 399                             string += '\n'
 400                         elif ch == 'r':
 401                             string += '\r'
 402                         elif ch == 't':
 403                             string += '\t'
 404                         elif ch == 'u':
 405                             value = 0
 406                             for _ in range(0, 4):
 407                                 ch = self.src[self.cursor]
 408                                 self.cursor += 1
 409                                 if ch not in '0123456789abcdefABCDEF':
 410                                     raise QAPIParseError(self,
 411                                                          '\\u escape needs 4 '
 412                                                          'hex digits')
 413                                 value = (value << 4) + int(ch, 16)
 414                             # If Python 2 and 3 didn't disagree so much on
 415                             # how to handle Unicode, then we could allow
 416                             # Unicode string defaults.  But most of QAPI is
 417                             # ASCII-only, so we aren't losing much for now.
 418                             if not value or value > 0x7f:
 419                                 raise QAPIParseError(self,
 420                                                      'For now, \\u escape '
 421                                                      'only supports non-zero '
 422                                                      'values up to \\u007f')
 423                             string += chr(value)
 424                         elif ch in '\\/\'"':
 425                             string += ch
 426                         else:
 427                             raise QAPIParseError(self,
 428                                                  "Unknown escape \\%s" % ch)
 429                         esc = False
 430                     elif ch == '\\':
 431                         esc = True
 432                     elif ch == "'":
 433                         self.val = string
 434                         return
 435                     else:
 436                         string += ch
 437             elif self.src.startswith('true', self.pos):
 438                 self.val = True
 439                 self.cursor += 3
 440                 return
 441             elif self.src.startswith('false', self.pos):
 442                 self.val = False
 443                 self.cursor += 4
 444                 return
 445             elif self.src.startswith('null', self.pos):
 446                 self.val = None
 447                 self.cursor += 3
 448                 return
 449             elif self.tok == '\n':
 450                 if self.cursor == len(self.src):
 451                     self.tok = None
 452                     return
 453                 self.line += 1
 454                 self.line_pos = self.cursor
 455             elif not self.tok.isspace():
 456                 raise QAPIParseError(self, 'Stray "%s"' % self.tok)
 457
 458     def get_members(self):
 459         expr = OrderedDict()
 460         if self.tok == '}':
 461             self.accept()
 462             return expr
 463         if self.tok != "'":
 464             raise QAPIParseError(self, 'Expected string or "}"')
 465         while True:
 466             key = self.val
 467             self.accept()
 468             if self.tok != ':':
 469                 raise QAPIParseError(self, 'Expected ":"')
 470             self.accept()
 471             if key in expr:
 472                 raise QAPIParseError(self, 'Duplicate key "%s"' % key)
 473             expr[key] = self.get_expr(True)
 474             if self.tok == '}':
 475                 self.accept()
 476                 return expr
 477             if self.tok != ',':
 478                 raise QAPIParseError(self, 'Expected "," or "}"')
 479             self.accept()
 480             if self.tok != "'":
 481                 raise QAPIParseError(self, 'Expected string')
 482
 483     def get_values(self):
 484         expr = []
 485         if self.tok == ']':
 486             self.accept()
 487             return expr
 488         if self.tok not in "{['tfn":
 489             raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
 490                                  'boolean or "null"')
 491         while True:
 492             expr.append(self.get_expr(True))
 493             if self.tok == ']':
 494                 self.accept()
 495                 return expr
 496             if self.tok != ',':
 497                 raise QAPIParseError(self, 'Expected "," or "]"')
 498             self.accept()
 499
 500     def get_expr(self, nested):
 501         if self.tok != '{' and not nested:
 502             raise QAPIParseError(self, 'Expected "{"')
 503         if self.tok == '{':
 504             self.accept()
 505             expr = self.get_members()
 506         elif self.tok == '[':
 507             self.accept()
 508             expr = self.get_values()
 509         elif self.tok in "'tfn":
 510             expr = self.val
 511             self.accept()
 512         else:
 513             raise QAPIParseError(self, 'Expected "{", "[", string, '
 514                                  'boolean or "null"')
 515         return expr
 516
 517     def get_doc(self, info):
 518         if self.val != '##':
 519             raise QAPIParseError(self, "Junk after '##' at start of "
 520                                  "documentation comment")
 521
 522         doc = QAPIDoc(self, info)
 523         self.accept(False)
 524         while self.tok == '#':
 525             if self.val.startswith('##'):
 526                 # End of doc comment
 527                 if self.val != '##':
 528                     raise QAPIParseError(self, "Junk after '##' at end of "
 529                                          "documentation comment")
 530                 doc.end_comment()
 531                 self.accept()
 532                 return doc
 533             else:
 534                 doc.append(self.val)
 535             self.accept(False)
 536
 537         raise QAPIParseError(self, "Documentation comment must end with '##'")
 538
 539
 540 #
 541 # Semantic analysis of schema expressions
 542 # TODO fold into QAPISchema
 543 # TODO catching name collisions in generated code would be nice
 544 #
 545
 546
 547 def find_base_members(base):
 548     if isinstance(base, dict):
 549         return base
 550     base_struct_define = struct_types.get(base)
 551     if not base_struct_define:
 552         return None
 553     return base_struct_define['data']
 554
 555
 556 # Return the qtype of an alternate branch, or None on error.
 557 def find_alternate_member_qtype(qapi_type):
 558     if qapi_type in builtin_types:
 559         return builtin_types[qapi_type]
 560     elif qapi_type in struct_types:
 561         return 'QTYPE_QDICT'
 562     elif qapi_type in enum_types:
 563         return 'QTYPE_QSTRING'
 564     elif qapi_type in union_types:
 565         return 'QTYPE_QDICT'
 566     return None
 567
 568
 569 # Return the discriminator enum define if discriminator is specified as an
 570 # enum type, otherwise return None.
 571 def discriminator_find_enum_define(expr):
 572     base = expr.get('base')
 573     discriminator = expr.get('discriminator')
 574
 575     if not (discriminator and base):
 576         return None
 577
 578     base_members = find_base_members(base)
 579     if not base_members:
 580         return None
 581
 582     discriminator_type = base_members.get(discriminator)
 583     if not discriminator_type:
 584         return None
 585
 586     return enum_types.get(discriminator_type)
 587
 588
 589 # Names must be letters, numbers, -, and _.  They must start with letter,
 590 # except for downstream extensions which must start with __RFQDN_.
 591 # Dots are only valid in the downstream extension prefix.
 592 valid_name = re.compile(r'^(__[a-zA-Z0-9.-]+_)?'
 593                         '[a-zA-Z][a-zA-Z0-9_-]*$')
 594
 595
 596 def check_name(info, source, name, allow_optional=False,
 597                enum_member=False):
 598     global valid_name
 599     membername = name
 600
 601     if not isinstance(name, str):
 602         raise QAPISemError(info, "%s requires a string name" % source)
 603     if name.startswith('*'):
 604         membername = name[1:]
 605         if not allow_optional:
 606             raise QAPISemError(info, "%s does not allow optional name '%s'"
 607                                % (source, name))
 608     # Enum members can start with a digit, because the generated C
 609     # code always prefixes it with the enum name
 610     if enum_member and membername[0].isdigit():
 611         membername = 'D' + membername
 612     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
 613     # and 'q_obj_*' implicit type names.
 614     if not valid_name.match(membername) or \
 615        c_name(membername, False).startswith('q_'):
 616         raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
 617
 618
 619 def add_name(name, info, meta, implicit=False):
 620     global all_names
 621     check_name(info, "'%s'" % meta, name)
 622     # FIXME should reject names that differ only in '_' vs. '.'
 623     # vs. '-', because they're liable to clash in generated C.
 624     if name in all_names:
 625         raise QAPISemError(info, "%s '%s' is already defined"
 626                            % (all_names[name], name))
 627     if not implicit and (name.endswith('Kind') or name.endswith('List')):
 628         raise QAPISemError(info, "%s '%s' should not end in '%s'"
 629                            % (meta, name, name[-4:]))
 630     all_names[name] = meta
 631
 632
 633 def check_type(info, source, value, allow_array=False,
 634                allow_dict=False, allow_optional=False,
 635                allow_metas=[]):
 636     global all_names
 637
 638     if value is None:
 639         return
 640
 641     # Check if array type for value is okay
 642     if isinstance(value, list):
 643         if not allow_array:
 644             raise QAPISemError(info, "%s cannot be an array" % source)
 645         if len(value) != 1 or not isinstance(value[0], str):
 646             raise QAPISemError(info,
 647                                "%s: array type must contain single type name" %
 648                                source)
 649         value = value[0]
 650
 651     # Check if type name for value is okay
 652     if isinstance(value, str):
 653         if value not in all_names:
 654             raise QAPISemError(info, "%s uses unknown type '%s'"
 655                                % (source, value))
 656         if not all_names[value] in allow_metas:
 657             raise QAPISemError(info, "%s cannot use %s type '%s'" %
 658                                (source, all_names[value], value))
 659         return
 660
 661     if not allow_dict:
 662         raise QAPISemError(info, "%s should be a type name" % source)
 663
 664     if not isinstance(value, OrderedDict):
 665         raise QAPISemError(info,
 666                            "%s should be a dictionary or type name" % source)
 667
 668     # value is a dictionary, check that each member is okay
 669     for (key, arg) in value.items():
 670         check_name(info, "Member of %s" % source, key,
 671                    allow_optional=allow_optional)
 672         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 673             raise QAPISemError(info, "Member of %s uses reserved name '%s'"
 674                                % (source, key))
 675         # Todo: allow dictionaries to represent default values of
 676         # an optional argument.
 677         check_type(info, "Member '%s' of %s" % (key, source), arg,
 678                    allow_array=True,
 679                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 680                                 'enum'])
 681
 682
 683 def check_command(expr, info):
 684     name = expr['command']
 685     boxed = expr.get('boxed', False)
 686
 687     args_meta = ['struct']
 688     if boxed:
 689         args_meta += ['union', 'alternate']
 690     check_type(info, "'data' for command '%s'" % name,
 691                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 692                allow_metas=args_meta)
 693     returns_meta = ['union', 'struct']
 694     if name in returns_whitelist:
 695         returns_meta += ['built-in', 'alternate', 'enum']
 696     check_type(info, "'returns' for command '%s'" % name,
 697                expr.get('returns'), allow_array=True,
 698                allow_optional=True, allow_metas=returns_meta)
 699
 700
 701 def check_event(expr, info):
 702     name = expr['event']
 703     boxed = expr.get('boxed', False)
 704
 705     meta = ['struct']
 706     if boxed:
 707         meta += ['union', 'alternate']
 708     check_type(info, "'data' for event '%s'" % name,
 709                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 710                allow_metas=meta)
 711
 712
 713 def check_union(expr, info):
 714     name = expr['union']
 715     base = expr.get('base')
 716     discriminator = expr.get('discriminator')
 717     members = expr['data']
 718
 719     # Two types of unions, determined by discriminator.
 720
 721     # With no discriminator it is a simple union.
 722     if discriminator is None:
 723         enum_define = None
 724         allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
 725         if base is not None:
 726             raise QAPISemError(info, "Simple union '%s' must not have a base" %
 727                                name)
 728
 729     # Else, it's a flat union.
 730     else:
 731         # The object must have a string or dictionary 'base'.
 732         check_type(info, "'base' for union '%s'" % name,
 733                    base, allow_dict=True, allow_optional=True,
 734                    allow_metas=['struct'])
 735         if not base:
 736             raise QAPISemError(info, "Flat union '%s' must have a base"
 737                                % name)
 738         base_members = find_base_members(base)
 739         assert base_members is not None
 740
 741         # The value of member 'discriminator' must name a non-optional
 742         # member of the base struct.
 743         check_name(info, "Discriminator of flat union '%s'" % name,
 744                    discriminator)
 745         discriminator_type = base_members.get(discriminator)
 746         if not discriminator_type:
 747             raise QAPISemError(info,
 748                                "Discriminator '%s' is not a member of base "
 749                                "struct '%s'"
 750                                % (discriminator, base))
 751         enum_define = enum_types.get(discriminator_type)
 752         allow_metas = ['struct']
 753         # Do not allow string discriminator
 754         if not enum_define:
 755             raise QAPISemError(info,
 756                                "Discriminator '%s' must be of enumeration "
 757                                "type" % discriminator)
 758
 759     # Check every branch; don't allow an empty union
 760     if len(members) == 0:
 761         raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
 762     for (key, value) in members.items():
 763         check_name(info, "Member of union '%s'" % name, key)
 764
 765         # Each value must name a known type
 766         check_type(info, "Member '%s' of union '%s'" % (key, name),
 767                    value, allow_array=not base, allow_metas=allow_metas)
 768
 769         # If the discriminator names an enum type, then all members
 770         # of 'data' must also be members of the enum type.
 771         if enum_define:
 772             if key not in enum_define['data']:
 773                 raise QAPISemError(info,
 774                                    "Discriminator value '%s' is not found in "
 775                                    "enum '%s'"
 776                                    % (key, enum_define['enum']))
 777
 778     # If discriminator is user-defined, ensure all values are covered
 779     if enum_define:
 780         for value in enum_define['data']:
 781             if value not in members.keys():
 782                 raise QAPISemError(info, "Union '%s' data missing '%s' branch"
 783                                    % (name, value))
 784
 785
 786 def check_alternate(expr, info):
 787     name = expr['alternate']
 788     members = expr['data']
 789     types_seen = {}
 790
 791     # Check every branch; require at least two branches
 792     if len(members) < 2:
 793         raise QAPISemError(info,
 794                            "Alternate '%s' should have at least two branches "
 795                            "in 'data'" % name)
 796     for (key, value) in members.items():
 797         check_name(info, "Member of alternate '%s'" % name, key)
 798
 799         # Ensure alternates have no type conflicts.
 800         check_type(info, "Member '%s' of alternate '%s'" % (key, name),
 801                    value,
 802                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 803         qtype = find_alternate_member_qtype(value)
 804         if not qtype:
 805             raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
 806                                "type '%s'" % (name, key, value))
 807         conflicting = set([qtype])
 808         if qtype == 'QTYPE_QSTRING':
 809             enum_expr = enum_types.get(value)
 810             if enum_expr:
 811                 for v in enum_expr['data']:
 812                     if v in ['on', 'off']:
 813                         conflicting.add('QTYPE_QBOOL')
 814                     if re.match(r'[-+0-9.]', v): # lazy, could be tightened
 815                         conflicting.add('QTYPE_QNUM')
 816             else:
 817                 conflicting.add('QTYPE_QNUM')
 818                 conflicting.add('QTYPE_QBOOL')
 819         for qt in conflicting:
 820             if qt in types_seen:
 821                 raise QAPISemError(info, "Alternate '%s' member '%s' can't "
 822                                    "be distinguished from member '%s'"
 823                                    % (name, key, types_seen[qt]))
 824             types_seen[qt] = key
 825
 826
 827 def check_enum(expr, info):
 828     name = expr['enum']
 829     members = expr.get('data')
 830     prefix = expr.get('prefix')
 831
 832     if not isinstance(members, list):
 833         raise QAPISemError(info,
 834                            "Enum '%s' requires an array for 'data'" % name)
 835     if prefix is not None and not isinstance(prefix, str):
 836         raise QAPISemError(info,
 837                            "Enum '%s' requires a string for 'prefix'" % name)
 838     for member in members:
 839         check_name(info, "Member of enum '%s'" % name, member,
 840                    enum_member=True)
 841
 842
 843 def check_struct(expr, info):
 844     name = expr['struct']
 845     members = expr['data']
 846
 847     check_type(info, "'data' for struct '%s'" % name, members,
 848                allow_dict=True, allow_optional=True)
 849     check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
 850                allow_metas=['struct'])
 851
 852
 853 def check_keys(expr_elem, meta, required, optional=[]):
 854     expr = expr_elem['expr']
 855     info = expr_elem['info']
 856     name = expr[meta]
 857     if not isinstance(name, str):
 858         raise QAPISemError(info, "'%s' key must have a string value" % meta)
 859     required = required + [meta]
 860     for (key, value) in expr.items():
 861         if key not in required and key not in optional:
 862             raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
 863                                % (key, meta, name))
 864         if (key == 'gen' or key == 'success-response') and value is not False:
 865             raise QAPISemError(info,
 866                                "'%s' of %s '%s' should only use false value"
 867                                % (key, meta, name))
 868         if key == 'boxed' and value is not True:
 869             raise QAPISemError(info,
 870                                "'%s' of %s '%s' should only use true value"
 871                                % (key, meta, name))
 872     for key in required:
 873         if key not in expr:
 874             raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
 875                                % (key, meta, name))
 876
 877
 878 def check_exprs(exprs):
 879     global all_names
 880
 881     # Populate name table with names of built-in types
 882     for builtin in builtin_types.keys():
 883         all_names[builtin] = 'built-in'
 884
 885     # Learn the types and check for valid expression keys
 886     for expr_elem in exprs:
 887         expr = expr_elem['expr']
 888         info = expr_elem['info']
 889         doc = expr_elem.get('doc')
 890
 891         if not doc and doc_required:
 892             raise QAPISemError(info,
 893                                "Expression missing documentation comment")
 894
 895         if 'enum' in expr:
 896             meta = 'enum'
 897             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 898             enum_types[expr[meta]] = expr
 899         elif 'union' in expr:
 900             meta = 'union'
 901             check_keys(expr_elem, 'union', ['data'],
 902                        ['base', 'discriminator'])
 903             union_types[expr[meta]] = expr
 904         elif 'alternate' in expr:
 905             meta = 'alternate'
 906             check_keys(expr_elem, 'alternate', ['data'])
 907         elif 'struct' in expr:
 908             meta = 'struct'
 909             check_keys(expr_elem, 'struct', ['data'], ['base'])
 910             struct_types[expr[meta]] = expr
 911         elif 'command' in expr:
 912             meta = 'command'
 913             check_keys(expr_elem, 'command', [],
 914                        ['data', 'returns', 'gen', 'success-response', 'boxed'])
 915         elif 'event' in expr:
 916             meta = 'event'
 917             check_keys(expr_elem, 'event', [], ['data', 'boxed'])
 918         else:
 919             raise QAPISemError(expr_elem['info'],
 920                                "Expression is missing metatype")
 921         name = expr[meta]
 922         add_name(name, info, meta)
 923         if doc and doc.symbol != name:
 924             raise QAPISemError(info, "Definition of '%s' follows documentation"
 925                                " for '%s'" % (name, doc.symbol))
 926
 927     # Try again for hidden UnionKind enum
 928     for expr_elem in exprs:
 929         expr = expr_elem['expr']
 930         if 'union' in expr and not discriminator_find_enum_define(expr):
 931             name = '%sKind' % expr['union']
 932         elif 'alternate' in expr:
 933             name = '%sKind' % expr['alternate']
 934         else:
 935             continue
 936         enum_types[name] = {'enum': name}
 937         add_name(name, info, 'enum', implicit=True)
 938
 939     # Validate that exprs make sense
 940     for expr_elem in exprs:
 941         expr = expr_elem['expr']
 942         info = expr_elem['info']
 943         doc = expr_elem.get('doc')
 944
 945         if 'enum' in expr:
 946             check_enum(expr, info)
 947         elif 'union' in expr:
 948             check_union(expr, info)
 949         elif 'alternate' in expr:
 950             check_alternate(expr, info)
 951         elif 'struct' in expr:
 952             check_struct(expr, info)
 953         elif 'command' in expr:
 954             check_command(expr, info)
 955         elif 'event' in expr:
 956             check_event(expr, info)
 957         else:
 958             assert False, 'unexpected meta type'
 959
 960         if doc:
 961             doc.check_expr(expr)
 962
 963     return exprs
 964
 965
 966 #
 967 # Schema compiler frontend
 968 #
 969
 970 class QAPISchemaEntity(object):
 971     def __init__(self, name, info, doc):
 972         assert isinstance(name, str)
 973         self.name = name
 974         # For explicitly defined entities, info points to the (explicit)
 975         # definition.  For builtins (and their arrays), info is None.
 976         # For implicitly defined entities, info points to a place that
 977         # triggered the implicit definition (there may be more than one
 978         # such place).
 979         self.info = info
 980         self.doc = doc
 981
 982     def c_name(self):
 983         return c_name(self.name)
 984
 985     def check(self, schema):
 986         pass
 987
 988     def is_implicit(self):
 989         return not self.info
 990
 991     def visit(self, visitor):
 992         pass
 993
 994
 995 class QAPISchemaVisitor(object):
 996     def visit_begin(self, schema):
 997         pass
 998
 999     def visit_end(self):
1000         pass
1001
1002     def visit_needed(self, entity):
1003         # Default to visiting everything
1004         return True
1005
1006     def visit_builtin_type(self, name, info, json_type):
1007         pass
1008
1009     def visit_enum_type(self, name, info, values, prefix):
1010         pass
1011
1012     def visit_array_type(self, name, info, element_type):
1013         pass
1014
1015     def visit_object_type(self, name, info, base, members, variants):
1016         pass
1017
1018     def visit_object_type_flat(self, name, info, members, variants):
1019         pass
1020
1021     def visit_alternate_type(self, name, info, variants):
1022         pass
1023
1024     def visit_command(self, name, info, arg_type, ret_type,
1025                       gen, success_response, boxed):
1026         pass
1027
1028     def visit_event(self, name, info, arg_type, boxed):
1029         pass
1030
1031
1032 class QAPISchemaType(QAPISchemaEntity):
1033     # Return the C type for common use.
1034     # For the types we commonly box, this is a pointer type.
1035     def c_type(self):
1036         pass
1037
1038     # Return the C type to be used in a parameter list.
1039     def c_param_type(self):
1040         return self.c_type()
1041
1042     # Return the C type to be used where we suppress boxing.
1043     def c_unboxed_type(self):
1044         return self.c_type()
1045
1046     def json_type(self):
1047         pass
1048
1049     def alternate_qtype(self):
1050         json2qtype = {
1051             'null':    'QTYPE_QNULL',
1052             'string':  'QTYPE_QSTRING',
1053             'number':  'QTYPE_QNUM',
1054             'int':     'QTYPE_QNUM',
1055             'boolean': 'QTYPE_QBOOL',
1056             'object':  'QTYPE_QDICT'
1057         }
1058         return json2qtype.get(self.json_type())
1059
1060     def doc_type(self):
1061         if self.is_implicit():
1062             return None
1063         return self.name
1064
1065
1066 class QAPISchemaBuiltinType(QAPISchemaType):
1067     def __init__(self, name, json_type, c_type):
1068         QAPISchemaType.__init__(self, name, None, None)
1069         assert not c_type or isinstance(c_type, str)
1070         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
1071                              'value')
1072         self._json_type_name = json_type
1073         self._c_type_name = c_type
1074
1075     def c_name(self):
1076         return self.name
1077
1078     def c_type(self):
1079         return self._c_type_name
1080
1081     def c_param_type(self):
1082         if self.name == 'str':
1083             return 'const ' + self._c_type_name
1084         return self._c_type_name
1085
1086     def json_type(self):
1087         return self._json_type_name
1088
1089     def doc_type(self):
1090         return self.json_type()
1091
1092     def visit(self, visitor):
1093         visitor.visit_builtin_type(self.name, self.info, self.json_type())
1094
1095
1096 class QAPISchemaEnumType(QAPISchemaType):
1097     def __init__(self, name, info, doc, values, prefix):
1098         QAPISchemaType.__init__(self, name, info, doc)
1099         for v in values:
1100             assert isinstance(v, QAPISchemaMember)
1101             v.set_owner(name)
1102         assert prefix is None or isinstance(prefix, str)
1103         self.values = values
1104         self.prefix = prefix
1105
1106     def check(self, schema):
1107         seen = {}
1108         for v in self.values:
1109             v.check_clash(self.info, seen)
1110             if self.doc:
1111                 self.doc.connect_member(v)
1112
1113     def is_implicit(self):
1114         # See QAPISchema._make_implicit_enum_type() and ._def_predefineds()
1115         return self.name.endswith('Kind') or self.name == 'QType'
1116
1117     def c_type(self):
1118         return c_name(self.name)
1119
1120     def member_names(self):
1121         return [v.name for v in self.values]
1122
1123     def json_type(self):
1124         return 'string'
1125
1126     def visit(self, visitor):
1127         visitor.visit_enum_type(self.name, self.info,
1128                                 self.member_names(), self.prefix)
1129
1130
1131 class QAPISchemaArrayType(QAPISchemaType):
1132     def __init__(self, name, info, element_type):
1133         QAPISchemaType.__init__(self, name, info, None)
1134         assert isinstance(element_type, str)
1135         self._element_type_name = element_type
1136         self.element_type = None
1137
1138     def check(self, schema):
1139         self.element_type = schema.lookup_type(self._element_type_name)
1140         assert self.element_type
1141
1142     def is_implicit(self):
1143         return True
1144
1145     def c_type(self):
1146         return c_name(self.name) + pointer_suffix
1147
1148     def json_type(self):
1149         return 'array'
1150
1151     def doc_type(self):
1152         elt_doc_type = self.element_type.doc_type()
1153         if not elt_doc_type:
1154             return None
1155         return 'array of ' + elt_doc_type
1156
1157     def visit(self, visitor):
1158         visitor.visit_array_type(self.name, self.info, self.element_type)
1159
1160
1161 class QAPISchemaObjectType(QAPISchemaType):
1162     def __init__(self, name, info, doc, base, local_members, variants):
1163         # struct has local_members, optional base, and no variants
1164         # flat union has base, variants, and no local_members
1165         # simple union has local_members, variants, and no base
1166         QAPISchemaType.__init__(self, name, info, doc)
1167         assert base is None or isinstance(base, str)
1168         for m in local_members:
1169             assert isinstance(m, QAPISchemaObjectTypeMember)
1170             m.set_owner(name)
1171         if variants is not None:
1172             assert isinstance(variants, QAPISchemaObjectTypeVariants)
1173             variants.set_owner(name)
1174         self._base_name = base
1175         self.base = None
1176         self.local_members = local_members
1177         self.variants = variants
1178         self.members = None
1179
1180     def check(self, schema):
1181         if self.members is False:               # check for cycles
1182             raise QAPISemError(self.info,
1183                                "Object %s contains itself" % self.name)
1184         if self.members:
1185             return
1186         self.members = False                    # mark as being checked
1187         seen = OrderedDict()
1188         if self._base_name:
1189             self.base = schema.lookup_type(self._base_name)
1190             assert isinstance(self.base, QAPISchemaObjectType)
1191             self.base.check(schema)
1192             self.base.check_clash(self.info, seen)
1193         for m in self.local_members:
1194             m.check(schema)
1195             m.check_clash(self.info, seen)
1196             if self.doc:
1197                 self.doc.connect_member(m)
1198         self.members = seen.values()
1199         if self.variants:
1200             self.variants.check(schema, seen)
1201             assert self.variants.tag_member in self.members
1202             self.variants.check_clash(self.info, seen)
1203         if self.doc:
1204             self.doc.check()
1205
1206     # Check that the members of this type do not cause duplicate JSON members,
1207     # and update seen to track the members seen so far. Report any errors
1208     # on behalf of info, which is not necessarily self.info
1209     def check_clash(self, info, seen):
1210         assert not self.variants       # not implemented
1211         for m in self.members:
1212             m.check_clash(info, seen)
1213
1214     def is_implicit(self):
1215         # See QAPISchema._make_implicit_object_type(), as well as
1216         # _def_predefineds()
1217         return self.name.startswith('q_')
1218
1219     def is_empty(self):
1220         assert self.members is not None
1221         return not self.members and not self.variants
1222
1223     def c_name(self):
1224         assert self.name != 'q_empty'
1225         return QAPISchemaType.c_name(self)
1226
1227     def c_type(self):
1228         assert not self.is_implicit()
1229         return c_name(self.name) + pointer_suffix
1230
1231     def c_unboxed_type(self):
1232         return c_name(self.name)
1233
1234     def json_type(self):
1235         return 'object'
1236
1237     def visit(self, visitor):
1238         visitor.visit_object_type(self.name, self.info,
1239                                   self.base, self.local_members, self.variants)
1240         visitor.visit_object_type_flat(self.name, self.info,
1241                                        self.members, self.variants)
1242
1243
1244 class QAPISchemaMember(object):
1245     role = 'member'
1246
1247     def __init__(self, name):
1248         assert isinstance(name, str)
1249         self.name = name
1250         self.owner = None
1251
1252     def set_owner(self, name):
1253         assert not self.owner
1254         self.owner = name
1255
1256     def check_clash(self, info, seen):
1257         cname = c_name(self.name)
1258         if cname.lower() != cname and self.owner not in name_case_whitelist:
1259             raise QAPISemError(info,
1260                                "%s should not use uppercase" % self.describe())
1261         if cname in seen:
1262             raise QAPISemError(info, "%s collides with %s" %
1263                                (self.describe(), seen[cname].describe()))
1264         seen[cname] = self
1265
1266     def _pretty_owner(self):
1267         owner = self.owner
1268         if owner.startswith('q_obj_'):
1269             # See QAPISchema._make_implicit_object_type() - reverse the
1270             # mapping there to create a nice human-readable description
1271             owner = owner[6:]
1272             if owner.endswith('-arg'):
1273                 return '(parameter of %s)' % owner[:-4]
1274             elif owner.endswith('-base'):
1275                 return '(base of %s)' % owner[:-5]
1276             else:
1277                 assert owner.endswith('-wrapper')
1278                 # Unreachable and not implemented
1279                 assert False
1280         if owner.endswith('Kind'):
1281             # See QAPISchema._make_implicit_enum_type()
1282             return '(branch of %s)' % owner[:-4]
1283         return '(%s of %s)' % (self.role, owner)
1284
1285     def describe(self):
1286         return "'%s' %s" % (self.name, self._pretty_owner())
1287
1288
1289 class QAPISchemaObjectTypeMember(QAPISchemaMember):
1290     def __init__(self, name, typ, optional):
1291         QAPISchemaMember.__init__(self, name)
1292         assert isinstance(typ, str)
1293         assert isinstance(optional, bool)
1294         self._type_name = typ
1295         self.type = None
1296         self.optional = optional
1297
1298     def check(self, schema):
1299         assert self.owner
1300         self.type = schema.lookup_type(self._type_name)
1301         assert self.type
1302
1303
1304 class QAPISchemaObjectTypeVariants(object):
1305     def __init__(self, tag_name, tag_member, variants):
1306         # Flat unions pass tag_name but not tag_member.
1307         # Simple unions and alternates pass tag_member but not tag_name.
1308         # After check(), tag_member is always set, and tag_name remains
1309         # a reliable witness of being used by a flat union.
1310         assert bool(tag_member) != bool(tag_name)
1311         assert (isinstance(tag_name, str) or
1312                 isinstance(tag_member, QAPISchemaObjectTypeMember))
1313         assert len(variants) > 0
1314         for v in variants:
1315             assert isinstance(v, QAPISchemaObjectTypeVariant)
1316         self._tag_name = tag_name
1317         self.tag_member = tag_member
1318         self.variants = variants
1319
1320     def set_owner(self, name):
1321         for v in self.variants:
1322             v.set_owner(name)
1323
1324     def check(self, schema, seen):
1325         if not self.tag_member:    # flat union
1326             self.tag_member = seen[c_name(self._tag_name)]
1327             assert self._tag_name == self.tag_member.name
1328         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
1329         for v in self.variants:
1330             v.check(schema)
1331             # Union names must match enum values; alternate names are
1332             # checked separately. Use 'seen' to tell the two apart.
1333             if seen:
1334                 assert v.name in self.tag_member.type.member_names()
1335                 assert isinstance(v.type, QAPISchemaObjectType)
1336                 v.type.check(schema)
1337
1338     def check_clash(self, info, seen):
1339         for v in self.variants:
1340             # Reset seen map for each variant, since qapi names from one
1341             # branch do not affect another branch
1342             assert isinstance(v.type, QAPISchemaObjectType)
1343             v.type.check_clash(info, dict(seen))
1344
1345
1346 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1347     role = 'branch'
1348
1349     def __init__(self, name, typ):
1350         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1351
1352
1353 class QAPISchemaAlternateType(QAPISchemaType):
1354     def __init__(self, name, info, doc, variants):
1355         QAPISchemaType.__init__(self, name, info, doc)
1356         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1357         assert variants.tag_member
1358         variants.set_owner(name)
1359         variants.tag_member.set_owner(self.name)
1360         self.variants = variants
1361
1362     def check(self, schema):
1363         self.variants.tag_member.check(schema)
1364         # Not calling self.variants.check_clash(), because there's nothing
1365         # to clash with
1366         self.variants.check(schema, {})
1367         # Alternate branch names have no relation to the tag enum values;
1368         # so we have to check for potential name collisions ourselves.
1369         seen = {}
1370         for v in self.variants.variants:
1371             v.check_clash(self.info, seen)
1372             if self.doc:
1373                 self.doc.connect_member(v)
1374         if self.doc:
1375             self.doc.check()
1376
1377     def c_type(self):
1378         return c_name(self.name) + pointer_suffix
1379
1380     def json_type(self):
1381         return 'value'
1382
1383     def visit(self, visitor):
1384         visitor.visit_alternate_type(self.name, self.info, self.variants)
1385
1386     def is_empty(self):
1387         return False
1388
1389
1390 class QAPISchemaCommand(QAPISchemaEntity):
1391     def __init__(self, name, info, doc, arg_type, ret_type,
1392                  gen, success_response, boxed):
1393         QAPISchemaEntity.__init__(self, name, info, doc)
1394         assert not arg_type or isinstance(arg_type, str)
1395         assert not ret_type or isinstance(ret_type, str)
1396         self._arg_type_name = arg_type
1397         self.arg_type = None
1398         self._ret_type_name = ret_type
1399         self.ret_type = None
1400         self.gen = gen
1401         self.success_response = success_response
1402         self.boxed = boxed
1403
1404     def check(self, schema):
1405         if self._arg_type_name:
1406             self.arg_type = schema.lookup_type(self._arg_type_name)
1407             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1408                     isinstance(self.arg_type, QAPISchemaAlternateType))
1409             self.arg_type.check(schema)
1410             if self.boxed:
1411                 if self.arg_type.is_empty():
1412                     raise QAPISemError(self.info,
1413                                        "Cannot use 'boxed' with empty type")
1414             else:
1415                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1416                 assert not self.arg_type.variants
1417         elif self.boxed:
1418             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1419         if self._ret_type_name:
1420             self.ret_type = schema.lookup_type(self._ret_type_name)
1421             assert isinstance(self.ret_type, QAPISchemaType)
1422
1423     def visit(self, visitor):
1424         visitor.visit_command(self.name, self.info,
1425                               self.arg_type, self.ret_type,
1426                               self.gen, self.success_response, self.boxed)
1427
1428
1429 class QAPISchemaEvent(QAPISchemaEntity):
1430     def __init__(self, name, info, doc, arg_type, boxed):
1431         QAPISchemaEntity.__init__(self, name, info, doc)
1432         assert not arg_type or isinstance(arg_type, str)
1433         self._arg_type_name = arg_type
1434         self.arg_type = None
1435         self.boxed = boxed
1436
1437     def check(self, schema):
1438         if self._arg_type_name:
1439             self.arg_type = schema.lookup_type(self._arg_type_name)
1440             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1441                     isinstance(self.arg_type, QAPISchemaAlternateType))
1442             self.arg_type.check(schema)
1443             if self.boxed:
1444                 if self.arg_type.is_empty():
1445                     raise QAPISemError(self.info,
1446                                        "Cannot use 'boxed' with empty type")
1447             else:
1448                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1449                 assert not self.arg_type.variants
1450         elif self.boxed:
1451             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1452
1453     def visit(self, visitor):
1454         visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
1455
1456
1457 class QAPISchema(object):
1458     def __init__(self, fname):
1459         try:
1460             parser = QAPISchemaParser(open(fname, 'r'))
1461             self.exprs = check_exprs(parser.exprs)
1462             self.docs = parser.docs
1463             self._entity_dict = {}
1464             self._predefining = True
1465             self._def_predefineds()
1466             self._predefining = False
1467             self._def_exprs()
1468             self.check()
1469         except QAPIError as err:
1470             print >>sys.stderr, err
1471             exit(1)
1472
1473     def _def_entity(self, ent):
1474         # Only the predefined types are allowed to not have info
1475         assert ent.info or self._predefining
1476         assert ent.name not in self._entity_dict
1477         self._entity_dict[ent.name] = ent
1478
1479     def lookup_entity(self, name, typ=None):
1480         ent = self._entity_dict.get(name)
1481         if typ and not isinstance(ent, typ):
1482             return None
1483         return ent
1484
1485     def lookup_type(self, name):
1486         return self.lookup_entity(name, QAPISchemaType)
1487
1488     def _def_builtin_type(self, name, json_type, c_type):
1489         self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
1490         # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
1491         # qapi-types.h from a single .c, all arrays of builtins must be
1492         # declared in the first file whether or not they are used.  Nicer
1493         # would be to use lazy instantiation, while figuring out how to
1494         # avoid compilation issues with multiple qapi-types.h.
1495         self._make_array_type(name, None)
1496
1497     def _def_predefineds(self):
1498         for t in [('str',    'string',  'char' + pointer_suffix),
1499                   ('number', 'number',  'double'),
1500                   ('int',    'int',     'int64_t'),
1501                   ('int8',   'int',     'int8_t'),
1502                   ('int16',  'int',     'int16_t'),
1503                   ('int32',  'int',     'int32_t'),
1504                   ('int64',  'int',     'int64_t'),
1505                   ('uint8',  'int',     'uint8_t'),
1506                   ('uint16', 'int',     'uint16_t'),
1507                   ('uint32', 'int',     'uint32_t'),
1508                   ('uint64', 'int',     'uint64_t'),
1509                   ('size',   'int',     'uint64_t'),
1510                   ('bool',   'boolean', 'bool'),
1511                   ('any',    'value',   'QObject' + pointer_suffix),
1512                   ('null',   'null',    'QNull' + pointer_suffix)]:
1513             self._def_builtin_type(*t)
1514         self.the_empty_object_type = QAPISchemaObjectType(
1515             'q_empty', None, None, None, [], None)
1516         self._def_entity(self.the_empty_object_type)
1517         qtype_values = self._make_enum_members(['none', 'qnull', 'qnum',
1518                                                 'qstring', 'qdict', 'qlist',
1519                                                 'qbool'])
1520         self._def_entity(QAPISchemaEnumType('QType', None, None,
1521                                             qtype_values, 'QTYPE'))
1522
1523     def _make_enum_members(self, values):
1524         return [QAPISchemaMember(v) for v in values]
1525
1526     def _make_implicit_enum_type(self, name, info, values):
1527         # See also QAPISchemaObjectTypeMember._pretty_owner()
1528         name = name + 'Kind'   # Use namespace reserved by add_name()
1529         self._def_entity(QAPISchemaEnumType(
1530             name, info, None, self._make_enum_members(values), None))
1531         return name
1532
1533     def _make_array_type(self, element_type, info):
1534         name = element_type + 'List'   # Use namespace reserved by add_name()
1535         if not self.lookup_type(name):
1536             self._def_entity(QAPISchemaArrayType(name, info, element_type))
1537         return name
1538
1539     def _make_implicit_object_type(self, name, info, doc, role, members):
1540         if not members:
1541             return None
1542         # See also QAPISchemaObjectTypeMember._pretty_owner()
1543         name = 'q_obj_%s-%s' % (name, role)
1544         if not self.lookup_entity(name, QAPISchemaObjectType):
1545             self._def_entity(QAPISchemaObjectType(name, info, doc, None,
1546                                                   members, None))
1547         return name
1548
1549     def _def_enum_type(self, expr, info, doc):
1550         name = expr['enum']
1551         data = expr['data']
1552         prefix = expr.get('prefix')
1553         self._def_entity(QAPISchemaEnumType(
1554             name, info, doc, self._make_enum_members(data), prefix))
1555
1556     def _make_member(self, name, typ, info):
1557         optional = False
1558         if name.startswith('*'):
1559             name = name[1:]
1560             optional = True
1561         if isinstance(typ, list):
1562             assert len(typ) == 1
1563             typ = self._make_array_type(typ[0], info)
1564         return QAPISchemaObjectTypeMember(name, typ, optional)
1565
1566     def _make_members(self, data, info):
1567         return [self._make_member(key, value, info)
1568                 for (key, value) in data.iteritems()]
1569
1570     def _def_struct_type(self, expr, info, doc):
1571         name = expr['struct']
1572         base = expr.get('base')
1573         data = expr['data']
1574         self._def_entity(QAPISchemaObjectType(name, info, doc, base,
1575                                               self._make_members(data, info),
1576                                               None))
1577
1578     def _make_variant(self, case, typ):
1579         return QAPISchemaObjectTypeVariant(case, typ)
1580
1581     def _make_simple_variant(self, case, typ, info):
1582         if isinstance(typ, list):
1583             assert len(typ) == 1
1584             typ = self._make_array_type(typ[0], info)
1585         typ = self._make_implicit_object_type(
1586             typ, info, None, 'wrapper', [self._make_member('data', typ, info)])
1587         return QAPISchemaObjectTypeVariant(case, typ)
1588
1589     def _def_union_type(self, expr, info, doc):
1590         name = expr['union']
1591         data = expr['data']
1592         base = expr.get('base')
1593         tag_name = expr.get('discriminator')
1594         tag_member = None
1595         if isinstance(base, dict):
1596             base = (self._make_implicit_object_type(
1597                 name, info, doc, 'base', self._make_members(base, info)))
1598         if tag_name:
1599             variants = [self._make_variant(key, value)
1600                         for (key, value) in data.iteritems()]
1601             members = []
1602         else:
1603             variants = [self._make_simple_variant(key, value, info)
1604                         for (key, value) in data.iteritems()]
1605             typ = self._make_implicit_enum_type(name, info,
1606                                                 [v.name for v in variants])
1607             tag_member = QAPISchemaObjectTypeMember('type', typ, False)
1608             members = [tag_member]
1609         self._def_entity(
1610             QAPISchemaObjectType(name, info, doc, base, members,
1611                                  QAPISchemaObjectTypeVariants(tag_name,
1612                                                               tag_member,
1613                                                               variants)))
1614
1615     def _def_alternate_type(self, expr, info, doc):
1616         name = expr['alternate']
1617         data = expr['data']
1618         variants = [self._make_variant(key, value)
1619                     for (key, value) in data.iteritems()]
1620         tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
1621         self._def_entity(
1622             QAPISchemaAlternateType(name, info, doc,
1623                                     QAPISchemaObjectTypeVariants(None,
1624                                                                  tag_member,
1625                                                                  variants)))
1626
1627     def _def_command(self, expr, info, doc):
1628         name = expr['command']
1629         data = expr.get('data')
1630         rets = expr.get('returns')
1631         gen = expr.get('gen', True)
1632         success_response = expr.get('success-response', True)
1633         boxed = expr.get('boxed', False)
1634         if isinstance(data, OrderedDict):
1635             data = self._make_implicit_object_type(
1636                 name, info, doc, 'arg', self._make_members(data, info))
1637         if isinstance(rets, list):
1638             assert len(rets) == 1
1639             rets = self._make_array_type(rets[0], info)
1640         self._def_entity(QAPISchemaCommand(name, info, doc, data, rets,
1641                                            gen, success_response, boxed))
1642
1643     def _def_event(self, expr, info, doc):
1644         name = expr['event']
1645         data = expr.get('data')
1646         boxed = expr.get('boxed', False)
1647         if isinstance(data, OrderedDict):
1648             data = self._make_implicit_object_type(
1649                 name, info, doc, 'arg', self._make_members(data, info))
1650         self._def_entity(QAPISchemaEvent(name, info, doc, data, boxed))
1651
1652     def _def_exprs(self):
1653         for expr_elem in self.exprs:
1654             expr = expr_elem['expr']
1655             info = expr_elem['info']
1656             doc = expr_elem.get('doc')
1657             if 'enum' in expr:
1658                 self._def_enum_type(expr, info, doc)
1659             elif 'struct' in expr:
1660                 self._def_struct_type(expr, info, doc)
1661             elif 'union' in expr:
1662                 self._def_union_type(expr, info, doc)
1663             elif 'alternate' in expr:
1664                 self._def_alternate_type(expr, info, doc)
1665             elif 'command' in expr:
1666                 self._def_command(expr, info, doc)
1667             elif 'event' in expr:
1668                 self._def_event(expr, info, doc)
1669             else:
1670                 assert False
1671
1672     def check(self):
1673         for ent in self._entity_dict.values():
1674             ent.check(self)
1675
1676     def visit(self, visitor):
1677         visitor.visit_begin(self)
1678         for (name, entity) in sorted(self._entity_dict.items()):
1679             if visitor.visit_needed(entity):
1680                 entity.visit(visitor)
1681         visitor.visit_end()
1682
1683
1684 #
1685 # Code generation helpers
1686 #
1687
1688 def camel_case(name):
1689     new_name = ''
1690     first = True
1691     for ch in name:
1692         if ch in ['_', '-']:
1693             first = True
1694         elif first:
1695             new_name += ch.upper()
1696             first = False
1697         else:
1698             new_name += ch.lower()
1699     return new_name
1700
1701
1702 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1703 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1704 # ENUM24_Name -> ENUM24_NAME
1705 def camel_to_upper(value):
1706     c_fun_str = c_name(value, False)
1707     if value.isupper():
1708         return c_fun_str
1709
1710     new_name = ''
1711     l = len(c_fun_str)
1712     for i in range(l):
1713         c = c_fun_str[i]
1714         # When c is upper and no '_' appears before, do more checks
1715         if c.isupper() and (i > 0) and c_fun_str[i - 1] != '_':
1716             if i < l - 1 and c_fun_str[i + 1].islower():
1717                 new_name += '_'
1718             elif c_fun_str[i - 1].isdigit():
1719                 new_name += '_'
1720         new_name += c
1721     return new_name.lstrip('_').upper()
1722
1723
1724 def c_enum_const(type_name, const_name, prefix=None):
1725     if prefix is not None:
1726         type_name = prefix
1727     return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
1728
1729 c_name_trans = string.maketrans('.-', '__')
1730
1731
1732 # Map @name to a valid C identifier.
1733 # If @protect, avoid returning certain ticklish identifiers (like
1734 # C keywords) by prepending 'q_'.
1735 #
1736 # Used for converting 'name' from a 'name':'type' qapi definition
1737 # into a generated struct member, as well as converting type names
1738 # into substrings of a generated C function name.
1739 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1740 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1741 def c_name(name, protect=True):
1742     # ANSI X3J11/88-090, 3.1.1
1743     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1744                      'default', 'do', 'double', 'else', 'enum', 'extern',
1745                      'float', 'for', 'goto', 'if', 'int', 'long', 'register',
1746                      'return', 'short', 'signed', 'sizeof', 'static',
1747                      'struct', 'switch', 'typedef', 'union', 'unsigned',
1748                      'void', 'volatile', 'while'])
1749     # ISO/IEC 9899:1999, 6.4.1
1750     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1751     # ISO/IEC 9899:2011, 6.4.1
1752     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
1753                      '_Noreturn', '_Static_assert', '_Thread_local'])
1754     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1755     # excluding _.*
1756     gcc_words = set(['asm', 'typeof'])
1757     # C++ ISO/IEC 14882:2003 2.11
1758     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1759                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1760                      'namespace', 'new', 'operator', 'private', 'protected',
1761                      'public', 'reinterpret_cast', 'static_cast', 'template',
1762                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1763                      'using', 'virtual', 'wchar_t',
1764                      # alternative representations
1765                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1766                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1767     # namespace pollution:
1768     polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
1769     name = name.translate(c_name_trans)
1770     if protect and (name in c89_words | c99_words | c11_words | gcc_words
1771                     | cpp_words | polluted_words):
1772         return 'q_' + name
1773     return name
1774
1775 eatspace = '\033EATSPACE.'
1776 pointer_suffix = ' *' + eatspace
1777
1778
1779 def genindent(count):
1780     ret = ''
1781     for _ in range(count):
1782         ret += ' '
1783     return ret
1784
1785 indent_level = 0
1786
1787
1788 def push_indent(indent_amount=4):
1789     global indent_level
1790     indent_level += indent_amount
1791
1792
1793 def pop_indent(indent_amount=4):
1794     global indent_level
1795     indent_level -= indent_amount
1796
1797
1798 # Generate @code with @kwds interpolated.
1799 # Obey indent_level, and strip eatspace.
1800 def cgen(code, **kwds):
1801     raw = code % kwds
1802     if indent_level:
1803         indent = genindent(indent_level)
1804         # re.subn() lacks flags support before Python 2.7, use re.compile()
1805         raw = re.subn(re.compile(r'^.', re.MULTILINE),
1806                       indent + r'\g<0>', raw)
1807         raw = raw[0]
1808     return re.sub(re.escape(eatspace) + r' *', '', raw)
1809
1810
1811 def mcgen(code, **kwds):
1812     if code[0] == '\n':
1813         code = code[1:]
1814     return cgen(code, **kwds)
1815
1816
1817 def guardname(filename):
1818     return c_name(filename, protect=False).upper()
1819
1820
1821 def guardstart(name):
1822     return mcgen('''
1823
1824 #ifndef %(name)s
1825 #define %(name)s
1826
1827 ''',
1828                  name=guardname(name))
1829
1830
1831 def guardend(name):
1832     return mcgen('''
1833
1834 #endif /* %(name)s */
1835
1836 ''',
1837                  name=guardname(name))
1838
1839
1840 def gen_enum_lookup(name, values, prefix=None):
1841     ret = mcgen('''
1842
1843 const QEnumLookup %(c_name)s_lookup = {
1844     .array = (const char *const[]) {
1845 ''',
1846                 c_name=c_name(name))
1847     for value in values:
1848         index = c_enum_const(name, value, prefix)
1849         ret += mcgen('''
1850         [%(index)s] = "%(value)s",
1851 ''',
1852                      index=index, value=value)
1853
1854     ret += mcgen('''
1855     },
1856     .size = %(max_index)s
1857 };
1858 ''',
1859                  max_index=c_enum_const(name, '_MAX', prefix))
1860     return ret
1861
1862
1863 def gen_enum(name, values, prefix=None):
1864     # append automatically generated _MAX value
1865     enum_values = values + ['_MAX']
1866
1867     ret = mcgen('''
1868
1869 typedef enum %(c_name)s {
1870 ''',
1871                 c_name=c_name(name))
1872
1873     i = 0
1874     for value in enum_values:
1875         ret += mcgen('''
1876     %(c_enum)s = %(i)d,
1877 ''',
1878                      c_enum=c_enum_const(name, value, prefix),
1879                      i=i)
1880         i += 1
1881
1882     ret += mcgen('''
1883 } %(c_name)s;
1884 ''',
1885                  c_name=c_name(name))
1886
1887     ret += mcgen('''
1888
1889 #define %(c_name)s_str(val) \\
1890     qapi_enum_lookup(&%(c_name)s_lookup, (val))
1891
1892 extern const QEnumLookup %(c_name)s_lookup;
1893 ''',
1894                  c_name=c_name(name))
1895     return ret
1896
1897
1898 def build_params(arg_type, boxed, extra):
1899     if not arg_type:
1900         assert not boxed
1901         return extra
1902     ret = ''
1903     sep = ''
1904     if boxed:
1905         ret += '%s arg' % arg_type.c_param_type()
1906         sep = ', '
1907     else:
1908         assert not arg_type.variants
1909         for memb in arg_type.members:
1910             ret += sep
1911             sep = ', '
1912             if memb.optional:
1913                 ret += 'bool has_%s, ' % c_name(memb.name)
1914             ret += '%s %s' % (memb.type.c_param_type(),
1915                               c_name(memb.name))
1916     if extra:
1917         ret += sep + extra
1918     return ret
1919
1920
1921 #
1922 # Common command line parsing
1923 #
1924
1925
1926 def parse_command_line(extra_options='', extra_long_options=[]):
1927
1928     try:
1929         opts, args = getopt.gnu_getopt(sys.argv[1:],
1930                                        'chp:o:' + extra_options,
1931                                        ['source', 'header', 'prefix=',
1932                                         'output-dir='] + extra_long_options)
1933     except getopt.GetoptError as err:
1934         print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
1935         sys.exit(1)
1936
1937     output_dir = ''
1938     prefix = ''
1939     do_c = False
1940     do_h = False
1941     extra_opts = []
1942
1943     for oa in opts:
1944         o, a = oa
1945         if o in ('-p', '--prefix'):
1946             match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1947             if match.end() != len(a):
1948                 print >>sys.stderr, \
1949                     "%s: 'funny character '%s' in argument of --prefix" \
1950                     % (sys.argv[0], a[match.end()])
1951                 sys.exit(1)
1952             prefix = a
1953         elif o in ('-o', '--output-dir'):
1954             output_dir = a + '/'
1955         elif o in ('-c', '--source'):
1956             do_c = True
1957         elif o in ('-h', '--header'):
1958             do_h = True
1959         else:
1960             extra_opts.append(oa)
1961
1962     if not do_c and not do_h:
1963         do_c = True
1964         do_h = True
1965
1966     if len(args) != 1:
1967         print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
1968         sys.exit(1)
1969     fname = args[0]
1970
1971     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
1972
1973 #
1974 # Generate output files with boilerplate
1975 #
1976
1977
1978 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
1979                 c_comment, h_comment):
1980     guard = guardname(prefix + h_file)
1981     c_file = output_dir + prefix + c_file
1982     h_file = output_dir + prefix + h_file
1983
1984     if output_dir:
1985         try:
1986             os.makedirs(output_dir)
1987         except os.error as e:
1988             if e.errno != errno.EEXIST:
1989                 raise
1990
1991     def maybe_open(really, name, opt):
1992         if really:
1993             return open(name, opt)
1994         else:
1995             import StringIO
1996             return StringIO.StringIO()
1997
1998     fdef = maybe_open(do_c, c_file, 'w')
1999     fdecl = maybe_open(do_h, h_file, 'w')
2000
2001     fdef.write(mcgen('''
2002 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2003 %(comment)s
2004 ''',
2005                      comment=c_comment))
2006
2007     fdecl.write(mcgen('''
2008 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2009 %(comment)s
2010 #ifndef %(guard)s
2011 #define %(guard)s
2012
2013 ''',
2014                       comment=h_comment, guard=guard))
2015
2016     return (fdef, fdecl)
2017
2018
2019 def close_output(fdef, fdecl):
2020     fdecl.write('''
2021 #endif
2022 ''')
2023     fdecl.close()
2024     fdef.close()