scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 import re
  15 from ordereddict import OrderedDict
  16 import errno
  17 import getopt
  18 import os
  19 import sys
  20 import string
  21
  22 builtin_types = {
  23     'str':      'QTYPE_QSTRING',
  24     'int':      'QTYPE_QINT',
  25     'number':   'QTYPE_QFLOAT',
  26     'bool':     'QTYPE_QBOOL',
  27     'int8':     'QTYPE_QINT',
  28     'int16':    'QTYPE_QINT',
  29     'int32':    'QTYPE_QINT',
  30     'int64':    'QTYPE_QINT',
  31     'uint8':    'QTYPE_QINT',
  32     'uint16':   'QTYPE_QINT',
  33     'uint32':   'QTYPE_QINT',
  34     'uint64':   'QTYPE_QINT',
  35     'size':     'QTYPE_QINT',
  36     'any':      None,           # any QType possible, actually
  37     'QType':    'QTYPE_QSTRING',
  38 }
  39
  40 # Are documentation comments required?
  41 doc_required = False
  42
  43 # Whitelist of commands allowed to return a non-dictionary
  44 returns_whitelist = []
  45
  46 # Whitelist of entities allowed to violate case conventions
  47 case_whitelist = [
  48     # From QMP:
  49     'ACPISlotType',         # DIMM, visible through query-acpi-ospm-status
  50     'CpuInfoMIPS',          # PC, visible through query-cpu
  51     'CpuInfoTricore',       # PC, visible through query-cpu
  52     'QapiErrorClass',       # all members, visible through errors
  53     'UuidInfo',             # UUID, visible through query-uuid
  54     'X86CPURegister32',     # all members, visible indirectly through qom-get
  55     'q_obj_CpuInfo-base',   # CPU, visible through query-cpu
  56 ]
  57
  58 enum_types = []
  59 struct_types = []
  60 union_types = []
  61 events = []
  62 all_names = {}
  63
  64 #
  65 # Parsing the schema into expressions
  66 #
  67
  68
  69 def error_path(parent):
  70     res = ""
  71     while parent:
  72         res = ("In file included from %s:%d:\n" % (parent['file'],
  73                                                    parent['line'])) + res
  74         parent = parent['parent']
  75     return res
  76
  77
  78 class QAPIError(Exception):
  79     def __init__(self, fname, line, col, incl_info, msg):
  80         Exception.__init__(self)
  81         self.fname = fname
  82         self.line = line
  83         self.col = col
  84         self.info = incl_info
  85         self.msg = msg
  86
  87     def __str__(self):
  88         loc = "%s:%d" % (self.fname, self.line)
  89         if self.col is not None:
  90             loc += ":%s" % self.col
  91         return error_path(self.info) + "%s: %s" % (loc, self.msg)
  92
  93
  94 class QAPIParseError(QAPIError):
  95     def __init__(self, parser, msg):
  96         col = 1
  97         for ch in parser.src[parser.line_pos:parser.pos]:
  98             if ch == '\t':
  99                 col = (col + 7) % 8 + 1
 100             else:
 101                 col += 1
 102         QAPIError.__init__(self, parser.fname, parser.line, col,
 103                            parser.incl_info, msg)
 104
 105
 106 class QAPISemError(QAPIError):
 107     def __init__(self, info, msg):
 108         QAPIError.__init__(self, info['file'], info['line'], None,
 109                            info['parent'], msg)
 110
 111
 112 class QAPIDoc(object):
 113     class Section(object):
 114         def __init__(self, name=None):
 115             # optional section name (argument/member or section name)
 116             self.name = name
 117             # the list of lines for this section
 118             self.content = []
 119
 120         def append(self, line):
 121             self.content.append(line)
 122
 123         def __repr__(self):
 124             return "\n".join(self.content).strip()
 125
 126     class ArgSection(Section):
 127         pass
 128
 129     def __init__(self, parser, info):
 130         # self.parser is used to report errors with QAPIParseError.  The
 131         # resulting error position depends on the state of the parser.
 132         # It happens to be the beginning of the comment.  More or less
 133         # servicable, but action at a distance.
 134         self.parser = parser
 135         self.info = info
 136         self.symbol = None
 137         self.body = QAPIDoc.Section()
 138         # dict mapping parameter name to ArgSection
 139         self.args = OrderedDict()
 140         # a list of Section
 141         self.sections = []
 142         # the current section
 143         self.section = self.body
 144         # associated expression (to be set by expression parser)
 145         self.expr = None
 146
 147     def has_section(self, name):
 148         """Return True if we have a section with this name."""
 149         for i in self.sections:
 150             if i.name == name:
 151                 return True
 152         return False
 153
 154     def append(self, line):
 155         """Parse a comment line and add it to the documentation."""
 156         line = line[1:]
 157         if not line:
 158             self._append_freeform(line)
 159             return
 160
 161         if line[0] != ' ':
 162             raise QAPIParseError(self.parser, "Missing space after #")
 163         line = line[1:]
 164
 165         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 166         # recognized, and get silently treated as ordinary text
 167         if self.symbol:
 168             self._append_symbol_line(line)
 169         elif not self.body.content and line.startswith("@"):
 170             if not line.endswith(":"):
 171                 raise QAPIParseError(self.parser, "Line should end with :")
 172             self.symbol = line[1:-1]
 173             # FIXME invalid names other than the empty string aren't flagged
 174             if not self.symbol:
 175                 raise QAPIParseError(self.parser, "Invalid name")
 176         else:
 177             self._append_freeform(line)
 178
 179     def _append_symbol_line(self, line):
 180         name = line.split(' ', 1)[0]
 181
 182         if name.startswith("@") and name.endswith(":"):
 183             line = line[len(name)+1:]
 184             self._start_args_section(name[1:-1])
 185         elif name in ("Returns:", "Since:",
 186                       # those are often singular or plural
 187                       "Note:", "Notes:",
 188                       "Example:", "Examples:",
 189                       "TODO:"):
 190             line = line[len(name)+1:]
 191             self._start_section(name[:-1])
 192
 193         self._append_freeform(line)
 194
 195     def _start_args_section(self, name):
 196         # FIXME invalid names other than the empty string aren't flagged
 197         if not name:
 198             raise QAPIParseError(self.parser, "Invalid parameter name")
 199         if name in self.args:
 200             raise QAPIParseError(self.parser,
 201                                  "'%s' parameter name duplicated" % name)
 202         if self.sections:
 203             raise QAPIParseError(self.parser,
 204                                  "'@%s:' can't follow '%s' section"
 205                                  % (name, self.sections[0].name))
 206         self.section = QAPIDoc.ArgSection(name)
 207         self.args[name] = self.section
 208
 209     def _start_section(self, name=""):
 210         if name in ("Returns", "Since") and self.has_section(name):
 211             raise QAPIParseError(self.parser,
 212                                  "Duplicated '%s' section" % name)
 213         self.section = QAPIDoc.Section(name)
 214         self.sections.append(self.section)
 215
 216     def _append_freeform(self, line):
 217         in_arg = isinstance(self.section, QAPIDoc.ArgSection)
 218         if (in_arg and self.section.content
 219                 and not self.section.content[-1]
 220                 and line and not line[0].isspace()):
 221             self._start_section()
 222         if (in_arg or not self.section.name
 223                 or not self.section.name.startswith("Example")):
 224             line = line.strip()
 225         self.section.append(line)
 226
 227
 228 class QAPISchemaParser(object):
 229
 230     def __init__(self, fp, previously_included=[], incl_info=None):
 231         abs_fname = os.path.abspath(fp.name)
 232         fname = fp.name
 233         self.fname = fname
 234         previously_included.append(abs_fname)
 235         self.incl_info = incl_info
 236         self.src = fp.read()
 237         if self.src == '' or self.src[-1] != '\n':
 238             self.src += '\n'
 239         self.cursor = 0
 240         self.line = 1
 241         self.line_pos = 0
 242         self.exprs = []
 243         self.docs = []
 244         self.accept()
 245
 246         while self.tok is not None:
 247             info = {'file': fname, 'line': self.line,
 248                     'parent': self.incl_info}
 249             if self.tok == '#':
 250                 doc = self.get_doc(info)
 251                 self.docs.append(doc)
 252                 continue
 253
 254             expr = self.get_expr(False)
 255             if 'include' in expr:
 256                 if len(expr) != 1:
 257                     raise QAPISemError(info, "Invalid 'include' directive")
 258                 include = expr["include"]
 259                 if not isinstance(include, str):
 260                     raise QAPISemError(info,
 261                                        "Value of 'include' must be a string")
 262                 self._include(include, info, os.path.dirname(abs_fname),
 263                               previously_included)
 264             elif "pragma" in expr:
 265                 if len(expr) != 1:
 266                     raise QAPISemError(info, "Invalid 'pragma' directive")
 267                 pragma = expr['pragma']
 268                 if not isinstance(pragma, dict):
 269                     raise QAPISemError(
 270                         info, "Value of 'pragma' must be a dictionary")
 271                 for name, value in pragma.iteritems():
 272                     self._pragma(name, value, info)
 273             else:
 274                 expr_elem = {'expr': expr,
 275                              'info': info}
 276                 if (self.docs
 277                         and self.docs[-1].info['file'] == fname
 278                         and not self.docs[-1].expr):
 279                     self.docs[-1].expr = expr
 280                     expr_elem['doc'] = self.docs[-1]
 281
 282                 self.exprs.append(expr_elem)
 283
 284     def _include(self, include, info, base_dir, previously_included):
 285         incl_abs_fname = os.path.join(base_dir, include)
 286         # catch inclusion cycle
 287         inf = info
 288         while inf:
 289             if incl_abs_fname == os.path.abspath(inf['file']):
 290                 raise QAPISemError(info, "Inclusion loop for %s" % include)
 291             inf = inf['parent']
 292
 293         # skip multiple include of the same file
 294         if incl_abs_fname in previously_included:
 295             return
 296         try:
 297             fobj = open(incl_abs_fname, 'r')
 298         except IOError as e:
 299             raise QAPISemError(info, '%s: %s' % (e.strerror, include))
 300         exprs_include = QAPISchemaParser(fobj, previously_included, info)
 301         self.exprs.extend(exprs_include.exprs)
 302         self.docs.extend(exprs_include.docs)
 303
 304     def _pragma(self, name, value, info):
 305         global doc_required, returns_whitelist
 306         if name == 'doc-required':
 307             if not isinstance(value, bool):
 308                 raise QAPISemError(info,
 309                                    "Pragma 'doc-required' must be boolean")
 310             doc_required = value
 311         elif name == 'returns-whitelist':
 312             if (not isinstance(value, list)
 313                     or any([not isinstance(elt, str) for elt in value])):
 314                 raise QAPISemError(info,
 315                                    "Pragma returns-whitelist must be"
 316                                    " a list of strings")
 317             returns_whitelist = value
 318         else:
 319             raise QAPISemError(info, "Unknown pragma '%s'" % name)
 320
 321     def accept(self, skip_comment=True):
 322         while True:
 323             self.tok = self.src[self.cursor]
 324             self.pos = self.cursor
 325             self.cursor += 1
 326             self.val = None
 327
 328             if self.tok == '#':
 329                 if self.src[self.cursor] == '#':
 330                     # Start of doc comment
 331                     skip_comment = False
 332                 self.cursor = self.src.find('\n', self.cursor)
 333                 if not skip_comment:
 334                     self.val = self.src[self.pos:self.cursor]
 335                     return
 336             elif self.tok in "{}:,[]":
 337                 return
 338             elif self.tok == "'":
 339                 string = ''
 340                 esc = False
 341                 while True:
 342                     ch = self.src[self.cursor]
 343                     self.cursor += 1
 344                     if ch == '\n':
 345                         raise QAPIParseError(self, 'Missing terminating "\'"')
 346                     if esc:
 347                         if ch == 'b':
 348                             string += '\b'
 349                         elif ch == 'f':
 350                             string += '\f'
 351                         elif ch == 'n':
 352                             string += '\n'
 353                         elif ch == 'r':
 354                             string += '\r'
 355                         elif ch == 't':
 356                             string += '\t'
 357                         elif ch == 'u':
 358                             value = 0
 359                             for _ in range(0, 4):
 360                                 ch = self.src[self.cursor]
 361                                 self.cursor += 1
 362                                 if ch not in "0123456789abcdefABCDEF":
 363                                     raise QAPIParseError(self,
 364                                                          '\\u escape needs 4 '
 365                                                          'hex digits')
 366                                 value = (value << 4) + int(ch, 16)
 367                             # If Python 2 and 3 didn't disagree so much on
 368                             # how to handle Unicode, then we could allow
 369                             # Unicode string defaults.  But most of QAPI is
 370                             # ASCII-only, so we aren't losing much for now.
 371                             if not value or value > 0x7f:
 372                                 raise QAPIParseError(self,
 373                                                      'For now, \\u escape '
 374                                                      'only supports non-zero '
 375                                                      'values up to \\u007f')
 376                             string += chr(value)
 377                         elif ch in "\\/'\"":
 378                             string += ch
 379                         else:
 380                             raise QAPIParseError(self,
 381                                                  "Unknown escape \\%s" % ch)
 382                         esc = False
 383                     elif ch == "\\":
 384                         esc = True
 385                     elif ch == "'":
 386                         self.val = string
 387                         return
 388                     else:
 389                         string += ch
 390             elif self.src.startswith("true", self.pos):
 391                 self.val = True
 392                 self.cursor += 3
 393                 return
 394             elif self.src.startswith("false", self.pos):
 395                 self.val = False
 396                 self.cursor += 4
 397                 return
 398             elif self.src.startswith("null", self.pos):
 399                 self.val = None
 400                 self.cursor += 3
 401                 return
 402             elif self.tok == '\n':
 403                 if self.cursor == len(self.src):
 404                     self.tok = None
 405                     return
 406                 self.line += 1
 407                 self.line_pos = self.cursor
 408             elif not self.tok.isspace():
 409                 raise QAPIParseError(self, 'Stray "%s"' % self.tok)
 410
 411     def get_members(self):
 412         expr = OrderedDict()
 413         if self.tok == '}':
 414             self.accept()
 415             return expr
 416         if self.tok != "'":
 417             raise QAPIParseError(self, 'Expected string or "}"')
 418         while True:
 419             key = self.val
 420             self.accept()
 421             if self.tok != ':':
 422                 raise QAPIParseError(self, 'Expected ":"')
 423             self.accept()
 424             if key in expr:
 425                 raise QAPIParseError(self, 'Duplicate key "%s"' % key)
 426             expr[key] = self.get_expr(True)
 427             if self.tok == '}':
 428                 self.accept()
 429                 return expr
 430             if self.tok != ',':
 431                 raise QAPIParseError(self, 'Expected "," or "}"')
 432             self.accept()
 433             if self.tok != "'":
 434                 raise QAPIParseError(self, 'Expected string')
 435
 436     def get_values(self):
 437         expr = []
 438         if self.tok == ']':
 439             self.accept()
 440             return expr
 441         if self.tok not in "{['tfn":
 442             raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
 443                                  'boolean or "null"')
 444         while True:
 445             expr.append(self.get_expr(True))
 446             if self.tok == ']':
 447                 self.accept()
 448                 return expr
 449             if self.tok != ',':
 450                 raise QAPIParseError(self, 'Expected "," or "]"')
 451             self.accept()
 452
 453     def get_expr(self, nested):
 454         if self.tok != '{' and not nested:
 455             raise QAPIParseError(self, 'Expected "{"')
 456         if self.tok == '{':
 457             self.accept()
 458             expr = self.get_members()
 459         elif self.tok == '[':
 460             self.accept()
 461             expr = self.get_values()
 462         elif self.tok in "'tfn":
 463             expr = self.val
 464             self.accept()
 465         else:
 466             raise QAPIParseError(self, 'Expected "{", "[" or string')
 467         return expr
 468
 469     def get_doc(self, info):
 470         if self.val != '##':
 471             raise QAPIParseError(self, "Junk after '##' at start of "
 472                                  "documentation comment")
 473
 474         doc = QAPIDoc(self, info)
 475         self.accept(False)
 476         while self.tok == '#':
 477             if self.val.startswith('##'):
 478                 # End of doc comment
 479                 if self.val != '##':
 480                     raise QAPIParseError(self, "Junk after '##' at end of "
 481                                          "documentation comment")
 482                 self.accept()
 483                 return doc
 484             else:
 485                 doc.append(self.val)
 486             self.accept(False)
 487
 488         raise QAPIParseError(self, "Documentation comment must end with '##'")
 489
 490
 491 #
 492 # Semantic analysis of schema expressions
 493 # TODO fold into QAPISchema
 494 # TODO catching name collisions in generated code would be nice
 495 #
 496
 497
 498 def find_base_members(base):
 499     if isinstance(base, dict):
 500         return base
 501     base_struct_define = find_struct(base)
 502     if not base_struct_define:
 503         return None
 504     return base_struct_define['data']
 505
 506
 507 # Return the qtype of an alternate branch, or None on error.
 508 def find_alternate_member_qtype(qapi_type):
 509     if qapi_type in builtin_types:
 510         return builtin_types[qapi_type]
 511     elif find_struct(qapi_type):
 512         return "QTYPE_QDICT"
 513     elif find_enum(qapi_type):
 514         return "QTYPE_QSTRING"
 515     elif find_union(qapi_type):
 516         return "QTYPE_QDICT"
 517     return None
 518
 519
 520 # Return the discriminator enum define if discriminator is specified as an
 521 # enum type, otherwise return None.
 522 def discriminator_find_enum_define(expr):
 523     base = expr.get('base')
 524     discriminator = expr.get('discriminator')
 525
 526     if not (discriminator and base):
 527         return None
 528
 529     base_members = find_base_members(base)
 530     if not base_members:
 531         return None
 532
 533     discriminator_type = base_members.get(discriminator)
 534     if not discriminator_type:
 535         return None
 536
 537     return find_enum(discriminator_type)
 538
 539
 540 # Names must be letters, numbers, -, and _.  They must start with letter,
 541 # except for downstream extensions which must start with __RFQDN_.
 542 # Dots are only valid in the downstream extension prefix.
 543 valid_name = re.compile('^(__[a-zA-Z0-9.-]+_)?'
 544                         '[a-zA-Z][a-zA-Z0-9_-]*$')
 545
 546
 547 def check_name(info, source, name, allow_optional=False,
 548                enum_member=False):
 549     global valid_name
 550     membername = name
 551
 552     if not isinstance(name, str):
 553         raise QAPISemError(info, "%s requires a string name" % source)
 554     if name.startswith('*'):
 555         membername = name[1:]
 556         if not allow_optional:
 557             raise QAPISemError(info, "%s does not allow optional name '%s'"
 558                                % (source, name))
 559     # Enum members can start with a digit, because the generated C
 560     # code always prefixes it with the enum name
 561     if enum_member and membername[0].isdigit():
 562         membername = 'D' + membername
 563     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
 564     # and 'q_obj_*' implicit type names.
 565     if not valid_name.match(membername) or \
 566        c_name(membername, False).startswith('q_'):
 567         raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
 568
 569
 570 def add_name(name, info, meta, implicit=False):
 571     global all_names
 572     check_name(info, "'%s'" % meta, name)
 573     # FIXME should reject names that differ only in '_' vs. '.'
 574     # vs. '-', because they're liable to clash in generated C.
 575     if name in all_names:
 576         raise QAPISemError(info, "%s '%s' is already defined"
 577                            % (all_names[name], name))
 578     if not implicit and (name.endswith('Kind') or name.endswith('List')):
 579         raise QAPISemError(info, "%s '%s' should not end in '%s'"
 580                            % (meta, name, name[-4:]))
 581     all_names[name] = meta
 582
 583
 584 def add_struct(definition, info):
 585     global struct_types
 586     name = definition['struct']
 587     add_name(name, info, 'struct')
 588     struct_types.append(definition)
 589
 590
 591 def find_struct(name):
 592     global struct_types
 593     for struct in struct_types:
 594         if struct['struct'] == name:
 595             return struct
 596     return None
 597
 598
 599 def add_union(definition, info):
 600     global union_types
 601     name = definition['union']
 602     add_name(name, info, 'union')
 603     union_types.append(definition)
 604
 605
 606 def find_union(name):
 607     global union_types
 608     for union in union_types:
 609         if union['union'] == name:
 610             return union
 611     return None
 612
 613
 614 def add_enum(name, info, enum_values=None, implicit=False):
 615     global enum_types
 616     add_name(name, info, 'enum', implicit)
 617     enum_types.append({"enum_name": name, "enum_values": enum_values})
 618
 619
 620 def find_enum(name):
 621     global enum_types
 622     for enum in enum_types:
 623         if enum['enum_name'] == name:
 624             return enum
 625     return None
 626
 627
 628 def is_enum(name):
 629     return find_enum(name) is not None
 630
 631
 632 def check_type(info, source, value, allow_array=False,
 633                allow_dict=False, allow_optional=False,
 634                allow_metas=[]):
 635     global all_names
 636
 637     if value is None:
 638         return
 639
 640     # Check if array type for value is okay
 641     if isinstance(value, list):
 642         if not allow_array:
 643             raise QAPISemError(info, "%s cannot be an array" % source)
 644         if len(value) != 1 or not isinstance(value[0], str):
 645             raise QAPISemError(info,
 646                                "%s: array type must contain single type name" %
 647                                source)
 648         value = value[0]
 649
 650     # Check if type name for value is okay
 651     if isinstance(value, str):
 652         if value not in all_names:
 653             raise QAPISemError(info, "%s uses unknown type '%s'"
 654                                % (source, value))
 655         if not all_names[value] in allow_metas:
 656             raise QAPISemError(info, "%s cannot use %s type '%s'" %
 657                                (source, all_names[value], value))
 658         return
 659
 660     if not allow_dict:
 661         raise QAPISemError(info, "%s should be a type name" % source)
 662
 663     if not isinstance(value, OrderedDict):
 664         raise QAPISemError(info,
 665                            "%s should be a dictionary or type name" % source)
 666
 667     # value is a dictionary, check that each member is okay
 668     for (key, arg) in value.items():
 669         check_name(info, "Member of %s" % source, key,
 670                    allow_optional=allow_optional)
 671         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 672             raise QAPISemError(info, "Member of %s uses reserved name '%s'"
 673                                % (source, key))
 674         # Todo: allow dictionaries to represent default values of
 675         # an optional argument.
 676         check_type(info, "Member '%s' of %s" % (key, source), arg,
 677                    allow_array=True,
 678                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 679                                 'enum'])
 680
 681
 682 def check_command(expr, info):
 683     name = expr['command']
 684     boxed = expr.get('boxed', False)
 685
 686     args_meta = ['struct']
 687     if boxed:
 688         args_meta += ['union', 'alternate']
 689     check_type(info, "'data' for command '%s'" % name,
 690                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 691                allow_metas=args_meta)
 692     returns_meta = ['union', 'struct']
 693     if name in returns_whitelist:
 694         returns_meta += ['built-in', 'alternate', 'enum']
 695     check_type(info, "'returns' for command '%s'" % name,
 696                expr.get('returns'), allow_array=True,
 697                allow_optional=True, allow_metas=returns_meta)
 698
 699
 700 def check_event(expr, info):
 701     global events
 702     name = expr['event']
 703     boxed = expr.get('boxed', False)
 704
 705     meta = ['struct']
 706     if boxed:
 707         meta += ['union', 'alternate']
 708     events.append(name)
 709     check_type(info, "'data' for event '%s'" % name,
 710                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 711                allow_metas=meta)
 712
 713
 714 def check_union(expr, info):
 715     name = expr['union']
 716     base = expr.get('base')
 717     discriminator = expr.get('discriminator')
 718     members = expr['data']
 719
 720     # Two types of unions, determined by discriminator.
 721
 722     # With no discriminator it is a simple union.
 723     if discriminator is None:
 724         enum_define = None
 725         allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
 726         if base is not None:
 727             raise QAPISemError(info, "Simple union '%s' must not have a base" %
 728                                name)
 729
 730     # Else, it's a flat union.
 731     else:
 732         # The object must have a string or dictionary 'base'.
 733         check_type(info, "'base' for union '%s'" % name,
 734                    base, allow_dict=True, allow_optional=True,
 735                    allow_metas=['struct'])
 736         if not base:
 737             raise QAPISemError(info, "Flat union '%s' must have a base"
 738                                % name)
 739         base_members = find_base_members(base)
 740         assert base_members
 741
 742         # The value of member 'discriminator' must name a non-optional
 743         # member of the base struct.
 744         check_name(info, "Discriminator of flat union '%s'" % name,
 745                    discriminator)
 746         discriminator_type = base_members.get(discriminator)
 747         if not discriminator_type:
 748             raise QAPISemError(info,
 749                                "Discriminator '%s' is not a member of base "
 750                                "struct '%s'"
 751                                % (discriminator, base))
 752         enum_define = find_enum(discriminator_type)
 753         allow_metas = ['struct']
 754         # Do not allow string discriminator
 755         if not enum_define:
 756             raise QAPISemError(info,
 757                                "Discriminator '%s' must be of enumeration "
 758                                "type" % discriminator)
 759
 760     # Check every branch; don't allow an empty union
 761     if len(members) == 0:
 762         raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
 763     for (key, value) in members.items():
 764         check_name(info, "Member of union '%s'" % name, key)
 765
 766         # Each value must name a known type
 767         check_type(info, "Member '%s' of union '%s'" % (key, name),
 768                    value, allow_array=not base, allow_metas=allow_metas)
 769
 770         # If the discriminator names an enum type, then all members
 771         # of 'data' must also be members of the enum type.
 772         if enum_define:
 773             if key not in enum_define['enum_values']:
 774                 raise QAPISemError(info,
 775                                    "Discriminator value '%s' is not found in "
 776                                    "enum '%s'"
 777                                    % (key, enum_define["enum_name"]))
 778
 779     # If discriminator is user-defined, ensure all values are covered
 780     if enum_define:
 781         for value in enum_define['enum_values']:
 782             if value not in members.keys():
 783                 raise QAPISemError(info, "Union '%s' data missing '%s' branch"
 784                                    % (name, value))
 785
 786
 787 def check_alternate(expr, info):
 788     name = expr['alternate']
 789     members = expr['data']
 790     types_seen = {}
 791
 792     # Check every branch; require at least two branches
 793     if len(members) < 2:
 794         raise QAPISemError(info,
 795                            "Alternate '%s' should have at least two branches "
 796                            "in 'data'" % name)
 797     for (key, value) in members.items():
 798         check_name(info, "Member of alternate '%s'" % name, key)
 799
 800         # Ensure alternates have no type conflicts.
 801         check_type(info, "Member '%s' of alternate '%s'" % (key, name),
 802                    value,
 803                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 804         qtype = find_alternate_member_qtype(value)
 805         if not qtype:
 806             raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
 807                                "type '%s'" % (name, key, value))
 808         if qtype in types_seen:
 809             raise QAPISemError(info, "Alternate '%s' member '%s' can't "
 810                                "be distinguished from member '%s'"
 811                                % (name, key, types_seen[qtype]))
 812         types_seen[qtype] = key
 813
 814
 815 def check_enum(expr, info):
 816     name = expr['enum']
 817     members = expr.get('data')
 818     prefix = expr.get('prefix')
 819
 820     if not isinstance(members, list):
 821         raise QAPISemError(info,
 822                            "Enum '%s' requires an array for 'data'" % name)
 823     if prefix is not None and not isinstance(prefix, str):
 824         raise QAPISemError(info,
 825                            "Enum '%s' requires a string for 'prefix'" % name)
 826     for member in members:
 827         check_name(info, "Member of enum '%s'" % name, member,
 828                    enum_member=True)
 829
 830
 831 def check_struct(expr, info):
 832     name = expr['struct']
 833     members = expr['data']
 834
 835     check_type(info, "'data' for struct '%s'" % name, members,
 836                allow_dict=True, allow_optional=True)
 837     check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
 838                allow_metas=['struct'])
 839
 840
 841 def check_keys(expr_elem, meta, required, optional=[]):
 842     expr = expr_elem['expr']
 843     info = expr_elem['info']
 844     name = expr[meta]
 845     if not isinstance(name, str):
 846         raise QAPISemError(info, "'%s' key must have a string value" % meta)
 847     required = required + [meta]
 848     for (key, value) in expr.items():
 849         if key not in required and key not in optional:
 850             raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
 851                                % (key, meta, name))
 852         if (key == 'gen' or key == 'success-response') and value is not False:
 853             raise QAPISemError(info,
 854                                "'%s' of %s '%s' should only use false value"
 855                                % (key, meta, name))
 856         if key == 'boxed' and value is not True:
 857             raise QAPISemError(info,
 858                                "'%s' of %s '%s' should only use true value"
 859                                % (key, meta, name))
 860     for key in required:
 861         if key not in expr:
 862             raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
 863                                % (key, meta, name))
 864
 865
 866 def check_exprs(exprs):
 867     global all_names
 868
 869     # Learn the types and check for valid expression keys
 870     for builtin in builtin_types.keys():
 871         all_names[builtin] = 'built-in'
 872     for expr_elem in exprs:
 873         expr = expr_elem['expr']
 874         info = expr_elem['info']
 875
 876         if 'doc' not in expr_elem and doc_required:
 877             raise QAPISemError(info,
 878                                "Expression missing documentation comment")
 879
 880         if 'enum' in expr:
 881             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 882             add_enum(expr['enum'], info, expr['data'])
 883         elif 'union' in expr:
 884             check_keys(expr_elem, 'union', ['data'],
 885                        ['base', 'discriminator'])
 886             add_union(expr, info)
 887         elif 'alternate' in expr:
 888             check_keys(expr_elem, 'alternate', ['data'])
 889             add_name(expr['alternate'], info, 'alternate')
 890         elif 'struct' in expr:
 891             check_keys(expr_elem, 'struct', ['data'], ['base'])
 892             add_struct(expr, info)
 893         elif 'command' in expr:
 894             check_keys(expr_elem, 'command', [],
 895                        ['data', 'returns', 'gen', 'success-response', 'boxed'])
 896             add_name(expr['command'], info, 'command')
 897         elif 'event' in expr:
 898             check_keys(expr_elem, 'event', [], ['data', 'boxed'])
 899             add_name(expr['event'], info, 'event')
 900         else:
 901             raise QAPISemError(expr_elem['info'],
 902                                "Expression is missing metatype")
 903
 904     # Try again for hidden UnionKind enum
 905     for expr_elem in exprs:
 906         expr = expr_elem['expr']
 907         if 'union' in expr:
 908             if not discriminator_find_enum_define(expr):
 909                 add_enum('%sKind' % expr['union'], expr_elem['info'],
 910                          implicit=True)
 911         elif 'alternate' in expr:
 912             add_enum('%sKind' % expr['alternate'], expr_elem['info'],
 913                      implicit=True)
 914
 915     # Validate that exprs make sense
 916     for expr_elem in exprs:
 917         expr = expr_elem['expr']
 918         info = expr_elem['info']
 919
 920         if 'enum' in expr:
 921             check_enum(expr, info)
 922         elif 'union' in expr:
 923             check_union(expr, info)
 924         elif 'alternate' in expr:
 925             check_alternate(expr, info)
 926         elif 'struct' in expr:
 927             check_struct(expr, info)
 928         elif 'command' in expr:
 929             check_command(expr, info)
 930         elif 'event' in expr:
 931             check_event(expr, info)
 932         else:
 933             assert False, 'unexpected meta type'
 934
 935     return exprs
 936
 937
 938 def check_freeform_doc(doc):
 939     if doc.symbol:
 940         raise QAPISemError(doc.info,
 941                            "Documention for '%s' is not followed"
 942                            " by the definition" % doc.symbol)
 943
 944     body = str(doc.body)
 945     if re.search(r'@\S+:', body, re.MULTILINE):
 946         raise QAPISemError(doc.info,
 947                            "Free-form documentation block must not contain"
 948                            " @NAME: sections")
 949
 950
 951 def check_definition_doc(doc, expr, info):
 952     for i in ('enum', 'union', 'alternate', 'struct', 'command', 'event'):
 953         if i in expr:
 954             meta = i
 955             break
 956
 957     name = expr[meta]
 958     if doc.symbol != name:
 959         raise QAPISemError(info, "Definition of '%s' follows documentation"
 960                            " for '%s'" % (name, doc.symbol))
 961     if doc.has_section('Returns') and 'command' not in expr:
 962         raise QAPISemError(info, "'Returns:' is only valid for commands")
 963
 964     if meta == 'union':
 965         args = expr.get('base', [])
 966     else:
 967         args = expr.get('data', [])
 968     if isinstance(args, str):
 969         return
 970     if isinstance(args, dict):
 971         args = args.keys()
 972     assert isinstance(args, list)
 973
 974     if (meta == 'alternate'
 975             or (meta == 'union' and not expr.get('discriminator'))):
 976         args.append('type')
 977
 978     for arg in args:
 979         if arg[0] == '*':
 980             opt = True
 981             desc = doc.args.get(arg[1:])
 982         else:
 983             opt = False
 984             desc = doc.args.get(arg)
 985         if not desc:
 986             continue
 987         desc_opt = "#optional" in str(desc)
 988         if desc_opt and not opt:
 989             raise QAPISemError(info, "Description has #optional, "
 990                                "but the declaration doesn't")
 991         if not desc_opt and opt:
 992             # silently fix the doc
 993             # TODO either fix the schema and make this an error,
 994             # or drop #optional entirely
 995             desc.append("#optional")
 996
 997     doc_args = set(doc.args.keys())
 998     args = set([name.strip('*') for name in args])
 999     if not doc_args.issubset(args):
1000         raise QAPISemError(info, "The following documented members are not in "
1001                            "the declaration: %s" % ", ".join(doc_args - args))
1002
1003
1004 def check_docs(docs):
1005     for doc in docs:
1006         for section in doc.args.values() + doc.sections:
1007             content = str(section)
1008             if not content or content.isspace():
1009                 raise QAPISemError(doc.info,
1010                                    "Empty doc section '%s'" % section.name)
1011
1012         if not doc.expr:
1013             check_freeform_doc(doc)
1014         else:
1015             check_definition_doc(doc, doc.expr, doc.info)
1016
1017     return docs
1018
1019
1020 #
1021 # Schema compiler frontend
1022 #
1023
1024 class QAPISchemaEntity(object):
1025     def __init__(self, name, info):
1026         assert isinstance(name, str)
1027         self.name = name
1028         # For explicitly defined entities, info points to the (explicit)
1029         # definition.  For builtins (and their arrays), info is None.
1030         # For implicitly defined entities, info points to a place that
1031         # triggered the implicit definition (there may be more than one
1032         # such place).
1033         self.info = info
1034
1035     def c_name(self):
1036         return c_name(self.name)
1037
1038     def check(self, schema):
1039         pass
1040
1041     def is_implicit(self):
1042         return not self.info
1043
1044     def visit(self, visitor):
1045         pass
1046
1047
1048 class QAPISchemaVisitor(object):
1049     def visit_begin(self, schema):
1050         pass
1051
1052     def visit_end(self):
1053         pass
1054
1055     def visit_needed(self, entity):
1056         # Default to visiting everything
1057         return True
1058
1059     def visit_builtin_type(self, name, info, json_type):
1060         pass
1061
1062     def visit_enum_type(self, name, info, values, prefix):
1063         pass
1064
1065     def visit_array_type(self, name, info, element_type):
1066         pass
1067
1068     def visit_object_type(self, name, info, base, members, variants):
1069         pass
1070
1071     def visit_object_type_flat(self, name, info, members, variants):
1072         pass
1073
1074     def visit_alternate_type(self, name, info, variants):
1075         pass
1076
1077     def visit_command(self, name, info, arg_type, ret_type,
1078                       gen, success_response, boxed):
1079         pass
1080
1081     def visit_event(self, name, info, arg_type, boxed):
1082         pass
1083
1084
1085 class QAPISchemaType(QAPISchemaEntity):
1086     # Return the C type for common use.
1087     # For the types we commonly box, this is a pointer type.
1088     def c_type(self):
1089         pass
1090
1091     # Return the C type to be used in a parameter list.
1092     def c_param_type(self):
1093         return self.c_type()
1094
1095     # Return the C type to be used where we suppress boxing.
1096     def c_unboxed_type(self):
1097         return self.c_type()
1098
1099     def json_type(self):
1100         pass
1101
1102     def alternate_qtype(self):
1103         json2qtype = {
1104             'string':  'QTYPE_QSTRING',
1105             'number':  'QTYPE_QFLOAT',
1106             'int':     'QTYPE_QINT',
1107             'boolean': 'QTYPE_QBOOL',
1108             'object':  'QTYPE_QDICT'
1109         }
1110         return json2qtype.get(self.json_type())
1111
1112
1113 class QAPISchemaBuiltinType(QAPISchemaType):
1114     def __init__(self, name, json_type, c_type):
1115         QAPISchemaType.__init__(self, name, None)
1116         assert not c_type or isinstance(c_type, str)
1117         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
1118                              'value')
1119         self._json_type_name = json_type
1120         self._c_type_name = c_type
1121
1122     def c_name(self):
1123         return self.name
1124
1125     def c_type(self):
1126         return self._c_type_name
1127
1128     def c_param_type(self):
1129         if self.name == 'str':
1130             return 'const ' + self._c_type_name
1131         return self._c_type_name
1132
1133     def json_type(self):
1134         return self._json_type_name
1135
1136     def visit(self, visitor):
1137         visitor.visit_builtin_type(self.name, self.info, self.json_type())
1138
1139
1140 class QAPISchemaEnumType(QAPISchemaType):
1141     def __init__(self, name, info, values, prefix):
1142         QAPISchemaType.__init__(self, name, info)
1143         for v in values:
1144             assert isinstance(v, QAPISchemaMember)
1145             v.set_owner(name)
1146         assert prefix is None or isinstance(prefix, str)
1147         self.values = values
1148         self.prefix = prefix
1149
1150     def check(self, schema):
1151         seen = {}
1152         for v in self.values:
1153             v.check_clash(self.info, seen)
1154
1155     def is_implicit(self):
1156         # See QAPISchema._make_implicit_enum_type()
1157         return self.name.endswith('Kind')
1158
1159     def c_type(self):
1160         return c_name(self.name)
1161
1162     def member_names(self):
1163         return [v.name for v in self.values]
1164
1165     def json_type(self):
1166         return 'string'
1167
1168     def visit(self, visitor):
1169         visitor.visit_enum_type(self.name, self.info,
1170                                 self.member_names(), self.prefix)
1171
1172
1173 class QAPISchemaArrayType(QAPISchemaType):
1174     def __init__(self, name, info, element_type):
1175         QAPISchemaType.__init__(self, name, info)
1176         assert isinstance(element_type, str)
1177         self._element_type_name = element_type
1178         self.element_type = None
1179
1180     def check(self, schema):
1181         self.element_type = schema.lookup_type(self._element_type_name)
1182         assert self.element_type
1183
1184     def is_implicit(self):
1185         return True
1186
1187     def c_type(self):
1188         return c_name(self.name) + pointer_suffix
1189
1190     def json_type(self):
1191         return 'array'
1192
1193     def visit(self, visitor):
1194         visitor.visit_array_type(self.name, self.info, self.element_type)
1195
1196
1197 class QAPISchemaObjectType(QAPISchemaType):
1198     def __init__(self, name, info, base, local_members, variants):
1199         # struct has local_members, optional base, and no variants
1200         # flat union has base, variants, and no local_members
1201         # simple union has local_members, variants, and no base
1202         QAPISchemaType.__init__(self, name, info)
1203         assert base is None or isinstance(base, str)
1204         for m in local_members:
1205             assert isinstance(m, QAPISchemaObjectTypeMember)
1206             m.set_owner(name)
1207         if variants is not None:
1208             assert isinstance(variants, QAPISchemaObjectTypeVariants)
1209             variants.set_owner(name)
1210         self._base_name = base
1211         self.base = None
1212         self.local_members = local_members
1213         self.variants = variants
1214         self.members = None
1215
1216     def check(self, schema):
1217         if self.members is False:               # check for cycles
1218             raise QAPISemError(self.info,
1219                                "Object %s contains itself" % self.name)
1220         if self.members:
1221             return
1222         self.members = False                    # mark as being checked
1223         seen = OrderedDict()
1224         if self._base_name:
1225             self.base = schema.lookup_type(self._base_name)
1226             assert isinstance(self.base, QAPISchemaObjectType)
1227             self.base.check(schema)
1228             self.base.check_clash(schema, self.info, seen)
1229         for m in self.local_members:
1230             m.check(schema)
1231             m.check_clash(self.info, seen)
1232         self.members = seen.values()
1233         if self.variants:
1234             self.variants.check(schema, seen)
1235             assert self.variants.tag_member in self.members
1236             self.variants.check_clash(schema, self.info, seen)
1237
1238     # Check that the members of this type do not cause duplicate JSON members,
1239     # and update seen to track the members seen so far. Report any errors
1240     # on behalf of info, which is not necessarily self.info
1241     def check_clash(self, schema, info, seen):
1242         assert not self.variants       # not implemented
1243         for m in self.members:
1244             m.check_clash(info, seen)
1245
1246     def is_implicit(self):
1247         # See QAPISchema._make_implicit_object_type(), as well as
1248         # _def_predefineds()
1249         return self.name.startswith('q_')
1250
1251     def is_empty(self):
1252         assert self.members is not None
1253         return not self.members and not self.variants
1254
1255     def c_name(self):
1256         assert self.name != 'q_empty'
1257         return QAPISchemaType.c_name(self)
1258
1259     def c_type(self):
1260         assert not self.is_implicit()
1261         return c_name(self.name) + pointer_suffix
1262
1263     def c_unboxed_type(self):
1264         return c_name(self.name)
1265
1266     def json_type(self):
1267         return 'object'
1268
1269     def visit(self, visitor):
1270         visitor.visit_object_type(self.name, self.info,
1271                                   self.base, self.local_members, self.variants)
1272         visitor.visit_object_type_flat(self.name, self.info,
1273                                        self.members, self.variants)
1274
1275
1276 class QAPISchemaMember(object):
1277     role = 'member'
1278
1279     def __init__(self, name):
1280         assert isinstance(name, str)
1281         self.name = name
1282         self.owner = None
1283
1284     def set_owner(self, name):
1285         assert not self.owner
1286         self.owner = name
1287
1288     def check_clash(self, info, seen):
1289         cname = c_name(self.name)
1290         if cname.lower() != cname and self.owner not in case_whitelist:
1291             raise QAPISemError(info,
1292                                "%s should not use uppercase" % self.describe())
1293         if cname in seen:
1294             raise QAPISemError(info, "%s collides with %s" %
1295                                (self.describe(), seen[cname].describe()))
1296         seen[cname] = self
1297
1298     def _pretty_owner(self):
1299         owner = self.owner
1300         if owner.startswith('q_obj_'):
1301             # See QAPISchema._make_implicit_object_type() - reverse the
1302             # mapping there to create a nice human-readable description
1303             owner = owner[6:]
1304             if owner.endswith('-arg'):
1305                 return '(parameter of %s)' % owner[:-4]
1306             elif owner.endswith('-base'):
1307                 return '(base of %s)' % owner[:-5]
1308             else:
1309                 assert owner.endswith('-wrapper')
1310                 # Unreachable and not implemented
1311                 assert False
1312         if owner.endswith('Kind'):
1313             # See QAPISchema._make_implicit_enum_type()
1314             return '(branch of %s)' % owner[:-4]
1315         return '(%s of %s)' % (self.role, owner)
1316
1317     def describe(self):
1318         return "'%s' %s" % (self.name, self._pretty_owner())
1319
1320
1321 class QAPISchemaObjectTypeMember(QAPISchemaMember):
1322     def __init__(self, name, typ, optional):
1323         QAPISchemaMember.__init__(self, name)
1324         assert isinstance(typ, str)
1325         assert isinstance(optional, bool)
1326         self._type_name = typ
1327         self.type = None
1328         self.optional = optional
1329
1330     def check(self, schema):
1331         assert self.owner
1332         self.type = schema.lookup_type(self._type_name)
1333         assert self.type
1334
1335
1336 class QAPISchemaObjectTypeVariants(object):
1337     def __init__(self, tag_name, tag_member, variants):
1338         # Flat unions pass tag_name but not tag_member.
1339         # Simple unions and alternates pass tag_member but not tag_name.
1340         # After check(), tag_member is always set, and tag_name remains
1341         # a reliable witness of being used by a flat union.
1342         assert bool(tag_member) != bool(tag_name)
1343         assert (isinstance(tag_name, str) or
1344                 isinstance(tag_member, QAPISchemaObjectTypeMember))
1345         assert len(variants) > 0
1346         for v in variants:
1347             assert isinstance(v, QAPISchemaObjectTypeVariant)
1348         self._tag_name = tag_name
1349         self.tag_member = tag_member
1350         self.variants = variants
1351
1352     def set_owner(self, name):
1353         for v in self.variants:
1354             v.set_owner(name)
1355
1356     def check(self, schema, seen):
1357         if not self.tag_member:    # flat union
1358             self.tag_member = seen[c_name(self._tag_name)]
1359             assert self._tag_name == self.tag_member.name
1360         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
1361         for v in self.variants:
1362             v.check(schema)
1363             # Union names must match enum values; alternate names are
1364             # checked separately. Use 'seen' to tell the two apart.
1365             if seen:
1366                 assert v.name in self.tag_member.type.member_names()
1367                 assert isinstance(v.type, QAPISchemaObjectType)
1368                 v.type.check(schema)
1369
1370     def check_clash(self, schema, info, seen):
1371         for v in self.variants:
1372             # Reset seen map for each variant, since qapi names from one
1373             # branch do not affect another branch
1374             assert isinstance(v.type, QAPISchemaObjectType)
1375             v.type.check_clash(schema, info, dict(seen))
1376
1377
1378 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1379     role = 'branch'
1380
1381     def __init__(self, name, typ):
1382         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1383
1384
1385 class QAPISchemaAlternateType(QAPISchemaType):
1386     def __init__(self, name, info, variants):
1387         QAPISchemaType.__init__(self, name, info)
1388         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1389         assert variants.tag_member
1390         variants.set_owner(name)
1391         variants.tag_member.set_owner(self.name)
1392         self.variants = variants
1393
1394     def check(self, schema):
1395         self.variants.tag_member.check(schema)
1396         # Not calling self.variants.check_clash(), because there's nothing
1397         # to clash with
1398         self.variants.check(schema, {})
1399         # Alternate branch names have no relation to the tag enum values;
1400         # so we have to check for potential name collisions ourselves.
1401         seen = {}
1402         for v in self.variants.variants:
1403             v.check_clash(self.info, seen)
1404
1405     def c_type(self):
1406         return c_name(self.name) + pointer_suffix
1407
1408     def json_type(self):
1409         return 'value'
1410
1411     def visit(self, visitor):
1412         visitor.visit_alternate_type(self.name, self.info, self.variants)
1413
1414     def is_empty(self):
1415         return False
1416
1417
1418 class QAPISchemaCommand(QAPISchemaEntity):
1419     def __init__(self, name, info, arg_type, ret_type, gen, success_response,
1420                  boxed):
1421         QAPISchemaEntity.__init__(self, name, info)
1422         assert not arg_type or isinstance(arg_type, str)
1423         assert not ret_type or isinstance(ret_type, str)
1424         self._arg_type_name = arg_type
1425         self.arg_type = None
1426         self._ret_type_name = ret_type
1427         self.ret_type = None
1428         self.gen = gen
1429         self.success_response = success_response
1430         self.boxed = boxed
1431
1432     def check(self, schema):
1433         if self._arg_type_name:
1434             self.arg_type = schema.lookup_type(self._arg_type_name)
1435             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1436                     isinstance(self.arg_type, QAPISchemaAlternateType))
1437             self.arg_type.check(schema)
1438             if self.boxed:
1439                 if self.arg_type.is_empty():
1440                     raise QAPISemError(self.info,
1441                                        "Cannot use 'boxed' with empty type")
1442             else:
1443                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1444                 assert not self.arg_type.variants
1445         elif self.boxed:
1446             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1447         if self._ret_type_name:
1448             self.ret_type = schema.lookup_type(self._ret_type_name)
1449             assert isinstance(self.ret_type, QAPISchemaType)
1450
1451     def visit(self, visitor):
1452         visitor.visit_command(self.name, self.info,
1453                               self.arg_type, self.ret_type,
1454                               self.gen, self.success_response, self.boxed)
1455
1456
1457 class QAPISchemaEvent(QAPISchemaEntity):
1458     def __init__(self, name, info, arg_type, boxed):
1459         QAPISchemaEntity.__init__(self, name, info)
1460         assert not arg_type or isinstance(arg_type, str)
1461         self._arg_type_name = arg_type
1462         self.arg_type = None
1463         self.boxed = boxed
1464
1465     def check(self, schema):
1466         if self._arg_type_name:
1467             self.arg_type = schema.lookup_type(self._arg_type_name)
1468             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1469                     isinstance(self.arg_type, QAPISchemaAlternateType))
1470             self.arg_type.check(schema)
1471             if self.boxed:
1472                 if self.arg_type.is_empty():
1473                     raise QAPISemError(self.info,
1474                                        "Cannot use 'boxed' with empty type")
1475             else:
1476                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1477                 assert not self.arg_type.variants
1478         elif self.boxed:
1479             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1480
1481     def visit(self, visitor):
1482         visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
1483
1484
1485 class QAPISchema(object):
1486     def __init__(self, fname):
1487         try:
1488             parser = QAPISchemaParser(open(fname, "r"))
1489             self.exprs = check_exprs(parser.exprs)
1490             self.docs = check_docs(parser.docs)
1491             self._entity_dict = {}
1492             self._predefining = True
1493             self._def_predefineds()
1494             self._predefining = False
1495             self._def_exprs()
1496             self.check()
1497         except QAPIError as err:
1498             print >>sys.stderr, err
1499             exit(1)
1500
1501     def _def_entity(self, ent):
1502         # Only the predefined types are allowed to not have info
1503         assert ent.info or self._predefining
1504         assert ent.name not in self._entity_dict
1505         self._entity_dict[ent.name] = ent
1506
1507     def lookup_entity(self, name, typ=None):
1508         ent = self._entity_dict.get(name)
1509         if typ and not isinstance(ent, typ):
1510             return None
1511         return ent
1512
1513     def lookup_type(self, name):
1514         return self.lookup_entity(name, QAPISchemaType)
1515
1516     def _def_builtin_type(self, name, json_type, c_type):
1517         self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
1518         # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
1519         # qapi-types.h from a single .c, all arrays of builtins must be
1520         # declared in the first file whether or not they are used.  Nicer
1521         # would be to use lazy instantiation, while figuring out how to
1522         # avoid compilation issues with multiple qapi-types.h.
1523         self._make_array_type(name, None)
1524
1525     def _def_predefineds(self):
1526         for t in [('str',    'string',  'char' + pointer_suffix),
1527                   ('number', 'number',  'double'),
1528                   ('int',    'int',     'int64_t'),
1529                   ('int8',   'int',     'int8_t'),
1530                   ('int16',  'int',     'int16_t'),
1531                   ('int32',  'int',     'int32_t'),
1532                   ('int64',  'int',     'int64_t'),
1533                   ('uint8',  'int',     'uint8_t'),
1534                   ('uint16', 'int',     'uint16_t'),
1535                   ('uint32', 'int',     'uint32_t'),
1536                   ('uint64', 'int',     'uint64_t'),
1537                   ('size',   'int',     'uint64_t'),
1538                   ('bool',   'boolean', 'bool'),
1539                   ('any',    'value',   'QObject' + pointer_suffix)]:
1540             self._def_builtin_type(*t)
1541         self.the_empty_object_type = QAPISchemaObjectType('q_empty', None,
1542                                                           None, [], None)
1543         self._def_entity(self.the_empty_object_type)
1544         qtype_values = self._make_enum_members(['none', 'qnull', 'qint',
1545                                                 'qstring', 'qdict', 'qlist',
1546                                                 'qfloat', 'qbool'])
1547         self._def_entity(QAPISchemaEnumType('QType', None, qtype_values,
1548                                             'QTYPE'))
1549
1550     def _make_enum_members(self, values):
1551         return [QAPISchemaMember(v) for v in values]
1552
1553     def _make_implicit_enum_type(self, name, info, values):
1554         # See also QAPISchemaObjectTypeMember._pretty_owner()
1555         name = name + 'Kind'   # Use namespace reserved by add_name()
1556         self._def_entity(QAPISchemaEnumType(
1557             name, info, self._make_enum_members(values), None))
1558         return name
1559
1560     def _make_array_type(self, element_type, info):
1561         name = element_type + 'List'   # Use namespace reserved by add_name()
1562         if not self.lookup_type(name):
1563             self._def_entity(QAPISchemaArrayType(name, info, element_type))
1564         return name
1565
1566     def _make_implicit_object_type(self, name, info, role, members):
1567         if not members:
1568             return None
1569         # See also QAPISchemaObjectTypeMember._pretty_owner()
1570         name = 'q_obj_%s-%s' % (name, role)
1571         if not self.lookup_entity(name, QAPISchemaObjectType):
1572             self._def_entity(QAPISchemaObjectType(name, info, None,
1573                                                   members, None))
1574         return name
1575
1576     def _def_enum_type(self, expr, info):
1577         name = expr['enum']
1578         data = expr['data']
1579         prefix = expr.get('prefix')
1580         self._def_entity(QAPISchemaEnumType(
1581             name, info, self._make_enum_members(data), prefix))
1582
1583     def _make_member(self, name, typ, info):
1584         optional = False
1585         if name.startswith('*'):
1586             name = name[1:]
1587             optional = True
1588         if isinstance(typ, list):
1589             assert len(typ) == 1
1590             typ = self._make_array_type(typ[0], info)
1591         return QAPISchemaObjectTypeMember(name, typ, optional)
1592
1593     def _make_members(self, data, info):
1594         return [self._make_member(key, value, info)
1595                 for (key, value) in data.iteritems()]
1596
1597     def _def_struct_type(self, expr, info):
1598         name = expr['struct']
1599         base = expr.get('base')
1600         data = expr['data']
1601         self._def_entity(QAPISchemaObjectType(name, info, base,
1602                                               self._make_members(data, info),
1603                                               None))
1604
1605     def _make_variant(self, case, typ):
1606         return QAPISchemaObjectTypeVariant(case, typ)
1607
1608     def _make_simple_variant(self, case, typ, info):
1609         if isinstance(typ, list):
1610             assert len(typ) == 1
1611             typ = self._make_array_type(typ[0], info)
1612         typ = self._make_implicit_object_type(
1613             typ, info, 'wrapper', [self._make_member('data', typ, info)])
1614         return QAPISchemaObjectTypeVariant(case, typ)
1615
1616     def _def_union_type(self, expr, info):
1617         name = expr['union']
1618         data = expr['data']
1619         base = expr.get('base')
1620         tag_name = expr.get('discriminator')
1621         tag_member = None
1622         if isinstance(base, dict):
1623             base = (self._make_implicit_object_type(
1624                     name, info, 'base', self._make_members(base, info)))
1625         if tag_name:
1626             variants = [self._make_variant(key, value)
1627                         for (key, value) in data.iteritems()]
1628             members = []
1629         else:
1630             variants = [self._make_simple_variant(key, value, info)
1631                         for (key, value) in data.iteritems()]
1632             typ = self._make_implicit_enum_type(name, info,
1633                                                 [v.name for v in variants])
1634             tag_member = QAPISchemaObjectTypeMember('type', typ, False)
1635             members = [tag_member]
1636         self._def_entity(
1637             QAPISchemaObjectType(name, info, base, members,
1638                                  QAPISchemaObjectTypeVariants(tag_name,
1639                                                               tag_member,
1640                                                               variants)))
1641
1642     def _def_alternate_type(self, expr, info):
1643         name = expr['alternate']
1644         data = expr['data']
1645         variants = [self._make_variant(key, value)
1646                     for (key, value) in data.iteritems()]
1647         tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
1648         self._def_entity(
1649             QAPISchemaAlternateType(name, info,
1650                                     QAPISchemaObjectTypeVariants(None,
1651                                                                  tag_member,
1652                                                                  variants)))
1653
1654     def _def_command(self, expr, info):
1655         name = expr['command']
1656         data = expr.get('data')
1657         rets = expr.get('returns')
1658         gen = expr.get('gen', True)
1659         success_response = expr.get('success-response', True)
1660         boxed = expr.get('boxed', False)
1661         if isinstance(data, OrderedDict):
1662             data = self._make_implicit_object_type(
1663                 name, info, 'arg', self._make_members(data, info))
1664         if isinstance(rets, list):
1665             assert len(rets) == 1
1666             rets = self._make_array_type(rets[0], info)
1667         self._def_entity(QAPISchemaCommand(name, info, data, rets, gen,
1668                                            success_response, boxed))
1669
1670     def _def_event(self, expr, info):
1671         name = expr['event']
1672         data = expr.get('data')
1673         boxed = expr.get('boxed', False)
1674         if isinstance(data, OrderedDict):
1675             data = self._make_implicit_object_type(
1676                 name, info, 'arg', self._make_members(data, info))
1677         self._def_entity(QAPISchemaEvent(name, info, data, boxed))
1678
1679     def _def_exprs(self):
1680         for expr_elem in self.exprs:
1681             expr = expr_elem['expr']
1682             info = expr_elem['info']
1683             if 'enum' in expr:
1684                 self._def_enum_type(expr, info)
1685             elif 'struct' in expr:
1686                 self._def_struct_type(expr, info)
1687             elif 'union' in expr:
1688                 self._def_union_type(expr, info)
1689             elif 'alternate' in expr:
1690                 self._def_alternate_type(expr, info)
1691             elif 'command' in expr:
1692                 self._def_command(expr, info)
1693             elif 'event' in expr:
1694                 self._def_event(expr, info)
1695             else:
1696                 assert False
1697
1698     def check(self):
1699         for ent in self._entity_dict.values():
1700             ent.check(self)
1701
1702     def visit(self, visitor):
1703         visitor.visit_begin(self)
1704         for (name, entity) in sorted(self._entity_dict.items()):
1705             if visitor.visit_needed(entity):
1706                 entity.visit(visitor)
1707         visitor.visit_end()
1708
1709
1710 #
1711 # Code generation helpers
1712 #
1713
1714 def camel_case(name):
1715     new_name = ''
1716     first = True
1717     for ch in name:
1718         if ch in ['_', '-']:
1719             first = True
1720         elif first:
1721             new_name += ch.upper()
1722             first = False
1723         else:
1724             new_name += ch.lower()
1725     return new_name
1726
1727
1728 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1729 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1730 # ENUM24_Name -> ENUM24_NAME
1731 def camel_to_upper(value):
1732     c_fun_str = c_name(value, False)
1733     if value.isupper():
1734         return c_fun_str
1735
1736     new_name = ''
1737     l = len(c_fun_str)
1738     for i in range(l):
1739         c = c_fun_str[i]
1740         # When c is upper and no "_" appears before, do more checks
1741         if c.isupper() and (i > 0) and c_fun_str[i - 1] != "_":
1742             if i < l - 1 and c_fun_str[i + 1].islower():
1743                 new_name += '_'
1744             elif c_fun_str[i - 1].isdigit():
1745                 new_name += '_'
1746         new_name += c
1747     return new_name.lstrip('_').upper()
1748
1749
1750 def c_enum_const(type_name, const_name, prefix=None):
1751     if prefix is not None:
1752         type_name = prefix
1753     return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
1754
1755 c_name_trans = string.maketrans('.-', '__')
1756
1757
1758 # Map @name to a valid C identifier.
1759 # If @protect, avoid returning certain ticklish identifiers (like
1760 # C keywords) by prepending "q_".
1761 #
1762 # Used for converting 'name' from a 'name':'type' qapi definition
1763 # into a generated struct member, as well as converting type names
1764 # into substrings of a generated C function name.
1765 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1766 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1767 def c_name(name, protect=True):
1768     # ANSI X3J11/88-090, 3.1.1
1769     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1770                      'default', 'do', 'double', 'else', 'enum', 'extern',
1771                      'float', 'for', 'goto', 'if', 'int', 'long', 'register',
1772                      'return', 'short', 'signed', 'sizeof', 'static',
1773                      'struct', 'switch', 'typedef', 'union', 'unsigned',
1774                      'void', 'volatile', 'while'])
1775     # ISO/IEC 9899:1999, 6.4.1
1776     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1777     # ISO/IEC 9899:2011, 6.4.1
1778     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
1779                      '_Noreturn', '_Static_assert', '_Thread_local'])
1780     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1781     # excluding _.*
1782     gcc_words = set(['asm', 'typeof'])
1783     # C++ ISO/IEC 14882:2003 2.11
1784     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1785                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1786                      'namespace', 'new', 'operator', 'private', 'protected',
1787                      'public', 'reinterpret_cast', 'static_cast', 'template',
1788                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1789                      'using', 'virtual', 'wchar_t',
1790                      # alternative representations
1791                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1792                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1793     # namespace pollution:
1794     polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
1795     name = name.translate(c_name_trans)
1796     if protect and (name in c89_words | c99_words | c11_words | gcc_words
1797                     | cpp_words | polluted_words):
1798         return "q_" + name
1799     return name
1800
1801 eatspace = '\033EATSPACE.'
1802 pointer_suffix = ' *' + eatspace
1803
1804
1805 def genindent(count):
1806     ret = ""
1807     for _ in range(count):
1808         ret += " "
1809     return ret
1810
1811 indent_level = 0
1812
1813
1814 def push_indent(indent_amount=4):
1815     global indent_level
1816     indent_level += indent_amount
1817
1818
1819 def pop_indent(indent_amount=4):
1820     global indent_level
1821     indent_level -= indent_amount
1822
1823
1824 # Generate @code with @kwds interpolated.
1825 # Obey indent_level, and strip eatspace.
1826 def cgen(code, **kwds):
1827     raw = code % kwds
1828     if indent_level:
1829         indent = genindent(indent_level)
1830         # re.subn() lacks flags support before Python 2.7, use re.compile()
1831         raw = re.subn(re.compile("^.", re.MULTILINE),
1832                       indent + r'\g<0>', raw)
1833         raw = raw[0]
1834     return re.sub(re.escape(eatspace) + ' *', '', raw)
1835
1836
1837 def mcgen(code, **kwds):
1838     if code[0] == '\n':
1839         code = code[1:]
1840     return cgen(code, **kwds)
1841
1842
1843 def guardname(filename):
1844     return c_name(filename, protect=False).upper()
1845
1846
1847 def guardstart(name):
1848     return mcgen('''
1849
1850 #ifndef %(name)s
1851 #define %(name)s
1852
1853 ''',
1854                  name=guardname(name))
1855
1856
1857 def guardend(name):
1858     return mcgen('''
1859
1860 #endif /* %(name)s */
1861
1862 ''',
1863                  name=guardname(name))
1864
1865
1866 def gen_enum_lookup(name, values, prefix=None):
1867     ret = mcgen('''
1868
1869 const char *const %(c_name)s_lookup[] = {
1870 ''',
1871                 c_name=c_name(name))
1872     for value in values:
1873         index = c_enum_const(name, value, prefix)
1874         ret += mcgen('''
1875     [%(index)s] = "%(value)s",
1876 ''',
1877                      index=index, value=value)
1878
1879     max_index = c_enum_const(name, '_MAX', prefix)
1880     ret += mcgen('''
1881     [%(max_index)s] = NULL,
1882 };
1883 ''',
1884                  max_index=max_index)
1885     return ret
1886
1887
1888 def gen_enum(name, values, prefix=None):
1889     # append automatically generated _MAX value
1890     enum_values = values + ['_MAX']
1891
1892     ret = mcgen('''
1893
1894 typedef enum %(c_name)s {
1895 ''',
1896                 c_name=c_name(name))
1897
1898     i = 0
1899     for value in enum_values:
1900         ret += mcgen('''
1901     %(c_enum)s = %(i)d,
1902 ''',
1903                      c_enum=c_enum_const(name, value, prefix),
1904                      i=i)
1905         i += 1
1906
1907     ret += mcgen('''
1908 } %(c_name)s;
1909 ''',
1910                  c_name=c_name(name))
1911
1912     ret += mcgen('''
1913
1914 extern const char *const %(c_name)s_lookup[];
1915 ''',
1916                  c_name=c_name(name))
1917     return ret
1918
1919
1920 def gen_params(arg_type, boxed, extra):
1921     if not arg_type:
1922         assert not boxed
1923         return extra
1924     ret = ''
1925     sep = ''
1926     if boxed:
1927         ret += '%s arg' % arg_type.c_param_type()
1928         sep = ', '
1929     else:
1930         assert not arg_type.variants
1931         for memb in arg_type.members:
1932             ret += sep
1933             sep = ', '
1934             if memb.optional:
1935                 ret += 'bool has_%s, ' % c_name(memb.name)
1936             ret += '%s %s' % (memb.type.c_param_type(),
1937                               c_name(memb.name))
1938     if extra:
1939         ret += sep + extra
1940     return ret
1941
1942
1943 #
1944 # Common command line parsing
1945 #
1946
1947
1948 def parse_command_line(extra_options="", extra_long_options=[]):
1949
1950     try:
1951         opts, args = getopt.gnu_getopt(sys.argv[1:],
1952                                        "chp:o:" + extra_options,
1953                                        ["source", "header", "prefix=",
1954                                         "output-dir="] + extra_long_options)
1955     except getopt.GetoptError as err:
1956         print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
1957         sys.exit(1)
1958
1959     output_dir = ""
1960     prefix = ""
1961     do_c = False
1962     do_h = False
1963     extra_opts = []
1964
1965     for oa in opts:
1966         o, a = oa
1967         if o in ("-p", "--prefix"):
1968             match = re.match('([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1969             if match.end() != len(a):
1970                 print >>sys.stderr, \
1971                     "%s: 'funny character '%s' in argument of --prefix" \
1972                     % (sys.argv[0], a[match.end()])
1973                 sys.exit(1)
1974             prefix = a
1975         elif o in ("-o", "--output-dir"):
1976             output_dir = a + "/"
1977         elif o in ("-c", "--source"):
1978             do_c = True
1979         elif o in ("-h", "--header"):
1980             do_h = True
1981         else:
1982             extra_opts.append(oa)
1983
1984     if not do_c and not do_h:
1985         do_c = True
1986         do_h = True
1987
1988     if len(args) != 1:
1989         print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
1990         sys.exit(1)
1991     fname = args[0]
1992
1993     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
1994
1995 #
1996 # Generate output files with boilerplate
1997 #
1998
1999
2000 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
2001                 c_comment, h_comment):
2002     guard = guardname(prefix + h_file)
2003     c_file = output_dir + prefix + c_file
2004     h_file = output_dir + prefix + h_file
2005
2006     if output_dir:
2007         try:
2008             os.makedirs(output_dir)
2009         except os.error as e:
2010             if e.errno != errno.EEXIST:
2011                 raise
2012
2013     def maybe_open(really, name, opt):
2014         if really:
2015             return open(name, opt)
2016         else:
2017             import StringIO
2018             return StringIO.StringIO()
2019
2020     fdef = maybe_open(do_c, c_file, 'w')
2021     fdecl = maybe_open(do_h, h_file, 'w')
2022
2023     fdef.write(mcgen('''
2024 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2025 %(comment)s
2026 ''',
2027                      comment=c_comment))
2028
2029     fdecl.write(mcgen('''
2030 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2031 %(comment)s
2032 #ifndef %(guard)s
2033 #define %(guard)s
2034
2035 ''',
2036                       comment=h_comment, guard=guard))
2037
2038     return (fdef, fdecl)
2039
2040
2041 def close_output(fdef, fdecl):
2042     fdecl.write('''
2043 #endif
2044 ''')
2045     fdecl.close()
2046     fdef.close()