scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 import errno
  15 import getopt
  16 import os
  17 import re
  18 import string
  19 import sys
  20 from ordereddict import OrderedDict
  21
  22 builtin_types = {
  23     'null':     'QTYPE_QNULL',
  24     'str':      'QTYPE_QSTRING',
  25     'int':      'QTYPE_QNUM',
  26     'number':   'QTYPE_QNUM',
  27     'bool':     'QTYPE_QBOOL',
  28     'int8':     'QTYPE_QNUM',
  29     'int16':    'QTYPE_QNUM',
  30     'int32':    'QTYPE_QNUM',
  31     'int64':    'QTYPE_QNUM',
  32     'uint8':    'QTYPE_QNUM',
  33     'uint16':   'QTYPE_QNUM',
  34     'uint32':   'QTYPE_QNUM',
  35     'uint64':   'QTYPE_QNUM',
  36     'size':     'QTYPE_QNUM',
  37     'any':      None,           # any QType possible, actually
  38     'QType':    'QTYPE_QSTRING',
  39 }
  40
  41 # Are documentation comments required?
  42 doc_required = False
  43
  44 # Whitelist of commands allowed to return a non-dictionary
  45 returns_whitelist = []
  46
  47 # Whitelist of entities allowed to violate case conventions
  48 name_case_whitelist = []
  49
  50 enum_types = {}
  51 struct_types = {}
  52 union_types = {}
  53 all_names = {}
  54
  55 #
  56 # Parsing the schema into expressions
  57 #
  58
  59
  60 def error_path(parent):
  61     res = ''
  62     while parent:
  63         res = ('In file included from %s:%d:\n' % (parent['file'],
  64                                                    parent['line'])) + res
  65         parent = parent['parent']
  66     return res
  67
  68
  69 class QAPIError(Exception):
  70     def __init__(self, fname, line, col, incl_info, msg):
  71         Exception.__init__(self)
  72         self.fname = fname
  73         self.line = line
  74         self.col = col
  75         self.info = incl_info
  76         self.msg = msg
  77
  78     def __str__(self):
  79         loc = '%s:%d' % (self.fname, self.line)
  80         if self.col is not None:
  81             loc += ':%s' % self.col
  82         return error_path(self.info) + '%s: %s' % (loc, self.msg)
  83
  84
  85 class QAPIParseError(QAPIError):
  86     def __init__(self, parser, msg):
  87         col = 1
  88         for ch in parser.src[parser.line_pos:parser.pos]:
  89             if ch == '\t':
  90                 col = (col + 7) % 8 + 1
  91             else:
  92                 col += 1
  93         QAPIError.__init__(self, parser.fname, parser.line, col,
  94                            parser.incl_info, msg)
  95
  96
  97 class QAPISemError(QAPIError):
  98     def __init__(self, info, msg):
  99         QAPIError.__init__(self, info['file'], info['line'], None,
 100                            info['parent'], msg)
 101
 102
 103 class QAPIDoc(object):
 104     class Section(object):
 105         def __init__(self, name=None):
 106             # optional section name (argument/member or section name)
 107             self.name = name
 108             # the list of lines for this section
 109             self.content = []
 110
 111         def append(self, line):
 112             self.content.append(line)
 113
 114         def __repr__(self):
 115             return '\n'.join(self.content).strip()
 116
 117     class ArgSection(Section):
 118         def __init__(self, name):
 119             QAPIDoc.Section.__init__(self, name)
 120             self.member = None
 121
 122         def connect(self, member):
 123             self.member = member
 124
 125     def __init__(self, parser, info):
 126         # self.parser is used to report errors with QAPIParseError.  The
 127         # resulting error position depends on the state of the parser.
 128         # It happens to be the beginning of the comment.  More or less
 129         # servicable, but action at a distance.
 130         self.parser = parser
 131         self.info = info
 132         self.symbol = None
 133         self.body = QAPIDoc.Section()
 134         # dict mapping parameter name to ArgSection
 135         self.args = OrderedDict()
 136         # a list of Section
 137         self.sections = []
 138         # the current section
 139         self.section = self.body
 140
 141     def has_section(self, name):
 142         """Return True if we have a section with this name."""
 143         for i in self.sections:
 144             if i.name == name:
 145                 return True
 146         return False
 147
 148     def append(self, line):
 149         """Parse a comment line and add it to the documentation."""
 150         line = line[1:]
 151         if not line:
 152             self._append_freeform(line)
 153             return
 154
 155         if line[0] != ' ':
 156             raise QAPIParseError(self.parser, "Missing space after #")
 157         line = line[1:]
 158
 159         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 160         # recognized, and get silently treated as ordinary text
 161         if self.symbol:
 162             self._append_symbol_line(line)
 163         elif not self.body.content and line.startswith('@'):
 164             if not line.endswith(':'):
 165                 raise QAPIParseError(self.parser, "Line should end with :")
 166             self.symbol = line[1:-1]
 167             # FIXME invalid names other than the empty string aren't flagged
 168             if not self.symbol:
 169                 raise QAPIParseError(self.parser, "Invalid name")
 170         else:
 171             self._append_freeform(line)
 172
 173     def end_comment(self):
 174         self._end_section()
 175
 176     def _append_symbol_line(self, line):
 177         name = line.split(' ', 1)[0]
 178
 179         if name.startswith('@') and name.endswith(':'):
 180             line = line[len(name)+1:]
 181             self._start_args_section(name[1:-1])
 182         elif name in ('Returns:', 'Since:',
 183                       # those are often singular or plural
 184                       'Note:', 'Notes:',
 185                       'Example:', 'Examples:',
 186                       'TODO:'):
 187             line = line[len(name)+1:]
 188             self._start_section(name[:-1])
 189
 190         self._append_freeform(line)
 191
 192     def _start_args_section(self, name):
 193         # FIXME invalid names other than the empty string aren't flagged
 194         if not name:
 195             raise QAPIParseError(self.parser, "Invalid parameter name")
 196         if name in self.args:
 197             raise QAPIParseError(self.parser,
 198                                  "'%s' parameter name duplicated" % name)
 199         if self.sections:
 200             raise QAPIParseError(self.parser,
 201                                  "'@%s:' can't follow '%s' section"
 202                                  % (name, self.sections[0].name))
 203         self._end_section()
 204         self.section = QAPIDoc.ArgSection(name)
 205         self.args[name] = self.section
 206
 207     def _start_section(self, name=''):
 208         if name in ('Returns', 'Since') and self.has_section(name):
 209             raise QAPIParseError(self.parser,
 210                                  "Duplicated '%s' section" % name)
 211         self._end_section()
 212         self.section = QAPIDoc.Section(name)
 213         self.sections.append(self.section)
 214
 215     def _end_section(self):
 216         if self.section:
 217             contents = str(self.section)
 218             if self.section.name and (not contents or contents.isspace()):
 219                 raise QAPIParseError(self.parser, "Empty doc section '%s'"
 220                                      % self.section.name)
 221             self.section = None
 222
 223     def _append_freeform(self, line):
 224         in_arg = isinstance(self.section, QAPIDoc.ArgSection)
 225         if (in_arg and self.section.content
 226                 and not self.section.content[-1]
 227                 and line and not line[0].isspace()):
 228             self._start_section()
 229         if (in_arg or not self.section.name
 230                 or not self.section.name.startswith('Example')):
 231             line = line.strip()
 232         match = re.match(r'(@\S+:)', line)
 233         if match:
 234             raise QAPIParseError(self.parser,
 235                                  "'%s' not allowed in free-form documentation"
 236                                  % match.group(1))
 237         self.section.append(line)
 238
 239     def connect_member(self, member):
 240         if member.name not in self.args:
 241             # Undocumented TODO outlaw
 242             self.args[member.name] = QAPIDoc.ArgSection(member.name)
 243         self.args[member.name].connect(member)
 244
 245     def check_expr(self, expr):
 246         if self.has_section('Returns') and 'command' not in expr:
 247             raise QAPISemError(self.info,
 248                                "'Returns:' is only valid for commands")
 249
 250     def check(self):
 251         bogus = [name for name, section in self.args.iteritems()
 252                  if not section.member]
 253         if bogus:
 254             raise QAPISemError(
 255                 self.info,
 256                 "The following documented members are not in "
 257                 "the declaration: %s" % ", ".join(bogus))
 258
 259
 260 class QAPISchemaParser(object):
 261
 262     def __init__(self, fp, previously_included=[], incl_info=None):
 263         abs_fname = os.path.abspath(fp.name)
 264         fname = fp.name
 265         self.fname = fname
 266         previously_included.append(abs_fname)
 267         self.incl_info = incl_info
 268         self.src = fp.read()
 269         if self.src == '' or self.src[-1] != '\n':
 270             self.src += '\n'
 271         self.cursor = 0
 272         self.line = 1
 273         self.line_pos = 0
 274         self.exprs = []
 275         self.docs = []
 276         self.cur_doc = None
 277         self.accept()
 278
 279         while self.tok is not None:
 280             info = {'file': fname, 'line': self.line,
 281                     'parent': self.incl_info}
 282             if self.tok == '#':
 283                 self.reject_expr_doc()
 284                 self.cur_doc = self.get_doc(info)
 285                 self.docs.append(self.cur_doc)
 286                 continue
 287
 288             expr = self.get_expr(False)
 289             if 'include' in expr:
 290                 self.reject_expr_doc()
 291                 if len(expr) != 1:
 292                     raise QAPISemError(info, "Invalid 'include' directive")
 293                 include = expr['include']
 294                 if not isinstance(include, str):
 295                     raise QAPISemError(info,
 296                                        "Value of 'include' must be a string")
 297                 self._include(include, info, os.path.dirname(abs_fname),
 298                               previously_included)
 299             elif "pragma" in expr:
 300                 self.reject_expr_doc()
 301                 if len(expr) != 1:
 302                     raise QAPISemError(info, "Invalid 'pragma' directive")
 303                 pragma = expr['pragma']
 304                 if not isinstance(pragma, dict):
 305                     raise QAPISemError(
 306                         info, "Value of 'pragma' must be a dictionary")
 307                 for name, value in pragma.iteritems():
 308                     self._pragma(name, value, info)
 309             else:
 310                 expr_elem = {'expr': expr,
 311                              'info': info}
 312                 if self.cur_doc:
 313                     if not self.cur_doc.symbol:
 314                         raise QAPISemError(
 315                             self.cur_doc.info,
 316                             "Expression documentation required")
 317                     expr_elem['doc'] = self.cur_doc
 318                 self.exprs.append(expr_elem)
 319             self.cur_doc = None
 320         self.reject_expr_doc()
 321
 322     def reject_expr_doc(self):
 323         if self.cur_doc and self.cur_doc.symbol:
 324             raise QAPISemError(
 325                 self.cur_doc.info,
 326                 "Documentation for '%s' is not followed by the definition"
 327                 % self.cur_doc.symbol)
 328
 329     def _include(self, include, info, base_dir, previously_included):
 330         incl_abs_fname = os.path.join(base_dir, include)
 331         # catch inclusion cycle
 332         inf = info
 333         while inf:
 334             if incl_abs_fname == os.path.abspath(inf['file']):
 335                 raise QAPISemError(info, "Inclusion loop for %s" % include)
 336             inf = inf['parent']
 337
 338         # skip multiple include of the same file
 339         if incl_abs_fname in previously_included:
 340             return
 341         try:
 342             fobj = open(incl_abs_fname, 'r')
 343         except IOError as e:
 344             raise QAPISemError(info, '%s: %s' % (e.strerror, include))
 345         exprs_include = QAPISchemaParser(fobj, previously_included, info)
 346         self.exprs.extend(exprs_include.exprs)
 347         self.docs.extend(exprs_include.docs)
 348
 349     def _pragma(self, name, value, info):
 350         global doc_required, returns_whitelist, name_case_whitelist
 351         if name == 'doc-required':
 352             if not isinstance(value, bool):
 353                 raise QAPISemError(info,
 354                                    "Pragma 'doc-required' must be boolean")
 355             doc_required = value
 356         elif name == 'returns-whitelist':
 357             if (not isinstance(value, list)
 358                     or any([not isinstance(elt, str) for elt in value])):
 359                 raise QAPISemError(info,
 360                                    "Pragma returns-whitelist must be"
 361                                    " a list of strings")
 362             returns_whitelist = value
 363         elif name == 'name-case-whitelist':
 364             if (not isinstance(value, list)
 365                     or any([not isinstance(elt, str) for elt in value])):
 366                 raise QAPISemError(info,
 367                                    "Pragma name-case-whitelist must be"
 368                                    " a list of strings")
 369             name_case_whitelist = value
 370         else:
 371             raise QAPISemError(info, "Unknown pragma '%s'" % name)
 372
 373     def accept(self, skip_comment=True):
 374         while True:
 375             self.tok = self.src[self.cursor]
 376             self.pos = self.cursor
 377             self.cursor += 1
 378             self.val = None
 379
 380             if self.tok == '#':
 381                 if self.src[self.cursor] == '#':
 382                     # Start of doc comment
 383                     skip_comment = False
 384                 self.cursor = self.src.find('\n', self.cursor)
 385                 if not skip_comment:
 386                     self.val = self.src[self.pos:self.cursor]
 387                     return
 388             elif self.tok in '{}:,[]':
 389                 return
 390             elif self.tok == "'":
 391                 string = ''
 392                 esc = False
 393                 while True:
 394                     ch = self.src[self.cursor]
 395                     self.cursor += 1
 396                     if ch == '\n':
 397                         raise QAPIParseError(self, 'Missing terminating "\'"')
 398                     if esc:
 399                         if ch == 'b':
 400                             string += '\b'
 401                         elif ch == 'f':
 402                             string += '\f'
 403                         elif ch == 'n':
 404                             string += '\n'
 405                         elif ch == 'r':
 406                             string += '\r'
 407                         elif ch == 't':
 408                             string += '\t'
 409                         elif ch == 'u':
 410                             value = 0
 411                             for _ in range(0, 4):
 412                                 ch = self.src[self.cursor]
 413                                 self.cursor += 1
 414                                 if ch not in '0123456789abcdefABCDEF':
 415                                     raise QAPIParseError(self,
 416                                                          '\\u escape needs 4 '
 417                                                          'hex digits')
 418                                 value = (value << 4) + int(ch, 16)
 419                             # If Python 2 and 3 didn't disagree so much on
 420                             # how to handle Unicode, then we could allow
 421                             # Unicode string defaults.  But most of QAPI is
 422                             # ASCII-only, so we aren't losing much for now.
 423                             if not value or value > 0x7f:
 424                                 raise QAPIParseError(self,
 425                                                      'For now, \\u escape '
 426                                                      'only supports non-zero '
 427                                                      'values up to \\u007f')
 428                             string += chr(value)
 429                         elif ch in '\\/\'"':
 430                             string += ch
 431                         else:
 432                             raise QAPIParseError(self,
 433                                                  "Unknown escape \\%s" % ch)
 434                         esc = False
 435                     elif ch == '\\':
 436                         esc = True
 437                     elif ch == "'":
 438                         self.val = string
 439                         return
 440                     else:
 441                         string += ch
 442             elif self.src.startswith('true', self.pos):
 443                 self.val = True
 444                 self.cursor += 3
 445                 return
 446             elif self.src.startswith('false', self.pos):
 447                 self.val = False
 448                 self.cursor += 4
 449                 return
 450             elif self.src.startswith('null', self.pos):
 451                 self.val = None
 452                 self.cursor += 3
 453                 return
 454             elif self.tok == '\n':
 455                 if self.cursor == len(self.src):
 456                     self.tok = None
 457                     return
 458                 self.line += 1
 459                 self.line_pos = self.cursor
 460             elif not self.tok.isspace():
 461                 raise QAPIParseError(self, 'Stray "%s"' % self.tok)
 462
 463     def get_members(self):
 464         expr = OrderedDict()
 465         if self.tok == '}':
 466             self.accept()
 467             return expr
 468         if self.tok != "'":
 469             raise QAPIParseError(self, 'Expected string or "}"')
 470         while True:
 471             key = self.val
 472             self.accept()
 473             if self.tok != ':':
 474                 raise QAPIParseError(self, 'Expected ":"')
 475             self.accept()
 476             if key in expr:
 477                 raise QAPIParseError(self, 'Duplicate key "%s"' % key)
 478             expr[key] = self.get_expr(True)
 479             if self.tok == '}':
 480                 self.accept()
 481                 return expr
 482             if self.tok != ',':
 483                 raise QAPIParseError(self, 'Expected "," or "}"')
 484             self.accept()
 485             if self.tok != "'":
 486                 raise QAPIParseError(self, 'Expected string')
 487
 488     def get_values(self):
 489         expr = []
 490         if self.tok == ']':
 491             self.accept()
 492             return expr
 493         if self.tok not in "{['tfn":
 494             raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
 495                                  'boolean or "null"')
 496         while True:
 497             expr.append(self.get_expr(True))
 498             if self.tok == ']':
 499                 self.accept()
 500                 return expr
 501             if self.tok != ',':
 502                 raise QAPIParseError(self, 'Expected "," or "]"')
 503             self.accept()
 504
 505     def get_expr(self, nested):
 506         if self.tok != '{' and not nested:
 507             raise QAPIParseError(self, 'Expected "{"')
 508         if self.tok == '{':
 509             self.accept()
 510             expr = self.get_members()
 511         elif self.tok == '[':
 512             self.accept()
 513             expr = self.get_values()
 514         elif self.tok in "'tfn":
 515             expr = self.val
 516             self.accept()
 517         else:
 518             raise QAPIParseError(self, 'Expected "{", "[", string, '
 519                                  'boolean or "null"')
 520         return expr
 521
 522     def get_doc(self, info):
 523         if self.val != '##':
 524             raise QAPIParseError(self, "Junk after '##' at start of "
 525                                  "documentation comment")
 526
 527         doc = QAPIDoc(self, info)
 528         self.accept(False)
 529         while self.tok == '#':
 530             if self.val.startswith('##'):
 531                 # End of doc comment
 532                 if self.val != '##':
 533                     raise QAPIParseError(self, "Junk after '##' at end of "
 534                                          "documentation comment")
 535                 doc.end_comment()
 536                 self.accept()
 537                 return doc
 538             else:
 539                 doc.append(self.val)
 540             self.accept(False)
 541
 542         raise QAPIParseError(self, "Documentation comment must end with '##'")
 543
 544
 545 #
 546 # Semantic analysis of schema expressions
 547 # TODO fold into QAPISchema
 548 # TODO catching name collisions in generated code would be nice
 549 #
 550
 551
 552 def find_base_members(base):
 553     if isinstance(base, dict):
 554         return base
 555     base_struct_define = struct_types.get(base)
 556     if not base_struct_define:
 557         return None
 558     return base_struct_define['data']
 559
 560
 561 # Return the qtype of an alternate branch, or None on error.
 562 def find_alternate_member_qtype(qapi_type):
 563     if qapi_type in builtin_types:
 564         return builtin_types[qapi_type]
 565     elif qapi_type in struct_types:
 566         return 'QTYPE_QDICT'
 567     elif qapi_type in enum_types:
 568         return 'QTYPE_QSTRING'
 569     elif qapi_type in union_types:
 570         return 'QTYPE_QDICT'
 571     return None
 572
 573
 574 # Return the discriminator enum define if discriminator is specified as an
 575 # enum type, otherwise return None.
 576 def discriminator_find_enum_define(expr):
 577     base = expr.get('base')
 578     discriminator = expr.get('discriminator')
 579
 580     if not (discriminator and base):
 581         return None
 582
 583     base_members = find_base_members(base)
 584     if not base_members:
 585         return None
 586
 587     discriminator_type = base_members.get(discriminator)
 588     if not discriminator_type:
 589         return None
 590
 591     return enum_types.get(discriminator_type)
 592
 593
 594 # Names must be letters, numbers, -, and _.  They must start with letter,
 595 # except for downstream extensions which must start with __RFQDN_.
 596 # Dots are only valid in the downstream extension prefix.
 597 valid_name = re.compile(r'^(__[a-zA-Z0-9.-]+_)?'
 598                         '[a-zA-Z][a-zA-Z0-9_-]*$')
 599
 600
 601 def check_name(info, source, name, allow_optional=False,
 602                enum_member=False):
 603     global valid_name
 604     membername = name
 605
 606     if not isinstance(name, str):
 607         raise QAPISemError(info, "%s requires a string name" % source)
 608     if name.startswith('*'):
 609         membername = name[1:]
 610         if not allow_optional:
 611             raise QAPISemError(info, "%s does not allow optional name '%s'"
 612                                % (source, name))
 613     # Enum members can start with a digit, because the generated C
 614     # code always prefixes it with the enum name
 615     if enum_member and membername[0].isdigit():
 616         membername = 'D' + membername
 617     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
 618     # and 'q_obj_*' implicit type names.
 619     if not valid_name.match(membername) or \
 620        c_name(membername, False).startswith('q_'):
 621         raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
 622
 623
 624 def add_name(name, info, meta, implicit=False):
 625     global all_names
 626     check_name(info, "'%s'" % meta, name)
 627     # FIXME should reject names that differ only in '_' vs. '.'
 628     # vs. '-', because they're liable to clash in generated C.
 629     if name in all_names:
 630         raise QAPISemError(info, "%s '%s' is already defined"
 631                            % (all_names[name], name))
 632     if not implicit and (name.endswith('Kind') or name.endswith('List')):
 633         raise QAPISemError(info, "%s '%s' should not end in '%s'"
 634                            % (meta, name, name[-4:]))
 635     all_names[name] = meta
 636
 637
 638 def check_type(info, source, value, allow_array=False,
 639                allow_dict=False, allow_optional=False,
 640                allow_metas=[]):
 641     global all_names
 642
 643     if value is None:
 644         return
 645
 646     # Check if array type for value is okay
 647     if isinstance(value, list):
 648         if not allow_array:
 649             raise QAPISemError(info, "%s cannot be an array" % source)
 650         if len(value) != 1 or not isinstance(value[0], str):
 651             raise QAPISemError(info,
 652                                "%s: array type must contain single type name" %
 653                                source)
 654         value = value[0]
 655
 656     # Check if type name for value is okay
 657     if isinstance(value, str):
 658         if value not in all_names:
 659             raise QAPISemError(info, "%s uses unknown type '%s'"
 660                                % (source, value))
 661         if not all_names[value] in allow_metas:
 662             raise QAPISemError(info, "%s cannot use %s type '%s'" %
 663                                (source, all_names[value], value))
 664         return
 665
 666     if not allow_dict:
 667         raise QAPISemError(info, "%s should be a type name" % source)
 668
 669     if not isinstance(value, OrderedDict):
 670         raise QAPISemError(info,
 671                            "%s should be a dictionary or type name" % source)
 672
 673     # value is a dictionary, check that each member is okay
 674     for (key, arg) in value.items():
 675         check_name(info, "Member of %s" % source, key,
 676                    allow_optional=allow_optional)
 677         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 678             raise QAPISemError(info, "Member of %s uses reserved name '%s'"
 679                                % (source, key))
 680         # Todo: allow dictionaries to represent default values of
 681         # an optional argument.
 682         check_type(info, "Member '%s' of %s" % (key, source), arg,
 683                    allow_array=True,
 684                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 685                                 'enum'])
 686
 687
 688 def check_command(expr, info):
 689     name = expr['command']
 690     boxed = expr.get('boxed', False)
 691
 692     args_meta = ['struct']
 693     if boxed:
 694         args_meta += ['union', 'alternate']
 695     check_type(info, "'data' for command '%s'" % name,
 696                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 697                allow_metas=args_meta)
 698     returns_meta = ['union', 'struct']
 699     if name in returns_whitelist:
 700         returns_meta += ['built-in', 'alternate', 'enum']
 701     check_type(info, "'returns' for command '%s'" % name,
 702                expr.get('returns'), allow_array=True,
 703                allow_optional=True, allow_metas=returns_meta)
 704
 705
 706 def check_event(expr, info):
 707     name = expr['event']
 708     boxed = expr.get('boxed', False)
 709
 710     meta = ['struct']
 711     if boxed:
 712         meta += ['union', 'alternate']
 713     check_type(info, "'data' for event '%s'" % name,
 714                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 715                allow_metas=meta)
 716
 717
 718 def check_union(expr, info):
 719     name = expr['union']
 720     base = expr.get('base')
 721     discriminator = expr.get('discriminator')
 722     members = expr['data']
 723
 724     # Two types of unions, determined by discriminator.
 725
 726     # With no discriminator it is a simple union.
 727     if discriminator is None:
 728         enum_define = None
 729         allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
 730         if base is not None:
 731             raise QAPISemError(info, "Simple union '%s' must not have a base" %
 732                                name)
 733
 734     # Else, it's a flat union.
 735     else:
 736         # The object must have a string or dictionary 'base'.
 737         check_type(info, "'base' for union '%s'" % name,
 738                    base, allow_dict=True, allow_optional=True,
 739                    allow_metas=['struct'])
 740         if not base:
 741             raise QAPISemError(info, "Flat union '%s' must have a base"
 742                                % name)
 743         base_members = find_base_members(base)
 744         assert base_members is not None
 745
 746         # The value of member 'discriminator' must name a non-optional
 747         # member of the base struct.
 748         check_name(info, "Discriminator of flat union '%s'" % name,
 749                    discriminator)
 750         discriminator_type = base_members.get(discriminator)
 751         if not discriminator_type:
 752             raise QAPISemError(info,
 753                                "Discriminator '%s' is not a member of base "
 754                                "struct '%s'"
 755                                % (discriminator, base))
 756         enum_define = enum_types.get(discriminator_type)
 757         allow_metas = ['struct']
 758         # Do not allow string discriminator
 759         if not enum_define:
 760             raise QAPISemError(info,
 761                                "Discriminator '%s' must be of enumeration "
 762                                "type" % discriminator)
 763
 764     # Check every branch; don't allow an empty union
 765     if len(members) == 0:
 766         raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
 767     for (key, value) in members.items():
 768         check_name(info, "Member of union '%s'" % name, key)
 769
 770         # Each value must name a known type
 771         check_type(info, "Member '%s' of union '%s'" % (key, name),
 772                    value, allow_array=not base, allow_metas=allow_metas)
 773
 774         # If the discriminator names an enum type, then all members
 775         # of 'data' must also be members of the enum type.
 776         if enum_define:
 777             if key not in enum_define['data']:
 778                 raise QAPISemError(info,
 779                                    "Discriminator value '%s' is not found in "
 780                                    "enum '%s'"
 781                                    % (key, enum_define['enum']))
 782
 783     # If discriminator is user-defined, ensure all values are covered
 784     if enum_define:
 785         for value in enum_define['data']:
 786             if value not in members.keys():
 787                 raise QAPISemError(info, "Union '%s' data missing '%s' branch"
 788                                    % (name, value))
 789
 790
 791 def check_alternate(expr, info):
 792     name = expr['alternate']
 793     members = expr['data']
 794     types_seen = {}
 795
 796     # Check every branch; require at least two branches
 797     if len(members) < 2:
 798         raise QAPISemError(info,
 799                            "Alternate '%s' should have at least two branches "
 800                            "in 'data'" % name)
 801     for (key, value) in members.items():
 802         check_name(info, "Member of alternate '%s'" % name, key)
 803
 804         # Ensure alternates have no type conflicts.
 805         check_type(info, "Member '%s' of alternate '%s'" % (key, name),
 806                    value,
 807                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 808         qtype = find_alternate_member_qtype(value)
 809         if not qtype:
 810             raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
 811                                "type '%s'" % (name, key, value))
 812         conflicting = set([qtype])
 813         if qtype == 'QTYPE_QSTRING':
 814             enum_expr = enum_types.get(value)
 815             if enum_expr:
 816                 for v in enum_expr['data']:
 817                     if v in ['on', 'off']:
 818                         conflicting.add('QTYPE_QBOOL')
 819                     if re.match(r'[-+0-9.]', v): # lazy, could be tightened
 820                         conflicting.add('QTYPE_QNUM')
 821             else:
 822                 conflicting.add('QTYPE_QNUM')
 823                 conflicting.add('QTYPE_QBOOL')
 824         for qt in conflicting:
 825             if qt in types_seen:
 826                 raise QAPISemError(info, "Alternate '%s' member '%s' can't "
 827                                    "be distinguished from member '%s'"
 828                                    % (name, key, types_seen[qt]))
 829             types_seen[qt] = key
 830
 831
 832 def check_enum(expr, info):
 833     name = expr['enum']
 834     members = expr.get('data')
 835     prefix = expr.get('prefix')
 836
 837     if not isinstance(members, list):
 838         raise QAPISemError(info,
 839                            "Enum '%s' requires an array for 'data'" % name)
 840     if prefix is not None and not isinstance(prefix, str):
 841         raise QAPISemError(info,
 842                            "Enum '%s' requires a string for 'prefix'" % name)
 843     for member in members:
 844         check_name(info, "Member of enum '%s'" % name, member,
 845                    enum_member=True)
 846
 847
 848 def check_struct(expr, info):
 849     name = expr['struct']
 850     members = expr['data']
 851
 852     check_type(info, "'data' for struct '%s'" % name, members,
 853                allow_dict=True, allow_optional=True)
 854     check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
 855                allow_metas=['struct'])
 856
 857
 858 def check_keys(expr_elem, meta, required, optional=[]):
 859     expr = expr_elem['expr']
 860     info = expr_elem['info']
 861     name = expr[meta]
 862     if not isinstance(name, str):
 863         raise QAPISemError(info, "'%s' key must have a string value" % meta)
 864     required = required + [meta]
 865     for (key, value) in expr.items():
 866         if key not in required and key not in optional:
 867             raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
 868                                % (key, meta, name))
 869         if (key == 'gen' or key == 'success-response') and value is not False:
 870             raise QAPISemError(info,
 871                                "'%s' of %s '%s' should only use false value"
 872                                % (key, meta, name))
 873         if key == 'boxed' and value is not True:
 874             raise QAPISemError(info,
 875                                "'%s' of %s '%s' should only use true value"
 876                                % (key, meta, name))
 877     for key in required:
 878         if key not in expr:
 879             raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
 880                                % (key, meta, name))
 881
 882
 883 def check_exprs(exprs):
 884     global all_names
 885
 886     # Populate name table with names of built-in types
 887     for builtin in builtin_types.keys():
 888         all_names[builtin] = 'built-in'
 889
 890     # Learn the types and check for valid expression keys
 891     for expr_elem in exprs:
 892         expr = expr_elem['expr']
 893         info = expr_elem['info']
 894         doc = expr_elem.get('doc')
 895
 896         if not doc and doc_required:
 897             raise QAPISemError(info,
 898                                "Expression missing documentation comment")
 899
 900         if 'enum' in expr:
 901             meta = 'enum'
 902             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 903             enum_types[expr[meta]] = expr
 904         elif 'union' in expr:
 905             meta = 'union'
 906             check_keys(expr_elem, 'union', ['data'],
 907                        ['base', 'discriminator'])
 908             union_types[expr[meta]] = expr
 909         elif 'alternate' in expr:
 910             meta = 'alternate'
 911             check_keys(expr_elem, 'alternate', ['data'])
 912         elif 'struct' in expr:
 913             meta = 'struct'
 914             check_keys(expr_elem, 'struct', ['data'], ['base'])
 915             struct_types[expr[meta]] = expr
 916         elif 'command' in expr:
 917             meta = 'command'
 918             check_keys(expr_elem, 'command', [],
 919                        ['data', 'returns', 'gen', 'success-response', 'boxed'])
 920         elif 'event' in expr:
 921             meta = 'event'
 922             check_keys(expr_elem, 'event', [], ['data', 'boxed'])
 923         else:
 924             raise QAPISemError(expr_elem['info'],
 925                                "Expression is missing metatype")
 926         name = expr[meta]
 927         add_name(name, info, meta)
 928         if doc and doc.symbol != name:
 929             raise QAPISemError(info, "Definition of '%s' follows documentation"
 930                                " for '%s'" % (name, doc.symbol))
 931
 932     # Try again for hidden UnionKind enum
 933     for expr_elem in exprs:
 934         expr = expr_elem['expr']
 935         if 'union' in expr and not discriminator_find_enum_define(expr):
 936             name = '%sKind' % expr['union']
 937         elif 'alternate' in expr:
 938             name = '%sKind' % expr['alternate']
 939         else:
 940             continue
 941         enum_types[name] = {'enum': name}
 942         add_name(name, info, 'enum', implicit=True)
 943
 944     # Validate that exprs make sense
 945     for expr_elem in exprs:
 946         expr = expr_elem['expr']
 947         info = expr_elem['info']
 948         doc = expr_elem.get('doc')
 949
 950         if 'enum' in expr:
 951             check_enum(expr, info)
 952         elif 'union' in expr:
 953             check_union(expr, info)
 954         elif 'alternate' in expr:
 955             check_alternate(expr, info)
 956         elif 'struct' in expr:
 957             check_struct(expr, info)
 958         elif 'command' in expr:
 959             check_command(expr, info)
 960         elif 'event' in expr:
 961             check_event(expr, info)
 962         else:
 963             assert False, 'unexpected meta type'
 964
 965         if doc:
 966             doc.check_expr(expr)
 967
 968     return exprs
 969
 970
 971 #
 972 # Schema compiler frontend
 973 #
 974
 975 class QAPISchemaEntity(object):
 976     def __init__(self, name, info, doc):
 977         assert isinstance(name, str)
 978         self.name = name
 979         # For explicitly defined entities, info points to the (explicit)
 980         # definition.  For builtins (and their arrays), info is None.
 981         # For implicitly defined entities, info points to a place that
 982         # triggered the implicit definition (there may be more than one
 983         # such place).
 984         self.info = info
 985         self.doc = doc
 986
 987     def c_name(self):
 988         return c_name(self.name)
 989
 990     def check(self, schema):
 991         pass
 992
 993     def is_implicit(self):
 994         return not self.info
 995
 996     def visit(self, visitor):
 997         pass
 998
 999
1000 class QAPISchemaVisitor(object):
1001     def visit_begin(self, schema):
1002         pass
1003
1004     def visit_end(self):
1005         pass
1006
1007     def visit_needed(self, entity):
1008         # Default to visiting everything
1009         return True
1010
1011     def visit_builtin_type(self, name, info, json_type):
1012         pass
1013
1014     def visit_enum_type(self, name, info, values, prefix):
1015         pass
1016
1017     def visit_array_type(self, name, info, element_type):
1018         pass
1019
1020     def visit_object_type(self, name, info, base, members, variants):
1021         pass
1022
1023     def visit_object_type_flat(self, name, info, members, variants):
1024         pass
1025
1026     def visit_alternate_type(self, name, info, variants):
1027         pass
1028
1029     def visit_command(self, name, info, arg_type, ret_type,
1030                       gen, success_response, boxed):
1031         pass
1032
1033     def visit_event(self, name, info, arg_type, boxed):
1034         pass
1035
1036
1037 class QAPISchemaType(QAPISchemaEntity):
1038     # Return the C type for common use.
1039     # For the types we commonly box, this is a pointer type.
1040     def c_type(self):
1041         pass
1042
1043     # Return the C type to be used in a parameter list.
1044     def c_param_type(self):
1045         return self.c_type()
1046
1047     # Return the C type to be used where we suppress boxing.
1048     def c_unboxed_type(self):
1049         return self.c_type()
1050
1051     def json_type(self):
1052         pass
1053
1054     def alternate_qtype(self):
1055         json2qtype = {
1056             'null':    'QTYPE_QNULL',
1057             'string':  'QTYPE_QSTRING',
1058             'number':  'QTYPE_QNUM',
1059             'int':     'QTYPE_QNUM',
1060             'boolean': 'QTYPE_QBOOL',
1061             'object':  'QTYPE_QDICT'
1062         }
1063         return json2qtype.get(self.json_type())
1064
1065     def doc_type(self):
1066         if self.is_implicit():
1067             return None
1068         return self.name
1069
1070
1071 class QAPISchemaBuiltinType(QAPISchemaType):
1072     def __init__(self, name, json_type, c_type):
1073         QAPISchemaType.__init__(self, name, None, None)
1074         assert not c_type or isinstance(c_type, str)
1075         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
1076                              'value')
1077         self._json_type_name = json_type
1078         self._c_type_name = c_type
1079
1080     def c_name(self):
1081         return self.name
1082
1083     def c_type(self):
1084         return self._c_type_name
1085
1086     def c_param_type(self):
1087         if self.name == 'str':
1088             return 'const ' + self._c_type_name
1089         return self._c_type_name
1090
1091     def json_type(self):
1092         return self._json_type_name
1093
1094     def doc_type(self):
1095         return self.json_type()
1096
1097     def visit(self, visitor):
1098         visitor.visit_builtin_type(self.name, self.info, self.json_type())
1099
1100
1101 class QAPISchemaEnumType(QAPISchemaType):
1102     def __init__(self, name, info, doc, values, prefix):
1103         QAPISchemaType.__init__(self, name, info, doc)
1104         for v in values:
1105             assert isinstance(v, QAPISchemaMember)
1106             v.set_owner(name)
1107         assert prefix is None or isinstance(prefix, str)
1108         self.values = values
1109         self.prefix = prefix
1110
1111     def check(self, schema):
1112         seen = {}
1113         for v in self.values:
1114             v.check_clash(self.info, seen)
1115             if self.doc:
1116                 self.doc.connect_member(v)
1117
1118     def is_implicit(self):
1119         # See QAPISchema._make_implicit_enum_type() and ._def_predefineds()
1120         return self.name.endswith('Kind') or self.name == 'QType'
1121
1122     def c_type(self):
1123         return c_name(self.name)
1124
1125     def member_names(self):
1126         return [v.name for v in self.values]
1127
1128     def json_type(self):
1129         return 'string'
1130
1131     def visit(self, visitor):
1132         visitor.visit_enum_type(self.name, self.info,
1133                                 self.member_names(), self.prefix)
1134
1135
1136 class QAPISchemaArrayType(QAPISchemaType):
1137     def __init__(self, name, info, element_type):
1138         QAPISchemaType.__init__(self, name, info, None)
1139         assert isinstance(element_type, str)
1140         self._element_type_name = element_type
1141         self.element_type = None
1142
1143     def check(self, schema):
1144         self.element_type = schema.lookup_type(self._element_type_name)
1145         assert self.element_type
1146
1147     def is_implicit(self):
1148         return True
1149
1150     def c_type(self):
1151         return c_name(self.name) + pointer_suffix
1152
1153     def json_type(self):
1154         return 'array'
1155
1156     def doc_type(self):
1157         elt_doc_type = self.element_type.doc_type()
1158         if not elt_doc_type:
1159             return None
1160         return 'array of ' + elt_doc_type
1161
1162     def visit(self, visitor):
1163         visitor.visit_array_type(self.name, self.info, self.element_type)
1164
1165
1166 class QAPISchemaObjectType(QAPISchemaType):
1167     def __init__(self, name, info, doc, base, local_members, variants):
1168         # struct has local_members, optional base, and no variants
1169         # flat union has base, variants, and no local_members
1170         # simple union has local_members, variants, and no base
1171         QAPISchemaType.__init__(self, name, info, doc)
1172         assert base is None or isinstance(base, str)
1173         for m in local_members:
1174             assert isinstance(m, QAPISchemaObjectTypeMember)
1175             m.set_owner(name)
1176         if variants is not None:
1177             assert isinstance(variants, QAPISchemaObjectTypeVariants)
1178             variants.set_owner(name)
1179         self._base_name = base
1180         self.base = None
1181         self.local_members = local_members
1182         self.variants = variants
1183         self.members = None
1184
1185     def check(self, schema):
1186         if self.members is False:               # check for cycles
1187             raise QAPISemError(self.info,
1188                                "Object %s contains itself" % self.name)
1189         if self.members:
1190             return
1191         self.members = False                    # mark as being checked
1192         seen = OrderedDict()
1193         if self._base_name:
1194             self.base = schema.lookup_type(self._base_name)
1195             assert isinstance(self.base, QAPISchemaObjectType)
1196             self.base.check(schema)
1197             self.base.check_clash(self.info, seen)
1198         for m in self.local_members:
1199             m.check(schema)
1200             m.check_clash(self.info, seen)
1201             if self.doc:
1202                 self.doc.connect_member(m)
1203         self.members = seen.values()
1204         if self.variants:
1205             self.variants.check(schema, seen)
1206             assert self.variants.tag_member in self.members
1207             self.variants.check_clash(self.info, seen)
1208         if self.doc:
1209             self.doc.check()
1210
1211     # Check that the members of this type do not cause duplicate JSON members,
1212     # and update seen to track the members seen so far. Report any errors
1213     # on behalf of info, which is not necessarily self.info
1214     def check_clash(self, info, seen):
1215         assert not self.variants       # not implemented
1216         for m in self.members:
1217             m.check_clash(info, seen)
1218
1219     def is_implicit(self):
1220         # See QAPISchema._make_implicit_object_type(), as well as
1221         # _def_predefineds()
1222         return self.name.startswith('q_')
1223
1224     def is_empty(self):
1225         assert self.members is not None
1226         return not self.members and not self.variants
1227
1228     def c_name(self):
1229         assert self.name != 'q_empty'
1230         return QAPISchemaType.c_name(self)
1231
1232     def c_type(self):
1233         assert not self.is_implicit()
1234         return c_name(self.name) + pointer_suffix
1235
1236     def c_unboxed_type(self):
1237         return c_name(self.name)
1238
1239     def json_type(self):
1240         return 'object'
1241
1242     def visit(self, visitor):
1243         visitor.visit_object_type(self.name, self.info,
1244                                   self.base, self.local_members, self.variants)
1245         visitor.visit_object_type_flat(self.name, self.info,
1246                                        self.members, self.variants)
1247
1248
1249 class QAPISchemaMember(object):
1250     role = 'member'
1251
1252     def __init__(self, name):
1253         assert isinstance(name, str)
1254         self.name = name
1255         self.owner = None
1256
1257     def set_owner(self, name):
1258         assert not self.owner
1259         self.owner = name
1260
1261     def check_clash(self, info, seen):
1262         cname = c_name(self.name)
1263         if cname.lower() != cname and self.owner not in name_case_whitelist:
1264             raise QAPISemError(info,
1265                                "%s should not use uppercase" % self.describe())
1266         if cname in seen:
1267             raise QAPISemError(info, "%s collides with %s" %
1268                                (self.describe(), seen[cname].describe()))
1269         seen[cname] = self
1270
1271     def _pretty_owner(self):
1272         owner = self.owner
1273         if owner.startswith('q_obj_'):
1274             # See QAPISchema._make_implicit_object_type() - reverse the
1275             # mapping there to create a nice human-readable description
1276             owner = owner[6:]
1277             if owner.endswith('-arg'):
1278                 return '(parameter of %s)' % owner[:-4]
1279             elif owner.endswith('-base'):
1280                 return '(base of %s)' % owner[:-5]
1281             else:
1282                 assert owner.endswith('-wrapper')
1283                 # Unreachable and not implemented
1284                 assert False
1285         if owner.endswith('Kind'):
1286             # See QAPISchema._make_implicit_enum_type()
1287             return '(branch of %s)' % owner[:-4]
1288         return '(%s of %s)' % (self.role, owner)
1289
1290     def describe(self):
1291         return "'%s' %s" % (self.name, self._pretty_owner())
1292
1293
1294 class QAPISchemaObjectTypeMember(QAPISchemaMember):
1295     def __init__(self, name, typ, optional):
1296         QAPISchemaMember.__init__(self, name)
1297         assert isinstance(typ, str)
1298         assert isinstance(optional, bool)
1299         self._type_name = typ
1300         self.type = None
1301         self.optional = optional
1302
1303     def check(self, schema):
1304         assert self.owner
1305         self.type = schema.lookup_type(self._type_name)
1306         assert self.type
1307
1308
1309 class QAPISchemaObjectTypeVariants(object):
1310     def __init__(self, tag_name, tag_member, variants):
1311         # Flat unions pass tag_name but not tag_member.
1312         # Simple unions and alternates pass tag_member but not tag_name.
1313         # After check(), tag_member is always set, and tag_name remains
1314         # a reliable witness of being used by a flat union.
1315         assert bool(tag_member) != bool(tag_name)
1316         assert (isinstance(tag_name, str) or
1317                 isinstance(tag_member, QAPISchemaObjectTypeMember))
1318         assert len(variants) > 0
1319         for v in variants:
1320             assert isinstance(v, QAPISchemaObjectTypeVariant)
1321         self._tag_name = tag_name
1322         self.tag_member = tag_member
1323         self.variants = variants
1324
1325     def set_owner(self, name):
1326         for v in self.variants:
1327             v.set_owner(name)
1328
1329     def check(self, schema, seen):
1330         if not self.tag_member:    # flat union
1331             self.tag_member = seen[c_name(self._tag_name)]
1332             assert self._tag_name == self.tag_member.name
1333         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
1334         for v in self.variants:
1335             v.check(schema)
1336             # Union names must match enum values; alternate names are
1337             # checked separately. Use 'seen' to tell the two apart.
1338             if seen:
1339                 assert v.name in self.tag_member.type.member_names()
1340                 assert isinstance(v.type, QAPISchemaObjectType)
1341                 v.type.check(schema)
1342
1343     def check_clash(self, info, seen):
1344         for v in self.variants:
1345             # Reset seen map for each variant, since qapi names from one
1346             # branch do not affect another branch
1347             assert isinstance(v.type, QAPISchemaObjectType)
1348             v.type.check_clash(info, dict(seen))
1349
1350
1351 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1352     role = 'branch'
1353
1354     def __init__(self, name, typ):
1355         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1356
1357
1358 class QAPISchemaAlternateType(QAPISchemaType):
1359     def __init__(self, name, info, doc, variants):
1360         QAPISchemaType.__init__(self, name, info, doc)
1361         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1362         assert variants.tag_member
1363         variants.set_owner(name)
1364         variants.tag_member.set_owner(self.name)
1365         self.variants = variants
1366
1367     def check(self, schema):
1368         self.variants.tag_member.check(schema)
1369         # Not calling self.variants.check_clash(), because there's nothing
1370         # to clash with
1371         self.variants.check(schema, {})
1372         # Alternate branch names have no relation to the tag enum values;
1373         # so we have to check for potential name collisions ourselves.
1374         seen = {}
1375         for v in self.variants.variants:
1376             v.check_clash(self.info, seen)
1377             if self.doc:
1378                 self.doc.connect_member(v)
1379         if self.doc:
1380             self.doc.check()
1381
1382     def c_type(self):
1383         return c_name(self.name) + pointer_suffix
1384
1385     def json_type(self):
1386         return 'value'
1387
1388     def visit(self, visitor):
1389         visitor.visit_alternate_type(self.name, self.info, self.variants)
1390
1391     def is_empty(self):
1392         return False
1393
1394
1395 class QAPISchemaCommand(QAPISchemaEntity):
1396     def __init__(self, name, info, doc, arg_type, ret_type,
1397                  gen, success_response, boxed):
1398         QAPISchemaEntity.__init__(self, name, info, doc)
1399         assert not arg_type or isinstance(arg_type, str)
1400         assert not ret_type or isinstance(ret_type, str)
1401         self._arg_type_name = arg_type
1402         self.arg_type = None
1403         self._ret_type_name = ret_type
1404         self.ret_type = None
1405         self.gen = gen
1406         self.success_response = success_response
1407         self.boxed = boxed
1408
1409     def check(self, schema):
1410         if self._arg_type_name:
1411             self.arg_type = schema.lookup_type(self._arg_type_name)
1412             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1413                     isinstance(self.arg_type, QAPISchemaAlternateType))
1414             self.arg_type.check(schema)
1415             if self.boxed:
1416                 if self.arg_type.is_empty():
1417                     raise QAPISemError(self.info,
1418                                        "Cannot use 'boxed' with empty type")
1419             else:
1420                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1421                 assert not self.arg_type.variants
1422         elif self.boxed:
1423             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1424         if self._ret_type_name:
1425             self.ret_type = schema.lookup_type(self._ret_type_name)
1426             assert isinstance(self.ret_type, QAPISchemaType)
1427
1428     def visit(self, visitor):
1429         visitor.visit_command(self.name, self.info,
1430                               self.arg_type, self.ret_type,
1431                               self.gen, self.success_response, self.boxed)
1432
1433
1434 class QAPISchemaEvent(QAPISchemaEntity):
1435     def __init__(self, name, info, doc, arg_type, boxed):
1436         QAPISchemaEntity.__init__(self, name, info, doc)
1437         assert not arg_type or isinstance(arg_type, str)
1438         self._arg_type_name = arg_type
1439         self.arg_type = None
1440         self.boxed = boxed
1441
1442     def check(self, schema):
1443         if self._arg_type_name:
1444             self.arg_type = schema.lookup_type(self._arg_type_name)
1445             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1446                     isinstance(self.arg_type, QAPISchemaAlternateType))
1447             self.arg_type.check(schema)
1448             if self.boxed:
1449                 if self.arg_type.is_empty():
1450                     raise QAPISemError(self.info,
1451                                        "Cannot use 'boxed' with empty type")
1452             else:
1453                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1454                 assert not self.arg_type.variants
1455         elif self.boxed:
1456             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1457
1458     def visit(self, visitor):
1459         visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
1460
1461
1462 class QAPISchema(object):
1463     def __init__(self, fname):
1464         try:
1465             parser = QAPISchemaParser(open(fname, 'r'))
1466             self.exprs = check_exprs(parser.exprs)
1467             self.docs = parser.docs
1468             self._entity_dict = {}
1469             self._predefining = True
1470             self._def_predefineds()
1471             self._predefining = False
1472             self._def_exprs()
1473             self.check()
1474         except QAPIError as err:
1475             print >>sys.stderr, err
1476             exit(1)
1477
1478     def _def_entity(self, ent):
1479         # Only the predefined types are allowed to not have info
1480         assert ent.info or self._predefining
1481         assert ent.name not in self._entity_dict
1482         self._entity_dict[ent.name] = ent
1483
1484     def lookup_entity(self, name, typ=None):
1485         ent = self._entity_dict.get(name)
1486         if typ and not isinstance(ent, typ):
1487             return None
1488         return ent
1489
1490     def lookup_type(self, name):
1491         return self.lookup_entity(name, QAPISchemaType)
1492
1493     def _def_builtin_type(self, name, json_type, c_type):
1494         self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
1495         # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
1496         # qapi-types.h from a single .c, all arrays of builtins must be
1497         # declared in the first file whether or not they are used.  Nicer
1498         # would be to use lazy instantiation, while figuring out how to
1499         # avoid compilation issues with multiple qapi-types.h.
1500         self._make_array_type(name, None)
1501
1502     def _def_predefineds(self):
1503         for t in [('str',    'string',  'char' + pointer_suffix),
1504                   ('number', 'number',  'double'),
1505                   ('int',    'int',     'int64_t'),
1506                   ('int8',   'int',     'int8_t'),
1507                   ('int16',  'int',     'int16_t'),
1508                   ('int32',  'int',     'int32_t'),
1509                   ('int64',  'int',     'int64_t'),
1510                   ('uint8',  'int',     'uint8_t'),
1511                   ('uint16', 'int',     'uint16_t'),
1512                   ('uint32', 'int',     'uint32_t'),
1513                   ('uint64', 'int',     'uint64_t'),
1514                   ('size',   'int',     'uint64_t'),
1515                   ('bool',   'boolean', 'bool'),
1516                   ('any',    'value',   'QObject' + pointer_suffix),
1517                   ('null',   'null',    'QNull' + pointer_suffix)]:
1518             self._def_builtin_type(*t)
1519         self.the_empty_object_type = QAPISchemaObjectType(
1520             'q_empty', None, None, None, [], None)
1521         self._def_entity(self.the_empty_object_type)
1522         qtype_values = self._make_enum_members(['none', 'qnull', 'qnum',
1523                                                 'qstring', 'qdict', 'qlist',
1524                                                 'qbool'])
1525         self._def_entity(QAPISchemaEnumType('QType', None, None,
1526                                             qtype_values, 'QTYPE'))
1527
1528     def _make_enum_members(self, values):
1529         return [QAPISchemaMember(v) for v in values]
1530
1531     def _make_implicit_enum_type(self, name, info, values):
1532         # See also QAPISchemaObjectTypeMember._pretty_owner()
1533         name = name + 'Kind'   # Use namespace reserved by add_name()
1534         self._def_entity(QAPISchemaEnumType(
1535             name, info, None, self._make_enum_members(values), None))
1536         return name
1537
1538     def _make_array_type(self, element_type, info):
1539         name = element_type + 'List'   # Use namespace reserved by add_name()
1540         if not self.lookup_type(name):
1541             self._def_entity(QAPISchemaArrayType(name, info, element_type))
1542         return name
1543
1544     def _make_implicit_object_type(self, name, info, doc, role, members):
1545         if not members:
1546             return None
1547         # See also QAPISchemaObjectTypeMember._pretty_owner()
1548         name = 'q_obj_%s-%s' % (name, role)
1549         if not self.lookup_entity(name, QAPISchemaObjectType):
1550             self._def_entity(QAPISchemaObjectType(name, info, doc, None,
1551                                                   members, None))
1552         return name
1553
1554     def _def_enum_type(self, expr, info, doc):
1555         name = expr['enum']
1556         data = expr['data']
1557         prefix = expr.get('prefix')
1558         self._def_entity(QAPISchemaEnumType(
1559             name, info, doc, self._make_enum_members(data), prefix))
1560
1561     def _make_member(self, name, typ, info):
1562         optional = False
1563         if name.startswith('*'):
1564             name = name[1:]
1565             optional = True
1566         if isinstance(typ, list):
1567             assert len(typ) == 1
1568             typ = self._make_array_type(typ[0], info)
1569         return QAPISchemaObjectTypeMember(name, typ, optional)
1570
1571     def _make_members(self, data, info):
1572         return [self._make_member(key, value, info)
1573                 for (key, value) in data.iteritems()]
1574
1575     def _def_struct_type(self, expr, info, doc):
1576         name = expr['struct']
1577         base = expr.get('base')
1578         data = expr['data']
1579         self._def_entity(QAPISchemaObjectType(name, info, doc, base,
1580                                               self._make_members(data, info),
1581                                               None))
1582
1583     def _make_variant(self, case, typ):
1584         return QAPISchemaObjectTypeVariant(case, typ)
1585
1586     def _make_simple_variant(self, case, typ, info):
1587         if isinstance(typ, list):
1588             assert len(typ) == 1
1589             typ = self._make_array_type(typ[0], info)
1590         typ = self._make_implicit_object_type(
1591             typ, info, None, 'wrapper', [self._make_member('data', typ, info)])
1592         return QAPISchemaObjectTypeVariant(case, typ)
1593
1594     def _def_union_type(self, expr, info, doc):
1595         name = expr['union']
1596         data = expr['data']
1597         base = expr.get('base')
1598         tag_name = expr.get('discriminator')
1599         tag_member = None
1600         if isinstance(base, dict):
1601             base = (self._make_implicit_object_type(
1602                 name, info, doc, 'base', self._make_members(base, info)))
1603         if tag_name:
1604             variants = [self._make_variant(key, value)
1605                         for (key, value) in data.iteritems()]
1606             members = []
1607         else:
1608             variants = [self._make_simple_variant(key, value, info)
1609                         for (key, value) in data.iteritems()]
1610             typ = self._make_implicit_enum_type(name, info,
1611                                                 [v.name for v in variants])
1612             tag_member = QAPISchemaObjectTypeMember('type', typ, False)
1613             members = [tag_member]
1614         self._def_entity(
1615             QAPISchemaObjectType(name, info, doc, base, members,
1616                                  QAPISchemaObjectTypeVariants(tag_name,
1617                                                               tag_member,
1618                                                               variants)))
1619
1620     def _def_alternate_type(self, expr, info, doc):
1621         name = expr['alternate']
1622         data = expr['data']
1623         variants = [self._make_variant(key, value)
1624                     for (key, value) in data.iteritems()]
1625         tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
1626         self._def_entity(
1627             QAPISchemaAlternateType(name, info, doc,
1628                                     QAPISchemaObjectTypeVariants(None,
1629                                                                  tag_member,
1630                                                                  variants)))
1631
1632     def _def_command(self, expr, info, doc):
1633         name = expr['command']
1634         data = expr.get('data')
1635         rets = expr.get('returns')
1636         gen = expr.get('gen', True)
1637         success_response = expr.get('success-response', True)
1638         boxed = expr.get('boxed', False)
1639         if isinstance(data, OrderedDict):
1640             data = self._make_implicit_object_type(
1641                 name, info, doc, 'arg', self._make_members(data, info))
1642         if isinstance(rets, list):
1643             assert len(rets) == 1
1644             rets = self._make_array_type(rets[0], info)
1645         self._def_entity(QAPISchemaCommand(name, info, doc, data, rets,
1646                                            gen, success_response, boxed))
1647
1648     def _def_event(self, expr, info, doc):
1649         name = expr['event']
1650         data = expr.get('data')
1651         boxed = expr.get('boxed', False)
1652         if isinstance(data, OrderedDict):
1653             data = self._make_implicit_object_type(
1654                 name, info, doc, 'arg', self._make_members(data, info))
1655         self._def_entity(QAPISchemaEvent(name, info, doc, data, boxed))
1656
1657     def _def_exprs(self):
1658         for expr_elem in self.exprs:
1659             expr = expr_elem['expr']
1660             info = expr_elem['info']
1661             doc = expr_elem.get('doc')
1662             if 'enum' in expr:
1663                 self._def_enum_type(expr, info, doc)
1664             elif 'struct' in expr:
1665                 self._def_struct_type(expr, info, doc)
1666             elif 'union' in expr:
1667                 self._def_union_type(expr, info, doc)
1668             elif 'alternate' in expr:
1669                 self._def_alternate_type(expr, info, doc)
1670             elif 'command' in expr:
1671                 self._def_command(expr, info, doc)
1672             elif 'event' in expr:
1673                 self._def_event(expr, info, doc)
1674             else:
1675                 assert False
1676
1677     def check(self):
1678         for ent in self._entity_dict.values():
1679             ent.check(self)
1680
1681     def visit(self, visitor):
1682         visitor.visit_begin(self)
1683         for (name, entity) in sorted(self._entity_dict.items()):
1684             if visitor.visit_needed(entity):
1685                 entity.visit(visitor)
1686         visitor.visit_end()
1687
1688
1689 #
1690 # Code generation helpers
1691 #
1692
1693 def camel_case(name):
1694     new_name = ''
1695     first = True
1696     for ch in name:
1697         if ch in ['_', '-']:
1698             first = True
1699         elif first:
1700             new_name += ch.upper()
1701             first = False
1702         else:
1703             new_name += ch.lower()
1704     return new_name
1705
1706
1707 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1708 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1709 # ENUM24_Name -> ENUM24_NAME
1710 def camel_to_upper(value):
1711     c_fun_str = c_name(value, False)
1712     if value.isupper():
1713         return c_fun_str
1714
1715     new_name = ''
1716     l = len(c_fun_str)
1717     for i in range(l):
1718         c = c_fun_str[i]
1719         # When c is upper and no '_' appears before, do more checks
1720         if c.isupper() and (i > 0) and c_fun_str[i - 1] != '_':
1721             if i < l - 1 and c_fun_str[i + 1].islower():
1722                 new_name += '_'
1723             elif c_fun_str[i - 1].isdigit():
1724                 new_name += '_'
1725         new_name += c
1726     return new_name.lstrip('_').upper()
1727
1728
1729 def c_enum_const(type_name, const_name, prefix=None):
1730     if prefix is not None:
1731         type_name = prefix
1732     return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
1733
1734 c_name_trans = string.maketrans('.-', '__')
1735
1736
1737 # Map @name to a valid C identifier.
1738 # If @protect, avoid returning certain ticklish identifiers (like
1739 # C keywords) by prepending 'q_'.
1740 #
1741 # Used for converting 'name' from a 'name':'type' qapi definition
1742 # into a generated struct member, as well as converting type names
1743 # into substrings of a generated C function name.
1744 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1745 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1746 def c_name(name, protect=True):
1747     # ANSI X3J11/88-090, 3.1.1
1748     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1749                      'default', 'do', 'double', 'else', 'enum', 'extern',
1750                      'float', 'for', 'goto', 'if', 'int', 'long', 'register',
1751                      'return', 'short', 'signed', 'sizeof', 'static',
1752                      'struct', 'switch', 'typedef', 'union', 'unsigned',
1753                      'void', 'volatile', 'while'])
1754     # ISO/IEC 9899:1999, 6.4.1
1755     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1756     # ISO/IEC 9899:2011, 6.4.1
1757     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
1758                      '_Noreturn', '_Static_assert', '_Thread_local'])
1759     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1760     # excluding _.*
1761     gcc_words = set(['asm', 'typeof'])
1762     # C++ ISO/IEC 14882:2003 2.11
1763     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1764                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1765                      'namespace', 'new', 'operator', 'private', 'protected',
1766                      'public', 'reinterpret_cast', 'static_cast', 'template',
1767                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1768                      'using', 'virtual', 'wchar_t',
1769                      # alternative representations
1770                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1771                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1772     # namespace pollution:
1773     polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
1774     name = name.translate(c_name_trans)
1775     if protect and (name in c89_words | c99_words | c11_words | gcc_words
1776                     | cpp_words | polluted_words):
1777         return 'q_' + name
1778     return name
1779
1780 eatspace = '\033EATSPACE.'
1781 pointer_suffix = ' *' + eatspace
1782
1783
1784 def genindent(count):
1785     ret = ''
1786     for _ in range(count):
1787         ret += ' '
1788     return ret
1789
1790 indent_level = 0
1791
1792
1793 def push_indent(indent_amount=4):
1794     global indent_level
1795     indent_level += indent_amount
1796
1797
1798 def pop_indent(indent_amount=4):
1799     global indent_level
1800     indent_level -= indent_amount
1801
1802
1803 # Generate @code with @kwds interpolated.
1804 # Obey indent_level, and strip eatspace.
1805 def cgen(code, **kwds):
1806     raw = code % kwds
1807     if indent_level:
1808         indent = genindent(indent_level)
1809         # re.subn() lacks flags support before Python 2.7, use re.compile()
1810         raw = re.subn(re.compile(r'^.', re.MULTILINE),
1811                       indent + r'\g<0>', raw)
1812         raw = raw[0]
1813     return re.sub(re.escape(eatspace) + r' *', '', raw)
1814
1815
1816 def mcgen(code, **kwds):
1817     if code[0] == '\n':
1818         code = code[1:]
1819     return cgen(code, **kwds)
1820
1821
1822 def guardname(filename):
1823     return c_name(filename, protect=False).upper()
1824
1825
1826 def guardstart(name):
1827     return mcgen('''
1828
1829 #ifndef %(name)s
1830 #define %(name)s
1831
1832 ''',
1833                  name=guardname(name))
1834
1835
1836 def guardend(name):
1837     return mcgen('''
1838
1839 #endif /* %(name)s */
1840
1841 ''',
1842                  name=guardname(name))
1843
1844
1845 def gen_enum_lookup(name, values, prefix=None):
1846     ret = mcgen('''
1847
1848 const QEnumLookup %(c_name)s_lookup = {
1849     .array = (const char *const[]) {
1850 ''',
1851                 c_name=c_name(name))
1852     for value in values:
1853         index = c_enum_const(name, value, prefix)
1854         ret += mcgen('''
1855         [%(index)s] = "%(value)s",
1856 ''',
1857                      index=index, value=value)
1858
1859     ret += mcgen('''
1860     },
1861     .size = %(max_index)s
1862 };
1863 ''',
1864                  max_index=c_enum_const(name, '_MAX', prefix))
1865     return ret
1866
1867
1868 def gen_enum(name, values, prefix=None):
1869     # append automatically generated _MAX value
1870     enum_values = values + ['_MAX']
1871
1872     ret = mcgen('''
1873
1874 typedef enum %(c_name)s {
1875 ''',
1876                 c_name=c_name(name))
1877
1878     i = 0
1879     for value in enum_values:
1880         ret += mcgen('''
1881     %(c_enum)s = %(i)d,
1882 ''',
1883                      c_enum=c_enum_const(name, value, prefix),
1884                      i=i)
1885         i += 1
1886
1887     ret += mcgen('''
1888 } %(c_name)s;
1889 ''',
1890                  c_name=c_name(name))
1891
1892     ret += mcgen('''
1893
1894 #define %(c_name)s_str(val) \\
1895     qapi_enum_lookup(&%(c_name)s_lookup, (val))
1896
1897 extern const QEnumLookup %(c_name)s_lookup;
1898 ''',
1899                  c_name=c_name(name))
1900     return ret
1901
1902
1903 def build_params(arg_type, boxed, extra):
1904     if not arg_type:
1905         assert not boxed
1906         return extra
1907     ret = ''
1908     sep = ''
1909     if boxed:
1910         ret += '%s arg' % arg_type.c_param_type()
1911         sep = ', '
1912     else:
1913         assert not arg_type.variants
1914         for memb in arg_type.members:
1915             ret += sep
1916             sep = ', '
1917             if memb.optional:
1918                 ret += 'bool has_%s, ' % c_name(memb.name)
1919             ret += '%s %s' % (memb.type.c_param_type(),
1920                               c_name(memb.name))
1921     if extra:
1922         ret += sep + extra
1923     return ret
1924
1925
1926 #
1927 # Common command line parsing
1928 #
1929
1930
1931 def parse_command_line(extra_options='', extra_long_options=[]):
1932
1933     try:
1934         opts, args = getopt.gnu_getopt(sys.argv[1:],
1935                                        'chp:o:' + extra_options,
1936                                        ['source', 'header', 'prefix=',
1937                                         'output-dir='] + extra_long_options)
1938     except getopt.GetoptError as err:
1939         print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
1940         sys.exit(1)
1941
1942     output_dir = ''
1943     prefix = ''
1944     do_c = False
1945     do_h = False
1946     extra_opts = []
1947
1948     for oa in opts:
1949         o, a = oa
1950         if o in ('-p', '--prefix'):
1951             match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1952             if match.end() != len(a):
1953                 print >>sys.stderr, \
1954                     "%s: 'funny character '%s' in argument of --prefix" \
1955                     % (sys.argv[0], a[match.end()])
1956                 sys.exit(1)
1957             prefix = a
1958         elif o in ('-o', '--output-dir'):
1959             output_dir = a + '/'
1960         elif o in ('-c', '--source'):
1961             do_c = True
1962         elif o in ('-h', '--header'):
1963             do_h = True
1964         else:
1965             extra_opts.append(oa)
1966
1967     if not do_c and not do_h:
1968         do_c = True
1969         do_h = True
1970
1971     if len(args) != 1:
1972         print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
1973         sys.exit(1)
1974     fname = args[0]
1975
1976     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
1977
1978 #
1979 # Generate output files with boilerplate
1980 #
1981
1982
1983 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
1984                 c_comment, h_comment):
1985     guard = guardname(prefix + h_file)
1986     c_file = output_dir + prefix + c_file
1987     h_file = output_dir + prefix + h_file
1988
1989     if output_dir:
1990         try:
1991             os.makedirs(output_dir)
1992         except os.error as e:
1993             if e.errno != errno.EEXIST:
1994                 raise
1995
1996     def maybe_open(really, name, opt):
1997         if really:
1998             return open(name, opt)
1999         else:
2000             import StringIO
2001             return StringIO.StringIO()
2002
2003     fdef = maybe_open(do_c, c_file, 'w')
2004     fdecl = maybe_open(do_h, h_file, 'w')
2005
2006     fdef.write(mcgen('''
2007 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2008 %(comment)s
2009 ''',
2010                      comment=c_comment))
2011
2012     fdecl.write(mcgen('''
2013 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2014 %(comment)s
2015 #ifndef %(guard)s
2016 #define %(guard)s
2017
2018 ''',
2019                       comment=h_comment, guard=guard))
2020
2021     return (fdef, fdecl)
2022
2023
2024 def close_output(fdef, fdecl):
2025     fdecl.write('''
2026 #endif
2027 ''')
2028     fdecl.close()
2029     fdef.close()