scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 import errno
  15 import getopt
  16 import os
  17 import re
  18 import string
  19 import sys
  20 from ordereddict import OrderedDict
  21
  22 builtin_types = {
  23     'null':     'QTYPE_QNULL',
  24     'str':      'QTYPE_QSTRING',
  25     'int':      'QTYPE_QNUM',
  26     'number':   'QTYPE_QNUM',
  27     'bool':     'QTYPE_QBOOL',
  28     'int8':     'QTYPE_QNUM',
  29     'int16':    'QTYPE_QNUM',
  30     'int32':    'QTYPE_QNUM',
  31     'int64':    'QTYPE_QNUM',
  32     'uint8':    'QTYPE_QNUM',
  33     'uint16':   'QTYPE_QNUM',
  34     'uint32':   'QTYPE_QNUM',
  35     'uint64':   'QTYPE_QNUM',
  36     'size':     'QTYPE_QNUM',
  37     'any':      None,           # any QType possible, actually
  38     'QType':    'QTYPE_QSTRING',
  39 }
  40
  41 # Are documentation comments required?
  42 doc_required = False
  43
  44 # Whitelist of commands allowed to return a non-dictionary
  45 returns_whitelist = []
  46
  47 # Whitelist of entities allowed to violate case conventions
  48 name_case_whitelist = []
  49
  50 enum_types = {}
  51 struct_types = {}
  52 union_types = {}
  53 all_names = {}
  54
  55 #
  56 # Parsing the schema into expressions
  57 #
  58
  59
  60 def error_path(parent):
  61     res = ''
  62     while parent:
  63         res = ('In file included from %s:%d:\n' % (parent['file'],
  64                                                    parent['line'])) + res
  65         parent = parent['parent']
  66     return res
  67
  68
  69 class QAPIError(Exception):
  70     def __init__(self, fname, line, col, incl_info, msg):
  71         Exception.__init__(self)
  72         self.fname = fname
  73         self.line = line
  74         self.col = col
  75         self.info = incl_info
  76         self.msg = msg
  77
  78     def __str__(self):
  79         loc = '%s:%d' % (self.fname, self.line)
  80         if self.col is not None:
  81             loc += ':%s' % self.col
  82         return error_path(self.info) + '%s: %s' % (loc, self.msg)
  83
  84
  85 class QAPIParseError(QAPIError):
  86     def __init__(self, parser, msg):
  87         col = 1
  88         for ch in parser.src[parser.line_pos:parser.pos]:
  89             if ch == '\t':
  90                 col = (col + 7) % 8 + 1
  91             else:
  92                 col += 1
  93         QAPIError.__init__(self, parser.fname, parser.line, col,
  94                            parser.incl_info, msg)
  95
  96
  97 class QAPISemError(QAPIError):
  98     def __init__(self, info, msg):
  99         QAPIError.__init__(self, info['file'], info['line'], None,
 100                            info['parent'], msg)
 101
 102
 103 class QAPIDoc(object):
 104     class Section(object):
 105         def __init__(self, name=None):
 106             # optional section name (argument/member or section name)
 107             self.name = name
 108             # the list of lines for this section
 109             self.content = []
 110
 111         def append(self, line):
 112             self.content.append(line)
 113
 114         def __repr__(self):
 115             return '\n'.join(self.content).strip()
 116
 117     class ArgSection(Section):
 118         def __init__(self, name):
 119             QAPIDoc.Section.__init__(self, name)
 120             self.member = None
 121
 122         def connect(self, member):
 123             self.member = member
 124
 125     def __init__(self, parser, info):
 126         # self.parser is used to report errors with QAPIParseError.  The
 127         # resulting error position depends on the state of the parser.
 128         # It happens to be the beginning of the comment.  More or less
 129         # servicable, but action at a distance.
 130         self.parser = parser
 131         self.info = info
 132         self.symbol = None
 133         self.body = QAPIDoc.Section()
 134         # dict mapping parameter name to ArgSection
 135         self.args = OrderedDict()
 136         # a list of Section
 137         self.sections = []
 138         # the current section
 139         self.section = self.body
 140
 141     def has_section(self, name):
 142         """Return True if we have a section with this name."""
 143         for i in self.sections:
 144             if i.name == name:
 145                 return True
 146         return False
 147
 148     def append(self, line):
 149         """Parse a comment line and add it to the documentation."""
 150         line = line[1:]
 151         if not line:
 152             self._append_freeform(line)
 153             return
 154
 155         if line[0] != ' ':
 156             raise QAPIParseError(self.parser, "Missing space after #")
 157         line = line[1:]
 158
 159         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 160         # recognized, and get silently treated as ordinary text
 161         if self.symbol:
 162             self._append_symbol_line(line)
 163         elif not self.body.content and line.startswith('@'):
 164             if not line.endswith(':'):
 165                 raise QAPIParseError(self.parser, "Line should end with :")
 166             self.symbol = line[1:-1]
 167             # FIXME invalid names other than the empty string aren't flagged
 168             if not self.symbol:
 169                 raise QAPIParseError(self.parser, "Invalid name")
 170         else:
 171             self._append_freeform(line)
 172
 173     def end_comment(self):
 174         self._end_section()
 175
 176     def _append_symbol_line(self, line):
 177         name = line.split(' ', 1)[0]
 178
 179         if name.startswith('@') and name.endswith(':'):
 180             line = line[len(name)+1:]
 181             self._start_args_section(name[1:-1])
 182         elif name in ('Returns:', 'Since:',
 183                       # those are often singular or plural
 184                       'Note:', 'Notes:',
 185                       'Example:', 'Examples:',
 186                       'TODO:'):
 187             line = line[len(name)+1:]
 188             self._start_section(name[:-1])
 189
 190         self._append_freeform(line)
 191
 192     def _start_args_section(self, name):
 193         # FIXME invalid names other than the empty string aren't flagged
 194         if not name:
 195             raise QAPIParseError(self.parser, "Invalid parameter name")
 196         if name in self.args:
 197             raise QAPIParseError(self.parser,
 198                                  "'%s' parameter name duplicated" % name)
 199         if self.sections:
 200             raise QAPIParseError(self.parser,
 201                                  "'@%s:' can't follow '%s' section"
 202                                  % (name, self.sections[0].name))
 203         self._end_section()
 204         self.section = QAPIDoc.ArgSection(name)
 205         self.args[name] = self.section
 206
 207     def _start_section(self, name=''):
 208         if name in ('Returns', 'Since') and self.has_section(name):
 209             raise QAPIParseError(self.parser,
 210                                  "Duplicated '%s' section" % name)
 211         self._end_section()
 212         self.section = QAPIDoc.Section(name)
 213         self.sections.append(self.section)
 214
 215     def _end_section(self):
 216         if self.section:
 217             contents = str(self.section)
 218             if self.section.name and (not contents or contents.isspace()):
 219                 raise QAPIParseError(self.parser, "Empty doc section '%s'"
 220                                      % self.section.name)
 221             self.section = None
 222
 223     def _append_freeform(self, line):
 224         in_arg = isinstance(self.section, QAPIDoc.ArgSection)
 225         if (in_arg and self.section.content
 226                 and not self.section.content[-1]
 227                 and line and not line[0].isspace()):
 228             self._start_section()
 229         if (in_arg or not self.section.name
 230                 or not self.section.name.startswith('Example')):
 231             line = line.strip()
 232         match = re.match(r'(@\S+:)', line)
 233         if match:
 234             raise QAPIParseError(self.parser,
 235                                  "'%s' not allowed in free-form documentation"
 236                                  % match.group(1))
 237         self.section.append(line)
 238
 239     def connect_member(self, member):
 240         if member.name not in self.args:
 241             # Undocumented TODO outlaw
 242             self.args[member.name] = QAPIDoc.ArgSection(member.name)
 243         self.args[member.name].connect(member)
 244
 245     def check_expr(self, expr):
 246         if self.has_section('Returns') and 'command' not in expr:
 247             raise QAPISemError(self.info,
 248                                "'Returns:' is only valid for commands")
 249
 250     def check(self):
 251         bogus = [name for name, section in self.args.iteritems()
 252                  if not section.member]
 253         if bogus:
 254             raise QAPISemError(
 255                 self.info,
 256                 "The following documented members are not in "
 257                 "the declaration: %s" % ", ".join(bogus))
 258
 259
 260 class QAPISchemaParser(object):
 261
 262     def __init__(self, fp, previously_included=[], incl_info=None):
 263         abs_fname = os.path.abspath(fp.name)
 264         self.fname = fp.name
 265         previously_included.append(abs_fname)
 266         self.incl_info = incl_info
 267         self.src = fp.read()
 268         if self.src == '' or self.src[-1] != '\n':
 269             self.src += '\n'
 270         self.cursor = 0
 271         self.line = 1
 272         self.line_pos = 0
 273         self.exprs = []
 274         self.docs = []
 275         self.cur_doc = None
 276         self.accept()
 277
 278         while self.tok is not None:
 279             info = {'file': self.fname, 'line': self.line,
 280                     'parent': self.incl_info}
 281             if self.tok == '#':
 282                 self.reject_expr_doc()
 283                 self.cur_doc = self.get_doc(info)
 284                 self.docs.append(self.cur_doc)
 285                 continue
 286
 287             expr = self.get_expr(False)
 288             if 'include' in expr:
 289                 self.reject_expr_doc()
 290                 if len(expr) != 1:
 291                     raise QAPISemError(info, "Invalid 'include' directive")
 292                 include = expr['include']
 293                 if not isinstance(include, str):
 294                     raise QAPISemError(info,
 295                                        "Value of 'include' must be a string")
 296                 self._include(include, info, os.path.dirname(abs_fname),
 297                               previously_included)
 298             elif "pragma" in expr:
 299                 self.reject_expr_doc()
 300                 if len(expr) != 1:
 301                     raise QAPISemError(info, "Invalid 'pragma' directive")
 302                 pragma = expr['pragma']
 303                 if not isinstance(pragma, dict):
 304                     raise QAPISemError(
 305                         info, "Value of 'pragma' must be a dictionary")
 306                 for name, value in pragma.iteritems():
 307                     self._pragma(name, value, info)
 308             else:
 309                 expr_elem = {'expr': expr,
 310                              'info': info}
 311                 if self.cur_doc:
 312                     if not self.cur_doc.symbol:
 313                         raise QAPISemError(
 314                             self.cur_doc.info,
 315                             "Expression documentation required")
 316                     expr_elem['doc'] = self.cur_doc
 317                 self.exprs.append(expr_elem)
 318             self.cur_doc = None
 319         self.reject_expr_doc()
 320
 321     def reject_expr_doc(self):
 322         if self.cur_doc and self.cur_doc.symbol:
 323             raise QAPISemError(
 324                 self.cur_doc.info,
 325                 "Documentation for '%s' is not followed by the definition"
 326                 % self.cur_doc.symbol)
 327
 328     def _include(self, include, info, base_dir, previously_included):
 329         incl_abs_fname = os.path.join(base_dir, include)
 330         # catch inclusion cycle
 331         inf = info
 332         while inf:
 333             if incl_abs_fname == os.path.abspath(inf['file']):
 334                 raise QAPISemError(info, "Inclusion loop for %s" % include)
 335             inf = inf['parent']
 336
 337         # skip multiple include of the same file
 338         if incl_abs_fname in previously_included:
 339             return
 340         try:
 341             fobj = open(incl_abs_fname, 'r')
 342         except IOError as e:
 343             raise QAPISemError(info, '%s: %s' % (e.strerror, include))
 344         exprs_include = QAPISchemaParser(fobj, previously_included, info)
 345         self.exprs.extend(exprs_include.exprs)
 346         self.docs.extend(exprs_include.docs)
 347
 348     def _pragma(self, name, value, info):
 349         global doc_required, returns_whitelist, name_case_whitelist
 350         if name == 'doc-required':
 351             if not isinstance(value, bool):
 352                 raise QAPISemError(info,
 353                                    "Pragma 'doc-required' must be boolean")
 354             doc_required = value
 355         elif name == 'returns-whitelist':
 356             if (not isinstance(value, list)
 357                     or any([not isinstance(elt, str) for elt in value])):
 358                 raise QAPISemError(info,
 359                                    "Pragma returns-whitelist must be"
 360                                    " a list of strings")
 361             returns_whitelist = value
 362         elif name == 'name-case-whitelist':
 363             if (not isinstance(value, list)
 364                     or any([not isinstance(elt, str) for elt in value])):
 365                 raise QAPISemError(info,
 366                                    "Pragma name-case-whitelist must be"
 367                                    " a list of strings")
 368             name_case_whitelist = value
 369         else:
 370             raise QAPISemError(info, "Unknown pragma '%s'" % name)
 371
 372     def accept(self, skip_comment=True):
 373         while True:
 374             self.tok = self.src[self.cursor]
 375             self.pos = self.cursor
 376             self.cursor += 1
 377             self.val = None
 378
 379             if self.tok == '#':
 380                 if self.src[self.cursor] == '#':
 381                     # Start of doc comment
 382                     skip_comment = False
 383                 self.cursor = self.src.find('\n', self.cursor)
 384                 if not skip_comment:
 385                     self.val = self.src[self.pos:self.cursor]
 386                     return
 387             elif self.tok in '{}:,[]':
 388                 return
 389             elif self.tok == "'":
 390                 string = ''
 391                 esc = False
 392                 while True:
 393                     ch = self.src[self.cursor]
 394                     self.cursor += 1
 395                     if ch == '\n':
 396                         raise QAPIParseError(self, 'Missing terminating "\'"')
 397                     if esc:
 398                         if ch == 'b':
 399                             string += '\b'
 400                         elif ch == 'f':
 401                             string += '\f'
 402                         elif ch == 'n':
 403                             string += '\n'
 404                         elif ch == 'r':
 405                             string += '\r'
 406                         elif ch == 't':
 407                             string += '\t'
 408                         elif ch == 'u':
 409                             value = 0
 410                             for _ in range(0, 4):
 411                                 ch = self.src[self.cursor]
 412                                 self.cursor += 1
 413                                 if ch not in '0123456789abcdefABCDEF':
 414                                     raise QAPIParseError(self,
 415                                                          '\\u escape needs 4 '
 416                                                          'hex digits')
 417                                 value = (value << 4) + int(ch, 16)
 418                             # If Python 2 and 3 didn't disagree so much on
 419                             # how to handle Unicode, then we could allow
 420                             # Unicode string defaults.  But most of QAPI is
 421                             # ASCII-only, so we aren't losing much for now.
 422                             if not value or value > 0x7f:
 423                                 raise QAPIParseError(self,
 424                                                      'For now, \\u escape '
 425                                                      'only supports non-zero '
 426                                                      'values up to \\u007f')
 427                             string += chr(value)
 428                         elif ch in '\\/\'"':
 429                             string += ch
 430                         else:
 431                             raise QAPIParseError(self,
 432                                                  "Unknown escape \\%s" % ch)
 433                         esc = False
 434                     elif ch == '\\':
 435                         esc = True
 436                     elif ch == "'":
 437                         self.val = string
 438                         return
 439                     else:
 440                         string += ch
 441             elif self.src.startswith('true', self.pos):
 442                 self.val = True
 443                 self.cursor += 3
 444                 return
 445             elif self.src.startswith('false', self.pos):
 446                 self.val = False
 447                 self.cursor += 4
 448                 return
 449             elif self.src.startswith('null', self.pos):
 450                 self.val = None
 451                 self.cursor += 3
 452                 return
 453             elif self.tok == '\n':
 454                 if self.cursor == len(self.src):
 455                     self.tok = None
 456                     return
 457                 self.line += 1
 458                 self.line_pos = self.cursor
 459             elif not self.tok.isspace():
 460                 raise QAPIParseError(self, 'Stray "%s"' % self.tok)
 461
 462     def get_members(self):
 463         expr = OrderedDict()
 464         if self.tok == '}':
 465             self.accept()
 466             return expr
 467         if self.tok != "'":
 468             raise QAPIParseError(self, 'Expected string or "}"')
 469         while True:
 470             key = self.val
 471             self.accept()
 472             if self.tok != ':':
 473                 raise QAPIParseError(self, 'Expected ":"')
 474             self.accept()
 475             if key in expr:
 476                 raise QAPIParseError(self, 'Duplicate key "%s"' % key)
 477             expr[key] = self.get_expr(True)
 478             if self.tok == '}':
 479                 self.accept()
 480                 return expr
 481             if self.tok != ',':
 482                 raise QAPIParseError(self, 'Expected "," or "}"')
 483             self.accept()
 484             if self.tok != "'":
 485                 raise QAPIParseError(self, 'Expected string')
 486
 487     def get_values(self):
 488         expr = []
 489         if self.tok == ']':
 490             self.accept()
 491             return expr
 492         if self.tok not in "{['tfn":
 493             raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
 494                                  'boolean or "null"')
 495         while True:
 496             expr.append(self.get_expr(True))
 497             if self.tok == ']':
 498                 self.accept()
 499                 return expr
 500             if self.tok != ',':
 501                 raise QAPIParseError(self, 'Expected "," or "]"')
 502             self.accept()
 503
 504     def get_expr(self, nested):
 505         if self.tok != '{' and not nested:
 506             raise QAPIParseError(self, 'Expected "{"')
 507         if self.tok == '{':
 508             self.accept()
 509             expr = self.get_members()
 510         elif self.tok == '[':
 511             self.accept()
 512             expr = self.get_values()
 513         elif self.tok in "'tfn":
 514             expr = self.val
 515             self.accept()
 516         else:
 517             raise QAPIParseError(self, 'Expected "{", "[", string, '
 518                                  'boolean or "null"')
 519         return expr
 520
 521     def get_doc(self, info):
 522         if self.val != '##':
 523             raise QAPIParseError(self, "Junk after '##' at start of "
 524                                  "documentation comment")
 525
 526         doc = QAPIDoc(self, info)
 527         self.accept(False)
 528         while self.tok == '#':
 529             if self.val.startswith('##'):
 530                 # End of doc comment
 531                 if self.val != '##':
 532                     raise QAPIParseError(self, "Junk after '##' at end of "
 533                                          "documentation comment")
 534                 doc.end_comment()
 535                 self.accept()
 536                 return doc
 537             else:
 538                 doc.append(self.val)
 539             self.accept(False)
 540
 541         raise QAPIParseError(self, "Documentation comment must end with '##'")
 542
 543
 544 #
 545 # Semantic analysis of schema expressions
 546 # TODO fold into QAPISchema
 547 # TODO catching name collisions in generated code would be nice
 548 #
 549
 550
 551 def find_base_members(base):
 552     if isinstance(base, dict):
 553         return base
 554     base_struct_define = struct_types.get(base)
 555     if not base_struct_define:
 556         return None
 557     return base_struct_define['data']
 558
 559
 560 # Return the qtype of an alternate branch, or None on error.
 561 def find_alternate_member_qtype(qapi_type):
 562     if qapi_type in builtin_types:
 563         return builtin_types[qapi_type]
 564     elif qapi_type in struct_types:
 565         return 'QTYPE_QDICT'
 566     elif qapi_type in enum_types:
 567         return 'QTYPE_QSTRING'
 568     elif qapi_type in union_types:
 569         return 'QTYPE_QDICT'
 570     return None
 571
 572
 573 # Return the discriminator enum define if discriminator is specified as an
 574 # enum type, otherwise return None.
 575 def discriminator_find_enum_define(expr):
 576     base = expr.get('base')
 577     discriminator = expr.get('discriminator')
 578
 579     if not (discriminator and base):
 580         return None
 581
 582     base_members = find_base_members(base)
 583     if not base_members:
 584         return None
 585
 586     discriminator_type = base_members.get(discriminator)
 587     if not discriminator_type:
 588         return None
 589
 590     return enum_types.get(discriminator_type)
 591
 592
 593 # Names must be letters, numbers, -, and _.  They must start with letter,
 594 # except for downstream extensions which must start with __RFQDN_.
 595 # Dots are only valid in the downstream extension prefix.
 596 valid_name = re.compile(r'^(__[a-zA-Z0-9.-]+_)?'
 597                         '[a-zA-Z][a-zA-Z0-9_-]*$')
 598
 599
 600 def check_name(info, source, name, allow_optional=False,
 601                enum_member=False):
 602     global valid_name
 603     membername = name
 604
 605     if not isinstance(name, str):
 606         raise QAPISemError(info, "%s requires a string name" % source)
 607     if name.startswith('*'):
 608         membername = name[1:]
 609         if not allow_optional:
 610             raise QAPISemError(info, "%s does not allow optional name '%s'"
 611                                % (source, name))
 612     # Enum members can start with a digit, because the generated C
 613     # code always prefixes it with the enum name
 614     if enum_member and membername[0].isdigit():
 615         membername = 'D' + membername
 616     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
 617     # and 'q_obj_*' implicit type names.
 618     if not valid_name.match(membername) or \
 619        c_name(membername, False).startswith('q_'):
 620         raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
 621
 622
 623 def add_name(name, info, meta, implicit=False):
 624     global all_names
 625     check_name(info, "'%s'" % meta, name)
 626     # FIXME should reject names that differ only in '_' vs. '.'
 627     # vs. '-', because they're liable to clash in generated C.
 628     if name in all_names:
 629         raise QAPISemError(info, "%s '%s' is already defined"
 630                            % (all_names[name], name))
 631     if not implicit and (name.endswith('Kind') or name.endswith('List')):
 632         raise QAPISemError(info, "%s '%s' should not end in '%s'"
 633                            % (meta, name, name[-4:]))
 634     all_names[name] = meta
 635
 636
 637 def check_type(info, source, value, allow_array=False,
 638                allow_dict=False, allow_optional=False,
 639                allow_metas=[]):
 640     global all_names
 641
 642     if value is None:
 643         return
 644
 645     # Check if array type for value is okay
 646     if isinstance(value, list):
 647         if not allow_array:
 648             raise QAPISemError(info, "%s cannot be an array" % source)
 649         if len(value) != 1 or not isinstance(value[0], str):
 650             raise QAPISemError(info,
 651                                "%s: array type must contain single type name" %
 652                                source)
 653         value = value[0]
 654
 655     # Check if type name for value is okay
 656     if isinstance(value, str):
 657         if value not in all_names:
 658             raise QAPISemError(info, "%s uses unknown type '%s'"
 659                                % (source, value))
 660         if not all_names[value] in allow_metas:
 661             raise QAPISemError(info, "%s cannot use %s type '%s'" %
 662                                (source, all_names[value], value))
 663         return
 664
 665     if not allow_dict:
 666         raise QAPISemError(info, "%s should be a type name" % source)
 667
 668     if not isinstance(value, OrderedDict):
 669         raise QAPISemError(info,
 670                            "%s should be a dictionary or type name" % source)
 671
 672     # value is a dictionary, check that each member is okay
 673     for (key, arg) in value.items():
 674         check_name(info, "Member of %s" % source, key,
 675                    allow_optional=allow_optional)
 676         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 677             raise QAPISemError(info, "Member of %s uses reserved name '%s'"
 678                                % (source, key))
 679         # Todo: allow dictionaries to represent default values of
 680         # an optional argument.
 681         check_type(info, "Member '%s' of %s" % (key, source), arg,
 682                    allow_array=True,
 683                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 684                                 'enum'])
 685
 686
 687 def check_command(expr, info):
 688     name = expr['command']
 689     boxed = expr.get('boxed', False)
 690
 691     args_meta = ['struct']
 692     if boxed:
 693         args_meta += ['union', 'alternate']
 694     check_type(info, "'data' for command '%s'" % name,
 695                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 696                allow_metas=args_meta)
 697     returns_meta = ['union', 'struct']
 698     if name in returns_whitelist:
 699         returns_meta += ['built-in', 'alternate', 'enum']
 700     check_type(info, "'returns' for command '%s'" % name,
 701                expr.get('returns'), allow_array=True,
 702                allow_optional=True, allow_metas=returns_meta)
 703
 704
 705 def check_event(expr, info):
 706     name = expr['event']
 707     boxed = expr.get('boxed', False)
 708
 709     meta = ['struct']
 710     if boxed:
 711         meta += ['union', 'alternate']
 712     check_type(info, "'data' for event '%s'" % name,
 713                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 714                allow_metas=meta)
 715
 716
 717 def check_union(expr, info):
 718     name = expr['union']
 719     base = expr.get('base')
 720     discriminator = expr.get('discriminator')
 721     members = expr['data']
 722
 723     # Two types of unions, determined by discriminator.
 724
 725     # With no discriminator it is a simple union.
 726     if discriminator is None:
 727         enum_define = None
 728         allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
 729         if base is not None:
 730             raise QAPISemError(info, "Simple union '%s' must not have a base" %
 731                                name)
 732
 733     # Else, it's a flat union.
 734     else:
 735         # The object must have a string or dictionary 'base'.
 736         check_type(info, "'base' for union '%s'" % name,
 737                    base, allow_dict=True, allow_optional=True,
 738                    allow_metas=['struct'])
 739         if not base:
 740             raise QAPISemError(info, "Flat union '%s' must have a base"
 741                                % name)
 742         base_members = find_base_members(base)
 743         assert base_members is not None
 744
 745         # The value of member 'discriminator' must name a non-optional
 746         # member of the base struct.
 747         check_name(info, "Discriminator of flat union '%s'" % name,
 748                    discriminator)
 749         discriminator_type = base_members.get(discriminator)
 750         if not discriminator_type:
 751             raise QAPISemError(info,
 752                                "Discriminator '%s' is not a member of base "
 753                                "struct '%s'"
 754                                % (discriminator, base))
 755         enum_define = enum_types.get(discriminator_type)
 756         allow_metas = ['struct']
 757         # Do not allow string discriminator
 758         if not enum_define:
 759             raise QAPISemError(info,
 760                                "Discriminator '%s' must be of enumeration "
 761                                "type" % discriminator)
 762
 763     # Check every branch; don't allow an empty union
 764     if len(members) == 0:
 765         raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
 766     for (key, value) in members.items():
 767         check_name(info, "Member of union '%s'" % name, key)
 768
 769         # Each value must name a known type
 770         check_type(info, "Member '%s' of union '%s'" % (key, name),
 771                    value, allow_array=not base, allow_metas=allow_metas)
 772
 773         # If the discriminator names an enum type, then all members
 774         # of 'data' must also be members of the enum type.
 775         if enum_define:
 776             if key not in enum_define['data']:
 777                 raise QAPISemError(info,
 778                                    "Discriminator value '%s' is not found in "
 779                                    "enum '%s'"
 780                                    % (key, enum_define['enum']))
 781
 782     # If discriminator is user-defined, ensure all values are covered
 783     if enum_define:
 784         for value in enum_define['data']:
 785             if value not in members.keys():
 786                 raise QAPISemError(info, "Union '%s' data missing '%s' branch"
 787                                    % (name, value))
 788
 789
 790 def check_alternate(expr, info):
 791     name = expr['alternate']
 792     members = expr['data']
 793     types_seen = {}
 794
 795     # Check every branch; require at least two branches
 796     if len(members) < 2:
 797         raise QAPISemError(info,
 798                            "Alternate '%s' should have at least two branches "
 799                            "in 'data'" % name)
 800     for (key, value) in members.items():
 801         check_name(info, "Member of alternate '%s'" % name, key)
 802
 803         # Ensure alternates have no type conflicts.
 804         check_type(info, "Member '%s' of alternate '%s'" % (key, name),
 805                    value,
 806                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 807         qtype = find_alternate_member_qtype(value)
 808         if not qtype:
 809             raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
 810                                "type '%s'" % (name, key, value))
 811         conflicting = set([qtype])
 812         if qtype == 'QTYPE_QSTRING':
 813             enum_expr = enum_types.get(value)
 814             if enum_expr:
 815                 for v in enum_expr['data']:
 816                     if v in ['on', 'off']:
 817                         conflicting.add('QTYPE_QBOOL')
 818                     if re.match(r'[-+0-9.]', v): # lazy, could be tightened
 819                         conflicting.add('QTYPE_QNUM')
 820             else:
 821                 conflicting.add('QTYPE_QNUM')
 822                 conflicting.add('QTYPE_QBOOL')
 823         for qt in conflicting:
 824             if qt in types_seen:
 825                 raise QAPISemError(info, "Alternate '%s' member '%s' can't "
 826                                    "be distinguished from member '%s'"
 827                                    % (name, key, types_seen[qt]))
 828             types_seen[qt] = key
 829
 830
 831 def check_enum(expr, info):
 832     name = expr['enum']
 833     members = expr.get('data')
 834     prefix = expr.get('prefix')
 835
 836     if not isinstance(members, list):
 837         raise QAPISemError(info,
 838                            "Enum '%s' requires an array for 'data'" % name)
 839     if prefix is not None and not isinstance(prefix, str):
 840         raise QAPISemError(info,
 841                            "Enum '%s' requires a string for 'prefix'" % name)
 842     for member in members:
 843         check_name(info, "Member of enum '%s'" % name, member,
 844                    enum_member=True)
 845
 846
 847 def check_struct(expr, info):
 848     name = expr['struct']
 849     members = expr['data']
 850
 851     check_type(info, "'data' for struct '%s'" % name, members,
 852                allow_dict=True, allow_optional=True)
 853     check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
 854                allow_metas=['struct'])
 855
 856
 857 def check_keys(expr_elem, meta, required, optional=[]):
 858     expr = expr_elem['expr']
 859     info = expr_elem['info']
 860     name = expr[meta]
 861     if not isinstance(name, str):
 862         raise QAPISemError(info, "'%s' key must have a string value" % meta)
 863     required = required + [meta]
 864     for (key, value) in expr.items():
 865         if key not in required and key not in optional:
 866             raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
 867                                % (key, meta, name))
 868         if (key == 'gen' or key == 'success-response') and value is not False:
 869             raise QAPISemError(info,
 870                                "'%s' of %s '%s' should only use false value"
 871                                % (key, meta, name))
 872         if key == 'boxed' and value is not True:
 873             raise QAPISemError(info,
 874                                "'%s' of %s '%s' should only use true value"
 875                                % (key, meta, name))
 876     for key in required:
 877         if key not in expr:
 878             raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
 879                                % (key, meta, name))
 880
 881
 882 def check_exprs(exprs):
 883     global all_names
 884
 885     # Populate name table with names of built-in types
 886     for builtin in builtin_types.keys():
 887         all_names[builtin] = 'built-in'
 888
 889     # Learn the types and check for valid expression keys
 890     for expr_elem in exprs:
 891         expr = expr_elem['expr']
 892         info = expr_elem['info']
 893         doc = expr_elem.get('doc')
 894
 895         if not doc and doc_required:
 896             raise QAPISemError(info,
 897                                "Expression missing documentation comment")
 898
 899         if 'enum' in expr:
 900             meta = 'enum'
 901             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 902             enum_types[expr[meta]] = expr
 903         elif 'union' in expr:
 904             meta = 'union'
 905             check_keys(expr_elem, 'union', ['data'],
 906                        ['base', 'discriminator'])
 907             union_types[expr[meta]] = expr
 908         elif 'alternate' in expr:
 909             meta = 'alternate'
 910             check_keys(expr_elem, 'alternate', ['data'])
 911         elif 'struct' in expr:
 912             meta = 'struct'
 913             check_keys(expr_elem, 'struct', ['data'], ['base'])
 914             struct_types[expr[meta]] = expr
 915         elif 'command' in expr:
 916             meta = 'command'
 917             check_keys(expr_elem, 'command', [],
 918                        ['data', 'returns', 'gen', 'success-response', 'boxed'])
 919         elif 'event' in expr:
 920             meta = 'event'
 921             check_keys(expr_elem, 'event', [], ['data', 'boxed'])
 922         else:
 923             raise QAPISemError(expr_elem['info'],
 924                                "Expression is missing metatype")
 925         name = expr[meta]
 926         add_name(name, info, meta)
 927         if doc and doc.symbol != name:
 928             raise QAPISemError(info, "Definition of '%s' follows documentation"
 929                                " for '%s'" % (name, doc.symbol))
 930
 931     # Try again for hidden UnionKind enum
 932     for expr_elem in exprs:
 933         expr = expr_elem['expr']
 934         if 'union' in expr and not discriminator_find_enum_define(expr):
 935             name = '%sKind' % expr['union']
 936         elif 'alternate' in expr:
 937             name = '%sKind' % expr['alternate']
 938         else:
 939             continue
 940         enum_types[name] = {'enum': name}
 941         add_name(name, info, 'enum', implicit=True)
 942
 943     # Validate that exprs make sense
 944     for expr_elem in exprs:
 945         expr = expr_elem['expr']
 946         info = expr_elem['info']
 947         doc = expr_elem.get('doc')
 948
 949         if 'enum' in expr:
 950             check_enum(expr, info)
 951         elif 'union' in expr:
 952             check_union(expr, info)
 953         elif 'alternate' in expr:
 954             check_alternate(expr, info)
 955         elif 'struct' in expr:
 956             check_struct(expr, info)
 957         elif 'command' in expr:
 958             check_command(expr, info)
 959         elif 'event' in expr:
 960             check_event(expr, info)
 961         else:
 962             assert False, 'unexpected meta type'
 963
 964         if doc:
 965             doc.check_expr(expr)
 966
 967     return exprs
 968
 969
 970 #
 971 # Schema compiler frontend
 972 #
 973
 974 class QAPISchemaEntity(object):
 975     def __init__(self, name, info, doc):
 976         assert isinstance(name, str)
 977         self.name = name
 978         # For explicitly defined entities, info points to the (explicit)
 979         # definition.  For builtins (and their arrays), info is None.
 980         # For implicitly defined entities, info points to a place that
 981         # triggered the implicit definition (there may be more than one
 982         # such place).
 983         self.info = info
 984         self.doc = doc
 985
 986     def c_name(self):
 987         return c_name(self.name)
 988
 989     def check(self, schema):
 990         pass
 991
 992     def is_implicit(self):
 993         return not self.info
 994
 995     def visit(self, visitor):
 996         pass
 997
 998
 999 class QAPISchemaVisitor(object):
1000     def visit_begin(self, schema):
1001         pass
1002
1003     def visit_end(self):
1004         pass
1005
1006     def visit_needed(self, entity):
1007         # Default to visiting everything
1008         return True
1009
1010     def visit_builtin_type(self, name, info, json_type):
1011         pass
1012
1013     def visit_enum_type(self, name, info, values, prefix):
1014         pass
1015
1016     def visit_array_type(self, name, info, element_type):
1017         pass
1018
1019     def visit_object_type(self, name, info, base, members, variants):
1020         pass
1021
1022     def visit_object_type_flat(self, name, info, members, variants):
1023         pass
1024
1025     def visit_alternate_type(self, name, info, variants):
1026         pass
1027
1028     def visit_command(self, name, info, arg_type, ret_type,
1029                       gen, success_response, boxed):
1030         pass
1031
1032     def visit_event(self, name, info, arg_type, boxed):
1033         pass
1034
1035
1036 class QAPISchemaType(QAPISchemaEntity):
1037     # Return the C type for common use.
1038     # For the types we commonly box, this is a pointer type.
1039     def c_type(self):
1040         pass
1041
1042     # Return the C type to be used in a parameter list.
1043     def c_param_type(self):
1044         return self.c_type()
1045
1046     # Return the C type to be used where we suppress boxing.
1047     def c_unboxed_type(self):
1048         return self.c_type()
1049
1050     def json_type(self):
1051         pass
1052
1053     def alternate_qtype(self):
1054         json2qtype = {
1055             'null':    'QTYPE_QNULL',
1056             'string':  'QTYPE_QSTRING',
1057             'number':  'QTYPE_QNUM',
1058             'int':     'QTYPE_QNUM',
1059             'boolean': 'QTYPE_QBOOL',
1060             'object':  'QTYPE_QDICT'
1061         }
1062         return json2qtype.get(self.json_type())
1063
1064     def doc_type(self):
1065         if self.is_implicit():
1066             return None
1067         return self.name
1068
1069
1070 class QAPISchemaBuiltinType(QAPISchemaType):
1071     def __init__(self, name, json_type, c_type):
1072         QAPISchemaType.__init__(self, name, None, None)
1073         assert not c_type or isinstance(c_type, str)
1074         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
1075                              'value')
1076         self._json_type_name = json_type
1077         self._c_type_name = c_type
1078
1079     def c_name(self):
1080         return self.name
1081
1082     def c_type(self):
1083         return self._c_type_name
1084
1085     def c_param_type(self):
1086         if self.name == 'str':
1087             return 'const ' + self._c_type_name
1088         return self._c_type_name
1089
1090     def json_type(self):
1091         return self._json_type_name
1092
1093     def doc_type(self):
1094         return self.json_type()
1095
1096     def visit(self, visitor):
1097         visitor.visit_builtin_type(self.name, self.info, self.json_type())
1098
1099
1100 class QAPISchemaEnumType(QAPISchemaType):
1101     def __init__(self, name, info, doc, values, prefix):
1102         QAPISchemaType.__init__(self, name, info, doc)
1103         for v in values:
1104             assert isinstance(v, QAPISchemaMember)
1105             v.set_owner(name)
1106         assert prefix is None or isinstance(prefix, str)
1107         self.values = values
1108         self.prefix = prefix
1109
1110     def check(self, schema):
1111         seen = {}
1112         for v in self.values:
1113             v.check_clash(self.info, seen)
1114             if self.doc:
1115                 self.doc.connect_member(v)
1116
1117     def is_implicit(self):
1118         # See QAPISchema._make_implicit_enum_type() and ._def_predefineds()
1119         return self.name.endswith('Kind') or self.name == 'QType'
1120
1121     def c_type(self):
1122         return c_name(self.name)
1123
1124     def member_names(self):
1125         return [v.name for v in self.values]
1126
1127     def json_type(self):
1128         return 'string'
1129
1130     def visit(self, visitor):
1131         visitor.visit_enum_type(self.name, self.info,
1132                                 self.member_names(), self.prefix)
1133
1134
1135 class QAPISchemaArrayType(QAPISchemaType):
1136     def __init__(self, name, info, element_type):
1137         QAPISchemaType.__init__(self, name, info, None)
1138         assert isinstance(element_type, str)
1139         self._element_type_name = element_type
1140         self.element_type = None
1141
1142     def check(self, schema):
1143         self.element_type = schema.lookup_type(self._element_type_name)
1144         assert self.element_type
1145
1146     def is_implicit(self):
1147         return True
1148
1149     def c_type(self):
1150         return c_name(self.name) + pointer_suffix
1151
1152     def json_type(self):
1153         return 'array'
1154
1155     def doc_type(self):
1156         elt_doc_type = self.element_type.doc_type()
1157         if not elt_doc_type:
1158             return None
1159         return 'array of ' + elt_doc_type
1160
1161     def visit(self, visitor):
1162         visitor.visit_array_type(self.name, self.info, self.element_type)
1163
1164
1165 class QAPISchemaObjectType(QAPISchemaType):
1166     def __init__(self, name, info, doc, base, local_members, variants):
1167         # struct has local_members, optional base, and no variants
1168         # flat union has base, variants, and no local_members
1169         # simple union has local_members, variants, and no base
1170         QAPISchemaType.__init__(self, name, info, doc)
1171         assert base is None or isinstance(base, str)
1172         for m in local_members:
1173             assert isinstance(m, QAPISchemaObjectTypeMember)
1174             m.set_owner(name)
1175         if variants is not None:
1176             assert isinstance(variants, QAPISchemaObjectTypeVariants)
1177             variants.set_owner(name)
1178         self._base_name = base
1179         self.base = None
1180         self.local_members = local_members
1181         self.variants = variants
1182         self.members = None
1183
1184     def check(self, schema):
1185         if self.members is False:               # check for cycles
1186             raise QAPISemError(self.info,
1187                                "Object %s contains itself" % self.name)
1188         if self.members:
1189             return
1190         self.members = False                    # mark as being checked
1191         seen = OrderedDict()
1192         if self._base_name:
1193             self.base = schema.lookup_type(self._base_name)
1194             assert isinstance(self.base, QAPISchemaObjectType)
1195             self.base.check(schema)
1196             self.base.check_clash(self.info, seen)
1197         for m in self.local_members:
1198             m.check(schema)
1199             m.check_clash(self.info, seen)
1200             if self.doc:
1201                 self.doc.connect_member(m)
1202         self.members = seen.values()
1203         if self.variants:
1204             self.variants.check(schema, seen)
1205             assert self.variants.tag_member in self.members
1206             self.variants.check_clash(self.info, seen)
1207         if self.doc:
1208             self.doc.check()
1209
1210     # Check that the members of this type do not cause duplicate JSON members,
1211     # and update seen to track the members seen so far. Report any errors
1212     # on behalf of info, which is not necessarily self.info
1213     def check_clash(self, info, seen):
1214         assert not self.variants       # not implemented
1215         for m in self.members:
1216             m.check_clash(info, seen)
1217
1218     def is_implicit(self):
1219         # See QAPISchema._make_implicit_object_type(), as well as
1220         # _def_predefineds()
1221         return self.name.startswith('q_')
1222
1223     def is_empty(self):
1224         assert self.members is not None
1225         return not self.members and not self.variants
1226
1227     def c_name(self):
1228         assert self.name != 'q_empty'
1229         return QAPISchemaType.c_name(self)
1230
1231     def c_type(self):
1232         assert not self.is_implicit()
1233         return c_name(self.name) + pointer_suffix
1234
1235     def c_unboxed_type(self):
1236         return c_name(self.name)
1237
1238     def json_type(self):
1239         return 'object'
1240
1241     def visit(self, visitor):
1242         visitor.visit_object_type(self.name, self.info,
1243                                   self.base, self.local_members, self.variants)
1244         visitor.visit_object_type_flat(self.name, self.info,
1245                                        self.members, self.variants)
1246
1247
1248 class QAPISchemaMember(object):
1249     role = 'member'
1250
1251     def __init__(self, name):
1252         assert isinstance(name, str)
1253         self.name = name
1254         self.owner = None
1255
1256     def set_owner(self, name):
1257         assert not self.owner
1258         self.owner = name
1259
1260     def check_clash(self, info, seen):
1261         cname = c_name(self.name)
1262         if cname.lower() != cname and self.owner not in name_case_whitelist:
1263             raise QAPISemError(info,
1264                                "%s should not use uppercase" % self.describe())
1265         if cname in seen:
1266             raise QAPISemError(info, "%s collides with %s" %
1267                                (self.describe(), seen[cname].describe()))
1268         seen[cname] = self
1269
1270     def _pretty_owner(self):
1271         owner = self.owner
1272         if owner.startswith('q_obj_'):
1273             # See QAPISchema._make_implicit_object_type() - reverse the
1274             # mapping there to create a nice human-readable description
1275             owner = owner[6:]
1276             if owner.endswith('-arg'):
1277                 return '(parameter of %s)' % owner[:-4]
1278             elif owner.endswith('-base'):
1279                 return '(base of %s)' % owner[:-5]
1280             else:
1281                 assert owner.endswith('-wrapper')
1282                 # Unreachable and not implemented
1283                 assert False
1284         if owner.endswith('Kind'):
1285             # See QAPISchema._make_implicit_enum_type()
1286             return '(branch of %s)' % owner[:-4]
1287         return '(%s of %s)' % (self.role, owner)
1288
1289     def describe(self):
1290         return "'%s' %s" % (self.name, self._pretty_owner())
1291
1292
1293 class QAPISchemaObjectTypeMember(QAPISchemaMember):
1294     def __init__(self, name, typ, optional):
1295         QAPISchemaMember.__init__(self, name)
1296         assert isinstance(typ, str)
1297         assert isinstance(optional, bool)
1298         self._type_name = typ
1299         self.type = None
1300         self.optional = optional
1301
1302     def check(self, schema):
1303         assert self.owner
1304         self.type = schema.lookup_type(self._type_name)
1305         assert self.type
1306
1307
1308 class QAPISchemaObjectTypeVariants(object):
1309     def __init__(self, tag_name, tag_member, variants):
1310         # Flat unions pass tag_name but not tag_member.
1311         # Simple unions and alternates pass tag_member but not tag_name.
1312         # After check(), tag_member is always set, and tag_name remains
1313         # a reliable witness of being used by a flat union.
1314         assert bool(tag_member) != bool(tag_name)
1315         assert (isinstance(tag_name, str) or
1316                 isinstance(tag_member, QAPISchemaObjectTypeMember))
1317         assert len(variants) > 0
1318         for v in variants:
1319             assert isinstance(v, QAPISchemaObjectTypeVariant)
1320         self._tag_name = tag_name
1321         self.tag_member = tag_member
1322         self.variants = variants
1323
1324     def set_owner(self, name):
1325         for v in self.variants:
1326             v.set_owner(name)
1327
1328     def check(self, schema, seen):
1329         if not self.tag_member:    # flat union
1330             self.tag_member = seen[c_name(self._tag_name)]
1331             assert self._tag_name == self.tag_member.name
1332         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
1333         for v in self.variants:
1334             v.check(schema)
1335             # Union names must match enum values; alternate names are
1336             # checked separately. Use 'seen' to tell the two apart.
1337             if seen:
1338                 assert v.name in self.tag_member.type.member_names()
1339                 assert isinstance(v.type, QAPISchemaObjectType)
1340                 v.type.check(schema)
1341
1342     def check_clash(self, info, seen):
1343         for v in self.variants:
1344             # Reset seen map for each variant, since qapi names from one
1345             # branch do not affect another branch
1346             assert isinstance(v.type, QAPISchemaObjectType)
1347             v.type.check_clash(info, dict(seen))
1348
1349
1350 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1351     role = 'branch'
1352
1353     def __init__(self, name, typ):
1354         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1355
1356
1357 class QAPISchemaAlternateType(QAPISchemaType):
1358     def __init__(self, name, info, doc, variants):
1359         QAPISchemaType.__init__(self, name, info, doc)
1360         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1361         assert variants.tag_member
1362         variants.set_owner(name)
1363         variants.tag_member.set_owner(self.name)
1364         self.variants = variants
1365
1366     def check(self, schema):
1367         self.variants.tag_member.check(schema)
1368         # Not calling self.variants.check_clash(), because there's nothing
1369         # to clash with
1370         self.variants.check(schema, {})
1371         # Alternate branch names have no relation to the tag enum values;
1372         # so we have to check for potential name collisions ourselves.
1373         seen = {}
1374         for v in self.variants.variants:
1375             v.check_clash(self.info, seen)
1376             if self.doc:
1377                 self.doc.connect_member(v)
1378         if self.doc:
1379             self.doc.check()
1380
1381     def c_type(self):
1382         return c_name(self.name) + pointer_suffix
1383
1384     def json_type(self):
1385         return 'value'
1386
1387     def visit(self, visitor):
1388         visitor.visit_alternate_type(self.name, self.info, self.variants)
1389
1390     def is_empty(self):
1391         return False
1392
1393
1394 class QAPISchemaCommand(QAPISchemaEntity):
1395     def __init__(self, name, info, doc, arg_type, ret_type,
1396                  gen, success_response, boxed):
1397         QAPISchemaEntity.__init__(self, name, info, doc)
1398         assert not arg_type or isinstance(arg_type, str)
1399         assert not ret_type or isinstance(ret_type, str)
1400         self._arg_type_name = arg_type
1401         self.arg_type = None
1402         self._ret_type_name = ret_type
1403         self.ret_type = None
1404         self.gen = gen
1405         self.success_response = success_response
1406         self.boxed = boxed
1407
1408     def check(self, schema):
1409         if self._arg_type_name:
1410             self.arg_type = schema.lookup_type(self._arg_type_name)
1411             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1412                     isinstance(self.arg_type, QAPISchemaAlternateType))
1413             self.arg_type.check(schema)
1414             if self.boxed:
1415                 if self.arg_type.is_empty():
1416                     raise QAPISemError(self.info,
1417                                        "Cannot use 'boxed' with empty type")
1418             else:
1419                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1420                 assert not self.arg_type.variants
1421         elif self.boxed:
1422             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1423         if self._ret_type_name:
1424             self.ret_type = schema.lookup_type(self._ret_type_name)
1425             assert isinstance(self.ret_type, QAPISchemaType)
1426
1427     def visit(self, visitor):
1428         visitor.visit_command(self.name, self.info,
1429                               self.arg_type, self.ret_type,
1430                               self.gen, self.success_response, self.boxed)
1431
1432
1433 class QAPISchemaEvent(QAPISchemaEntity):
1434     def __init__(self, name, info, doc, arg_type, boxed):
1435         QAPISchemaEntity.__init__(self, name, info, doc)
1436         assert not arg_type or isinstance(arg_type, str)
1437         self._arg_type_name = arg_type
1438         self.arg_type = None
1439         self.boxed = boxed
1440
1441     def check(self, schema):
1442         if self._arg_type_name:
1443             self.arg_type = schema.lookup_type(self._arg_type_name)
1444             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1445                     isinstance(self.arg_type, QAPISchemaAlternateType))
1446             self.arg_type.check(schema)
1447             if self.boxed:
1448                 if self.arg_type.is_empty():
1449                     raise QAPISemError(self.info,
1450                                        "Cannot use 'boxed' with empty type")
1451             else:
1452                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1453                 assert not self.arg_type.variants
1454         elif self.boxed:
1455             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1456
1457     def visit(self, visitor):
1458         visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
1459
1460
1461 class QAPISchema(object):
1462     def __init__(self, fname):
1463         try:
1464             parser = QAPISchemaParser(open(fname, 'r'))
1465             self.exprs = check_exprs(parser.exprs)
1466             self.docs = parser.docs
1467             self._entity_dict = {}
1468             self._predefining = True
1469             self._def_predefineds()
1470             self._predefining = False
1471             self._def_exprs()
1472             self.check()
1473         except QAPIError as err:
1474             print >>sys.stderr, err
1475             exit(1)
1476
1477     def _def_entity(self, ent):
1478         # Only the predefined types are allowed to not have info
1479         assert ent.info or self._predefining
1480         assert ent.name not in self._entity_dict
1481         self._entity_dict[ent.name] = ent
1482
1483     def lookup_entity(self, name, typ=None):
1484         ent = self._entity_dict.get(name)
1485         if typ and not isinstance(ent, typ):
1486             return None
1487         return ent
1488
1489     def lookup_type(self, name):
1490         return self.lookup_entity(name, QAPISchemaType)
1491
1492     def _def_builtin_type(self, name, json_type, c_type):
1493         self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
1494         # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
1495         # qapi-types.h from a single .c, all arrays of builtins must be
1496         # declared in the first file whether or not they are used.  Nicer
1497         # would be to use lazy instantiation, while figuring out how to
1498         # avoid compilation issues with multiple qapi-types.h.
1499         self._make_array_type(name, None)
1500
1501     def _def_predefineds(self):
1502         for t in [('str',    'string',  'char' + pointer_suffix),
1503                   ('number', 'number',  'double'),
1504                   ('int',    'int',     'int64_t'),
1505                   ('int8',   'int',     'int8_t'),
1506                   ('int16',  'int',     'int16_t'),
1507                   ('int32',  'int',     'int32_t'),
1508                   ('int64',  'int',     'int64_t'),
1509                   ('uint8',  'int',     'uint8_t'),
1510                   ('uint16', 'int',     'uint16_t'),
1511                   ('uint32', 'int',     'uint32_t'),
1512                   ('uint64', 'int',     'uint64_t'),
1513                   ('size',   'int',     'uint64_t'),
1514                   ('bool',   'boolean', 'bool'),
1515                   ('any',    'value',   'QObject' + pointer_suffix),
1516                   ('null',   'null',    'QNull' + pointer_suffix)]:
1517             self._def_builtin_type(*t)
1518         self.the_empty_object_type = QAPISchemaObjectType(
1519             'q_empty', None, None, None, [], None)
1520         self._def_entity(self.the_empty_object_type)
1521         qtype_values = self._make_enum_members(['none', 'qnull', 'qnum',
1522                                                 'qstring', 'qdict', 'qlist',
1523                                                 'qbool'])
1524         self._def_entity(QAPISchemaEnumType('QType', None, None,
1525                                             qtype_values, 'QTYPE'))
1526
1527     def _make_enum_members(self, values):
1528         return [QAPISchemaMember(v) for v in values]
1529
1530     def _make_implicit_enum_type(self, name, info, values):
1531         # See also QAPISchemaObjectTypeMember._pretty_owner()
1532         name = name + 'Kind'   # Use namespace reserved by add_name()
1533         self._def_entity(QAPISchemaEnumType(
1534             name, info, None, self._make_enum_members(values), None))
1535         return name
1536
1537     def _make_array_type(self, element_type, info):
1538         name = element_type + 'List'   # Use namespace reserved by add_name()
1539         if not self.lookup_type(name):
1540             self._def_entity(QAPISchemaArrayType(name, info, element_type))
1541         return name
1542
1543     def _make_implicit_object_type(self, name, info, doc, role, members):
1544         if not members:
1545             return None
1546         # See also QAPISchemaObjectTypeMember._pretty_owner()
1547         name = 'q_obj_%s-%s' % (name, role)
1548         if not self.lookup_entity(name, QAPISchemaObjectType):
1549             self._def_entity(QAPISchemaObjectType(name, info, doc, None,
1550                                                   members, None))
1551         return name
1552
1553     def _def_enum_type(self, expr, info, doc):
1554         name = expr['enum']
1555         data = expr['data']
1556         prefix = expr.get('prefix')
1557         self._def_entity(QAPISchemaEnumType(
1558             name, info, doc, self._make_enum_members(data), prefix))
1559
1560     def _make_member(self, name, typ, info):
1561         optional = False
1562         if name.startswith('*'):
1563             name = name[1:]
1564             optional = True
1565         if isinstance(typ, list):
1566             assert len(typ) == 1
1567             typ = self._make_array_type(typ[0], info)
1568         return QAPISchemaObjectTypeMember(name, typ, optional)
1569
1570     def _make_members(self, data, info):
1571         return [self._make_member(key, value, info)
1572                 for (key, value) in data.iteritems()]
1573
1574     def _def_struct_type(self, expr, info, doc):
1575         name = expr['struct']
1576         base = expr.get('base')
1577         data = expr['data']
1578         self._def_entity(QAPISchemaObjectType(name, info, doc, base,
1579                                               self._make_members(data, info),
1580                                               None))
1581
1582     def _make_variant(self, case, typ):
1583         return QAPISchemaObjectTypeVariant(case, typ)
1584
1585     def _make_simple_variant(self, case, typ, info):
1586         if isinstance(typ, list):
1587             assert len(typ) == 1
1588             typ = self._make_array_type(typ[0], info)
1589         typ = self._make_implicit_object_type(
1590             typ, info, None, 'wrapper', [self._make_member('data', typ, info)])
1591         return QAPISchemaObjectTypeVariant(case, typ)
1592
1593     def _def_union_type(self, expr, info, doc):
1594         name = expr['union']
1595         data = expr['data']
1596         base = expr.get('base')
1597         tag_name = expr.get('discriminator')
1598         tag_member = None
1599         if isinstance(base, dict):
1600             base = (self._make_implicit_object_type(
1601                 name, info, doc, 'base', self._make_members(base, info)))
1602         if tag_name:
1603             variants = [self._make_variant(key, value)
1604                         for (key, value) in data.iteritems()]
1605             members = []
1606         else:
1607             variants = [self._make_simple_variant(key, value, info)
1608                         for (key, value) in data.iteritems()]
1609             typ = self._make_implicit_enum_type(name, info,
1610                                                 [v.name for v in variants])
1611             tag_member = QAPISchemaObjectTypeMember('type', typ, False)
1612             members = [tag_member]
1613         self._def_entity(
1614             QAPISchemaObjectType(name, info, doc, base, members,
1615                                  QAPISchemaObjectTypeVariants(tag_name,
1616                                                               tag_member,
1617                                                               variants)))
1618
1619     def _def_alternate_type(self, expr, info, doc):
1620         name = expr['alternate']
1621         data = expr['data']
1622         variants = [self._make_variant(key, value)
1623                     for (key, value) in data.iteritems()]
1624         tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
1625         self._def_entity(
1626             QAPISchemaAlternateType(name, info, doc,
1627                                     QAPISchemaObjectTypeVariants(None,
1628                                                                  tag_member,
1629                                                                  variants)))
1630
1631     def _def_command(self, expr, info, doc):
1632         name = expr['command']
1633         data = expr.get('data')
1634         rets = expr.get('returns')
1635         gen = expr.get('gen', True)
1636         success_response = expr.get('success-response', True)
1637         boxed = expr.get('boxed', False)
1638         if isinstance(data, OrderedDict):
1639             data = self._make_implicit_object_type(
1640                 name, info, doc, 'arg', self._make_members(data, info))
1641         if isinstance(rets, list):
1642             assert len(rets) == 1
1643             rets = self._make_array_type(rets[0], info)
1644         self._def_entity(QAPISchemaCommand(name, info, doc, data, rets,
1645                                            gen, success_response, boxed))
1646
1647     def _def_event(self, expr, info, doc):
1648         name = expr['event']
1649         data = expr.get('data')
1650         boxed = expr.get('boxed', False)
1651         if isinstance(data, OrderedDict):
1652             data = self._make_implicit_object_type(
1653                 name, info, doc, 'arg', self._make_members(data, info))
1654         self._def_entity(QAPISchemaEvent(name, info, doc, data, boxed))
1655
1656     def _def_exprs(self):
1657         for expr_elem in self.exprs:
1658             expr = expr_elem['expr']
1659             info = expr_elem['info']
1660             doc = expr_elem.get('doc')
1661             if 'enum' in expr:
1662                 self._def_enum_type(expr, info, doc)
1663             elif 'struct' in expr:
1664                 self._def_struct_type(expr, info, doc)
1665             elif 'union' in expr:
1666                 self._def_union_type(expr, info, doc)
1667             elif 'alternate' in expr:
1668                 self._def_alternate_type(expr, info, doc)
1669             elif 'command' in expr:
1670                 self._def_command(expr, info, doc)
1671             elif 'event' in expr:
1672                 self._def_event(expr, info, doc)
1673             else:
1674                 assert False
1675
1676     def check(self):
1677         for ent in self._entity_dict.values():
1678             ent.check(self)
1679
1680     def visit(self, visitor):
1681         visitor.visit_begin(self)
1682         for (name, entity) in sorted(self._entity_dict.items()):
1683             if visitor.visit_needed(entity):
1684                 entity.visit(visitor)
1685         visitor.visit_end()
1686
1687
1688 #
1689 # Code generation helpers
1690 #
1691
1692 def camel_case(name):
1693     new_name = ''
1694     first = True
1695     for ch in name:
1696         if ch in ['_', '-']:
1697             first = True
1698         elif first:
1699             new_name += ch.upper()
1700             first = False
1701         else:
1702             new_name += ch.lower()
1703     return new_name
1704
1705
1706 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1707 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1708 # ENUM24_Name -> ENUM24_NAME
1709 def camel_to_upper(value):
1710     c_fun_str = c_name(value, False)
1711     if value.isupper():
1712         return c_fun_str
1713
1714     new_name = ''
1715     l = len(c_fun_str)
1716     for i in range(l):
1717         c = c_fun_str[i]
1718         # When c is upper and no '_' appears before, do more checks
1719         if c.isupper() and (i > 0) and c_fun_str[i - 1] != '_':
1720             if i < l - 1 and c_fun_str[i + 1].islower():
1721                 new_name += '_'
1722             elif c_fun_str[i - 1].isdigit():
1723                 new_name += '_'
1724         new_name += c
1725     return new_name.lstrip('_').upper()
1726
1727
1728 def c_enum_const(type_name, const_name, prefix=None):
1729     if prefix is not None:
1730         type_name = prefix
1731     return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
1732
1733 c_name_trans = string.maketrans('.-', '__')
1734
1735
1736 # Map @name to a valid C identifier.
1737 # If @protect, avoid returning certain ticklish identifiers (like
1738 # C keywords) by prepending 'q_'.
1739 #
1740 # Used for converting 'name' from a 'name':'type' qapi definition
1741 # into a generated struct member, as well as converting type names
1742 # into substrings of a generated C function name.
1743 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1744 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1745 def c_name(name, protect=True):
1746     # ANSI X3J11/88-090, 3.1.1
1747     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1748                      'default', 'do', 'double', 'else', 'enum', 'extern',
1749                      'float', 'for', 'goto', 'if', 'int', 'long', 'register',
1750                      'return', 'short', 'signed', 'sizeof', 'static',
1751                      'struct', 'switch', 'typedef', 'union', 'unsigned',
1752                      'void', 'volatile', 'while'])
1753     # ISO/IEC 9899:1999, 6.4.1
1754     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1755     # ISO/IEC 9899:2011, 6.4.1
1756     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
1757                      '_Noreturn', '_Static_assert', '_Thread_local'])
1758     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1759     # excluding _.*
1760     gcc_words = set(['asm', 'typeof'])
1761     # C++ ISO/IEC 14882:2003 2.11
1762     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1763                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1764                      'namespace', 'new', 'operator', 'private', 'protected',
1765                      'public', 'reinterpret_cast', 'static_cast', 'template',
1766                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1767                      'using', 'virtual', 'wchar_t',
1768                      # alternative representations
1769                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1770                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1771     # namespace pollution:
1772     polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
1773     name = name.translate(c_name_trans)
1774     if protect and (name in c89_words | c99_words | c11_words | gcc_words
1775                     | cpp_words | polluted_words):
1776         return 'q_' + name
1777     return name
1778
1779 eatspace = '\033EATSPACE.'
1780 pointer_suffix = ' *' + eatspace
1781
1782
1783 def genindent(count):
1784     ret = ''
1785     for _ in range(count):
1786         ret += ' '
1787     return ret
1788
1789 indent_level = 0
1790
1791
1792 def push_indent(indent_amount=4):
1793     global indent_level
1794     indent_level += indent_amount
1795
1796
1797 def pop_indent(indent_amount=4):
1798     global indent_level
1799     indent_level -= indent_amount
1800
1801
1802 # Generate @code with @kwds interpolated.
1803 # Obey indent_level, and strip eatspace.
1804 def cgen(code, **kwds):
1805     raw = code % kwds
1806     if indent_level:
1807         indent = genindent(indent_level)
1808         # re.subn() lacks flags support before Python 2.7, use re.compile()
1809         raw = re.subn(re.compile(r'^.', re.MULTILINE),
1810                       indent + r'\g<0>', raw)
1811         raw = raw[0]
1812     return re.sub(re.escape(eatspace) + r' *', '', raw)
1813
1814
1815 def mcgen(code, **kwds):
1816     if code[0] == '\n':
1817         code = code[1:]
1818     return cgen(code, **kwds)
1819
1820
1821 def guardname(filename):
1822     return c_name(filename, protect=False).upper()
1823
1824
1825 def guardstart(name):
1826     return mcgen('''
1827
1828 #ifndef %(name)s
1829 #define %(name)s
1830
1831 ''',
1832                  name=guardname(name))
1833
1834
1835 def guardend(name):
1836     return mcgen('''
1837
1838 #endif /* %(name)s */
1839
1840 ''',
1841                  name=guardname(name))
1842
1843
1844 def gen_enum_lookup(name, values, prefix=None):
1845     ret = mcgen('''
1846
1847 const QEnumLookup %(c_name)s_lookup = {
1848     .array = (const char *const[]) {
1849 ''',
1850                 c_name=c_name(name))
1851     for value in values:
1852         index = c_enum_const(name, value, prefix)
1853         ret += mcgen('''
1854         [%(index)s] = "%(value)s",
1855 ''',
1856                      index=index, value=value)
1857
1858     ret += mcgen('''
1859     },
1860     .size = %(max_index)s
1861 };
1862 ''',
1863                  max_index=c_enum_const(name, '_MAX', prefix))
1864     return ret
1865
1866
1867 def gen_enum(name, values, prefix=None):
1868     # append automatically generated _MAX value
1869     enum_values = values + ['_MAX']
1870
1871     ret = mcgen('''
1872
1873 typedef enum %(c_name)s {
1874 ''',
1875                 c_name=c_name(name))
1876
1877     i = 0
1878     for value in enum_values:
1879         ret += mcgen('''
1880     %(c_enum)s = %(i)d,
1881 ''',
1882                      c_enum=c_enum_const(name, value, prefix),
1883                      i=i)
1884         i += 1
1885
1886     ret += mcgen('''
1887 } %(c_name)s;
1888 ''',
1889                  c_name=c_name(name))
1890
1891     ret += mcgen('''
1892
1893 #define %(c_name)s_str(val) \\
1894     qapi_enum_lookup(&%(c_name)s_lookup, (val))
1895
1896 extern const QEnumLookup %(c_name)s_lookup;
1897 ''',
1898                  c_name=c_name(name))
1899     return ret
1900
1901
1902 def build_params(arg_type, boxed, extra):
1903     if not arg_type:
1904         assert not boxed
1905         return extra
1906     ret = ''
1907     sep = ''
1908     if boxed:
1909         ret += '%s arg' % arg_type.c_param_type()
1910         sep = ', '
1911     else:
1912         assert not arg_type.variants
1913         for memb in arg_type.members:
1914             ret += sep
1915             sep = ', '
1916             if memb.optional:
1917                 ret += 'bool has_%s, ' % c_name(memb.name)
1918             ret += '%s %s' % (memb.type.c_param_type(),
1919                               c_name(memb.name))
1920     if extra:
1921         ret += sep + extra
1922     return ret
1923
1924
1925 #
1926 # Common command line parsing
1927 #
1928
1929
1930 def parse_command_line(extra_options='', extra_long_options=[]):
1931
1932     try:
1933         opts, args = getopt.gnu_getopt(sys.argv[1:],
1934                                        'chp:o:' + extra_options,
1935                                        ['source', 'header', 'prefix=',
1936                                         'output-dir='] + extra_long_options)
1937     except getopt.GetoptError as err:
1938         print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
1939         sys.exit(1)
1940
1941     output_dir = ''
1942     prefix = ''
1943     do_c = False
1944     do_h = False
1945     extra_opts = []
1946
1947     for oa in opts:
1948         o, a = oa
1949         if o in ('-p', '--prefix'):
1950             match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1951             if match.end() != len(a):
1952                 print >>sys.stderr, \
1953                     "%s: 'funny character '%s' in argument of --prefix" \
1954                     % (sys.argv[0], a[match.end()])
1955                 sys.exit(1)
1956             prefix = a
1957         elif o in ('-o', '--output-dir'):
1958             output_dir = a + '/'
1959         elif o in ('-c', '--source'):
1960             do_c = True
1961         elif o in ('-h', '--header'):
1962             do_h = True
1963         else:
1964             extra_opts.append(oa)
1965
1966     if not do_c and not do_h:
1967         do_c = True
1968         do_h = True
1969
1970     if len(args) != 1:
1971         print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
1972         sys.exit(1)
1973     fname = args[0]
1974
1975     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
1976
1977 #
1978 # Generate output files with boilerplate
1979 #
1980
1981
1982 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
1983                 c_comment, h_comment):
1984     guard = guardname(prefix + h_file)
1985     c_file = output_dir + prefix + c_file
1986     h_file = output_dir + prefix + h_file
1987
1988     if output_dir:
1989         try:
1990             os.makedirs(output_dir)
1991         except os.error as e:
1992             if e.errno != errno.EEXIST:
1993                 raise
1994
1995     def maybe_open(really, name, opt):
1996         if really:
1997             return open(name, opt)
1998         else:
1999             import StringIO
2000             return StringIO.StringIO()
2001
2002     fdef = maybe_open(do_c, c_file, 'w')
2003     fdecl = maybe_open(do_h, h_file, 'w')
2004
2005     fdef.write(mcgen('''
2006 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2007 %(comment)s
2008 ''',
2009                      comment=c_comment))
2010
2011     fdecl.write(mcgen('''
2012 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2013 %(comment)s
2014 #ifndef %(guard)s
2015 #define %(guard)s
2016
2017 ''',
2018                       comment=h_comment, guard=guard))
2019
2020     return (fdef, fdecl)
2021
2022
2023 def close_output(fdef, fdecl):
2024     fdecl.write('''
2025 #endif
2026 ''')
2027     fdecl.close()
2028     fdef.close()