scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 import errno
  15 import getopt
  16 import os
  17 import re
  18 import string
  19 import sys
  20 from ordereddict import OrderedDict
  21
  22 builtin_types = {
  23     'str':      'QTYPE_QSTRING',
  24     'int':      'QTYPE_QINT',
  25     'number':   'QTYPE_QFLOAT',
  26     'bool':     'QTYPE_QBOOL',
  27     'int8':     'QTYPE_QINT',
  28     'int16':    'QTYPE_QINT',
  29     'int32':    'QTYPE_QINT',
  30     'int64':    'QTYPE_QINT',
  31     'uint8':    'QTYPE_QINT',
  32     'uint16':   'QTYPE_QINT',
  33     'uint32':   'QTYPE_QINT',
  34     'uint64':   'QTYPE_QINT',
  35     'size':     'QTYPE_QINT',
  36     'any':      None,           # any QType possible, actually
  37     'QType':    'QTYPE_QSTRING',
  38 }
  39
  40 # Are documentation comments required?
  41 doc_required = False
  42
  43 # Whitelist of commands allowed to return a non-dictionary
  44 returns_whitelist = []
  45
  46 # Whitelist of entities allowed to violate case conventions
  47 name_case_whitelist = []
  48
  49 enum_types = {}
  50 struct_types = {}
  51 union_types = {}
  52 all_names = {}
  53
  54 #
  55 # Parsing the schema into expressions
  56 #
  57
  58
  59 def error_path(parent):
  60     res = ''
  61     while parent:
  62         res = ('In file included from %s:%d:\n' % (parent['file'],
  63                                                    parent['line'])) + res
  64         parent = parent['parent']
  65     return res
  66
  67
  68 class QAPIError(Exception):
  69     def __init__(self, fname, line, col, incl_info, msg):
  70         Exception.__init__(self)
  71         self.fname = fname
  72         self.line = line
  73         self.col = col
  74         self.info = incl_info
  75         self.msg = msg
  76
  77     def __str__(self):
  78         loc = '%s:%d' % (self.fname, self.line)
  79         if self.col is not None:
  80             loc += ':%s' % self.col
  81         return error_path(self.info) + '%s: %s' % (loc, self.msg)
  82
  83
  84 class QAPIParseError(QAPIError):
  85     def __init__(self, parser, msg):
  86         col = 1
  87         for ch in parser.src[parser.line_pos:parser.pos]:
  88             if ch == '\t':
  89                 col = (col + 7) % 8 + 1
  90             else:
  91                 col += 1
  92         QAPIError.__init__(self, parser.fname, parser.line, col,
  93                            parser.incl_info, msg)
  94
  95
  96 class QAPISemError(QAPIError):
  97     def __init__(self, info, msg):
  98         QAPIError.__init__(self, info['file'], info['line'], None,
  99                            info['parent'], msg)
 100
 101
 102 class QAPIDoc(object):
 103     class Section(object):
 104         def __init__(self, name=None):
 105             # optional section name (argument/member or section name)
 106             self.name = name
 107             # the list of lines for this section
 108             self.content = []
 109             self.optional = False
 110
 111         def append(self, line):
 112             self.content.append(line)
 113
 114         def __repr__(self):
 115             return '\n'.join(self.content).strip()
 116
 117     class ArgSection(Section):
 118         def __init__(self, name):
 119             QAPIDoc.Section.__init__(self, name)
 120             self.member = None
 121
 122         def connect(self, member):
 123             self.member = member
 124
 125     def __init__(self, parser, info):
 126         # self.parser is used to report errors with QAPIParseError.  The
 127         # resulting error position depends on the state of the parser.
 128         # It happens to be the beginning of the comment.  More or less
 129         # servicable, but action at a distance.
 130         self.parser = parser
 131         self.info = info
 132         self.symbol = None
 133         self.body = QAPIDoc.Section()
 134         # dict mapping parameter name to ArgSection
 135         self.args = OrderedDict()
 136         # a list of Section
 137         self.sections = []
 138         # the current section
 139         self.section = self.body
 140
 141     def has_section(self, name):
 142         """Return True if we have a section with this name."""
 143         for i in self.sections:
 144             if i.name == name:
 145                 return True
 146         return False
 147
 148     def append(self, line):
 149         """Parse a comment line and add it to the documentation."""
 150         line = line[1:]
 151         if not line:
 152             self._append_freeform(line)
 153             return
 154
 155         if line[0] != ' ':
 156             raise QAPIParseError(self.parser, "Missing space after #")
 157         line = line[1:]
 158
 159         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 160         # recognized, and get silently treated as ordinary text
 161         if self.symbol:
 162             self._append_symbol_line(line)
 163         elif not self.body.content and line.startswith('@'):
 164             if not line.endswith(':'):
 165                 raise QAPIParseError(self.parser, "Line should end with :")
 166             self.symbol = line[1:-1]
 167             # FIXME invalid names other than the empty string aren't flagged
 168             if not self.symbol:
 169                 raise QAPIParseError(self.parser, "Invalid name")
 170         else:
 171             self._append_freeform(line)
 172
 173     def end_comment(self):
 174         self._end_section()
 175
 176     def _append_symbol_line(self, line):
 177         name = line.split(' ', 1)[0]
 178
 179         if name.startswith('@') and name.endswith(':'):
 180             line = line[len(name)+1:]
 181             self._start_args_section(name[1:-1])
 182         elif name in ('Returns:', 'Since:',
 183                       # those are often singular or plural
 184                       'Note:', 'Notes:',
 185                       'Example:', 'Examples:',
 186                       'TODO:'):
 187             line = line[len(name)+1:]
 188             self._start_section(name[:-1])
 189
 190         self._append_freeform(line)
 191
 192     def _start_args_section(self, name):
 193         # FIXME invalid names other than the empty string aren't flagged
 194         if not name:
 195             raise QAPIParseError(self.parser, "Invalid parameter name")
 196         if name in self.args:
 197             raise QAPIParseError(self.parser,
 198                                  "'%s' parameter name duplicated" % name)
 199         if self.sections:
 200             raise QAPIParseError(self.parser,
 201                                  "'@%s:' can't follow '%s' section"
 202                                  % (name, self.sections[0].name))
 203         self._end_section()
 204         self.section = QAPIDoc.ArgSection(name)
 205         self.args[name] = self.section
 206
 207     def _start_section(self, name=''):
 208         if name in ('Returns', 'Since') and self.has_section(name):
 209             raise QAPIParseError(self.parser,
 210                                  "Duplicated '%s' section" % name)
 211         self._end_section()
 212         self.section = QAPIDoc.Section(name)
 213         self.sections.append(self.section)
 214
 215     def _end_section(self):
 216         if self.section:
 217             contents = str(self.section)
 218             if self.section.name and (not contents or contents.isspace()):
 219                 raise QAPIParseError(self.parser, "Empty doc section '%s'"
 220                                      % self.section.name)
 221             self.section = None
 222
 223     def _append_freeform(self, line):
 224         in_arg = isinstance(self.section, QAPIDoc.ArgSection)
 225         if (in_arg and self.section.content
 226                 and not self.section.content[-1]
 227                 and line and not line[0].isspace()):
 228             self._start_section()
 229         if (in_arg or not self.section.name
 230                 or not self.section.name.startswith('Example')):
 231             line = line.strip()
 232         match = re.match(r'(@\S+:)', line)
 233         if match:
 234             raise QAPIParseError(self.parser,
 235                                  "'%s' not allowed in free-form documentation"
 236                                  % match.group(1))
 237         # TODO Drop this once the dust has settled
 238         if (isinstance(self.section, QAPIDoc.ArgSection)
 239                 and '#optional' in line):
 240             raise QAPISemError(self.info, "Please drop the #optional tag")
 241         self.section.append(line)
 242
 243     def connect_member(self, member):
 244         if member.name not in self.args:
 245             # Undocumented TODO outlaw
 246             self.args[member.name] = QAPIDoc.ArgSection(member.name)
 247         self.args[member.name].connect(member)
 248
 249     def check_expr(self, expr):
 250         if self.has_section('Returns') and 'command' not in expr:
 251             raise QAPISemError(self.info,
 252                                "'Returns:' is only valid for commands")
 253
 254     def check(self):
 255         bogus = [name for name, section in self.args.iteritems()
 256                  if not section.member]
 257         if bogus:
 258             raise QAPISemError(
 259                 self.info,
 260                 "The following documented members are not in "
 261                 "the declaration: %s" % ", ".join(bogus))
 262
 263
 264 class QAPISchemaParser(object):
 265
 266     def __init__(self, fp, previously_included=[], incl_info=None):
 267         abs_fname = os.path.abspath(fp.name)
 268         fname = fp.name
 269         self.fname = fname
 270         previously_included.append(abs_fname)
 271         self.incl_info = incl_info
 272         self.src = fp.read()
 273         if self.src == '' or self.src[-1] != '\n':
 274             self.src += '\n'
 275         self.cursor = 0
 276         self.line = 1
 277         self.line_pos = 0
 278         self.exprs = []
 279         self.docs = []
 280         self.cur_doc = None
 281         self.accept()
 282
 283         while self.tok is not None:
 284             info = {'file': fname, 'line': self.line,
 285                     'parent': self.incl_info}
 286             if self.tok == '#':
 287                 self.reject_expr_doc()
 288                 self.cur_doc = self.get_doc(info)
 289                 self.docs.append(self.cur_doc)
 290                 continue
 291
 292             expr = self.get_expr(False)
 293             if 'include' in expr:
 294                 self.reject_expr_doc()
 295                 if len(expr) != 1:
 296                     raise QAPISemError(info, "Invalid 'include' directive")
 297                 include = expr['include']
 298                 if not isinstance(include, str):
 299                     raise QAPISemError(info,
 300                                        "Value of 'include' must be a string")
 301                 self._include(include, info, os.path.dirname(abs_fname),
 302                               previously_included)
 303             elif "pragma" in expr:
 304                 self.reject_expr_doc()
 305                 if len(expr) != 1:
 306                     raise QAPISemError(info, "Invalid 'pragma' directive")
 307                 pragma = expr['pragma']
 308                 if not isinstance(pragma, dict):
 309                     raise QAPISemError(
 310                         info, "Value of 'pragma' must be a dictionary")
 311                 for name, value in pragma.iteritems():
 312                     self._pragma(name, value, info)
 313             else:
 314                 expr_elem = {'expr': expr,
 315                              'info': info}
 316                 if self.cur_doc:
 317                     if not self.cur_doc.symbol:
 318                         raise QAPISemError(
 319                             self.cur_doc.info,
 320                             "Expression documentation required")
 321                     expr_elem['doc'] = self.cur_doc
 322                 self.exprs.append(expr_elem)
 323             self.cur_doc = None
 324         self.reject_expr_doc()
 325
 326     def reject_expr_doc(self):
 327         if self.cur_doc and self.cur_doc.symbol:
 328             raise QAPISemError(
 329                 self.cur_doc.info,
 330                 "Documentation for '%s' is not followed by the definition"
 331                 % self.cur_doc.symbol)
 332
 333     def _include(self, include, info, base_dir, previously_included):
 334         incl_abs_fname = os.path.join(base_dir, include)
 335         # catch inclusion cycle
 336         inf = info
 337         while inf:
 338             if incl_abs_fname == os.path.abspath(inf['file']):
 339                 raise QAPISemError(info, "Inclusion loop for %s" % include)
 340             inf = inf['parent']
 341
 342         # skip multiple include of the same file
 343         if incl_abs_fname in previously_included:
 344             return
 345         try:
 346             fobj = open(incl_abs_fname, 'r')
 347         except IOError as e:
 348             raise QAPISemError(info, '%s: %s' % (e.strerror, include))
 349         exprs_include = QAPISchemaParser(fobj, previously_included, info)
 350         self.exprs.extend(exprs_include.exprs)
 351         self.docs.extend(exprs_include.docs)
 352
 353     def _pragma(self, name, value, info):
 354         global doc_required, returns_whitelist, name_case_whitelist
 355         if name == 'doc-required':
 356             if not isinstance(value, bool):
 357                 raise QAPISemError(info,
 358                                    "Pragma 'doc-required' must be boolean")
 359             doc_required = value
 360         elif name == 'returns-whitelist':
 361             if (not isinstance(value, list)
 362                     or any([not isinstance(elt, str) for elt in value])):
 363                 raise QAPISemError(info,
 364                                    "Pragma returns-whitelist must be"
 365                                    " a list of strings")
 366             returns_whitelist = value
 367         elif name == 'name-case-whitelist':
 368             if (not isinstance(value, list)
 369                     or any([not isinstance(elt, str) for elt in value])):
 370                 raise QAPISemError(info,
 371                                    "Pragma name-case-whitelist must be"
 372                                    " a list of strings")
 373             name_case_whitelist = value
 374         else:
 375             raise QAPISemError(info, "Unknown pragma '%s'" % name)
 376
 377     def accept(self, skip_comment=True):
 378         while True:
 379             self.tok = self.src[self.cursor]
 380             self.pos = self.cursor
 381             self.cursor += 1
 382             self.val = None
 383
 384             if self.tok == '#':
 385                 if self.src[self.cursor] == '#':
 386                     # Start of doc comment
 387                     skip_comment = False
 388                 self.cursor = self.src.find('\n', self.cursor)
 389                 if not skip_comment:
 390                     self.val = self.src[self.pos:self.cursor]
 391                     return
 392             elif self.tok in '{}:,[]':
 393                 return
 394             elif self.tok == "'":
 395                 string = ''
 396                 esc = False
 397                 while True:
 398                     ch = self.src[self.cursor]
 399                     self.cursor += 1
 400                     if ch == '\n':
 401                         raise QAPIParseError(self, 'Missing terminating "\'"')
 402                     if esc:
 403                         if ch == 'b':
 404                             string += '\b'
 405                         elif ch == 'f':
 406                             string += '\f'
 407                         elif ch == 'n':
 408                             string += '\n'
 409                         elif ch == 'r':
 410                             string += '\r'
 411                         elif ch == 't':
 412                             string += '\t'
 413                         elif ch == 'u':
 414                             value = 0
 415                             for _ in range(0, 4):
 416                                 ch = self.src[self.cursor]
 417                                 self.cursor += 1
 418                                 if ch not in '0123456789abcdefABCDEF':
 419                                     raise QAPIParseError(self,
 420                                                          '\\u escape needs 4 '
 421                                                          'hex digits')
 422                                 value = (value << 4) + int(ch, 16)
 423                             # If Python 2 and 3 didn't disagree so much on
 424                             # how to handle Unicode, then we could allow
 425                             # Unicode string defaults.  But most of QAPI is
 426                             # ASCII-only, so we aren't losing much for now.
 427                             if not value or value > 0x7f:
 428                                 raise QAPIParseError(self,
 429                                                      'For now, \\u escape '
 430                                                      'only supports non-zero '
 431                                                      'values up to \\u007f')
 432                             string += chr(value)
 433                         elif ch in '\\/\'"':
 434                             string += ch
 435                         else:
 436                             raise QAPIParseError(self,
 437                                                  "Unknown escape \\%s" % ch)
 438                         esc = False
 439                     elif ch == '\\':
 440                         esc = True
 441                     elif ch == "'":
 442                         self.val = string
 443                         return
 444                     else:
 445                         string += ch
 446             elif self.src.startswith('true', self.pos):
 447                 self.val = True
 448                 self.cursor += 3
 449                 return
 450             elif self.src.startswith('false', self.pos):
 451                 self.val = False
 452                 self.cursor += 4
 453                 return
 454             elif self.src.startswith('null', self.pos):
 455                 self.val = None
 456                 self.cursor += 3
 457                 return
 458             elif self.tok == '\n':
 459                 if self.cursor == len(self.src):
 460                     self.tok = None
 461                     return
 462                 self.line += 1
 463                 self.line_pos = self.cursor
 464             elif not self.tok.isspace():
 465                 raise QAPIParseError(self, 'Stray "%s"' % self.tok)
 466
 467     def get_members(self):
 468         expr = OrderedDict()
 469         if self.tok == '}':
 470             self.accept()
 471             return expr
 472         if self.tok != "'":
 473             raise QAPIParseError(self, 'Expected string or "}"')
 474         while True:
 475             key = self.val
 476             self.accept()
 477             if self.tok != ':':
 478                 raise QAPIParseError(self, 'Expected ":"')
 479             self.accept()
 480             if key in expr:
 481                 raise QAPIParseError(self, 'Duplicate key "%s"' % key)
 482             expr[key] = self.get_expr(True)
 483             if self.tok == '}':
 484                 self.accept()
 485                 return expr
 486             if self.tok != ',':
 487                 raise QAPIParseError(self, 'Expected "," or "}"')
 488             self.accept()
 489             if self.tok != "'":
 490                 raise QAPIParseError(self, 'Expected string')
 491
 492     def get_values(self):
 493         expr = []
 494         if self.tok == ']':
 495             self.accept()
 496             return expr
 497         if self.tok not in "{['tfn":
 498             raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
 499                                  'boolean or "null"')
 500         while True:
 501             expr.append(self.get_expr(True))
 502             if self.tok == ']':
 503                 self.accept()
 504                 return expr
 505             if self.tok != ',':
 506                 raise QAPIParseError(self, 'Expected "," or "]"')
 507             self.accept()
 508
 509     def get_expr(self, nested):
 510         if self.tok != '{' and not nested:
 511             raise QAPIParseError(self, 'Expected "{"')
 512         if self.tok == '{':
 513             self.accept()
 514             expr = self.get_members()
 515         elif self.tok == '[':
 516             self.accept()
 517             expr = self.get_values()
 518         elif self.tok in "'tfn":
 519             expr = self.val
 520             self.accept()
 521         else:
 522             raise QAPIParseError(self, 'Expected "{", "[", string, '
 523                                  'boolean or "null"')
 524         return expr
 525
 526     def get_doc(self, info):
 527         if self.val != '##':
 528             raise QAPIParseError(self, "Junk after '##' at start of "
 529                                  "documentation comment")
 530
 531         doc = QAPIDoc(self, info)
 532         self.accept(False)
 533         while self.tok == '#':
 534             if self.val.startswith('##'):
 535                 # End of doc comment
 536                 if self.val != '##':
 537                     raise QAPIParseError(self, "Junk after '##' at end of "
 538                                          "documentation comment")
 539                 doc.end_comment()
 540                 self.accept()
 541                 return doc
 542             else:
 543                 doc.append(self.val)
 544             self.accept(False)
 545
 546         raise QAPIParseError(self, "Documentation comment must end with '##'")
 547
 548
 549 #
 550 # Semantic analysis of schema expressions
 551 # TODO fold into QAPISchema
 552 # TODO catching name collisions in generated code would be nice
 553 #
 554
 555
 556 def find_base_members(base):
 557     if isinstance(base, dict):
 558         return base
 559     base_struct_define = struct_types.get(base)
 560     if not base_struct_define:
 561         return None
 562     return base_struct_define['data']
 563
 564
 565 # Return the qtype of an alternate branch, or None on error.
 566 def find_alternate_member_qtype(qapi_type):
 567     if qapi_type in builtin_types:
 568         return builtin_types[qapi_type]
 569     elif qapi_type in struct_types:
 570         return 'QTYPE_QDICT'
 571     elif qapi_type in enum_types:
 572         return 'QTYPE_QSTRING'
 573     elif qapi_type in union_types:
 574         return 'QTYPE_QDICT'
 575     return None
 576
 577
 578 # Return the discriminator enum define if discriminator is specified as an
 579 # enum type, otherwise return None.
 580 def discriminator_find_enum_define(expr):
 581     base = expr.get('base')
 582     discriminator = expr.get('discriminator')
 583
 584     if not (discriminator and base):
 585         return None
 586
 587     base_members = find_base_members(base)
 588     if not base_members:
 589         return None
 590
 591     discriminator_type = base_members.get(discriminator)
 592     if not discriminator_type:
 593         return None
 594
 595     return enum_types.get(discriminator_type)
 596
 597
 598 # Names must be letters, numbers, -, and _.  They must start with letter,
 599 # except for downstream extensions which must start with __RFQDN_.
 600 # Dots are only valid in the downstream extension prefix.
 601 valid_name = re.compile(r'^(__[a-zA-Z0-9.-]+_)?'
 602                         '[a-zA-Z][a-zA-Z0-9_-]*$')
 603
 604
 605 def check_name(info, source, name, allow_optional=False,
 606                enum_member=False):
 607     global valid_name
 608     membername = name
 609
 610     if not isinstance(name, str):
 611         raise QAPISemError(info, "%s requires a string name" % source)
 612     if name.startswith('*'):
 613         membername = name[1:]
 614         if not allow_optional:
 615             raise QAPISemError(info, "%s does not allow optional name '%s'"
 616                                % (source, name))
 617     # Enum members can start with a digit, because the generated C
 618     # code always prefixes it with the enum name
 619     if enum_member and membername[0].isdigit():
 620         membername = 'D' + membername
 621     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
 622     # and 'q_obj_*' implicit type names.
 623     if not valid_name.match(membername) or \
 624        c_name(membername, False).startswith('q_'):
 625         raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
 626
 627
 628 def add_name(name, info, meta, implicit=False):
 629     global all_names
 630     check_name(info, "'%s'" % meta, name)
 631     # FIXME should reject names that differ only in '_' vs. '.'
 632     # vs. '-', because they're liable to clash in generated C.
 633     if name in all_names:
 634         raise QAPISemError(info, "%s '%s' is already defined"
 635                            % (all_names[name], name))
 636     if not implicit and (name.endswith('Kind') or name.endswith('List')):
 637         raise QAPISemError(info, "%s '%s' should not end in '%s'"
 638                            % (meta, name, name[-4:]))
 639     all_names[name] = meta
 640
 641
 642 def check_type(info, source, value, allow_array=False,
 643                allow_dict=False, allow_optional=False,
 644                allow_metas=[]):
 645     global all_names
 646
 647     if value is None:
 648         return
 649
 650     # Check if array type for value is okay
 651     if isinstance(value, list):
 652         if not allow_array:
 653             raise QAPISemError(info, "%s cannot be an array" % source)
 654         if len(value) != 1 or not isinstance(value[0], str):
 655             raise QAPISemError(info,
 656                                "%s: array type must contain single type name" %
 657                                source)
 658         value = value[0]
 659
 660     # Check if type name for value is okay
 661     if isinstance(value, str):
 662         if value not in all_names:
 663             raise QAPISemError(info, "%s uses unknown type '%s'"
 664                                % (source, value))
 665         if not all_names[value] in allow_metas:
 666             raise QAPISemError(info, "%s cannot use %s type '%s'" %
 667                                (source, all_names[value], value))
 668         return
 669
 670     if not allow_dict:
 671         raise QAPISemError(info, "%s should be a type name" % source)
 672
 673     if not isinstance(value, OrderedDict):
 674         raise QAPISemError(info,
 675                            "%s should be a dictionary or type name" % source)
 676
 677     # value is a dictionary, check that each member is okay
 678     for (key, arg) in value.items():
 679         check_name(info, "Member of %s" % source, key,
 680                    allow_optional=allow_optional)
 681         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 682             raise QAPISemError(info, "Member of %s uses reserved name '%s'"
 683                                % (source, key))
 684         # Todo: allow dictionaries to represent default values of
 685         # an optional argument.
 686         check_type(info, "Member '%s' of %s" % (key, source), arg,
 687                    allow_array=True,
 688                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 689                                 'enum'])
 690
 691
 692 def check_command(expr, info):
 693     name = expr['command']
 694     boxed = expr.get('boxed', False)
 695
 696     args_meta = ['struct']
 697     if boxed:
 698         args_meta += ['union', 'alternate']
 699     check_type(info, "'data' for command '%s'" % name,
 700                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 701                allow_metas=args_meta)
 702     returns_meta = ['union', 'struct']
 703     if name in returns_whitelist:
 704         returns_meta += ['built-in', 'alternate', 'enum']
 705     check_type(info, "'returns' for command '%s'" % name,
 706                expr.get('returns'), allow_array=True,
 707                allow_optional=True, allow_metas=returns_meta)
 708
 709
 710 def check_event(expr, info):
 711     name = expr['event']
 712     boxed = expr.get('boxed', False)
 713
 714     meta = ['struct']
 715     if boxed:
 716         meta += ['union', 'alternate']
 717     check_type(info, "'data' for event '%s'" % name,
 718                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 719                allow_metas=meta)
 720
 721
 722 def check_union(expr, info):
 723     name = expr['union']
 724     base = expr.get('base')
 725     discriminator = expr.get('discriminator')
 726     members = expr['data']
 727
 728     # Two types of unions, determined by discriminator.
 729
 730     # With no discriminator it is a simple union.
 731     if discriminator is None:
 732         enum_define = None
 733         allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
 734         if base is not None:
 735             raise QAPISemError(info, "Simple union '%s' must not have a base" %
 736                                name)
 737
 738     # Else, it's a flat union.
 739     else:
 740         # The object must have a string or dictionary 'base'.
 741         check_type(info, "'base' for union '%s'" % name,
 742                    base, allow_dict=True, allow_optional=True,
 743                    allow_metas=['struct'])
 744         if not base:
 745             raise QAPISemError(info, "Flat union '%s' must have a base"
 746                                % name)
 747         base_members = find_base_members(base)
 748         assert base_members is not None
 749
 750         # The value of member 'discriminator' must name a non-optional
 751         # member of the base struct.
 752         check_name(info, "Discriminator of flat union '%s'" % name,
 753                    discriminator)
 754         discriminator_type = base_members.get(discriminator)
 755         if not discriminator_type:
 756             raise QAPISemError(info,
 757                                "Discriminator '%s' is not a member of base "
 758                                "struct '%s'"
 759                                % (discriminator, base))
 760         enum_define = enum_types.get(discriminator_type)
 761         allow_metas = ['struct']
 762         # Do not allow string discriminator
 763         if not enum_define:
 764             raise QAPISemError(info,
 765                                "Discriminator '%s' must be of enumeration "
 766                                "type" % discriminator)
 767
 768     # Check every branch; don't allow an empty union
 769     if len(members) == 0:
 770         raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
 771     for (key, value) in members.items():
 772         check_name(info, "Member of union '%s'" % name, key)
 773
 774         # Each value must name a known type
 775         check_type(info, "Member '%s' of union '%s'" % (key, name),
 776                    value, allow_array=not base, allow_metas=allow_metas)
 777
 778         # If the discriminator names an enum type, then all members
 779         # of 'data' must also be members of the enum type.
 780         if enum_define:
 781             if key not in enum_define['data']:
 782                 raise QAPISemError(info,
 783                                    "Discriminator value '%s' is not found in "
 784                                    "enum '%s'"
 785                                    % (key, enum_define['enum']))
 786
 787     # If discriminator is user-defined, ensure all values are covered
 788     if enum_define:
 789         for value in enum_define['data']:
 790             if value not in members.keys():
 791                 raise QAPISemError(info, "Union '%s' data missing '%s' branch"
 792                                    % (name, value))
 793
 794
 795 def check_alternate(expr, info):
 796     name = expr['alternate']
 797     members = expr['data']
 798     types_seen = {}
 799
 800     # Check every branch; require at least two branches
 801     if len(members) < 2:
 802         raise QAPISemError(info,
 803                            "Alternate '%s' should have at least two branches "
 804                            "in 'data'" % name)
 805     for (key, value) in members.items():
 806         check_name(info, "Member of alternate '%s'" % name, key)
 807
 808         # Ensure alternates have no type conflicts.
 809         check_type(info, "Member '%s' of alternate '%s'" % (key, name),
 810                    value,
 811                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 812         qtype = find_alternate_member_qtype(value)
 813         if not qtype:
 814             raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
 815                                "type '%s'" % (name, key, value))
 816         if qtype in types_seen:
 817             raise QAPISemError(info, "Alternate '%s' member '%s' can't "
 818                                "be distinguished from member '%s'"
 819                                % (name, key, types_seen[qtype]))
 820         types_seen[qtype] = key
 821
 822
 823 def check_enum(expr, info):
 824     name = expr['enum']
 825     members = expr.get('data')
 826     prefix = expr.get('prefix')
 827
 828     if not isinstance(members, list):
 829         raise QAPISemError(info,
 830                            "Enum '%s' requires an array for 'data'" % name)
 831     if prefix is not None and not isinstance(prefix, str):
 832         raise QAPISemError(info,
 833                            "Enum '%s' requires a string for 'prefix'" % name)
 834     for member in members:
 835         check_name(info, "Member of enum '%s'" % name, member,
 836                    enum_member=True)
 837
 838
 839 def check_struct(expr, info):
 840     name = expr['struct']
 841     members = expr['data']
 842
 843     check_type(info, "'data' for struct '%s'" % name, members,
 844                allow_dict=True, allow_optional=True)
 845     check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
 846                allow_metas=['struct'])
 847
 848
 849 def check_keys(expr_elem, meta, required, optional=[]):
 850     expr = expr_elem['expr']
 851     info = expr_elem['info']
 852     name = expr[meta]
 853     if not isinstance(name, str):
 854         raise QAPISemError(info, "'%s' key must have a string value" % meta)
 855     required = required + [meta]
 856     for (key, value) in expr.items():
 857         if key not in required and key not in optional:
 858             raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
 859                                % (key, meta, name))
 860         if (key == 'gen' or key == 'success-response') and value is not False:
 861             raise QAPISemError(info,
 862                                "'%s' of %s '%s' should only use false value"
 863                                % (key, meta, name))
 864         if key == 'boxed' and value is not True:
 865             raise QAPISemError(info,
 866                                "'%s' of %s '%s' should only use true value"
 867                                % (key, meta, name))
 868     for key in required:
 869         if key not in expr:
 870             raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
 871                                % (key, meta, name))
 872
 873
 874 def check_exprs(exprs):
 875     global all_names
 876
 877     # Populate name table with names of built-in types
 878     for builtin in builtin_types.keys():
 879         all_names[builtin] = 'built-in'
 880
 881     # Learn the types and check for valid expression keys
 882     for expr_elem in exprs:
 883         expr = expr_elem['expr']
 884         info = expr_elem['info']
 885         doc = expr_elem.get('doc')
 886
 887         if not doc and doc_required:
 888             raise QAPISemError(info,
 889                                "Expression missing documentation comment")
 890
 891         if 'enum' in expr:
 892             meta = 'enum'
 893             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 894             enum_types[expr[meta]] = expr
 895         elif 'union' in expr:
 896             meta = 'union'
 897             check_keys(expr_elem, 'union', ['data'],
 898                        ['base', 'discriminator'])
 899             union_types[expr[meta]] = expr
 900         elif 'alternate' in expr:
 901             meta = 'alternate'
 902             check_keys(expr_elem, 'alternate', ['data'])
 903         elif 'struct' in expr:
 904             meta = 'struct'
 905             check_keys(expr_elem, 'struct', ['data'], ['base'])
 906             struct_types[expr[meta]] = expr
 907         elif 'command' in expr:
 908             meta = 'command'
 909             check_keys(expr_elem, 'command', [],
 910                        ['data', 'returns', 'gen', 'success-response', 'boxed'])
 911         elif 'event' in expr:
 912             meta = 'event'
 913             check_keys(expr_elem, 'event', [], ['data', 'boxed'])
 914         else:
 915             raise QAPISemError(expr_elem['info'],
 916                                "Expression is missing metatype")
 917         name = expr[meta]
 918         add_name(name, info, meta)
 919         if doc and doc.symbol != name:
 920             raise QAPISemError(info, "Definition of '%s' follows documentation"
 921                                " for '%s'" % (name, doc.symbol))
 922
 923     # Try again for hidden UnionKind enum
 924     for expr_elem in exprs:
 925         expr = expr_elem['expr']
 926         if 'union' in expr and not discriminator_find_enum_define(expr):
 927             name = '%sKind' % expr['union']
 928         elif 'alternate' in expr:
 929             name = '%sKind' % expr['alternate']
 930         else:
 931             continue
 932         enum_types[name] = {'enum': name}
 933         add_name(name, info, 'enum', implicit=True)
 934
 935     # Validate that exprs make sense
 936     for expr_elem in exprs:
 937         expr = expr_elem['expr']
 938         info = expr_elem['info']
 939         doc = expr_elem.get('doc')
 940
 941         if 'enum' in expr:
 942             check_enum(expr, info)
 943         elif 'union' in expr:
 944             check_union(expr, info)
 945         elif 'alternate' in expr:
 946             check_alternate(expr, info)
 947         elif 'struct' in expr:
 948             check_struct(expr, info)
 949         elif 'command' in expr:
 950             check_command(expr, info)
 951         elif 'event' in expr:
 952             check_event(expr, info)
 953         else:
 954             assert False, 'unexpected meta type'
 955
 956         if doc:
 957             doc.check_expr(expr)
 958
 959     return exprs
 960
 961
 962 #
 963 # Schema compiler frontend
 964 #
 965
 966 class QAPISchemaEntity(object):
 967     def __init__(self, name, info, doc):
 968         assert isinstance(name, str)
 969         self.name = name
 970         # For explicitly defined entities, info points to the (explicit)
 971         # definition.  For builtins (and their arrays), info is None.
 972         # For implicitly defined entities, info points to a place that
 973         # triggered the implicit definition (there may be more than one
 974         # such place).
 975         self.info = info
 976         self.doc = doc
 977
 978     def c_name(self):
 979         return c_name(self.name)
 980
 981     def check(self, schema):
 982         pass
 983
 984     def is_implicit(self):
 985         return not self.info
 986
 987     def visit(self, visitor):
 988         pass
 989
 990
 991 class QAPISchemaVisitor(object):
 992     def visit_begin(self, schema):
 993         pass
 994
 995     def visit_end(self):
 996         pass
 997
 998     def visit_needed(self, entity):
 999         # Default to visiting everything
1000         return True
1001
1002     def visit_builtin_type(self, name, info, json_type):
1003         pass
1004
1005     def visit_enum_type(self, name, info, values, prefix):
1006         pass
1007
1008     def visit_array_type(self, name, info, element_type):
1009         pass
1010
1011     def visit_object_type(self, name, info, base, members, variants):
1012         pass
1013
1014     def visit_object_type_flat(self, name, info, members, variants):
1015         pass
1016
1017     def visit_alternate_type(self, name, info, variants):
1018         pass
1019
1020     def visit_command(self, name, info, arg_type, ret_type,
1021                       gen, success_response, boxed):
1022         pass
1023
1024     def visit_event(self, name, info, arg_type, boxed):
1025         pass
1026
1027
1028 class QAPISchemaType(QAPISchemaEntity):
1029     # Return the C type for common use.
1030     # For the types we commonly box, this is a pointer type.
1031     def c_type(self):
1032         pass
1033
1034     # Return the C type to be used in a parameter list.
1035     def c_param_type(self):
1036         return self.c_type()
1037
1038     # Return the C type to be used where we suppress boxing.
1039     def c_unboxed_type(self):
1040         return self.c_type()
1041
1042     def json_type(self):
1043         pass
1044
1045     def alternate_qtype(self):
1046         json2qtype = {
1047             'string':  'QTYPE_QSTRING',
1048             'number':  'QTYPE_QFLOAT',
1049             'int':     'QTYPE_QINT',
1050             'boolean': 'QTYPE_QBOOL',
1051             'object':  'QTYPE_QDICT'
1052         }
1053         return json2qtype.get(self.json_type())
1054
1055     def doc_type(self):
1056         if self.is_implicit():
1057             return None
1058         return self.name
1059
1060
1061 class QAPISchemaBuiltinType(QAPISchemaType):
1062     def __init__(self, name, json_type, c_type):
1063         QAPISchemaType.__init__(self, name, None, None)
1064         assert not c_type or isinstance(c_type, str)
1065         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
1066                              'value')
1067         self._json_type_name = json_type
1068         self._c_type_name = c_type
1069
1070     def c_name(self):
1071         return self.name
1072
1073     def c_type(self):
1074         return self._c_type_name
1075
1076     def c_param_type(self):
1077         if self.name == 'str':
1078             return 'const ' + self._c_type_name
1079         return self._c_type_name
1080
1081     def json_type(self):
1082         return self._json_type_name
1083
1084     def doc_type(self):
1085         return self.json_type()
1086
1087     def visit(self, visitor):
1088         visitor.visit_builtin_type(self.name, self.info, self.json_type())
1089
1090
1091 class QAPISchemaEnumType(QAPISchemaType):
1092     def __init__(self, name, info, doc, values, prefix):
1093         QAPISchemaType.__init__(self, name, info, doc)
1094         for v in values:
1095             assert isinstance(v, QAPISchemaMember)
1096             v.set_owner(name)
1097         assert prefix is None or isinstance(prefix, str)
1098         self.values = values
1099         self.prefix = prefix
1100
1101     def check(self, schema):
1102         seen = {}
1103         for v in self.values:
1104             v.check_clash(self.info, seen)
1105             if self.doc:
1106                 self.doc.connect_member(v)
1107
1108     def is_implicit(self):
1109         # See QAPISchema._make_implicit_enum_type() and ._def_predefineds()
1110         return self.name.endswith('Kind') or self.name == 'QType'
1111
1112     def c_type(self):
1113         return c_name(self.name)
1114
1115     def member_names(self):
1116         return [v.name for v in self.values]
1117
1118     def json_type(self):
1119         return 'string'
1120
1121     def visit(self, visitor):
1122         visitor.visit_enum_type(self.name, self.info,
1123                                 self.member_names(), self.prefix)
1124
1125
1126 class QAPISchemaArrayType(QAPISchemaType):
1127     def __init__(self, name, info, element_type):
1128         QAPISchemaType.__init__(self, name, info, None)
1129         assert isinstance(element_type, str)
1130         self._element_type_name = element_type
1131         self.element_type = None
1132
1133     def check(self, schema):
1134         self.element_type = schema.lookup_type(self._element_type_name)
1135         assert self.element_type
1136
1137     def is_implicit(self):
1138         return True
1139
1140     def c_type(self):
1141         return c_name(self.name) + pointer_suffix
1142
1143     def json_type(self):
1144         return 'array'
1145
1146     def doc_type(self):
1147         elt_doc_type = self.element_type.doc_type()
1148         if not elt_doc_type:
1149             return None
1150         return 'array of ' + elt_doc_type
1151
1152     def visit(self, visitor):
1153         visitor.visit_array_type(self.name, self.info, self.element_type)
1154
1155
1156 class QAPISchemaObjectType(QAPISchemaType):
1157     def __init__(self, name, info, doc, base, local_members, variants):
1158         # struct has local_members, optional base, and no variants
1159         # flat union has base, variants, and no local_members
1160         # simple union has local_members, variants, and no base
1161         QAPISchemaType.__init__(self, name, info, doc)
1162         assert base is None or isinstance(base, str)
1163         for m in local_members:
1164             assert isinstance(m, QAPISchemaObjectTypeMember)
1165             m.set_owner(name)
1166         if variants is not None:
1167             assert isinstance(variants, QAPISchemaObjectTypeVariants)
1168             variants.set_owner(name)
1169         self._base_name = base
1170         self.base = None
1171         self.local_members = local_members
1172         self.variants = variants
1173         self.members = None
1174
1175     def check(self, schema):
1176         if self.members is False:               # check for cycles
1177             raise QAPISemError(self.info,
1178                                "Object %s contains itself" % self.name)
1179         if self.members:
1180             return
1181         self.members = False                    # mark as being checked
1182         seen = OrderedDict()
1183         if self._base_name:
1184             self.base = schema.lookup_type(self._base_name)
1185             assert isinstance(self.base, QAPISchemaObjectType)
1186             self.base.check(schema)
1187             self.base.check_clash(self.info, seen)
1188         for m in self.local_members:
1189             m.check(schema)
1190             m.check_clash(self.info, seen)
1191             if self.doc:
1192                 self.doc.connect_member(m)
1193         self.members = seen.values()
1194         if self.variants:
1195             self.variants.check(schema, seen)
1196             assert self.variants.tag_member in self.members
1197             self.variants.check_clash(self.info, seen)
1198         if self.doc:
1199             self.doc.check()
1200
1201     # Check that the members of this type do not cause duplicate JSON members,
1202     # and update seen to track the members seen so far. Report any errors
1203     # on behalf of info, which is not necessarily self.info
1204     def check_clash(self, info, seen):
1205         assert not self.variants       # not implemented
1206         for m in self.members:
1207             m.check_clash(info, seen)
1208
1209     def is_implicit(self):
1210         # See QAPISchema._make_implicit_object_type(), as well as
1211         # _def_predefineds()
1212         return self.name.startswith('q_')
1213
1214     def is_empty(self):
1215         assert self.members is not None
1216         return not self.members and not self.variants
1217
1218     def c_name(self):
1219         assert self.name != 'q_empty'
1220         return QAPISchemaType.c_name(self)
1221
1222     def c_type(self):
1223         assert not self.is_implicit()
1224         return c_name(self.name) + pointer_suffix
1225
1226     def c_unboxed_type(self):
1227         return c_name(self.name)
1228
1229     def json_type(self):
1230         return 'object'
1231
1232     def visit(self, visitor):
1233         visitor.visit_object_type(self.name, self.info,
1234                                   self.base, self.local_members, self.variants)
1235         visitor.visit_object_type_flat(self.name, self.info,
1236                                        self.members, self.variants)
1237
1238
1239 class QAPISchemaMember(object):
1240     role = 'member'
1241
1242     def __init__(self, name):
1243         assert isinstance(name, str)
1244         self.name = name
1245         self.owner = None
1246
1247     def set_owner(self, name):
1248         assert not self.owner
1249         self.owner = name
1250
1251     def check_clash(self, info, seen):
1252         cname = c_name(self.name)
1253         if cname.lower() != cname and self.owner not in name_case_whitelist:
1254             raise QAPISemError(info,
1255                                "%s should not use uppercase" % self.describe())
1256         if cname in seen:
1257             raise QAPISemError(info, "%s collides with %s" %
1258                                (self.describe(), seen[cname].describe()))
1259         seen[cname] = self
1260
1261     def _pretty_owner(self):
1262         owner = self.owner
1263         if owner.startswith('q_obj_'):
1264             # See QAPISchema._make_implicit_object_type() - reverse the
1265             # mapping there to create a nice human-readable description
1266             owner = owner[6:]
1267             if owner.endswith('-arg'):
1268                 return '(parameter of %s)' % owner[:-4]
1269             elif owner.endswith('-base'):
1270                 return '(base of %s)' % owner[:-5]
1271             else:
1272                 assert owner.endswith('-wrapper')
1273                 # Unreachable and not implemented
1274                 assert False
1275         if owner.endswith('Kind'):
1276             # See QAPISchema._make_implicit_enum_type()
1277             return '(branch of %s)' % owner[:-4]
1278         return '(%s of %s)' % (self.role, owner)
1279
1280     def describe(self):
1281         return "'%s' %s" % (self.name, self._pretty_owner())
1282
1283
1284 class QAPISchemaObjectTypeMember(QAPISchemaMember):
1285     def __init__(self, name, typ, optional):
1286         QAPISchemaMember.__init__(self, name)
1287         assert isinstance(typ, str)
1288         assert isinstance(optional, bool)
1289         self._type_name = typ
1290         self.type = None
1291         self.optional = optional
1292
1293     def check(self, schema):
1294         assert self.owner
1295         self.type = schema.lookup_type(self._type_name)
1296         assert self.type
1297
1298
1299 class QAPISchemaObjectTypeVariants(object):
1300     def __init__(self, tag_name, tag_member, variants):
1301         # Flat unions pass tag_name but not tag_member.
1302         # Simple unions and alternates pass tag_member but not tag_name.
1303         # After check(), tag_member is always set, and tag_name remains
1304         # a reliable witness of being used by a flat union.
1305         assert bool(tag_member) != bool(tag_name)
1306         assert (isinstance(tag_name, str) or
1307                 isinstance(tag_member, QAPISchemaObjectTypeMember))
1308         assert len(variants) > 0
1309         for v in variants:
1310             assert isinstance(v, QAPISchemaObjectTypeVariant)
1311         self._tag_name = tag_name
1312         self.tag_member = tag_member
1313         self.variants = variants
1314
1315     def set_owner(self, name):
1316         for v in self.variants:
1317             v.set_owner(name)
1318
1319     def check(self, schema, seen):
1320         if not self.tag_member:    # flat union
1321             self.tag_member = seen[c_name(self._tag_name)]
1322             assert self._tag_name == self.tag_member.name
1323         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
1324         for v in self.variants:
1325             v.check(schema)
1326             # Union names must match enum values; alternate names are
1327             # checked separately. Use 'seen' to tell the two apart.
1328             if seen:
1329                 assert v.name in self.tag_member.type.member_names()
1330                 assert isinstance(v.type, QAPISchemaObjectType)
1331                 v.type.check(schema)
1332
1333     def check_clash(self, info, seen):
1334         for v in self.variants:
1335             # Reset seen map for each variant, since qapi names from one
1336             # branch do not affect another branch
1337             assert isinstance(v.type, QAPISchemaObjectType)
1338             v.type.check_clash(info, dict(seen))
1339
1340
1341 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1342     role = 'branch'
1343
1344     def __init__(self, name, typ):
1345         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1346
1347
1348 class QAPISchemaAlternateType(QAPISchemaType):
1349     def __init__(self, name, info, doc, variants):
1350         QAPISchemaType.__init__(self, name, info, doc)
1351         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1352         assert variants.tag_member
1353         variants.set_owner(name)
1354         variants.tag_member.set_owner(self.name)
1355         self.variants = variants
1356
1357     def check(self, schema):
1358         self.variants.tag_member.check(schema)
1359         # Not calling self.variants.check_clash(), because there's nothing
1360         # to clash with
1361         self.variants.check(schema, {})
1362         # Alternate branch names have no relation to the tag enum values;
1363         # so we have to check for potential name collisions ourselves.
1364         seen = {}
1365         for v in self.variants.variants:
1366             v.check_clash(self.info, seen)
1367             if self.doc:
1368                 self.doc.connect_member(v)
1369         if self.doc:
1370             self.doc.check()
1371
1372     def c_type(self):
1373         return c_name(self.name) + pointer_suffix
1374
1375     def json_type(self):
1376         return 'value'
1377
1378     def visit(self, visitor):
1379         visitor.visit_alternate_type(self.name, self.info, self.variants)
1380
1381     def is_empty(self):
1382         return False
1383
1384
1385 class QAPISchemaCommand(QAPISchemaEntity):
1386     def __init__(self, name, info, doc, arg_type, ret_type,
1387                  gen, success_response, boxed):
1388         QAPISchemaEntity.__init__(self, name, info, doc)
1389         assert not arg_type or isinstance(arg_type, str)
1390         assert not ret_type or isinstance(ret_type, str)
1391         self._arg_type_name = arg_type
1392         self.arg_type = None
1393         self._ret_type_name = ret_type
1394         self.ret_type = None
1395         self.gen = gen
1396         self.success_response = success_response
1397         self.boxed = boxed
1398
1399     def check(self, schema):
1400         if self._arg_type_name:
1401             self.arg_type = schema.lookup_type(self._arg_type_name)
1402             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1403                     isinstance(self.arg_type, QAPISchemaAlternateType))
1404             self.arg_type.check(schema)
1405             if self.boxed:
1406                 if self.arg_type.is_empty():
1407                     raise QAPISemError(self.info,
1408                                        "Cannot use 'boxed' with empty type")
1409             else:
1410                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1411                 assert not self.arg_type.variants
1412         elif self.boxed:
1413             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1414         if self._ret_type_name:
1415             self.ret_type = schema.lookup_type(self._ret_type_name)
1416             assert isinstance(self.ret_type, QAPISchemaType)
1417
1418     def visit(self, visitor):
1419         visitor.visit_command(self.name, self.info,
1420                               self.arg_type, self.ret_type,
1421                               self.gen, self.success_response, self.boxed)
1422
1423
1424 class QAPISchemaEvent(QAPISchemaEntity):
1425     def __init__(self, name, info, doc, arg_type, boxed):
1426         QAPISchemaEntity.__init__(self, name, info, doc)
1427         assert not arg_type or isinstance(arg_type, str)
1428         self._arg_type_name = arg_type
1429         self.arg_type = None
1430         self.boxed = boxed
1431
1432     def check(self, schema):
1433         if self._arg_type_name:
1434             self.arg_type = schema.lookup_type(self._arg_type_name)
1435             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1436                     isinstance(self.arg_type, QAPISchemaAlternateType))
1437             self.arg_type.check(schema)
1438             if self.boxed:
1439                 if self.arg_type.is_empty():
1440                     raise QAPISemError(self.info,
1441                                        "Cannot use 'boxed' with empty type")
1442             else:
1443                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1444                 assert not self.arg_type.variants
1445         elif self.boxed:
1446             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1447
1448     def visit(self, visitor):
1449         visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
1450
1451
1452 class QAPISchema(object):
1453     def __init__(self, fname):
1454         try:
1455             parser = QAPISchemaParser(open(fname, 'r'))
1456             self.exprs = check_exprs(parser.exprs)
1457             self.docs = parser.docs
1458             self._entity_dict = {}
1459             self._predefining = True
1460             self._def_predefineds()
1461             self._predefining = False
1462             self._def_exprs()
1463             self.check()
1464         except QAPIError as err:
1465             print >>sys.stderr, err
1466             exit(1)
1467
1468     def _def_entity(self, ent):
1469         # Only the predefined types are allowed to not have info
1470         assert ent.info or self._predefining
1471         assert ent.name not in self._entity_dict
1472         self._entity_dict[ent.name] = ent
1473
1474     def lookup_entity(self, name, typ=None):
1475         ent = self._entity_dict.get(name)
1476         if typ and not isinstance(ent, typ):
1477             return None
1478         return ent
1479
1480     def lookup_type(self, name):
1481         return self.lookup_entity(name, QAPISchemaType)
1482
1483     def _def_builtin_type(self, name, json_type, c_type):
1484         self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
1485         # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
1486         # qapi-types.h from a single .c, all arrays of builtins must be
1487         # declared in the first file whether or not they are used.  Nicer
1488         # would be to use lazy instantiation, while figuring out how to
1489         # avoid compilation issues with multiple qapi-types.h.
1490         self._make_array_type(name, None)
1491
1492     def _def_predefineds(self):
1493         for t in [('str',    'string',  'char' + pointer_suffix),
1494                   ('number', 'number',  'double'),
1495                   ('int',    'int',     'int64_t'),
1496                   ('int8',   'int',     'int8_t'),
1497                   ('int16',  'int',     'int16_t'),
1498                   ('int32',  'int',     'int32_t'),
1499                   ('int64',  'int',     'int64_t'),
1500                   ('uint8',  'int',     'uint8_t'),
1501                   ('uint16', 'int',     'uint16_t'),
1502                   ('uint32', 'int',     'uint32_t'),
1503                   ('uint64', 'int',     'uint64_t'),
1504                   ('size',   'int',     'uint64_t'),
1505                   ('bool',   'boolean', 'bool'),
1506                   ('any',    'value',   'QObject' + pointer_suffix)]:
1507             self._def_builtin_type(*t)
1508         self.the_empty_object_type = QAPISchemaObjectType(
1509             'q_empty', None, None, None, [], None)
1510         self._def_entity(self.the_empty_object_type)
1511         qtype_values = self._make_enum_members(['none', 'qnull', 'qint',
1512                                                 'qstring', 'qdict', 'qlist',
1513                                                 'qfloat', 'qbool'])
1514         self._def_entity(QAPISchemaEnumType('QType', None, None,
1515                                             qtype_values, 'QTYPE'))
1516
1517     def _make_enum_members(self, values):
1518         return [QAPISchemaMember(v) for v in values]
1519
1520     def _make_implicit_enum_type(self, name, info, values):
1521         # See also QAPISchemaObjectTypeMember._pretty_owner()
1522         name = name + 'Kind'   # Use namespace reserved by add_name()
1523         self._def_entity(QAPISchemaEnumType(
1524             name, info, None, self._make_enum_members(values), None))
1525         return name
1526
1527     def _make_array_type(self, element_type, info):
1528         name = element_type + 'List'   # Use namespace reserved by add_name()
1529         if not self.lookup_type(name):
1530             self._def_entity(QAPISchemaArrayType(name, info, element_type))
1531         return name
1532
1533     def _make_implicit_object_type(self, name, info, doc, role, members):
1534         if not members:
1535             return None
1536         # See also QAPISchemaObjectTypeMember._pretty_owner()
1537         name = 'q_obj_%s-%s' % (name, role)
1538         if not self.lookup_entity(name, QAPISchemaObjectType):
1539             self._def_entity(QAPISchemaObjectType(name, info, doc, None,
1540                                                   members, None))
1541         return name
1542
1543     def _def_enum_type(self, expr, info, doc):
1544         name = expr['enum']
1545         data = expr['data']
1546         prefix = expr.get('prefix')
1547         self._def_entity(QAPISchemaEnumType(
1548             name, info, doc, self._make_enum_members(data), prefix))
1549
1550     def _make_member(self, name, typ, info):
1551         optional = False
1552         if name.startswith('*'):
1553             name = name[1:]
1554             optional = True
1555         if isinstance(typ, list):
1556             assert len(typ) == 1
1557             typ = self._make_array_type(typ[0], info)
1558         return QAPISchemaObjectTypeMember(name, typ, optional)
1559
1560     def _make_members(self, data, info):
1561         return [self._make_member(key, value, info)
1562                 for (key, value) in data.iteritems()]
1563
1564     def _def_struct_type(self, expr, info, doc):
1565         name = expr['struct']
1566         base = expr.get('base')
1567         data = expr['data']
1568         self._def_entity(QAPISchemaObjectType(name, info, doc, base,
1569                                               self._make_members(data, info),
1570                                               None))
1571
1572     def _make_variant(self, case, typ):
1573         return QAPISchemaObjectTypeVariant(case, typ)
1574
1575     def _make_simple_variant(self, case, typ, info):
1576         if isinstance(typ, list):
1577             assert len(typ) == 1
1578             typ = self._make_array_type(typ[0], info)
1579         typ = self._make_implicit_object_type(
1580             typ, info, None, 'wrapper', [self._make_member('data', typ, info)])
1581         return QAPISchemaObjectTypeVariant(case, typ)
1582
1583     def _def_union_type(self, expr, info, doc):
1584         name = expr['union']
1585         data = expr['data']
1586         base = expr.get('base')
1587         tag_name = expr.get('discriminator')
1588         tag_member = None
1589         if isinstance(base, dict):
1590             base = (self._make_implicit_object_type(
1591                 name, info, doc, 'base', self._make_members(base, info)))
1592         if tag_name:
1593             variants = [self._make_variant(key, value)
1594                         for (key, value) in data.iteritems()]
1595             members = []
1596         else:
1597             variants = [self._make_simple_variant(key, value, info)
1598                         for (key, value) in data.iteritems()]
1599             typ = self._make_implicit_enum_type(name, info,
1600                                                 [v.name for v in variants])
1601             tag_member = QAPISchemaObjectTypeMember('type', typ, False)
1602             members = [tag_member]
1603         self._def_entity(
1604             QAPISchemaObjectType(name, info, doc, base, members,
1605                                  QAPISchemaObjectTypeVariants(tag_name,
1606                                                               tag_member,
1607                                                               variants)))
1608
1609     def _def_alternate_type(self, expr, info, doc):
1610         name = expr['alternate']
1611         data = expr['data']
1612         variants = [self._make_variant(key, value)
1613                     for (key, value) in data.iteritems()]
1614         tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
1615         self._def_entity(
1616             QAPISchemaAlternateType(name, info, doc,
1617                                     QAPISchemaObjectTypeVariants(None,
1618                                                                  tag_member,
1619                                                                  variants)))
1620
1621     def _def_command(self, expr, info, doc):
1622         name = expr['command']
1623         data = expr.get('data')
1624         rets = expr.get('returns')
1625         gen = expr.get('gen', True)
1626         success_response = expr.get('success-response', True)
1627         boxed = expr.get('boxed', False)
1628         if isinstance(data, OrderedDict):
1629             data = self._make_implicit_object_type(
1630                 name, info, doc, 'arg', self._make_members(data, info))
1631         if isinstance(rets, list):
1632             assert len(rets) == 1
1633             rets = self._make_array_type(rets[0], info)
1634         self._def_entity(QAPISchemaCommand(name, info, doc, data, rets,
1635                                            gen, success_response, boxed))
1636
1637     def _def_event(self, expr, info, doc):
1638         name = expr['event']
1639         data = expr.get('data')
1640         boxed = expr.get('boxed', False)
1641         if isinstance(data, OrderedDict):
1642             data = self._make_implicit_object_type(
1643                 name, info, doc, 'arg', self._make_members(data, info))
1644         self._def_entity(QAPISchemaEvent(name, info, doc, data, boxed))
1645
1646     def _def_exprs(self):
1647         for expr_elem in self.exprs:
1648             expr = expr_elem['expr']
1649             info = expr_elem['info']
1650             doc = expr_elem.get('doc')
1651             if 'enum' in expr:
1652                 self._def_enum_type(expr, info, doc)
1653             elif 'struct' in expr:
1654                 self._def_struct_type(expr, info, doc)
1655             elif 'union' in expr:
1656                 self._def_union_type(expr, info, doc)
1657             elif 'alternate' in expr:
1658                 self._def_alternate_type(expr, info, doc)
1659             elif 'command' in expr:
1660                 self._def_command(expr, info, doc)
1661             elif 'event' in expr:
1662                 self._def_event(expr, info, doc)
1663             else:
1664                 assert False
1665
1666     def check(self):
1667         for ent in self._entity_dict.values():
1668             ent.check(self)
1669
1670     def visit(self, visitor):
1671         visitor.visit_begin(self)
1672         for (name, entity) in sorted(self._entity_dict.items()):
1673             if visitor.visit_needed(entity):
1674                 entity.visit(visitor)
1675         visitor.visit_end()
1676
1677
1678 #
1679 # Code generation helpers
1680 #
1681
1682 def camel_case(name):
1683     new_name = ''
1684     first = True
1685     for ch in name:
1686         if ch in ['_', '-']:
1687             first = True
1688         elif first:
1689             new_name += ch.upper()
1690             first = False
1691         else:
1692             new_name += ch.lower()
1693     return new_name
1694
1695
1696 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1697 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1698 # ENUM24_Name -> ENUM24_NAME
1699 def camel_to_upper(value):
1700     c_fun_str = c_name(value, False)
1701     if value.isupper():
1702         return c_fun_str
1703
1704     new_name = ''
1705     l = len(c_fun_str)
1706     for i in range(l):
1707         c = c_fun_str[i]
1708         # When c is upper and no '_' appears before, do more checks
1709         if c.isupper() and (i > 0) and c_fun_str[i - 1] != '_':
1710             if i < l - 1 and c_fun_str[i + 1].islower():
1711                 new_name += '_'
1712             elif c_fun_str[i - 1].isdigit():
1713                 new_name += '_'
1714         new_name += c
1715     return new_name.lstrip('_').upper()
1716
1717
1718 def c_enum_const(type_name, const_name, prefix=None):
1719     if prefix is not None:
1720         type_name = prefix
1721     return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
1722
1723 c_name_trans = string.maketrans('.-', '__')
1724
1725
1726 # Map @name to a valid C identifier.
1727 # If @protect, avoid returning certain ticklish identifiers (like
1728 # C keywords) by prepending 'q_'.
1729 #
1730 # Used for converting 'name' from a 'name':'type' qapi definition
1731 # into a generated struct member, as well as converting type names
1732 # into substrings of a generated C function name.
1733 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1734 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1735 def c_name(name, protect=True):
1736     # ANSI X3J11/88-090, 3.1.1
1737     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1738                      'default', 'do', 'double', 'else', 'enum', 'extern',
1739                      'float', 'for', 'goto', 'if', 'int', 'long', 'register',
1740                      'return', 'short', 'signed', 'sizeof', 'static',
1741                      'struct', 'switch', 'typedef', 'union', 'unsigned',
1742                      'void', 'volatile', 'while'])
1743     # ISO/IEC 9899:1999, 6.4.1
1744     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1745     # ISO/IEC 9899:2011, 6.4.1
1746     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
1747                      '_Noreturn', '_Static_assert', '_Thread_local'])
1748     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1749     # excluding _.*
1750     gcc_words = set(['asm', 'typeof'])
1751     # C++ ISO/IEC 14882:2003 2.11
1752     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1753                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1754                      'namespace', 'new', 'operator', 'private', 'protected',
1755                      'public', 'reinterpret_cast', 'static_cast', 'template',
1756                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1757                      'using', 'virtual', 'wchar_t',
1758                      # alternative representations
1759                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1760                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1761     # namespace pollution:
1762     polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
1763     name = name.translate(c_name_trans)
1764     if protect and (name in c89_words | c99_words | c11_words | gcc_words
1765                     | cpp_words | polluted_words):
1766         return 'q_' + name
1767     return name
1768
1769 eatspace = '\033EATSPACE.'
1770 pointer_suffix = ' *' + eatspace
1771
1772
1773 def genindent(count):
1774     ret = ''
1775     for _ in range(count):
1776         ret += ' '
1777     return ret
1778
1779 indent_level = 0
1780
1781
1782 def push_indent(indent_amount=4):
1783     global indent_level
1784     indent_level += indent_amount
1785
1786
1787 def pop_indent(indent_amount=4):
1788     global indent_level
1789     indent_level -= indent_amount
1790
1791
1792 # Generate @code with @kwds interpolated.
1793 # Obey indent_level, and strip eatspace.
1794 def cgen(code, **kwds):
1795     raw = code % kwds
1796     if indent_level:
1797         indent = genindent(indent_level)
1798         # re.subn() lacks flags support before Python 2.7, use re.compile()
1799         raw = re.subn(re.compile(r'^.', re.MULTILINE),
1800                       indent + r'\g<0>', raw)
1801         raw = raw[0]
1802     return re.sub(re.escape(eatspace) + r' *', '', raw)
1803
1804
1805 def mcgen(code, **kwds):
1806     if code[0] == '\n':
1807         code = code[1:]
1808     return cgen(code, **kwds)
1809
1810
1811 def guardname(filename):
1812     return c_name(filename, protect=False).upper()
1813
1814
1815 def guardstart(name):
1816     return mcgen('''
1817
1818 #ifndef %(name)s
1819 #define %(name)s
1820
1821 ''',
1822                  name=guardname(name))
1823
1824
1825 def guardend(name):
1826     return mcgen('''
1827
1828 #endif /* %(name)s */
1829
1830 ''',
1831                  name=guardname(name))
1832
1833
1834 def gen_enum_lookup(name, values, prefix=None):
1835     ret = mcgen('''
1836
1837 const char *const %(c_name)s_lookup[] = {
1838 ''',
1839                 c_name=c_name(name))
1840     for value in values:
1841         index = c_enum_const(name, value, prefix)
1842         ret += mcgen('''
1843     [%(index)s] = "%(value)s",
1844 ''',
1845                      index=index, value=value)
1846
1847     max_index = c_enum_const(name, '_MAX', prefix)
1848     ret += mcgen('''
1849     [%(max_index)s] = NULL,
1850 };
1851 ''',
1852                  max_index=max_index)
1853     return ret
1854
1855
1856 def gen_enum(name, values, prefix=None):
1857     # append automatically generated _MAX value
1858     enum_values = values + ['_MAX']
1859
1860     ret = mcgen('''
1861
1862 typedef enum %(c_name)s {
1863 ''',
1864                 c_name=c_name(name))
1865
1866     i = 0
1867     for value in enum_values:
1868         ret += mcgen('''
1869     %(c_enum)s = %(i)d,
1870 ''',
1871                      c_enum=c_enum_const(name, value, prefix),
1872                      i=i)
1873         i += 1
1874
1875     ret += mcgen('''
1876 } %(c_name)s;
1877 ''',
1878                  c_name=c_name(name))
1879
1880     ret += mcgen('''
1881
1882 extern const char *const %(c_name)s_lookup[];
1883 ''',
1884                  c_name=c_name(name))
1885     return ret
1886
1887
1888 def gen_params(arg_type, boxed, extra):
1889     if not arg_type:
1890         assert not boxed
1891         return extra
1892     ret = ''
1893     sep = ''
1894     if boxed:
1895         ret += '%s arg' % arg_type.c_param_type()
1896         sep = ', '
1897     else:
1898         assert not arg_type.variants
1899         for memb in arg_type.members:
1900             ret += sep
1901             sep = ', '
1902             if memb.optional:
1903                 ret += 'bool has_%s, ' % c_name(memb.name)
1904             ret += '%s %s' % (memb.type.c_param_type(),
1905                               c_name(memb.name))
1906     if extra:
1907         ret += sep + extra
1908     return ret
1909
1910
1911 #
1912 # Common command line parsing
1913 #
1914
1915
1916 def parse_command_line(extra_options='', extra_long_options=[]):
1917
1918     try:
1919         opts, args = getopt.gnu_getopt(sys.argv[1:],
1920                                        'chp:o:' + extra_options,
1921                                        ['source', 'header', 'prefix=',
1922                                         'output-dir='] + extra_long_options)
1923     except getopt.GetoptError as err:
1924         print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
1925         sys.exit(1)
1926
1927     output_dir = ''
1928     prefix = ''
1929     do_c = False
1930     do_h = False
1931     extra_opts = []
1932
1933     for oa in opts:
1934         o, a = oa
1935         if o in ('-p', '--prefix'):
1936             match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1937             if match.end() != len(a):
1938                 print >>sys.stderr, \
1939                     "%s: 'funny character '%s' in argument of --prefix" \
1940                     % (sys.argv[0], a[match.end()])
1941                 sys.exit(1)
1942             prefix = a
1943         elif o in ('-o', '--output-dir'):
1944             output_dir = a + '/'
1945         elif o in ('-c', '--source'):
1946             do_c = True
1947         elif o in ('-h', '--header'):
1948             do_h = True
1949         else:
1950             extra_opts.append(oa)
1951
1952     if not do_c and not do_h:
1953         do_c = True
1954         do_h = True
1955
1956     if len(args) != 1:
1957         print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
1958         sys.exit(1)
1959     fname = args[0]
1960
1961     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
1962
1963 #
1964 # Generate output files with boilerplate
1965 #
1966
1967
1968 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
1969                 c_comment, h_comment):
1970     guard = guardname(prefix + h_file)
1971     c_file = output_dir + prefix + c_file
1972     h_file = output_dir + prefix + h_file
1973
1974     if output_dir:
1975         try:
1976             os.makedirs(output_dir)
1977         except os.error as e:
1978             if e.errno != errno.EEXIST:
1979                 raise
1980
1981     def maybe_open(really, name, opt):
1982         if really:
1983             return open(name, opt)
1984         else:
1985             import StringIO
1986             return StringIO.StringIO()
1987
1988     fdef = maybe_open(do_c, c_file, 'w')
1989     fdecl = maybe_open(do_h, h_file, 'w')
1990
1991     fdef.write(mcgen('''
1992 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
1993 %(comment)s
1994 ''',
1995                      comment=c_comment))
1996
1997     fdecl.write(mcgen('''
1998 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
1999 %(comment)s
2000 #ifndef %(guard)s
2001 #define %(guard)s
2002
2003 ''',
2004                       comment=h_comment, guard=guard))
2005
2006     return (fdef, fdecl)
2007
2008
2009 def close_output(fdef, fdecl):
2010     fdecl.write('''
2011 #endif
2012 ''')
2013     fdecl.close()
2014     fdef.close()