scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 from __future__ import print_function
  15 import errno
  16 import getopt
  17 import os
  18 import re
  19 import string
  20 import sys
  21 from ordereddict import OrderedDict
  22
  23 builtin_types = {
  24     'null':     'QTYPE_QNULL',
  25     'str':      'QTYPE_QSTRING',
  26     'int':      'QTYPE_QNUM',
  27     'number':   'QTYPE_QNUM',
  28     'bool':     'QTYPE_QBOOL',
  29     'int8':     'QTYPE_QNUM',
  30     'int16':    'QTYPE_QNUM',
  31     'int32':    'QTYPE_QNUM',
  32     'int64':    'QTYPE_QNUM',
  33     'uint8':    'QTYPE_QNUM',
  34     'uint16':   'QTYPE_QNUM',
  35     'uint32':   'QTYPE_QNUM',
  36     'uint64':   'QTYPE_QNUM',
  37     'size':     'QTYPE_QNUM',
  38     'any':      None,           # any QType possible, actually
  39     'QType':    'QTYPE_QSTRING',
  40 }
  41
  42 # Are documentation comments required?
  43 doc_required = False
  44
  45 # Whitelist of commands allowed to return a non-dictionary
  46 returns_whitelist = []
  47
  48 # Whitelist of entities allowed to violate case conventions
  49 name_case_whitelist = []
  50
  51 enum_types = {}
  52 struct_types = {}
  53 union_types = {}
  54 all_names = {}
  55
  56 #
  57 # Parsing the schema into expressions
  58 #
  59
  60
  61 def error_path(parent):
  62     res = ''
  63     while parent:
  64         res = ('In file included from %s:%d:\n' % (parent['file'],
  65                                                    parent['line'])) + res
  66         parent = parent['parent']
  67     return res
  68
  69
  70 class QAPIError(Exception):
  71     def __init__(self, fname, line, col, incl_info, msg):
  72         Exception.__init__(self)
  73         self.fname = fname
  74         self.line = line
  75         self.col = col
  76         self.info = incl_info
  77         self.msg = msg
  78
  79     def __str__(self):
  80         loc = '%s:%d' % (self.fname, self.line)
  81         if self.col is not None:
  82             loc += ':%s' % self.col
  83         return error_path(self.info) + '%s: %s' % (loc, self.msg)
  84
  85
  86 class QAPIParseError(QAPIError):
  87     def __init__(self, parser, msg):
  88         col = 1
  89         for ch in parser.src[parser.line_pos:parser.pos]:
  90             if ch == '\t':
  91                 col = (col + 7) % 8 + 1
  92             else:
  93                 col += 1
  94         QAPIError.__init__(self, parser.fname, parser.line, col,
  95                            parser.incl_info, msg)
  96
  97
  98 class QAPISemError(QAPIError):
  99     def __init__(self, info, msg):
 100         QAPIError.__init__(self, info['file'], info['line'], None,
 101                            info['parent'], msg)
 102
 103
 104 class QAPIDoc(object):
 105     class Section(object):
 106         def __init__(self, name=None):
 107             # optional section name (argument/member or section name)
 108             self.name = name
 109             # the list of lines for this section
 110             self.text = ''
 111
 112         def append(self, line):
 113             self.text += line.rstrip() + '\n'
 114
 115     class ArgSection(Section):
 116         def __init__(self, name):
 117             QAPIDoc.Section.__init__(self, name)
 118             self.member = None
 119
 120         def connect(self, member):
 121             self.member = member
 122
 123     def __init__(self, parser, info):
 124         # self._parser is used to report errors with QAPIParseError.  The
 125         # resulting error position depends on the state of the parser.
 126         # It happens to be the beginning of the comment.  More or less
 127         # servicable, but action at a distance.
 128         self._parser = parser
 129         self.info = info
 130         self.symbol = None
 131         self.body = QAPIDoc.Section()
 132         # dict mapping parameter name to ArgSection
 133         self.args = OrderedDict()
 134         # a list of Section
 135         self.sections = []
 136         # the current section
 137         self._section = self.body
 138
 139     def has_section(self, name):
 140         """Return True if we have a section with this name."""
 141         for i in self.sections:
 142             if i.name == name:
 143                 return True
 144         return False
 145
 146     def append(self, line):
 147         """Parse a comment line and add it to the documentation."""
 148         line = line[1:]
 149         if not line:
 150             self._append_freeform(line)
 151             return
 152
 153         if line[0] != ' ':
 154             raise QAPIParseError(self._parser, "Missing space after #")
 155         line = line[1:]
 156
 157         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 158         # recognized, and get silently treated as ordinary text
 159         if self.symbol:
 160             self._append_symbol_line(line)
 161         elif not self.body.text and line.startswith('@'):
 162             if not line.endswith(':'):
 163                 raise QAPIParseError(self._parser, "Line should end with :")
 164             self.symbol = line[1:-1]
 165             # FIXME invalid names other than the empty string aren't flagged
 166             if not self.symbol:
 167                 raise QAPIParseError(self._parser, "Invalid name")
 168         else:
 169             self._append_freeform(line)
 170
 171     def end_comment(self):
 172         self._end_section()
 173
 174     def _append_symbol_line(self, line):
 175         name = line.split(' ', 1)[0]
 176
 177         if name.startswith('@') and name.endswith(':'):
 178             line = line[len(name)+1:]
 179             self._start_args_section(name[1:-1])
 180         elif name in ('Returns:', 'Since:',
 181                       # those are often singular or plural
 182                       'Note:', 'Notes:',
 183                       'Example:', 'Examples:',
 184                       'TODO:'):
 185             line = line[len(name)+1:]
 186             self._start_section(name[:-1])
 187
 188         self._append_freeform(line)
 189
 190     def _start_args_section(self, name):
 191         # FIXME invalid names other than the empty string aren't flagged
 192         if not name:
 193             raise QAPIParseError(self._parser, "Invalid parameter name")
 194         if name in self.args:
 195             raise QAPIParseError(self._parser,
 196                                  "'%s' parameter name duplicated" % name)
 197         if self.sections:
 198             raise QAPIParseError(self._parser,
 199                                  "'@%s:' can't follow '%s' section"
 200                                  % (name, self.sections[0].name))
 201         self._end_section()
 202         self._section = QAPIDoc.ArgSection(name)
 203         self.args[name] = self._section
 204
 205     def _start_section(self, name=None):
 206         if name in ('Returns', 'Since') and self.has_section(name):
 207             raise QAPIParseError(self._parser,
 208                                  "Duplicated '%s' section" % name)
 209         self._end_section()
 210         self._section = QAPIDoc.Section(name)
 211         self.sections.append(self._section)
 212
 213     def _end_section(self):
 214         if self._section:
 215             text = self._section.text = self._section.text.strip()
 216             if self._section.name and (not text or text.isspace()):
 217                 raise QAPIParseError(self._parser, "Empty doc section '%s'"
 218                                      % self._section.name)
 219             self._section = None
 220
 221     def _append_freeform(self, line):
 222         in_arg = isinstance(self._section, QAPIDoc.ArgSection)
 223         if (in_arg and self._section.text.endswith('\n\n')
 224                 and line and not line[0].isspace()):
 225             self._start_section()
 226         if (in_arg or not self._section.name
 227                 or not self._section.name.startswith('Example')):
 228             line = line.strip()
 229         match = re.match(r'(@\S+:)', line)
 230         if match:
 231             raise QAPIParseError(self._parser,
 232                                  "'%s' not allowed in free-form documentation"
 233                                  % match.group(1))
 234         self._section.append(line)
 235
 236     def connect_member(self, member):
 237         if member.name not in self.args:
 238             # Undocumented TODO outlaw
 239             self.args[member.name] = QAPIDoc.ArgSection(member.name)
 240         self.args[member.name].connect(member)
 241
 242     def check_expr(self, expr):
 243         if self.has_section('Returns') and 'command' not in expr:
 244             raise QAPISemError(self.info,
 245                                "'Returns:' is only valid for commands")
 246
 247     def check(self):
 248         bogus = [name for name, section in self.args.iteritems()
 249                  if not section.member]
 250         if bogus:
 251             raise QAPISemError(
 252                 self.info,
 253                 "The following documented members are not in "
 254                 "the declaration: %s" % ", ".join(bogus))
 255
 256
 257 class QAPISchemaParser(object):
 258
 259     def __init__(self, fp, previously_included=[], incl_info=None):
 260         abs_fname = os.path.abspath(fp.name)
 261         self.fname = fp.name
 262         previously_included.append(abs_fname)
 263         self.incl_info = incl_info
 264         self.src = fp.read()
 265         if self.src == '' or self.src[-1] != '\n':
 266             self.src += '\n'
 267         self.cursor = 0
 268         self.line = 1
 269         self.line_pos = 0
 270         self.exprs = []
 271         self.docs = []
 272         self.accept()
 273         cur_doc = None
 274
 275         while self.tok is not None:
 276             info = {'file': self.fname, 'line': self.line,
 277                     'parent': self.incl_info}
 278             if self.tok == '#':
 279                 self.reject_expr_doc(cur_doc)
 280                 cur_doc = self.get_doc(info)
 281                 self.docs.append(cur_doc)
 282                 continue
 283
 284             expr = self.get_expr(False)
 285             if 'include' in expr:
 286                 self.reject_expr_doc(cur_doc)
 287                 if len(expr) != 1:
 288                     raise QAPISemError(info, "Invalid 'include' directive")
 289                 include = expr['include']
 290                 if not isinstance(include, str):
 291                     raise QAPISemError(info,
 292                                        "Value of 'include' must be a string")
 293                 self._include(include, info, os.path.dirname(abs_fname),
 294                               previously_included)
 295             elif "pragma" in expr:
 296                 self.reject_expr_doc(cur_doc)
 297                 if len(expr) != 1:
 298                     raise QAPISemError(info, "Invalid 'pragma' directive")
 299                 pragma = expr['pragma']
 300                 if not isinstance(pragma, dict):
 301                     raise QAPISemError(
 302                         info, "Value of 'pragma' must be a dictionary")
 303                 for name, value in pragma.iteritems():
 304                     self._pragma(name, value, info)
 305             else:
 306                 expr_elem = {'expr': expr,
 307                              'info': info}
 308                 if cur_doc:
 309                     if not cur_doc.symbol:
 310                         raise QAPISemError(
 311                             cur_doc.info, "Expression documentation required")
 312                     expr_elem['doc'] = cur_doc
 313                 self.exprs.append(expr_elem)
 314             cur_doc = None
 315         self.reject_expr_doc(cur_doc)
 316
 317     @staticmethod
 318     def reject_expr_doc(doc):
 319         if doc and doc.symbol:
 320             raise QAPISemError(
 321                 doc.info,
 322                 "Documentation for '%s' is not followed by the definition"
 323                 % doc.symbol)
 324
 325     def _include(self, include, info, base_dir, previously_included):
 326         incl_abs_fname = os.path.join(base_dir, include)
 327         # catch inclusion cycle
 328         inf = info
 329         while inf:
 330             if incl_abs_fname == os.path.abspath(inf['file']):
 331                 raise QAPISemError(info, "Inclusion loop for %s" % include)
 332             inf = inf['parent']
 333
 334         # skip multiple include of the same file
 335         if incl_abs_fname in previously_included:
 336             return
 337         try:
 338             fobj = open(incl_abs_fname, 'r')
 339         except IOError as e:
 340             raise QAPISemError(info, '%s: %s' % (e.strerror, include))
 341         exprs_include = QAPISchemaParser(fobj, previously_included, info)
 342         self.exprs.extend(exprs_include.exprs)
 343         self.docs.extend(exprs_include.docs)
 344
 345     def _pragma(self, name, value, info):
 346         global doc_required, returns_whitelist, name_case_whitelist
 347         if name == 'doc-required':
 348             if not isinstance(value, bool):
 349                 raise QAPISemError(info,
 350                                    "Pragma 'doc-required' must be boolean")
 351             doc_required = value
 352         elif name == 'returns-whitelist':
 353             if (not isinstance(value, list)
 354                     or any([not isinstance(elt, str) for elt in value])):
 355                 raise QAPISemError(info,
 356                                    "Pragma returns-whitelist must be"
 357                                    " a list of strings")
 358             returns_whitelist = value
 359         elif name == 'name-case-whitelist':
 360             if (not isinstance(value, list)
 361                     or any([not isinstance(elt, str) for elt in value])):
 362                 raise QAPISemError(info,
 363                                    "Pragma name-case-whitelist must be"
 364                                    " a list of strings")
 365             name_case_whitelist = value
 366         else:
 367             raise QAPISemError(info, "Unknown pragma '%s'" % name)
 368
 369     def accept(self, skip_comment=True):
 370         while True:
 371             self.tok = self.src[self.cursor]
 372             self.pos = self.cursor
 373             self.cursor += 1
 374             self.val = None
 375
 376             if self.tok == '#':
 377                 if self.src[self.cursor] == '#':
 378                     # Start of doc comment
 379                     skip_comment = False
 380                 self.cursor = self.src.find('\n', self.cursor)
 381                 if not skip_comment:
 382                     self.val = self.src[self.pos:self.cursor]
 383                     return
 384             elif self.tok in '{}:,[]':
 385                 return
 386             elif self.tok == "'":
 387                 string = ''
 388                 esc = False
 389                 while True:
 390                     ch = self.src[self.cursor]
 391                     self.cursor += 1
 392                     if ch == '\n':
 393                         raise QAPIParseError(self, 'Missing terminating "\'"')
 394                     if esc:
 395                         if ch == 'b':
 396                             string += '\b'
 397                         elif ch == 'f':
 398                             string += '\f'
 399                         elif ch == 'n':
 400                             string += '\n'
 401                         elif ch == 'r':
 402                             string += '\r'
 403                         elif ch == 't':
 404                             string += '\t'
 405                         elif ch == 'u':
 406                             value = 0
 407                             for _ in range(0, 4):
 408                                 ch = self.src[self.cursor]
 409                                 self.cursor += 1
 410                                 if ch not in '0123456789abcdefABCDEF':
 411                                     raise QAPIParseError(self,
 412                                                          '\\u escape needs 4 '
 413                                                          'hex digits')
 414                                 value = (value << 4) + int(ch, 16)
 415                             # If Python 2 and 3 didn't disagree so much on
 416                             # how to handle Unicode, then we could allow
 417                             # Unicode string defaults.  But most of QAPI is
 418                             # ASCII-only, so we aren't losing much for now.
 419                             if not value or value > 0x7f:
 420                                 raise QAPIParseError(self,
 421                                                      'For now, \\u escape '
 422                                                      'only supports non-zero '
 423                                                      'values up to \\u007f')
 424                             string += chr(value)
 425                         elif ch in '\\/\'"':
 426                             string += ch
 427                         else:
 428                             raise QAPIParseError(self,
 429                                                  "Unknown escape \\%s" % ch)
 430                         esc = False
 431                     elif ch == '\\':
 432                         esc = True
 433                     elif ch == "'":
 434                         self.val = string
 435                         return
 436                     else:
 437                         string += ch
 438             elif self.src.startswith('true', self.pos):
 439                 self.val = True
 440                 self.cursor += 3
 441                 return
 442             elif self.src.startswith('false', self.pos):
 443                 self.val = False
 444                 self.cursor += 4
 445                 return
 446             elif self.src.startswith('null', self.pos):
 447                 self.val = None
 448                 self.cursor += 3
 449                 return
 450             elif self.tok == '\n':
 451                 if self.cursor == len(self.src):
 452                     self.tok = None
 453                     return
 454                 self.line += 1
 455                 self.line_pos = self.cursor
 456             elif not self.tok.isspace():
 457                 raise QAPIParseError(self, 'Stray "%s"' % self.tok)
 458
 459     def get_members(self):
 460         expr = OrderedDict()
 461         if self.tok == '}':
 462             self.accept()
 463             return expr
 464         if self.tok != "'":
 465             raise QAPIParseError(self, 'Expected string or "}"')
 466         while True:
 467             key = self.val
 468             self.accept()
 469             if self.tok != ':':
 470                 raise QAPIParseError(self, 'Expected ":"')
 471             self.accept()
 472             if key in expr:
 473                 raise QAPIParseError(self, 'Duplicate key "%s"' % key)
 474             expr[key] = self.get_expr(True)
 475             if self.tok == '}':
 476                 self.accept()
 477                 return expr
 478             if self.tok != ',':
 479                 raise QAPIParseError(self, 'Expected "," or "}"')
 480             self.accept()
 481             if self.tok != "'":
 482                 raise QAPIParseError(self, 'Expected string')
 483
 484     def get_values(self):
 485         expr = []
 486         if self.tok == ']':
 487             self.accept()
 488             return expr
 489         if self.tok not in "{['tfn":
 490             raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
 491                                  'boolean or "null"')
 492         while True:
 493             expr.append(self.get_expr(True))
 494             if self.tok == ']':
 495                 self.accept()
 496                 return expr
 497             if self.tok != ',':
 498                 raise QAPIParseError(self, 'Expected "," or "]"')
 499             self.accept()
 500
 501     def get_expr(self, nested):
 502         if self.tok != '{' and not nested:
 503             raise QAPIParseError(self, 'Expected "{"')
 504         if self.tok == '{':
 505             self.accept()
 506             expr = self.get_members()
 507         elif self.tok == '[':
 508             self.accept()
 509             expr = self.get_values()
 510         elif self.tok in "'tfn":
 511             expr = self.val
 512             self.accept()
 513         else:
 514             raise QAPIParseError(self, 'Expected "{", "[", string, '
 515                                  'boolean or "null"')
 516         return expr
 517
 518     def get_doc(self, info):
 519         if self.val != '##':
 520             raise QAPIParseError(self, "Junk after '##' at start of "
 521                                  "documentation comment")
 522
 523         doc = QAPIDoc(self, info)
 524         self.accept(False)
 525         while self.tok == '#':
 526             if self.val.startswith('##'):
 527                 # End of doc comment
 528                 if self.val != '##':
 529                     raise QAPIParseError(self, "Junk after '##' at end of "
 530                                          "documentation comment")
 531                 doc.end_comment()
 532                 self.accept()
 533                 return doc
 534             else:
 535                 doc.append(self.val)
 536             self.accept(False)
 537
 538         raise QAPIParseError(self, "Documentation comment must end with '##'")
 539
 540
 541 #
 542 # Semantic analysis of schema expressions
 543 # TODO fold into QAPISchema
 544 # TODO catching name collisions in generated code would be nice
 545 #
 546
 547
 548 def find_base_members(base):
 549     if isinstance(base, dict):
 550         return base
 551     base_struct_define = struct_types.get(base)
 552     if not base_struct_define:
 553         return None
 554     return base_struct_define['data']
 555
 556
 557 # Return the qtype of an alternate branch, or None on error.
 558 def find_alternate_member_qtype(qapi_type):
 559     if qapi_type in builtin_types:
 560         return builtin_types[qapi_type]
 561     elif qapi_type in struct_types:
 562         return 'QTYPE_QDICT'
 563     elif qapi_type in enum_types:
 564         return 'QTYPE_QSTRING'
 565     elif qapi_type in union_types:
 566         return 'QTYPE_QDICT'
 567     return None
 568
 569
 570 # Return the discriminator enum define if discriminator is specified as an
 571 # enum type, otherwise return None.
 572 def discriminator_find_enum_define(expr):
 573     base = expr.get('base')
 574     discriminator = expr.get('discriminator')
 575
 576     if not (discriminator and base):
 577         return None
 578
 579     base_members = find_base_members(base)
 580     if not base_members:
 581         return None
 582
 583     discriminator_type = base_members.get(discriminator)
 584     if not discriminator_type:
 585         return None
 586
 587     return enum_types.get(discriminator_type)
 588
 589
 590 # Names must be letters, numbers, -, and _.  They must start with letter,
 591 # except for downstream extensions which must start with __RFQDN_.
 592 # Dots are only valid in the downstream extension prefix.
 593 valid_name = re.compile(r'^(__[a-zA-Z0-9.-]+_)?'
 594                         '[a-zA-Z][a-zA-Z0-9_-]*$')
 595
 596
 597 def check_name(info, source, name, allow_optional=False,
 598                enum_member=False):
 599     global valid_name
 600     membername = name
 601
 602     if not isinstance(name, str):
 603         raise QAPISemError(info, "%s requires a string name" % source)
 604     if name.startswith('*'):
 605         membername = name[1:]
 606         if not allow_optional:
 607             raise QAPISemError(info, "%s does not allow optional name '%s'"
 608                                % (source, name))
 609     # Enum members can start with a digit, because the generated C
 610     # code always prefixes it with the enum name
 611     if enum_member and membername[0].isdigit():
 612         membername = 'D' + membername
 613     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
 614     # and 'q_obj_*' implicit type names.
 615     if not valid_name.match(membername) or \
 616        c_name(membername, False).startswith('q_'):
 617         raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
 618
 619
 620 def add_name(name, info, meta, implicit=False):
 621     global all_names
 622     check_name(info, "'%s'" % meta, name)
 623     # FIXME should reject names that differ only in '_' vs. '.'
 624     # vs. '-', because they're liable to clash in generated C.
 625     if name in all_names:
 626         raise QAPISemError(info, "%s '%s' is already defined"
 627                            % (all_names[name], name))
 628     if not implicit and (name.endswith('Kind') or name.endswith('List')):
 629         raise QAPISemError(info, "%s '%s' should not end in '%s'"
 630                            % (meta, name, name[-4:]))
 631     all_names[name] = meta
 632
 633
 634 def check_type(info, source, value, allow_array=False,
 635                allow_dict=False, allow_optional=False,
 636                allow_metas=[]):
 637     global all_names
 638
 639     if value is None:
 640         return
 641
 642     # Check if array type for value is okay
 643     if isinstance(value, list):
 644         if not allow_array:
 645             raise QAPISemError(info, "%s cannot be an array" % source)
 646         if len(value) != 1 or not isinstance(value[0], str):
 647             raise QAPISemError(info,
 648                                "%s: array type must contain single type name" %
 649                                source)
 650         value = value[0]
 651
 652     # Check if type name for value is okay
 653     if isinstance(value, str):
 654         if value not in all_names:
 655             raise QAPISemError(info, "%s uses unknown type '%s'"
 656                                % (source, value))
 657         if not all_names[value] in allow_metas:
 658             raise QAPISemError(info, "%s cannot use %s type '%s'" %
 659                                (source, all_names[value], value))
 660         return
 661
 662     if not allow_dict:
 663         raise QAPISemError(info, "%s should be a type name" % source)
 664
 665     if not isinstance(value, OrderedDict):
 666         raise QAPISemError(info,
 667                            "%s should be a dictionary or type name" % source)
 668
 669     # value is a dictionary, check that each member is okay
 670     for (key, arg) in value.items():
 671         check_name(info, "Member of %s" % source, key,
 672                    allow_optional=allow_optional)
 673         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 674             raise QAPISemError(info, "Member of %s uses reserved name '%s'"
 675                                % (source, key))
 676         # Todo: allow dictionaries to represent default values of
 677         # an optional argument.
 678         check_type(info, "Member '%s' of %s" % (key, source), arg,
 679                    allow_array=True,
 680                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 681                                 'enum'])
 682
 683
 684 def check_command(expr, info):
 685     name = expr['command']
 686     boxed = expr.get('boxed', False)
 687
 688     args_meta = ['struct']
 689     if boxed:
 690         args_meta += ['union', 'alternate']
 691     check_type(info, "'data' for command '%s'" % name,
 692                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 693                allow_metas=args_meta)
 694     returns_meta = ['union', 'struct']
 695     if name in returns_whitelist:
 696         returns_meta += ['built-in', 'alternate', 'enum']
 697     check_type(info, "'returns' for command '%s'" % name,
 698                expr.get('returns'), allow_array=True,
 699                allow_optional=True, allow_metas=returns_meta)
 700
 701
 702 def check_event(expr, info):
 703     name = expr['event']
 704     boxed = expr.get('boxed', False)
 705
 706     meta = ['struct']
 707     if boxed:
 708         meta += ['union', 'alternate']
 709     check_type(info, "'data' for event '%s'" % name,
 710                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 711                allow_metas=meta)
 712
 713
 714 def check_union(expr, info):
 715     name = expr['union']
 716     base = expr.get('base')
 717     discriminator = expr.get('discriminator')
 718     members = expr['data']
 719
 720     # Two types of unions, determined by discriminator.
 721
 722     # With no discriminator it is a simple union.
 723     if discriminator is None:
 724         enum_define = None
 725         allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
 726         if base is not None:
 727             raise QAPISemError(info, "Simple union '%s' must not have a base" %
 728                                name)
 729
 730     # Else, it's a flat union.
 731     else:
 732         # The object must have a string or dictionary 'base'.
 733         check_type(info, "'base' for union '%s'" % name,
 734                    base, allow_dict=True, allow_optional=True,
 735                    allow_metas=['struct'])
 736         if not base:
 737             raise QAPISemError(info, "Flat union '%s' must have a base"
 738                                % name)
 739         base_members = find_base_members(base)
 740         assert base_members is not None
 741
 742         # The value of member 'discriminator' must name a non-optional
 743         # member of the base struct.
 744         check_name(info, "Discriminator of flat union '%s'" % name,
 745                    discriminator)
 746         discriminator_type = base_members.get(discriminator)
 747         if not discriminator_type:
 748             raise QAPISemError(info,
 749                                "Discriminator '%s' is not a member of base "
 750                                "struct '%s'"
 751                                % (discriminator, base))
 752         enum_define = enum_types.get(discriminator_type)
 753         allow_metas = ['struct']
 754         # Do not allow string discriminator
 755         if not enum_define:
 756             raise QAPISemError(info,
 757                                "Discriminator '%s' must be of enumeration "
 758                                "type" % discriminator)
 759
 760     # Check every branch; don't allow an empty union
 761     if len(members) == 0:
 762         raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
 763     for (key, value) in members.items():
 764         check_name(info, "Member of union '%s'" % name, key)
 765
 766         # Each value must name a known type
 767         check_type(info, "Member '%s' of union '%s'" % (key, name),
 768                    value, allow_array=not base, allow_metas=allow_metas)
 769
 770         # If the discriminator names an enum type, then all members
 771         # of 'data' must also be members of the enum type.
 772         if enum_define:
 773             if key not in enum_define['data']:
 774                 raise QAPISemError(info,
 775                                    "Discriminator value '%s' is not found in "
 776                                    "enum '%s'"
 777                                    % (key, enum_define['enum']))
 778
 779     # If discriminator is user-defined, ensure all values are covered
 780     if enum_define:
 781         for value in enum_define['data']:
 782             if value not in members.keys():
 783                 raise QAPISemError(info, "Union '%s' data missing '%s' branch"
 784                                    % (name, value))
 785
 786
 787 def check_alternate(expr, info):
 788     name = expr['alternate']
 789     members = expr['data']
 790     types_seen = {}
 791
 792     # Check every branch; require at least two branches
 793     if len(members) < 2:
 794         raise QAPISemError(info,
 795                            "Alternate '%s' should have at least two branches "
 796                            "in 'data'" % name)
 797     for (key, value) in members.items():
 798         check_name(info, "Member of alternate '%s'" % name, key)
 799
 800         # Ensure alternates have no type conflicts.
 801         check_type(info, "Member '%s' of alternate '%s'" % (key, name),
 802                    value,
 803                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 804         qtype = find_alternate_member_qtype(value)
 805         if not qtype:
 806             raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
 807                                "type '%s'" % (name, key, value))
 808         conflicting = set([qtype])
 809         if qtype == 'QTYPE_QSTRING':
 810             enum_expr = enum_types.get(value)
 811             if enum_expr:
 812                 for v in enum_expr['data']:
 813                     if v in ['on', 'off']:
 814                         conflicting.add('QTYPE_QBOOL')
 815                     if re.match(r'[-+0-9.]', v): # lazy, could be tightened
 816                         conflicting.add('QTYPE_QNUM')
 817             else:
 818                 conflicting.add('QTYPE_QNUM')
 819                 conflicting.add('QTYPE_QBOOL')
 820         for qt in conflicting:
 821             if qt in types_seen:
 822                 raise QAPISemError(info, "Alternate '%s' member '%s' can't "
 823                                    "be distinguished from member '%s'"
 824                                    % (name, key, types_seen[qt]))
 825             types_seen[qt] = key
 826
 827
 828 def check_enum(expr, info):
 829     name = expr['enum']
 830     members = expr.get('data')
 831     prefix = expr.get('prefix')
 832
 833     if not isinstance(members, list):
 834         raise QAPISemError(info,
 835                            "Enum '%s' requires an array for 'data'" % name)
 836     if prefix is not None and not isinstance(prefix, str):
 837         raise QAPISemError(info,
 838                            "Enum '%s' requires a string for 'prefix'" % name)
 839     for member in members:
 840         check_name(info, "Member of enum '%s'" % name, member,
 841                    enum_member=True)
 842
 843
 844 def check_struct(expr, info):
 845     name = expr['struct']
 846     members = expr['data']
 847
 848     check_type(info, "'data' for struct '%s'" % name, members,
 849                allow_dict=True, allow_optional=True)
 850     check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
 851                allow_metas=['struct'])
 852
 853
 854 def check_keys(expr_elem, meta, required, optional=[]):
 855     expr = expr_elem['expr']
 856     info = expr_elem['info']
 857     name = expr[meta]
 858     if not isinstance(name, str):
 859         raise QAPISemError(info, "'%s' key must have a string value" % meta)
 860     required = required + [meta]
 861     for (key, value) in expr.items():
 862         if key not in required and key not in optional:
 863             raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
 864                                % (key, meta, name))
 865         if (key == 'gen' or key == 'success-response') and value is not False:
 866             raise QAPISemError(info,
 867                                "'%s' of %s '%s' should only use false value"
 868                                % (key, meta, name))
 869         if key == 'boxed' and value is not True:
 870             raise QAPISemError(info,
 871                                "'%s' of %s '%s' should only use true value"
 872                                % (key, meta, name))
 873     for key in required:
 874         if key not in expr:
 875             raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
 876                                % (key, meta, name))
 877
 878
 879 def check_exprs(exprs):
 880     global all_names
 881
 882     # Populate name table with names of built-in types
 883     for builtin in builtin_types.keys():
 884         all_names[builtin] = 'built-in'
 885
 886     # Learn the types and check for valid expression keys
 887     for expr_elem in exprs:
 888         expr = expr_elem['expr']
 889         info = expr_elem['info']
 890         doc = expr_elem.get('doc')
 891
 892         if not doc and doc_required:
 893             raise QAPISemError(info,
 894                                "Expression missing documentation comment")
 895
 896         if 'enum' in expr:
 897             meta = 'enum'
 898             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 899             enum_types[expr[meta]] = expr
 900         elif 'union' in expr:
 901             meta = 'union'
 902             check_keys(expr_elem, 'union', ['data'],
 903                        ['base', 'discriminator'])
 904             union_types[expr[meta]] = expr
 905         elif 'alternate' in expr:
 906             meta = 'alternate'
 907             check_keys(expr_elem, 'alternate', ['data'])
 908         elif 'struct' in expr:
 909             meta = 'struct'
 910             check_keys(expr_elem, 'struct', ['data'], ['base'])
 911             struct_types[expr[meta]] = expr
 912         elif 'command' in expr:
 913             meta = 'command'
 914             check_keys(expr_elem, 'command', [],
 915                        ['data', 'returns', 'gen', 'success-response', 'boxed'])
 916         elif 'event' in expr:
 917             meta = 'event'
 918             check_keys(expr_elem, 'event', [], ['data', 'boxed'])
 919         else:
 920             raise QAPISemError(expr_elem['info'],
 921                                "Expression is missing metatype")
 922         name = expr[meta]
 923         add_name(name, info, meta)
 924         if doc and doc.symbol != name:
 925             raise QAPISemError(info, "Definition of '%s' follows documentation"
 926                                " for '%s'" % (name, doc.symbol))
 927
 928     # Try again for hidden UnionKind enum
 929     for expr_elem in exprs:
 930         expr = expr_elem['expr']
 931         if 'union' in expr and not discriminator_find_enum_define(expr):
 932             name = '%sKind' % expr['union']
 933         elif 'alternate' in expr:
 934             name = '%sKind' % expr['alternate']
 935         else:
 936             continue
 937         enum_types[name] = {'enum': name}
 938         add_name(name, info, 'enum', implicit=True)
 939
 940     # Validate that exprs make sense
 941     for expr_elem in exprs:
 942         expr = expr_elem['expr']
 943         info = expr_elem['info']
 944         doc = expr_elem.get('doc')
 945
 946         if 'enum' in expr:
 947             check_enum(expr, info)
 948         elif 'union' in expr:
 949             check_union(expr, info)
 950         elif 'alternate' in expr:
 951             check_alternate(expr, info)
 952         elif 'struct' in expr:
 953             check_struct(expr, info)
 954         elif 'command' in expr:
 955             check_command(expr, info)
 956         elif 'event' in expr:
 957             check_event(expr, info)
 958         else:
 959             assert False, 'unexpected meta type'
 960
 961         if doc:
 962             doc.check_expr(expr)
 963
 964     return exprs
 965
 966
 967 #
 968 # Schema compiler frontend
 969 #
 970
 971 class QAPISchemaEntity(object):
 972     def __init__(self, name, info, doc):
 973         assert isinstance(name, str)
 974         self.name = name
 975         # For explicitly defined entities, info points to the (explicit)
 976         # definition.  For builtins (and their arrays), info is None.
 977         # For implicitly defined entities, info points to a place that
 978         # triggered the implicit definition (there may be more than one
 979         # such place).
 980         self.info = info
 981         self.doc = doc
 982
 983     def c_name(self):
 984         return c_name(self.name)
 985
 986     def check(self, schema):
 987         pass
 988
 989     def is_implicit(self):
 990         return not self.info
 991
 992     def visit(self, visitor):
 993         pass
 994
 995
 996 class QAPISchemaVisitor(object):
 997     def visit_begin(self, schema):
 998         pass
 999
1000     def visit_end(self):
1001         pass
1002
1003     def visit_needed(self, entity):
1004         # Default to visiting everything
1005         return True
1006
1007     def visit_builtin_type(self, name, info, json_type):
1008         pass
1009
1010     def visit_enum_type(self, name, info, values, prefix):
1011         pass
1012
1013     def visit_array_type(self, name, info, element_type):
1014         pass
1015
1016     def visit_object_type(self, name, info, base, members, variants):
1017         pass
1018
1019     def visit_object_type_flat(self, name, info, members, variants):
1020         pass
1021
1022     def visit_alternate_type(self, name, info, variants):
1023         pass
1024
1025     def visit_command(self, name, info, arg_type, ret_type,
1026                       gen, success_response, boxed):
1027         pass
1028
1029     def visit_event(self, name, info, arg_type, boxed):
1030         pass
1031
1032
1033 class QAPISchemaType(QAPISchemaEntity):
1034     # Return the C type for common use.
1035     # For the types we commonly box, this is a pointer type.
1036     def c_type(self):
1037         pass
1038
1039     # Return the C type to be used in a parameter list.
1040     def c_param_type(self):
1041         return self.c_type()
1042
1043     # Return the C type to be used where we suppress boxing.
1044     def c_unboxed_type(self):
1045         return self.c_type()
1046
1047     def json_type(self):
1048         pass
1049
1050     def alternate_qtype(self):
1051         json2qtype = {
1052             'null':    'QTYPE_QNULL',
1053             'string':  'QTYPE_QSTRING',
1054             'number':  'QTYPE_QNUM',
1055             'int':     'QTYPE_QNUM',
1056             'boolean': 'QTYPE_QBOOL',
1057             'object':  'QTYPE_QDICT'
1058         }
1059         return json2qtype.get(self.json_type())
1060
1061     def doc_type(self):
1062         if self.is_implicit():
1063             return None
1064         return self.name
1065
1066
1067 class QAPISchemaBuiltinType(QAPISchemaType):
1068     def __init__(self, name, json_type, c_type):
1069         QAPISchemaType.__init__(self, name, None, None)
1070         assert not c_type or isinstance(c_type, str)
1071         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
1072                              'value')
1073         self._json_type_name = json_type
1074         self._c_type_name = c_type
1075
1076     def c_name(self):
1077         return self.name
1078
1079     def c_type(self):
1080         return self._c_type_name
1081
1082     def c_param_type(self):
1083         if self.name == 'str':
1084             return 'const ' + self._c_type_name
1085         return self._c_type_name
1086
1087     def json_type(self):
1088         return self._json_type_name
1089
1090     def doc_type(self):
1091         return self.json_type()
1092
1093     def visit(self, visitor):
1094         visitor.visit_builtin_type(self.name, self.info, self.json_type())
1095
1096
1097 class QAPISchemaEnumType(QAPISchemaType):
1098     def __init__(self, name, info, doc, values, prefix):
1099         QAPISchemaType.__init__(self, name, info, doc)
1100         for v in values:
1101             assert isinstance(v, QAPISchemaMember)
1102             v.set_owner(name)
1103         assert prefix is None or isinstance(prefix, str)
1104         self.values = values
1105         self.prefix = prefix
1106
1107     def check(self, schema):
1108         seen = {}
1109         for v in self.values:
1110             v.check_clash(self.info, seen)
1111             if self.doc:
1112                 self.doc.connect_member(v)
1113
1114     def is_implicit(self):
1115         # See QAPISchema._make_implicit_enum_type() and ._def_predefineds()
1116         return self.name.endswith('Kind') or self.name == 'QType'
1117
1118     def c_type(self):
1119         return c_name(self.name)
1120
1121     def member_names(self):
1122         return [v.name for v in self.values]
1123
1124     def json_type(self):
1125         return 'string'
1126
1127     def visit(self, visitor):
1128         visitor.visit_enum_type(self.name, self.info,
1129                                 self.member_names(), self.prefix)
1130
1131
1132 class QAPISchemaArrayType(QAPISchemaType):
1133     def __init__(self, name, info, element_type):
1134         QAPISchemaType.__init__(self, name, info, None)
1135         assert isinstance(element_type, str)
1136         self._element_type_name = element_type
1137         self.element_type = None
1138
1139     def check(self, schema):
1140         self.element_type = schema.lookup_type(self._element_type_name)
1141         assert self.element_type
1142
1143     def is_implicit(self):
1144         return True
1145
1146     def c_type(self):
1147         return c_name(self.name) + pointer_suffix
1148
1149     def json_type(self):
1150         return 'array'
1151
1152     def doc_type(self):
1153         elt_doc_type = self.element_type.doc_type()
1154         if not elt_doc_type:
1155             return None
1156         return 'array of ' + elt_doc_type
1157
1158     def visit(self, visitor):
1159         visitor.visit_array_type(self.name, self.info, self.element_type)
1160
1161
1162 class QAPISchemaObjectType(QAPISchemaType):
1163     def __init__(self, name, info, doc, base, local_members, variants):
1164         # struct has local_members, optional base, and no variants
1165         # flat union has base, variants, and no local_members
1166         # simple union has local_members, variants, and no base
1167         QAPISchemaType.__init__(self, name, info, doc)
1168         assert base is None or isinstance(base, str)
1169         for m in local_members:
1170             assert isinstance(m, QAPISchemaObjectTypeMember)
1171             m.set_owner(name)
1172         if variants is not None:
1173             assert isinstance(variants, QAPISchemaObjectTypeVariants)
1174             variants.set_owner(name)
1175         self._base_name = base
1176         self.base = None
1177         self.local_members = local_members
1178         self.variants = variants
1179         self.members = None
1180
1181     def check(self, schema):
1182         if self.members is False:               # check for cycles
1183             raise QAPISemError(self.info,
1184                                "Object %s contains itself" % self.name)
1185         if self.members:
1186             return
1187         self.members = False                    # mark as being checked
1188         seen = OrderedDict()
1189         if self._base_name:
1190             self.base = schema.lookup_type(self._base_name)
1191             assert isinstance(self.base, QAPISchemaObjectType)
1192             self.base.check(schema)
1193             self.base.check_clash(self.info, seen)
1194         for m in self.local_members:
1195             m.check(schema)
1196             m.check_clash(self.info, seen)
1197             if self.doc:
1198                 self.doc.connect_member(m)
1199         self.members = seen.values()
1200         if self.variants:
1201             self.variants.check(schema, seen)
1202             assert self.variants.tag_member in self.members
1203             self.variants.check_clash(self.info, seen)
1204         if self.doc:
1205             self.doc.check()
1206
1207     # Check that the members of this type do not cause duplicate JSON members,
1208     # and update seen to track the members seen so far. Report any errors
1209     # on behalf of info, which is not necessarily self.info
1210     def check_clash(self, info, seen):
1211         assert not self.variants       # not implemented
1212         for m in self.members:
1213             m.check_clash(info, seen)
1214
1215     def is_implicit(self):
1216         # See QAPISchema._make_implicit_object_type(), as well as
1217         # _def_predefineds()
1218         return self.name.startswith('q_')
1219
1220     def is_empty(self):
1221         assert self.members is not None
1222         return not self.members and not self.variants
1223
1224     def c_name(self):
1225         assert self.name != 'q_empty'
1226         return QAPISchemaType.c_name(self)
1227
1228     def c_type(self):
1229         assert not self.is_implicit()
1230         return c_name(self.name) + pointer_suffix
1231
1232     def c_unboxed_type(self):
1233         return c_name(self.name)
1234
1235     def json_type(self):
1236         return 'object'
1237
1238     def visit(self, visitor):
1239         visitor.visit_object_type(self.name, self.info,
1240                                   self.base, self.local_members, self.variants)
1241         visitor.visit_object_type_flat(self.name, self.info,
1242                                        self.members, self.variants)
1243
1244
1245 class QAPISchemaMember(object):
1246     role = 'member'
1247
1248     def __init__(self, name):
1249         assert isinstance(name, str)
1250         self.name = name
1251         self.owner = None
1252
1253     def set_owner(self, name):
1254         assert not self.owner
1255         self.owner = name
1256
1257     def check_clash(self, info, seen):
1258         cname = c_name(self.name)
1259         if cname.lower() != cname and self.owner not in name_case_whitelist:
1260             raise QAPISemError(info,
1261                                "%s should not use uppercase" % self.describe())
1262         if cname in seen:
1263             raise QAPISemError(info, "%s collides with %s" %
1264                                (self.describe(), seen[cname].describe()))
1265         seen[cname] = self
1266
1267     def _pretty_owner(self):
1268         owner = self.owner
1269         if owner.startswith('q_obj_'):
1270             # See QAPISchema._make_implicit_object_type() - reverse the
1271             # mapping there to create a nice human-readable description
1272             owner = owner[6:]
1273             if owner.endswith('-arg'):
1274                 return '(parameter of %s)' % owner[:-4]
1275             elif owner.endswith('-base'):
1276                 return '(base of %s)' % owner[:-5]
1277             else:
1278                 assert owner.endswith('-wrapper')
1279                 # Unreachable and not implemented
1280                 assert False
1281         if owner.endswith('Kind'):
1282             # See QAPISchema._make_implicit_enum_type()
1283             return '(branch of %s)' % owner[:-4]
1284         return '(%s of %s)' % (self.role, owner)
1285
1286     def describe(self):
1287         return "'%s' %s" % (self.name, self._pretty_owner())
1288
1289
1290 class QAPISchemaObjectTypeMember(QAPISchemaMember):
1291     def __init__(self, name, typ, optional):
1292         QAPISchemaMember.__init__(self, name)
1293         assert isinstance(typ, str)
1294         assert isinstance(optional, bool)
1295         self._type_name = typ
1296         self.type = None
1297         self.optional = optional
1298
1299     def check(self, schema):
1300         assert self.owner
1301         self.type = schema.lookup_type(self._type_name)
1302         assert self.type
1303
1304
1305 class QAPISchemaObjectTypeVariants(object):
1306     def __init__(self, tag_name, tag_member, variants):
1307         # Flat unions pass tag_name but not tag_member.
1308         # Simple unions and alternates pass tag_member but not tag_name.
1309         # After check(), tag_member is always set, and tag_name remains
1310         # a reliable witness of being used by a flat union.
1311         assert bool(tag_member) != bool(tag_name)
1312         assert (isinstance(tag_name, str) or
1313                 isinstance(tag_member, QAPISchemaObjectTypeMember))
1314         assert len(variants) > 0
1315         for v in variants:
1316             assert isinstance(v, QAPISchemaObjectTypeVariant)
1317         self._tag_name = tag_name
1318         self.tag_member = tag_member
1319         self.variants = variants
1320
1321     def set_owner(self, name):
1322         for v in self.variants:
1323             v.set_owner(name)
1324
1325     def check(self, schema, seen):
1326         if not self.tag_member:    # flat union
1327             self.tag_member = seen[c_name(self._tag_name)]
1328             assert self._tag_name == self.tag_member.name
1329         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
1330         for v in self.variants:
1331             v.check(schema)
1332             # Union names must match enum values; alternate names are
1333             # checked separately. Use 'seen' to tell the two apart.
1334             if seen:
1335                 assert v.name in self.tag_member.type.member_names()
1336                 assert isinstance(v.type, QAPISchemaObjectType)
1337                 v.type.check(schema)
1338
1339     def check_clash(self, info, seen):
1340         for v in self.variants:
1341             # Reset seen map for each variant, since qapi names from one
1342             # branch do not affect another branch
1343             assert isinstance(v.type, QAPISchemaObjectType)
1344             v.type.check_clash(info, dict(seen))
1345
1346
1347 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1348     role = 'branch'
1349
1350     def __init__(self, name, typ):
1351         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1352
1353
1354 class QAPISchemaAlternateType(QAPISchemaType):
1355     def __init__(self, name, info, doc, variants):
1356         QAPISchemaType.__init__(self, name, info, doc)
1357         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1358         assert variants.tag_member
1359         variants.set_owner(name)
1360         variants.tag_member.set_owner(self.name)
1361         self.variants = variants
1362
1363     def check(self, schema):
1364         self.variants.tag_member.check(schema)
1365         # Not calling self.variants.check_clash(), because there's nothing
1366         # to clash with
1367         self.variants.check(schema, {})
1368         # Alternate branch names have no relation to the tag enum values;
1369         # so we have to check for potential name collisions ourselves.
1370         seen = {}
1371         for v in self.variants.variants:
1372             v.check_clash(self.info, seen)
1373             if self.doc:
1374                 self.doc.connect_member(v)
1375         if self.doc:
1376             self.doc.check()
1377
1378     def c_type(self):
1379         return c_name(self.name) + pointer_suffix
1380
1381     def json_type(self):
1382         return 'value'
1383
1384     def visit(self, visitor):
1385         visitor.visit_alternate_type(self.name, self.info, self.variants)
1386
1387     def is_empty(self):
1388         return False
1389
1390
1391 class QAPISchemaCommand(QAPISchemaEntity):
1392     def __init__(self, name, info, doc, arg_type, ret_type,
1393                  gen, success_response, boxed):
1394         QAPISchemaEntity.__init__(self, name, info, doc)
1395         assert not arg_type or isinstance(arg_type, str)
1396         assert not ret_type or isinstance(ret_type, str)
1397         self._arg_type_name = arg_type
1398         self.arg_type = None
1399         self._ret_type_name = ret_type
1400         self.ret_type = None
1401         self.gen = gen
1402         self.success_response = success_response
1403         self.boxed = boxed
1404
1405     def check(self, schema):
1406         if self._arg_type_name:
1407             self.arg_type = schema.lookup_type(self._arg_type_name)
1408             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1409                     isinstance(self.arg_type, QAPISchemaAlternateType))
1410             self.arg_type.check(schema)
1411             if self.boxed:
1412                 if self.arg_type.is_empty():
1413                     raise QAPISemError(self.info,
1414                                        "Cannot use 'boxed' with empty type")
1415             else:
1416                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1417                 assert not self.arg_type.variants
1418         elif self.boxed:
1419             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1420         if self._ret_type_name:
1421             self.ret_type = schema.lookup_type(self._ret_type_name)
1422             assert isinstance(self.ret_type, QAPISchemaType)
1423
1424     def visit(self, visitor):
1425         visitor.visit_command(self.name, self.info,
1426                               self.arg_type, self.ret_type,
1427                               self.gen, self.success_response, self.boxed)
1428
1429
1430 class QAPISchemaEvent(QAPISchemaEntity):
1431     def __init__(self, name, info, doc, arg_type, boxed):
1432         QAPISchemaEntity.__init__(self, name, info, doc)
1433         assert not arg_type or isinstance(arg_type, str)
1434         self._arg_type_name = arg_type
1435         self.arg_type = None
1436         self.boxed = boxed
1437
1438     def check(self, schema):
1439         if self._arg_type_name:
1440             self.arg_type = schema.lookup_type(self._arg_type_name)
1441             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1442                     isinstance(self.arg_type, QAPISchemaAlternateType))
1443             self.arg_type.check(schema)
1444             if self.boxed:
1445                 if self.arg_type.is_empty():
1446                     raise QAPISemError(self.info,
1447                                        "Cannot use 'boxed' with empty type")
1448             else:
1449                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1450                 assert not self.arg_type.variants
1451         elif self.boxed:
1452             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1453
1454     def visit(self, visitor):
1455         visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
1456
1457
1458 class QAPISchema(object):
1459     def __init__(self, fname):
1460         try:
1461             parser = QAPISchemaParser(open(fname, 'r'))
1462             self.exprs = check_exprs(parser.exprs)
1463             self.docs = parser.docs
1464             self._entity_dict = {}
1465             self._predefining = True
1466             self._def_predefineds()
1467             self._predefining = False
1468             self._def_exprs()
1469             self.check()
1470         except QAPIError as err:
1471             print(err, file=sys.stderr)
1472             exit(1)
1473
1474     def _def_entity(self, ent):
1475         # Only the predefined types are allowed to not have info
1476         assert ent.info or self._predefining
1477         assert ent.name not in self._entity_dict
1478         self._entity_dict[ent.name] = ent
1479
1480     def lookup_entity(self, name, typ=None):
1481         ent = self._entity_dict.get(name)
1482         if typ and not isinstance(ent, typ):
1483             return None
1484         return ent
1485
1486     def lookup_type(self, name):
1487         return self.lookup_entity(name, QAPISchemaType)
1488
1489     def _def_builtin_type(self, name, json_type, c_type):
1490         self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
1491         # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
1492         # qapi-types.h from a single .c, all arrays of builtins must be
1493         # declared in the first file whether or not they are used.  Nicer
1494         # would be to use lazy instantiation, while figuring out how to
1495         # avoid compilation issues with multiple qapi-types.h.
1496         self._make_array_type(name, None)
1497
1498     def _def_predefineds(self):
1499         for t in [('str',    'string',  'char' + pointer_suffix),
1500                   ('number', 'number',  'double'),
1501                   ('int',    'int',     'int64_t'),
1502                   ('int8',   'int',     'int8_t'),
1503                   ('int16',  'int',     'int16_t'),
1504                   ('int32',  'int',     'int32_t'),
1505                   ('int64',  'int',     'int64_t'),
1506                   ('uint8',  'int',     'uint8_t'),
1507                   ('uint16', 'int',     'uint16_t'),
1508                   ('uint32', 'int',     'uint32_t'),
1509                   ('uint64', 'int',     'uint64_t'),
1510                   ('size',   'int',     'uint64_t'),
1511                   ('bool',   'boolean', 'bool'),
1512                   ('any',    'value',   'QObject' + pointer_suffix),
1513                   ('null',   'null',    'QNull' + pointer_suffix)]:
1514             self._def_builtin_type(*t)
1515         self.the_empty_object_type = QAPISchemaObjectType(
1516             'q_empty', None, None, None, [], None)
1517         self._def_entity(self.the_empty_object_type)
1518         qtype_values = self._make_enum_members(['none', 'qnull', 'qnum',
1519                                                 'qstring', 'qdict', 'qlist',
1520                                                 'qbool'])
1521         self._def_entity(QAPISchemaEnumType('QType', None, None,
1522                                             qtype_values, 'QTYPE'))
1523
1524     def _make_enum_members(self, values):
1525         return [QAPISchemaMember(v) for v in values]
1526
1527     def _make_implicit_enum_type(self, name, info, values):
1528         # See also QAPISchemaObjectTypeMember._pretty_owner()
1529         name = name + 'Kind'   # Use namespace reserved by add_name()
1530         self._def_entity(QAPISchemaEnumType(
1531             name, info, None, self._make_enum_members(values), None))
1532         return name
1533
1534     def _make_array_type(self, element_type, info):
1535         name = element_type + 'List'   # Use namespace reserved by add_name()
1536         if not self.lookup_type(name):
1537             self._def_entity(QAPISchemaArrayType(name, info, element_type))
1538         return name
1539
1540     def _make_implicit_object_type(self, name, info, doc, role, members):
1541         if not members:
1542             return None
1543         # See also QAPISchemaObjectTypeMember._pretty_owner()
1544         name = 'q_obj_%s-%s' % (name, role)
1545         if not self.lookup_entity(name, QAPISchemaObjectType):
1546             self._def_entity(QAPISchemaObjectType(name, info, doc, None,
1547                                                   members, None))
1548         return name
1549
1550     def _def_enum_type(self, expr, info, doc):
1551         name = expr['enum']
1552         data = expr['data']
1553         prefix = expr.get('prefix')
1554         self._def_entity(QAPISchemaEnumType(
1555             name, info, doc, self._make_enum_members(data), prefix))
1556
1557     def _make_member(self, name, typ, info):
1558         optional = False
1559         if name.startswith('*'):
1560             name = name[1:]
1561             optional = True
1562         if isinstance(typ, list):
1563             assert len(typ) == 1
1564             typ = self._make_array_type(typ[0], info)
1565         return QAPISchemaObjectTypeMember(name, typ, optional)
1566
1567     def _make_members(self, data, info):
1568         return [self._make_member(key, value, info)
1569                 for (key, value) in data.iteritems()]
1570
1571     def _def_struct_type(self, expr, info, doc):
1572         name = expr['struct']
1573         base = expr.get('base')
1574         data = expr['data']
1575         self._def_entity(QAPISchemaObjectType(name, info, doc, base,
1576                                               self._make_members(data, info),
1577                                               None))
1578
1579     def _make_variant(self, case, typ):
1580         return QAPISchemaObjectTypeVariant(case, typ)
1581
1582     def _make_simple_variant(self, case, typ, info):
1583         if isinstance(typ, list):
1584             assert len(typ) == 1
1585             typ = self._make_array_type(typ[0], info)
1586         typ = self._make_implicit_object_type(
1587             typ, info, None, 'wrapper', [self._make_member('data', typ, info)])
1588         return QAPISchemaObjectTypeVariant(case, typ)
1589
1590     def _def_union_type(self, expr, info, doc):
1591         name = expr['union']
1592         data = expr['data']
1593         base = expr.get('base')
1594         tag_name = expr.get('discriminator')
1595         tag_member = None
1596         if isinstance(base, dict):
1597             base = (self._make_implicit_object_type(
1598                 name, info, doc, 'base', self._make_members(base, info)))
1599         if tag_name:
1600             variants = [self._make_variant(key, value)
1601                         for (key, value) in data.iteritems()]
1602             members = []
1603         else:
1604             variants = [self._make_simple_variant(key, value, info)
1605                         for (key, value) in data.iteritems()]
1606             typ = self._make_implicit_enum_type(name, info,
1607                                                 [v.name for v in variants])
1608             tag_member = QAPISchemaObjectTypeMember('type', typ, False)
1609             members = [tag_member]
1610         self._def_entity(
1611             QAPISchemaObjectType(name, info, doc, base, members,
1612                                  QAPISchemaObjectTypeVariants(tag_name,
1613                                                               tag_member,
1614                                                               variants)))
1615
1616     def _def_alternate_type(self, expr, info, doc):
1617         name = expr['alternate']
1618         data = expr['data']
1619         variants = [self._make_variant(key, value)
1620                     for (key, value) in data.iteritems()]
1621         tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
1622         self._def_entity(
1623             QAPISchemaAlternateType(name, info, doc,
1624                                     QAPISchemaObjectTypeVariants(None,
1625                                                                  tag_member,
1626                                                                  variants)))
1627
1628     def _def_command(self, expr, info, doc):
1629         name = expr['command']
1630         data = expr.get('data')
1631         rets = expr.get('returns')
1632         gen = expr.get('gen', True)
1633         success_response = expr.get('success-response', True)
1634         boxed = expr.get('boxed', False)
1635         if isinstance(data, OrderedDict):
1636             data = self._make_implicit_object_type(
1637                 name, info, doc, 'arg', self._make_members(data, info))
1638         if isinstance(rets, list):
1639             assert len(rets) == 1
1640             rets = self._make_array_type(rets[0], info)
1641         self._def_entity(QAPISchemaCommand(name, info, doc, data, rets,
1642                                            gen, success_response, boxed))
1643
1644     def _def_event(self, expr, info, doc):
1645         name = expr['event']
1646         data = expr.get('data')
1647         boxed = expr.get('boxed', False)
1648         if isinstance(data, OrderedDict):
1649             data = self._make_implicit_object_type(
1650                 name, info, doc, 'arg', self._make_members(data, info))
1651         self._def_entity(QAPISchemaEvent(name, info, doc, data, boxed))
1652
1653     def _def_exprs(self):
1654         for expr_elem in self.exprs:
1655             expr = expr_elem['expr']
1656             info = expr_elem['info']
1657             doc = expr_elem.get('doc')
1658             if 'enum' in expr:
1659                 self._def_enum_type(expr, info, doc)
1660             elif 'struct' in expr:
1661                 self._def_struct_type(expr, info, doc)
1662             elif 'union' in expr:
1663                 self._def_union_type(expr, info, doc)
1664             elif 'alternate' in expr:
1665                 self._def_alternate_type(expr, info, doc)
1666             elif 'command' in expr:
1667                 self._def_command(expr, info, doc)
1668             elif 'event' in expr:
1669                 self._def_event(expr, info, doc)
1670             else:
1671                 assert False
1672
1673     def check(self):
1674         for ent in self._entity_dict.values():
1675             ent.check(self)
1676
1677     def visit(self, visitor):
1678         visitor.visit_begin(self)
1679         for (name, entity) in sorted(self._entity_dict.items()):
1680             if visitor.visit_needed(entity):
1681                 entity.visit(visitor)
1682         visitor.visit_end()
1683
1684
1685 #
1686 # Code generation helpers
1687 #
1688
1689 def camel_case(name):
1690     new_name = ''
1691     first = True
1692     for ch in name:
1693         if ch in ['_', '-']:
1694             first = True
1695         elif first:
1696             new_name += ch.upper()
1697             first = False
1698         else:
1699             new_name += ch.lower()
1700     return new_name
1701
1702
1703 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1704 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1705 # ENUM24_Name -> ENUM24_NAME
1706 def camel_to_upper(value):
1707     c_fun_str = c_name(value, False)
1708     if value.isupper():
1709         return c_fun_str
1710
1711     new_name = ''
1712     l = len(c_fun_str)
1713     for i in range(l):
1714         c = c_fun_str[i]
1715         # When c is upper and no '_' appears before, do more checks
1716         if c.isupper() and (i > 0) and c_fun_str[i - 1] != '_':
1717             if i < l - 1 and c_fun_str[i + 1].islower():
1718                 new_name += '_'
1719             elif c_fun_str[i - 1].isdigit():
1720                 new_name += '_'
1721         new_name += c
1722     return new_name.lstrip('_').upper()
1723
1724
1725 def c_enum_const(type_name, const_name, prefix=None):
1726     if prefix is not None:
1727         type_name = prefix
1728     return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
1729
1730 c_name_trans = string.maketrans('.-', '__')
1731
1732
1733 # Map @name to a valid C identifier.
1734 # If @protect, avoid returning certain ticklish identifiers (like
1735 # C keywords) by prepending 'q_'.
1736 #
1737 # Used for converting 'name' from a 'name':'type' qapi definition
1738 # into a generated struct member, as well as converting type names
1739 # into substrings of a generated C function name.
1740 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1741 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1742 def c_name(name, protect=True):
1743     # ANSI X3J11/88-090, 3.1.1
1744     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1745                      'default', 'do', 'double', 'else', 'enum', 'extern',
1746                      'float', 'for', 'goto', 'if', 'int', 'long', 'register',
1747                      'return', 'short', 'signed', 'sizeof', 'static',
1748                      'struct', 'switch', 'typedef', 'union', 'unsigned',
1749                      'void', 'volatile', 'while'])
1750     # ISO/IEC 9899:1999, 6.4.1
1751     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1752     # ISO/IEC 9899:2011, 6.4.1
1753     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
1754                      '_Noreturn', '_Static_assert', '_Thread_local'])
1755     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1756     # excluding _.*
1757     gcc_words = set(['asm', 'typeof'])
1758     # C++ ISO/IEC 14882:2003 2.11
1759     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1760                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1761                      'namespace', 'new', 'operator', 'private', 'protected',
1762                      'public', 'reinterpret_cast', 'static_cast', 'template',
1763                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1764                      'using', 'virtual', 'wchar_t',
1765                      # alternative representations
1766                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1767                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1768     # namespace pollution:
1769     polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
1770     name = name.translate(c_name_trans)
1771     if protect and (name in c89_words | c99_words | c11_words | gcc_words
1772                     | cpp_words | polluted_words):
1773         return 'q_' + name
1774     return name
1775
1776 eatspace = '\033EATSPACE.'
1777 pointer_suffix = ' *' + eatspace
1778
1779
1780 def genindent(count):
1781     ret = ''
1782     for _ in range(count):
1783         ret += ' '
1784     return ret
1785
1786 indent_level = 0
1787
1788
1789 def push_indent(indent_amount=4):
1790     global indent_level
1791     indent_level += indent_amount
1792
1793
1794 def pop_indent(indent_amount=4):
1795     global indent_level
1796     indent_level -= indent_amount
1797
1798
1799 # Generate @code with @kwds interpolated.
1800 # Obey indent_level, and strip eatspace.
1801 def cgen(code, **kwds):
1802     raw = code % kwds
1803     if indent_level:
1804         indent = genindent(indent_level)
1805         # re.subn() lacks flags support before Python 2.7, use re.compile()
1806         raw = re.subn(re.compile(r'^.', re.MULTILINE),
1807                       indent + r'\g<0>', raw)
1808         raw = raw[0]
1809     return re.sub(re.escape(eatspace) + r' *', '', raw)
1810
1811
1812 def mcgen(code, **kwds):
1813     if code[0] == '\n':
1814         code = code[1:]
1815     return cgen(code, **kwds)
1816
1817
1818 def guardname(filename):
1819     return c_name(filename, protect=False).upper()
1820
1821
1822 def guardstart(name):
1823     return mcgen('''
1824
1825 #ifndef %(name)s
1826 #define %(name)s
1827
1828 ''',
1829                  name=guardname(name))
1830
1831
1832 def guardend(name):
1833     return mcgen('''
1834
1835 #endif /* %(name)s */
1836
1837 ''',
1838                  name=guardname(name))
1839
1840
1841 def gen_enum_lookup(name, values, prefix=None):
1842     ret = mcgen('''
1843
1844 const QEnumLookup %(c_name)s_lookup = {
1845     .array = (const char *const[]) {
1846 ''',
1847                 c_name=c_name(name))
1848     for value in values:
1849         index = c_enum_const(name, value, prefix)
1850         ret += mcgen('''
1851         [%(index)s] = "%(value)s",
1852 ''',
1853                      index=index, value=value)
1854
1855     ret += mcgen('''
1856     },
1857     .size = %(max_index)s
1858 };
1859 ''',
1860                  max_index=c_enum_const(name, '_MAX', prefix))
1861     return ret
1862
1863
1864 def gen_enum(name, values, prefix=None):
1865     # append automatically generated _MAX value
1866     enum_values = values + ['_MAX']
1867
1868     ret = mcgen('''
1869
1870 typedef enum %(c_name)s {
1871 ''',
1872                 c_name=c_name(name))
1873
1874     i = 0
1875     for value in enum_values:
1876         ret += mcgen('''
1877     %(c_enum)s = %(i)d,
1878 ''',
1879                      c_enum=c_enum_const(name, value, prefix),
1880                      i=i)
1881         i += 1
1882
1883     ret += mcgen('''
1884 } %(c_name)s;
1885 ''',
1886                  c_name=c_name(name))
1887
1888     ret += mcgen('''
1889
1890 #define %(c_name)s_str(val) \\
1891     qapi_enum_lookup(&%(c_name)s_lookup, (val))
1892
1893 extern const QEnumLookup %(c_name)s_lookup;
1894 ''',
1895                  c_name=c_name(name))
1896     return ret
1897
1898
1899 def build_params(arg_type, boxed, extra):
1900     if not arg_type:
1901         assert not boxed
1902         return extra
1903     ret = ''
1904     sep = ''
1905     if boxed:
1906         ret += '%s arg' % arg_type.c_param_type()
1907         sep = ', '
1908     else:
1909         assert not arg_type.variants
1910         for memb in arg_type.members:
1911             ret += sep
1912             sep = ', '
1913             if memb.optional:
1914                 ret += 'bool has_%s, ' % c_name(memb.name)
1915             ret += '%s %s' % (memb.type.c_param_type(),
1916                               c_name(memb.name))
1917     if extra:
1918         ret += sep + extra
1919     return ret
1920
1921
1922 #
1923 # Common command line parsing
1924 #
1925
1926
1927 def parse_command_line(extra_options='', extra_long_options=[]):
1928
1929     try:
1930         opts, args = getopt.gnu_getopt(sys.argv[1:],
1931                                        'chp:o:' + extra_options,
1932                                        ['source', 'header', 'prefix=',
1933                                         'output-dir='] + extra_long_options)
1934     except getopt.GetoptError as err:
1935         print("%s: %s" % (sys.argv[0], str(err)), file=sys.stderr)
1936         sys.exit(1)
1937
1938     output_dir = ''
1939     prefix = ''
1940     do_c = False
1941     do_h = False
1942     extra_opts = []
1943
1944     for oa in opts:
1945         o, a = oa
1946         if o in ('-p', '--prefix'):
1947             match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1948             if match.end() != len(a):
1949                 print("%s: 'funny character '%s' in argument of --prefix" \
1950                       % (sys.argv[0], a[match.end()]), file=sys.stderr)
1951                 sys.exit(1)
1952             prefix = a
1953         elif o in ('-o', '--output-dir'):
1954             output_dir = a + '/'
1955         elif o in ('-c', '--source'):
1956             do_c = True
1957         elif o in ('-h', '--header'):
1958             do_h = True
1959         else:
1960             extra_opts.append(oa)
1961
1962     if not do_c and not do_h:
1963         do_c = True
1964         do_h = True
1965
1966     if len(args) != 1:
1967         print("%s: need exactly one argument" % sys.argv[0], file=sys.stderr)
1968         sys.exit(1)
1969     fname = args[0]
1970
1971     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
1972
1973 #
1974 # Generate output files with boilerplate
1975 #
1976
1977
1978 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
1979                 c_comment, h_comment):
1980     guard = guardname(prefix + h_file)
1981     c_file = output_dir + prefix + c_file
1982     h_file = output_dir + prefix + h_file
1983
1984     if output_dir:
1985         try:
1986             os.makedirs(output_dir)
1987         except os.error as e:
1988             if e.errno != errno.EEXIST:
1989                 raise
1990
1991     def maybe_open(really, name, opt):
1992         if really:
1993             return open(name, opt)
1994         else:
1995             import StringIO
1996             return StringIO.StringIO()
1997
1998     fdef = maybe_open(do_c, c_file, 'w')
1999     fdecl = maybe_open(do_h, h_file, 'w')
2000
2001     fdef.write(mcgen('''
2002 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2003 %(comment)s
2004 ''',
2005                      comment=c_comment))
2006
2007     fdecl.write(mcgen('''
2008 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2009 %(comment)s
2010 #ifndef %(guard)s
2011 #define %(guard)s
2012
2013 ''',
2014                       comment=h_comment, guard=guard))
2015
2016     return (fdef, fdecl)
2017
2018
2019 def close_output(fdef, fdecl):
2020     fdecl.write('''
2021 #endif
2022 ''')
2023     fdecl.close()
2024     fdef.close()