scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 import errno
  15 import getopt
  16 import os
  17 import re
  18 import string
  19 import sys
  20 from ordereddict import OrderedDict
  21
  22 builtin_types = {
  23     'str':      'QTYPE_QSTRING',
  24     'int':      'QTYPE_QINT',
  25     'number':   'QTYPE_QFLOAT',
  26     'bool':     'QTYPE_QBOOL',
  27     'int8':     'QTYPE_QINT',
  28     'int16':    'QTYPE_QINT',
  29     'int32':    'QTYPE_QINT',
  30     'int64':    'QTYPE_QINT',
  31     'uint8':    'QTYPE_QINT',
  32     'uint16':   'QTYPE_QINT',
  33     'uint32':   'QTYPE_QINT',
  34     'uint64':   'QTYPE_QINT',
  35     'size':     'QTYPE_QINT',
  36     'any':      None,           # any QType possible, actually
  37     'QType':    'QTYPE_QSTRING',
  38 }
  39
  40 # Are documentation comments required?
  41 doc_required = False
  42
  43 # Whitelist of commands allowed to return a non-dictionary
  44 returns_whitelist = []
  45
  46 # Whitelist of entities allowed to violate case conventions
  47 name_case_whitelist = []
  48
  49 enum_types = {}
  50 struct_types = {}
  51 union_types = {}
  52 all_names = {}
  53
  54 #
  55 # Parsing the schema into expressions
  56 #
  57
  58
  59 def error_path(parent):
  60     res = ''
  61     while parent:
  62         res = ('In file included from %s:%d:\n' % (parent['file'],
  63                                                    parent['line'])) + res
  64         parent = parent['parent']
  65     return res
  66
  67
  68 class QAPIError(Exception):
  69     def __init__(self, fname, line, col, incl_info, msg):
  70         Exception.__init__(self)
  71         self.fname = fname
  72         self.line = line
  73         self.col = col
  74         self.info = incl_info
  75         self.msg = msg
  76
  77     def __str__(self):
  78         loc = '%s:%d' % (self.fname, self.line)
  79         if self.col is not None:
  80             loc += ':%s' % self.col
  81         return error_path(self.info) + '%s: %s' % (loc, self.msg)
  82
  83
  84 class QAPIParseError(QAPIError):
  85     def __init__(self, parser, msg):
  86         col = 1
  87         for ch in parser.src[parser.line_pos:parser.pos]:
  88             if ch == '\t':
  89                 col = (col + 7) % 8 + 1
  90             else:
  91                 col += 1
  92         QAPIError.__init__(self, parser.fname, parser.line, col,
  93                            parser.incl_info, msg)
  94
  95
  96 class QAPISemError(QAPIError):
  97     def __init__(self, info, msg):
  98         QAPIError.__init__(self, info['file'], info['line'], None,
  99                            info['parent'], msg)
 100
 101
 102 class QAPIDoc(object):
 103     class Section(object):
 104         def __init__(self, name=None):
 105             # optional section name (argument/member or section name)
 106             self.name = name
 107             # the list of lines for this section
 108             self.content = []
 109
 110         def append(self, line):
 111             self.content.append(line)
 112
 113         def __repr__(self):
 114             return '\n'.join(self.content).strip()
 115
 116     class ArgSection(Section):
 117         def __init__(self, name):
 118             QAPIDoc.Section.__init__(self, name)
 119             self.member = None
 120
 121         def connect(self, member):
 122             self.member = member
 123
 124     def __init__(self, parser, info):
 125         # self.parser is used to report errors with QAPIParseError.  The
 126         # resulting error position depends on the state of the parser.
 127         # It happens to be the beginning of the comment.  More or less
 128         # servicable, but action at a distance.
 129         self.parser = parser
 130         self.info = info
 131         self.symbol = None
 132         self.body = QAPIDoc.Section()
 133         # dict mapping parameter name to ArgSection
 134         self.args = OrderedDict()
 135         # a list of Section
 136         self.sections = []
 137         # the current section
 138         self.section = self.body
 139
 140     def has_section(self, name):
 141         """Return True if we have a section with this name."""
 142         for i in self.sections:
 143             if i.name == name:
 144                 return True
 145         return False
 146
 147     def append(self, line):
 148         """Parse a comment line and add it to the documentation."""
 149         line = line[1:]
 150         if not line:
 151             self._append_freeform(line)
 152             return
 153
 154         if line[0] != ' ':
 155             raise QAPIParseError(self.parser, "Missing space after #")
 156         line = line[1:]
 157
 158         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 159         # recognized, and get silently treated as ordinary text
 160         if self.symbol:
 161             self._append_symbol_line(line)
 162         elif not self.body.content and line.startswith('@'):
 163             if not line.endswith(':'):
 164                 raise QAPIParseError(self.parser, "Line should end with :")
 165             self.symbol = line[1:-1]
 166             # FIXME invalid names other than the empty string aren't flagged
 167             if not self.symbol:
 168                 raise QAPIParseError(self.parser, "Invalid name")
 169         else:
 170             self._append_freeform(line)
 171
 172     def end_comment(self):
 173         self._end_section()
 174
 175     def _append_symbol_line(self, line):
 176         name = line.split(' ', 1)[0]
 177
 178         if name.startswith('@') and name.endswith(':'):
 179             line = line[len(name)+1:]
 180             self._start_args_section(name[1:-1])
 181         elif name in ('Returns:', 'Since:',
 182                       # those are often singular or plural
 183                       'Note:', 'Notes:',
 184                       'Example:', 'Examples:',
 185                       'TODO:'):
 186             line = line[len(name)+1:]
 187             self._start_section(name[:-1])
 188
 189         self._append_freeform(line)
 190
 191     def _start_args_section(self, name):
 192         # FIXME invalid names other than the empty string aren't flagged
 193         if not name:
 194             raise QAPIParseError(self.parser, "Invalid parameter name")
 195         if name in self.args:
 196             raise QAPIParseError(self.parser,
 197                                  "'%s' parameter name duplicated" % name)
 198         if self.sections:
 199             raise QAPIParseError(self.parser,
 200                                  "'@%s:' can't follow '%s' section"
 201                                  % (name, self.sections[0].name))
 202         self._end_section()
 203         self.section = QAPIDoc.ArgSection(name)
 204         self.args[name] = self.section
 205
 206     def _start_section(self, name=''):
 207         if name in ('Returns', 'Since') and self.has_section(name):
 208             raise QAPIParseError(self.parser,
 209                                  "Duplicated '%s' section" % name)
 210         self._end_section()
 211         self.section = QAPIDoc.Section(name)
 212         self.sections.append(self.section)
 213
 214     def _end_section(self):
 215         if self.section:
 216             contents = str(self.section)
 217             if self.section.name and (not contents or contents.isspace()):
 218                 raise QAPIParseError(self.parser, "Empty doc section '%s'"
 219                                      % self.section.name)
 220             self.section = None
 221
 222     def _append_freeform(self, line):
 223         in_arg = isinstance(self.section, QAPIDoc.ArgSection)
 224         if (in_arg and self.section.content
 225                 and not self.section.content[-1]
 226                 and line and not line[0].isspace()):
 227             self._start_section()
 228         if (in_arg or not self.section.name
 229                 or not self.section.name.startswith('Example')):
 230             line = line.strip()
 231         match = re.match(r'(@\S+:)', line)
 232         if match:
 233             raise QAPIParseError(self.parser,
 234                                  "'%s' not allowed in free-form documentation"
 235                                  % match.group(1))
 236         # TODO Drop this once the dust has settled
 237         if (isinstance(self.section, QAPIDoc.ArgSection)
 238                 and '#optional' in line):
 239             raise QAPISemError(self.info, "Please drop the #optional tag")
 240         self.section.append(line)
 241
 242     def connect_member(self, member):
 243         if member.name not in self.args:
 244             # Undocumented TODO outlaw
 245             self.args[member.name] = QAPIDoc.ArgSection(member.name)
 246         self.args[member.name].connect(member)
 247
 248     def check_expr(self, expr):
 249         if self.has_section('Returns') and 'command' not in expr:
 250             raise QAPISemError(self.info,
 251                                "'Returns:' is only valid for commands")
 252
 253     def check(self):
 254         bogus = [name for name, section in self.args.iteritems()
 255                  if not section.member]
 256         if bogus:
 257             raise QAPISemError(
 258                 self.info,
 259                 "The following documented members are not in "
 260                 "the declaration: %s" % ", ".join(bogus))
 261
 262
 263 class QAPISchemaParser(object):
 264
 265     def __init__(self, fp, previously_included=[], incl_info=None):
 266         abs_fname = os.path.abspath(fp.name)
 267         fname = fp.name
 268         self.fname = fname
 269         previously_included.append(abs_fname)
 270         self.incl_info = incl_info
 271         self.src = fp.read()
 272         if self.src == '' or self.src[-1] != '\n':
 273             self.src += '\n'
 274         self.cursor = 0
 275         self.line = 1
 276         self.line_pos = 0
 277         self.exprs = []
 278         self.docs = []
 279         self.cur_doc = None
 280         self.accept()
 281
 282         while self.tok is not None:
 283             info = {'file': fname, 'line': self.line,
 284                     'parent': self.incl_info}
 285             if self.tok == '#':
 286                 self.reject_expr_doc()
 287                 self.cur_doc = self.get_doc(info)
 288                 self.docs.append(self.cur_doc)
 289                 continue
 290
 291             expr = self.get_expr(False)
 292             if 'include' in expr:
 293                 self.reject_expr_doc()
 294                 if len(expr) != 1:
 295                     raise QAPISemError(info, "Invalid 'include' directive")
 296                 include = expr['include']
 297                 if not isinstance(include, str):
 298                     raise QAPISemError(info,
 299                                        "Value of 'include' must be a string")
 300                 self._include(include, info, os.path.dirname(abs_fname),
 301                               previously_included)
 302             elif "pragma" in expr:
 303                 self.reject_expr_doc()
 304                 if len(expr) != 1:
 305                     raise QAPISemError(info, "Invalid 'pragma' directive")
 306                 pragma = expr['pragma']
 307                 if not isinstance(pragma, dict):
 308                     raise QAPISemError(
 309                         info, "Value of 'pragma' must be a dictionary")
 310                 for name, value in pragma.iteritems():
 311                     self._pragma(name, value, info)
 312             else:
 313                 expr_elem = {'expr': expr,
 314                              'info': info}
 315                 if self.cur_doc:
 316                     if not self.cur_doc.symbol:
 317                         raise QAPISemError(
 318                             self.cur_doc.info,
 319                             "Expression documentation required")
 320                     expr_elem['doc'] = self.cur_doc
 321                 self.exprs.append(expr_elem)
 322             self.cur_doc = None
 323         self.reject_expr_doc()
 324
 325     def reject_expr_doc(self):
 326         if self.cur_doc and self.cur_doc.symbol:
 327             raise QAPISemError(
 328                 self.cur_doc.info,
 329                 "Documentation for '%s' is not followed by the definition"
 330                 % self.cur_doc.symbol)
 331
 332     def _include(self, include, info, base_dir, previously_included):
 333         incl_abs_fname = os.path.join(base_dir, include)
 334         # catch inclusion cycle
 335         inf = info
 336         while inf:
 337             if incl_abs_fname == os.path.abspath(inf['file']):
 338                 raise QAPISemError(info, "Inclusion loop for %s" % include)
 339             inf = inf['parent']
 340
 341         # skip multiple include of the same file
 342         if incl_abs_fname in previously_included:
 343             return
 344         try:
 345             fobj = open(incl_abs_fname, 'r')
 346         except IOError as e:
 347             raise QAPISemError(info, '%s: %s' % (e.strerror, include))
 348         exprs_include = QAPISchemaParser(fobj, previously_included, info)
 349         self.exprs.extend(exprs_include.exprs)
 350         self.docs.extend(exprs_include.docs)
 351
 352     def _pragma(self, name, value, info):
 353         global doc_required, returns_whitelist, name_case_whitelist
 354         if name == 'doc-required':
 355             if not isinstance(value, bool):
 356                 raise QAPISemError(info,
 357                                    "Pragma 'doc-required' must be boolean")
 358             doc_required = value
 359         elif name == 'returns-whitelist':
 360             if (not isinstance(value, list)
 361                     or any([not isinstance(elt, str) for elt in value])):
 362                 raise QAPISemError(info,
 363                                    "Pragma returns-whitelist must be"
 364                                    " a list of strings")
 365             returns_whitelist = value
 366         elif name == 'name-case-whitelist':
 367             if (not isinstance(value, list)
 368                     or any([not isinstance(elt, str) for elt in value])):
 369                 raise QAPISemError(info,
 370                                    "Pragma name-case-whitelist must be"
 371                                    " a list of strings")
 372             name_case_whitelist = value
 373         else:
 374             raise QAPISemError(info, "Unknown pragma '%s'" % name)
 375
 376     def accept(self, skip_comment=True):
 377         while True:
 378             self.tok = self.src[self.cursor]
 379             self.pos = self.cursor
 380             self.cursor += 1
 381             self.val = None
 382
 383             if self.tok == '#':
 384                 if self.src[self.cursor] == '#':
 385                     # Start of doc comment
 386                     skip_comment = False
 387                 self.cursor = self.src.find('\n', self.cursor)
 388                 if not skip_comment:
 389                     self.val = self.src[self.pos:self.cursor]
 390                     return
 391             elif self.tok in '{}:,[]':
 392                 return
 393             elif self.tok == "'":
 394                 string = ''
 395                 esc = False
 396                 while True:
 397                     ch = self.src[self.cursor]
 398                     self.cursor += 1
 399                     if ch == '\n':
 400                         raise QAPIParseError(self, 'Missing terminating "\'"')
 401                     if esc:
 402                         if ch == 'b':
 403                             string += '\b'
 404                         elif ch == 'f':
 405                             string += '\f'
 406                         elif ch == 'n':
 407                             string += '\n'
 408                         elif ch == 'r':
 409                             string += '\r'
 410                         elif ch == 't':
 411                             string += '\t'
 412                         elif ch == 'u':
 413                             value = 0
 414                             for _ in range(0, 4):
 415                                 ch = self.src[self.cursor]
 416                                 self.cursor += 1
 417                                 if ch not in '0123456789abcdefABCDEF':
 418                                     raise QAPIParseError(self,
 419                                                          '\\u escape needs 4 '
 420                                                          'hex digits')
 421                                 value = (value << 4) + int(ch, 16)
 422                             # If Python 2 and 3 didn't disagree so much on
 423                             # how to handle Unicode, then we could allow
 424                             # Unicode string defaults.  But most of QAPI is
 425                             # ASCII-only, so we aren't losing much for now.
 426                             if not value or value > 0x7f:
 427                                 raise QAPIParseError(self,
 428                                                      'For now, \\u escape '
 429                                                      'only supports non-zero '
 430                                                      'values up to \\u007f')
 431                             string += chr(value)
 432                         elif ch in '\\/\'"':
 433                             string += ch
 434                         else:
 435                             raise QAPIParseError(self,
 436                                                  "Unknown escape \\%s" % ch)
 437                         esc = False
 438                     elif ch == '\\':
 439                         esc = True
 440                     elif ch == "'":
 441                         self.val = string
 442                         return
 443                     else:
 444                         string += ch
 445             elif self.src.startswith('true', self.pos):
 446                 self.val = True
 447                 self.cursor += 3
 448                 return
 449             elif self.src.startswith('false', self.pos):
 450                 self.val = False
 451                 self.cursor += 4
 452                 return
 453             elif self.src.startswith('null', self.pos):
 454                 self.val = None
 455                 self.cursor += 3
 456                 return
 457             elif self.tok == '\n':
 458                 if self.cursor == len(self.src):
 459                     self.tok = None
 460                     return
 461                 self.line += 1
 462                 self.line_pos = self.cursor
 463             elif not self.tok.isspace():
 464                 raise QAPIParseError(self, 'Stray "%s"' % self.tok)
 465
 466     def get_members(self):
 467         expr = OrderedDict()
 468         if self.tok == '}':
 469             self.accept()
 470             return expr
 471         if self.tok != "'":
 472             raise QAPIParseError(self, 'Expected string or "}"')
 473         while True:
 474             key = self.val
 475             self.accept()
 476             if self.tok != ':':
 477                 raise QAPIParseError(self, 'Expected ":"')
 478             self.accept()
 479             if key in expr:
 480                 raise QAPIParseError(self, 'Duplicate key "%s"' % key)
 481             expr[key] = self.get_expr(True)
 482             if self.tok == '}':
 483                 self.accept()
 484                 return expr
 485             if self.tok != ',':
 486                 raise QAPIParseError(self, 'Expected "," or "}"')
 487             self.accept()
 488             if self.tok != "'":
 489                 raise QAPIParseError(self, 'Expected string')
 490
 491     def get_values(self):
 492         expr = []
 493         if self.tok == ']':
 494             self.accept()
 495             return expr
 496         if self.tok not in "{['tfn":
 497             raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
 498                                  'boolean or "null"')
 499         while True:
 500             expr.append(self.get_expr(True))
 501             if self.tok == ']':
 502                 self.accept()
 503                 return expr
 504             if self.tok != ',':
 505                 raise QAPIParseError(self, 'Expected "," or "]"')
 506             self.accept()
 507
 508     def get_expr(self, nested):
 509         if self.tok != '{' and not nested:
 510             raise QAPIParseError(self, 'Expected "{"')
 511         if self.tok == '{':
 512             self.accept()
 513             expr = self.get_members()
 514         elif self.tok == '[':
 515             self.accept()
 516             expr = self.get_values()
 517         elif self.tok in "'tfn":
 518             expr = self.val
 519             self.accept()
 520         else:
 521             raise QAPIParseError(self, 'Expected "{", "[", string, '
 522                                  'boolean or "null"')
 523         return expr
 524
 525     def get_doc(self, info):
 526         if self.val != '##':
 527             raise QAPIParseError(self, "Junk after '##' at start of "
 528                                  "documentation comment")
 529
 530         doc = QAPIDoc(self, info)
 531         self.accept(False)
 532         while self.tok == '#':
 533             if self.val.startswith('##'):
 534                 # End of doc comment
 535                 if self.val != '##':
 536                     raise QAPIParseError(self, "Junk after '##' at end of "
 537                                          "documentation comment")
 538                 doc.end_comment()
 539                 self.accept()
 540                 return doc
 541             else:
 542                 doc.append(self.val)
 543             self.accept(False)
 544
 545         raise QAPIParseError(self, "Documentation comment must end with '##'")
 546
 547
 548 #
 549 # Semantic analysis of schema expressions
 550 # TODO fold into QAPISchema
 551 # TODO catching name collisions in generated code would be nice
 552 #
 553
 554
 555 def find_base_members(base):
 556     if isinstance(base, dict):
 557         return base
 558     base_struct_define = struct_types.get(base)
 559     if not base_struct_define:
 560         return None
 561     return base_struct_define['data']
 562
 563
 564 # Return the qtype of an alternate branch, or None on error.
 565 def find_alternate_member_qtype(qapi_type):
 566     if qapi_type in builtin_types:
 567         return builtin_types[qapi_type]
 568     elif qapi_type in struct_types:
 569         return 'QTYPE_QDICT'
 570     elif qapi_type in enum_types:
 571         return 'QTYPE_QSTRING'
 572     elif qapi_type in union_types:
 573         return 'QTYPE_QDICT'
 574     return None
 575
 576
 577 # Return the discriminator enum define if discriminator is specified as an
 578 # enum type, otherwise return None.
 579 def discriminator_find_enum_define(expr):
 580     base = expr.get('base')
 581     discriminator = expr.get('discriminator')
 582
 583     if not (discriminator and base):
 584         return None
 585
 586     base_members = find_base_members(base)
 587     if not base_members:
 588         return None
 589
 590     discriminator_type = base_members.get(discriminator)
 591     if not discriminator_type:
 592         return None
 593
 594     return enum_types.get(discriminator_type)
 595
 596
 597 # Names must be letters, numbers, -, and _.  They must start with letter,
 598 # except for downstream extensions which must start with __RFQDN_.
 599 # Dots are only valid in the downstream extension prefix.
 600 valid_name = re.compile(r'^(__[a-zA-Z0-9.-]+_)?'
 601                         '[a-zA-Z][a-zA-Z0-9_-]*$')
 602
 603
 604 def check_name(info, source, name, allow_optional=False,
 605                enum_member=False):
 606     global valid_name
 607     membername = name
 608
 609     if not isinstance(name, str):
 610         raise QAPISemError(info, "%s requires a string name" % source)
 611     if name.startswith('*'):
 612         membername = name[1:]
 613         if not allow_optional:
 614             raise QAPISemError(info, "%s does not allow optional name '%s'"
 615                                % (source, name))
 616     # Enum members can start with a digit, because the generated C
 617     # code always prefixes it with the enum name
 618     if enum_member and membername[0].isdigit():
 619         membername = 'D' + membername
 620     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
 621     # and 'q_obj_*' implicit type names.
 622     if not valid_name.match(membername) or \
 623        c_name(membername, False).startswith('q_'):
 624         raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
 625
 626
 627 def add_name(name, info, meta, implicit=False):
 628     global all_names
 629     check_name(info, "'%s'" % meta, name)
 630     # FIXME should reject names that differ only in '_' vs. '.'
 631     # vs. '-', because they're liable to clash in generated C.
 632     if name in all_names:
 633         raise QAPISemError(info, "%s '%s' is already defined"
 634                            % (all_names[name], name))
 635     if not implicit and (name.endswith('Kind') or name.endswith('List')):
 636         raise QAPISemError(info, "%s '%s' should not end in '%s'"
 637                            % (meta, name, name[-4:]))
 638     all_names[name] = meta
 639
 640
 641 def check_type(info, source, value, allow_array=False,
 642                allow_dict=False, allow_optional=False,
 643                allow_metas=[]):
 644     global all_names
 645
 646     if value is None:
 647         return
 648
 649     # Check if array type for value is okay
 650     if isinstance(value, list):
 651         if not allow_array:
 652             raise QAPISemError(info, "%s cannot be an array" % source)
 653         if len(value) != 1 or not isinstance(value[0], str):
 654             raise QAPISemError(info,
 655                                "%s: array type must contain single type name" %
 656                                source)
 657         value = value[0]
 658
 659     # Check if type name for value is okay
 660     if isinstance(value, str):
 661         if value not in all_names:
 662             raise QAPISemError(info, "%s uses unknown type '%s'"
 663                                % (source, value))
 664         if not all_names[value] in allow_metas:
 665             raise QAPISemError(info, "%s cannot use %s type '%s'" %
 666                                (source, all_names[value], value))
 667         return
 668
 669     if not allow_dict:
 670         raise QAPISemError(info, "%s should be a type name" % source)
 671
 672     if not isinstance(value, OrderedDict):
 673         raise QAPISemError(info,
 674                            "%s should be a dictionary or type name" % source)
 675
 676     # value is a dictionary, check that each member is okay
 677     for (key, arg) in value.items():
 678         check_name(info, "Member of %s" % source, key,
 679                    allow_optional=allow_optional)
 680         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 681             raise QAPISemError(info, "Member of %s uses reserved name '%s'"
 682                                % (source, key))
 683         # Todo: allow dictionaries to represent default values of
 684         # an optional argument.
 685         check_type(info, "Member '%s' of %s" % (key, source), arg,
 686                    allow_array=True,
 687                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 688                                 'enum'])
 689
 690
 691 def check_command(expr, info):
 692     name = expr['command']
 693     boxed = expr.get('boxed', False)
 694
 695     args_meta = ['struct']
 696     if boxed:
 697         args_meta += ['union', 'alternate']
 698     check_type(info, "'data' for command '%s'" % name,
 699                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 700                allow_metas=args_meta)
 701     returns_meta = ['union', 'struct']
 702     if name in returns_whitelist:
 703         returns_meta += ['built-in', 'alternate', 'enum']
 704     check_type(info, "'returns' for command '%s'" % name,
 705                expr.get('returns'), allow_array=True,
 706                allow_optional=True, allow_metas=returns_meta)
 707
 708
 709 def check_event(expr, info):
 710     name = expr['event']
 711     boxed = expr.get('boxed', False)
 712
 713     meta = ['struct']
 714     if boxed:
 715         meta += ['union', 'alternate']
 716     check_type(info, "'data' for event '%s'" % name,
 717                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 718                allow_metas=meta)
 719
 720
 721 def check_union(expr, info):
 722     name = expr['union']
 723     base = expr.get('base')
 724     discriminator = expr.get('discriminator')
 725     members = expr['data']
 726
 727     # Two types of unions, determined by discriminator.
 728
 729     # With no discriminator it is a simple union.
 730     if discriminator is None:
 731         enum_define = None
 732         allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
 733         if base is not None:
 734             raise QAPISemError(info, "Simple union '%s' must not have a base" %
 735                                name)
 736
 737     # Else, it's a flat union.
 738     else:
 739         # The object must have a string or dictionary 'base'.
 740         check_type(info, "'base' for union '%s'" % name,
 741                    base, allow_dict=True, allow_optional=True,
 742                    allow_metas=['struct'])
 743         if not base:
 744             raise QAPISemError(info, "Flat union '%s' must have a base"
 745                                % name)
 746         base_members = find_base_members(base)
 747         assert base_members is not None
 748
 749         # The value of member 'discriminator' must name a non-optional
 750         # member of the base struct.
 751         check_name(info, "Discriminator of flat union '%s'" % name,
 752                    discriminator)
 753         discriminator_type = base_members.get(discriminator)
 754         if not discriminator_type:
 755             raise QAPISemError(info,
 756                                "Discriminator '%s' is not a member of base "
 757                                "struct '%s'"
 758                                % (discriminator, base))
 759         enum_define = enum_types.get(discriminator_type)
 760         allow_metas = ['struct']
 761         # Do not allow string discriminator
 762         if not enum_define:
 763             raise QAPISemError(info,
 764                                "Discriminator '%s' must be of enumeration "
 765                                "type" % discriminator)
 766
 767     # Check every branch; don't allow an empty union
 768     if len(members) == 0:
 769         raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
 770     for (key, value) in members.items():
 771         check_name(info, "Member of union '%s'" % name, key)
 772
 773         # Each value must name a known type
 774         check_type(info, "Member '%s' of union '%s'" % (key, name),
 775                    value, allow_array=not base, allow_metas=allow_metas)
 776
 777         # If the discriminator names an enum type, then all members
 778         # of 'data' must also be members of the enum type.
 779         if enum_define:
 780             if key not in enum_define['data']:
 781                 raise QAPISemError(info,
 782                                    "Discriminator value '%s' is not found in "
 783                                    "enum '%s'"
 784                                    % (key, enum_define['enum']))
 785
 786     # If discriminator is user-defined, ensure all values are covered
 787     if enum_define:
 788         for value in enum_define['data']:
 789             if value not in members.keys():
 790                 raise QAPISemError(info, "Union '%s' data missing '%s' branch"
 791                                    % (name, value))
 792
 793
 794 def check_alternate(expr, info):
 795     name = expr['alternate']
 796     members = expr['data']
 797     types_seen = {}
 798
 799     # Check every branch; require at least two branches
 800     if len(members) < 2:
 801         raise QAPISemError(info,
 802                            "Alternate '%s' should have at least two branches "
 803                            "in 'data'" % name)
 804     for (key, value) in members.items():
 805         check_name(info, "Member of alternate '%s'" % name, key)
 806
 807         # Ensure alternates have no type conflicts.
 808         check_type(info, "Member '%s' of alternate '%s'" % (key, name),
 809                    value,
 810                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 811         qtype = find_alternate_member_qtype(value)
 812         if not qtype:
 813             raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
 814                                "type '%s'" % (name, key, value))
 815         conflicting = set([qtype])
 816         if qtype == 'QTYPE_QSTRING':
 817             enum_expr = enum_types.get(value)
 818             if enum_expr:
 819                 for v in enum_expr['data']:
 820                     if v in ['on', 'off']:
 821                         conflicting.add('QTYPE_QBOOL')
 822                     if re.match(r'[-+0-9.]', v): # lazy, could be tightened
 823                         conflicting.add('QTYPE_QINT')
 824                         conflicting.add('QTYPE_QFLOAT')
 825             else:
 826                 conflicting.add('QTYPE_QINT')
 827                 conflicting.add('QTYPE_QFLOAT')
 828                 conflicting.add('QTYPE_QBOOL')
 829         if conflicting & set(types_seen):
 830             raise QAPISemError(info, "Alternate '%s' member '%s' can't "
 831                                "be distinguished from member '%s'"
 832                                % (name, key, types_seen[qtype]))
 833         for qt in conflicting:
 834             types_seen[qt] = key
 835
 836
 837 def check_enum(expr, info):
 838     name = expr['enum']
 839     members = expr.get('data')
 840     prefix = expr.get('prefix')
 841
 842     if not isinstance(members, list):
 843         raise QAPISemError(info,
 844                            "Enum '%s' requires an array for 'data'" % name)
 845     if prefix is not None and not isinstance(prefix, str):
 846         raise QAPISemError(info,
 847                            "Enum '%s' requires a string for 'prefix'" % name)
 848     for member in members:
 849         check_name(info, "Member of enum '%s'" % name, member,
 850                    enum_member=True)
 851
 852
 853 def check_struct(expr, info):
 854     name = expr['struct']
 855     members = expr['data']
 856
 857     check_type(info, "'data' for struct '%s'" % name, members,
 858                allow_dict=True, allow_optional=True)
 859     check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
 860                allow_metas=['struct'])
 861
 862
 863 def check_keys(expr_elem, meta, required, optional=[]):
 864     expr = expr_elem['expr']
 865     info = expr_elem['info']
 866     name = expr[meta]
 867     if not isinstance(name, str):
 868         raise QAPISemError(info, "'%s' key must have a string value" % meta)
 869     required = required + [meta]
 870     for (key, value) in expr.items():
 871         if key not in required and key not in optional:
 872             raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
 873                                % (key, meta, name))
 874         if (key == 'gen' or key == 'success-response') and value is not False:
 875             raise QAPISemError(info,
 876                                "'%s' of %s '%s' should only use false value"
 877                                % (key, meta, name))
 878         if key == 'boxed' and value is not True:
 879             raise QAPISemError(info,
 880                                "'%s' of %s '%s' should only use true value"
 881                                % (key, meta, name))
 882     for key in required:
 883         if key not in expr:
 884             raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
 885                                % (key, meta, name))
 886
 887
 888 def check_exprs(exprs):
 889     global all_names
 890
 891     # Populate name table with names of built-in types
 892     for builtin in builtin_types.keys():
 893         all_names[builtin] = 'built-in'
 894
 895     # Learn the types and check for valid expression keys
 896     for expr_elem in exprs:
 897         expr = expr_elem['expr']
 898         info = expr_elem['info']
 899         doc = expr_elem.get('doc')
 900
 901         if not doc and doc_required:
 902             raise QAPISemError(info,
 903                                "Expression missing documentation comment")
 904
 905         if 'enum' in expr:
 906             meta = 'enum'
 907             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 908             enum_types[expr[meta]] = expr
 909         elif 'union' in expr:
 910             meta = 'union'
 911             check_keys(expr_elem, 'union', ['data'],
 912                        ['base', 'discriminator'])
 913             union_types[expr[meta]] = expr
 914         elif 'alternate' in expr:
 915             meta = 'alternate'
 916             check_keys(expr_elem, 'alternate', ['data'])
 917         elif 'struct' in expr:
 918             meta = 'struct'
 919             check_keys(expr_elem, 'struct', ['data'], ['base'])
 920             struct_types[expr[meta]] = expr
 921         elif 'command' in expr:
 922             meta = 'command'
 923             check_keys(expr_elem, 'command', [],
 924                        ['data', 'returns', 'gen', 'success-response', 'boxed'])
 925         elif 'event' in expr:
 926             meta = 'event'
 927             check_keys(expr_elem, 'event', [], ['data', 'boxed'])
 928         else:
 929             raise QAPISemError(expr_elem['info'],
 930                                "Expression is missing metatype")
 931         name = expr[meta]
 932         add_name(name, info, meta)
 933         if doc and doc.symbol != name:
 934             raise QAPISemError(info, "Definition of '%s' follows documentation"
 935                                " for '%s'" % (name, doc.symbol))
 936
 937     # Try again for hidden UnionKind enum
 938     for expr_elem in exprs:
 939         expr = expr_elem['expr']
 940         if 'union' in expr and not discriminator_find_enum_define(expr):
 941             name = '%sKind' % expr['union']
 942         elif 'alternate' in expr:
 943             name = '%sKind' % expr['alternate']
 944         else:
 945             continue
 946         enum_types[name] = {'enum': name}
 947         add_name(name, info, 'enum', implicit=True)
 948
 949     # Validate that exprs make sense
 950     for expr_elem in exprs:
 951         expr = expr_elem['expr']
 952         info = expr_elem['info']
 953         doc = expr_elem.get('doc')
 954
 955         if 'enum' in expr:
 956             check_enum(expr, info)
 957         elif 'union' in expr:
 958             check_union(expr, info)
 959         elif 'alternate' in expr:
 960             check_alternate(expr, info)
 961         elif 'struct' in expr:
 962             check_struct(expr, info)
 963         elif 'command' in expr:
 964             check_command(expr, info)
 965         elif 'event' in expr:
 966             check_event(expr, info)
 967         else:
 968             assert False, 'unexpected meta type'
 969
 970         if doc:
 971             doc.check_expr(expr)
 972
 973     return exprs
 974
 975
 976 #
 977 # Schema compiler frontend
 978 #
 979
 980 class QAPISchemaEntity(object):
 981     def __init__(self, name, info, doc):
 982         assert isinstance(name, str)
 983         self.name = name
 984         # For explicitly defined entities, info points to the (explicit)
 985         # definition.  For builtins (and their arrays), info is None.
 986         # For implicitly defined entities, info points to a place that
 987         # triggered the implicit definition (there may be more than one
 988         # such place).
 989         self.info = info
 990         self.doc = doc
 991
 992     def c_name(self):
 993         return c_name(self.name)
 994
 995     def check(self, schema):
 996         pass
 997
 998     def is_implicit(self):
 999         return not self.info
1000
1001     def visit(self, visitor):
1002         pass
1003
1004
1005 class QAPISchemaVisitor(object):
1006     def visit_begin(self, schema):
1007         pass
1008
1009     def visit_end(self):
1010         pass
1011
1012     def visit_needed(self, entity):
1013         # Default to visiting everything
1014         return True
1015
1016     def visit_builtin_type(self, name, info, json_type):
1017         pass
1018
1019     def visit_enum_type(self, name, info, values, prefix):
1020         pass
1021
1022     def visit_array_type(self, name, info, element_type):
1023         pass
1024
1025     def visit_object_type(self, name, info, base, members, variants):
1026         pass
1027
1028     def visit_object_type_flat(self, name, info, members, variants):
1029         pass
1030
1031     def visit_alternate_type(self, name, info, variants):
1032         pass
1033
1034     def visit_command(self, name, info, arg_type, ret_type,
1035                       gen, success_response, boxed):
1036         pass
1037
1038     def visit_event(self, name, info, arg_type, boxed):
1039         pass
1040
1041
1042 class QAPISchemaType(QAPISchemaEntity):
1043     # Return the C type for common use.
1044     # For the types we commonly box, this is a pointer type.
1045     def c_type(self):
1046         pass
1047
1048     # Return the C type to be used in a parameter list.
1049     def c_param_type(self):
1050         return self.c_type()
1051
1052     # Return the C type to be used where we suppress boxing.
1053     def c_unboxed_type(self):
1054         return self.c_type()
1055
1056     def json_type(self):
1057         pass
1058
1059     def alternate_qtype(self):
1060         json2qtype = {
1061             'string':  'QTYPE_QSTRING',
1062             'number':  'QTYPE_QFLOAT',
1063             'int':     'QTYPE_QINT',
1064             'boolean': 'QTYPE_QBOOL',
1065             'object':  'QTYPE_QDICT'
1066         }
1067         return json2qtype.get(self.json_type())
1068
1069     def doc_type(self):
1070         if self.is_implicit():
1071             return None
1072         return self.name
1073
1074
1075 class QAPISchemaBuiltinType(QAPISchemaType):
1076     def __init__(self, name, json_type, c_type):
1077         QAPISchemaType.__init__(self, name, None, None)
1078         assert not c_type or isinstance(c_type, str)
1079         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
1080                              'value')
1081         self._json_type_name = json_type
1082         self._c_type_name = c_type
1083
1084     def c_name(self):
1085         return self.name
1086
1087     def c_type(self):
1088         return self._c_type_name
1089
1090     def c_param_type(self):
1091         if self.name == 'str':
1092             return 'const ' + self._c_type_name
1093         return self._c_type_name
1094
1095     def json_type(self):
1096         return self._json_type_name
1097
1098     def doc_type(self):
1099         return self.json_type()
1100
1101     def visit(self, visitor):
1102         visitor.visit_builtin_type(self.name, self.info, self.json_type())
1103
1104
1105 class QAPISchemaEnumType(QAPISchemaType):
1106     def __init__(self, name, info, doc, values, prefix):
1107         QAPISchemaType.__init__(self, name, info, doc)
1108         for v in values:
1109             assert isinstance(v, QAPISchemaMember)
1110             v.set_owner(name)
1111         assert prefix is None or isinstance(prefix, str)
1112         self.values = values
1113         self.prefix = prefix
1114
1115     def check(self, schema):
1116         seen = {}
1117         for v in self.values:
1118             v.check_clash(self.info, seen)
1119             if self.doc:
1120                 self.doc.connect_member(v)
1121
1122     def is_implicit(self):
1123         # See QAPISchema._make_implicit_enum_type() and ._def_predefineds()
1124         return self.name.endswith('Kind') or self.name == 'QType'
1125
1126     def c_type(self):
1127         return c_name(self.name)
1128
1129     def member_names(self):
1130         return [v.name for v in self.values]
1131
1132     def json_type(self):
1133         return 'string'
1134
1135     def visit(self, visitor):
1136         visitor.visit_enum_type(self.name, self.info,
1137                                 self.member_names(), self.prefix)
1138
1139
1140 class QAPISchemaArrayType(QAPISchemaType):
1141     def __init__(self, name, info, element_type):
1142         QAPISchemaType.__init__(self, name, info, None)
1143         assert isinstance(element_type, str)
1144         self._element_type_name = element_type
1145         self.element_type = None
1146
1147     def check(self, schema):
1148         self.element_type = schema.lookup_type(self._element_type_name)
1149         assert self.element_type
1150
1151     def is_implicit(self):
1152         return True
1153
1154     def c_type(self):
1155         return c_name(self.name) + pointer_suffix
1156
1157     def json_type(self):
1158         return 'array'
1159
1160     def doc_type(self):
1161         elt_doc_type = self.element_type.doc_type()
1162         if not elt_doc_type:
1163             return None
1164         return 'array of ' + elt_doc_type
1165
1166     def visit(self, visitor):
1167         visitor.visit_array_type(self.name, self.info, self.element_type)
1168
1169
1170 class QAPISchemaObjectType(QAPISchemaType):
1171     def __init__(self, name, info, doc, base, local_members, variants):
1172         # struct has local_members, optional base, and no variants
1173         # flat union has base, variants, and no local_members
1174         # simple union has local_members, variants, and no base
1175         QAPISchemaType.__init__(self, name, info, doc)
1176         assert base is None or isinstance(base, str)
1177         for m in local_members:
1178             assert isinstance(m, QAPISchemaObjectTypeMember)
1179             m.set_owner(name)
1180         if variants is not None:
1181             assert isinstance(variants, QAPISchemaObjectTypeVariants)
1182             variants.set_owner(name)
1183         self._base_name = base
1184         self.base = None
1185         self.local_members = local_members
1186         self.variants = variants
1187         self.members = None
1188
1189     def check(self, schema):
1190         if self.members is False:               # check for cycles
1191             raise QAPISemError(self.info,
1192                                "Object %s contains itself" % self.name)
1193         if self.members:
1194             return
1195         self.members = False                    # mark as being checked
1196         seen = OrderedDict()
1197         if self._base_name:
1198             self.base = schema.lookup_type(self._base_name)
1199             assert isinstance(self.base, QAPISchemaObjectType)
1200             self.base.check(schema)
1201             self.base.check_clash(self.info, seen)
1202         for m in self.local_members:
1203             m.check(schema)
1204             m.check_clash(self.info, seen)
1205             if self.doc:
1206                 self.doc.connect_member(m)
1207         self.members = seen.values()
1208         if self.variants:
1209             self.variants.check(schema, seen)
1210             assert self.variants.tag_member in self.members
1211             self.variants.check_clash(self.info, seen)
1212         if self.doc:
1213             self.doc.check()
1214
1215     # Check that the members of this type do not cause duplicate JSON members,
1216     # and update seen to track the members seen so far. Report any errors
1217     # on behalf of info, which is not necessarily self.info
1218     def check_clash(self, info, seen):
1219         assert not self.variants       # not implemented
1220         for m in self.members:
1221             m.check_clash(info, seen)
1222
1223     def is_implicit(self):
1224         # See QAPISchema._make_implicit_object_type(), as well as
1225         # _def_predefineds()
1226         return self.name.startswith('q_')
1227
1228     def is_empty(self):
1229         assert self.members is not None
1230         return not self.members and not self.variants
1231
1232     def c_name(self):
1233         assert self.name != 'q_empty'
1234         return QAPISchemaType.c_name(self)
1235
1236     def c_type(self):
1237         assert not self.is_implicit()
1238         return c_name(self.name) + pointer_suffix
1239
1240     def c_unboxed_type(self):
1241         return c_name(self.name)
1242
1243     def json_type(self):
1244         return 'object'
1245
1246     def visit(self, visitor):
1247         visitor.visit_object_type(self.name, self.info,
1248                                   self.base, self.local_members, self.variants)
1249         visitor.visit_object_type_flat(self.name, self.info,
1250                                        self.members, self.variants)
1251
1252
1253 class QAPISchemaMember(object):
1254     role = 'member'
1255
1256     def __init__(self, name):
1257         assert isinstance(name, str)
1258         self.name = name
1259         self.owner = None
1260
1261     def set_owner(self, name):
1262         assert not self.owner
1263         self.owner = name
1264
1265     def check_clash(self, info, seen):
1266         cname = c_name(self.name)
1267         if cname.lower() != cname and self.owner not in name_case_whitelist:
1268             raise QAPISemError(info,
1269                                "%s should not use uppercase" % self.describe())
1270         if cname in seen:
1271             raise QAPISemError(info, "%s collides with %s" %
1272                                (self.describe(), seen[cname].describe()))
1273         seen[cname] = self
1274
1275     def _pretty_owner(self):
1276         owner = self.owner
1277         if owner.startswith('q_obj_'):
1278             # See QAPISchema._make_implicit_object_type() - reverse the
1279             # mapping there to create a nice human-readable description
1280             owner = owner[6:]
1281             if owner.endswith('-arg'):
1282                 return '(parameter of %s)' % owner[:-4]
1283             elif owner.endswith('-base'):
1284                 return '(base of %s)' % owner[:-5]
1285             else:
1286                 assert owner.endswith('-wrapper')
1287                 # Unreachable and not implemented
1288                 assert False
1289         if owner.endswith('Kind'):
1290             # See QAPISchema._make_implicit_enum_type()
1291             return '(branch of %s)' % owner[:-4]
1292         return '(%s of %s)' % (self.role, owner)
1293
1294     def describe(self):
1295         return "'%s' %s" % (self.name, self._pretty_owner())
1296
1297
1298 class QAPISchemaObjectTypeMember(QAPISchemaMember):
1299     def __init__(self, name, typ, optional):
1300         QAPISchemaMember.__init__(self, name)
1301         assert isinstance(typ, str)
1302         assert isinstance(optional, bool)
1303         self._type_name = typ
1304         self.type = None
1305         self.optional = optional
1306
1307     def check(self, schema):
1308         assert self.owner
1309         self.type = schema.lookup_type(self._type_name)
1310         assert self.type
1311
1312
1313 class QAPISchemaObjectTypeVariants(object):
1314     def __init__(self, tag_name, tag_member, variants):
1315         # Flat unions pass tag_name but not tag_member.
1316         # Simple unions and alternates pass tag_member but not tag_name.
1317         # After check(), tag_member is always set, and tag_name remains
1318         # a reliable witness of being used by a flat union.
1319         assert bool(tag_member) != bool(tag_name)
1320         assert (isinstance(tag_name, str) or
1321                 isinstance(tag_member, QAPISchemaObjectTypeMember))
1322         assert len(variants) > 0
1323         for v in variants:
1324             assert isinstance(v, QAPISchemaObjectTypeVariant)
1325         self._tag_name = tag_name
1326         self.tag_member = tag_member
1327         self.variants = variants
1328
1329     def set_owner(self, name):
1330         for v in self.variants:
1331             v.set_owner(name)
1332
1333     def check(self, schema, seen):
1334         if not self.tag_member:    # flat union
1335             self.tag_member = seen[c_name(self._tag_name)]
1336             assert self._tag_name == self.tag_member.name
1337         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
1338         for v in self.variants:
1339             v.check(schema)
1340             # Union names must match enum values; alternate names are
1341             # checked separately. Use 'seen' to tell the two apart.
1342             if seen:
1343                 assert v.name in self.tag_member.type.member_names()
1344                 assert isinstance(v.type, QAPISchemaObjectType)
1345                 v.type.check(schema)
1346
1347     def check_clash(self, info, seen):
1348         for v in self.variants:
1349             # Reset seen map for each variant, since qapi names from one
1350             # branch do not affect another branch
1351             assert isinstance(v.type, QAPISchemaObjectType)
1352             v.type.check_clash(info, dict(seen))
1353
1354
1355 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1356     role = 'branch'
1357
1358     def __init__(self, name, typ):
1359         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1360
1361
1362 class QAPISchemaAlternateType(QAPISchemaType):
1363     def __init__(self, name, info, doc, variants):
1364         QAPISchemaType.__init__(self, name, info, doc)
1365         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1366         assert variants.tag_member
1367         variants.set_owner(name)
1368         variants.tag_member.set_owner(self.name)
1369         self.variants = variants
1370
1371     def check(self, schema):
1372         self.variants.tag_member.check(schema)
1373         # Not calling self.variants.check_clash(), because there's nothing
1374         # to clash with
1375         self.variants.check(schema, {})
1376         # Alternate branch names have no relation to the tag enum values;
1377         # so we have to check for potential name collisions ourselves.
1378         seen = {}
1379         for v in self.variants.variants:
1380             v.check_clash(self.info, seen)
1381             if self.doc:
1382                 self.doc.connect_member(v)
1383         if self.doc:
1384             self.doc.check()
1385
1386     def c_type(self):
1387         return c_name(self.name) + pointer_suffix
1388
1389     def json_type(self):
1390         return 'value'
1391
1392     def visit(self, visitor):
1393         visitor.visit_alternate_type(self.name, self.info, self.variants)
1394
1395     def is_empty(self):
1396         return False
1397
1398
1399 class QAPISchemaCommand(QAPISchemaEntity):
1400     def __init__(self, name, info, doc, arg_type, ret_type,
1401                  gen, success_response, boxed):
1402         QAPISchemaEntity.__init__(self, name, info, doc)
1403         assert not arg_type or isinstance(arg_type, str)
1404         assert not ret_type or isinstance(ret_type, str)
1405         self._arg_type_name = arg_type
1406         self.arg_type = None
1407         self._ret_type_name = ret_type
1408         self.ret_type = None
1409         self.gen = gen
1410         self.success_response = success_response
1411         self.boxed = boxed
1412
1413     def check(self, schema):
1414         if self._arg_type_name:
1415             self.arg_type = schema.lookup_type(self._arg_type_name)
1416             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1417                     isinstance(self.arg_type, QAPISchemaAlternateType))
1418             self.arg_type.check(schema)
1419             if self.boxed:
1420                 if self.arg_type.is_empty():
1421                     raise QAPISemError(self.info,
1422                                        "Cannot use 'boxed' with empty type")
1423             else:
1424                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1425                 assert not self.arg_type.variants
1426         elif self.boxed:
1427             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1428         if self._ret_type_name:
1429             self.ret_type = schema.lookup_type(self._ret_type_name)
1430             assert isinstance(self.ret_type, QAPISchemaType)
1431
1432     def visit(self, visitor):
1433         visitor.visit_command(self.name, self.info,
1434                               self.arg_type, self.ret_type,
1435                               self.gen, self.success_response, self.boxed)
1436
1437
1438 class QAPISchemaEvent(QAPISchemaEntity):
1439     def __init__(self, name, info, doc, arg_type, boxed):
1440         QAPISchemaEntity.__init__(self, name, info, doc)
1441         assert not arg_type or isinstance(arg_type, str)
1442         self._arg_type_name = arg_type
1443         self.arg_type = None
1444         self.boxed = boxed
1445
1446     def check(self, schema):
1447         if self._arg_type_name:
1448             self.arg_type = schema.lookup_type(self._arg_type_name)
1449             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1450                     isinstance(self.arg_type, QAPISchemaAlternateType))
1451             self.arg_type.check(schema)
1452             if self.boxed:
1453                 if self.arg_type.is_empty():
1454                     raise QAPISemError(self.info,
1455                                        "Cannot use 'boxed' with empty type")
1456             else:
1457                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1458                 assert not self.arg_type.variants
1459         elif self.boxed:
1460             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1461
1462     def visit(self, visitor):
1463         visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
1464
1465
1466 class QAPISchema(object):
1467     def __init__(self, fname):
1468         try:
1469             parser = QAPISchemaParser(open(fname, 'r'))
1470             self.exprs = check_exprs(parser.exprs)
1471             self.docs = parser.docs
1472             self._entity_dict = {}
1473             self._predefining = True
1474             self._def_predefineds()
1475             self._predefining = False
1476             self._def_exprs()
1477             self.check()
1478         except QAPIError as err:
1479             print >>sys.stderr, err
1480             exit(1)
1481
1482     def _def_entity(self, ent):
1483         # Only the predefined types are allowed to not have info
1484         assert ent.info or self._predefining
1485         assert ent.name not in self._entity_dict
1486         self._entity_dict[ent.name] = ent
1487
1488     def lookup_entity(self, name, typ=None):
1489         ent = self._entity_dict.get(name)
1490         if typ and not isinstance(ent, typ):
1491             return None
1492         return ent
1493
1494     def lookup_type(self, name):
1495         return self.lookup_entity(name, QAPISchemaType)
1496
1497     def _def_builtin_type(self, name, json_type, c_type):
1498         self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
1499         # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
1500         # qapi-types.h from a single .c, all arrays of builtins must be
1501         # declared in the first file whether or not they are used.  Nicer
1502         # would be to use lazy instantiation, while figuring out how to
1503         # avoid compilation issues with multiple qapi-types.h.
1504         self._make_array_type(name, None)
1505
1506     def _def_predefineds(self):
1507         for t in [('str',    'string',  'char' + pointer_suffix),
1508                   ('number', 'number',  'double'),
1509                   ('int',    'int',     'int64_t'),
1510                   ('int8',   'int',     'int8_t'),
1511                   ('int16',  'int',     'int16_t'),
1512                   ('int32',  'int',     'int32_t'),
1513                   ('int64',  'int',     'int64_t'),
1514                   ('uint8',  'int',     'uint8_t'),
1515                   ('uint16', 'int',     'uint16_t'),
1516                   ('uint32', 'int',     'uint32_t'),
1517                   ('uint64', 'int',     'uint64_t'),
1518                   ('size',   'int',     'uint64_t'),
1519                   ('bool',   'boolean', 'bool'),
1520                   ('any',    'value',   'QObject' + pointer_suffix)]:
1521             self._def_builtin_type(*t)
1522         self.the_empty_object_type = QAPISchemaObjectType(
1523             'q_empty', None, None, None, [], None)
1524         self._def_entity(self.the_empty_object_type)
1525         qtype_values = self._make_enum_members(['none', 'qnull', 'qint',
1526                                                 'qstring', 'qdict', 'qlist',
1527                                                 'qfloat', 'qbool'])
1528         self._def_entity(QAPISchemaEnumType('QType', None, None,
1529                                             qtype_values, 'QTYPE'))
1530
1531     def _make_enum_members(self, values):
1532         return [QAPISchemaMember(v) for v in values]
1533
1534     def _make_implicit_enum_type(self, name, info, values):
1535         # See also QAPISchemaObjectTypeMember._pretty_owner()
1536         name = name + 'Kind'   # Use namespace reserved by add_name()
1537         self._def_entity(QAPISchemaEnumType(
1538             name, info, None, self._make_enum_members(values), None))
1539         return name
1540
1541     def _make_array_type(self, element_type, info):
1542         name = element_type + 'List'   # Use namespace reserved by add_name()
1543         if not self.lookup_type(name):
1544             self._def_entity(QAPISchemaArrayType(name, info, element_type))
1545         return name
1546
1547     def _make_implicit_object_type(self, name, info, doc, role, members):
1548         if not members:
1549             return None
1550         # See also QAPISchemaObjectTypeMember._pretty_owner()
1551         name = 'q_obj_%s-%s' % (name, role)
1552         if not self.lookup_entity(name, QAPISchemaObjectType):
1553             self._def_entity(QAPISchemaObjectType(name, info, doc, None,
1554                                                   members, None))
1555         return name
1556
1557     def _def_enum_type(self, expr, info, doc):
1558         name = expr['enum']
1559         data = expr['data']
1560         prefix = expr.get('prefix')
1561         self._def_entity(QAPISchemaEnumType(
1562             name, info, doc, self._make_enum_members(data), prefix))
1563
1564     def _make_member(self, name, typ, info):
1565         optional = False
1566         if name.startswith('*'):
1567             name = name[1:]
1568             optional = True
1569         if isinstance(typ, list):
1570             assert len(typ) == 1
1571             typ = self._make_array_type(typ[0], info)
1572         return QAPISchemaObjectTypeMember(name, typ, optional)
1573
1574     def _make_members(self, data, info):
1575         return [self._make_member(key, value, info)
1576                 for (key, value) in data.iteritems()]
1577
1578     def _def_struct_type(self, expr, info, doc):
1579         name = expr['struct']
1580         base = expr.get('base')
1581         data = expr['data']
1582         self._def_entity(QAPISchemaObjectType(name, info, doc, base,
1583                                               self._make_members(data, info),
1584                                               None))
1585
1586     def _make_variant(self, case, typ):
1587         return QAPISchemaObjectTypeVariant(case, typ)
1588
1589     def _make_simple_variant(self, case, typ, info):
1590         if isinstance(typ, list):
1591             assert len(typ) == 1
1592             typ = self._make_array_type(typ[0], info)
1593         typ = self._make_implicit_object_type(
1594             typ, info, None, 'wrapper', [self._make_member('data', typ, info)])
1595         return QAPISchemaObjectTypeVariant(case, typ)
1596
1597     def _def_union_type(self, expr, info, doc):
1598         name = expr['union']
1599         data = expr['data']
1600         base = expr.get('base')
1601         tag_name = expr.get('discriminator')
1602         tag_member = None
1603         if isinstance(base, dict):
1604             base = (self._make_implicit_object_type(
1605                 name, info, doc, 'base', self._make_members(base, info)))
1606         if tag_name:
1607             variants = [self._make_variant(key, value)
1608                         for (key, value) in data.iteritems()]
1609             members = []
1610         else:
1611             variants = [self._make_simple_variant(key, value, info)
1612                         for (key, value) in data.iteritems()]
1613             typ = self._make_implicit_enum_type(name, info,
1614                                                 [v.name for v in variants])
1615             tag_member = QAPISchemaObjectTypeMember('type', typ, False)
1616             members = [tag_member]
1617         self._def_entity(
1618             QAPISchemaObjectType(name, info, doc, base, members,
1619                                  QAPISchemaObjectTypeVariants(tag_name,
1620                                                               tag_member,
1621                                                               variants)))
1622
1623     def _def_alternate_type(self, expr, info, doc):
1624         name = expr['alternate']
1625         data = expr['data']
1626         variants = [self._make_variant(key, value)
1627                     for (key, value) in data.iteritems()]
1628         tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
1629         self._def_entity(
1630             QAPISchemaAlternateType(name, info, doc,
1631                                     QAPISchemaObjectTypeVariants(None,
1632                                                                  tag_member,
1633                                                                  variants)))
1634
1635     def _def_command(self, expr, info, doc):
1636         name = expr['command']
1637         data = expr.get('data')
1638         rets = expr.get('returns')
1639         gen = expr.get('gen', True)
1640         success_response = expr.get('success-response', True)
1641         boxed = expr.get('boxed', False)
1642         if isinstance(data, OrderedDict):
1643             data = self._make_implicit_object_type(
1644                 name, info, doc, 'arg', self._make_members(data, info))
1645         if isinstance(rets, list):
1646             assert len(rets) == 1
1647             rets = self._make_array_type(rets[0], info)
1648         self._def_entity(QAPISchemaCommand(name, info, doc, data, rets,
1649                                            gen, success_response, boxed))
1650
1651     def _def_event(self, expr, info, doc):
1652         name = expr['event']
1653         data = expr.get('data')
1654         boxed = expr.get('boxed', False)
1655         if isinstance(data, OrderedDict):
1656             data = self._make_implicit_object_type(
1657                 name, info, doc, 'arg', self._make_members(data, info))
1658         self._def_entity(QAPISchemaEvent(name, info, doc, data, boxed))
1659
1660     def _def_exprs(self):
1661         for expr_elem in self.exprs:
1662             expr = expr_elem['expr']
1663             info = expr_elem['info']
1664             doc = expr_elem.get('doc')
1665             if 'enum' in expr:
1666                 self._def_enum_type(expr, info, doc)
1667             elif 'struct' in expr:
1668                 self._def_struct_type(expr, info, doc)
1669             elif 'union' in expr:
1670                 self._def_union_type(expr, info, doc)
1671             elif 'alternate' in expr:
1672                 self._def_alternate_type(expr, info, doc)
1673             elif 'command' in expr:
1674                 self._def_command(expr, info, doc)
1675             elif 'event' in expr:
1676                 self._def_event(expr, info, doc)
1677             else:
1678                 assert False
1679
1680     def check(self):
1681         for ent in self._entity_dict.values():
1682             ent.check(self)
1683
1684     def visit(self, visitor):
1685         visitor.visit_begin(self)
1686         for (name, entity) in sorted(self._entity_dict.items()):
1687             if visitor.visit_needed(entity):
1688                 entity.visit(visitor)
1689         visitor.visit_end()
1690
1691
1692 #
1693 # Code generation helpers
1694 #
1695
1696 def camel_case(name):
1697     new_name = ''
1698     first = True
1699     for ch in name:
1700         if ch in ['_', '-']:
1701             first = True
1702         elif first:
1703             new_name += ch.upper()
1704             first = False
1705         else:
1706             new_name += ch.lower()
1707     return new_name
1708
1709
1710 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1711 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1712 # ENUM24_Name -> ENUM24_NAME
1713 def camel_to_upper(value):
1714     c_fun_str = c_name(value, False)
1715     if value.isupper():
1716         return c_fun_str
1717
1718     new_name = ''
1719     l = len(c_fun_str)
1720     for i in range(l):
1721         c = c_fun_str[i]
1722         # When c is upper and no '_' appears before, do more checks
1723         if c.isupper() and (i > 0) and c_fun_str[i - 1] != '_':
1724             if i < l - 1 and c_fun_str[i + 1].islower():
1725                 new_name += '_'
1726             elif c_fun_str[i - 1].isdigit():
1727                 new_name += '_'
1728         new_name += c
1729     return new_name.lstrip('_').upper()
1730
1731
1732 def c_enum_const(type_name, const_name, prefix=None):
1733     if prefix is not None:
1734         type_name = prefix
1735     return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
1736
1737 c_name_trans = string.maketrans('.-', '__')
1738
1739
1740 # Map @name to a valid C identifier.
1741 # If @protect, avoid returning certain ticklish identifiers (like
1742 # C keywords) by prepending 'q_'.
1743 #
1744 # Used for converting 'name' from a 'name':'type' qapi definition
1745 # into a generated struct member, as well as converting type names
1746 # into substrings of a generated C function name.
1747 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1748 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1749 def c_name(name, protect=True):
1750     # ANSI X3J11/88-090, 3.1.1
1751     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1752                      'default', 'do', 'double', 'else', 'enum', 'extern',
1753                      'float', 'for', 'goto', 'if', 'int', 'long', 'register',
1754                      'return', 'short', 'signed', 'sizeof', 'static',
1755                      'struct', 'switch', 'typedef', 'union', 'unsigned',
1756                      'void', 'volatile', 'while'])
1757     # ISO/IEC 9899:1999, 6.4.1
1758     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1759     # ISO/IEC 9899:2011, 6.4.1
1760     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
1761                      '_Noreturn', '_Static_assert', '_Thread_local'])
1762     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1763     # excluding _.*
1764     gcc_words = set(['asm', 'typeof'])
1765     # C++ ISO/IEC 14882:2003 2.11
1766     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1767                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1768                      'namespace', 'new', 'operator', 'private', 'protected',
1769                      'public', 'reinterpret_cast', 'static_cast', 'template',
1770                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1771                      'using', 'virtual', 'wchar_t',
1772                      # alternative representations
1773                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1774                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1775     # namespace pollution:
1776     polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
1777     name = name.translate(c_name_trans)
1778     if protect and (name in c89_words | c99_words | c11_words | gcc_words
1779                     | cpp_words | polluted_words):
1780         return 'q_' + name
1781     return name
1782
1783 eatspace = '\033EATSPACE.'
1784 pointer_suffix = ' *' + eatspace
1785
1786
1787 def genindent(count):
1788     ret = ''
1789     for _ in range(count):
1790         ret += ' '
1791     return ret
1792
1793 indent_level = 0
1794
1795
1796 def push_indent(indent_amount=4):
1797     global indent_level
1798     indent_level += indent_amount
1799
1800
1801 def pop_indent(indent_amount=4):
1802     global indent_level
1803     indent_level -= indent_amount
1804
1805
1806 # Generate @code with @kwds interpolated.
1807 # Obey indent_level, and strip eatspace.
1808 def cgen(code, **kwds):
1809     raw = code % kwds
1810     if indent_level:
1811         indent = genindent(indent_level)
1812         # re.subn() lacks flags support before Python 2.7, use re.compile()
1813         raw = re.subn(re.compile(r'^.', re.MULTILINE),
1814                       indent + r'\g<0>', raw)
1815         raw = raw[0]
1816     return re.sub(re.escape(eatspace) + r' *', '', raw)
1817
1818
1819 def mcgen(code, **kwds):
1820     if code[0] == '\n':
1821         code = code[1:]
1822     return cgen(code, **kwds)
1823
1824
1825 def guardname(filename):
1826     return c_name(filename, protect=False).upper()
1827
1828
1829 def guardstart(name):
1830     return mcgen('''
1831
1832 #ifndef %(name)s
1833 #define %(name)s
1834
1835 ''',
1836                  name=guardname(name))
1837
1838
1839 def guardend(name):
1840     return mcgen('''
1841
1842 #endif /* %(name)s */
1843
1844 ''',
1845                  name=guardname(name))
1846
1847
1848 def gen_enum_lookup(name, values, prefix=None):
1849     ret = mcgen('''
1850
1851 const char *const %(c_name)s_lookup[] = {
1852 ''',
1853                 c_name=c_name(name))
1854     for value in values:
1855         index = c_enum_const(name, value, prefix)
1856         ret += mcgen('''
1857     [%(index)s] = "%(value)s",
1858 ''',
1859                      index=index, value=value)
1860
1861     max_index = c_enum_const(name, '_MAX', prefix)
1862     ret += mcgen('''
1863     [%(max_index)s] = NULL,
1864 };
1865 ''',
1866                  max_index=max_index)
1867     return ret
1868
1869
1870 def gen_enum(name, values, prefix=None):
1871     # append automatically generated _MAX value
1872     enum_values = values + ['_MAX']
1873
1874     ret = mcgen('''
1875
1876 typedef enum %(c_name)s {
1877 ''',
1878                 c_name=c_name(name))
1879
1880     i = 0
1881     for value in enum_values:
1882         ret += mcgen('''
1883     %(c_enum)s = %(i)d,
1884 ''',
1885                      c_enum=c_enum_const(name, value, prefix),
1886                      i=i)
1887         i += 1
1888
1889     ret += mcgen('''
1890 } %(c_name)s;
1891 ''',
1892                  c_name=c_name(name))
1893
1894     ret += mcgen('''
1895
1896 extern const char *const %(c_name)s_lookup[];
1897 ''',
1898                  c_name=c_name(name))
1899     return ret
1900
1901
1902 def gen_params(arg_type, boxed, extra):
1903     if not arg_type:
1904         assert not boxed
1905         return extra
1906     ret = ''
1907     sep = ''
1908     if boxed:
1909         ret += '%s arg' % arg_type.c_param_type()
1910         sep = ', '
1911     else:
1912         assert not arg_type.variants
1913         for memb in arg_type.members:
1914             ret += sep
1915             sep = ', '
1916             if memb.optional:
1917                 ret += 'bool has_%s, ' % c_name(memb.name)
1918             ret += '%s %s' % (memb.type.c_param_type(),
1919                               c_name(memb.name))
1920     if extra:
1921         ret += sep + extra
1922     return ret
1923
1924
1925 #
1926 # Common command line parsing
1927 #
1928
1929
1930 def parse_command_line(extra_options='', extra_long_options=[]):
1931
1932     try:
1933         opts, args = getopt.gnu_getopt(sys.argv[1:],
1934                                        'chp:o:' + extra_options,
1935                                        ['source', 'header', 'prefix=',
1936                                         'output-dir='] + extra_long_options)
1937     except getopt.GetoptError as err:
1938         print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
1939         sys.exit(1)
1940
1941     output_dir = ''
1942     prefix = ''
1943     do_c = False
1944     do_h = False
1945     extra_opts = []
1946
1947     for oa in opts:
1948         o, a = oa
1949         if o in ('-p', '--prefix'):
1950             match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1951             if match.end() != len(a):
1952                 print >>sys.stderr, \
1953                     "%s: 'funny character '%s' in argument of --prefix" \
1954                     % (sys.argv[0], a[match.end()])
1955                 sys.exit(1)
1956             prefix = a
1957         elif o in ('-o', '--output-dir'):
1958             output_dir = a + '/'
1959         elif o in ('-c', '--source'):
1960             do_c = True
1961         elif o in ('-h', '--header'):
1962             do_h = True
1963         else:
1964             extra_opts.append(oa)
1965
1966     if not do_c and not do_h:
1967         do_c = True
1968         do_h = True
1969
1970     if len(args) != 1:
1971         print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
1972         sys.exit(1)
1973     fname = args[0]
1974
1975     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
1976
1977 #
1978 # Generate output files with boilerplate
1979 #
1980
1981
1982 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
1983                 c_comment, h_comment):
1984     guard = guardname(prefix + h_file)
1985     c_file = output_dir + prefix + c_file
1986     h_file = output_dir + prefix + h_file
1987
1988     if output_dir:
1989         try:
1990             os.makedirs(output_dir)
1991         except os.error as e:
1992             if e.errno != errno.EEXIST:
1993                 raise
1994
1995     def maybe_open(really, name, opt):
1996         if really:
1997             return open(name, opt)
1998         else:
1999             import StringIO
2000             return StringIO.StringIO()
2001
2002     fdef = maybe_open(do_c, c_file, 'w')
2003     fdecl = maybe_open(do_h, h_file, 'w')
2004
2005     fdef.write(mcgen('''
2006 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2007 %(comment)s
2008 ''',
2009                      comment=c_comment))
2010
2011     fdecl.write(mcgen('''
2012 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2013 %(comment)s
2014 #ifndef %(guard)s
2015 #define %(guard)s
2016
2017 ''',
2018                       comment=h_comment, guard=guard))
2019
2020     return (fdef, fdecl)
2021
2022
2023 def close_output(fdef, fdecl):
2024     fdecl.write('''
2025 #endif
2026 ''')
2027     fdecl.close()
2028     fdef.close()