scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 import errno
  15 import getopt
  16 import os
  17 import re
  18 import string
  19 import sys
  20 from ordereddict import OrderedDict
  21
  22 builtin_types = {
  23     'str':      'QTYPE_QSTRING',
  24     'int':      'QTYPE_QINT',
  25     'number':   'QTYPE_QFLOAT',
  26     'bool':     'QTYPE_QBOOL',
  27     'int8':     'QTYPE_QINT',
  28     'int16':    'QTYPE_QINT',
  29     'int32':    'QTYPE_QINT',
  30     'int64':    'QTYPE_QINT',
  31     'uint8':    'QTYPE_QINT',
  32     'uint16':   'QTYPE_QINT',
  33     'uint32':   'QTYPE_QINT',
  34     'uint64':   'QTYPE_QINT',
  35     'size':     'QTYPE_QINT',
  36     'any':      None,           # any QType possible, actually
  37     'QType':    'QTYPE_QSTRING',
  38 }
  39
  40 # Are documentation comments required?
  41 doc_required = False
  42
  43 # Whitelist of commands allowed to return a non-dictionary
  44 returns_whitelist = []
  45
  46 # Whitelist of entities allowed to violate case conventions
  47 name_case_whitelist = []
  48
  49 enum_types = {}
  50 struct_types = {}
  51 union_types = {}
  52 all_names = {}
  53
  54 #
  55 # Parsing the schema into expressions
  56 #
  57
  58
  59 def error_path(parent):
  60     res = ''
  61     while parent:
  62         res = ('In file included from %s:%d:\n' % (parent['file'],
  63                                                    parent['line'])) + res
  64         parent = parent['parent']
  65     return res
  66
  67
  68 class QAPIError(Exception):
  69     def __init__(self, fname, line, col, incl_info, msg):
  70         Exception.__init__(self)
  71         self.fname = fname
  72         self.line = line
  73         self.col = col
  74         self.info = incl_info
  75         self.msg = msg
  76
  77     def __str__(self):
  78         loc = '%s:%d' % (self.fname, self.line)
  79         if self.col is not None:
  80             loc += ':%s' % self.col
  81         return error_path(self.info) + '%s: %s' % (loc, self.msg)
  82
  83
  84 class QAPIParseError(QAPIError):
  85     def __init__(self, parser, msg):
  86         col = 1
  87         for ch in parser.src[parser.line_pos:parser.pos]:
  88             if ch == '\t':
  89                 col = (col + 7) % 8 + 1
  90             else:
  91                 col += 1
  92         QAPIError.__init__(self, parser.fname, parser.line, col,
  93                            parser.incl_info, msg)
  94
  95
  96 class QAPISemError(QAPIError):
  97     def __init__(self, info, msg):
  98         QAPIError.__init__(self, info['file'], info['line'], None,
  99                            info['parent'], msg)
 100
 101
 102 class QAPIDoc(object):
 103     class Section(object):
 104         def __init__(self, name=None):
 105             # optional section name (argument/member or section name)
 106             self.name = name
 107             # the list of lines for this section
 108             self.content = []
 109
 110         def append(self, line):
 111             self.content.append(line)
 112
 113         def __repr__(self):
 114             return '\n'.join(self.content).strip()
 115
 116     class ArgSection(Section):
 117         def __init__(self, name):
 118             QAPIDoc.Section.__init__(self, name)
 119             self.member = None
 120
 121         def connect(self, member):
 122             self.member = member
 123
 124     def __init__(self, parser, info):
 125         # self.parser is used to report errors with QAPIParseError.  The
 126         # resulting error position depends on the state of the parser.
 127         # It happens to be the beginning of the comment.  More or less
 128         # servicable, but action at a distance.
 129         self.parser = parser
 130         self.info = info
 131         self.symbol = None
 132         self.body = QAPIDoc.Section()
 133         # dict mapping parameter name to ArgSection
 134         self.args = OrderedDict()
 135         # a list of Section
 136         self.sections = []
 137         # the current section
 138         self.section = self.body
 139
 140     def has_section(self, name):
 141         """Return True if we have a section with this name."""
 142         for i in self.sections:
 143             if i.name == name:
 144                 return True
 145         return False
 146
 147     def append(self, line):
 148         """Parse a comment line and add it to the documentation."""
 149         line = line[1:]
 150         if not line:
 151             self._append_freeform(line)
 152             return
 153
 154         if line[0] != ' ':
 155             raise QAPIParseError(self.parser, "Missing space after #")
 156         line = line[1:]
 157
 158         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 159         # recognized, and get silently treated as ordinary text
 160         if self.symbol:
 161             self._append_symbol_line(line)
 162         elif not self.body.content and line.startswith('@'):
 163             if not line.endswith(':'):
 164                 raise QAPIParseError(self.parser, "Line should end with :")
 165             self.symbol = line[1:-1]
 166             # FIXME invalid names other than the empty string aren't flagged
 167             if not self.symbol:
 168                 raise QAPIParseError(self.parser, "Invalid name")
 169         else:
 170             self._append_freeform(line)
 171
 172     def end_comment(self):
 173         self._end_section()
 174
 175     def _append_symbol_line(self, line):
 176         name = line.split(' ', 1)[0]
 177
 178         if name.startswith('@') and name.endswith(':'):
 179             line = line[len(name)+1:]
 180             self._start_args_section(name[1:-1])
 181         elif name in ('Returns:', 'Since:',
 182                       # those are often singular or plural
 183                       'Note:', 'Notes:',
 184                       'Example:', 'Examples:',
 185                       'TODO:'):
 186             line = line[len(name)+1:]
 187             self._start_section(name[:-1])
 188
 189         self._append_freeform(line)
 190
 191     def _start_args_section(self, name):
 192         # FIXME invalid names other than the empty string aren't flagged
 193         if not name:
 194             raise QAPIParseError(self.parser, "Invalid parameter name")
 195         if name in self.args:
 196             raise QAPIParseError(self.parser,
 197                                  "'%s' parameter name duplicated" % name)
 198         if self.sections:
 199             raise QAPIParseError(self.parser,
 200                                  "'@%s:' can't follow '%s' section"
 201                                  % (name, self.sections[0].name))
 202         self._end_section()
 203         self.section = QAPIDoc.ArgSection(name)
 204         self.args[name] = self.section
 205
 206     def _start_section(self, name=''):
 207         if name in ('Returns', 'Since') and self.has_section(name):
 208             raise QAPIParseError(self.parser,
 209                                  "Duplicated '%s' section" % name)
 210         self._end_section()
 211         self.section = QAPIDoc.Section(name)
 212         self.sections.append(self.section)
 213
 214     def _end_section(self):
 215         if self.section:
 216             contents = str(self.section)
 217             if self.section.name and (not contents or contents.isspace()):
 218                 raise QAPIParseError(self.parser, "Empty doc section '%s'"
 219                                      % self.section.name)
 220             self.section = None
 221
 222     def _append_freeform(self, line):
 223         in_arg = isinstance(self.section, QAPIDoc.ArgSection)
 224         if (in_arg and self.section.content
 225                 and not self.section.content[-1]
 226                 and line and not line[0].isspace()):
 227             self._start_section()
 228         if (in_arg or not self.section.name
 229                 or not self.section.name.startswith('Example')):
 230             line = line.strip()
 231         match = re.match(r'(@\S+:)', line)
 232         if match:
 233             raise QAPIParseError(self.parser,
 234                                  "'%s' not allowed in free-form documentation"
 235                                  % match.group(1))
 236         # TODO Drop this once the dust has settled
 237         if (isinstance(self.section, QAPIDoc.ArgSection)
 238                 and '#optional' in line):
 239             raise QAPISemError(self.info, "Please drop the #optional tag")
 240         self.section.append(line)
 241
 242     def connect_member(self, member):
 243         if member.name not in self.args:
 244             # Undocumented TODO outlaw
 245             self.args[member.name] = QAPIDoc.ArgSection(member.name)
 246         self.args[member.name].connect(member)
 247
 248     def check_expr(self, expr):
 249         if self.has_section('Returns') and 'command' not in expr:
 250             raise QAPISemError(self.info,
 251                                "'Returns:' is only valid for commands")
 252
 253     def check(self):
 254         bogus = [name for name, section in self.args.iteritems()
 255                  if not section.member]
 256         if bogus:
 257             raise QAPISemError(
 258                 self.info,
 259                 "The following documented members are not in "
 260                 "the declaration: %s" % ", ".join(bogus))
 261
 262
 263 class QAPISchemaParser(object):
 264
 265     def __init__(self, fp, previously_included=[], incl_info=None):
 266         abs_fname = os.path.abspath(fp.name)
 267         fname = fp.name
 268         self.fname = fname
 269         previously_included.append(abs_fname)
 270         self.incl_info = incl_info
 271         self.src = fp.read()
 272         if self.src == '' or self.src[-1] != '\n':
 273             self.src += '\n'
 274         self.cursor = 0
 275         self.line = 1
 276         self.line_pos = 0
 277         self.exprs = []
 278         self.docs = []
 279         self.cur_doc = None
 280         self.accept()
 281
 282         while self.tok is not None:
 283             info = {'file': fname, 'line': self.line,
 284                     'parent': self.incl_info}
 285             if self.tok == '#':
 286                 self.reject_expr_doc()
 287                 self.cur_doc = self.get_doc(info)
 288                 self.docs.append(self.cur_doc)
 289                 continue
 290
 291             expr = self.get_expr(False)
 292             if 'include' in expr:
 293                 self.reject_expr_doc()
 294                 if len(expr) != 1:
 295                     raise QAPISemError(info, "Invalid 'include' directive")
 296                 include = expr['include']
 297                 if not isinstance(include, str):
 298                     raise QAPISemError(info,
 299                                        "Value of 'include' must be a string")
 300                 self._include(include, info, os.path.dirname(abs_fname),
 301                               previously_included)
 302             elif "pragma" in expr:
 303                 self.reject_expr_doc()
 304                 if len(expr) != 1:
 305                     raise QAPISemError(info, "Invalid 'pragma' directive")
 306                 pragma = expr['pragma']
 307                 if not isinstance(pragma, dict):
 308                     raise QAPISemError(
 309                         info, "Value of 'pragma' must be a dictionary")
 310                 for name, value in pragma.iteritems():
 311                     self._pragma(name, value, info)
 312             else:
 313                 expr_elem = {'expr': expr,
 314                              'info': info}
 315                 if self.cur_doc:
 316                     if not self.cur_doc.symbol:
 317                         raise QAPISemError(
 318                             self.cur_doc.info,
 319                             "Expression documentation required")
 320                     expr_elem['doc'] = self.cur_doc
 321                 self.exprs.append(expr_elem)
 322             self.cur_doc = None
 323         self.reject_expr_doc()
 324
 325     def reject_expr_doc(self):
 326         if self.cur_doc and self.cur_doc.symbol:
 327             raise QAPISemError(
 328                 self.cur_doc.info,
 329                 "Documentation for '%s' is not followed by the definition"
 330                 % self.cur_doc.symbol)
 331
 332     def _include(self, include, info, base_dir, previously_included):
 333         incl_abs_fname = os.path.join(base_dir, include)
 334         # catch inclusion cycle
 335         inf = info
 336         while inf:
 337             if incl_abs_fname == os.path.abspath(inf['file']):
 338                 raise QAPISemError(info, "Inclusion loop for %s" % include)
 339             inf = inf['parent']
 340
 341         # skip multiple include of the same file
 342         if incl_abs_fname in previously_included:
 343             return
 344         try:
 345             fobj = open(incl_abs_fname, 'r')
 346         except IOError as e:
 347             raise QAPISemError(info, '%s: %s' % (e.strerror, include))
 348         exprs_include = QAPISchemaParser(fobj, previously_included, info)
 349         self.exprs.extend(exprs_include.exprs)
 350         self.docs.extend(exprs_include.docs)
 351
 352     def _pragma(self, name, value, info):
 353         global doc_required, returns_whitelist, name_case_whitelist
 354         if name == 'doc-required':
 355             if not isinstance(value, bool):
 356                 raise QAPISemError(info,
 357                                    "Pragma 'doc-required' must be boolean")
 358             doc_required = value
 359         elif name == 'returns-whitelist':
 360             if (not isinstance(value, list)
 361                     or any([not isinstance(elt, str) for elt in value])):
 362                 raise QAPISemError(info,
 363                                    "Pragma returns-whitelist must be"
 364                                    " a list of strings")
 365             returns_whitelist = value
 366         elif name == 'name-case-whitelist':
 367             if (not isinstance(value, list)
 368                     or any([not isinstance(elt, str) for elt in value])):
 369                 raise QAPISemError(info,
 370                                    "Pragma name-case-whitelist must be"
 371                                    " a list of strings")
 372             name_case_whitelist = value
 373         else:
 374             raise QAPISemError(info, "Unknown pragma '%s'" % name)
 375
 376     def accept(self, skip_comment=True):
 377         while True:
 378             self.tok = self.src[self.cursor]
 379             self.pos = self.cursor
 380             self.cursor += 1
 381             self.val = None
 382
 383             if self.tok == '#':
 384                 if self.src[self.cursor] == '#':
 385                     # Start of doc comment
 386                     skip_comment = False
 387                 self.cursor = self.src.find('\n', self.cursor)
 388                 if not skip_comment:
 389                     self.val = self.src[self.pos:self.cursor]
 390                     return
 391             elif self.tok in '{}:,[]':
 392                 return
 393             elif self.tok == "'":
 394                 string = ''
 395                 esc = False
 396                 while True:
 397                     ch = self.src[self.cursor]
 398                     self.cursor += 1
 399                     if ch == '\n':
 400                         raise QAPIParseError(self, 'Missing terminating "\'"')
 401                     if esc:
 402                         if ch == 'b':
 403                             string += '\b'
 404                         elif ch == 'f':
 405                             string += '\f'
 406                         elif ch == 'n':
 407                             string += '\n'
 408                         elif ch == 'r':
 409                             string += '\r'
 410                         elif ch == 't':
 411                             string += '\t'
 412                         elif ch == 'u':
 413                             value = 0
 414                             for _ in range(0, 4):
 415                                 ch = self.src[self.cursor]
 416                                 self.cursor += 1
 417                                 if ch not in '0123456789abcdefABCDEF':
 418                                     raise QAPIParseError(self,
 419                                                          '\\u escape needs 4 '
 420                                                          'hex digits')
 421                                 value = (value << 4) + int(ch, 16)
 422                             # If Python 2 and 3 didn't disagree so much on
 423                             # how to handle Unicode, then we could allow
 424                             # Unicode string defaults.  But most of QAPI is
 425                             # ASCII-only, so we aren't losing much for now.
 426                             if not value or value > 0x7f:
 427                                 raise QAPIParseError(self,
 428                                                      'For now, \\u escape '
 429                                                      'only supports non-zero '
 430                                                      'values up to \\u007f')
 431                             string += chr(value)
 432                         elif ch in '\\/\'"':
 433                             string += ch
 434                         else:
 435                             raise QAPIParseError(self,
 436                                                  "Unknown escape \\%s" % ch)
 437                         esc = False
 438                     elif ch == '\\':
 439                         esc = True
 440                     elif ch == "'":
 441                         self.val = string
 442                         return
 443                     else:
 444                         string += ch
 445             elif self.src.startswith('true', self.pos):
 446                 self.val = True
 447                 self.cursor += 3
 448                 return
 449             elif self.src.startswith('false', self.pos):
 450                 self.val = False
 451                 self.cursor += 4
 452                 return
 453             elif self.src.startswith('null', self.pos):
 454                 self.val = None
 455                 self.cursor += 3
 456                 return
 457             elif self.tok == '\n':
 458                 if self.cursor == len(self.src):
 459                     self.tok = None
 460                     return
 461                 self.line += 1
 462                 self.line_pos = self.cursor
 463             elif not self.tok.isspace():
 464                 raise QAPIParseError(self, 'Stray "%s"' % self.tok)
 465
 466     def get_members(self):
 467         expr = OrderedDict()
 468         if self.tok == '}':
 469             self.accept()
 470             return expr
 471         if self.tok != "'":
 472             raise QAPIParseError(self, 'Expected string or "}"')
 473         while True:
 474             key = self.val
 475             self.accept()
 476             if self.tok != ':':
 477                 raise QAPIParseError(self, 'Expected ":"')
 478             self.accept()
 479             if key in expr:
 480                 raise QAPIParseError(self, 'Duplicate key "%s"' % key)
 481             expr[key] = self.get_expr(True)
 482             if self.tok == '}':
 483                 self.accept()
 484                 return expr
 485             if self.tok != ',':
 486                 raise QAPIParseError(self, 'Expected "," or "}"')
 487             self.accept()
 488             if self.tok != "'":
 489                 raise QAPIParseError(self, 'Expected string')
 490
 491     def get_values(self):
 492         expr = []
 493         if self.tok == ']':
 494             self.accept()
 495             return expr
 496         if self.tok not in "{['tfn":
 497             raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
 498                                  'boolean or "null"')
 499         while True:
 500             expr.append(self.get_expr(True))
 501             if self.tok == ']':
 502                 self.accept()
 503                 return expr
 504             if self.tok != ',':
 505                 raise QAPIParseError(self, 'Expected "," or "]"')
 506             self.accept()
 507
 508     def get_expr(self, nested):
 509         if self.tok != '{' and not nested:
 510             raise QAPIParseError(self, 'Expected "{"')
 511         if self.tok == '{':
 512             self.accept()
 513             expr = self.get_members()
 514         elif self.tok == '[':
 515             self.accept()
 516             expr = self.get_values()
 517         elif self.tok in "'tfn":
 518             expr = self.val
 519             self.accept()
 520         else:
 521             raise QAPIParseError(self, 'Expected "{", "[", string, '
 522                                  'boolean or "null"')
 523         return expr
 524
 525     def get_doc(self, info):
 526         if self.val != '##':
 527             raise QAPIParseError(self, "Junk after '##' at start of "
 528                                  "documentation comment")
 529
 530         doc = QAPIDoc(self, info)
 531         self.accept(False)
 532         while self.tok == '#':
 533             if self.val.startswith('##'):
 534                 # End of doc comment
 535                 if self.val != '##':
 536                     raise QAPIParseError(self, "Junk after '##' at end of "
 537                                          "documentation comment")
 538                 doc.end_comment()
 539                 self.accept()
 540                 return doc
 541             else:
 542                 doc.append(self.val)
 543             self.accept(False)
 544
 545         raise QAPIParseError(self, "Documentation comment must end with '##'")
 546
 547
 548 #
 549 # Semantic analysis of schema expressions
 550 # TODO fold into QAPISchema
 551 # TODO catching name collisions in generated code would be nice
 552 #
 553
 554
 555 def find_base_members(base):
 556     if isinstance(base, dict):
 557         return base
 558     base_struct_define = struct_types.get(base)
 559     if not base_struct_define:
 560         return None
 561     return base_struct_define['data']
 562
 563
 564 # Return the qtype of an alternate branch, or None on error.
 565 def find_alternate_member_qtype(qapi_type):
 566     if qapi_type in builtin_types:
 567         return builtin_types[qapi_type]
 568     elif qapi_type in struct_types:
 569         return 'QTYPE_QDICT'
 570     elif qapi_type in enum_types:
 571         return 'QTYPE_QSTRING'
 572     elif qapi_type in union_types:
 573         return 'QTYPE_QDICT'
 574     return None
 575
 576
 577 # Return the discriminator enum define if discriminator is specified as an
 578 # enum type, otherwise return None.
 579 def discriminator_find_enum_define(expr):
 580     base = expr.get('base')
 581     discriminator = expr.get('discriminator')
 582
 583     if not (discriminator and base):
 584         return None
 585
 586     base_members = find_base_members(base)
 587     if not base_members:
 588         return None
 589
 590     discriminator_type = base_members.get(discriminator)
 591     if not discriminator_type:
 592         return None
 593
 594     return enum_types.get(discriminator_type)
 595
 596
 597 # Names must be letters, numbers, -, and _.  They must start with letter,
 598 # except for downstream extensions which must start with __RFQDN_.
 599 # Dots are only valid in the downstream extension prefix.
 600 valid_name = re.compile(r'^(__[a-zA-Z0-9.-]+_)?'
 601                         '[a-zA-Z][a-zA-Z0-9_-]*$')
 602
 603
 604 def check_name(info, source, name, allow_optional=False,
 605                enum_member=False):
 606     global valid_name
 607     membername = name
 608
 609     if not isinstance(name, str):
 610         raise QAPISemError(info, "%s requires a string name" % source)
 611     if name.startswith('*'):
 612         membername = name[1:]
 613         if not allow_optional:
 614             raise QAPISemError(info, "%s does not allow optional name '%s'"
 615                                % (source, name))
 616     # Enum members can start with a digit, because the generated C
 617     # code always prefixes it with the enum name
 618     if enum_member and membername[0].isdigit():
 619         membername = 'D' + membername
 620     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
 621     # and 'q_obj_*' implicit type names.
 622     if not valid_name.match(membername) or \
 623        c_name(membername, False).startswith('q_'):
 624         raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
 625
 626
 627 def add_name(name, info, meta, implicit=False):
 628     global all_names
 629     check_name(info, "'%s'" % meta, name)
 630     # FIXME should reject names that differ only in '_' vs. '.'
 631     # vs. '-', because they're liable to clash in generated C.
 632     if name in all_names:
 633         raise QAPISemError(info, "%s '%s' is already defined"
 634                            % (all_names[name], name))
 635     if not implicit and (name.endswith('Kind') or name.endswith('List')):
 636         raise QAPISemError(info, "%s '%s' should not end in '%s'"
 637                            % (meta, name, name[-4:]))
 638     all_names[name] = meta
 639
 640
 641 def check_type(info, source, value, allow_array=False,
 642                allow_dict=False, allow_optional=False,
 643                allow_metas=[]):
 644     global all_names
 645
 646     if value is None:
 647         return
 648
 649     # Check if array type for value is okay
 650     if isinstance(value, list):
 651         if not allow_array:
 652             raise QAPISemError(info, "%s cannot be an array" % source)
 653         if len(value) != 1 or not isinstance(value[0], str):
 654             raise QAPISemError(info,
 655                                "%s: array type must contain single type name" %
 656                                source)
 657         value = value[0]
 658
 659     # Check if type name for value is okay
 660     if isinstance(value, str):
 661         if value not in all_names:
 662             raise QAPISemError(info, "%s uses unknown type '%s'"
 663                                % (source, value))
 664         if not all_names[value] in allow_metas:
 665             raise QAPISemError(info, "%s cannot use %s type '%s'" %
 666                                (source, all_names[value], value))
 667         return
 668
 669     if not allow_dict:
 670         raise QAPISemError(info, "%s should be a type name" % source)
 671
 672     if not isinstance(value, OrderedDict):
 673         raise QAPISemError(info,
 674                            "%s should be a dictionary or type name" % source)
 675
 676     # value is a dictionary, check that each member is okay
 677     for (key, arg) in value.items():
 678         check_name(info, "Member of %s" % source, key,
 679                    allow_optional=allow_optional)
 680         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 681             raise QAPISemError(info, "Member of %s uses reserved name '%s'"
 682                                % (source, key))
 683         # Todo: allow dictionaries to represent default values of
 684         # an optional argument.
 685         check_type(info, "Member '%s' of %s" % (key, source), arg,
 686                    allow_array=True,
 687                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 688                                 'enum'])
 689
 690
 691 def check_command(expr, info):
 692     name = expr['command']
 693     boxed = expr.get('boxed', False)
 694
 695     args_meta = ['struct']
 696     if boxed:
 697         args_meta += ['union', 'alternate']
 698     check_type(info, "'data' for command '%s'" % name,
 699                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 700                allow_metas=args_meta)
 701     returns_meta = ['union', 'struct']
 702     if name in returns_whitelist:
 703         returns_meta += ['built-in', 'alternate', 'enum']
 704     check_type(info, "'returns' for command '%s'" % name,
 705                expr.get('returns'), allow_array=True,
 706                allow_optional=True, allow_metas=returns_meta)
 707
 708
 709 def check_event(expr, info):
 710     name = expr['event']
 711     boxed = expr.get('boxed', False)
 712
 713     meta = ['struct']
 714     if boxed:
 715         meta += ['union', 'alternate']
 716     check_type(info, "'data' for event '%s'" % name,
 717                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 718                allow_metas=meta)
 719
 720
 721 def check_union(expr, info):
 722     name = expr['union']
 723     base = expr.get('base')
 724     discriminator = expr.get('discriminator')
 725     members = expr['data']
 726
 727     # Two types of unions, determined by discriminator.
 728
 729     # With no discriminator it is a simple union.
 730     if discriminator is None:
 731         enum_define = None
 732         allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
 733         if base is not None:
 734             raise QAPISemError(info, "Simple union '%s' must not have a base" %
 735                                name)
 736
 737     # Else, it's a flat union.
 738     else:
 739         # The object must have a string or dictionary 'base'.
 740         check_type(info, "'base' for union '%s'" % name,
 741                    base, allow_dict=True, allow_optional=True,
 742                    allow_metas=['struct'])
 743         if not base:
 744             raise QAPISemError(info, "Flat union '%s' must have a base"
 745                                % name)
 746         base_members = find_base_members(base)
 747         assert base_members is not None
 748
 749         # The value of member 'discriminator' must name a non-optional
 750         # member of the base struct.
 751         check_name(info, "Discriminator of flat union '%s'" % name,
 752                    discriminator)
 753         discriminator_type = base_members.get(discriminator)
 754         if not discriminator_type:
 755             raise QAPISemError(info,
 756                                "Discriminator '%s' is not a member of base "
 757                                "struct '%s'"
 758                                % (discriminator, base))
 759         enum_define = enum_types.get(discriminator_type)
 760         allow_metas = ['struct']
 761         # Do not allow string discriminator
 762         if not enum_define:
 763             raise QAPISemError(info,
 764                                "Discriminator '%s' must be of enumeration "
 765                                "type" % discriminator)
 766
 767     # Check every branch; don't allow an empty union
 768     if len(members) == 0:
 769         raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
 770     for (key, value) in members.items():
 771         check_name(info, "Member of union '%s'" % name, key)
 772
 773         # Each value must name a known type
 774         check_type(info, "Member '%s' of union '%s'" % (key, name),
 775                    value, allow_array=not base, allow_metas=allow_metas)
 776
 777         # If the discriminator names an enum type, then all members
 778         # of 'data' must also be members of the enum type.
 779         if enum_define:
 780             if key not in enum_define['data']:
 781                 raise QAPISemError(info,
 782                                    "Discriminator value '%s' is not found in "
 783                                    "enum '%s'"
 784                                    % (key, enum_define['enum']))
 785
 786     # If discriminator is user-defined, ensure all values are covered
 787     if enum_define:
 788         for value in enum_define['data']:
 789             if value not in members.keys():
 790                 raise QAPISemError(info, "Union '%s' data missing '%s' branch"
 791                                    % (name, value))
 792
 793
 794 def check_alternate(expr, info):
 795     name = expr['alternate']
 796     members = expr['data']
 797     types_seen = {}
 798
 799     # Check every branch; require at least two branches
 800     if len(members) < 2:
 801         raise QAPISemError(info,
 802                            "Alternate '%s' should have at least two branches "
 803                            "in 'data'" % name)
 804     for (key, value) in members.items():
 805         check_name(info, "Member of alternate '%s'" % name, key)
 806
 807         # Ensure alternates have no type conflicts.
 808         check_type(info, "Member '%s' of alternate '%s'" % (key, name),
 809                    value,
 810                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 811         qtype = find_alternate_member_qtype(value)
 812         if not qtype:
 813             raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
 814                                "type '%s'" % (name, key, value))
 815         if qtype in types_seen:
 816             raise QAPISemError(info, "Alternate '%s' member '%s' can't "
 817                                "be distinguished from member '%s'"
 818                                % (name, key, types_seen[qtype]))
 819         types_seen[qtype] = key
 820
 821
 822 def check_enum(expr, info):
 823     name = expr['enum']
 824     members = expr.get('data')
 825     prefix = expr.get('prefix')
 826
 827     if not isinstance(members, list):
 828         raise QAPISemError(info,
 829                            "Enum '%s' requires an array for 'data'" % name)
 830     if prefix is not None and not isinstance(prefix, str):
 831         raise QAPISemError(info,
 832                            "Enum '%s' requires a string for 'prefix'" % name)
 833     for member in members:
 834         check_name(info, "Member of enum '%s'" % name, member,
 835                    enum_member=True)
 836
 837
 838 def check_struct(expr, info):
 839     name = expr['struct']
 840     members = expr['data']
 841
 842     check_type(info, "'data' for struct '%s'" % name, members,
 843                allow_dict=True, allow_optional=True)
 844     check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
 845                allow_metas=['struct'])
 846
 847
 848 def check_keys(expr_elem, meta, required, optional=[]):
 849     expr = expr_elem['expr']
 850     info = expr_elem['info']
 851     name = expr[meta]
 852     if not isinstance(name, str):
 853         raise QAPISemError(info, "'%s' key must have a string value" % meta)
 854     required = required + [meta]
 855     for (key, value) in expr.items():
 856         if key not in required and key not in optional:
 857             raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
 858                                % (key, meta, name))
 859         if (key == 'gen' or key == 'success-response') and value is not False:
 860             raise QAPISemError(info,
 861                                "'%s' of %s '%s' should only use false value"
 862                                % (key, meta, name))
 863         if key == 'boxed' and value is not True:
 864             raise QAPISemError(info,
 865                                "'%s' of %s '%s' should only use true value"
 866                                % (key, meta, name))
 867     for key in required:
 868         if key not in expr:
 869             raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
 870                                % (key, meta, name))
 871
 872
 873 def check_exprs(exprs):
 874     global all_names
 875
 876     # Populate name table with names of built-in types
 877     for builtin in builtin_types.keys():
 878         all_names[builtin] = 'built-in'
 879
 880     # Learn the types and check for valid expression keys
 881     for expr_elem in exprs:
 882         expr = expr_elem['expr']
 883         info = expr_elem['info']
 884         doc = expr_elem.get('doc')
 885
 886         if not doc and doc_required:
 887             raise QAPISemError(info,
 888                                "Expression missing documentation comment")
 889
 890         if 'enum' in expr:
 891             meta = 'enum'
 892             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 893             enum_types[expr[meta]] = expr
 894         elif 'union' in expr:
 895             meta = 'union'
 896             check_keys(expr_elem, 'union', ['data'],
 897                        ['base', 'discriminator'])
 898             union_types[expr[meta]] = expr
 899         elif 'alternate' in expr:
 900             meta = 'alternate'
 901             check_keys(expr_elem, 'alternate', ['data'])
 902         elif 'struct' in expr:
 903             meta = 'struct'
 904             check_keys(expr_elem, 'struct', ['data'], ['base'])
 905             struct_types[expr[meta]] = expr
 906         elif 'command' in expr:
 907             meta = 'command'
 908             check_keys(expr_elem, 'command', [],
 909                        ['data', 'returns', 'gen', 'success-response', 'boxed'])
 910         elif 'event' in expr:
 911             meta = 'event'
 912             check_keys(expr_elem, 'event', [], ['data', 'boxed'])
 913         else:
 914             raise QAPISemError(expr_elem['info'],
 915                                "Expression is missing metatype")
 916         name = expr[meta]
 917         add_name(name, info, meta)
 918         if doc and doc.symbol != name:
 919             raise QAPISemError(info, "Definition of '%s' follows documentation"
 920                                " for '%s'" % (name, doc.symbol))
 921
 922     # Try again for hidden UnionKind enum
 923     for expr_elem in exprs:
 924         expr = expr_elem['expr']
 925         if 'union' in expr and not discriminator_find_enum_define(expr):
 926             name = '%sKind' % expr['union']
 927         elif 'alternate' in expr:
 928             name = '%sKind' % expr['alternate']
 929         else:
 930             continue
 931         enum_types[name] = {'enum': name}
 932         add_name(name, info, 'enum', implicit=True)
 933
 934     # Validate that exprs make sense
 935     for expr_elem in exprs:
 936         expr = expr_elem['expr']
 937         info = expr_elem['info']
 938         doc = expr_elem.get('doc')
 939
 940         if 'enum' in expr:
 941             check_enum(expr, info)
 942         elif 'union' in expr:
 943             check_union(expr, info)
 944         elif 'alternate' in expr:
 945             check_alternate(expr, info)
 946         elif 'struct' in expr:
 947             check_struct(expr, info)
 948         elif 'command' in expr:
 949             check_command(expr, info)
 950         elif 'event' in expr:
 951             check_event(expr, info)
 952         else:
 953             assert False, 'unexpected meta type'
 954
 955         if doc:
 956             doc.check_expr(expr)
 957
 958     return exprs
 959
 960
 961 #
 962 # Schema compiler frontend
 963 #
 964
 965 class QAPISchemaEntity(object):
 966     def __init__(self, name, info, doc):
 967         assert isinstance(name, str)
 968         self.name = name
 969         # For explicitly defined entities, info points to the (explicit)
 970         # definition.  For builtins (and their arrays), info is None.
 971         # For implicitly defined entities, info points to a place that
 972         # triggered the implicit definition (there may be more than one
 973         # such place).
 974         self.info = info
 975         self.doc = doc
 976
 977     def c_name(self):
 978         return c_name(self.name)
 979
 980     def check(self, schema):
 981         pass
 982
 983     def is_implicit(self):
 984         return not self.info
 985
 986     def visit(self, visitor):
 987         pass
 988
 989
 990 class QAPISchemaVisitor(object):
 991     def visit_begin(self, schema):
 992         pass
 993
 994     def visit_end(self):
 995         pass
 996
 997     def visit_needed(self, entity):
 998         # Default to visiting everything
 999         return True
1000
1001     def visit_builtin_type(self, name, info, json_type):
1002         pass
1003
1004     def visit_enum_type(self, name, info, values, prefix):
1005         pass
1006
1007     def visit_array_type(self, name, info, element_type):
1008         pass
1009
1010     def visit_object_type(self, name, info, base, members, variants):
1011         pass
1012
1013     def visit_object_type_flat(self, name, info, members, variants):
1014         pass
1015
1016     def visit_alternate_type(self, name, info, variants):
1017         pass
1018
1019     def visit_command(self, name, info, arg_type, ret_type,
1020                       gen, success_response, boxed):
1021         pass
1022
1023     def visit_event(self, name, info, arg_type, boxed):
1024         pass
1025
1026
1027 class QAPISchemaType(QAPISchemaEntity):
1028     # Return the C type for common use.
1029     # For the types we commonly box, this is a pointer type.
1030     def c_type(self):
1031         pass
1032
1033     # Return the C type to be used in a parameter list.
1034     def c_param_type(self):
1035         return self.c_type()
1036
1037     # Return the C type to be used where we suppress boxing.
1038     def c_unboxed_type(self):
1039         return self.c_type()
1040
1041     def json_type(self):
1042         pass
1043
1044     def alternate_qtype(self):
1045         json2qtype = {
1046             'string':  'QTYPE_QSTRING',
1047             'number':  'QTYPE_QFLOAT',
1048             'int':     'QTYPE_QINT',
1049             'boolean': 'QTYPE_QBOOL',
1050             'object':  'QTYPE_QDICT'
1051         }
1052         return json2qtype.get(self.json_type())
1053
1054     def doc_type(self):
1055         if self.is_implicit():
1056             return None
1057         return self.name
1058
1059
1060 class QAPISchemaBuiltinType(QAPISchemaType):
1061     def __init__(self, name, json_type, c_type):
1062         QAPISchemaType.__init__(self, name, None, None)
1063         assert not c_type or isinstance(c_type, str)
1064         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
1065                              'value')
1066         self._json_type_name = json_type
1067         self._c_type_name = c_type
1068
1069     def c_name(self):
1070         return self.name
1071
1072     def c_type(self):
1073         return self._c_type_name
1074
1075     def c_param_type(self):
1076         if self.name == 'str':
1077             return 'const ' + self._c_type_name
1078         return self._c_type_name
1079
1080     def json_type(self):
1081         return self._json_type_name
1082
1083     def doc_type(self):
1084         return self.json_type()
1085
1086     def visit(self, visitor):
1087         visitor.visit_builtin_type(self.name, self.info, self.json_type())
1088
1089
1090 class QAPISchemaEnumType(QAPISchemaType):
1091     def __init__(self, name, info, doc, values, prefix):
1092         QAPISchemaType.__init__(self, name, info, doc)
1093         for v in values:
1094             assert isinstance(v, QAPISchemaMember)
1095             v.set_owner(name)
1096         assert prefix is None or isinstance(prefix, str)
1097         self.values = values
1098         self.prefix = prefix
1099
1100     def check(self, schema):
1101         seen = {}
1102         for v in self.values:
1103             v.check_clash(self.info, seen)
1104             if self.doc:
1105                 self.doc.connect_member(v)
1106
1107     def is_implicit(self):
1108         # See QAPISchema._make_implicit_enum_type() and ._def_predefineds()
1109         return self.name.endswith('Kind') or self.name == 'QType'
1110
1111     def c_type(self):
1112         return c_name(self.name)
1113
1114     def member_names(self):
1115         return [v.name for v in self.values]
1116
1117     def json_type(self):
1118         return 'string'
1119
1120     def visit(self, visitor):
1121         visitor.visit_enum_type(self.name, self.info,
1122                                 self.member_names(), self.prefix)
1123
1124
1125 class QAPISchemaArrayType(QAPISchemaType):
1126     def __init__(self, name, info, element_type):
1127         QAPISchemaType.__init__(self, name, info, None)
1128         assert isinstance(element_type, str)
1129         self._element_type_name = element_type
1130         self.element_type = None
1131
1132     def check(self, schema):
1133         self.element_type = schema.lookup_type(self._element_type_name)
1134         assert self.element_type
1135
1136     def is_implicit(self):
1137         return True
1138
1139     def c_type(self):
1140         return c_name(self.name) + pointer_suffix
1141
1142     def json_type(self):
1143         return 'array'
1144
1145     def doc_type(self):
1146         elt_doc_type = self.element_type.doc_type()
1147         if not elt_doc_type:
1148             return None
1149         return 'array of ' + elt_doc_type
1150
1151     def visit(self, visitor):
1152         visitor.visit_array_type(self.name, self.info, self.element_type)
1153
1154
1155 class QAPISchemaObjectType(QAPISchemaType):
1156     def __init__(self, name, info, doc, base, local_members, variants):
1157         # struct has local_members, optional base, and no variants
1158         # flat union has base, variants, and no local_members
1159         # simple union has local_members, variants, and no base
1160         QAPISchemaType.__init__(self, name, info, doc)
1161         assert base is None or isinstance(base, str)
1162         for m in local_members:
1163             assert isinstance(m, QAPISchemaObjectTypeMember)
1164             m.set_owner(name)
1165         if variants is not None:
1166             assert isinstance(variants, QAPISchemaObjectTypeVariants)
1167             variants.set_owner(name)
1168         self._base_name = base
1169         self.base = None
1170         self.local_members = local_members
1171         self.variants = variants
1172         self.members = None
1173
1174     def check(self, schema):
1175         if self.members is False:               # check for cycles
1176             raise QAPISemError(self.info,
1177                                "Object %s contains itself" % self.name)
1178         if self.members:
1179             return
1180         self.members = False                    # mark as being checked
1181         seen = OrderedDict()
1182         if self._base_name:
1183             self.base = schema.lookup_type(self._base_name)
1184             assert isinstance(self.base, QAPISchemaObjectType)
1185             self.base.check(schema)
1186             self.base.check_clash(self.info, seen)
1187         for m in self.local_members:
1188             m.check(schema)
1189             m.check_clash(self.info, seen)
1190             if self.doc:
1191                 self.doc.connect_member(m)
1192         self.members = seen.values()
1193         if self.variants:
1194             self.variants.check(schema, seen)
1195             assert self.variants.tag_member in self.members
1196             self.variants.check_clash(self.info, seen)
1197         if self.doc:
1198             self.doc.check()
1199
1200     # Check that the members of this type do not cause duplicate JSON members,
1201     # and update seen to track the members seen so far. Report any errors
1202     # on behalf of info, which is not necessarily self.info
1203     def check_clash(self, info, seen):
1204         assert not self.variants       # not implemented
1205         for m in self.members:
1206             m.check_clash(info, seen)
1207
1208     def is_implicit(self):
1209         # See QAPISchema._make_implicit_object_type(), as well as
1210         # _def_predefineds()
1211         return self.name.startswith('q_')
1212
1213     def is_empty(self):
1214         assert self.members is not None
1215         return not self.members and not self.variants
1216
1217     def c_name(self):
1218         assert self.name != 'q_empty'
1219         return QAPISchemaType.c_name(self)
1220
1221     def c_type(self):
1222         assert not self.is_implicit()
1223         return c_name(self.name) + pointer_suffix
1224
1225     def c_unboxed_type(self):
1226         return c_name(self.name)
1227
1228     def json_type(self):
1229         return 'object'
1230
1231     def visit(self, visitor):
1232         visitor.visit_object_type(self.name, self.info,
1233                                   self.base, self.local_members, self.variants)
1234         visitor.visit_object_type_flat(self.name, self.info,
1235                                        self.members, self.variants)
1236
1237
1238 class QAPISchemaMember(object):
1239     role = 'member'
1240
1241     def __init__(self, name):
1242         assert isinstance(name, str)
1243         self.name = name
1244         self.owner = None
1245
1246     def set_owner(self, name):
1247         assert not self.owner
1248         self.owner = name
1249
1250     def check_clash(self, info, seen):
1251         cname = c_name(self.name)
1252         if cname.lower() != cname and self.owner not in name_case_whitelist:
1253             raise QAPISemError(info,
1254                                "%s should not use uppercase" % self.describe())
1255         if cname in seen:
1256             raise QAPISemError(info, "%s collides with %s" %
1257                                (self.describe(), seen[cname].describe()))
1258         seen[cname] = self
1259
1260     def _pretty_owner(self):
1261         owner = self.owner
1262         if owner.startswith('q_obj_'):
1263             # See QAPISchema._make_implicit_object_type() - reverse the
1264             # mapping there to create a nice human-readable description
1265             owner = owner[6:]
1266             if owner.endswith('-arg'):
1267                 return '(parameter of %s)' % owner[:-4]
1268             elif owner.endswith('-base'):
1269                 return '(base of %s)' % owner[:-5]
1270             else:
1271                 assert owner.endswith('-wrapper')
1272                 # Unreachable and not implemented
1273                 assert False
1274         if owner.endswith('Kind'):
1275             # See QAPISchema._make_implicit_enum_type()
1276             return '(branch of %s)' % owner[:-4]
1277         return '(%s of %s)' % (self.role, owner)
1278
1279     def describe(self):
1280         return "'%s' %s" % (self.name, self._pretty_owner())
1281
1282
1283 class QAPISchemaObjectTypeMember(QAPISchemaMember):
1284     def __init__(self, name, typ, optional):
1285         QAPISchemaMember.__init__(self, name)
1286         assert isinstance(typ, str)
1287         assert isinstance(optional, bool)
1288         self._type_name = typ
1289         self.type = None
1290         self.optional = optional
1291
1292     def check(self, schema):
1293         assert self.owner
1294         self.type = schema.lookup_type(self._type_name)
1295         assert self.type
1296
1297
1298 class QAPISchemaObjectTypeVariants(object):
1299     def __init__(self, tag_name, tag_member, variants):
1300         # Flat unions pass tag_name but not tag_member.
1301         # Simple unions and alternates pass tag_member but not tag_name.
1302         # After check(), tag_member is always set, and tag_name remains
1303         # a reliable witness of being used by a flat union.
1304         assert bool(tag_member) != bool(tag_name)
1305         assert (isinstance(tag_name, str) or
1306                 isinstance(tag_member, QAPISchemaObjectTypeMember))
1307         assert len(variants) > 0
1308         for v in variants:
1309             assert isinstance(v, QAPISchemaObjectTypeVariant)
1310         self._tag_name = tag_name
1311         self.tag_member = tag_member
1312         self.variants = variants
1313
1314     def set_owner(self, name):
1315         for v in self.variants:
1316             v.set_owner(name)
1317
1318     def check(self, schema, seen):
1319         if not self.tag_member:    # flat union
1320             self.tag_member = seen[c_name(self._tag_name)]
1321             assert self._tag_name == self.tag_member.name
1322         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
1323         for v in self.variants:
1324             v.check(schema)
1325             # Union names must match enum values; alternate names are
1326             # checked separately. Use 'seen' to tell the two apart.
1327             if seen:
1328                 assert v.name in self.tag_member.type.member_names()
1329                 assert isinstance(v.type, QAPISchemaObjectType)
1330                 v.type.check(schema)
1331
1332     def check_clash(self, info, seen):
1333         for v in self.variants:
1334             # Reset seen map for each variant, since qapi names from one
1335             # branch do not affect another branch
1336             assert isinstance(v.type, QAPISchemaObjectType)
1337             v.type.check_clash(info, dict(seen))
1338
1339
1340 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1341     role = 'branch'
1342
1343     def __init__(self, name, typ):
1344         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1345
1346
1347 class QAPISchemaAlternateType(QAPISchemaType):
1348     def __init__(self, name, info, doc, variants):
1349         QAPISchemaType.__init__(self, name, info, doc)
1350         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1351         assert variants.tag_member
1352         variants.set_owner(name)
1353         variants.tag_member.set_owner(self.name)
1354         self.variants = variants
1355
1356     def check(self, schema):
1357         self.variants.tag_member.check(schema)
1358         # Not calling self.variants.check_clash(), because there's nothing
1359         # to clash with
1360         self.variants.check(schema, {})
1361         # Alternate branch names have no relation to the tag enum values;
1362         # so we have to check for potential name collisions ourselves.
1363         seen = {}
1364         for v in self.variants.variants:
1365             v.check_clash(self.info, seen)
1366             if self.doc:
1367                 self.doc.connect_member(v)
1368         if self.doc:
1369             self.doc.check()
1370
1371     def c_type(self):
1372         return c_name(self.name) + pointer_suffix
1373
1374     def json_type(self):
1375         return 'value'
1376
1377     def visit(self, visitor):
1378         visitor.visit_alternate_type(self.name, self.info, self.variants)
1379
1380     def is_empty(self):
1381         return False
1382
1383
1384 class QAPISchemaCommand(QAPISchemaEntity):
1385     def __init__(self, name, info, doc, arg_type, ret_type,
1386                  gen, success_response, boxed):
1387         QAPISchemaEntity.__init__(self, name, info, doc)
1388         assert not arg_type or isinstance(arg_type, str)
1389         assert not ret_type or isinstance(ret_type, str)
1390         self._arg_type_name = arg_type
1391         self.arg_type = None
1392         self._ret_type_name = ret_type
1393         self.ret_type = None
1394         self.gen = gen
1395         self.success_response = success_response
1396         self.boxed = boxed
1397
1398     def check(self, schema):
1399         if self._arg_type_name:
1400             self.arg_type = schema.lookup_type(self._arg_type_name)
1401             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1402                     isinstance(self.arg_type, QAPISchemaAlternateType))
1403             self.arg_type.check(schema)
1404             if self.boxed:
1405                 if self.arg_type.is_empty():
1406                     raise QAPISemError(self.info,
1407                                        "Cannot use 'boxed' with empty type")
1408             else:
1409                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1410                 assert not self.arg_type.variants
1411         elif self.boxed:
1412             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1413         if self._ret_type_name:
1414             self.ret_type = schema.lookup_type(self._ret_type_name)
1415             assert isinstance(self.ret_type, QAPISchemaType)
1416
1417     def visit(self, visitor):
1418         visitor.visit_command(self.name, self.info,
1419                               self.arg_type, self.ret_type,
1420                               self.gen, self.success_response, self.boxed)
1421
1422
1423 class QAPISchemaEvent(QAPISchemaEntity):
1424     def __init__(self, name, info, doc, arg_type, boxed):
1425         QAPISchemaEntity.__init__(self, name, info, doc)
1426         assert not arg_type or isinstance(arg_type, str)
1427         self._arg_type_name = arg_type
1428         self.arg_type = None
1429         self.boxed = boxed
1430
1431     def check(self, schema):
1432         if self._arg_type_name:
1433             self.arg_type = schema.lookup_type(self._arg_type_name)
1434             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1435                     isinstance(self.arg_type, QAPISchemaAlternateType))
1436             self.arg_type.check(schema)
1437             if self.boxed:
1438                 if self.arg_type.is_empty():
1439                     raise QAPISemError(self.info,
1440                                        "Cannot use 'boxed' with empty type")
1441             else:
1442                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1443                 assert not self.arg_type.variants
1444         elif self.boxed:
1445             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1446
1447     def visit(self, visitor):
1448         visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
1449
1450
1451 class QAPISchema(object):
1452     def __init__(self, fname):
1453         try:
1454             parser = QAPISchemaParser(open(fname, 'r'))
1455             self.exprs = check_exprs(parser.exprs)
1456             self.docs = parser.docs
1457             self._entity_dict = {}
1458             self._predefining = True
1459             self._def_predefineds()
1460             self._predefining = False
1461             self._def_exprs()
1462             self.check()
1463         except QAPIError as err:
1464             print >>sys.stderr, err
1465             exit(1)
1466
1467     def _def_entity(self, ent):
1468         # Only the predefined types are allowed to not have info
1469         assert ent.info or self._predefining
1470         assert ent.name not in self._entity_dict
1471         self._entity_dict[ent.name] = ent
1472
1473     def lookup_entity(self, name, typ=None):
1474         ent = self._entity_dict.get(name)
1475         if typ and not isinstance(ent, typ):
1476             return None
1477         return ent
1478
1479     def lookup_type(self, name):
1480         return self.lookup_entity(name, QAPISchemaType)
1481
1482     def _def_builtin_type(self, name, json_type, c_type):
1483         self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
1484         # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
1485         # qapi-types.h from a single .c, all arrays of builtins must be
1486         # declared in the first file whether or not they are used.  Nicer
1487         # would be to use lazy instantiation, while figuring out how to
1488         # avoid compilation issues with multiple qapi-types.h.
1489         self._make_array_type(name, None)
1490
1491     def _def_predefineds(self):
1492         for t in [('str',    'string',  'char' + pointer_suffix),
1493                   ('number', 'number',  'double'),
1494                   ('int',    'int',     'int64_t'),
1495                   ('int8',   'int',     'int8_t'),
1496                   ('int16',  'int',     'int16_t'),
1497                   ('int32',  'int',     'int32_t'),
1498                   ('int64',  'int',     'int64_t'),
1499                   ('uint8',  'int',     'uint8_t'),
1500                   ('uint16', 'int',     'uint16_t'),
1501                   ('uint32', 'int',     'uint32_t'),
1502                   ('uint64', 'int',     'uint64_t'),
1503                   ('size',   'int',     'uint64_t'),
1504                   ('bool',   'boolean', 'bool'),
1505                   ('any',    'value',   'QObject' + pointer_suffix)]:
1506             self._def_builtin_type(*t)
1507         self.the_empty_object_type = QAPISchemaObjectType(
1508             'q_empty', None, None, None, [], None)
1509         self._def_entity(self.the_empty_object_type)
1510         qtype_values = self._make_enum_members(['none', 'qnull', 'qint',
1511                                                 'qstring', 'qdict', 'qlist',
1512                                                 'qfloat', 'qbool'])
1513         self._def_entity(QAPISchemaEnumType('QType', None, None,
1514                                             qtype_values, 'QTYPE'))
1515
1516     def _make_enum_members(self, values):
1517         return [QAPISchemaMember(v) for v in values]
1518
1519     def _make_implicit_enum_type(self, name, info, values):
1520         # See also QAPISchemaObjectTypeMember._pretty_owner()
1521         name = name + 'Kind'   # Use namespace reserved by add_name()
1522         self._def_entity(QAPISchemaEnumType(
1523             name, info, None, self._make_enum_members(values), None))
1524         return name
1525
1526     def _make_array_type(self, element_type, info):
1527         name = element_type + 'List'   # Use namespace reserved by add_name()
1528         if not self.lookup_type(name):
1529             self._def_entity(QAPISchemaArrayType(name, info, element_type))
1530         return name
1531
1532     def _make_implicit_object_type(self, name, info, doc, role, members):
1533         if not members:
1534             return None
1535         # See also QAPISchemaObjectTypeMember._pretty_owner()
1536         name = 'q_obj_%s-%s' % (name, role)
1537         if not self.lookup_entity(name, QAPISchemaObjectType):
1538             self._def_entity(QAPISchemaObjectType(name, info, doc, None,
1539                                                   members, None))
1540         return name
1541
1542     def _def_enum_type(self, expr, info, doc):
1543         name = expr['enum']
1544         data = expr['data']
1545         prefix = expr.get('prefix')
1546         self._def_entity(QAPISchemaEnumType(
1547             name, info, doc, self._make_enum_members(data), prefix))
1548
1549     def _make_member(self, name, typ, info):
1550         optional = False
1551         if name.startswith('*'):
1552             name = name[1:]
1553             optional = True
1554         if isinstance(typ, list):
1555             assert len(typ) == 1
1556             typ = self._make_array_type(typ[0], info)
1557         return QAPISchemaObjectTypeMember(name, typ, optional)
1558
1559     def _make_members(self, data, info):
1560         return [self._make_member(key, value, info)
1561                 for (key, value) in data.iteritems()]
1562
1563     def _def_struct_type(self, expr, info, doc):
1564         name = expr['struct']
1565         base = expr.get('base')
1566         data = expr['data']
1567         self._def_entity(QAPISchemaObjectType(name, info, doc, base,
1568                                               self._make_members(data, info),
1569                                               None))
1570
1571     def _make_variant(self, case, typ):
1572         return QAPISchemaObjectTypeVariant(case, typ)
1573
1574     def _make_simple_variant(self, case, typ, info):
1575         if isinstance(typ, list):
1576             assert len(typ) == 1
1577             typ = self._make_array_type(typ[0], info)
1578         typ = self._make_implicit_object_type(
1579             typ, info, None, 'wrapper', [self._make_member('data', typ, info)])
1580         return QAPISchemaObjectTypeVariant(case, typ)
1581
1582     def _def_union_type(self, expr, info, doc):
1583         name = expr['union']
1584         data = expr['data']
1585         base = expr.get('base')
1586         tag_name = expr.get('discriminator')
1587         tag_member = None
1588         if isinstance(base, dict):
1589             base = (self._make_implicit_object_type(
1590                 name, info, doc, 'base', self._make_members(base, info)))
1591         if tag_name:
1592             variants = [self._make_variant(key, value)
1593                         for (key, value) in data.iteritems()]
1594             members = []
1595         else:
1596             variants = [self._make_simple_variant(key, value, info)
1597                         for (key, value) in data.iteritems()]
1598             typ = self._make_implicit_enum_type(name, info,
1599                                                 [v.name for v in variants])
1600             tag_member = QAPISchemaObjectTypeMember('type', typ, False)
1601             members = [tag_member]
1602         self._def_entity(
1603             QAPISchemaObjectType(name, info, doc, base, members,
1604                                  QAPISchemaObjectTypeVariants(tag_name,
1605                                                               tag_member,
1606                                                               variants)))
1607
1608     def _def_alternate_type(self, expr, info, doc):
1609         name = expr['alternate']
1610         data = expr['data']
1611         variants = [self._make_variant(key, value)
1612                     for (key, value) in data.iteritems()]
1613         tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
1614         self._def_entity(
1615             QAPISchemaAlternateType(name, info, doc,
1616                                     QAPISchemaObjectTypeVariants(None,
1617                                                                  tag_member,
1618                                                                  variants)))
1619
1620     def _def_command(self, expr, info, doc):
1621         name = expr['command']
1622         data = expr.get('data')
1623         rets = expr.get('returns')
1624         gen = expr.get('gen', True)
1625         success_response = expr.get('success-response', True)
1626         boxed = expr.get('boxed', False)
1627         if isinstance(data, OrderedDict):
1628             data = self._make_implicit_object_type(
1629                 name, info, doc, 'arg', self._make_members(data, info))
1630         if isinstance(rets, list):
1631             assert len(rets) == 1
1632             rets = self._make_array_type(rets[0], info)
1633         self._def_entity(QAPISchemaCommand(name, info, doc, data, rets,
1634                                            gen, success_response, boxed))
1635
1636     def _def_event(self, expr, info, doc):
1637         name = expr['event']
1638         data = expr.get('data')
1639         boxed = expr.get('boxed', False)
1640         if isinstance(data, OrderedDict):
1641             data = self._make_implicit_object_type(
1642                 name, info, doc, 'arg', self._make_members(data, info))
1643         self._def_entity(QAPISchemaEvent(name, info, doc, data, boxed))
1644
1645     def _def_exprs(self):
1646         for expr_elem in self.exprs:
1647             expr = expr_elem['expr']
1648             info = expr_elem['info']
1649             doc = expr_elem.get('doc')
1650             if 'enum' in expr:
1651                 self._def_enum_type(expr, info, doc)
1652             elif 'struct' in expr:
1653                 self._def_struct_type(expr, info, doc)
1654             elif 'union' in expr:
1655                 self._def_union_type(expr, info, doc)
1656             elif 'alternate' in expr:
1657                 self._def_alternate_type(expr, info, doc)
1658             elif 'command' in expr:
1659                 self._def_command(expr, info, doc)
1660             elif 'event' in expr:
1661                 self._def_event(expr, info, doc)
1662             else:
1663                 assert False
1664
1665     def check(self):
1666         for ent in self._entity_dict.values():
1667             ent.check(self)
1668
1669     def visit(self, visitor):
1670         visitor.visit_begin(self)
1671         for (name, entity) in sorted(self._entity_dict.items()):
1672             if visitor.visit_needed(entity):
1673                 entity.visit(visitor)
1674         visitor.visit_end()
1675
1676
1677 #
1678 # Code generation helpers
1679 #
1680
1681 def camel_case(name):
1682     new_name = ''
1683     first = True
1684     for ch in name:
1685         if ch in ['_', '-']:
1686             first = True
1687         elif first:
1688             new_name += ch.upper()
1689             first = False
1690         else:
1691             new_name += ch.lower()
1692     return new_name
1693
1694
1695 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1696 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1697 # ENUM24_Name -> ENUM24_NAME
1698 def camel_to_upper(value):
1699     c_fun_str = c_name(value, False)
1700     if value.isupper():
1701         return c_fun_str
1702
1703     new_name = ''
1704     l = len(c_fun_str)
1705     for i in range(l):
1706         c = c_fun_str[i]
1707         # When c is upper and no '_' appears before, do more checks
1708         if c.isupper() and (i > 0) and c_fun_str[i - 1] != '_':
1709             if i < l - 1 and c_fun_str[i + 1].islower():
1710                 new_name += '_'
1711             elif c_fun_str[i - 1].isdigit():
1712                 new_name += '_'
1713         new_name += c
1714     return new_name.lstrip('_').upper()
1715
1716
1717 def c_enum_const(type_name, const_name, prefix=None):
1718     if prefix is not None:
1719         type_name = prefix
1720     return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
1721
1722 c_name_trans = string.maketrans('.-', '__')
1723
1724
1725 # Map @name to a valid C identifier.
1726 # If @protect, avoid returning certain ticklish identifiers (like
1727 # C keywords) by prepending 'q_'.
1728 #
1729 # Used for converting 'name' from a 'name':'type' qapi definition
1730 # into a generated struct member, as well as converting type names
1731 # into substrings of a generated C function name.
1732 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1733 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1734 def c_name(name, protect=True):
1735     # ANSI X3J11/88-090, 3.1.1
1736     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1737                      'default', 'do', 'double', 'else', 'enum', 'extern',
1738                      'float', 'for', 'goto', 'if', 'int', 'long', 'register',
1739                      'return', 'short', 'signed', 'sizeof', 'static',
1740                      'struct', 'switch', 'typedef', 'union', 'unsigned',
1741                      'void', 'volatile', 'while'])
1742     # ISO/IEC 9899:1999, 6.4.1
1743     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1744     # ISO/IEC 9899:2011, 6.4.1
1745     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
1746                      '_Noreturn', '_Static_assert', '_Thread_local'])
1747     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1748     # excluding _.*
1749     gcc_words = set(['asm', 'typeof'])
1750     # C++ ISO/IEC 14882:2003 2.11
1751     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1752                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1753                      'namespace', 'new', 'operator', 'private', 'protected',
1754                      'public', 'reinterpret_cast', 'static_cast', 'template',
1755                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1756                      'using', 'virtual', 'wchar_t',
1757                      # alternative representations
1758                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1759                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1760     # namespace pollution:
1761     polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
1762     name = name.translate(c_name_trans)
1763     if protect and (name in c89_words | c99_words | c11_words | gcc_words
1764                     | cpp_words | polluted_words):
1765         return 'q_' + name
1766     return name
1767
1768 eatspace = '\033EATSPACE.'
1769 pointer_suffix = ' *' + eatspace
1770
1771
1772 def genindent(count):
1773     ret = ''
1774     for _ in range(count):
1775         ret += ' '
1776     return ret
1777
1778 indent_level = 0
1779
1780
1781 def push_indent(indent_amount=4):
1782     global indent_level
1783     indent_level += indent_amount
1784
1785
1786 def pop_indent(indent_amount=4):
1787     global indent_level
1788     indent_level -= indent_amount
1789
1790
1791 # Generate @code with @kwds interpolated.
1792 # Obey indent_level, and strip eatspace.
1793 def cgen(code, **kwds):
1794     raw = code % kwds
1795     if indent_level:
1796         indent = genindent(indent_level)
1797         # re.subn() lacks flags support before Python 2.7, use re.compile()
1798         raw = re.subn(re.compile(r'^.', re.MULTILINE),
1799                       indent + r'\g<0>', raw)
1800         raw = raw[0]
1801     return re.sub(re.escape(eatspace) + r' *', '', raw)
1802
1803
1804 def mcgen(code, **kwds):
1805     if code[0] == '\n':
1806         code = code[1:]
1807     return cgen(code, **kwds)
1808
1809
1810 def guardname(filename):
1811     return c_name(filename, protect=False).upper()
1812
1813
1814 def guardstart(name):
1815     return mcgen('''
1816
1817 #ifndef %(name)s
1818 #define %(name)s
1819
1820 ''',
1821                  name=guardname(name))
1822
1823
1824 def guardend(name):
1825     return mcgen('''
1826
1827 #endif /* %(name)s */
1828
1829 ''',
1830                  name=guardname(name))
1831
1832
1833 def gen_enum_lookup(name, values, prefix=None):
1834     ret = mcgen('''
1835
1836 const char *const %(c_name)s_lookup[] = {
1837 ''',
1838                 c_name=c_name(name))
1839     for value in values:
1840         index = c_enum_const(name, value, prefix)
1841         ret += mcgen('''
1842     [%(index)s] = "%(value)s",
1843 ''',
1844                      index=index, value=value)
1845
1846     max_index = c_enum_const(name, '_MAX', prefix)
1847     ret += mcgen('''
1848     [%(max_index)s] = NULL,
1849 };
1850 ''',
1851                  max_index=max_index)
1852     return ret
1853
1854
1855 def gen_enum(name, values, prefix=None):
1856     # append automatically generated _MAX value
1857     enum_values = values + ['_MAX']
1858
1859     ret = mcgen('''
1860
1861 typedef enum %(c_name)s {
1862 ''',
1863                 c_name=c_name(name))
1864
1865     i = 0
1866     for value in enum_values:
1867         ret += mcgen('''
1868     %(c_enum)s = %(i)d,
1869 ''',
1870                      c_enum=c_enum_const(name, value, prefix),
1871                      i=i)
1872         i += 1
1873
1874     ret += mcgen('''
1875 } %(c_name)s;
1876 ''',
1877                  c_name=c_name(name))
1878
1879     ret += mcgen('''
1880
1881 extern const char *const %(c_name)s_lookup[];
1882 ''',
1883                  c_name=c_name(name))
1884     return ret
1885
1886
1887 def gen_params(arg_type, boxed, extra):
1888     if not arg_type:
1889         assert not boxed
1890         return extra
1891     ret = ''
1892     sep = ''
1893     if boxed:
1894         ret += '%s arg' % arg_type.c_param_type()
1895         sep = ', '
1896     else:
1897         assert not arg_type.variants
1898         for memb in arg_type.members:
1899             ret += sep
1900             sep = ', '
1901             if memb.optional:
1902                 ret += 'bool has_%s, ' % c_name(memb.name)
1903             ret += '%s %s' % (memb.type.c_param_type(),
1904                               c_name(memb.name))
1905     if extra:
1906         ret += sep + extra
1907     return ret
1908
1909
1910 #
1911 # Common command line parsing
1912 #
1913
1914
1915 def parse_command_line(extra_options='', extra_long_options=[]):
1916
1917     try:
1918         opts, args = getopt.gnu_getopt(sys.argv[1:],
1919                                        'chp:o:' + extra_options,
1920                                        ['source', 'header', 'prefix=',
1921                                         'output-dir='] + extra_long_options)
1922     except getopt.GetoptError as err:
1923         print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
1924         sys.exit(1)
1925
1926     output_dir = ''
1927     prefix = ''
1928     do_c = False
1929     do_h = False
1930     extra_opts = []
1931
1932     for oa in opts:
1933         o, a = oa
1934         if o in ('-p', '--prefix'):
1935             match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1936             if match.end() != len(a):
1937                 print >>sys.stderr, \
1938                     "%s: 'funny character '%s' in argument of --prefix" \
1939                     % (sys.argv[0], a[match.end()])
1940                 sys.exit(1)
1941             prefix = a
1942         elif o in ('-o', '--output-dir'):
1943             output_dir = a + '/'
1944         elif o in ('-c', '--source'):
1945             do_c = True
1946         elif o in ('-h', '--header'):
1947             do_h = True
1948         else:
1949             extra_opts.append(oa)
1950
1951     if not do_c and not do_h:
1952         do_c = True
1953         do_h = True
1954
1955     if len(args) != 1:
1956         print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
1957         sys.exit(1)
1958     fname = args[0]
1959
1960     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
1961
1962 #
1963 # Generate output files with boilerplate
1964 #
1965
1966
1967 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
1968                 c_comment, h_comment):
1969     guard = guardname(prefix + h_file)
1970     c_file = output_dir + prefix + c_file
1971     h_file = output_dir + prefix + h_file
1972
1973     if output_dir:
1974         try:
1975             os.makedirs(output_dir)
1976         except os.error as e:
1977             if e.errno != errno.EEXIST:
1978                 raise
1979
1980     def maybe_open(really, name, opt):
1981         if really:
1982             return open(name, opt)
1983         else:
1984             import StringIO
1985             return StringIO.StringIO()
1986
1987     fdef = maybe_open(do_c, c_file, 'w')
1988     fdecl = maybe_open(do_h, h_file, 'w')
1989
1990     fdef.write(mcgen('''
1991 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
1992 %(comment)s
1993 ''',
1994                      comment=c_comment))
1995
1996     fdecl.write(mcgen('''
1997 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
1998 %(comment)s
1999 #ifndef %(guard)s
2000 #define %(guard)s
2001
2002 ''',
2003                       comment=h_comment, guard=guard))
2004
2005     return (fdef, fdecl)
2006
2007
2008 def close_output(fdef, fdecl):
2009     fdecl.write('''
2010 #endif
2011 ''')
2012     fdecl.close()
2013     fdef.close()