scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 import errno
  15 import getopt
  16 import os
  17 import re
  18 import string
  19 import sys
  20 from ordereddict import OrderedDict
  21
  22 builtin_types = {
  23     'str':      'QTYPE_QSTRING',
  24     'int':      'QTYPE_QNUM',
  25     'number':   'QTYPE_QNUM',
  26     'bool':     'QTYPE_QBOOL',
  27     'int8':     'QTYPE_QNUM',
  28     'int16':    'QTYPE_QNUM',
  29     'int32':    'QTYPE_QNUM',
  30     'int64':    'QTYPE_QNUM',
  31     'uint8':    'QTYPE_QNUM',
  32     'uint16':   'QTYPE_QNUM',
  33     'uint32':   'QTYPE_QNUM',
  34     'uint64':   'QTYPE_QNUM',
  35     'size':     'QTYPE_QNUM',
  36     'any':      None,           # any QType possible, actually
  37     'QType':    'QTYPE_QSTRING',
  38 }
  39
  40 # Are documentation comments required?
  41 doc_required = False
  42
  43 # Whitelist of commands allowed to return a non-dictionary
  44 returns_whitelist = []
  45
  46 # Whitelist of entities allowed to violate case conventions
  47 name_case_whitelist = []
  48
  49 enum_types = {}
  50 struct_types = {}
  51 union_types = {}
  52 all_names = {}
  53
  54 #
  55 # Parsing the schema into expressions
  56 #
  57
  58
  59 def error_path(parent):
  60     res = ''
  61     while parent:
  62         res = ('In file included from %s:%d:\n' % (parent['file'],
  63                                                    parent['line'])) + res
  64         parent = parent['parent']
  65     return res
  66
  67
  68 class QAPIError(Exception):
  69     def __init__(self, fname, line, col, incl_info, msg):
  70         Exception.__init__(self)
  71         self.fname = fname
  72         self.line = line
  73         self.col = col
  74         self.info = incl_info
  75         self.msg = msg
  76
  77     def __str__(self):
  78         loc = '%s:%d' % (self.fname, self.line)
  79         if self.col is not None:
  80             loc += ':%s' % self.col
  81         return error_path(self.info) + '%s: %s' % (loc, self.msg)
  82
  83
  84 class QAPIParseError(QAPIError):
  85     def __init__(self, parser, msg):
  86         col = 1
  87         for ch in parser.src[parser.line_pos:parser.pos]:
  88             if ch == '\t':
  89                 col = (col + 7) % 8 + 1
  90             else:
  91                 col += 1
  92         QAPIError.__init__(self, parser.fname, parser.line, col,
  93                            parser.incl_info, msg)
  94
  95
  96 class QAPISemError(QAPIError):
  97     def __init__(self, info, msg):
  98         QAPIError.__init__(self, info['file'], info['line'], None,
  99                            info['parent'], msg)
 100
 101
 102 class QAPIDoc(object):
 103     class Section(object):
 104         def __init__(self, name=None):
 105             # optional section name (argument/member or section name)
 106             self.name = name
 107             # the list of lines for this section
 108             self.content = []
 109
 110         def append(self, line):
 111             self.content.append(line)
 112
 113         def __repr__(self):
 114             return '\n'.join(self.content).strip()
 115
 116     class ArgSection(Section):
 117         def __init__(self, name):
 118             QAPIDoc.Section.__init__(self, name)
 119             self.member = None
 120
 121         def connect(self, member):
 122             self.member = member
 123
 124     def __init__(self, parser, info):
 125         # self.parser is used to report errors with QAPIParseError.  The
 126         # resulting error position depends on the state of the parser.
 127         # It happens to be the beginning of the comment.  More or less
 128         # servicable, but action at a distance.
 129         self.parser = parser
 130         self.info = info
 131         self.symbol = None
 132         self.body = QAPIDoc.Section()
 133         # dict mapping parameter name to ArgSection
 134         self.args = OrderedDict()
 135         # a list of Section
 136         self.sections = []
 137         # the current section
 138         self.section = self.body
 139
 140     def has_section(self, name):
 141         """Return True if we have a section with this name."""
 142         for i in self.sections:
 143             if i.name == name:
 144                 return True
 145         return False
 146
 147     def append(self, line):
 148         """Parse a comment line and add it to the documentation."""
 149         line = line[1:]
 150         if not line:
 151             self._append_freeform(line)
 152             return
 153
 154         if line[0] != ' ':
 155             raise QAPIParseError(self.parser, "Missing space after #")
 156         line = line[1:]
 157
 158         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 159         # recognized, and get silently treated as ordinary text
 160         if self.symbol:
 161             self._append_symbol_line(line)
 162         elif not self.body.content and line.startswith('@'):
 163             if not line.endswith(':'):
 164                 raise QAPIParseError(self.parser, "Line should end with :")
 165             self.symbol = line[1:-1]
 166             # FIXME invalid names other than the empty string aren't flagged
 167             if not self.symbol:
 168                 raise QAPIParseError(self.parser, "Invalid name")
 169         else:
 170             self._append_freeform(line)
 171
 172     def end_comment(self):
 173         self._end_section()
 174
 175     def _append_symbol_line(self, line):
 176         name = line.split(' ', 1)[0]
 177
 178         if name.startswith('@') and name.endswith(':'):
 179             line = line[len(name)+1:]
 180             self._start_args_section(name[1:-1])
 181         elif name in ('Returns:', 'Since:',
 182                       # those are often singular or plural
 183                       'Note:', 'Notes:',
 184                       'Example:', 'Examples:',
 185                       'TODO:'):
 186             line = line[len(name)+1:]
 187             self._start_section(name[:-1])
 188
 189         self._append_freeform(line)
 190
 191     def _start_args_section(self, name):
 192         # FIXME invalid names other than the empty string aren't flagged
 193         if not name:
 194             raise QAPIParseError(self.parser, "Invalid parameter name")
 195         if name in self.args:
 196             raise QAPIParseError(self.parser,
 197                                  "'%s' parameter name duplicated" % name)
 198         if self.sections:
 199             raise QAPIParseError(self.parser,
 200                                  "'@%s:' can't follow '%s' section"
 201                                  % (name, self.sections[0].name))
 202         self._end_section()
 203         self.section = QAPIDoc.ArgSection(name)
 204         self.args[name] = self.section
 205
 206     def _start_section(self, name=''):
 207         if name in ('Returns', 'Since') and self.has_section(name):
 208             raise QAPIParseError(self.parser,
 209                                  "Duplicated '%s' section" % name)
 210         self._end_section()
 211         self.section = QAPIDoc.Section(name)
 212         self.sections.append(self.section)
 213
 214     def _end_section(self):
 215         if self.section:
 216             contents = str(self.section)
 217             if self.section.name and (not contents or contents.isspace()):
 218                 raise QAPIParseError(self.parser, "Empty doc section '%s'"
 219                                      % self.section.name)
 220             self.section = None
 221
 222     def _append_freeform(self, line):
 223         in_arg = isinstance(self.section, QAPIDoc.ArgSection)
 224         if (in_arg and self.section.content
 225                 and not self.section.content[-1]
 226                 and line and not line[0].isspace()):
 227             self._start_section()
 228         if (in_arg or not self.section.name
 229                 or not self.section.name.startswith('Example')):
 230             line = line.strip()
 231         match = re.match(r'(@\S+:)', line)
 232         if match:
 233             raise QAPIParseError(self.parser,
 234                                  "'%s' not allowed in free-form documentation"
 235                                  % match.group(1))
 236         # TODO Drop this once the dust has settled
 237         if (isinstance(self.section, QAPIDoc.ArgSection)
 238                 and '#optional' in line):
 239             raise QAPISemError(self.info, "Please drop the #optional tag")
 240         self.section.append(line)
 241
 242     def connect_member(self, member):
 243         if member.name not in self.args:
 244             # Undocumented TODO outlaw
 245             self.args[member.name] = QAPIDoc.ArgSection(member.name)
 246         self.args[member.name].connect(member)
 247
 248     def check_expr(self, expr):
 249         if self.has_section('Returns') and 'command' not in expr:
 250             raise QAPISemError(self.info,
 251                                "'Returns:' is only valid for commands")
 252
 253     def check(self):
 254         bogus = [name for name, section in self.args.iteritems()
 255                  if not section.member]
 256         if bogus:
 257             raise QAPISemError(
 258                 self.info,
 259                 "The following documented members are not in "
 260                 "the declaration: %s" % ", ".join(bogus))
 261
 262
 263 class QAPISchemaParser(object):
 264
 265     def __init__(self, fp, previously_included=[], incl_info=None):
 266         abs_fname = os.path.abspath(fp.name)
 267         fname = fp.name
 268         self.fname = fname
 269         previously_included.append(abs_fname)
 270         self.incl_info = incl_info
 271         self.src = fp.read()
 272         if self.src == '' or self.src[-1] != '\n':
 273             self.src += '\n'
 274         self.cursor = 0
 275         self.line = 1
 276         self.line_pos = 0
 277         self.exprs = []
 278         self.docs = []
 279         self.cur_doc = None
 280         self.accept()
 281
 282         while self.tok is not None:
 283             info = {'file': fname, 'line': self.line,
 284                     'parent': self.incl_info}
 285             if self.tok == '#':
 286                 self.reject_expr_doc()
 287                 self.cur_doc = self.get_doc(info)
 288                 self.docs.append(self.cur_doc)
 289                 continue
 290
 291             expr = self.get_expr(False)
 292             if 'include' in expr:
 293                 self.reject_expr_doc()
 294                 if len(expr) != 1:
 295                     raise QAPISemError(info, "Invalid 'include' directive")
 296                 include = expr['include']
 297                 if not isinstance(include, str):
 298                     raise QAPISemError(info,
 299                                        "Value of 'include' must be a string")
 300                 self._include(include, info, os.path.dirname(abs_fname),
 301                               previously_included)
 302             elif "pragma" in expr:
 303                 self.reject_expr_doc()
 304                 if len(expr) != 1:
 305                     raise QAPISemError(info, "Invalid 'pragma' directive")
 306                 pragma = expr['pragma']
 307                 if not isinstance(pragma, dict):
 308                     raise QAPISemError(
 309                         info, "Value of 'pragma' must be a dictionary")
 310                 for name, value in pragma.iteritems():
 311                     self._pragma(name, value, info)
 312             else:
 313                 expr_elem = {'expr': expr,
 314                              'info': info}
 315                 if self.cur_doc:
 316                     if not self.cur_doc.symbol:
 317                         raise QAPISemError(
 318                             self.cur_doc.info,
 319                             "Expression documentation required")
 320                     expr_elem['doc'] = self.cur_doc
 321                 self.exprs.append(expr_elem)
 322             self.cur_doc = None
 323         self.reject_expr_doc()
 324
 325     def reject_expr_doc(self):
 326         if self.cur_doc and self.cur_doc.symbol:
 327             raise QAPISemError(
 328                 self.cur_doc.info,
 329                 "Documentation for '%s' is not followed by the definition"
 330                 % self.cur_doc.symbol)
 331
 332     def _include(self, include, info, base_dir, previously_included):
 333         incl_abs_fname = os.path.join(base_dir, include)
 334         # catch inclusion cycle
 335         inf = info
 336         while inf:
 337             if incl_abs_fname == os.path.abspath(inf['file']):
 338                 raise QAPISemError(info, "Inclusion loop for %s" % include)
 339             inf = inf['parent']
 340
 341         # skip multiple include of the same file
 342         if incl_abs_fname in previously_included:
 343             return
 344         try:
 345             fobj = open(incl_abs_fname, 'r')
 346         except IOError as e:
 347             raise QAPISemError(info, '%s: %s' % (e.strerror, include))
 348         exprs_include = QAPISchemaParser(fobj, previously_included, info)
 349         self.exprs.extend(exprs_include.exprs)
 350         self.docs.extend(exprs_include.docs)
 351
 352     def _pragma(self, name, value, info):
 353         global doc_required, returns_whitelist, name_case_whitelist
 354         if name == 'doc-required':
 355             if not isinstance(value, bool):
 356                 raise QAPISemError(info,
 357                                    "Pragma 'doc-required' must be boolean")
 358             doc_required = value
 359         elif name == 'returns-whitelist':
 360             if (not isinstance(value, list)
 361                     or any([not isinstance(elt, str) for elt in value])):
 362                 raise QAPISemError(info,
 363                                    "Pragma returns-whitelist must be"
 364                                    " a list of strings")
 365             returns_whitelist = value
 366         elif name == 'name-case-whitelist':
 367             if (not isinstance(value, list)
 368                     or any([not isinstance(elt, str) for elt in value])):
 369                 raise QAPISemError(info,
 370                                    "Pragma name-case-whitelist must be"
 371                                    " a list of strings")
 372             name_case_whitelist = value
 373         else:
 374             raise QAPISemError(info, "Unknown pragma '%s'" % name)
 375
 376     def accept(self, skip_comment=True):
 377         while True:
 378             self.tok = self.src[self.cursor]
 379             self.pos = self.cursor
 380             self.cursor += 1
 381             self.val = None
 382
 383             if self.tok == '#':
 384                 if self.src[self.cursor] == '#':
 385                     # Start of doc comment
 386                     skip_comment = False
 387                 self.cursor = self.src.find('\n', self.cursor)
 388                 if not skip_comment:
 389                     self.val = self.src[self.pos:self.cursor]
 390                     return
 391             elif self.tok in '{}:,[]':
 392                 return
 393             elif self.tok == "'":
 394                 string = ''
 395                 esc = False
 396                 while True:
 397                     ch = self.src[self.cursor]
 398                     self.cursor += 1
 399                     if ch == '\n':
 400                         raise QAPIParseError(self, 'Missing terminating "\'"')
 401                     if esc:
 402                         if ch == 'b':
 403                             string += '\b'
 404                         elif ch == 'f':
 405                             string += '\f'
 406                         elif ch == 'n':
 407                             string += '\n'
 408                         elif ch == 'r':
 409                             string += '\r'
 410                         elif ch == 't':
 411                             string += '\t'
 412                         elif ch == 'u':
 413                             value = 0
 414                             for _ in range(0, 4):
 415                                 ch = self.src[self.cursor]
 416                                 self.cursor += 1
 417                                 if ch not in '0123456789abcdefABCDEF':
 418                                     raise QAPIParseError(self,
 419                                                          '\\u escape needs 4 '
 420                                                          'hex digits')
 421                                 value = (value << 4) + int(ch, 16)
 422                             # If Python 2 and 3 didn't disagree so much on
 423                             # how to handle Unicode, then we could allow
 424                             # Unicode string defaults.  But most of QAPI is
 425                             # ASCII-only, so we aren't losing much for now.
 426                             if not value or value > 0x7f:
 427                                 raise QAPIParseError(self,
 428                                                      'For now, \\u escape '
 429                                                      'only supports non-zero '
 430                                                      'values up to \\u007f')
 431                             string += chr(value)
 432                         elif ch in '\\/\'"':
 433                             string += ch
 434                         else:
 435                             raise QAPIParseError(self,
 436                                                  "Unknown escape \\%s" % ch)
 437                         esc = False
 438                     elif ch == '\\':
 439                         esc = True
 440                     elif ch == "'":
 441                         self.val = string
 442                         return
 443                     else:
 444                         string += ch
 445             elif self.src.startswith('true', self.pos):
 446                 self.val = True
 447                 self.cursor += 3
 448                 return
 449             elif self.src.startswith('false', self.pos):
 450                 self.val = False
 451                 self.cursor += 4
 452                 return
 453             elif self.src.startswith('null', self.pos):
 454                 self.val = None
 455                 self.cursor += 3
 456                 return
 457             elif self.tok == '\n':
 458                 if self.cursor == len(self.src):
 459                     self.tok = None
 460                     return
 461                 self.line += 1
 462                 self.line_pos = self.cursor
 463             elif not self.tok.isspace():
 464                 raise QAPIParseError(self, 'Stray "%s"' % self.tok)
 465
 466     def get_members(self):
 467         expr = OrderedDict()
 468         if self.tok == '}':
 469             self.accept()
 470             return expr
 471         if self.tok != "'":
 472             raise QAPIParseError(self, 'Expected string or "}"')
 473         while True:
 474             key = self.val
 475             self.accept()
 476             if self.tok != ':':
 477                 raise QAPIParseError(self, 'Expected ":"')
 478             self.accept()
 479             if key in expr:
 480                 raise QAPIParseError(self, 'Duplicate key "%s"' % key)
 481             expr[key] = self.get_expr(True)
 482             if self.tok == '}':
 483                 self.accept()
 484                 return expr
 485             if self.tok != ',':
 486                 raise QAPIParseError(self, 'Expected "," or "}"')
 487             self.accept()
 488             if self.tok != "'":
 489                 raise QAPIParseError(self, 'Expected string')
 490
 491     def get_values(self):
 492         expr = []
 493         if self.tok == ']':
 494             self.accept()
 495             return expr
 496         if self.tok not in "{['tfn":
 497             raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
 498                                  'boolean or "null"')
 499         while True:
 500             expr.append(self.get_expr(True))
 501             if self.tok == ']':
 502                 self.accept()
 503                 return expr
 504             if self.tok != ',':
 505                 raise QAPIParseError(self, 'Expected "," or "]"')
 506             self.accept()
 507
 508     def get_expr(self, nested):
 509         if self.tok != '{' and not nested:
 510             raise QAPIParseError(self, 'Expected "{"')
 511         if self.tok == '{':
 512             self.accept()
 513             expr = self.get_members()
 514         elif self.tok == '[':
 515             self.accept()
 516             expr = self.get_values()
 517         elif self.tok in "'tfn":
 518             expr = self.val
 519             self.accept()
 520         else:
 521             raise QAPIParseError(self, 'Expected "{", "[", string, '
 522                                  'boolean or "null"')
 523         return expr
 524
 525     def get_doc(self, info):
 526         if self.val != '##':
 527             raise QAPIParseError(self, "Junk after '##' at start of "
 528                                  "documentation comment")
 529
 530         doc = QAPIDoc(self, info)
 531         self.accept(False)
 532         while self.tok == '#':
 533             if self.val.startswith('##'):
 534                 # End of doc comment
 535                 if self.val != '##':
 536                     raise QAPIParseError(self, "Junk after '##' at end of "
 537                                          "documentation comment")
 538                 doc.end_comment()
 539                 self.accept()
 540                 return doc
 541             else:
 542                 doc.append(self.val)
 543             self.accept(False)
 544
 545         raise QAPIParseError(self, "Documentation comment must end with '##'")
 546
 547
 548 #
 549 # Semantic analysis of schema expressions
 550 # TODO fold into QAPISchema
 551 # TODO catching name collisions in generated code would be nice
 552 #
 553
 554
 555 def find_base_members(base):
 556     if isinstance(base, dict):
 557         return base
 558     base_struct_define = struct_types.get(base)
 559     if not base_struct_define:
 560         return None
 561     return base_struct_define['data']
 562
 563
 564 # Return the qtype of an alternate branch, or None on error.
 565 def find_alternate_member_qtype(qapi_type):
 566     if qapi_type in builtin_types:
 567         return builtin_types[qapi_type]
 568     elif qapi_type in struct_types:
 569         return 'QTYPE_QDICT'
 570     elif qapi_type in enum_types:
 571         return 'QTYPE_QSTRING'
 572     elif qapi_type in union_types:
 573         return 'QTYPE_QDICT'
 574     return None
 575
 576
 577 # Return the discriminator enum define if discriminator is specified as an
 578 # enum type, otherwise return None.
 579 def discriminator_find_enum_define(expr):
 580     base = expr.get('base')
 581     discriminator = expr.get('discriminator')
 582
 583     if not (discriminator and base):
 584         return None
 585
 586     base_members = find_base_members(base)
 587     if not base_members:
 588         return None
 589
 590     discriminator_type = base_members.get(discriminator)
 591     if not discriminator_type:
 592         return None
 593
 594     return enum_types.get(discriminator_type)
 595
 596
 597 # Names must be letters, numbers, -, and _.  They must start with letter,
 598 # except for downstream extensions which must start with __RFQDN_.
 599 # Dots are only valid in the downstream extension prefix.
 600 valid_name = re.compile(r'^(__[a-zA-Z0-9.-]+_)?'
 601                         '[a-zA-Z][a-zA-Z0-9_-]*$')
 602
 603
 604 def check_name(info, source, name, allow_optional=False,
 605                enum_member=False):
 606     global valid_name
 607     membername = name
 608
 609     if not isinstance(name, str):
 610         raise QAPISemError(info, "%s requires a string name" % source)
 611     if name.startswith('*'):
 612         membername = name[1:]
 613         if not allow_optional:
 614             raise QAPISemError(info, "%s does not allow optional name '%s'"
 615                                % (source, name))
 616     # Enum members can start with a digit, because the generated C
 617     # code always prefixes it with the enum name
 618     if enum_member and membername[0].isdigit():
 619         membername = 'D' + membername
 620     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
 621     # and 'q_obj_*' implicit type names.
 622     if not valid_name.match(membername) or \
 623        c_name(membername, False).startswith('q_'):
 624         raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
 625
 626
 627 def add_name(name, info, meta, implicit=False):
 628     global all_names
 629     check_name(info, "'%s'" % meta, name)
 630     # FIXME should reject names that differ only in '_' vs. '.'
 631     # vs. '-', because they're liable to clash in generated C.
 632     if name in all_names:
 633         raise QAPISemError(info, "%s '%s' is already defined"
 634                            % (all_names[name], name))
 635     if not implicit and (name.endswith('Kind') or name.endswith('List')):
 636         raise QAPISemError(info, "%s '%s' should not end in '%s'"
 637                            % (meta, name, name[-4:]))
 638     all_names[name] = meta
 639
 640
 641 def check_type(info, source, value, allow_array=False,
 642                allow_dict=False, allow_optional=False,
 643                allow_metas=[]):
 644     global all_names
 645
 646     if value is None:
 647         return
 648
 649     # Check if array type for value is okay
 650     if isinstance(value, list):
 651         if not allow_array:
 652             raise QAPISemError(info, "%s cannot be an array" % source)
 653         if len(value) != 1 or not isinstance(value[0], str):
 654             raise QAPISemError(info,
 655                                "%s: array type must contain single type name" %
 656                                source)
 657         value = value[0]
 658
 659     # Check if type name for value is okay
 660     if isinstance(value, str):
 661         if value not in all_names:
 662             raise QAPISemError(info, "%s uses unknown type '%s'"
 663                                % (source, value))
 664         if not all_names[value] in allow_metas:
 665             raise QAPISemError(info, "%s cannot use %s type '%s'" %
 666                                (source, all_names[value], value))
 667         return
 668
 669     if not allow_dict:
 670         raise QAPISemError(info, "%s should be a type name" % source)
 671
 672     if not isinstance(value, OrderedDict):
 673         raise QAPISemError(info,
 674                            "%s should be a dictionary or type name" % source)
 675
 676     # value is a dictionary, check that each member is okay
 677     for (key, arg) in value.items():
 678         check_name(info, "Member of %s" % source, key,
 679                    allow_optional=allow_optional)
 680         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 681             raise QAPISemError(info, "Member of %s uses reserved name '%s'"
 682                                % (source, key))
 683         # Todo: allow dictionaries to represent default values of
 684         # an optional argument.
 685         check_type(info, "Member '%s' of %s" % (key, source), arg,
 686                    allow_array=True,
 687                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 688                                 'enum'])
 689
 690
 691 def check_command(expr, info):
 692     name = expr['command']
 693     boxed = expr.get('boxed', False)
 694
 695     args_meta = ['struct']
 696     if boxed:
 697         args_meta += ['union', 'alternate']
 698     check_type(info, "'data' for command '%s'" % name,
 699                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 700                allow_metas=args_meta)
 701     returns_meta = ['union', 'struct']
 702     if name in returns_whitelist:
 703         returns_meta += ['built-in', 'alternate', 'enum']
 704     check_type(info, "'returns' for command '%s'" % name,
 705                expr.get('returns'), allow_array=True,
 706                allow_optional=True, allow_metas=returns_meta)
 707
 708
 709 def check_event(expr, info):
 710     name = expr['event']
 711     boxed = expr.get('boxed', False)
 712
 713     meta = ['struct']
 714     if boxed:
 715         meta += ['union', 'alternate']
 716     check_type(info, "'data' for event '%s'" % name,
 717                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 718                allow_metas=meta)
 719
 720
 721 def check_union(expr, info):
 722     name = expr['union']
 723     base = expr.get('base')
 724     discriminator = expr.get('discriminator')
 725     members = expr['data']
 726
 727     # Two types of unions, determined by discriminator.
 728
 729     # With no discriminator it is a simple union.
 730     if discriminator is None:
 731         enum_define = None
 732         allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
 733         if base is not None:
 734             raise QAPISemError(info, "Simple union '%s' must not have a base" %
 735                                name)
 736
 737     # Else, it's a flat union.
 738     else:
 739         # The object must have a string or dictionary 'base'.
 740         check_type(info, "'base' for union '%s'" % name,
 741                    base, allow_dict=True, allow_optional=True,
 742                    allow_metas=['struct'])
 743         if not base:
 744             raise QAPISemError(info, "Flat union '%s' must have a base"
 745                                % name)
 746         base_members = find_base_members(base)
 747         assert base_members is not None
 748
 749         # The value of member 'discriminator' must name a non-optional
 750         # member of the base struct.
 751         check_name(info, "Discriminator of flat union '%s'" % name,
 752                    discriminator)
 753         discriminator_type = base_members.get(discriminator)
 754         if not discriminator_type:
 755             raise QAPISemError(info,
 756                                "Discriminator '%s' is not a member of base "
 757                                "struct '%s'"
 758                                % (discriminator, base))
 759         enum_define = enum_types.get(discriminator_type)
 760         allow_metas = ['struct']
 761         # Do not allow string discriminator
 762         if not enum_define:
 763             raise QAPISemError(info,
 764                                "Discriminator '%s' must be of enumeration "
 765                                "type" % discriminator)
 766
 767     # Check every branch; don't allow an empty union
 768     if len(members) == 0:
 769         raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
 770     for (key, value) in members.items():
 771         check_name(info, "Member of union '%s'" % name, key)
 772
 773         # Each value must name a known type
 774         check_type(info, "Member '%s' of union '%s'" % (key, name),
 775                    value, allow_array=not base, allow_metas=allow_metas)
 776
 777         # If the discriminator names an enum type, then all members
 778         # of 'data' must also be members of the enum type.
 779         if enum_define:
 780             if key not in enum_define['data']:
 781                 raise QAPISemError(info,
 782                                    "Discriminator value '%s' is not found in "
 783                                    "enum '%s'"
 784                                    % (key, enum_define['enum']))
 785
 786     # If discriminator is user-defined, ensure all values are covered
 787     if enum_define:
 788         for value in enum_define['data']:
 789             if value not in members.keys():
 790                 raise QAPISemError(info, "Union '%s' data missing '%s' branch"
 791                                    % (name, value))
 792
 793
 794 def check_alternate(expr, info):
 795     name = expr['alternate']
 796     members = expr['data']
 797     types_seen = {}
 798
 799     # Check every branch; require at least two branches
 800     if len(members) < 2:
 801         raise QAPISemError(info,
 802                            "Alternate '%s' should have at least two branches "
 803                            "in 'data'" % name)
 804     for (key, value) in members.items():
 805         check_name(info, "Member of alternate '%s'" % name, key)
 806
 807         # Ensure alternates have no type conflicts.
 808         check_type(info, "Member '%s' of alternate '%s'" % (key, name),
 809                    value,
 810                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 811         qtype = find_alternate_member_qtype(value)
 812         if not qtype:
 813             raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
 814                                "type '%s'" % (name, key, value))
 815         conflicting = set([qtype])
 816         if qtype == 'QTYPE_QSTRING':
 817             enum_expr = enum_types.get(value)
 818             if enum_expr:
 819                 for v in enum_expr['data']:
 820                     if v in ['on', 'off']:
 821                         conflicting.add('QTYPE_QBOOL')
 822                     if re.match(r'[-+0-9.]', v): # lazy, could be tightened
 823                         conflicting.add('QTYPE_QNUM')
 824             else:
 825                 conflicting.add('QTYPE_QNUM')
 826                 conflicting.add('QTYPE_QBOOL')
 827         if conflicting & set(types_seen):
 828             raise QAPISemError(info, "Alternate '%s' member '%s' can't "
 829                                "be distinguished from member '%s'"
 830                                % (name, key, types_seen[qtype]))
 831         for qt in conflicting:
 832             types_seen[qt] = key
 833
 834
 835 def check_enum(expr, info):
 836     name = expr['enum']
 837     members = expr.get('data')
 838     prefix = expr.get('prefix')
 839
 840     if not isinstance(members, list):
 841         raise QAPISemError(info,
 842                            "Enum '%s' requires an array for 'data'" % name)
 843     if prefix is not None and not isinstance(prefix, str):
 844         raise QAPISemError(info,
 845                            "Enum '%s' requires a string for 'prefix'" % name)
 846     for member in members:
 847         check_name(info, "Member of enum '%s'" % name, member,
 848                    enum_member=True)
 849
 850
 851 def check_struct(expr, info):
 852     name = expr['struct']
 853     members = expr['data']
 854
 855     check_type(info, "'data' for struct '%s'" % name, members,
 856                allow_dict=True, allow_optional=True)
 857     check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
 858                allow_metas=['struct'])
 859
 860
 861 def check_keys(expr_elem, meta, required, optional=[]):
 862     expr = expr_elem['expr']
 863     info = expr_elem['info']
 864     name = expr[meta]
 865     if not isinstance(name, str):
 866         raise QAPISemError(info, "'%s' key must have a string value" % meta)
 867     required = required + [meta]
 868     for (key, value) in expr.items():
 869         if key not in required and key not in optional:
 870             raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
 871                                % (key, meta, name))
 872         if (key == 'gen' or key == 'success-response') and value is not False:
 873             raise QAPISemError(info,
 874                                "'%s' of %s '%s' should only use false value"
 875                                % (key, meta, name))
 876         if key == 'boxed' and value is not True:
 877             raise QAPISemError(info,
 878                                "'%s' of %s '%s' should only use true value"
 879                                % (key, meta, name))
 880     for key in required:
 881         if key not in expr:
 882             raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
 883                                % (key, meta, name))
 884
 885
 886 def check_exprs(exprs):
 887     global all_names
 888
 889     # Populate name table with names of built-in types
 890     for builtin in builtin_types.keys():
 891         all_names[builtin] = 'built-in'
 892
 893     # Learn the types and check for valid expression keys
 894     for expr_elem in exprs:
 895         expr = expr_elem['expr']
 896         info = expr_elem['info']
 897         doc = expr_elem.get('doc')
 898
 899         if not doc and doc_required:
 900             raise QAPISemError(info,
 901                                "Expression missing documentation comment")
 902
 903         if 'enum' in expr:
 904             meta = 'enum'
 905             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 906             enum_types[expr[meta]] = expr
 907         elif 'union' in expr:
 908             meta = 'union'
 909             check_keys(expr_elem, 'union', ['data'],
 910                        ['base', 'discriminator'])
 911             union_types[expr[meta]] = expr
 912         elif 'alternate' in expr:
 913             meta = 'alternate'
 914             check_keys(expr_elem, 'alternate', ['data'])
 915         elif 'struct' in expr:
 916             meta = 'struct'
 917             check_keys(expr_elem, 'struct', ['data'], ['base'])
 918             struct_types[expr[meta]] = expr
 919         elif 'command' in expr:
 920             meta = 'command'
 921             check_keys(expr_elem, 'command', [],
 922                        ['data', 'returns', 'gen', 'success-response', 'boxed'])
 923         elif 'event' in expr:
 924             meta = 'event'
 925             check_keys(expr_elem, 'event', [], ['data', 'boxed'])
 926         else:
 927             raise QAPISemError(expr_elem['info'],
 928                                "Expression is missing metatype")
 929         name = expr[meta]
 930         add_name(name, info, meta)
 931         if doc and doc.symbol != name:
 932             raise QAPISemError(info, "Definition of '%s' follows documentation"
 933                                " for '%s'" % (name, doc.symbol))
 934
 935     # Try again for hidden UnionKind enum
 936     for expr_elem in exprs:
 937         expr = expr_elem['expr']
 938         if 'union' in expr and not discriminator_find_enum_define(expr):
 939             name = '%sKind' % expr['union']
 940         elif 'alternate' in expr:
 941             name = '%sKind' % expr['alternate']
 942         else:
 943             continue
 944         enum_types[name] = {'enum': name}
 945         add_name(name, info, 'enum', implicit=True)
 946
 947     # Validate that exprs make sense
 948     for expr_elem in exprs:
 949         expr = expr_elem['expr']
 950         info = expr_elem['info']
 951         doc = expr_elem.get('doc')
 952
 953         if 'enum' in expr:
 954             check_enum(expr, info)
 955         elif 'union' in expr:
 956             check_union(expr, info)
 957         elif 'alternate' in expr:
 958             check_alternate(expr, info)
 959         elif 'struct' in expr:
 960             check_struct(expr, info)
 961         elif 'command' in expr:
 962             check_command(expr, info)
 963         elif 'event' in expr:
 964             check_event(expr, info)
 965         else:
 966             assert False, 'unexpected meta type'
 967
 968         if doc:
 969             doc.check_expr(expr)
 970
 971     return exprs
 972
 973
 974 #
 975 # Schema compiler frontend
 976 #
 977
 978 class QAPISchemaEntity(object):
 979     def __init__(self, name, info, doc):
 980         assert isinstance(name, str)
 981         self.name = name
 982         # For explicitly defined entities, info points to the (explicit)
 983         # definition.  For builtins (and their arrays), info is None.
 984         # For implicitly defined entities, info points to a place that
 985         # triggered the implicit definition (there may be more than one
 986         # such place).
 987         self.info = info
 988         self.doc = doc
 989
 990     def c_name(self):
 991         return c_name(self.name)
 992
 993     def check(self, schema):
 994         pass
 995
 996     def is_implicit(self):
 997         return not self.info
 998
 999     def visit(self, visitor):
1000         pass
1001
1002
1003 class QAPISchemaVisitor(object):
1004     def visit_begin(self, schema):
1005         pass
1006
1007     def visit_end(self):
1008         pass
1009
1010     def visit_needed(self, entity):
1011         # Default to visiting everything
1012         return True
1013
1014     def visit_builtin_type(self, name, info, json_type):
1015         pass
1016
1017     def visit_enum_type(self, name, info, values, prefix):
1018         pass
1019
1020     def visit_array_type(self, name, info, element_type):
1021         pass
1022
1023     def visit_object_type(self, name, info, base, members, variants):
1024         pass
1025
1026     def visit_object_type_flat(self, name, info, members, variants):
1027         pass
1028
1029     def visit_alternate_type(self, name, info, variants):
1030         pass
1031
1032     def visit_command(self, name, info, arg_type, ret_type,
1033                       gen, success_response, boxed):
1034         pass
1035
1036     def visit_event(self, name, info, arg_type, boxed):
1037         pass
1038
1039
1040 class QAPISchemaType(QAPISchemaEntity):
1041     # Return the C type for common use.
1042     # For the types we commonly box, this is a pointer type.
1043     def c_type(self):
1044         pass
1045
1046     # Return the C type to be used in a parameter list.
1047     def c_param_type(self):
1048         return self.c_type()
1049
1050     # Return the C type to be used where we suppress boxing.
1051     def c_unboxed_type(self):
1052         return self.c_type()
1053
1054     def json_type(self):
1055         pass
1056
1057     def alternate_qtype(self):
1058         json2qtype = {
1059             'string':  'QTYPE_QSTRING',
1060             'number':  'QTYPE_QNUM',
1061             'int':     'QTYPE_QNUM',
1062             'boolean': 'QTYPE_QBOOL',
1063             'object':  'QTYPE_QDICT'
1064         }
1065         return json2qtype.get(self.json_type())
1066
1067     def doc_type(self):
1068         if self.is_implicit():
1069             return None
1070         return self.name
1071
1072
1073 class QAPISchemaBuiltinType(QAPISchemaType):
1074     def __init__(self, name, json_type, c_type):
1075         QAPISchemaType.__init__(self, name, None, None)
1076         assert not c_type or isinstance(c_type, str)
1077         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
1078                              'value')
1079         self._json_type_name = json_type
1080         self._c_type_name = c_type
1081
1082     def c_name(self):
1083         return self.name
1084
1085     def c_type(self):
1086         return self._c_type_name
1087
1088     def c_param_type(self):
1089         if self.name == 'str':
1090             return 'const ' + self._c_type_name
1091         return self._c_type_name
1092
1093     def json_type(self):
1094         return self._json_type_name
1095
1096     def doc_type(self):
1097         return self.json_type()
1098
1099     def visit(self, visitor):
1100         visitor.visit_builtin_type(self.name, self.info, self.json_type())
1101
1102
1103 class QAPISchemaEnumType(QAPISchemaType):
1104     def __init__(self, name, info, doc, values, prefix):
1105         QAPISchemaType.__init__(self, name, info, doc)
1106         for v in values:
1107             assert isinstance(v, QAPISchemaMember)
1108             v.set_owner(name)
1109         assert prefix is None or isinstance(prefix, str)
1110         self.values = values
1111         self.prefix = prefix
1112
1113     def check(self, schema):
1114         seen = {}
1115         for v in self.values:
1116             v.check_clash(self.info, seen)
1117             if self.doc:
1118                 self.doc.connect_member(v)
1119
1120     def is_implicit(self):
1121         # See QAPISchema._make_implicit_enum_type() and ._def_predefineds()
1122         return self.name.endswith('Kind') or self.name == 'QType'
1123
1124     def c_type(self):
1125         return c_name(self.name)
1126
1127     def member_names(self):
1128         return [v.name for v in self.values]
1129
1130     def json_type(self):
1131         return 'string'
1132
1133     def visit(self, visitor):
1134         visitor.visit_enum_type(self.name, self.info,
1135                                 self.member_names(), self.prefix)
1136
1137
1138 class QAPISchemaArrayType(QAPISchemaType):
1139     def __init__(self, name, info, element_type):
1140         QAPISchemaType.__init__(self, name, info, None)
1141         assert isinstance(element_type, str)
1142         self._element_type_name = element_type
1143         self.element_type = None
1144
1145     def check(self, schema):
1146         self.element_type = schema.lookup_type(self._element_type_name)
1147         assert self.element_type
1148
1149     def is_implicit(self):
1150         return True
1151
1152     def c_type(self):
1153         return c_name(self.name) + pointer_suffix
1154
1155     def json_type(self):
1156         return 'array'
1157
1158     def doc_type(self):
1159         elt_doc_type = self.element_type.doc_type()
1160         if not elt_doc_type:
1161             return None
1162         return 'array of ' + elt_doc_type
1163
1164     def visit(self, visitor):
1165         visitor.visit_array_type(self.name, self.info, self.element_type)
1166
1167
1168 class QAPISchemaObjectType(QAPISchemaType):
1169     def __init__(self, name, info, doc, base, local_members, variants):
1170         # struct has local_members, optional base, and no variants
1171         # flat union has base, variants, and no local_members
1172         # simple union has local_members, variants, and no base
1173         QAPISchemaType.__init__(self, name, info, doc)
1174         assert base is None or isinstance(base, str)
1175         for m in local_members:
1176             assert isinstance(m, QAPISchemaObjectTypeMember)
1177             m.set_owner(name)
1178         if variants is not None:
1179             assert isinstance(variants, QAPISchemaObjectTypeVariants)
1180             variants.set_owner(name)
1181         self._base_name = base
1182         self.base = None
1183         self.local_members = local_members
1184         self.variants = variants
1185         self.members = None
1186
1187     def check(self, schema):
1188         if self.members is False:               # check for cycles
1189             raise QAPISemError(self.info,
1190                                "Object %s contains itself" % self.name)
1191         if self.members:
1192             return
1193         self.members = False                    # mark as being checked
1194         seen = OrderedDict()
1195         if self._base_name:
1196             self.base = schema.lookup_type(self._base_name)
1197             assert isinstance(self.base, QAPISchemaObjectType)
1198             self.base.check(schema)
1199             self.base.check_clash(self.info, seen)
1200         for m in self.local_members:
1201             m.check(schema)
1202             m.check_clash(self.info, seen)
1203             if self.doc:
1204                 self.doc.connect_member(m)
1205         self.members = seen.values()
1206         if self.variants:
1207             self.variants.check(schema, seen)
1208             assert self.variants.tag_member in self.members
1209             self.variants.check_clash(self.info, seen)
1210         if self.doc:
1211             self.doc.check()
1212
1213     # Check that the members of this type do not cause duplicate JSON members,
1214     # and update seen to track the members seen so far. Report any errors
1215     # on behalf of info, which is not necessarily self.info
1216     def check_clash(self, info, seen):
1217         assert not self.variants       # not implemented
1218         for m in self.members:
1219             m.check_clash(info, seen)
1220
1221     def is_implicit(self):
1222         # See QAPISchema._make_implicit_object_type(), as well as
1223         # _def_predefineds()
1224         return self.name.startswith('q_')
1225
1226     def is_empty(self):
1227         assert self.members is not None
1228         return not self.members and not self.variants
1229
1230     def c_name(self):
1231         assert self.name != 'q_empty'
1232         return QAPISchemaType.c_name(self)
1233
1234     def c_type(self):
1235         assert not self.is_implicit()
1236         return c_name(self.name) + pointer_suffix
1237
1238     def c_unboxed_type(self):
1239         return c_name(self.name)
1240
1241     def json_type(self):
1242         return 'object'
1243
1244     def visit(self, visitor):
1245         visitor.visit_object_type(self.name, self.info,
1246                                   self.base, self.local_members, self.variants)
1247         visitor.visit_object_type_flat(self.name, self.info,
1248                                        self.members, self.variants)
1249
1250
1251 class QAPISchemaMember(object):
1252     role = 'member'
1253
1254     def __init__(self, name):
1255         assert isinstance(name, str)
1256         self.name = name
1257         self.owner = None
1258
1259     def set_owner(self, name):
1260         assert not self.owner
1261         self.owner = name
1262
1263     def check_clash(self, info, seen):
1264         cname = c_name(self.name)
1265         if cname.lower() != cname and self.owner not in name_case_whitelist:
1266             raise QAPISemError(info,
1267                                "%s should not use uppercase" % self.describe())
1268         if cname in seen:
1269             raise QAPISemError(info, "%s collides with %s" %
1270                                (self.describe(), seen[cname].describe()))
1271         seen[cname] = self
1272
1273     def _pretty_owner(self):
1274         owner = self.owner
1275         if owner.startswith('q_obj_'):
1276             # See QAPISchema._make_implicit_object_type() - reverse the
1277             # mapping there to create a nice human-readable description
1278             owner = owner[6:]
1279             if owner.endswith('-arg'):
1280                 return '(parameter of %s)' % owner[:-4]
1281             elif owner.endswith('-base'):
1282                 return '(base of %s)' % owner[:-5]
1283             else:
1284                 assert owner.endswith('-wrapper')
1285                 # Unreachable and not implemented
1286                 assert False
1287         if owner.endswith('Kind'):
1288             # See QAPISchema._make_implicit_enum_type()
1289             return '(branch of %s)' % owner[:-4]
1290         return '(%s of %s)' % (self.role, owner)
1291
1292     def describe(self):
1293         return "'%s' %s" % (self.name, self._pretty_owner())
1294
1295
1296 class QAPISchemaObjectTypeMember(QAPISchemaMember):
1297     def __init__(self, name, typ, optional):
1298         QAPISchemaMember.__init__(self, name)
1299         assert isinstance(typ, str)
1300         assert isinstance(optional, bool)
1301         self._type_name = typ
1302         self.type = None
1303         self.optional = optional
1304
1305     def check(self, schema):
1306         assert self.owner
1307         self.type = schema.lookup_type(self._type_name)
1308         assert self.type
1309
1310
1311 class QAPISchemaObjectTypeVariants(object):
1312     def __init__(self, tag_name, tag_member, variants):
1313         # Flat unions pass tag_name but not tag_member.
1314         # Simple unions and alternates pass tag_member but not tag_name.
1315         # After check(), tag_member is always set, and tag_name remains
1316         # a reliable witness of being used by a flat union.
1317         assert bool(tag_member) != bool(tag_name)
1318         assert (isinstance(tag_name, str) or
1319                 isinstance(tag_member, QAPISchemaObjectTypeMember))
1320         assert len(variants) > 0
1321         for v in variants:
1322             assert isinstance(v, QAPISchemaObjectTypeVariant)
1323         self._tag_name = tag_name
1324         self.tag_member = tag_member
1325         self.variants = variants
1326
1327     def set_owner(self, name):
1328         for v in self.variants:
1329             v.set_owner(name)
1330
1331     def check(self, schema, seen):
1332         if not self.tag_member:    # flat union
1333             self.tag_member = seen[c_name(self._tag_name)]
1334             assert self._tag_name == self.tag_member.name
1335         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
1336         for v in self.variants:
1337             v.check(schema)
1338             # Union names must match enum values; alternate names are
1339             # checked separately. Use 'seen' to tell the two apart.
1340             if seen:
1341                 assert v.name in self.tag_member.type.member_names()
1342                 assert isinstance(v.type, QAPISchemaObjectType)
1343                 v.type.check(schema)
1344
1345     def check_clash(self, info, seen):
1346         for v in self.variants:
1347             # Reset seen map for each variant, since qapi names from one
1348             # branch do not affect another branch
1349             assert isinstance(v.type, QAPISchemaObjectType)
1350             v.type.check_clash(info, dict(seen))
1351
1352
1353 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1354     role = 'branch'
1355
1356     def __init__(self, name, typ):
1357         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1358
1359
1360 class QAPISchemaAlternateType(QAPISchemaType):
1361     def __init__(self, name, info, doc, variants):
1362         QAPISchemaType.__init__(self, name, info, doc)
1363         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1364         assert variants.tag_member
1365         variants.set_owner(name)
1366         variants.tag_member.set_owner(self.name)
1367         self.variants = variants
1368
1369     def check(self, schema):
1370         self.variants.tag_member.check(schema)
1371         # Not calling self.variants.check_clash(), because there's nothing
1372         # to clash with
1373         self.variants.check(schema, {})
1374         # Alternate branch names have no relation to the tag enum values;
1375         # so we have to check for potential name collisions ourselves.
1376         seen = {}
1377         for v in self.variants.variants:
1378             v.check_clash(self.info, seen)
1379             if self.doc:
1380                 self.doc.connect_member(v)
1381         if self.doc:
1382             self.doc.check()
1383
1384     def c_type(self):
1385         return c_name(self.name) + pointer_suffix
1386
1387     def json_type(self):
1388         return 'value'
1389
1390     def visit(self, visitor):
1391         visitor.visit_alternate_type(self.name, self.info, self.variants)
1392
1393     def is_empty(self):
1394         return False
1395
1396
1397 class QAPISchemaCommand(QAPISchemaEntity):
1398     def __init__(self, name, info, doc, arg_type, ret_type,
1399                  gen, success_response, boxed):
1400         QAPISchemaEntity.__init__(self, name, info, doc)
1401         assert not arg_type or isinstance(arg_type, str)
1402         assert not ret_type or isinstance(ret_type, str)
1403         self._arg_type_name = arg_type
1404         self.arg_type = None
1405         self._ret_type_name = ret_type
1406         self.ret_type = None
1407         self.gen = gen
1408         self.success_response = success_response
1409         self.boxed = boxed
1410
1411     def check(self, schema):
1412         if self._arg_type_name:
1413             self.arg_type = schema.lookup_type(self._arg_type_name)
1414             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1415                     isinstance(self.arg_type, QAPISchemaAlternateType))
1416             self.arg_type.check(schema)
1417             if self.boxed:
1418                 if self.arg_type.is_empty():
1419                     raise QAPISemError(self.info,
1420                                        "Cannot use 'boxed' with empty type")
1421             else:
1422                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1423                 assert not self.arg_type.variants
1424         elif self.boxed:
1425             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1426         if self._ret_type_name:
1427             self.ret_type = schema.lookup_type(self._ret_type_name)
1428             assert isinstance(self.ret_type, QAPISchemaType)
1429
1430     def visit(self, visitor):
1431         visitor.visit_command(self.name, self.info,
1432                               self.arg_type, self.ret_type,
1433                               self.gen, self.success_response, self.boxed)
1434
1435
1436 class QAPISchemaEvent(QAPISchemaEntity):
1437     def __init__(self, name, info, doc, arg_type, boxed):
1438         QAPISchemaEntity.__init__(self, name, info, doc)
1439         assert not arg_type or isinstance(arg_type, str)
1440         self._arg_type_name = arg_type
1441         self.arg_type = None
1442         self.boxed = boxed
1443
1444     def check(self, schema):
1445         if self._arg_type_name:
1446             self.arg_type = schema.lookup_type(self._arg_type_name)
1447             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1448                     isinstance(self.arg_type, QAPISchemaAlternateType))
1449             self.arg_type.check(schema)
1450             if self.boxed:
1451                 if self.arg_type.is_empty():
1452                     raise QAPISemError(self.info,
1453                                        "Cannot use 'boxed' with empty type")
1454             else:
1455                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1456                 assert not self.arg_type.variants
1457         elif self.boxed:
1458             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1459
1460     def visit(self, visitor):
1461         visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
1462
1463
1464 class QAPISchema(object):
1465     def __init__(self, fname):
1466         try:
1467             parser = QAPISchemaParser(open(fname, 'r'))
1468             self.exprs = check_exprs(parser.exprs)
1469             self.docs = parser.docs
1470             self._entity_dict = {}
1471             self._predefining = True
1472             self._def_predefineds()
1473             self._predefining = False
1474             self._def_exprs()
1475             self.check()
1476         except QAPIError as err:
1477             print >>sys.stderr, err
1478             exit(1)
1479
1480     def _def_entity(self, ent):
1481         # Only the predefined types are allowed to not have info
1482         assert ent.info or self._predefining
1483         assert ent.name not in self._entity_dict
1484         self._entity_dict[ent.name] = ent
1485
1486     def lookup_entity(self, name, typ=None):
1487         ent = self._entity_dict.get(name)
1488         if typ and not isinstance(ent, typ):
1489             return None
1490         return ent
1491
1492     def lookup_type(self, name):
1493         return self.lookup_entity(name, QAPISchemaType)
1494
1495     def _def_builtin_type(self, name, json_type, c_type):
1496         self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
1497         # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
1498         # qapi-types.h from a single .c, all arrays of builtins must be
1499         # declared in the first file whether or not they are used.  Nicer
1500         # would be to use lazy instantiation, while figuring out how to
1501         # avoid compilation issues with multiple qapi-types.h.
1502         self._make_array_type(name, None)
1503
1504     def _def_predefineds(self):
1505         for t in [('str',    'string',  'char' + pointer_suffix),
1506                   ('number', 'number',  'double'),
1507                   ('int',    'int',     'int64_t'),
1508                   ('int8',   'int',     'int8_t'),
1509                   ('int16',  'int',     'int16_t'),
1510                   ('int32',  'int',     'int32_t'),
1511                   ('int64',  'int',     'int64_t'),
1512                   ('uint8',  'int',     'uint8_t'),
1513                   ('uint16', 'int',     'uint16_t'),
1514                   ('uint32', 'int',     'uint32_t'),
1515                   ('uint64', 'int',     'uint64_t'),
1516                   ('size',   'int',     'uint64_t'),
1517                   ('bool',   'boolean', 'bool'),
1518                   ('any',    'value',   'QObject' + pointer_suffix)]:
1519             self._def_builtin_type(*t)
1520         self.the_empty_object_type = QAPISchemaObjectType(
1521             'q_empty', None, None, None, [], None)
1522         self._def_entity(self.the_empty_object_type)
1523         qtype_values = self._make_enum_members(['none', 'qnull', 'qnum',
1524                                                 'qstring', 'qdict', 'qlist',
1525                                                 'qbool'])
1526         self._def_entity(QAPISchemaEnumType('QType', None, None,
1527                                             qtype_values, 'QTYPE'))
1528
1529     def _make_enum_members(self, values):
1530         return [QAPISchemaMember(v) for v in values]
1531
1532     def _make_implicit_enum_type(self, name, info, values):
1533         # See also QAPISchemaObjectTypeMember._pretty_owner()
1534         name = name + 'Kind'   # Use namespace reserved by add_name()
1535         self._def_entity(QAPISchemaEnumType(
1536             name, info, None, self._make_enum_members(values), None))
1537         return name
1538
1539     def _make_array_type(self, element_type, info):
1540         name = element_type + 'List'   # Use namespace reserved by add_name()
1541         if not self.lookup_type(name):
1542             self._def_entity(QAPISchemaArrayType(name, info, element_type))
1543         return name
1544
1545     def _make_implicit_object_type(self, name, info, doc, role, members):
1546         if not members:
1547             return None
1548         # See also QAPISchemaObjectTypeMember._pretty_owner()
1549         name = 'q_obj_%s-%s' % (name, role)
1550         if not self.lookup_entity(name, QAPISchemaObjectType):
1551             self._def_entity(QAPISchemaObjectType(name, info, doc, None,
1552                                                   members, None))
1553         return name
1554
1555     def _def_enum_type(self, expr, info, doc):
1556         name = expr['enum']
1557         data = expr['data']
1558         prefix = expr.get('prefix')
1559         self._def_entity(QAPISchemaEnumType(
1560             name, info, doc, self._make_enum_members(data), prefix))
1561
1562     def _make_member(self, name, typ, info):
1563         optional = False
1564         if name.startswith('*'):
1565             name = name[1:]
1566             optional = True
1567         if isinstance(typ, list):
1568             assert len(typ) == 1
1569             typ = self._make_array_type(typ[0], info)
1570         return QAPISchemaObjectTypeMember(name, typ, optional)
1571
1572     def _make_members(self, data, info):
1573         return [self._make_member(key, value, info)
1574                 for (key, value) in data.iteritems()]
1575
1576     def _def_struct_type(self, expr, info, doc):
1577         name = expr['struct']
1578         base = expr.get('base')
1579         data = expr['data']
1580         self._def_entity(QAPISchemaObjectType(name, info, doc, base,
1581                                               self._make_members(data, info),
1582                                               None))
1583
1584     def _make_variant(self, case, typ):
1585         return QAPISchemaObjectTypeVariant(case, typ)
1586
1587     def _make_simple_variant(self, case, typ, info):
1588         if isinstance(typ, list):
1589             assert len(typ) == 1
1590             typ = self._make_array_type(typ[0], info)
1591         typ = self._make_implicit_object_type(
1592             typ, info, None, 'wrapper', [self._make_member('data', typ, info)])
1593         return QAPISchemaObjectTypeVariant(case, typ)
1594
1595     def _def_union_type(self, expr, info, doc):
1596         name = expr['union']
1597         data = expr['data']
1598         base = expr.get('base')
1599         tag_name = expr.get('discriminator')
1600         tag_member = None
1601         if isinstance(base, dict):
1602             base = (self._make_implicit_object_type(
1603                 name, info, doc, 'base', self._make_members(base, info)))
1604         if tag_name:
1605             variants = [self._make_variant(key, value)
1606                         for (key, value) in data.iteritems()]
1607             members = []
1608         else:
1609             variants = [self._make_simple_variant(key, value, info)
1610                         for (key, value) in data.iteritems()]
1611             typ = self._make_implicit_enum_type(name, info,
1612                                                 [v.name for v in variants])
1613             tag_member = QAPISchemaObjectTypeMember('type', typ, False)
1614             members = [tag_member]
1615         self._def_entity(
1616             QAPISchemaObjectType(name, info, doc, base, members,
1617                                  QAPISchemaObjectTypeVariants(tag_name,
1618                                                               tag_member,
1619                                                               variants)))
1620
1621     def _def_alternate_type(self, expr, info, doc):
1622         name = expr['alternate']
1623         data = expr['data']
1624         variants = [self._make_variant(key, value)
1625                     for (key, value) in data.iteritems()]
1626         tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
1627         self._def_entity(
1628             QAPISchemaAlternateType(name, info, doc,
1629                                     QAPISchemaObjectTypeVariants(None,
1630                                                                  tag_member,
1631                                                                  variants)))
1632
1633     def _def_command(self, expr, info, doc):
1634         name = expr['command']
1635         data = expr.get('data')
1636         rets = expr.get('returns')
1637         gen = expr.get('gen', True)
1638         success_response = expr.get('success-response', True)
1639         boxed = expr.get('boxed', False)
1640         if isinstance(data, OrderedDict):
1641             data = self._make_implicit_object_type(
1642                 name, info, doc, 'arg', self._make_members(data, info))
1643         if isinstance(rets, list):
1644             assert len(rets) == 1
1645             rets = self._make_array_type(rets[0], info)
1646         self._def_entity(QAPISchemaCommand(name, info, doc, data, rets,
1647                                            gen, success_response, boxed))
1648
1649     def _def_event(self, expr, info, doc):
1650         name = expr['event']
1651         data = expr.get('data')
1652         boxed = expr.get('boxed', False)
1653         if isinstance(data, OrderedDict):
1654             data = self._make_implicit_object_type(
1655                 name, info, doc, 'arg', self._make_members(data, info))
1656         self._def_entity(QAPISchemaEvent(name, info, doc, data, boxed))
1657
1658     def _def_exprs(self):
1659         for expr_elem in self.exprs:
1660             expr = expr_elem['expr']
1661             info = expr_elem['info']
1662             doc = expr_elem.get('doc')
1663             if 'enum' in expr:
1664                 self._def_enum_type(expr, info, doc)
1665             elif 'struct' in expr:
1666                 self._def_struct_type(expr, info, doc)
1667             elif 'union' in expr:
1668                 self._def_union_type(expr, info, doc)
1669             elif 'alternate' in expr:
1670                 self._def_alternate_type(expr, info, doc)
1671             elif 'command' in expr:
1672                 self._def_command(expr, info, doc)
1673             elif 'event' in expr:
1674                 self._def_event(expr, info, doc)
1675             else:
1676                 assert False
1677
1678     def check(self):
1679         for ent in self._entity_dict.values():
1680             ent.check(self)
1681
1682     def visit(self, visitor):
1683         visitor.visit_begin(self)
1684         for (name, entity) in sorted(self._entity_dict.items()):
1685             if visitor.visit_needed(entity):
1686                 entity.visit(visitor)
1687         visitor.visit_end()
1688
1689
1690 #
1691 # Code generation helpers
1692 #
1693
1694 def camel_case(name):
1695     new_name = ''
1696     first = True
1697     for ch in name:
1698         if ch in ['_', '-']:
1699             first = True
1700         elif first:
1701             new_name += ch.upper()
1702             first = False
1703         else:
1704             new_name += ch.lower()
1705     return new_name
1706
1707
1708 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1709 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1710 # ENUM24_Name -> ENUM24_NAME
1711 def camel_to_upper(value):
1712     c_fun_str = c_name(value, False)
1713     if value.isupper():
1714         return c_fun_str
1715
1716     new_name = ''
1717     l = len(c_fun_str)
1718     for i in range(l):
1719         c = c_fun_str[i]
1720         # When c is upper and no '_' appears before, do more checks
1721         if c.isupper() and (i > 0) and c_fun_str[i - 1] != '_':
1722             if i < l - 1 and c_fun_str[i + 1].islower():
1723                 new_name += '_'
1724             elif c_fun_str[i - 1].isdigit():
1725                 new_name += '_'
1726         new_name += c
1727     return new_name.lstrip('_').upper()
1728
1729
1730 def c_enum_const(type_name, const_name, prefix=None):
1731     if prefix is not None:
1732         type_name = prefix
1733     return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
1734
1735 c_name_trans = string.maketrans('.-', '__')
1736
1737
1738 # Map @name to a valid C identifier.
1739 # If @protect, avoid returning certain ticklish identifiers (like
1740 # C keywords) by prepending 'q_'.
1741 #
1742 # Used for converting 'name' from a 'name':'type' qapi definition
1743 # into a generated struct member, as well as converting type names
1744 # into substrings of a generated C function name.
1745 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1746 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1747 def c_name(name, protect=True):
1748     # ANSI X3J11/88-090, 3.1.1
1749     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1750                      'default', 'do', 'double', 'else', 'enum', 'extern',
1751                      'float', 'for', 'goto', 'if', 'int', 'long', 'register',
1752                      'return', 'short', 'signed', 'sizeof', 'static',
1753                      'struct', 'switch', 'typedef', 'union', 'unsigned',
1754                      'void', 'volatile', 'while'])
1755     # ISO/IEC 9899:1999, 6.4.1
1756     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1757     # ISO/IEC 9899:2011, 6.4.1
1758     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
1759                      '_Noreturn', '_Static_assert', '_Thread_local'])
1760     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1761     # excluding _.*
1762     gcc_words = set(['asm', 'typeof'])
1763     # C++ ISO/IEC 14882:2003 2.11
1764     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1765                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1766                      'namespace', 'new', 'operator', 'private', 'protected',
1767                      'public', 'reinterpret_cast', 'static_cast', 'template',
1768                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1769                      'using', 'virtual', 'wchar_t',
1770                      # alternative representations
1771                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1772                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1773     # namespace pollution:
1774     polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
1775     name = name.translate(c_name_trans)
1776     if protect and (name in c89_words | c99_words | c11_words | gcc_words
1777                     | cpp_words | polluted_words):
1778         return 'q_' + name
1779     return name
1780
1781 eatspace = '\033EATSPACE.'
1782 pointer_suffix = ' *' + eatspace
1783
1784
1785 def genindent(count):
1786     ret = ''
1787     for _ in range(count):
1788         ret += ' '
1789     return ret
1790
1791 indent_level = 0
1792
1793
1794 def push_indent(indent_amount=4):
1795     global indent_level
1796     indent_level += indent_amount
1797
1798
1799 def pop_indent(indent_amount=4):
1800     global indent_level
1801     indent_level -= indent_amount
1802
1803
1804 # Generate @code with @kwds interpolated.
1805 # Obey indent_level, and strip eatspace.
1806 def cgen(code, **kwds):
1807     raw = code % kwds
1808     if indent_level:
1809         indent = genindent(indent_level)
1810         # re.subn() lacks flags support before Python 2.7, use re.compile()
1811         raw = re.subn(re.compile(r'^.', re.MULTILINE),
1812                       indent + r'\g<0>', raw)
1813         raw = raw[0]
1814     return re.sub(re.escape(eatspace) + r' *', '', raw)
1815
1816
1817 def mcgen(code, **kwds):
1818     if code[0] == '\n':
1819         code = code[1:]
1820     return cgen(code, **kwds)
1821
1822
1823 def guardname(filename):
1824     return c_name(filename, protect=False).upper()
1825
1826
1827 def guardstart(name):
1828     return mcgen('''
1829
1830 #ifndef %(name)s
1831 #define %(name)s
1832
1833 ''',
1834                  name=guardname(name))
1835
1836
1837 def guardend(name):
1838     return mcgen('''
1839
1840 #endif /* %(name)s */
1841
1842 ''',
1843                  name=guardname(name))
1844
1845
1846 def gen_enum_lookup(name, values, prefix=None):
1847     ret = mcgen('''
1848
1849 const char *const %(c_name)s_lookup[] = {
1850 ''',
1851                 c_name=c_name(name))
1852     for value in values:
1853         index = c_enum_const(name, value, prefix)
1854         ret += mcgen('''
1855     [%(index)s] = "%(value)s",
1856 ''',
1857                      index=index, value=value)
1858
1859     max_index = c_enum_const(name, '_MAX', prefix)
1860     ret += mcgen('''
1861     [%(max_index)s] = NULL,
1862 };
1863 ''',
1864                  max_index=max_index)
1865     return ret
1866
1867
1868 def gen_enum(name, values, prefix=None):
1869     # append automatically generated _MAX value
1870     enum_values = values + ['_MAX']
1871
1872     ret = mcgen('''
1873
1874 typedef enum %(c_name)s {
1875 ''',
1876                 c_name=c_name(name))
1877
1878     i = 0
1879     for value in enum_values:
1880         ret += mcgen('''
1881     %(c_enum)s = %(i)d,
1882 ''',
1883                      c_enum=c_enum_const(name, value, prefix),
1884                      i=i)
1885         i += 1
1886
1887     ret += mcgen('''
1888 } %(c_name)s;
1889 ''',
1890                  c_name=c_name(name))
1891
1892     ret += mcgen('''
1893
1894 extern const char *const %(c_name)s_lookup[];
1895 ''',
1896                  c_name=c_name(name))
1897     return ret
1898
1899
1900 def gen_params(arg_type, boxed, extra):
1901     if not arg_type:
1902         assert not boxed
1903         return extra
1904     ret = ''
1905     sep = ''
1906     if boxed:
1907         ret += '%s arg' % arg_type.c_param_type()
1908         sep = ', '
1909     else:
1910         assert not arg_type.variants
1911         for memb in arg_type.members:
1912             ret += sep
1913             sep = ', '
1914             if memb.optional:
1915                 ret += 'bool has_%s, ' % c_name(memb.name)
1916             ret += '%s %s' % (memb.type.c_param_type(),
1917                               c_name(memb.name))
1918     if extra:
1919         ret += sep + extra
1920     return ret
1921
1922
1923 #
1924 # Common command line parsing
1925 #
1926
1927
1928 def parse_command_line(extra_options='', extra_long_options=[]):
1929
1930     try:
1931         opts, args = getopt.gnu_getopt(sys.argv[1:],
1932                                        'chp:o:' + extra_options,
1933                                        ['source', 'header', 'prefix=',
1934                                         'output-dir='] + extra_long_options)
1935     except getopt.GetoptError as err:
1936         print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
1937         sys.exit(1)
1938
1939     output_dir = ''
1940     prefix = ''
1941     do_c = False
1942     do_h = False
1943     extra_opts = []
1944
1945     for oa in opts:
1946         o, a = oa
1947         if o in ('-p', '--prefix'):
1948             match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1949             if match.end() != len(a):
1950                 print >>sys.stderr, \
1951                     "%s: 'funny character '%s' in argument of --prefix" \
1952                     % (sys.argv[0], a[match.end()])
1953                 sys.exit(1)
1954             prefix = a
1955         elif o in ('-o', '--output-dir'):
1956             output_dir = a + '/'
1957         elif o in ('-c', '--source'):
1958             do_c = True
1959         elif o in ('-h', '--header'):
1960             do_h = True
1961         else:
1962             extra_opts.append(oa)
1963
1964     if not do_c and not do_h:
1965         do_c = True
1966         do_h = True
1967
1968     if len(args) != 1:
1969         print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
1970         sys.exit(1)
1971     fname = args[0]
1972
1973     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
1974
1975 #
1976 # Generate output files with boilerplate
1977 #
1978
1979
1980 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
1981                 c_comment, h_comment):
1982     guard = guardname(prefix + h_file)
1983     c_file = output_dir + prefix + c_file
1984     h_file = output_dir + prefix + h_file
1985
1986     if output_dir:
1987         try:
1988             os.makedirs(output_dir)
1989         except os.error as e:
1990             if e.errno != errno.EEXIST:
1991                 raise
1992
1993     def maybe_open(really, name, opt):
1994         if really:
1995             return open(name, opt)
1996         else:
1997             import StringIO
1998             return StringIO.StringIO()
1999
2000     fdef = maybe_open(do_c, c_file, 'w')
2001     fdecl = maybe_open(do_h, h_file, 'w')
2002
2003     fdef.write(mcgen('''
2004 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2005 %(comment)s
2006 ''',
2007                      comment=c_comment))
2008
2009     fdecl.write(mcgen('''
2010 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2011 %(comment)s
2012 #ifndef %(guard)s
2013 #define %(guard)s
2014
2015 ''',
2016                       comment=h_comment, guard=guard))
2017
2018     return (fdef, fdecl)
2019
2020
2021 def close_output(fdef, fdecl):
2022     fdecl.write('''
2023 #endif
2024 ''')
2025     fdecl.close()
2026     fdef.close()