scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 import re
  15 from ordereddict import OrderedDict
  16 import errno
  17 import getopt
  18 import os
  19 import sys
  20 import string
  21
  22 builtin_types = {
  23     'str':      'QTYPE_QSTRING',
  24     'int':      'QTYPE_QINT',
  25     'number':   'QTYPE_QFLOAT',
  26     'bool':     'QTYPE_QBOOL',
  27     'int8':     'QTYPE_QINT',
  28     'int16':    'QTYPE_QINT',
  29     'int32':    'QTYPE_QINT',
  30     'int64':    'QTYPE_QINT',
  31     'uint8':    'QTYPE_QINT',
  32     'uint16':   'QTYPE_QINT',
  33     'uint32':   'QTYPE_QINT',
  34     'uint64':   'QTYPE_QINT',
  35     'size':     'QTYPE_QINT',
  36     'any':      None,           # any QType possible, actually
  37     'QType':    'QTYPE_QSTRING',
  38 }
  39
  40 # Are documentation comments required?
  41 doc_required = False
  42
  43 # Whitelist of commands allowed to return a non-dictionary
  44 returns_whitelist = [
  45     # From QMP:
  46     'human-monitor-command',
  47     'qom-get',
  48     'query-migrate-cache-size',
  49     'query-tpm-models',
  50     'query-tpm-types',
  51     'ringbuf-read',
  52
  53     # From QGA:
  54     'guest-file-open',
  55     'guest-fsfreeze-freeze',
  56     'guest-fsfreeze-freeze-list',
  57     'guest-fsfreeze-status',
  58     'guest-fsfreeze-thaw',
  59     'guest-get-time',
  60     'guest-set-vcpus',
  61     'guest-sync',
  62     'guest-sync-delimited',
  63 ]
  64
  65 # Whitelist of entities allowed to violate case conventions
  66 case_whitelist = [
  67     # From QMP:
  68     'ACPISlotType',         # DIMM, visible through query-acpi-ospm-status
  69     'CpuInfoMIPS',          # PC, visible through query-cpu
  70     'CpuInfoTricore',       # PC, visible through query-cpu
  71     'QapiErrorClass',       # all members, visible through errors
  72     'UuidInfo',             # UUID, visible through query-uuid
  73     'X86CPURegister32',     # all members, visible indirectly through qom-get
  74     'q_obj_CpuInfo-base',   # CPU, visible through query-cpu
  75 ]
  76
  77 enum_types = []
  78 struct_types = []
  79 union_types = []
  80 events = []
  81 all_names = {}
  82
  83 #
  84 # Parsing the schema into expressions
  85 #
  86
  87
  88 def error_path(parent):
  89     res = ""
  90     while parent:
  91         res = ("In file included from %s:%d:\n" % (parent['file'],
  92                                                    parent['line'])) + res
  93         parent = parent['parent']
  94     return res
  95
  96
  97 class QAPIError(Exception):
  98     def __init__(self, fname, line, col, incl_info, msg):
  99         Exception.__init__(self)
 100         self.fname = fname
 101         self.line = line
 102         self.col = col
 103         self.info = incl_info
 104         self.msg = msg
 105
 106     def __str__(self):
 107         loc = "%s:%d" % (self.fname, self.line)
 108         if self.col is not None:
 109             loc += ":%s" % self.col
 110         return error_path(self.info) + "%s: %s" % (loc, self.msg)
 111
 112
 113 class QAPIParseError(QAPIError):
 114     def __init__(self, parser, msg):
 115         col = 1
 116         for ch in parser.src[parser.line_pos:parser.pos]:
 117             if ch == '\t':
 118                 col = (col + 7) % 8 + 1
 119             else:
 120                 col += 1
 121         QAPIError.__init__(self, parser.fname, parser.line, col,
 122                            parser.incl_info, msg)
 123
 124
 125 class QAPISemError(QAPIError):
 126     def __init__(self, info, msg):
 127         QAPIError.__init__(self, info['file'], info['line'], None,
 128                            info['parent'], msg)
 129
 130
 131 class QAPIDoc(object):
 132     class Section(object):
 133         def __init__(self, name=None):
 134             # optional section name (argument/member or section name)
 135             self.name = name
 136             # the list of lines for this section
 137             self.content = []
 138
 139         def append(self, line):
 140             self.content.append(line)
 141
 142         def __repr__(self):
 143             return "\n".join(self.content).strip()
 144
 145     class ArgSection(Section):
 146         pass
 147
 148     def __init__(self, parser, info):
 149         # self.parser is used to report errors with QAPIParseError.  The
 150         # resulting error position depends on the state of the parser.
 151         # It happens to be the beginning of the comment.  More or less
 152         # servicable, but action at a distance.
 153         self.parser = parser
 154         self.info = info
 155         self.symbol = None
 156         self.body = QAPIDoc.Section()
 157         # dict mapping parameter name to ArgSection
 158         self.args = OrderedDict()
 159         # a list of Section
 160         self.sections = []
 161         # the current section
 162         self.section = self.body
 163         # associated expression (to be set by expression parser)
 164         self.expr = None
 165
 166     def has_section(self, name):
 167         """Return True if we have a section with this name."""
 168         for i in self.sections:
 169             if i.name == name:
 170                 return True
 171         return False
 172
 173     def append(self, line):
 174         """Parse a comment line and add it to the documentation."""
 175         line = line[1:]
 176         if not line:
 177             self._append_freeform(line)
 178             return
 179
 180         if line[0] != ' ':
 181             raise QAPIParseError(self.parser, "Missing space after #")
 182         line = line[1:]
 183
 184         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 185         # recognized, and get silently treated as ordinary text
 186         if self.symbol:
 187             self._append_symbol_line(line)
 188         elif not self.body.content and line.startswith("@"):
 189             if not line.endswith(":"):
 190                 raise QAPIParseError(self.parser, "Line should end with :")
 191             self.symbol = line[1:-1]
 192             # FIXME invalid names other than the empty string aren't flagged
 193             if not self.symbol:
 194                 raise QAPIParseError(self.parser, "Invalid name")
 195         else:
 196             self._append_freeform(line)
 197
 198     def _append_symbol_line(self, line):
 199         name = line.split(' ', 1)[0]
 200
 201         if name.startswith("@") and name.endswith(":"):
 202             line = line[len(name)+1:]
 203             self._start_args_section(name[1:-1])
 204         elif name in ("Returns:", "Since:",
 205                       # those are often singular or plural
 206                       "Note:", "Notes:",
 207                       "Example:", "Examples:",
 208                       "TODO:"):
 209             line = line[len(name)+1:]
 210             self._start_section(name[:-1])
 211
 212         self._append_freeform(line)
 213
 214     def _start_args_section(self, name):
 215         # FIXME invalid names other than the empty string aren't flagged
 216         if not name:
 217             raise QAPIParseError(self.parser, "Invalid parameter name")
 218         if name in self.args:
 219             raise QAPIParseError(self.parser,
 220                                  "'%s' parameter name duplicated" % name)
 221         if self.sections:
 222             raise QAPIParseError(self.parser,
 223                                  "'@%s:' can't follow '%s' section"
 224                                  % (name, self.sections[0].name))
 225         self.section = QAPIDoc.ArgSection(name)
 226         self.args[name] = self.section
 227
 228     def _start_section(self, name=""):
 229         if name in ("Returns", "Since") and self.has_section(name):
 230             raise QAPIParseError(self.parser,
 231                                  "Duplicated '%s' section" % name)
 232         self.section = QAPIDoc.Section(name)
 233         self.sections.append(self.section)
 234
 235     def _append_freeform(self, line):
 236         in_arg = isinstance(self.section, QAPIDoc.ArgSection)
 237         if (in_arg and self.section.content
 238                 and not self.section.content[-1]
 239                 and line and not line[0].isspace()):
 240             self._start_section()
 241         if (in_arg or not self.section.name
 242                 or not self.section.name.startswith("Example")):
 243             line = line.strip()
 244         self.section.append(line)
 245
 246
 247 class QAPISchemaParser(object):
 248
 249     def __init__(self, fp, previously_included=[], incl_info=None):
 250         abs_fname = os.path.abspath(fp.name)
 251         fname = fp.name
 252         self.fname = fname
 253         previously_included.append(abs_fname)
 254         self.incl_info = incl_info
 255         self.src = fp.read()
 256         if self.src == '' or self.src[-1] != '\n':
 257             self.src += '\n'
 258         self.cursor = 0
 259         self.line = 1
 260         self.line_pos = 0
 261         self.exprs = []
 262         self.docs = []
 263         self.accept()
 264
 265         while self.tok is not None:
 266             info = {'file': fname, 'line': self.line,
 267                     'parent': self.incl_info}
 268             if self.tok == '#':
 269                 doc = self.get_doc(info)
 270                 self.docs.append(doc)
 271                 continue
 272
 273             expr = self.get_expr(False)
 274             if 'include' in expr:
 275                 if len(expr) != 1:
 276                     raise QAPISemError(info, "Invalid 'include' directive")
 277                 include = expr["include"]
 278                 if not isinstance(include, str):
 279                     raise QAPISemError(info,
 280                                        "Value of 'include' must be a string")
 281                 self._include(include, info, os.path.dirname(abs_fname),
 282                               previously_included)
 283             elif "pragma" in expr:
 284                 if len(expr) != 1:
 285                     raise QAPISemError(info, "Invalid 'pragma' directive")
 286                 pragma = expr['pragma']
 287                 if not isinstance(pragma, dict):
 288                     raise QAPISemError(
 289                         info, "Value of 'pragma' must be a dictionary")
 290                 for name, value in pragma.iteritems():
 291                     self._pragma(name, value, info)
 292             else:
 293                 expr_elem = {'expr': expr,
 294                              'info': info}
 295                 if (self.docs
 296                         and self.docs[-1].info['file'] == fname
 297                         and not self.docs[-1].expr):
 298                     self.docs[-1].expr = expr
 299                     expr_elem['doc'] = self.docs[-1]
 300
 301                 self.exprs.append(expr_elem)
 302
 303     def _include(self, include, info, base_dir, previously_included):
 304         incl_abs_fname = os.path.join(base_dir, include)
 305         # catch inclusion cycle
 306         inf = info
 307         while inf:
 308             if incl_abs_fname == os.path.abspath(inf['file']):
 309                 raise QAPISemError(info, "Inclusion loop for %s" % include)
 310             inf = inf['parent']
 311
 312         # skip multiple include of the same file
 313         if incl_abs_fname in previously_included:
 314             return
 315         try:
 316             fobj = open(incl_abs_fname, 'r')
 317         except IOError as e:
 318             raise QAPISemError(info, '%s: %s' % (e.strerror, include))
 319         exprs_include = QAPISchemaParser(fobj, previously_included, info)
 320         self.exprs.extend(exprs_include.exprs)
 321         self.docs.extend(exprs_include.docs)
 322
 323     def _pragma(self, name, value, info):
 324         global doc_required
 325         if name == 'doc-required':
 326             if not isinstance(value, bool):
 327                 raise QAPISemError(info,
 328                                    "Pragma 'doc-required' must be boolean")
 329             doc_required = value
 330         else:
 331             raise QAPISemError(info, "Unknown pragma '%s'" % name)
 332
 333     def accept(self, skip_comment=True):
 334         while True:
 335             self.tok = self.src[self.cursor]
 336             self.pos = self.cursor
 337             self.cursor += 1
 338             self.val = None
 339
 340             if self.tok == '#':
 341                 if self.src[self.cursor] == '#':
 342                     # Start of doc comment
 343                     skip_comment = False
 344                 self.cursor = self.src.find('\n', self.cursor)
 345                 if not skip_comment:
 346                     self.val = self.src[self.pos:self.cursor]
 347                     return
 348             elif self.tok in "{}:,[]":
 349                 return
 350             elif self.tok == "'":
 351                 string = ''
 352                 esc = False
 353                 while True:
 354                     ch = self.src[self.cursor]
 355                     self.cursor += 1
 356                     if ch == '\n':
 357                         raise QAPIParseError(self, 'Missing terminating "\'"')
 358                     if esc:
 359                         if ch == 'b':
 360                             string += '\b'
 361                         elif ch == 'f':
 362                             string += '\f'
 363                         elif ch == 'n':
 364                             string += '\n'
 365                         elif ch == 'r':
 366                             string += '\r'
 367                         elif ch == 't':
 368                             string += '\t'
 369                         elif ch == 'u':
 370                             value = 0
 371                             for _ in range(0, 4):
 372                                 ch = self.src[self.cursor]
 373                                 self.cursor += 1
 374                                 if ch not in "0123456789abcdefABCDEF":
 375                                     raise QAPIParseError(self,
 376                                                          '\\u escape needs 4 '
 377                                                          'hex digits')
 378                                 value = (value << 4) + int(ch, 16)
 379                             # If Python 2 and 3 didn't disagree so much on
 380                             # how to handle Unicode, then we could allow
 381                             # Unicode string defaults.  But most of QAPI is
 382                             # ASCII-only, so we aren't losing much for now.
 383                             if not value or value > 0x7f:
 384                                 raise QAPIParseError(self,
 385                                                      'For now, \\u escape '
 386                                                      'only supports non-zero '
 387                                                      'values up to \\u007f')
 388                             string += chr(value)
 389                         elif ch in "\\/'\"":
 390                             string += ch
 391                         else:
 392                             raise QAPIParseError(self,
 393                                                  "Unknown escape \\%s" % ch)
 394                         esc = False
 395                     elif ch == "\\":
 396                         esc = True
 397                     elif ch == "'":
 398                         self.val = string
 399                         return
 400                     else:
 401                         string += ch
 402             elif self.src.startswith("true", self.pos):
 403                 self.val = True
 404                 self.cursor += 3
 405                 return
 406             elif self.src.startswith("false", self.pos):
 407                 self.val = False
 408                 self.cursor += 4
 409                 return
 410             elif self.src.startswith("null", self.pos):
 411                 self.val = None
 412                 self.cursor += 3
 413                 return
 414             elif self.tok == '\n':
 415                 if self.cursor == len(self.src):
 416                     self.tok = None
 417                     return
 418                 self.line += 1
 419                 self.line_pos = self.cursor
 420             elif not self.tok.isspace():
 421                 raise QAPIParseError(self, 'Stray "%s"' % self.tok)
 422
 423     def get_members(self):
 424         expr = OrderedDict()
 425         if self.tok == '}':
 426             self.accept()
 427             return expr
 428         if self.tok != "'":
 429             raise QAPIParseError(self, 'Expected string or "}"')
 430         while True:
 431             key = self.val
 432             self.accept()
 433             if self.tok != ':':
 434                 raise QAPIParseError(self, 'Expected ":"')
 435             self.accept()
 436             if key in expr:
 437                 raise QAPIParseError(self, 'Duplicate key "%s"' % key)
 438             expr[key] = self.get_expr(True)
 439             if self.tok == '}':
 440                 self.accept()
 441                 return expr
 442             if self.tok != ',':
 443                 raise QAPIParseError(self, 'Expected "," or "}"')
 444             self.accept()
 445             if self.tok != "'":
 446                 raise QAPIParseError(self, 'Expected string')
 447
 448     def get_values(self):
 449         expr = []
 450         if self.tok == ']':
 451             self.accept()
 452             return expr
 453         if self.tok not in "{['tfn":
 454             raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
 455                                  'boolean or "null"')
 456         while True:
 457             expr.append(self.get_expr(True))
 458             if self.tok == ']':
 459                 self.accept()
 460                 return expr
 461             if self.tok != ',':
 462                 raise QAPIParseError(self, 'Expected "," or "]"')
 463             self.accept()
 464
 465     def get_expr(self, nested):
 466         if self.tok != '{' and not nested:
 467             raise QAPIParseError(self, 'Expected "{"')
 468         if self.tok == '{':
 469             self.accept()
 470             expr = self.get_members()
 471         elif self.tok == '[':
 472             self.accept()
 473             expr = self.get_values()
 474         elif self.tok in "'tfn":
 475             expr = self.val
 476             self.accept()
 477         else:
 478             raise QAPIParseError(self, 'Expected "{", "[" or string')
 479         return expr
 480
 481     def get_doc(self, info):
 482         if self.val != '##':
 483             raise QAPIParseError(self, "Junk after '##' at start of "
 484                                  "documentation comment")
 485
 486         doc = QAPIDoc(self, info)
 487         self.accept(False)
 488         while self.tok == '#':
 489             if self.val.startswith('##'):
 490                 # End of doc comment
 491                 if self.val != '##':
 492                     raise QAPIParseError(self, "Junk after '##' at end of "
 493                                          "documentation comment")
 494                 self.accept()
 495                 return doc
 496             else:
 497                 doc.append(self.val)
 498             self.accept(False)
 499
 500         raise QAPIParseError(self, "Documentation comment must end with '##'")
 501
 502
 503 #
 504 # Semantic analysis of schema expressions
 505 # TODO fold into QAPISchema
 506 # TODO catching name collisions in generated code would be nice
 507 #
 508
 509
 510 def find_base_members(base):
 511     if isinstance(base, dict):
 512         return base
 513     base_struct_define = find_struct(base)
 514     if not base_struct_define:
 515         return None
 516     return base_struct_define['data']
 517
 518
 519 # Return the qtype of an alternate branch, or None on error.
 520 def find_alternate_member_qtype(qapi_type):
 521     if qapi_type in builtin_types:
 522         return builtin_types[qapi_type]
 523     elif find_struct(qapi_type):
 524         return "QTYPE_QDICT"
 525     elif find_enum(qapi_type):
 526         return "QTYPE_QSTRING"
 527     elif find_union(qapi_type):
 528         return "QTYPE_QDICT"
 529     return None
 530
 531
 532 # Return the discriminator enum define if discriminator is specified as an
 533 # enum type, otherwise return None.
 534 def discriminator_find_enum_define(expr):
 535     base = expr.get('base')
 536     discriminator = expr.get('discriminator')
 537
 538     if not (discriminator and base):
 539         return None
 540
 541     base_members = find_base_members(base)
 542     if not base_members:
 543         return None
 544
 545     discriminator_type = base_members.get(discriminator)
 546     if not discriminator_type:
 547         return None
 548
 549     return find_enum(discriminator_type)
 550
 551
 552 # Names must be letters, numbers, -, and _.  They must start with letter,
 553 # except for downstream extensions which must start with __RFQDN_.
 554 # Dots are only valid in the downstream extension prefix.
 555 valid_name = re.compile('^(__[a-zA-Z0-9.-]+_)?'
 556                         '[a-zA-Z][a-zA-Z0-9_-]*$')
 557
 558
 559 def check_name(info, source, name, allow_optional=False,
 560                enum_member=False):
 561     global valid_name
 562     membername = name
 563
 564     if not isinstance(name, str):
 565         raise QAPISemError(info, "%s requires a string name" % source)
 566     if name.startswith('*'):
 567         membername = name[1:]
 568         if not allow_optional:
 569             raise QAPISemError(info, "%s does not allow optional name '%s'"
 570                                % (source, name))
 571     # Enum members can start with a digit, because the generated C
 572     # code always prefixes it with the enum name
 573     if enum_member and membername[0].isdigit():
 574         membername = 'D' + membername
 575     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
 576     # and 'q_obj_*' implicit type names.
 577     if not valid_name.match(membername) or \
 578        c_name(membername, False).startswith('q_'):
 579         raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
 580
 581
 582 def add_name(name, info, meta, implicit=False):
 583     global all_names
 584     check_name(info, "'%s'" % meta, name)
 585     # FIXME should reject names that differ only in '_' vs. '.'
 586     # vs. '-', because they're liable to clash in generated C.
 587     if name in all_names:
 588         raise QAPISemError(info, "%s '%s' is already defined"
 589                            % (all_names[name], name))
 590     if not implicit and (name.endswith('Kind') or name.endswith('List')):
 591         raise QAPISemError(info, "%s '%s' should not end in '%s'"
 592                            % (meta, name, name[-4:]))
 593     all_names[name] = meta
 594
 595
 596 def add_struct(definition, info):
 597     global struct_types
 598     name = definition['struct']
 599     add_name(name, info, 'struct')
 600     struct_types.append(definition)
 601
 602
 603 def find_struct(name):
 604     global struct_types
 605     for struct in struct_types:
 606         if struct['struct'] == name:
 607             return struct
 608     return None
 609
 610
 611 def add_union(definition, info):
 612     global union_types
 613     name = definition['union']
 614     add_name(name, info, 'union')
 615     union_types.append(definition)
 616
 617
 618 def find_union(name):
 619     global union_types
 620     for union in union_types:
 621         if union['union'] == name:
 622             return union
 623     return None
 624
 625
 626 def add_enum(name, info, enum_values=None, implicit=False):
 627     global enum_types
 628     add_name(name, info, 'enum', implicit)
 629     enum_types.append({"enum_name": name, "enum_values": enum_values})
 630
 631
 632 def find_enum(name):
 633     global enum_types
 634     for enum in enum_types:
 635         if enum['enum_name'] == name:
 636             return enum
 637     return None
 638
 639
 640 def is_enum(name):
 641     return find_enum(name) is not None
 642
 643
 644 def check_type(info, source, value, allow_array=False,
 645                allow_dict=False, allow_optional=False,
 646                allow_metas=[]):
 647     global all_names
 648
 649     if value is None:
 650         return
 651
 652     # Check if array type for value is okay
 653     if isinstance(value, list):
 654         if not allow_array:
 655             raise QAPISemError(info, "%s cannot be an array" % source)
 656         if len(value) != 1 or not isinstance(value[0], str):
 657             raise QAPISemError(info,
 658                                "%s: array type must contain single type name" %
 659                                source)
 660         value = value[0]
 661
 662     # Check if type name for value is okay
 663     if isinstance(value, str):
 664         if value not in all_names:
 665             raise QAPISemError(info, "%s uses unknown type '%s'"
 666                                % (source, value))
 667         if not all_names[value] in allow_metas:
 668             raise QAPISemError(info, "%s cannot use %s type '%s'" %
 669                                (source, all_names[value], value))
 670         return
 671
 672     if not allow_dict:
 673         raise QAPISemError(info, "%s should be a type name" % source)
 674
 675     if not isinstance(value, OrderedDict):
 676         raise QAPISemError(info,
 677                            "%s should be a dictionary or type name" % source)
 678
 679     # value is a dictionary, check that each member is okay
 680     for (key, arg) in value.items():
 681         check_name(info, "Member of %s" % source, key,
 682                    allow_optional=allow_optional)
 683         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 684             raise QAPISemError(info, "Member of %s uses reserved name '%s'"
 685                                % (source, key))
 686         # Todo: allow dictionaries to represent default values of
 687         # an optional argument.
 688         check_type(info, "Member '%s' of %s" % (key, source), arg,
 689                    allow_array=True,
 690                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 691                                 'enum'])
 692
 693
 694 def check_command(expr, info):
 695     name = expr['command']
 696     boxed = expr.get('boxed', False)
 697
 698     args_meta = ['struct']
 699     if boxed:
 700         args_meta += ['union', 'alternate']
 701     check_type(info, "'data' for command '%s'" % name,
 702                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 703                allow_metas=args_meta)
 704     returns_meta = ['union', 'struct']
 705     if name in returns_whitelist:
 706         returns_meta += ['built-in', 'alternate', 'enum']
 707     check_type(info, "'returns' for command '%s'" % name,
 708                expr.get('returns'), allow_array=True,
 709                allow_optional=True, allow_metas=returns_meta)
 710
 711
 712 def check_event(expr, info):
 713     global events
 714     name = expr['event']
 715     boxed = expr.get('boxed', False)
 716
 717     meta = ['struct']
 718     if boxed:
 719         meta += ['union', 'alternate']
 720     events.append(name)
 721     check_type(info, "'data' for event '%s'" % name,
 722                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 723                allow_metas=meta)
 724
 725
 726 def check_union(expr, info):
 727     name = expr['union']
 728     base = expr.get('base')
 729     discriminator = expr.get('discriminator')
 730     members = expr['data']
 731
 732     # Two types of unions, determined by discriminator.
 733
 734     # With no discriminator it is a simple union.
 735     if discriminator is None:
 736         enum_define = None
 737         allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
 738         if base is not None:
 739             raise QAPISemError(info, "Simple union '%s' must not have a base" %
 740                                name)
 741
 742     # Else, it's a flat union.
 743     else:
 744         # The object must have a string or dictionary 'base'.
 745         check_type(info, "'base' for union '%s'" % name,
 746                    base, allow_dict=True, allow_optional=True,
 747                    allow_metas=['struct'])
 748         if not base:
 749             raise QAPISemError(info, "Flat union '%s' must have a base"
 750                                % name)
 751         base_members = find_base_members(base)
 752         assert base_members
 753
 754         # The value of member 'discriminator' must name a non-optional
 755         # member of the base struct.
 756         check_name(info, "Discriminator of flat union '%s'" % name,
 757                    discriminator)
 758         discriminator_type = base_members.get(discriminator)
 759         if not discriminator_type:
 760             raise QAPISemError(info,
 761                                "Discriminator '%s' is not a member of base "
 762                                "struct '%s'"
 763                                % (discriminator, base))
 764         enum_define = find_enum(discriminator_type)
 765         allow_metas = ['struct']
 766         # Do not allow string discriminator
 767         if not enum_define:
 768             raise QAPISemError(info,
 769                                "Discriminator '%s' must be of enumeration "
 770                                "type" % discriminator)
 771
 772     # Check every branch; don't allow an empty union
 773     if len(members) == 0:
 774         raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
 775     for (key, value) in members.items():
 776         check_name(info, "Member of union '%s'" % name, key)
 777
 778         # Each value must name a known type
 779         check_type(info, "Member '%s' of union '%s'" % (key, name),
 780                    value, allow_array=not base, allow_metas=allow_metas)
 781
 782         # If the discriminator names an enum type, then all members
 783         # of 'data' must also be members of the enum type.
 784         if enum_define:
 785             if key not in enum_define['enum_values']:
 786                 raise QAPISemError(info,
 787                                    "Discriminator value '%s' is not found in "
 788                                    "enum '%s'"
 789                                    % (key, enum_define["enum_name"]))
 790
 791     # If discriminator is user-defined, ensure all values are covered
 792     if enum_define:
 793         for value in enum_define['enum_values']:
 794             if value not in members.keys():
 795                 raise QAPISemError(info, "Union '%s' data missing '%s' branch"
 796                                    % (name, value))
 797
 798
 799 def check_alternate(expr, info):
 800     name = expr['alternate']
 801     members = expr['data']
 802     types_seen = {}
 803
 804     # Check every branch; require at least two branches
 805     if len(members) < 2:
 806         raise QAPISemError(info,
 807                            "Alternate '%s' should have at least two branches "
 808                            "in 'data'" % name)
 809     for (key, value) in members.items():
 810         check_name(info, "Member of alternate '%s'" % name, key)
 811
 812         # Ensure alternates have no type conflicts.
 813         check_type(info, "Member '%s' of alternate '%s'" % (key, name),
 814                    value,
 815                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 816         qtype = find_alternate_member_qtype(value)
 817         if not qtype:
 818             raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
 819                                "type '%s'" % (name, key, value))
 820         if qtype in types_seen:
 821             raise QAPISemError(info, "Alternate '%s' member '%s' can't "
 822                                "be distinguished from member '%s'"
 823                                % (name, key, types_seen[qtype]))
 824         types_seen[qtype] = key
 825
 826
 827 def check_enum(expr, info):
 828     name = expr['enum']
 829     members = expr.get('data')
 830     prefix = expr.get('prefix')
 831
 832     if not isinstance(members, list):
 833         raise QAPISemError(info,
 834                            "Enum '%s' requires an array for 'data'" % name)
 835     if prefix is not None and not isinstance(prefix, str):
 836         raise QAPISemError(info,
 837                            "Enum '%s' requires a string for 'prefix'" % name)
 838     for member in members:
 839         check_name(info, "Member of enum '%s'" % name, member,
 840                    enum_member=True)
 841
 842
 843 def check_struct(expr, info):
 844     name = expr['struct']
 845     members = expr['data']
 846
 847     check_type(info, "'data' for struct '%s'" % name, members,
 848                allow_dict=True, allow_optional=True)
 849     check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
 850                allow_metas=['struct'])
 851
 852
 853 def check_keys(expr_elem, meta, required, optional=[]):
 854     expr = expr_elem['expr']
 855     info = expr_elem['info']
 856     name = expr[meta]
 857     if not isinstance(name, str):
 858         raise QAPISemError(info, "'%s' key must have a string value" % meta)
 859     required = required + [meta]
 860     for (key, value) in expr.items():
 861         if key not in required and key not in optional:
 862             raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
 863                                % (key, meta, name))
 864         if (key == 'gen' or key == 'success-response') and value is not False:
 865             raise QAPISemError(info,
 866                                "'%s' of %s '%s' should only use false value"
 867                                % (key, meta, name))
 868         if key == 'boxed' and value is not True:
 869             raise QAPISemError(info,
 870                                "'%s' of %s '%s' should only use true value"
 871                                % (key, meta, name))
 872     for key in required:
 873         if key not in expr:
 874             raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
 875                                % (key, meta, name))
 876
 877
 878 def check_exprs(exprs):
 879     global all_names
 880
 881     # Learn the types and check for valid expression keys
 882     for builtin in builtin_types.keys():
 883         all_names[builtin] = 'built-in'
 884     for expr_elem in exprs:
 885         expr = expr_elem['expr']
 886         info = expr_elem['info']
 887
 888         if 'doc' not in expr_elem and doc_required:
 889             raise QAPISemError(info,
 890                                "Expression missing documentation comment")
 891
 892         if 'enum' in expr:
 893             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 894             add_enum(expr['enum'], info, expr['data'])
 895         elif 'union' in expr:
 896             check_keys(expr_elem, 'union', ['data'],
 897                        ['base', 'discriminator'])
 898             add_union(expr, info)
 899         elif 'alternate' in expr:
 900             check_keys(expr_elem, 'alternate', ['data'])
 901             add_name(expr['alternate'], info, 'alternate')
 902         elif 'struct' in expr:
 903             check_keys(expr_elem, 'struct', ['data'], ['base'])
 904             add_struct(expr, info)
 905         elif 'command' in expr:
 906             check_keys(expr_elem, 'command', [],
 907                        ['data', 'returns', 'gen', 'success-response', 'boxed'])
 908             add_name(expr['command'], info, 'command')
 909         elif 'event' in expr:
 910             check_keys(expr_elem, 'event', [], ['data', 'boxed'])
 911             add_name(expr['event'], info, 'event')
 912         else:
 913             raise QAPISemError(expr_elem['info'],
 914                                "Expression is missing metatype")
 915
 916     # Try again for hidden UnionKind enum
 917     for expr_elem in exprs:
 918         expr = expr_elem['expr']
 919         if 'union' in expr:
 920             if not discriminator_find_enum_define(expr):
 921                 add_enum('%sKind' % expr['union'], expr_elem['info'],
 922                          implicit=True)
 923         elif 'alternate' in expr:
 924             add_enum('%sKind' % expr['alternate'], expr_elem['info'],
 925                      implicit=True)
 926
 927     # Validate that exprs make sense
 928     for expr_elem in exprs:
 929         expr = expr_elem['expr']
 930         info = expr_elem['info']
 931
 932         if 'enum' in expr:
 933             check_enum(expr, info)
 934         elif 'union' in expr:
 935             check_union(expr, info)
 936         elif 'alternate' in expr:
 937             check_alternate(expr, info)
 938         elif 'struct' in expr:
 939             check_struct(expr, info)
 940         elif 'command' in expr:
 941             check_command(expr, info)
 942         elif 'event' in expr:
 943             check_event(expr, info)
 944         else:
 945             assert False, 'unexpected meta type'
 946
 947     return exprs
 948
 949
 950 def check_freeform_doc(doc):
 951     if doc.symbol:
 952         raise QAPISemError(doc.info,
 953                            "Documention for '%s' is not followed"
 954                            " by the definition" % doc.symbol)
 955
 956     body = str(doc.body)
 957     if re.search(r'@\S+:', body, re.MULTILINE):
 958         raise QAPISemError(doc.info,
 959                            "Free-form documentation block must not contain"
 960                            " @NAME: sections")
 961
 962
 963 def check_definition_doc(doc, expr, info):
 964     for i in ('enum', 'union', 'alternate', 'struct', 'command', 'event'):
 965         if i in expr:
 966             meta = i
 967             break
 968
 969     name = expr[meta]
 970     if doc.symbol != name:
 971         raise QAPISemError(info, "Definition of '%s' follows documentation"
 972                            " for '%s'" % (name, doc.symbol))
 973     if doc.has_section('Returns') and 'command' not in expr:
 974         raise QAPISemError(info, "'Returns:' is only valid for commands")
 975
 976     if meta == 'union':
 977         args = expr.get('base', [])
 978     else:
 979         args = expr.get('data', [])
 980     if isinstance(args, str):
 981         return
 982     if isinstance(args, dict):
 983         args = args.keys()
 984     assert isinstance(args, list)
 985
 986     if (meta == 'alternate'
 987             or (meta == 'union' and not expr.get('discriminator'))):
 988         args.append('type')
 989
 990     for arg in args:
 991         if arg[0] == '*':
 992             opt = True
 993             desc = doc.args.get(arg[1:])
 994         else:
 995             opt = False
 996             desc = doc.args.get(arg)
 997         if not desc:
 998             continue
 999         desc_opt = "#optional" in str(desc)
1000         if desc_opt and not opt:
1001             raise QAPISemError(info, "Description has #optional, "
1002                                "but the declaration doesn't")
1003         if not desc_opt and opt:
1004             # silently fix the doc
1005             # TODO either fix the schema and make this an error,
1006             # or drop #optional entirely
1007             desc.append("#optional")
1008
1009     doc_args = set(doc.args.keys())
1010     args = set([name.strip('*') for name in args])
1011     if not doc_args.issubset(args):
1012         raise QAPISemError(info, "The following documented members are not in "
1013                            "the declaration: %s" % ", ".join(doc_args - args))
1014
1015
1016 def check_docs(docs):
1017     for doc in docs:
1018         for section in doc.args.values() + doc.sections:
1019             content = str(section)
1020             if not content or content.isspace():
1021                 raise QAPISemError(doc.info,
1022                                    "Empty doc section '%s'" % section.name)
1023
1024         if not doc.expr:
1025             check_freeform_doc(doc)
1026         else:
1027             check_definition_doc(doc, doc.expr, doc.info)
1028
1029     return docs
1030
1031
1032 #
1033 # Schema compiler frontend
1034 #
1035
1036 class QAPISchemaEntity(object):
1037     def __init__(self, name, info):
1038         assert isinstance(name, str)
1039         self.name = name
1040         # For explicitly defined entities, info points to the (explicit)
1041         # definition.  For builtins (and their arrays), info is None.
1042         # For implicitly defined entities, info points to a place that
1043         # triggered the implicit definition (there may be more than one
1044         # such place).
1045         self.info = info
1046
1047     def c_name(self):
1048         return c_name(self.name)
1049
1050     def check(self, schema):
1051         pass
1052
1053     def is_implicit(self):
1054         return not self.info
1055
1056     def visit(self, visitor):
1057         pass
1058
1059
1060 class QAPISchemaVisitor(object):
1061     def visit_begin(self, schema):
1062         pass
1063
1064     def visit_end(self):
1065         pass
1066
1067     def visit_needed(self, entity):
1068         # Default to visiting everything
1069         return True
1070
1071     def visit_builtin_type(self, name, info, json_type):
1072         pass
1073
1074     def visit_enum_type(self, name, info, values, prefix):
1075         pass
1076
1077     def visit_array_type(self, name, info, element_type):
1078         pass
1079
1080     def visit_object_type(self, name, info, base, members, variants):
1081         pass
1082
1083     def visit_object_type_flat(self, name, info, members, variants):
1084         pass
1085
1086     def visit_alternate_type(self, name, info, variants):
1087         pass
1088
1089     def visit_command(self, name, info, arg_type, ret_type,
1090                       gen, success_response, boxed):
1091         pass
1092
1093     def visit_event(self, name, info, arg_type, boxed):
1094         pass
1095
1096
1097 class QAPISchemaType(QAPISchemaEntity):
1098     # Return the C type for common use.
1099     # For the types we commonly box, this is a pointer type.
1100     def c_type(self):
1101         pass
1102
1103     # Return the C type to be used in a parameter list.
1104     def c_param_type(self):
1105         return self.c_type()
1106
1107     # Return the C type to be used where we suppress boxing.
1108     def c_unboxed_type(self):
1109         return self.c_type()
1110
1111     def json_type(self):
1112         pass
1113
1114     def alternate_qtype(self):
1115         json2qtype = {
1116             'string':  'QTYPE_QSTRING',
1117             'number':  'QTYPE_QFLOAT',
1118             'int':     'QTYPE_QINT',
1119             'boolean': 'QTYPE_QBOOL',
1120             'object':  'QTYPE_QDICT'
1121         }
1122         return json2qtype.get(self.json_type())
1123
1124
1125 class QAPISchemaBuiltinType(QAPISchemaType):
1126     def __init__(self, name, json_type, c_type):
1127         QAPISchemaType.__init__(self, name, None)
1128         assert not c_type or isinstance(c_type, str)
1129         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
1130                              'value')
1131         self._json_type_name = json_type
1132         self._c_type_name = c_type
1133
1134     def c_name(self):
1135         return self.name
1136
1137     def c_type(self):
1138         return self._c_type_name
1139
1140     def c_param_type(self):
1141         if self.name == 'str':
1142             return 'const ' + self._c_type_name
1143         return self._c_type_name
1144
1145     def json_type(self):
1146         return self._json_type_name
1147
1148     def visit(self, visitor):
1149         visitor.visit_builtin_type(self.name, self.info, self.json_type())
1150
1151
1152 class QAPISchemaEnumType(QAPISchemaType):
1153     def __init__(self, name, info, values, prefix):
1154         QAPISchemaType.__init__(self, name, info)
1155         for v in values:
1156             assert isinstance(v, QAPISchemaMember)
1157             v.set_owner(name)
1158         assert prefix is None or isinstance(prefix, str)
1159         self.values = values
1160         self.prefix = prefix
1161
1162     def check(self, schema):
1163         seen = {}
1164         for v in self.values:
1165             v.check_clash(self.info, seen)
1166
1167     def is_implicit(self):
1168         # See QAPISchema._make_implicit_enum_type()
1169         return self.name.endswith('Kind')
1170
1171     def c_type(self):
1172         return c_name(self.name)
1173
1174     def member_names(self):
1175         return [v.name for v in self.values]
1176
1177     def json_type(self):
1178         return 'string'
1179
1180     def visit(self, visitor):
1181         visitor.visit_enum_type(self.name, self.info,
1182                                 self.member_names(), self.prefix)
1183
1184
1185 class QAPISchemaArrayType(QAPISchemaType):
1186     def __init__(self, name, info, element_type):
1187         QAPISchemaType.__init__(self, name, info)
1188         assert isinstance(element_type, str)
1189         self._element_type_name = element_type
1190         self.element_type = None
1191
1192     def check(self, schema):
1193         self.element_type = schema.lookup_type(self._element_type_name)
1194         assert self.element_type
1195
1196     def is_implicit(self):
1197         return True
1198
1199     def c_type(self):
1200         return c_name(self.name) + pointer_suffix
1201
1202     def json_type(self):
1203         return 'array'
1204
1205     def visit(self, visitor):
1206         visitor.visit_array_type(self.name, self.info, self.element_type)
1207
1208
1209 class QAPISchemaObjectType(QAPISchemaType):
1210     def __init__(self, name, info, base, local_members, variants):
1211         # struct has local_members, optional base, and no variants
1212         # flat union has base, variants, and no local_members
1213         # simple union has local_members, variants, and no base
1214         QAPISchemaType.__init__(self, name, info)
1215         assert base is None or isinstance(base, str)
1216         for m in local_members:
1217             assert isinstance(m, QAPISchemaObjectTypeMember)
1218             m.set_owner(name)
1219         if variants is not None:
1220             assert isinstance(variants, QAPISchemaObjectTypeVariants)
1221             variants.set_owner(name)
1222         self._base_name = base
1223         self.base = None
1224         self.local_members = local_members
1225         self.variants = variants
1226         self.members = None
1227
1228     def check(self, schema):
1229         if self.members is False:               # check for cycles
1230             raise QAPISemError(self.info,
1231                                "Object %s contains itself" % self.name)
1232         if self.members:
1233             return
1234         self.members = False                    # mark as being checked
1235         seen = OrderedDict()
1236         if self._base_name:
1237             self.base = schema.lookup_type(self._base_name)
1238             assert isinstance(self.base, QAPISchemaObjectType)
1239             self.base.check(schema)
1240             self.base.check_clash(schema, self.info, seen)
1241         for m in self.local_members:
1242             m.check(schema)
1243             m.check_clash(self.info, seen)
1244         self.members = seen.values()
1245         if self.variants:
1246             self.variants.check(schema, seen)
1247             assert self.variants.tag_member in self.members
1248             self.variants.check_clash(schema, self.info, seen)
1249
1250     # Check that the members of this type do not cause duplicate JSON members,
1251     # and update seen to track the members seen so far. Report any errors
1252     # on behalf of info, which is not necessarily self.info
1253     def check_clash(self, schema, info, seen):
1254         assert not self.variants       # not implemented
1255         for m in self.members:
1256             m.check_clash(info, seen)
1257
1258     def is_implicit(self):
1259         # See QAPISchema._make_implicit_object_type(), as well as
1260         # _def_predefineds()
1261         return self.name.startswith('q_')
1262
1263     def is_empty(self):
1264         assert self.members is not None
1265         return not self.members and not self.variants
1266
1267     def c_name(self):
1268         assert self.name != 'q_empty'
1269         return QAPISchemaType.c_name(self)
1270
1271     def c_type(self):
1272         assert not self.is_implicit()
1273         return c_name(self.name) + pointer_suffix
1274
1275     def c_unboxed_type(self):
1276         return c_name(self.name)
1277
1278     def json_type(self):
1279         return 'object'
1280
1281     def visit(self, visitor):
1282         visitor.visit_object_type(self.name, self.info,
1283                                   self.base, self.local_members, self.variants)
1284         visitor.visit_object_type_flat(self.name, self.info,
1285                                        self.members, self.variants)
1286
1287
1288 class QAPISchemaMember(object):
1289     role = 'member'
1290
1291     def __init__(self, name):
1292         assert isinstance(name, str)
1293         self.name = name
1294         self.owner = None
1295
1296     def set_owner(self, name):
1297         assert not self.owner
1298         self.owner = name
1299
1300     def check_clash(self, info, seen):
1301         cname = c_name(self.name)
1302         if cname.lower() != cname and self.owner not in case_whitelist:
1303             raise QAPISemError(info,
1304                                "%s should not use uppercase" % self.describe())
1305         if cname in seen:
1306             raise QAPISemError(info, "%s collides with %s" %
1307                                (self.describe(), seen[cname].describe()))
1308         seen[cname] = self
1309
1310     def _pretty_owner(self):
1311         owner = self.owner
1312         if owner.startswith('q_obj_'):
1313             # See QAPISchema._make_implicit_object_type() - reverse the
1314             # mapping there to create a nice human-readable description
1315             owner = owner[6:]
1316             if owner.endswith('-arg'):
1317                 return '(parameter of %s)' % owner[:-4]
1318             elif owner.endswith('-base'):
1319                 return '(base of %s)' % owner[:-5]
1320             else:
1321                 assert owner.endswith('-wrapper')
1322                 # Unreachable and not implemented
1323                 assert False
1324         if owner.endswith('Kind'):
1325             # See QAPISchema._make_implicit_enum_type()
1326             return '(branch of %s)' % owner[:-4]
1327         return '(%s of %s)' % (self.role, owner)
1328
1329     def describe(self):
1330         return "'%s' %s" % (self.name, self._pretty_owner())
1331
1332
1333 class QAPISchemaObjectTypeMember(QAPISchemaMember):
1334     def __init__(self, name, typ, optional):
1335         QAPISchemaMember.__init__(self, name)
1336         assert isinstance(typ, str)
1337         assert isinstance(optional, bool)
1338         self._type_name = typ
1339         self.type = None
1340         self.optional = optional
1341
1342     def check(self, schema):
1343         assert self.owner
1344         self.type = schema.lookup_type(self._type_name)
1345         assert self.type
1346
1347
1348 class QAPISchemaObjectTypeVariants(object):
1349     def __init__(self, tag_name, tag_member, variants):
1350         # Flat unions pass tag_name but not tag_member.
1351         # Simple unions and alternates pass tag_member but not tag_name.
1352         # After check(), tag_member is always set, and tag_name remains
1353         # a reliable witness of being used by a flat union.
1354         assert bool(tag_member) != bool(tag_name)
1355         assert (isinstance(tag_name, str) or
1356                 isinstance(tag_member, QAPISchemaObjectTypeMember))
1357         assert len(variants) > 0
1358         for v in variants:
1359             assert isinstance(v, QAPISchemaObjectTypeVariant)
1360         self._tag_name = tag_name
1361         self.tag_member = tag_member
1362         self.variants = variants
1363
1364     def set_owner(self, name):
1365         for v in self.variants:
1366             v.set_owner(name)
1367
1368     def check(self, schema, seen):
1369         if not self.tag_member:    # flat union
1370             self.tag_member = seen[c_name(self._tag_name)]
1371             assert self._tag_name == self.tag_member.name
1372         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
1373         for v in self.variants:
1374             v.check(schema)
1375             # Union names must match enum values; alternate names are
1376             # checked separately. Use 'seen' to tell the two apart.
1377             if seen:
1378                 assert v.name in self.tag_member.type.member_names()
1379                 assert isinstance(v.type, QAPISchemaObjectType)
1380                 v.type.check(schema)
1381
1382     def check_clash(self, schema, info, seen):
1383         for v in self.variants:
1384             # Reset seen map for each variant, since qapi names from one
1385             # branch do not affect another branch
1386             assert isinstance(v.type, QAPISchemaObjectType)
1387             v.type.check_clash(schema, info, dict(seen))
1388
1389
1390 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1391     role = 'branch'
1392
1393     def __init__(self, name, typ):
1394         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1395
1396
1397 class QAPISchemaAlternateType(QAPISchemaType):
1398     def __init__(self, name, info, variants):
1399         QAPISchemaType.__init__(self, name, info)
1400         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1401         assert variants.tag_member
1402         variants.set_owner(name)
1403         variants.tag_member.set_owner(self.name)
1404         self.variants = variants
1405
1406     def check(self, schema):
1407         self.variants.tag_member.check(schema)
1408         # Not calling self.variants.check_clash(), because there's nothing
1409         # to clash with
1410         self.variants.check(schema, {})
1411         # Alternate branch names have no relation to the tag enum values;
1412         # so we have to check for potential name collisions ourselves.
1413         seen = {}
1414         for v in self.variants.variants:
1415             v.check_clash(self.info, seen)
1416
1417     def c_type(self):
1418         return c_name(self.name) + pointer_suffix
1419
1420     def json_type(self):
1421         return 'value'
1422
1423     def visit(self, visitor):
1424         visitor.visit_alternate_type(self.name, self.info, self.variants)
1425
1426     def is_empty(self):
1427         return False
1428
1429
1430 class QAPISchemaCommand(QAPISchemaEntity):
1431     def __init__(self, name, info, arg_type, ret_type, gen, success_response,
1432                  boxed):
1433         QAPISchemaEntity.__init__(self, name, info)
1434         assert not arg_type or isinstance(arg_type, str)
1435         assert not ret_type or isinstance(ret_type, str)
1436         self._arg_type_name = arg_type
1437         self.arg_type = None
1438         self._ret_type_name = ret_type
1439         self.ret_type = None
1440         self.gen = gen
1441         self.success_response = success_response
1442         self.boxed = boxed
1443
1444     def check(self, schema):
1445         if self._arg_type_name:
1446             self.arg_type = schema.lookup_type(self._arg_type_name)
1447             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1448                     isinstance(self.arg_type, QAPISchemaAlternateType))
1449             self.arg_type.check(schema)
1450             if self.boxed:
1451                 if self.arg_type.is_empty():
1452                     raise QAPISemError(self.info,
1453                                        "Cannot use 'boxed' with empty type")
1454             else:
1455                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1456                 assert not self.arg_type.variants
1457         elif self.boxed:
1458             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1459         if self._ret_type_name:
1460             self.ret_type = schema.lookup_type(self._ret_type_name)
1461             assert isinstance(self.ret_type, QAPISchemaType)
1462
1463     def visit(self, visitor):
1464         visitor.visit_command(self.name, self.info,
1465                               self.arg_type, self.ret_type,
1466                               self.gen, self.success_response, self.boxed)
1467
1468
1469 class QAPISchemaEvent(QAPISchemaEntity):
1470     def __init__(self, name, info, arg_type, boxed):
1471         QAPISchemaEntity.__init__(self, name, info)
1472         assert not arg_type or isinstance(arg_type, str)
1473         self._arg_type_name = arg_type
1474         self.arg_type = None
1475         self.boxed = boxed
1476
1477     def check(self, schema):
1478         if self._arg_type_name:
1479             self.arg_type = schema.lookup_type(self._arg_type_name)
1480             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1481                     isinstance(self.arg_type, QAPISchemaAlternateType))
1482             self.arg_type.check(schema)
1483             if self.boxed:
1484                 if self.arg_type.is_empty():
1485                     raise QAPISemError(self.info,
1486                                        "Cannot use 'boxed' with empty type")
1487             else:
1488                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1489                 assert not self.arg_type.variants
1490         elif self.boxed:
1491             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1492
1493     def visit(self, visitor):
1494         visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
1495
1496
1497 class QAPISchema(object):
1498     def __init__(self, fname):
1499         try:
1500             parser = QAPISchemaParser(open(fname, "r"))
1501             self.exprs = check_exprs(parser.exprs)
1502             self.docs = check_docs(parser.docs)
1503             self._entity_dict = {}
1504             self._predefining = True
1505             self._def_predefineds()
1506             self._predefining = False
1507             self._def_exprs()
1508             self.check()
1509         except QAPIError as err:
1510             print >>sys.stderr, err
1511             exit(1)
1512
1513     def _def_entity(self, ent):
1514         # Only the predefined types are allowed to not have info
1515         assert ent.info or self._predefining
1516         assert ent.name not in self._entity_dict
1517         self._entity_dict[ent.name] = ent
1518
1519     def lookup_entity(self, name, typ=None):
1520         ent = self._entity_dict.get(name)
1521         if typ and not isinstance(ent, typ):
1522             return None
1523         return ent
1524
1525     def lookup_type(self, name):
1526         return self.lookup_entity(name, QAPISchemaType)
1527
1528     def _def_builtin_type(self, name, json_type, c_type):
1529         self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
1530         # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
1531         # qapi-types.h from a single .c, all arrays of builtins must be
1532         # declared in the first file whether or not they are used.  Nicer
1533         # would be to use lazy instantiation, while figuring out how to
1534         # avoid compilation issues with multiple qapi-types.h.
1535         self._make_array_type(name, None)
1536
1537     def _def_predefineds(self):
1538         for t in [('str',    'string',  'char' + pointer_suffix),
1539                   ('number', 'number',  'double'),
1540                   ('int',    'int',     'int64_t'),
1541                   ('int8',   'int',     'int8_t'),
1542                   ('int16',  'int',     'int16_t'),
1543                   ('int32',  'int',     'int32_t'),
1544                   ('int64',  'int',     'int64_t'),
1545                   ('uint8',  'int',     'uint8_t'),
1546                   ('uint16', 'int',     'uint16_t'),
1547                   ('uint32', 'int',     'uint32_t'),
1548                   ('uint64', 'int',     'uint64_t'),
1549                   ('size',   'int',     'uint64_t'),
1550                   ('bool',   'boolean', 'bool'),
1551                   ('any',    'value',   'QObject' + pointer_suffix)]:
1552             self._def_builtin_type(*t)
1553         self.the_empty_object_type = QAPISchemaObjectType('q_empty', None,
1554                                                           None, [], None)
1555         self._def_entity(self.the_empty_object_type)
1556         qtype_values = self._make_enum_members(['none', 'qnull', 'qint',
1557                                                 'qstring', 'qdict', 'qlist',
1558                                                 'qfloat', 'qbool'])
1559         self._def_entity(QAPISchemaEnumType('QType', None, qtype_values,
1560                                             'QTYPE'))
1561
1562     def _make_enum_members(self, values):
1563         return [QAPISchemaMember(v) for v in values]
1564
1565     def _make_implicit_enum_type(self, name, info, values):
1566         # See also QAPISchemaObjectTypeMember._pretty_owner()
1567         name = name + 'Kind'   # Use namespace reserved by add_name()
1568         self._def_entity(QAPISchemaEnumType(
1569             name, info, self._make_enum_members(values), None))
1570         return name
1571
1572     def _make_array_type(self, element_type, info):
1573         name = element_type + 'List'   # Use namespace reserved by add_name()
1574         if not self.lookup_type(name):
1575             self._def_entity(QAPISchemaArrayType(name, info, element_type))
1576         return name
1577
1578     def _make_implicit_object_type(self, name, info, role, members):
1579         if not members:
1580             return None
1581         # See also QAPISchemaObjectTypeMember._pretty_owner()
1582         name = 'q_obj_%s-%s' % (name, role)
1583         if not self.lookup_entity(name, QAPISchemaObjectType):
1584             self._def_entity(QAPISchemaObjectType(name, info, None,
1585                                                   members, None))
1586         return name
1587
1588     def _def_enum_type(self, expr, info):
1589         name = expr['enum']
1590         data = expr['data']
1591         prefix = expr.get('prefix')
1592         self._def_entity(QAPISchemaEnumType(
1593             name, info, self._make_enum_members(data), prefix))
1594
1595     def _make_member(self, name, typ, info):
1596         optional = False
1597         if name.startswith('*'):
1598             name = name[1:]
1599             optional = True
1600         if isinstance(typ, list):
1601             assert len(typ) == 1
1602             typ = self._make_array_type(typ[0], info)
1603         return QAPISchemaObjectTypeMember(name, typ, optional)
1604
1605     def _make_members(self, data, info):
1606         return [self._make_member(key, value, info)
1607                 for (key, value) in data.iteritems()]
1608
1609     def _def_struct_type(self, expr, info):
1610         name = expr['struct']
1611         base = expr.get('base')
1612         data = expr['data']
1613         self._def_entity(QAPISchemaObjectType(name, info, base,
1614                                               self._make_members(data, info),
1615                                               None))
1616
1617     def _make_variant(self, case, typ):
1618         return QAPISchemaObjectTypeVariant(case, typ)
1619
1620     def _make_simple_variant(self, case, typ, info):
1621         if isinstance(typ, list):
1622             assert len(typ) == 1
1623             typ = self._make_array_type(typ[0], info)
1624         typ = self._make_implicit_object_type(
1625             typ, info, 'wrapper', [self._make_member('data', typ, info)])
1626         return QAPISchemaObjectTypeVariant(case, typ)
1627
1628     def _def_union_type(self, expr, info):
1629         name = expr['union']
1630         data = expr['data']
1631         base = expr.get('base')
1632         tag_name = expr.get('discriminator')
1633         tag_member = None
1634         if isinstance(base, dict):
1635             base = (self._make_implicit_object_type(
1636                     name, info, 'base', self._make_members(base, info)))
1637         if tag_name:
1638             variants = [self._make_variant(key, value)
1639                         for (key, value) in data.iteritems()]
1640             members = []
1641         else:
1642             variants = [self._make_simple_variant(key, value, info)
1643                         for (key, value) in data.iteritems()]
1644             typ = self._make_implicit_enum_type(name, info,
1645                                                 [v.name for v in variants])
1646             tag_member = QAPISchemaObjectTypeMember('type', typ, False)
1647             members = [tag_member]
1648         self._def_entity(
1649             QAPISchemaObjectType(name, info, base, members,
1650                                  QAPISchemaObjectTypeVariants(tag_name,
1651                                                               tag_member,
1652                                                               variants)))
1653
1654     def _def_alternate_type(self, expr, info):
1655         name = expr['alternate']
1656         data = expr['data']
1657         variants = [self._make_variant(key, value)
1658                     for (key, value) in data.iteritems()]
1659         tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
1660         self._def_entity(
1661             QAPISchemaAlternateType(name, info,
1662                                     QAPISchemaObjectTypeVariants(None,
1663                                                                  tag_member,
1664                                                                  variants)))
1665
1666     def _def_command(self, expr, info):
1667         name = expr['command']
1668         data = expr.get('data')
1669         rets = expr.get('returns')
1670         gen = expr.get('gen', True)
1671         success_response = expr.get('success-response', True)
1672         boxed = expr.get('boxed', False)
1673         if isinstance(data, OrderedDict):
1674             data = self._make_implicit_object_type(
1675                 name, info, 'arg', self._make_members(data, info))
1676         if isinstance(rets, list):
1677             assert len(rets) == 1
1678             rets = self._make_array_type(rets[0], info)
1679         self._def_entity(QAPISchemaCommand(name, info, data, rets, gen,
1680                                            success_response, boxed))
1681
1682     def _def_event(self, expr, info):
1683         name = expr['event']
1684         data = expr.get('data')
1685         boxed = expr.get('boxed', False)
1686         if isinstance(data, OrderedDict):
1687             data = self._make_implicit_object_type(
1688                 name, info, 'arg', self._make_members(data, info))
1689         self._def_entity(QAPISchemaEvent(name, info, data, boxed))
1690
1691     def _def_exprs(self):
1692         for expr_elem in self.exprs:
1693             expr = expr_elem['expr']
1694             info = expr_elem['info']
1695             if 'enum' in expr:
1696                 self._def_enum_type(expr, info)
1697             elif 'struct' in expr:
1698                 self._def_struct_type(expr, info)
1699             elif 'union' in expr:
1700                 self._def_union_type(expr, info)
1701             elif 'alternate' in expr:
1702                 self._def_alternate_type(expr, info)
1703             elif 'command' in expr:
1704                 self._def_command(expr, info)
1705             elif 'event' in expr:
1706                 self._def_event(expr, info)
1707             else:
1708                 assert False
1709
1710     def check(self):
1711         for ent in self._entity_dict.values():
1712             ent.check(self)
1713
1714     def visit(self, visitor):
1715         visitor.visit_begin(self)
1716         for (name, entity) in sorted(self._entity_dict.items()):
1717             if visitor.visit_needed(entity):
1718                 entity.visit(visitor)
1719         visitor.visit_end()
1720
1721
1722 #
1723 # Code generation helpers
1724 #
1725
1726 def camel_case(name):
1727     new_name = ''
1728     first = True
1729     for ch in name:
1730         if ch in ['_', '-']:
1731             first = True
1732         elif first:
1733             new_name += ch.upper()
1734             first = False
1735         else:
1736             new_name += ch.lower()
1737     return new_name
1738
1739
1740 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1741 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1742 # ENUM24_Name -> ENUM24_NAME
1743 def camel_to_upper(value):
1744     c_fun_str = c_name(value, False)
1745     if value.isupper():
1746         return c_fun_str
1747
1748     new_name = ''
1749     l = len(c_fun_str)
1750     for i in range(l):
1751         c = c_fun_str[i]
1752         # When c is upper and no "_" appears before, do more checks
1753         if c.isupper() and (i > 0) and c_fun_str[i - 1] != "_":
1754             if i < l - 1 and c_fun_str[i + 1].islower():
1755                 new_name += '_'
1756             elif c_fun_str[i - 1].isdigit():
1757                 new_name += '_'
1758         new_name += c
1759     return new_name.lstrip('_').upper()
1760
1761
1762 def c_enum_const(type_name, const_name, prefix=None):
1763     if prefix is not None:
1764         type_name = prefix
1765     return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
1766
1767 c_name_trans = string.maketrans('.-', '__')
1768
1769
1770 # Map @name to a valid C identifier.
1771 # If @protect, avoid returning certain ticklish identifiers (like
1772 # C keywords) by prepending "q_".
1773 #
1774 # Used for converting 'name' from a 'name':'type' qapi definition
1775 # into a generated struct member, as well as converting type names
1776 # into substrings of a generated C function name.
1777 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1778 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1779 def c_name(name, protect=True):
1780     # ANSI X3J11/88-090, 3.1.1
1781     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1782                      'default', 'do', 'double', 'else', 'enum', 'extern',
1783                      'float', 'for', 'goto', 'if', 'int', 'long', 'register',
1784                      'return', 'short', 'signed', 'sizeof', 'static',
1785                      'struct', 'switch', 'typedef', 'union', 'unsigned',
1786                      'void', 'volatile', 'while'])
1787     # ISO/IEC 9899:1999, 6.4.1
1788     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1789     # ISO/IEC 9899:2011, 6.4.1
1790     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
1791                      '_Noreturn', '_Static_assert', '_Thread_local'])
1792     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1793     # excluding _.*
1794     gcc_words = set(['asm', 'typeof'])
1795     # C++ ISO/IEC 14882:2003 2.11
1796     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1797                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1798                      'namespace', 'new', 'operator', 'private', 'protected',
1799                      'public', 'reinterpret_cast', 'static_cast', 'template',
1800                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1801                      'using', 'virtual', 'wchar_t',
1802                      # alternative representations
1803                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1804                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1805     # namespace pollution:
1806     polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
1807     name = name.translate(c_name_trans)
1808     if protect and (name in c89_words | c99_words | c11_words | gcc_words
1809                     | cpp_words | polluted_words):
1810         return "q_" + name
1811     return name
1812
1813 eatspace = '\033EATSPACE.'
1814 pointer_suffix = ' *' + eatspace
1815
1816
1817 def genindent(count):
1818     ret = ""
1819     for _ in range(count):
1820         ret += " "
1821     return ret
1822
1823 indent_level = 0
1824
1825
1826 def push_indent(indent_amount=4):
1827     global indent_level
1828     indent_level += indent_amount
1829
1830
1831 def pop_indent(indent_amount=4):
1832     global indent_level
1833     indent_level -= indent_amount
1834
1835
1836 # Generate @code with @kwds interpolated.
1837 # Obey indent_level, and strip eatspace.
1838 def cgen(code, **kwds):
1839     raw = code % kwds
1840     if indent_level:
1841         indent = genindent(indent_level)
1842         # re.subn() lacks flags support before Python 2.7, use re.compile()
1843         raw = re.subn(re.compile("^.", re.MULTILINE),
1844                       indent + r'\g<0>', raw)
1845         raw = raw[0]
1846     return re.sub(re.escape(eatspace) + ' *', '', raw)
1847
1848
1849 def mcgen(code, **kwds):
1850     if code[0] == '\n':
1851         code = code[1:]
1852     return cgen(code, **kwds)
1853
1854
1855 def guardname(filename):
1856     return c_name(filename, protect=False).upper()
1857
1858
1859 def guardstart(name):
1860     return mcgen('''
1861
1862 #ifndef %(name)s
1863 #define %(name)s
1864
1865 ''',
1866                  name=guardname(name))
1867
1868
1869 def guardend(name):
1870     return mcgen('''
1871
1872 #endif /* %(name)s */
1873
1874 ''',
1875                  name=guardname(name))
1876
1877
1878 def gen_enum_lookup(name, values, prefix=None):
1879     ret = mcgen('''
1880
1881 const char *const %(c_name)s_lookup[] = {
1882 ''',
1883                 c_name=c_name(name))
1884     for value in values:
1885         index = c_enum_const(name, value, prefix)
1886         ret += mcgen('''
1887     [%(index)s] = "%(value)s",
1888 ''',
1889                      index=index, value=value)
1890
1891     max_index = c_enum_const(name, '_MAX', prefix)
1892     ret += mcgen('''
1893     [%(max_index)s] = NULL,
1894 };
1895 ''',
1896                  max_index=max_index)
1897     return ret
1898
1899
1900 def gen_enum(name, values, prefix=None):
1901     # append automatically generated _MAX value
1902     enum_values = values + ['_MAX']
1903
1904     ret = mcgen('''
1905
1906 typedef enum %(c_name)s {
1907 ''',
1908                 c_name=c_name(name))
1909
1910     i = 0
1911     for value in enum_values:
1912         ret += mcgen('''
1913     %(c_enum)s = %(i)d,
1914 ''',
1915                      c_enum=c_enum_const(name, value, prefix),
1916                      i=i)
1917         i += 1
1918
1919     ret += mcgen('''
1920 } %(c_name)s;
1921 ''',
1922                  c_name=c_name(name))
1923
1924     ret += mcgen('''
1925
1926 extern const char *const %(c_name)s_lookup[];
1927 ''',
1928                  c_name=c_name(name))
1929     return ret
1930
1931
1932 def gen_params(arg_type, boxed, extra):
1933     if not arg_type:
1934         assert not boxed
1935         return extra
1936     ret = ''
1937     sep = ''
1938     if boxed:
1939         ret += '%s arg' % arg_type.c_param_type()
1940         sep = ', '
1941     else:
1942         assert not arg_type.variants
1943         for memb in arg_type.members:
1944             ret += sep
1945             sep = ', '
1946             if memb.optional:
1947                 ret += 'bool has_%s, ' % c_name(memb.name)
1948             ret += '%s %s' % (memb.type.c_param_type(),
1949                               c_name(memb.name))
1950     if extra:
1951         ret += sep + extra
1952     return ret
1953
1954
1955 #
1956 # Common command line parsing
1957 #
1958
1959
1960 def parse_command_line(extra_options="", extra_long_options=[]):
1961
1962     try:
1963         opts, args = getopt.gnu_getopt(sys.argv[1:],
1964                                        "chp:o:" + extra_options,
1965                                        ["source", "header", "prefix=",
1966                                         "output-dir="] + extra_long_options)
1967     except getopt.GetoptError as err:
1968         print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
1969         sys.exit(1)
1970
1971     output_dir = ""
1972     prefix = ""
1973     do_c = False
1974     do_h = False
1975     extra_opts = []
1976
1977     for oa in opts:
1978         o, a = oa
1979         if o in ("-p", "--prefix"):
1980             match = re.match('([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1981             if match.end() != len(a):
1982                 print >>sys.stderr, \
1983                     "%s: 'funny character '%s' in argument of --prefix" \
1984                     % (sys.argv[0], a[match.end()])
1985                 sys.exit(1)
1986             prefix = a
1987         elif o in ("-o", "--output-dir"):
1988             output_dir = a + "/"
1989         elif o in ("-c", "--source"):
1990             do_c = True
1991         elif o in ("-h", "--header"):
1992             do_h = True
1993         else:
1994             extra_opts.append(oa)
1995
1996     if not do_c and not do_h:
1997         do_c = True
1998         do_h = True
1999
2000     if len(args) != 1:
2001         print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
2002         sys.exit(1)
2003     fname = args[0]
2004
2005     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
2006
2007 #
2008 # Generate output files with boilerplate
2009 #
2010
2011
2012 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
2013                 c_comment, h_comment):
2014     guard = guardname(prefix + h_file)
2015     c_file = output_dir + prefix + c_file
2016     h_file = output_dir + prefix + h_file
2017
2018     if output_dir:
2019         try:
2020             os.makedirs(output_dir)
2021         except os.error as e:
2022             if e.errno != errno.EEXIST:
2023                 raise
2024
2025     def maybe_open(really, name, opt):
2026         if really:
2027             return open(name, opt)
2028         else:
2029             import StringIO
2030             return StringIO.StringIO()
2031
2032     fdef = maybe_open(do_c, c_file, 'w')
2033     fdecl = maybe_open(do_h, h_file, 'w')
2034
2035     fdef.write(mcgen('''
2036 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2037 %(comment)s
2038 ''',
2039                      comment=c_comment))
2040
2041     fdecl.write(mcgen('''
2042 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2043 %(comment)s
2044 #ifndef %(guard)s
2045 #define %(guard)s
2046
2047 ''',
2048                       comment=h_comment, guard=guard))
2049
2050     return (fdef, fdecl)
2051
2052
2053 def close_output(fdef, fdecl):
2054     fdecl.write('''
2055 #endif
2056 ''')
2057     fdecl.close()
2058     fdef.close()