scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 import re
  15 from ordereddict import OrderedDict
  16 import errno
  17 import getopt
  18 import os
  19 import sys
  20 import string
  21
  22 builtin_types = {
  23     'str':      'QTYPE_QSTRING',
  24     'int':      'QTYPE_QINT',
  25     'number':   'QTYPE_QFLOAT',
  26     'bool':     'QTYPE_QBOOL',
  27     'int8':     'QTYPE_QINT',
  28     'int16':    'QTYPE_QINT',
  29     'int32':    'QTYPE_QINT',
  30     'int64':    'QTYPE_QINT',
  31     'uint8':    'QTYPE_QINT',
  32     'uint16':   'QTYPE_QINT',
  33     'uint32':   'QTYPE_QINT',
  34     'uint64':   'QTYPE_QINT',
  35     'size':     'QTYPE_QINT',
  36     'any':      None,           # any QType possible, actually
  37     'QType':    'QTYPE_QSTRING',
  38 }
  39
  40 # Whitelist of commands allowed to return a non-dictionary
  41 returns_whitelist = [
  42     # From QMP:
  43     'human-monitor-command',
  44     'qom-get',
  45     'query-migrate-cache-size',
  46     'query-tpm-models',
  47     'query-tpm-types',
  48     'ringbuf-read',
  49
  50     # From QGA:
  51     'guest-file-open',
  52     'guest-fsfreeze-freeze',
  53     'guest-fsfreeze-freeze-list',
  54     'guest-fsfreeze-status',
  55     'guest-fsfreeze-thaw',
  56     'guest-get-time',
  57     'guest-set-vcpus',
  58     'guest-sync',
  59     'guest-sync-delimited',
  60 ]
  61
  62 # Whitelist of entities allowed to violate case conventions
  63 case_whitelist = [
  64     # From QMP:
  65     'ACPISlotType',         # DIMM, visible through query-acpi-ospm-status
  66     'CpuInfoMIPS',          # PC, visible through query-cpu
  67     'CpuInfoTricore',       # PC, visible through query-cpu
  68     'QapiErrorClass',       # all members, visible through errors
  69     'UuidInfo',             # UUID, visible through query-uuid
  70     'X86CPURegister32',     # all members, visible indirectly through qom-get
  71     'q_obj_CpuInfo-base',   # CPU, visible through query-cpu
  72 ]
  73
  74 enum_types = []
  75 struct_types = []
  76 union_types = []
  77 events = []
  78 all_names = {}
  79
  80 #
  81 # Parsing the schema into expressions
  82 #
  83
  84
  85 def error_path(parent):
  86     res = ""
  87     while parent:
  88         res = ("In file included from %s:%d:\n" % (parent['file'],
  89                                                    parent['line'])) + res
  90         parent = parent['parent']
  91     return res
  92
  93
  94 class QAPIError(Exception):
  95     def __init__(self, fname, line, col, incl_info, msg):
  96         Exception.__init__(self)
  97         self.fname = fname
  98         self.line = line
  99         self.col = col
 100         self.info = incl_info
 101         self.msg = msg
 102
 103     def __str__(self):
 104         loc = "%s:%d" % (self.fname, self.line)
 105         if self.col is not None:
 106             loc += ":%s" % self.col
 107         return error_path(self.info) + "%s: %s" % (loc, self.msg)
 108
 109
 110 class QAPIParseError(QAPIError):
 111     def __init__(self, parser, msg):
 112         col = 1
 113         for ch in parser.src[parser.line_pos:parser.pos]:
 114             if ch == '\t':
 115                 col = (col + 7) % 8 + 1
 116             else:
 117                 col += 1
 118         QAPIError.__init__(self, parser.fname, parser.line, col,
 119                            parser.incl_info, msg)
 120
 121
 122 class QAPISemError(QAPIError):
 123     def __init__(self, info, msg):
 124         QAPIError.__init__(self, info['file'], info['line'], None,
 125                            info['parent'], msg)
 126
 127
 128 class QAPIDoc(object):
 129     class Section(object):
 130         def __init__(self, name=None):
 131             # optional section name (argument/member or section name)
 132             self.name = name
 133             # the list of lines for this section
 134             self.content = []
 135
 136         def append(self, line):
 137             self.content.append(line)
 138
 139         def __repr__(self):
 140             return "\n".join(self.content).strip()
 141
 142     class ArgSection(Section):
 143         pass
 144
 145     def __init__(self, parser, info):
 146         # self.parser is used to report errors with QAPIParseError.  The
 147         # resulting error position depends on the state of the parser.
 148         # It happens to be the beginning of the comment.  More or less
 149         # servicable, but action at a distance.
 150         self.parser = parser
 151         self.info = info
 152         self.symbol = None
 153         self.body = QAPIDoc.Section()
 154         # dict mapping parameter name to ArgSection
 155         self.args = OrderedDict()
 156         # a list of Section
 157         self.sections = []
 158         # the current section
 159         self.section = self.body
 160         # associated expression (to be set by expression parser)
 161         self.expr = None
 162
 163     def has_section(self, name):
 164         """Return True if we have a section with this name."""
 165         for i in self.sections:
 166             if i.name == name:
 167                 return True
 168         return False
 169
 170     def append(self, line):
 171         """Parse a comment line and add it to the documentation."""
 172         line = line[1:]
 173         if not line:
 174             self._append_freeform(line)
 175             return
 176
 177         if line[0] != ' ':
 178             raise QAPIParseError(self.parser, "Missing space after #")
 179         line = line[1:]
 180
 181         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 182         # recognized, and get silently treated as ordinary text
 183         if self.symbol:
 184             self._append_symbol_line(line)
 185         elif not self.body.content and line.startswith("@"):
 186             if not line.endswith(":"):
 187                 raise QAPIParseError(self.parser, "Line should end with :")
 188             self.symbol = line[1:-1]
 189             # FIXME invalid names other than the empty string aren't flagged
 190             if not self.symbol:
 191                 raise QAPIParseError(self.parser, "Invalid name")
 192         else:
 193             self._append_freeform(line)
 194
 195     def _append_symbol_line(self, line):
 196         name = line.split(' ', 1)[0]
 197
 198         if name.startswith("@") and name.endswith(":"):
 199             line = line[len(name)+1:]
 200             self._start_args_section(name[1:-1])
 201         elif name in ("Returns:", "Since:",
 202                       # those are often singular or plural
 203                       "Note:", "Notes:",
 204                       "Example:", "Examples:",
 205                       "TODO:"):
 206             line = line[len(name)+1:]
 207             self._start_section(name[:-1])
 208
 209         self._append_freeform(line)
 210
 211     def _start_args_section(self, name):
 212         # FIXME invalid names other than the empty string aren't flagged
 213         if not name:
 214             raise QAPIParseError(self.parser, "Invalid parameter name")
 215         if name in self.args:
 216             raise QAPIParseError(self.parser,
 217                                  "'%s' parameter name duplicated" % name)
 218         if self.sections:
 219             raise QAPIParseError(self.parser,
 220                                  "'@%s:' can't follow '%s' section"
 221                                  % (name, self.sections[0].name))
 222         self.section = QAPIDoc.ArgSection(name)
 223         self.args[name] = self.section
 224
 225     def _start_section(self, name=""):
 226         if name in ("Returns", "Since") and self.has_section(name):
 227             raise QAPIParseError(self.parser,
 228                                  "Duplicated '%s' section" % name)
 229         self.section = QAPIDoc.Section(name)
 230         self.sections.append(self.section)
 231
 232     def _append_freeform(self, line):
 233         in_arg = isinstance(self.section, QAPIDoc.ArgSection)
 234         if (in_arg and self.section.content
 235                 and not self.section.content[-1]
 236                 and line and not line[0].isspace()):
 237             self._start_section()
 238         if (in_arg or not self.section.name
 239                 or not self.section.name.startswith("Example")):
 240             line = line.strip()
 241         self.section.append(line)
 242
 243
 244 class QAPISchemaParser(object):
 245
 246     def __init__(self, fp, previously_included=[], incl_info=None):
 247         abs_fname = os.path.abspath(fp.name)
 248         fname = fp.name
 249         self.fname = fname
 250         previously_included.append(abs_fname)
 251         self.incl_info = incl_info
 252         self.src = fp.read()
 253         if self.src == '' or self.src[-1] != '\n':
 254             self.src += '\n'
 255         self.cursor = 0
 256         self.line = 1
 257         self.line_pos = 0
 258         self.exprs = []
 259         self.docs = []
 260         self.accept()
 261
 262         while self.tok is not None:
 263             info = {'file': fname, 'line': self.line,
 264                     'parent': self.incl_info}
 265             if self.tok == '#':
 266                 doc = self.get_doc(info)
 267                 self.docs.append(doc)
 268                 continue
 269
 270             expr = self.get_expr(False)
 271             if 'include' in expr:
 272                 if len(expr) != 1:
 273                     raise QAPISemError(info, "Invalid 'include' directive")
 274                 include = expr["include"]
 275                 if not isinstance(include, str):
 276                     raise QAPISemError(info,
 277                                        "Value of 'include' must be a string")
 278                 self._include(include, info, os.path.dirname(abs_fname),
 279                               previously_included)
 280             else:
 281                 expr_elem = {'expr': expr,
 282                              'info': info}
 283                 if (self.docs
 284                         and self.docs[-1].info['file'] == fname
 285                         and not self.docs[-1].expr):
 286                     self.docs[-1].expr = expr
 287                     expr_elem['doc'] = self.docs[-1]
 288
 289                 self.exprs.append(expr_elem)
 290
 291     def _include(self, include, info, base_dir, previously_included):
 292         incl_abs_fname = os.path.join(base_dir, include)
 293         # catch inclusion cycle
 294         inf = info
 295         while inf:
 296             if incl_abs_fname == os.path.abspath(inf['file']):
 297                 raise QAPISemError(info, "Inclusion loop for %s" % include)
 298             inf = inf['parent']
 299
 300         # skip multiple include of the same file
 301         if incl_abs_fname in previously_included:
 302             return
 303         try:
 304             fobj = open(incl_abs_fname, 'r')
 305         except IOError as e:
 306             raise QAPISemError(info, '%s: %s' % (e.strerror, include))
 307         exprs_include = QAPISchemaParser(fobj, previously_included, info)
 308         self.exprs.extend(exprs_include.exprs)
 309         self.docs.extend(exprs_include.docs)
 310
 311     def accept(self, skip_comment=True):
 312         while True:
 313             self.tok = self.src[self.cursor]
 314             self.pos = self.cursor
 315             self.cursor += 1
 316             self.val = None
 317
 318             if self.tok == '#':
 319                 if self.src[self.cursor] == '#':
 320                     # Start of doc comment
 321                     skip_comment = False
 322                 self.cursor = self.src.find('\n', self.cursor)
 323                 if not skip_comment:
 324                     self.val = self.src[self.pos:self.cursor]
 325                     return
 326             elif self.tok in "{}:,[]":
 327                 return
 328             elif self.tok == "'":
 329                 string = ''
 330                 esc = False
 331                 while True:
 332                     ch = self.src[self.cursor]
 333                     self.cursor += 1
 334                     if ch == '\n':
 335                         raise QAPIParseError(self, 'Missing terminating "\'"')
 336                     if esc:
 337                         if ch == 'b':
 338                             string += '\b'
 339                         elif ch == 'f':
 340                             string += '\f'
 341                         elif ch == 'n':
 342                             string += '\n'
 343                         elif ch == 'r':
 344                             string += '\r'
 345                         elif ch == 't':
 346                             string += '\t'
 347                         elif ch == 'u':
 348                             value = 0
 349                             for _ in range(0, 4):
 350                                 ch = self.src[self.cursor]
 351                                 self.cursor += 1
 352                                 if ch not in "0123456789abcdefABCDEF":
 353                                     raise QAPIParseError(self,
 354                                                          '\\u escape needs 4 '
 355                                                          'hex digits')
 356                                 value = (value << 4) + int(ch, 16)
 357                             # If Python 2 and 3 didn't disagree so much on
 358                             # how to handle Unicode, then we could allow
 359                             # Unicode string defaults.  But most of QAPI is
 360                             # ASCII-only, so we aren't losing much for now.
 361                             if not value or value > 0x7f:
 362                                 raise QAPIParseError(self,
 363                                                      'For now, \\u escape '
 364                                                      'only supports non-zero '
 365                                                      'values up to \\u007f')
 366                             string += chr(value)
 367                         elif ch in "\\/'\"":
 368                             string += ch
 369                         else:
 370                             raise QAPIParseError(self,
 371                                                  "Unknown escape \\%s" % ch)
 372                         esc = False
 373                     elif ch == "\\":
 374                         esc = True
 375                     elif ch == "'":
 376                         self.val = string
 377                         return
 378                     else:
 379                         string += ch
 380             elif self.src.startswith("true", self.pos):
 381                 self.val = True
 382                 self.cursor += 3
 383                 return
 384             elif self.src.startswith("false", self.pos):
 385                 self.val = False
 386                 self.cursor += 4
 387                 return
 388             elif self.src.startswith("null", self.pos):
 389                 self.val = None
 390                 self.cursor += 3
 391                 return
 392             elif self.tok == '\n':
 393                 if self.cursor == len(self.src):
 394                     self.tok = None
 395                     return
 396                 self.line += 1
 397                 self.line_pos = self.cursor
 398             elif not self.tok.isspace():
 399                 raise QAPIParseError(self, 'Stray "%s"' % self.tok)
 400
 401     def get_members(self):
 402         expr = OrderedDict()
 403         if self.tok == '}':
 404             self.accept()
 405             return expr
 406         if self.tok != "'":
 407             raise QAPIParseError(self, 'Expected string or "}"')
 408         while True:
 409             key = self.val
 410             self.accept()
 411             if self.tok != ':':
 412                 raise QAPIParseError(self, 'Expected ":"')
 413             self.accept()
 414             if key in expr:
 415                 raise QAPIParseError(self, 'Duplicate key "%s"' % key)
 416             expr[key] = self.get_expr(True)
 417             if self.tok == '}':
 418                 self.accept()
 419                 return expr
 420             if self.tok != ',':
 421                 raise QAPIParseError(self, 'Expected "," or "}"')
 422             self.accept()
 423             if self.tok != "'":
 424                 raise QAPIParseError(self, 'Expected string')
 425
 426     def get_values(self):
 427         expr = []
 428         if self.tok == ']':
 429             self.accept()
 430             return expr
 431         if self.tok not in "{['tfn":
 432             raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
 433                                  'boolean or "null"')
 434         while True:
 435             expr.append(self.get_expr(True))
 436             if self.tok == ']':
 437                 self.accept()
 438                 return expr
 439             if self.tok != ',':
 440                 raise QAPIParseError(self, 'Expected "," or "]"')
 441             self.accept()
 442
 443     def get_expr(self, nested):
 444         if self.tok != '{' and not nested:
 445             raise QAPIParseError(self, 'Expected "{"')
 446         if self.tok == '{':
 447             self.accept()
 448             expr = self.get_members()
 449         elif self.tok == '[':
 450             self.accept()
 451             expr = self.get_values()
 452         elif self.tok in "'tfn":
 453             expr = self.val
 454             self.accept()
 455         else:
 456             raise QAPIParseError(self, 'Expected "{", "[" or string')
 457         return expr
 458
 459     def get_doc(self, info):
 460         if self.val != '##':
 461             raise QAPIParseError(self, "Junk after '##' at start of "
 462                                  "documentation comment")
 463
 464         doc = QAPIDoc(self, info)
 465         self.accept(False)
 466         while self.tok == '#':
 467             if self.val.startswith('##'):
 468                 # End of doc comment
 469                 if self.val != '##':
 470                     raise QAPIParseError(self, "Junk after '##' at end of "
 471                                          "documentation comment")
 472                 self.accept()
 473                 return doc
 474             else:
 475                 doc.append(self.val)
 476             self.accept(False)
 477
 478         raise QAPIParseError(self, "Documentation comment must end with '##'")
 479
 480
 481 #
 482 # Semantic analysis of schema expressions
 483 # TODO fold into QAPISchema
 484 # TODO catching name collisions in generated code would be nice
 485 #
 486
 487
 488 def find_base_members(base):
 489     if isinstance(base, dict):
 490         return base
 491     base_struct_define = find_struct(base)
 492     if not base_struct_define:
 493         return None
 494     return base_struct_define['data']
 495
 496
 497 # Return the qtype of an alternate branch, or None on error.
 498 def find_alternate_member_qtype(qapi_type):
 499     if qapi_type in builtin_types:
 500         return builtin_types[qapi_type]
 501     elif find_struct(qapi_type):
 502         return "QTYPE_QDICT"
 503     elif find_enum(qapi_type):
 504         return "QTYPE_QSTRING"
 505     elif find_union(qapi_type):
 506         return "QTYPE_QDICT"
 507     return None
 508
 509
 510 # Return the discriminator enum define if discriminator is specified as an
 511 # enum type, otherwise return None.
 512 def discriminator_find_enum_define(expr):
 513     base = expr.get('base')
 514     discriminator = expr.get('discriminator')
 515
 516     if not (discriminator and base):
 517         return None
 518
 519     base_members = find_base_members(base)
 520     if not base_members:
 521         return None
 522
 523     discriminator_type = base_members.get(discriminator)
 524     if not discriminator_type:
 525         return None
 526
 527     return find_enum(discriminator_type)
 528
 529
 530 # Names must be letters, numbers, -, and _.  They must start with letter,
 531 # except for downstream extensions which must start with __RFQDN_.
 532 # Dots are only valid in the downstream extension prefix.
 533 valid_name = re.compile('^(__[a-zA-Z0-9.-]+_)?'
 534                         '[a-zA-Z][a-zA-Z0-9_-]*$')
 535
 536
 537 def check_name(info, source, name, allow_optional=False,
 538                enum_member=False):
 539     global valid_name
 540     membername = name
 541
 542     if not isinstance(name, str):
 543         raise QAPISemError(info, "%s requires a string name" % source)
 544     if name.startswith('*'):
 545         membername = name[1:]
 546         if not allow_optional:
 547             raise QAPISemError(info, "%s does not allow optional name '%s'"
 548                                % (source, name))
 549     # Enum members can start with a digit, because the generated C
 550     # code always prefixes it with the enum name
 551     if enum_member and membername[0].isdigit():
 552         membername = 'D' + membername
 553     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
 554     # and 'q_obj_*' implicit type names.
 555     if not valid_name.match(membername) or \
 556        c_name(membername, False).startswith('q_'):
 557         raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
 558
 559
 560 def add_name(name, info, meta, implicit=False):
 561     global all_names
 562     check_name(info, "'%s'" % meta, name)
 563     # FIXME should reject names that differ only in '_' vs. '.'
 564     # vs. '-', because they're liable to clash in generated C.
 565     if name in all_names:
 566         raise QAPISemError(info, "%s '%s' is already defined"
 567                            % (all_names[name], name))
 568     if not implicit and (name.endswith('Kind') or name.endswith('List')):
 569         raise QAPISemError(info, "%s '%s' should not end in '%s'"
 570                            % (meta, name, name[-4:]))
 571     all_names[name] = meta
 572
 573
 574 def add_struct(definition, info):
 575     global struct_types
 576     name = definition['struct']
 577     add_name(name, info, 'struct')
 578     struct_types.append(definition)
 579
 580
 581 def find_struct(name):
 582     global struct_types
 583     for struct in struct_types:
 584         if struct['struct'] == name:
 585             return struct
 586     return None
 587
 588
 589 def add_union(definition, info):
 590     global union_types
 591     name = definition['union']
 592     add_name(name, info, 'union')
 593     union_types.append(definition)
 594
 595
 596 def find_union(name):
 597     global union_types
 598     for union in union_types:
 599         if union['union'] == name:
 600             return union
 601     return None
 602
 603
 604 def add_enum(name, info, enum_values=None, implicit=False):
 605     global enum_types
 606     add_name(name, info, 'enum', implicit)
 607     enum_types.append({"enum_name": name, "enum_values": enum_values})
 608
 609
 610 def find_enum(name):
 611     global enum_types
 612     for enum in enum_types:
 613         if enum['enum_name'] == name:
 614             return enum
 615     return None
 616
 617
 618 def is_enum(name):
 619     return find_enum(name) is not None
 620
 621
 622 def check_type(info, source, value, allow_array=False,
 623                allow_dict=False, allow_optional=False,
 624                allow_metas=[]):
 625     global all_names
 626
 627     if value is None:
 628         return
 629
 630     # Check if array type for value is okay
 631     if isinstance(value, list):
 632         if not allow_array:
 633             raise QAPISemError(info, "%s cannot be an array" % source)
 634         if len(value) != 1 or not isinstance(value[0], str):
 635             raise QAPISemError(info,
 636                                "%s: array type must contain single type name" %
 637                                source)
 638         value = value[0]
 639
 640     # Check if type name for value is okay
 641     if isinstance(value, str):
 642         if value not in all_names:
 643             raise QAPISemError(info, "%s uses unknown type '%s'"
 644                                % (source, value))
 645         if not all_names[value] in allow_metas:
 646             raise QAPISemError(info, "%s cannot use %s type '%s'" %
 647                                (source, all_names[value], value))
 648         return
 649
 650     if not allow_dict:
 651         raise QAPISemError(info, "%s should be a type name" % source)
 652
 653     if not isinstance(value, OrderedDict):
 654         raise QAPISemError(info,
 655                            "%s should be a dictionary or type name" % source)
 656
 657     # value is a dictionary, check that each member is okay
 658     for (key, arg) in value.items():
 659         check_name(info, "Member of %s" % source, key,
 660                    allow_optional=allow_optional)
 661         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 662             raise QAPISemError(info, "Member of %s uses reserved name '%s'"
 663                                % (source, key))
 664         # Todo: allow dictionaries to represent default values of
 665         # an optional argument.
 666         check_type(info, "Member '%s' of %s" % (key, source), arg,
 667                    allow_array=True,
 668                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 669                                 'enum'])
 670
 671
 672 def check_command(expr, info):
 673     name = expr['command']
 674     boxed = expr.get('boxed', False)
 675
 676     args_meta = ['struct']
 677     if boxed:
 678         args_meta += ['union', 'alternate']
 679     check_type(info, "'data' for command '%s'" % name,
 680                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 681                allow_metas=args_meta)
 682     returns_meta = ['union', 'struct']
 683     if name in returns_whitelist:
 684         returns_meta += ['built-in', 'alternate', 'enum']
 685     check_type(info, "'returns' for command '%s'" % name,
 686                expr.get('returns'), allow_array=True,
 687                allow_optional=True, allow_metas=returns_meta)
 688
 689
 690 def check_event(expr, info):
 691     global events
 692     name = expr['event']
 693     boxed = expr.get('boxed', False)
 694
 695     meta = ['struct']
 696     if boxed:
 697         meta += ['union', 'alternate']
 698     events.append(name)
 699     check_type(info, "'data' for event '%s'" % name,
 700                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 701                allow_metas=meta)
 702
 703
 704 def check_union(expr, info):
 705     name = expr['union']
 706     base = expr.get('base')
 707     discriminator = expr.get('discriminator')
 708     members = expr['data']
 709
 710     # Two types of unions, determined by discriminator.
 711
 712     # With no discriminator it is a simple union.
 713     if discriminator is None:
 714         enum_define = None
 715         allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
 716         if base is not None:
 717             raise QAPISemError(info, "Simple union '%s' must not have a base" %
 718                                name)
 719
 720     # Else, it's a flat union.
 721     else:
 722         # The object must have a string or dictionary 'base'.
 723         check_type(info, "'base' for union '%s'" % name,
 724                    base, allow_dict=True, allow_optional=True,
 725                    allow_metas=['struct'])
 726         if not base:
 727             raise QAPISemError(info, "Flat union '%s' must have a base"
 728                                % name)
 729         base_members = find_base_members(base)
 730         assert base_members
 731
 732         # The value of member 'discriminator' must name a non-optional
 733         # member of the base struct.
 734         check_name(info, "Discriminator of flat union '%s'" % name,
 735                    discriminator)
 736         discriminator_type = base_members.get(discriminator)
 737         if not discriminator_type:
 738             raise QAPISemError(info,
 739                                "Discriminator '%s' is not a member of base "
 740                                "struct '%s'"
 741                                % (discriminator, base))
 742         enum_define = find_enum(discriminator_type)
 743         allow_metas = ['struct']
 744         # Do not allow string discriminator
 745         if not enum_define:
 746             raise QAPISemError(info,
 747                                "Discriminator '%s' must be of enumeration "
 748                                "type" % discriminator)
 749
 750     # Check every branch; don't allow an empty union
 751     if len(members) == 0:
 752         raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
 753     for (key, value) in members.items():
 754         check_name(info, "Member of union '%s'" % name, key)
 755
 756         # Each value must name a known type
 757         check_type(info, "Member '%s' of union '%s'" % (key, name),
 758                    value, allow_array=not base, allow_metas=allow_metas)
 759
 760         # If the discriminator names an enum type, then all members
 761         # of 'data' must also be members of the enum type.
 762         if enum_define:
 763             if key not in enum_define['enum_values']:
 764                 raise QAPISemError(info,
 765                                    "Discriminator value '%s' is not found in "
 766                                    "enum '%s'"
 767                                    % (key, enum_define["enum_name"]))
 768
 769     # If discriminator is user-defined, ensure all values are covered
 770     if enum_define:
 771         for value in enum_define['enum_values']:
 772             if value not in members.keys():
 773                 raise QAPISemError(info, "Union '%s' data missing '%s' branch"
 774                                    % (name, value))
 775
 776
 777 def check_alternate(expr, info):
 778     name = expr['alternate']
 779     members = expr['data']
 780     types_seen = {}
 781
 782     # Check every branch; require at least two branches
 783     if len(members) < 2:
 784         raise QAPISemError(info,
 785                            "Alternate '%s' should have at least two branches "
 786                            "in 'data'" % name)
 787     for (key, value) in members.items():
 788         check_name(info, "Member of alternate '%s'" % name, key)
 789
 790         # Ensure alternates have no type conflicts.
 791         check_type(info, "Member '%s' of alternate '%s'" % (key, name),
 792                    value,
 793                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 794         qtype = find_alternate_member_qtype(value)
 795         if not qtype:
 796             raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
 797                                "type '%s'" % (name, key, value))
 798         if qtype in types_seen:
 799             raise QAPISemError(info, "Alternate '%s' member '%s' can't "
 800                                "be distinguished from member '%s'"
 801                                % (name, key, types_seen[qtype]))
 802         types_seen[qtype] = key
 803
 804
 805 def check_enum(expr, info):
 806     name = expr['enum']
 807     members = expr.get('data')
 808     prefix = expr.get('prefix')
 809
 810     if not isinstance(members, list):
 811         raise QAPISemError(info,
 812                            "Enum '%s' requires an array for 'data'" % name)
 813     if prefix is not None and not isinstance(prefix, str):
 814         raise QAPISemError(info,
 815                            "Enum '%s' requires a string for 'prefix'" % name)
 816     for member in members:
 817         check_name(info, "Member of enum '%s'" % name, member,
 818                    enum_member=True)
 819
 820
 821 def check_struct(expr, info):
 822     name = expr['struct']
 823     members = expr['data']
 824
 825     check_type(info, "'data' for struct '%s'" % name, members,
 826                allow_dict=True, allow_optional=True)
 827     check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
 828                allow_metas=['struct'])
 829
 830
 831 def check_keys(expr_elem, meta, required, optional=[]):
 832     expr = expr_elem['expr']
 833     info = expr_elem['info']
 834     name = expr[meta]
 835     if not isinstance(name, str):
 836         raise QAPISemError(info, "'%s' key must have a string value" % meta)
 837     required = required + [meta]
 838     for (key, value) in expr.items():
 839         if key not in required and key not in optional:
 840             raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
 841                                % (key, meta, name))
 842         if (key == 'gen' or key == 'success-response') and value is not False:
 843             raise QAPISemError(info,
 844                                "'%s' of %s '%s' should only use false value"
 845                                % (key, meta, name))
 846         if key == 'boxed' and value is not True:
 847             raise QAPISemError(info,
 848                                "'%s' of %s '%s' should only use true value"
 849                                % (key, meta, name))
 850     for key in required:
 851         if key not in expr:
 852             raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
 853                                % (key, meta, name))
 854
 855
 856 def check_exprs(exprs):
 857     global all_names
 858
 859     # Learn the types and check for valid expression keys
 860     for builtin in builtin_types.keys():
 861         all_names[builtin] = 'built-in'
 862     for expr_elem in exprs:
 863         expr = expr_elem['expr']
 864         info = expr_elem['info']
 865
 866         if 'doc' not in expr_elem:
 867             raise QAPISemError(info,
 868                                "Expression missing documentation comment")
 869
 870         if 'enum' in expr:
 871             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 872             add_enum(expr['enum'], info, expr['data'])
 873         elif 'union' in expr:
 874             check_keys(expr_elem, 'union', ['data'],
 875                        ['base', 'discriminator'])
 876             add_union(expr, info)
 877         elif 'alternate' in expr:
 878             check_keys(expr_elem, 'alternate', ['data'])
 879             add_name(expr['alternate'], info, 'alternate')
 880         elif 'struct' in expr:
 881             check_keys(expr_elem, 'struct', ['data'], ['base'])
 882             add_struct(expr, info)
 883         elif 'command' in expr:
 884             check_keys(expr_elem, 'command', [],
 885                        ['data', 'returns', 'gen', 'success-response', 'boxed'])
 886             add_name(expr['command'], info, 'command')
 887         elif 'event' in expr:
 888             check_keys(expr_elem, 'event', [], ['data', 'boxed'])
 889             add_name(expr['event'], info, 'event')
 890         else:
 891             raise QAPISemError(expr_elem['info'],
 892                                "Expression is missing metatype")
 893
 894     # Try again for hidden UnionKind enum
 895     for expr_elem in exprs:
 896         expr = expr_elem['expr']
 897         if 'union' in expr:
 898             if not discriminator_find_enum_define(expr):
 899                 add_enum('%sKind' % expr['union'], expr_elem['info'],
 900                          implicit=True)
 901         elif 'alternate' in expr:
 902             add_enum('%sKind' % expr['alternate'], expr_elem['info'],
 903                      implicit=True)
 904
 905     # Validate that exprs make sense
 906     for expr_elem in exprs:
 907         expr = expr_elem['expr']
 908         info = expr_elem['info']
 909
 910         if 'enum' in expr:
 911             check_enum(expr, info)
 912         elif 'union' in expr:
 913             check_union(expr, info)
 914         elif 'alternate' in expr:
 915             check_alternate(expr, info)
 916         elif 'struct' in expr:
 917             check_struct(expr, info)
 918         elif 'command' in expr:
 919             check_command(expr, info)
 920         elif 'event' in expr:
 921             check_event(expr, info)
 922         else:
 923             assert False, 'unexpected meta type'
 924
 925     return exprs
 926
 927
 928 def check_freeform_doc(doc):
 929     if doc.symbol:
 930         raise QAPISemError(doc.info,
 931                            "Documention for '%s' is not followed"
 932                            " by the definition" % doc.symbol)
 933
 934     body = str(doc.body)
 935     if re.search(r'@\S+:', body, re.MULTILINE):
 936         raise QAPISemError(doc.info,
 937                            "Free-form documentation block must not contain"
 938                            " @NAME: sections")
 939
 940
 941 def check_definition_doc(doc, expr, info):
 942     for i in ('enum', 'union', 'alternate', 'struct', 'command', 'event'):
 943         if i in expr:
 944             meta = i
 945             break
 946
 947     name = expr[meta]
 948     if doc.symbol != name:
 949         raise QAPISemError(info, "Definition of '%s' follows documentation"
 950                            " for '%s'" % (name, doc.symbol))
 951     if doc.has_section('Returns') and 'command' not in expr:
 952         raise QAPISemError(info, "'Returns:' is only valid for commands")
 953
 954     if meta == 'union':
 955         args = expr.get('base', [])
 956     else:
 957         args = expr.get('data', [])
 958     if isinstance(args, str):
 959         return
 960     if isinstance(args, dict):
 961         args = args.keys()
 962     assert isinstance(args, list)
 963
 964     if (meta == 'alternate'
 965             or (meta == 'union' and not expr.get('discriminator'))):
 966         args.append('type')
 967
 968     for arg in args:
 969         if arg[0] == '*':
 970             opt = True
 971             desc = doc.args.get(arg[1:])
 972         else:
 973             opt = False
 974             desc = doc.args.get(arg)
 975         if not desc:
 976             continue
 977         desc_opt = "#optional" in str(desc)
 978         if desc_opt and not opt:
 979             raise QAPISemError(info, "Description has #optional, "
 980                                "but the declaration doesn't")
 981         if not desc_opt and opt:
 982             # silently fix the doc
 983             # TODO either fix the schema and make this an error,
 984             # or drop #optional entirely
 985             desc.append("#optional")
 986
 987     doc_args = set(doc.args.keys())
 988     args = set([name.strip('*') for name in args])
 989     if not doc_args.issubset(args):
 990         raise QAPISemError(info, "The following documented members are not in "
 991                            "the declaration: %s" % ", ".join(doc_args - args))
 992
 993
 994 def check_docs(docs):
 995     for doc in docs:
 996         for section in doc.args.values() + doc.sections:
 997             content = str(section)
 998             if not content or content.isspace():
 999                 raise QAPISemError(doc.info,
1000                                    "Empty doc section '%s'" % section.name)
1001
1002         if not doc.expr:
1003             check_freeform_doc(doc)
1004         else:
1005             check_definition_doc(doc, doc.expr, doc.info)
1006
1007     return docs
1008
1009
1010 #
1011 # Schema compiler frontend
1012 #
1013
1014 class QAPISchemaEntity(object):
1015     def __init__(self, name, info):
1016         assert isinstance(name, str)
1017         self.name = name
1018         # For explicitly defined entities, info points to the (explicit)
1019         # definition.  For builtins (and their arrays), info is None.
1020         # For implicitly defined entities, info points to a place that
1021         # triggered the implicit definition (there may be more than one
1022         # such place).
1023         self.info = info
1024
1025     def c_name(self):
1026         return c_name(self.name)
1027
1028     def check(self, schema):
1029         pass
1030
1031     def is_implicit(self):
1032         return not self.info
1033
1034     def visit(self, visitor):
1035         pass
1036
1037
1038 class QAPISchemaVisitor(object):
1039     def visit_begin(self, schema):
1040         pass
1041
1042     def visit_end(self):
1043         pass
1044
1045     def visit_needed(self, entity):
1046         # Default to visiting everything
1047         return True
1048
1049     def visit_builtin_type(self, name, info, json_type):
1050         pass
1051
1052     def visit_enum_type(self, name, info, values, prefix):
1053         pass
1054
1055     def visit_array_type(self, name, info, element_type):
1056         pass
1057
1058     def visit_object_type(self, name, info, base, members, variants):
1059         pass
1060
1061     def visit_object_type_flat(self, name, info, members, variants):
1062         pass
1063
1064     def visit_alternate_type(self, name, info, variants):
1065         pass
1066
1067     def visit_command(self, name, info, arg_type, ret_type,
1068                       gen, success_response, boxed):
1069         pass
1070
1071     def visit_event(self, name, info, arg_type, boxed):
1072         pass
1073
1074
1075 class QAPISchemaType(QAPISchemaEntity):
1076     # Return the C type for common use.
1077     # For the types we commonly box, this is a pointer type.
1078     def c_type(self):
1079         pass
1080
1081     # Return the C type to be used in a parameter list.
1082     def c_param_type(self):
1083         return self.c_type()
1084
1085     # Return the C type to be used where we suppress boxing.
1086     def c_unboxed_type(self):
1087         return self.c_type()
1088
1089     def json_type(self):
1090         pass
1091
1092     def alternate_qtype(self):
1093         json2qtype = {
1094             'string':  'QTYPE_QSTRING',
1095             'number':  'QTYPE_QFLOAT',
1096             'int':     'QTYPE_QINT',
1097             'boolean': 'QTYPE_QBOOL',
1098             'object':  'QTYPE_QDICT'
1099         }
1100         return json2qtype.get(self.json_type())
1101
1102
1103 class QAPISchemaBuiltinType(QAPISchemaType):
1104     def __init__(self, name, json_type, c_type):
1105         QAPISchemaType.__init__(self, name, None)
1106         assert not c_type or isinstance(c_type, str)
1107         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
1108                              'value')
1109         self._json_type_name = json_type
1110         self._c_type_name = c_type
1111
1112     def c_name(self):
1113         return self.name
1114
1115     def c_type(self):
1116         return self._c_type_name
1117
1118     def c_param_type(self):
1119         if self.name == 'str':
1120             return 'const ' + self._c_type_name
1121         return self._c_type_name
1122
1123     def json_type(self):
1124         return self._json_type_name
1125
1126     def visit(self, visitor):
1127         visitor.visit_builtin_type(self.name, self.info, self.json_type())
1128
1129
1130 class QAPISchemaEnumType(QAPISchemaType):
1131     def __init__(self, name, info, values, prefix):
1132         QAPISchemaType.__init__(self, name, info)
1133         for v in values:
1134             assert isinstance(v, QAPISchemaMember)
1135             v.set_owner(name)
1136         assert prefix is None or isinstance(prefix, str)
1137         self.values = values
1138         self.prefix = prefix
1139
1140     def check(self, schema):
1141         seen = {}
1142         for v in self.values:
1143             v.check_clash(self.info, seen)
1144
1145     def is_implicit(self):
1146         # See QAPISchema._make_implicit_enum_type()
1147         return self.name.endswith('Kind')
1148
1149     def c_type(self):
1150         return c_name(self.name)
1151
1152     def member_names(self):
1153         return [v.name for v in self.values]
1154
1155     def json_type(self):
1156         return 'string'
1157
1158     def visit(self, visitor):
1159         visitor.visit_enum_type(self.name, self.info,
1160                                 self.member_names(), self.prefix)
1161
1162
1163 class QAPISchemaArrayType(QAPISchemaType):
1164     def __init__(self, name, info, element_type):
1165         QAPISchemaType.__init__(self, name, info)
1166         assert isinstance(element_type, str)
1167         self._element_type_name = element_type
1168         self.element_type = None
1169
1170     def check(self, schema):
1171         self.element_type = schema.lookup_type(self._element_type_name)
1172         assert self.element_type
1173
1174     def is_implicit(self):
1175         return True
1176
1177     def c_type(self):
1178         return c_name(self.name) + pointer_suffix
1179
1180     def json_type(self):
1181         return 'array'
1182
1183     def visit(self, visitor):
1184         visitor.visit_array_type(self.name, self.info, self.element_type)
1185
1186
1187 class QAPISchemaObjectType(QAPISchemaType):
1188     def __init__(self, name, info, base, local_members, variants):
1189         # struct has local_members, optional base, and no variants
1190         # flat union has base, variants, and no local_members
1191         # simple union has local_members, variants, and no base
1192         QAPISchemaType.__init__(self, name, info)
1193         assert base is None or isinstance(base, str)
1194         for m in local_members:
1195             assert isinstance(m, QAPISchemaObjectTypeMember)
1196             m.set_owner(name)
1197         if variants is not None:
1198             assert isinstance(variants, QAPISchemaObjectTypeVariants)
1199             variants.set_owner(name)
1200         self._base_name = base
1201         self.base = None
1202         self.local_members = local_members
1203         self.variants = variants
1204         self.members = None
1205
1206     def check(self, schema):
1207         if self.members is False:               # check for cycles
1208             raise QAPISemError(self.info,
1209                                "Object %s contains itself" % self.name)
1210         if self.members:
1211             return
1212         self.members = False                    # mark as being checked
1213         seen = OrderedDict()
1214         if self._base_name:
1215             self.base = schema.lookup_type(self._base_name)
1216             assert isinstance(self.base, QAPISchemaObjectType)
1217             self.base.check(schema)
1218             self.base.check_clash(schema, self.info, seen)
1219         for m in self.local_members:
1220             m.check(schema)
1221             m.check_clash(self.info, seen)
1222         self.members = seen.values()
1223         if self.variants:
1224             self.variants.check(schema, seen)
1225             assert self.variants.tag_member in self.members
1226             self.variants.check_clash(schema, self.info, seen)
1227
1228     # Check that the members of this type do not cause duplicate JSON members,
1229     # and update seen to track the members seen so far. Report any errors
1230     # on behalf of info, which is not necessarily self.info
1231     def check_clash(self, schema, info, seen):
1232         assert not self.variants       # not implemented
1233         for m in self.members:
1234             m.check_clash(info, seen)
1235
1236     def is_implicit(self):
1237         # See QAPISchema._make_implicit_object_type(), as well as
1238         # _def_predefineds()
1239         return self.name.startswith('q_')
1240
1241     def is_empty(self):
1242         assert self.members is not None
1243         return not self.members and not self.variants
1244
1245     def c_name(self):
1246         assert self.name != 'q_empty'
1247         return QAPISchemaType.c_name(self)
1248
1249     def c_type(self):
1250         assert not self.is_implicit()
1251         return c_name(self.name) + pointer_suffix
1252
1253     def c_unboxed_type(self):
1254         return c_name(self.name)
1255
1256     def json_type(self):
1257         return 'object'
1258
1259     def visit(self, visitor):
1260         visitor.visit_object_type(self.name, self.info,
1261                                   self.base, self.local_members, self.variants)
1262         visitor.visit_object_type_flat(self.name, self.info,
1263                                        self.members, self.variants)
1264
1265
1266 class QAPISchemaMember(object):
1267     role = 'member'
1268
1269     def __init__(self, name):
1270         assert isinstance(name, str)
1271         self.name = name
1272         self.owner = None
1273
1274     def set_owner(self, name):
1275         assert not self.owner
1276         self.owner = name
1277
1278     def check_clash(self, info, seen):
1279         cname = c_name(self.name)
1280         if cname.lower() != cname and self.owner not in case_whitelist:
1281             raise QAPISemError(info,
1282                                "%s should not use uppercase" % self.describe())
1283         if cname in seen:
1284             raise QAPISemError(info, "%s collides with %s" %
1285                                (self.describe(), seen[cname].describe()))
1286         seen[cname] = self
1287
1288     def _pretty_owner(self):
1289         owner = self.owner
1290         if owner.startswith('q_obj_'):
1291             # See QAPISchema._make_implicit_object_type() - reverse the
1292             # mapping there to create a nice human-readable description
1293             owner = owner[6:]
1294             if owner.endswith('-arg'):
1295                 return '(parameter of %s)' % owner[:-4]
1296             elif owner.endswith('-base'):
1297                 return '(base of %s)' % owner[:-5]
1298             else:
1299                 assert owner.endswith('-wrapper')
1300                 # Unreachable and not implemented
1301                 assert False
1302         if owner.endswith('Kind'):
1303             # See QAPISchema._make_implicit_enum_type()
1304             return '(branch of %s)' % owner[:-4]
1305         return '(%s of %s)' % (self.role, owner)
1306
1307     def describe(self):
1308         return "'%s' %s" % (self.name, self._pretty_owner())
1309
1310
1311 class QAPISchemaObjectTypeMember(QAPISchemaMember):
1312     def __init__(self, name, typ, optional):
1313         QAPISchemaMember.__init__(self, name)
1314         assert isinstance(typ, str)
1315         assert isinstance(optional, bool)
1316         self._type_name = typ
1317         self.type = None
1318         self.optional = optional
1319
1320     def check(self, schema):
1321         assert self.owner
1322         self.type = schema.lookup_type(self._type_name)
1323         assert self.type
1324
1325
1326 class QAPISchemaObjectTypeVariants(object):
1327     def __init__(self, tag_name, tag_member, variants):
1328         # Flat unions pass tag_name but not tag_member.
1329         # Simple unions and alternates pass tag_member but not tag_name.
1330         # After check(), tag_member is always set, and tag_name remains
1331         # a reliable witness of being used by a flat union.
1332         assert bool(tag_member) != bool(tag_name)
1333         assert (isinstance(tag_name, str) or
1334                 isinstance(tag_member, QAPISchemaObjectTypeMember))
1335         assert len(variants) > 0
1336         for v in variants:
1337             assert isinstance(v, QAPISchemaObjectTypeVariant)
1338         self._tag_name = tag_name
1339         self.tag_member = tag_member
1340         self.variants = variants
1341
1342     def set_owner(self, name):
1343         for v in self.variants:
1344             v.set_owner(name)
1345
1346     def check(self, schema, seen):
1347         if not self.tag_member:    # flat union
1348             self.tag_member = seen[c_name(self._tag_name)]
1349             assert self._tag_name == self.tag_member.name
1350         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
1351         for v in self.variants:
1352             v.check(schema)
1353             # Union names must match enum values; alternate names are
1354             # checked separately. Use 'seen' to tell the two apart.
1355             if seen:
1356                 assert v.name in self.tag_member.type.member_names()
1357                 assert isinstance(v.type, QAPISchemaObjectType)
1358                 v.type.check(schema)
1359
1360     def check_clash(self, schema, info, seen):
1361         for v in self.variants:
1362             # Reset seen map for each variant, since qapi names from one
1363             # branch do not affect another branch
1364             assert isinstance(v.type, QAPISchemaObjectType)
1365             v.type.check_clash(schema, info, dict(seen))
1366
1367
1368 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1369     role = 'branch'
1370
1371     def __init__(self, name, typ):
1372         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1373
1374
1375 class QAPISchemaAlternateType(QAPISchemaType):
1376     def __init__(self, name, info, variants):
1377         QAPISchemaType.__init__(self, name, info)
1378         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1379         assert variants.tag_member
1380         variants.set_owner(name)
1381         variants.tag_member.set_owner(self.name)
1382         self.variants = variants
1383
1384     def check(self, schema):
1385         self.variants.tag_member.check(schema)
1386         # Not calling self.variants.check_clash(), because there's nothing
1387         # to clash with
1388         self.variants.check(schema, {})
1389         # Alternate branch names have no relation to the tag enum values;
1390         # so we have to check for potential name collisions ourselves.
1391         seen = {}
1392         for v in self.variants.variants:
1393             v.check_clash(self.info, seen)
1394
1395     def c_type(self):
1396         return c_name(self.name) + pointer_suffix
1397
1398     def json_type(self):
1399         return 'value'
1400
1401     def visit(self, visitor):
1402         visitor.visit_alternate_type(self.name, self.info, self.variants)
1403
1404     def is_empty(self):
1405         return False
1406
1407
1408 class QAPISchemaCommand(QAPISchemaEntity):
1409     def __init__(self, name, info, arg_type, ret_type, gen, success_response,
1410                  boxed):
1411         QAPISchemaEntity.__init__(self, name, info)
1412         assert not arg_type or isinstance(arg_type, str)
1413         assert not ret_type or isinstance(ret_type, str)
1414         self._arg_type_name = arg_type
1415         self.arg_type = None
1416         self._ret_type_name = ret_type
1417         self.ret_type = None
1418         self.gen = gen
1419         self.success_response = success_response
1420         self.boxed = boxed
1421
1422     def check(self, schema):
1423         if self._arg_type_name:
1424             self.arg_type = schema.lookup_type(self._arg_type_name)
1425             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1426                     isinstance(self.arg_type, QAPISchemaAlternateType))
1427             self.arg_type.check(schema)
1428             if self.boxed:
1429                 if self.arg_type.is_empty():
1430                     raise QAPISemError(self.info,
1431                                        "Cannot use 'boxed' with empty type")
1432             else:
1433                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1434                 assert not self.arg_type.variants
1435         elif self.boxed:
1436             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1437         if self._ret_type_name:
1438             self.ret_type = schema.lookup_type(self._ret_type_name)
1439             assert isinstance(self.ret_type, QAPISchemaType)
1440
1441     def visit(self, visitor):
1442         visitor.visit_command(self.name, self.info,
1443                               self.arg_type, self.ret_type,
1444                               self.gen, self.success_response, self.boxed)
1445
1446
1447 class QAPISchemaEvent(QAPISchemaEntity):
1448     def __init__(self, name, info, arg_type, boxed):
1449         QAPISchemaEntity.__init__(self, name, info)
1450         assert not arg_type or isinstance(arg_type, str)
1451         self._arg_type_name = arg_type
1452         self.arg_type = None
1453         self.boxed = boxed
1454
1455     def check(self, schema):
1456         if self._arg_type_name:
1457             self.arg_type = schema.lookup_type(self._arg_type_name)
1458             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1459                     isinstance(self.arg_type, QAPISchemaAlternateType))
1460             self.arg_type.check(schema)
1461             if self.boxed:
1462                 if self.arg_type.is_empty():
1463                     raise QAPISemError(self.info,
1464                                        "Cannot use 'boxed' with empty type")
1465             else:
1466                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1467                 assert not self.arg_type.variants
1468         elif self.boxed:
1469             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1470
1471     def visit(self, visitor):
1472         visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
1473
1474
1475 class QAPISchema(object):
1476     def __init__(self, fname):
1477         try:
1478             parser = QAPISchemaParser(open(fname, "r"))
1479             self.exprs = check_exprs(parser.exprs)
1480             self.docs = check_docs(parser.docs)
1481             self._entity_dict = {}
1482             self._predefining = True
1483             self._def_predefineds()
1484             self._predefining = False
1485             self._def_exprs()
1486             self.check()
1487         except QAPIError as err:
1488             print >>sys.stderr, err
1489             exit(1)
1490
1491     def _def_entity(self, ent):
1492         # Only the predefined types are allowed to not have info
1493         assert ent.info or self._predefining
1494         assert ent.name not in self._entity_dict
1495         self._entity_dict[ent.name] = ent
1496
1497     def lookup_entity(self, name, typ=None):
1498         ent = self._entity_dict.get(name)
1499         if typ and not isinstance(ent, typ):
1500             return None
1501         return ent
1502
1503     def lookup_type(self, name):
1504         return self.lookup_entity(name, QAPISchemaType)
1505
1506     def _def_builtin_type(self, name, json_type, c_type):
1507         self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
1508         # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
1509         # qapi-types.h from a single .c, all arrays of builtins must be
1510         # declared in the first file whether or not they are used.  Nicer
1511         # would be to use lazy instantiation, while figuring out how to
1512         # avoid compilation issues with multiple qapi-types.h.
1513         self._make_array_type(name, None)
1514
1515     def _def_predefineds(self):
1516         for t in [('str',    'string',  'char' + pointer_suffix),
1517                   ('number', 'number',  'double'),
1518                   ('int',    'int',     'int64_t'),
1519                   ('int8',   'int',     'int8_t'),
1520                   ('int16',  'int',     'int16_t'),
1521                   ('int32',  'int',     'int32_t'),
1522                   ('int64',  'int',     'int64_t'),
1523                   ('uint8',  'int',     'uint8_t'),
1524                   ('uint16', 'int',     'uint16_t'),
1525                   ('uint32', 'int',     'uint32_t'),
1526                   ('uint64', 'int',     'uint64_t'),
1527                   ('size',   'int',     'uint64_t'),
1528                   ('bool',   'boolean', 'bool'),
1529                   ('any',    'value',   'QObject' + pointer_suffix)]:
1530             self._def_builtin_type(*t)
1531         self.the_empty_object_type = QAPISchemaObjectType('q_empty', None,
1532                                                           None, [], None)
1533         self._def_entity(self.the_empty_object_type)
1534         qtype_values = self._make_enum_members(['none', 'qnull', 'qint',
1535                                                 'qstring', 'qdict', 'qlist',
1536                                                 'qfloat', 'qbool'])
1537         self._def_entity(QAPISchemaEnumType('QType', None, qtype_values,
1538                                             'QTYPE'))
1539
1540     def _make_enum_members(self, values):
1541         return [QAPISchemaMember(v) for v in values]
1542
1543     def _make_implicit_enum_type(self, name, info, values):
1544         # See also QAPISchemaObjectTypeMember._pretty_owner()
1545         name = name + 'Kind'   # Use namespace reserved by add_name()
1546         self._def_entity(QAPISchemaEnumType(
1547             name, info, self._make_enum_members(values), None))
1548         return name
1549
1550     def _make_array_type(self, element_type, info):
1551         name = element_type + 'List'   # Use namespace reserved by add_name()
1552         if not self.lookup_type(name):
1553             self._def_entity(QAPISchemaArrayType(name, info, element_type))
1554         return name
1555
1556     def _make_implicit_object_type(self, name, info, role, members):
1557         if not members:
1558             return None
1559         # See also QAPISchemaObjectTypeMember._pretty_owner()
1560         name = 'q_obj_%s-%s' % (name, role)
1561         if not self.lookup_entity(name, QAPISchemaObjectType):
1562             self._def_entity(QAPISchemaObjectType(name, info, None,
1563                                                   members, None))
1564         return name
1565
1566     def _def_enum_type(self, expr, info):
1567         name = expr['enum']
1568         data = expr['data']
1569         prefix = expr.get('prefix')
1570         self._def_entity(QAPISchemaEnumType(
1571             name, info, self._make_enum_members(data), prefix))
1572
1573     def _make_member(self, name, typ, info):
1574         optional = False
1575         if name.startswith('*'):
1576             name = name[1:]
1577             optional = True
1578         if isinstance(typ, list):
1579             assert len(typ) == 1
1580             typ = self._make_array_type(typ[0], info)
1581         return QAPISchemaObjectTypeMember(name, typ, optional)
1582
1583     def _make_members(self, data, info):
1584         return [self._make_member(key, value, info)
1585                 for (key, value) in data.iteritems()]
1586
1587     def _def_struct_type(self, expr, info):
1588         name = expr['struct']
1589         base = expr.get('base')
1590         data = expr['data']
1591         self._def_entity(QAPISchemaObjectType(name, info, base,
1592                                               self._make_members(data, info),
1593                                               None))
1594
1595     def _make_variant(self, case, typ):
1596         return QAPISchemaObjectTypeVariant(case, typ)
1597
1598     def _make_simple_variant(self, case, typ, info):
1599         if isinstance(typ, list):
1600             assert len(typ) == 1
1601             typ = self._make_array_type(typ[0], info)
1602         typ = self._make_implicit_object_type(
1603             typ, info, 'wrapper', [self._make_member('data', typ, info)])
1604         return QAPISchemaObjectTypeVariant(case, typ)
1605
1606     def _def_union_type(self, expr, info):
1607         name = expr['union']
1608         data = expr['data']
1609         base = expr.get('base')
1610         tag_name = expr.get('discriminator')
1611         tag_member = None
1612         if isinstance(base, dict):
1613             base = (self._make_implicit_object_type(
1614                     name, info, 'base', self._make_members(base, info)))
1615         if tag_name:
1616             variants = [self._make_variant(key, value)
1617                         for (key, value) in data.iteritems()]
1618             members = []
1619         else:
1620             variants = [self._make_simple_variant(key, value, info)
1621                         for (key, value) in data.iteritems()]
1622             typ = self._make_implicit_enum_type(name, info,
1623                                                 [v.name for v in variants])
1624             tag_member = QAPISchemaObjectTypeMember('type', typ, False)
1625             members = [tag_member]
1626         self._def_entity(
1627             QAPISchemaObjectType(name, info, base, members,
1628                                  QAPISchemaObjectTypeVariants(tag_name,
1629                                                               tag_member,
1630                                                               variants)))
1631
1632     def _def_alternate_type(self, expr, info):
1633         name = expr['alternate']
1634         data = expr['data']
1635         variants = [self._make_variant(key, value)
1636                     for (key, value) in data.iteritems()]
1637         tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
1638         self._def_entity(
1639             QAPISchemaAlternateType(name, info,
1640                                     QAPISchemaObjectTypeVariants(None,
1641                                                                  tag_member,
1642                                                                  variants)))
1643
1644     def _def_command(self, expr, info):
1645         name = expr['command']
1646         data = expr.get('data')
1647         rets = expr.get('returns')
1648         gen = expr.get('gen', True)
1649         success_response = expr.get('success-response', True)
1650         boxed = expr.get('boxed', False)
1651         if isinstance(data, OrderedDict):
1652             data = self._make_implicit_object_type(
1653                 name, info, 'arg', self._make_members(data, info))
1654         if isinstance(rets, list):
1655             assert len(rets) == 1
1656             rets = self._make_array_type(rets[0], info)
1657         self._def_entity(QAPISchemaCommand(name, info, data, rets, gen,
1658                                            success_response, boxed))
1659
1660     def _def_event(self, expr, info):
1661         name = expr['event']
1662         data = expr.get('data')
1663         boxed = expr.get('boxed', False)
1664         if isinstance(data, OrderedDict):
1665             data = self._make_implicit_object_type(
1666                 name, info, 'arg', self._make_members(data, info))
1667         self._def_entity(QAPISchemaEvent(name, info, data, boxed))
1668
1669     def _def_exprs(self):
1670         for expr_elem in self.exprs:
1671             expr = expr_elem['expr']
1672             info = expr_elem['info']
1673             if 'enum' in expr:
1674                 self._def_enum_type(expr, info)
1675             elif 'struct' in expr:
1676                 self._def_struct_type(expr, info)
1677             elif 'union' in expr:
1678                 self._def_union_type(expr, info)
1679             elif 'alternate' in expr:
1680                 self._def_alternate_type(expr, info)
1681             elif 'command' in expr:
1682                 self._def_command(expr, info)
1683             elif 'event' in expr:
1684                 self._def_event(expr, info)
1685             else:
1686                 assert False
1687
1688     def check(self):
1689         for ent in self._entity_dict.values():
1690             ent.check(self)
1691
1692     def visit(self, visitor):
1693         visitor.visit_begin(self)
1694         for (name, entity) in sorted(self._entity_dict.items()):
1695             if visitor.visit_needed(entity):
1696                 entity.visit(visitor)
1697         visitor.visit_end()
1698
1699
1700 #
1701 # Code generation helpers
1702 #
1703
1704 def camel_case(name):
1705     new_name = ''
1706     first = True
1707     for ch in name:
1708         if ch in ['_', '-']:
1709             first = True
1710         elif first:
1711             new_name += ch.upper()
1712             first = False
1713         else:
1714             new_name += ch.lower()
1715     return new_name
1716
1717
1718 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1719 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1720 # ENUM24_Name -> ENUM24_NAME
1721 def camel_to_upper(value):
1722     c_fun_str = c_name(value, False)
1723     if value.isupper():
1724         return c_fun_str
1725
1726     new_name = ''
1727     l = len(c_fun_str)
1728     for i in range(l):
1729         c = c_fun_str[i]
1730         # When c is upper and no "_" appears before, do more checks
1731         if c.isupper() and (i > 0) and c_fun_str[i - 1] != "_":
1732             if i < l - 1 and c_fun_str[i + 1].islower():
1733                 new_name += '_'
1734             elif c_fun_str[i - 1].isdigit():
1735                 new_name += '_'
1736         new_name += c
1737     return new_name.lstrip('_').upper()
1738
1739
1740 def c_enum_const(type_name, const_name, prefix=None):
1741     if prefix is not None:
1742         type_name = prefix
1743     return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
1744
1745 c_name_trans = string.maketrans('.-', '__')
1746
1747
1748 # Map @name to a valid C identifier.
1749 # If @protect, avoid returning certain ticklish identifiers (like
1750 # C keywords) by prepending "q_".
1751 #
1752 # Used for converting 'name' from a 'name':'type' qapi definition
1753 # into a generated struct member, as well as converting type names
1754 # into substrings of a generated C function name.
1755 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1756 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1757 def c_name(name, protect=True):
1758     # ANSI X3J11/88-090, 3.1.1
1759     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1760                      'default', 'do', 'double', 'else', 'enum', 'extern',
1761                      'float', 'for', 'goto', 'if', 'int', 'long', 'register',
1762                      'return', 'short', 'signed', 'sizeof', 'static',
1763                      'struct', 'switch', 'typedef', 'union', 'unsigned',
1764                      'void', 'volatile', 'while'])
1765     # ISO/IEC 9899:1999, 6.4.1
1766     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1767     # ISO/IEC 9899:2011, 6.4.1
1768     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
1769                      '_Noreturn', '_Static_assert', '_Thread_local'])
1770     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1771     # excluding _.*
1772     gcc_words = set(['asm', 'typeof'])
1773     # C++ ISO/IEC 14882:2003 2.11
1774     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1775                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1776                      'namespace', 'new', 'operator', 'private', 'protected',
1777                      'public', 'reinterpret_cast', 'static_cast', 'template',
1778                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1779                      'using', 'virtual', 'wchar_t',
1780                      # alternative representations
1781                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1782                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1783     # namespace pollution:
1784     polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
1785     name = name.translate(c_name_trans)
1786     if protect and (name in c89_words | c99_words | c11_words | gcc_words
1787                     | cpp_words | polluted_words):
1788         return "q_" + name
1789     return name
1790
1791 eatspace = '\033EATSPACE.'
1792 pointer_suffix = ' *' + eatspace
1793
1794
1795 def genindent(count):
1796     ret = ""
1797     for _ in range(count):
1798         ret += " "
1799     return ret
1800
1801 indent_level = 0
1802
1803
1804 def push_indent(indent_amount=4):
1805     global indent_level
1806     indent_level += indent_amount
1807
1808
1809 def pop_indent(indent_amount=4):
1810     global indent_level
1811     indent_level -= indent_amount
1812
1813
1814 # Generate @code with @kwds interpolated.
1815 # Obey indent_level, and strip eatspace.
1816 def cgen(code, **kwds):
1817     raw = code % kwds
1818     if indent_level:
1819         indent = genindent(indent_level)
1820         # re.subn() lacks flags support before Python 2.7, use re.compile()
1821         raw = re.subn(re.compile("^.", re.MULTILINE),
1822                       indent + r'\g<0>', raw)
1823         raw = raw[0]
1824     return re.sub(re.escape(eatspace) + ' *', '', raw)
1825
1826
1827 def mcgen(code, **kwds):
1828     if code[0] == '\n':
1829         code = code[1:]
1830     return cgen(code, **kwds)
1831
1832
1833 def guardname(filename):
1834     return c_name(filename, protect=False).upper()
1835
1836
1837 def guardstart(name):
1838     return mcgen('''
1839
1840 #ifndef %(name)s
1841 #define %(name)s
1842
1843 ''',
1844                  name=guardname(name))
1845
1846
1847 def guardend(name):
1848     return mcgen('''
1849
1850 #endif /* %(name)s */
1851
1852 ''',
1853                  name=guardname(name))
1854
1855
1856 def gen_enum_lookup(name, values, prefix=None):
1857     ret = mcgen('''
1858
1859 const char *const %(c_name)s_lookup[] = {
1860 ''',
1861                 c_name=c_name(name))
1862     for value in values:
1863         index = c_enum_const(name, value, prefix)
1864         ret += mcgen('''
1865     [%(index)s] = "%(value)s",
1866 ''',
1867                      index=index, value=value)
1868
1869     max_index = c_enum_const(name, '_MAX', prefix)
1870     ret += mcgen('''
1871     [%(max_index)s] = NULL,
1872 };
1873 ''',
1874                  max_index=max_index)
1875     return ret
1876
1877
1878 def gen_enum(name, values, prefix=None):
1879     # append automatically generated _MAX value
1880     enum_values = values + ['_MAX']
1881
1882     ret = mcgen('''
1883
1884 typedef enum %(c_name)s {
1885 ''',
1886                 c_name=c_name(name))
1887
1888     i = 0
1889     for value in enum_values:
1890         ret += mcgen('''
1891     %(c_enum)s = %(i)d,
1892 ''',
1893                      c_enum=c_enum_const(name, value, prefix),
1894                      i=i)
1895         i += 1
1896
1897     ret += mcgen('''
1898 } %(c_name)s;
1899 ''',
1900                  c_name=c_name(name))
1901
1902     ret += mcgen('''
1903
1904 extern const char *const %(c_name)s_lookup[];
1905 ''',
1906                  c_name=c_name(name))
1907     return ret
1908
1909
1910 def gen_params(arg_type, boxed, extra):
1911     if not arg_type:
1912         assert not boxed
1913         return extra
1914     ret = ''
1915     sep = ''
1916     if boxed:
1917         ret += '%s arg' % arg_type.c_param_type()
1918         sep = ', '
1919     else:
1920         assert not arg_type.variants
1921         for memb in arg_type.members:
1922             ret += sep
1923             sep = ', '
1924             if memb.optional:
1925                 ret += 'bool has_%s, ' % c_name(memb.name)
1926             ret += '%s %s' % (memb.type.c_param_type(),
1927                               c_name(memb.name))
1928     if extra:
1929         ret += sep + extra
1930     return ret
1931
1932
1933 #
1934 # Common command line parsing
1935 #
1936
1937
1938 def parse_command_line(extra_options="", extra_long_options=[]):
1939
1940     try:
1941         opts, args = getopt.gnu_getopt(sys.argv[1:],
1942                                        "chp:o:" + extra_options,
1943                                        ["source", "header", "prefix=",
1944                                         "output-dir="] + extra_long_options)
1945     except getopt.GetoptError as err:
1946         print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
1947         sys.exit(1)
1948
1949     output_dir = ""
1950     prefix = ""
1951     do_c = False
1952     do_h = False
1953     extra_opts = []
1954
1955     for oa in opts:
1956         o, a = oa
1957         if o in ("-p", "--prefix"):
1958             match = re.match('([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1959             if match.end() != len(a):
1960                 print >>sys.stderr, \
1961                     "%s: 'funny character '%s' in argument of --prefix" \
1962                     % (sys.argv[0], a[match.end()])
1963                 sys.exit(1)
1964             prefix = a
1965         elif o in ("-o", "--output-dir"):
1966             output_dir = a + "/"
1967         elif o in ("-c", "--source"):
1968             do_c = True
1969         elif o in ("-h", "--header"):
1970             do_h = True
1971         else:
1972             extra_opts.append(oa)
1973
1974     if not do_c and not do_h:
1975         do_c = True
1976         do_h = True
1977
1978     if len(args) != 1:
1979         print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
1980         sys.exit(1)
1981     fname = args[0]
1982
1983     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
1984
1985 #
1986 # Generate output files with boilerplate
1987 #
1988
1989
1990 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
1991                 c_comment, h_comment):
1992     guard = guardname(prefix + h_file)
1993     c_file = output_dir + prefix + c_file
1994     h_file = output_dir + prefix + h_file
1995
1996     if output_dir:
1997         try:
1998             os.makedirs(output_dir)
1999         except os.error as e:
2000             if e.errno != errno.EEXIST:
2001                 raise
2002
2003     def maybe_open(really, name, opt):
2004         if really:
2005             return open(name, opt)
2006         else:
2007             import StringIO
2008             return StringIO.StringIO()
2009
2010     fdef = maybe_open(do_c, c_file, 'w')
2011     fdecl = maybe_open(do_h, h_file, 'w')
2012
2013     fdef.write(mcgen('''
2014 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2015 %(comment)s
2016 ''',
2017                      comment=c_comment))
2018
2019     fdecl.write(mcgen('''
2020 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2021 %(comment)s
2022 #ifndef %(guard)s
2023 #define %(guard)s
2024
2025 ''',
2026                       comment=h_comment, guard=guard))
2027
2028     return (fdef, fdecl)
2029
2030
2031 def close_output(fdef, fdecl):
2032     fdecl.write('''
2033 #endif
2034 ''')
2035     fdecl.close()
2036     fdef.close()