scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 from __future__ import print_function
  15 import errno
  16 import getopt
  17 import os
  18 import re
  19 import string
  20 import sys
  21 try:
  22     from collections import OrderedDict
  23 except:
  24     from ordereddict import OrderedDict
  25 try:
  26     from StringIO import StringIO
  27 except ImportError:
  28     from io import StringIO
  29
  30 builtin_types = {
  31     'null':     'QTYPE_QNULL',
  32     'str':      'QTYPE_QSTRING',
  33     'int':      'QTYPE_QNUM',
  34     'number':   'QTYPE_QNUM',
  35     'bool':     'QTYPE_QBOOL',
  36     'int8':     'QTYPE_QNUM',
  37     'int16':    'QTYPE_QNUM',
  38     'int32':    'QTYPE_QNUM',
  39     'int64':    'QTYPE_QNUM',
  40     'uint8':    'QTYPE_QNUM',
  41     'uint16':   'QTYPE_QNUM',
  42     'uint32':   'QTYPE_QNUM',
  43     'uint64':   'QTYPE_QNUM',
  44     'size':     'QTYPE_QNUM',
  45     'any':      None,           # any QType possible, actually
  46     'QType':    'QTYPE_QSTRING',
  47 }
  48
  49 # Are documentation comments required?
  50 doc_required = False
  51
  52 # Whitelist of commands allowed to return a non-dictionary
  53 returns_whitelist = []
  54
  55 # Whitelist of entities allowed to violate case conventions
  56 name_case_whitelist = []
  57
  58 enum_types = {}
  59 struct_types = {}
  60 union_types = {}
  61 all_names = {}
  62
  63 #
  64 # Parsing the schema into expressions
  65 #
  66
  67
  68 def error_path(parent):
  69     res = ''
  70     while parent:
  71         res = ('In file included from %s:%d:\n' % (parent['file'],
  72                                                    parent['line'])) + res
  73         parent = parent['parent']
  74     return res
  75
  76
  77 class QAPIError(Exception):
  78     def __init__(self, fname, line, col, incl_info, msg):
  79         Exception.__init__(self)
  80         self.fname = fname
  81         self.line = line
  82         self.col = col
  83         self.info = incl_info
  84         self.msg = msg
  85
  86     def __str__(self):
  87         loc = '%s:%d' % (self.fname, self.line)
  88         if self.col is not None:
  89             loc += ':%s' % self.col
  90         return error_path(self.info) + '%s: %s' % (loc, self.msg)
  91
  92
  93 class QAPIParseError(QAPIError):
  94     def __init__(self, parser, msg):
  95         col = 1
  96         for ch in parser.src[parser.line_pos:parser.pos]:
  97             if ch == '\t':
  98                 col = (col + 7) % 8 + 1
  99             else:
 100                 col += 1
 101         QAPIError.__init__(self, parser.fname, parser.line, col,
 102                            parser.incl_info, msg)
 103
 104
 105 class QAPISemError(QAPIError):
 106     def __init__(self, info, msg):
 107         QAPIError.__init__(self, info['file'], info['line'], None,
 108                            info['parent'], msg)
 109
 110
 111 class QAPIDoc(object):
 112     class Section(object):
 113         def __init__(self, name=None):
 114             # optional section name (argument/member or section name)
 115             self.name = name
 116             # the list of lines for this section
 117             self.text = ''
 118
 119         def append(self, line):
 120             self.text += line.rstrip() + '\n'
 121
 122     class ArgSection(Section):
 123         def __init__(self, name):
 124             QAPIDoc.Section.__init__(self, name)
 125             self.member = None
 126
 127         def connect(self, member):
 128             self.member = member
 129
 130     def __init__(self, parser, info):
 131         # self._parser is used to report errors with QAPIParseError.  The
 132         # resulting error position depends on the state of the parser.
 133         # It happens to be the beginning of the comment.  More or less
 134         # servicable, but action at a distance.
 135         self._parser = parser
 136         self.info = info
 137         self.symbol = None
 138         self.body = QAPIDoc.Section()
 139         # dict mapping parameter name to ArgSection
 140         self.args = OrderedDict()
 141         # a list of Section
 142         self.sections = []
 143         # the current section
 144         self._section = self.body
 145
 146     def has_section(self, name):
 147         """Return True if we have a section with this name."""
 148         for i in self.sections:
 149             if i.name == name:
 150                 return True
 151         return False
 152
 153     def append(self, line):
 154         """Parse a comment line and add it to the documentation."""
 155         line = line[1:]
 156         if not line:
 157             self._append_freeform(line)
 158             return
 159
 160         if line[0] != ' ':
 161             raise QAPIParseError(self._parser, "Missing space after #")
 162         line = line[1:]
 163
 164         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 165         # recognized, and get silently treated as ordinary text
 166         if self.symbol:
 167             self._append_symbol_line(line)
 168         elif not self.body.text and line.startswith('@'):
 169             if not line.endswith(':'):
 170                 raise QAPIParseError(self._parser, "Line should end with :")
 171             self.symbol = line[1:-1]
 172             # FIXME invalid names other than the empty string aren't flagged
 173             if not self.symbol:
 174                 raise QAPIParseError(self._parser, "Invalid name")
 175         else:
 176             self._append_freeform(line)
 177
 178     def end_comment(self):
 179         self._end_section()
 180
 181     def _append_symbol_line(self, line):
 182         name = line.split(' ', 1)[0]
 183
 184         if name.startswith('@') and name.endswith(':'):
 185             line = line[len(name)+1:]
 186             self._start_args_section(name[1:-1])
 187         elif name in ('Returns:', 'Since:',
 188                       # those are often singular or plural
 189                       'Note:', 'Notes:',
 190                       'Example:', 'Examples:',
 191                       'TODO:'):
 192             line = line[len(name)+1:]
 193             self._start_section(name[:-1])
 194
 195         self._append_freeform(line)
 196
 197     def _start_args_section(self, name):
 198         # FIXME invalid names other than the empty string aren't flagged
 199         if not name:
 200             raise QAPIParseError(self._parser, "Invalid parameter name")
 201         if name in self.args:
 202             raise QAPIParseError(self._parser,
 203                                  "'%s' parameter name duplicated" % name)
 204         if self.sections:
 205             raise QAPIParseError(self._parser,
 206                                  "'@%s:' can't follow '%s' section"
 207                                  % (name, self.sections[0].name))
 208         self._end_section()
 209         self._section = QAPIDoc.ArgSection(name)
 210         self.args[name] = self._section
 211
 212     def _start_section(self, name=None):
 213         if name in ('Returns', 'Since') and self.has_section(name):
 214             raise QAPIParseError(self._parser,
 215                                  "Duplicated '%s' section" % name)
 216         self._end_section()
 217         self._section = QAPIDoc.Section(name)
 218         self.sections.append(self._section)
 219
 220     def _end_section(self):
 221         if self._section:
 222             text = self._section.text = self._section.text.strip()
 223             if self._section.name and (not text or text.isspace()):
 224                 raise QAPIParseError(self._parser, "Empty doc section '%s'"
 225                                      % self._section.name)
 226             self._section = None
 227
 228     def _append_freeform(self, line):
 229         in_arg = isinstance(self._section, QAPIDoc.ArgSection)
 230         if (in_arg and self._section.text.endswith('\n\n')
 231                 and line and not line[0].isspace()):
 232             self._start_section()
 233         if (in_arg or not self._section.name
 234                 or not self._section.name.startswith('Example')):
 235             line = line.strip()
 236         match = re.match(r'(@\S+:)', line)
 237         if match:
 238             raise QAPIParseError(self._parser,
 239                                  "'%s' not allowed in free-form documentation"
 240                                  % match.group(1))
 241         self._section.append(line)
 242
 243     def connect_member(self, member):
 244         if member.name not in self.args:
 245             # Undocumented TODO outlaw
 246             self.args[member.name] = QAPIDoc.ArgSection(member.name)
 247         self.args[member.name].connect(member)
 248
 249     def check_expr(self, expr):
 250         if self.has_section('Returns') and 'command' not in expr:
 251             raise QAPISemError(self.info,
 252                                "'Returns:' is only valid for commands")
 253
 254     def check(self):
 255         bogus = [name for name, section in self.args.items()
 256                  if not section.member]
 257         if bogus:
 258             raise QAPISemError(
 259                 self.info,
 260                 "The following documented members are not in "
 261                 "the declaration: %s" % ", ".join(bogus))
 262
 263
 264 class QAPISchemaParser(object):
 265
 266     def __init__(self, fp, previously_included=[], incl_info=None):
 267         abs_fname = os.path.abspath(fp.name)
 268         self.fname = fp.name
 269         previously_included.append(abs_fname)
 270         self.incl_info = incl_info
 271         self.src = fp.read()
 272         if self.src == '' or self.src[-1] != '\n':
 273             self.src += '\n'
 274         self.cursor = 0
 275         self.line = 1
 276         self.line_pos = 0
 277         self.exprs = []
 278         self.docs = []
 279         self.accept()
 280         cur_doc = None
 281
 282         while self.tok is not None:
 283             info = {'file': self.fname, 'line': self.line,
 284                     'parent': self.incl_info}
 285             if self.tok == '#':
 286                 self.reject_expr_doc(cur_doc)
 287                 cur_doc = self.get_doc(info)
 288                 self.docs.append(cur_doc)
 289                 continue
 290
 291             expr = self.get_expr(False)
 292             if 'include' in expr:
 293                 self.reject_expr_doc(cur_doc)
 294                 if len(expr) != 1:
 295                     raise QAPISemError(info, "Invalid 'include' directive")
 296                 include = expr['include']
 297                 if not isinstance(include, str):
 298                     raise QAPISemError(info,
 299                                        "Value of 'include' must be a string")
 300                 self._include(include, info, os.path.dirname(abs_fname),
 301                               previously_included)
 302             elif "pragma" in expr:
 303                 self.reject_expr_doc(cur_doc)
 304                 if len(expr) != 1:
 305                     raise QAPISemError(info, "Invalid 'pragma' directive")
 306                 pragma = expr['pragma']
 307                 if not isinstance(pragma, dict):
 308                     raise QAPISemError(
 309                         info, "Value of 'pragma' must be a dictionary")
 310                 for name, value in pragma.items():
 311                     self._pragma(name, value, info)
 312             else:
 313                 expr_elem = {'expr': expr,
 314                              'info': info}
 315                 if cur_doc:
 316                     if not cur_doc.symbol:
 317                         raise QAPISemError(
 318                             cur_doc.info, "Expression documentation required")
 319                     expr_elem['doc'] = cur_doc
 320                 self.exprs.append(expr_elem)
 321             cur_doc = None
 322         self.reject_expr_doc(cur_doc)
 323
 324     @staticmethod
 325     def reject_expr_doc(doc):
 326         if doc and doc.symbol:
 327             raise QAPISemError(
 328                 doc.info,
 329                 "Documentation for '%s' is not followed by the definition"
 330                 % doc.symbol)
 331
 332     def _include(self, include, info, base_dir, previously_included):
 333         incl_abs_fname = os.path.join(base_dir, include)
 334         # catch inclusion cycle
 335         inf = info
 336         while inf:
 337             if incl_abs_fname == os.path.abspath(inf['file']):
 338                 raise QAPISemError(info, "Inclusion loop for %s" % include)
 339             inf = inf['parent']
 340
 341         # skip multiple include of the same file
 342         if incl_abs_fname in previously_included:
 343             return
 344         try:
 345             fobj = open(incl_abs_fname, 'r')
 346         except IOError as e:
 347             raise QAPISemError(info, '%s: %s' % (e.strerror, include))
 348         exprs_include = QAPISchemaParser(fobj, previously_included, info)
 349         self.exprs.extend(exprs_include.exprs)
 350         self.docs.extend(exprs_include.docs)
 351
 352     def _pragma(self, name, value, info):
 353         global doc_required, returns_whitelist, name_case_whitelist
 354         if name == 'doc-required':
 355             if not isinstance(value, bool):
 356                 raise QAPISemError(info,
 357                                    "Pragma 'doc-required' must be boolean")
 358             doc_required = value
 359         elif name == 'returns-whitelist':
 360             if (not isinstance(value, list)
 361                     or any([not isinstance(elt, str) for elt in value])):
 362                 raise QAPISemError(info,
 363                                    "Pragma returns-whitelist must be"
 364                                    " a list of strings")
 365             returns_whitelist = value
 366         elif name == 'name-case-whitelist':
 367             if (not isinstance(value, list)
 368                     or any([not isinstance(elt, str) for elt in value])):
 369                 raise QAPISemError(info,
 370                                    "Pragma name-case-whitelist must be"
 371                                    " a list of strings")
 372             name_case_whitelist = value
 373         else:
 374             raise QAPISemError(info, "Unknown pragma '%s'" % name)
 375
 376     def accept(self, skip_comment=True):
 377         while True:
 378             self.tok = self.src[self.cursor]
 379             self.pos = self.cursor
 380             self.cursor += 1
 381             self.val = None
 382
 383             if self.tok == '#':
 384                 if self.src[self.cursor] == '#':
 385                     # Start of doc comment
 386                     skip_comment = False
 387                 self.cursor = self.src.find('\n', self.cursor)
 388                 if not skip_comment:
 389                     self.val = self.src[self.pos:self.cursor]
 390                     return
 391             elif self.tok in '{}:,[]':
 392                 return
 393             elif self.tok == "'":
 394                 string = ''
 395                 esc = False
 396                 while True:
 397                     ch = self.src[self.cursor]
 398                     self.cursor += 1
 399                     if ch == '\n':
 400                         raise QAPIParseError(self, 'Missing terminating "\'"')
 401                     if esc:
 402                         if ch == 'b':
 403                             string += '\b'
 404                         elif ch == 'f':
 405                             string += '\f'
 406                         elif ch == 'n':
 407                             string += '\n'
 408                         elif ch == 'r':
 409                             string += '\r'
 410                         elif ch == 't':
 411                             string += '\t'
 412                         elif ch == 'u':
 413                             value = 0
 414                             for _ in range(0, 4):
 415                                 ch = self.src[self.cursor]
 416                                 self.cursor += 1
 417                                 if ch not in '0123456789abcdefABCDEF':
 418                                     raise QAPIParseError(self,
 419                                                          '\\u escape needs 4 '
 420                                                          'hex digits')
 421                                 value = (value << 4) + int(ch, 16)
 422                             # If Python 2 and 3 didn't disagree so much on
 423                             # how to handle Unicode, then we could allow
 424                             # Unicode string defaults.  But most of QAPI is
 425                             # ASCII-only, so we aren't losing much for now.
 426                             if not value or value > 0x7f:
 427                                 raise QAPIParseError(self,
 428                                                      'For now, \\u escape '
 429                                                      'only supports non-zero '
 430                                                      'values up to \\u007f')
 431                             string += chr(value)
 432                         elif ch in '\\/\'"':
 433                             string += ch
 434                         else:
 435                             raise QAPIParseError(self,
 436                                                  "Unknown escape \\%s" % ch)
 437                         esc = False
 438                     elif ch == '\\':
 439                         esc = True
 440                     elif ch == "'":
 441                         self.val = string
 442                         return
 443                     else:
 444                         string += ch
 445             elif self.src.startswith('true', self.pos):
 446                 self.val = True
 447                 self.cursor += 3
 448                 return
 449             elif self.src.startswith('false', self.pos):
 450                 self.val = False
 451                 self.cursor += 4
 452                 return
 453             elif self.src.startswith('null', self.pos):
 454                 self.val = None
 455                 self.cursor += 3
 456                 return
 457             elif self.tok == '\n':
 458                 if self.cursor == len(self.src):
 459                     self.tok = None
 460                     return
 461                 self.line += 1
 462                 self.line_pos = self.cursor
 463             elif not self.tok.isspace():
 464                 raise QAPIParseError(self, 'Stray "%s"' % self.tok)
 465
 466     def get_members(self):
 467         expr = OrderedDict()
 468         if self.tok == '}':
 469             self.accept()
 470             return expr
 471         if self.tok != "'":
 472             raise QAPIParseError(self, 'Expected string or "}"')
 473         while True:
 474             key = self.val
 475             self.accept()
 476             if self.tok != ':':
 477                 raise QAPIParseError(self, 'Expected ":"')
 478             self.accept()
 479             if key in expr:
 480                 raise QAPIParseError(self, 'Duplicate key "%s"' % key)
 481             expr[key] = self.get_expr(True)
 482             if self.tok == '}':
 483                 self.accept()
 484                 return expr
 485             if self.tok != ',':
 486                 raise QAPIParseError(self, 'Expected "," or "}"')
 487             self.accept()
 488             if self.tok != "'":
 489                 raise QAPIParseError(self, 'Expected string')
 490
 491     def get_values(self):
 492         expr = []
 493         if self.tok == ']':
 494             self.accept()
 495             return expr
 496         if self.tok not in "{['tfn":
 497             raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
 498                                  'boolean or "null"')
 499         while True:
 500             expr.append(self.get_expr(True))
 501             if self.tok == ']':
 502                 self.accept()
 503                 return expr
 504             if self.tok != ',':
 505                 raise QAPIParseError(self, 'Expected "," or "]"')
 506             self.accept()
 507
 508     def get_expr(self, nested):
 509         if self.tok != '{' and not nested:
 510             raise QAPIParseError(self, 'Expected "{"')
 511         if self.tok == '{':
 512             self.accept()
 513             expr = self.get_members()
 514         elif self.tok == '[':
 515             self.accept()
 516             expr = self.get_values()
 517         elif self.tok in "'tfn":
 518             expr = self.val
 519             self.accept()
 520         else:
 521             raise QAPIParseError(self, 'Expected "{", "[", string, '
 522                                  'boolean or "null"')
 523         return expr
 524
 525     def get_doc(self, info):
 526         if self.val != '##':
 527             raise QAPIParseError(self, "Junk after '##' at start of "
 528                                  "documentation comment")
 529
 530         doc = QAPIDoc(self, info)
 531         self.accept(False)
 532         while self.tok == '#':
 533             if self.val.startswith('##'):
 534                 # End of doc comment
 535                 if self.val != '##':
 536                     raise QAPIParseError(self, "Junk after '##' at end of "
 537                                          "documentation comment")
 538                 doc.end_comment()
 539                 self.accept()
 540                 return doc
 541             else:
 542                 doc.append(self.val)
 543             self.accept(False)
 544
 545         raise QAPIParseError(self, "Documentation comment must end with '##'")
 546
 547
 548 #
 549 # Semantic analysis of schema expressions
 550 # TODO fold into QAPISchema
 551 # TODO catching name collisions in generated code would be nice
 552 #
 553
 554
 555 def find_base_members(base):
 556     if isinstance(base, dict):
 557         return base
 558     base_struct_define = struct_types.get(base)
 559     if not base_struct_define:
 560         return None
 561     return base_struct_define['data']
 562
 563
 564 # Return the qtype of an alternate branch, or None on error.
 565 def find_alternate_member_qtype(qapi_type):
 566     if qapi_type in builtin_types:
 567         return builtin_types[qapi_type]
 568     elif qapi_type in struct_types:
 569         return 'QTYPE_QDICT'
 570     elif qapi_type in enum_types:
 571         return 'QTYPE_QSTRING'
 572     elif qapi_type in union_types:
 573         return 'QTYPE_QDICT'
 574     return None
 575
 576
 577 # Return the discriminator enum define if discriminator is specified as an
 578 # enum type, otherwise return None.
 579 def discriminator_find_enum_define(expr):
 580     base = expr.get('base')
 581     discriminator = expr.get('discriminator')
 582
 583     if not (discriminator and base):
 584         return None
 585
 586     base_members = find_base_members(base)
 587     if not base_members:
 588         return None
 589
 590     discriminator_type = base_members.get(discriminator)
 591     if not discriminator_type:
 592         return None
 593
 594     return enum_types.get(discriminator_type)
 595
 596
 597 # Names must be letters, numbers, -, and _.  They must start with letter,
 598 # except for downstream extensions which must start with __RFQDN_.
 599 # Dots are only valid in the downstream extension prefix.
 600 valid_name = re.compile(r'^(__[a-zA-Z0-9.-]+_)?'
 601                         '[a-zA-Z][a-zA-Z0-9_-]*$')
 602
 603
 604 def check_name(info, source, name, allow_optional=False,
 605                enum_member=False):
 606     global valid_name
 607     membername = name
 608
 609     if not isinstance(name, str):
 610         raise QAPISemError(info, "%s requires a string name" % source)
 611     if name.startswith('*'):
 612         membername = name[1:]
 613         if not allow_optional:
 614             raise QAPISemError(info, "%s does not allow optional name '%s'"
 615                                % (source, name))
 616     # Enum members can start with a digit, because the generated C
 617     # code always prefixes it with the enum name
 618     if enum_member and membername[0].isdigit():
 619         membername = 'D' + membername
 620     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
 621     # and 'q_obj_*' implicit type names.
 622     if not valid_name.match(membername) or \
 623        c_name(membername, False).startswith('q_'):
 624         raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
 625
 626
 627 def add_name(name, info, meta, implicit=False):
 628     global all_names
 629     check_name(info, "'%s'" % meta, name)
 630     # FIXME should reject names that differ only in '_' vs. '.'
 631     # vs. '-', because they're liable to clash in generated C.
 632     if name in all_names:
 633         raise QAPISemError(info, "%s '%s' is already defined"
 634                            % (all_names[name], name))
 635     if not implicit and (name.endswith('Kind') or name.endswith('List')):
 636         raise QAPISemError(info, "%s '%s' should not end in '%s'"
 637                            % (meta, name, name[-4:]))
 638     all_names[name] = meta
 639
 640
 641 def check_type(info, source, value, allow_array=False,
 642                allow_dict=False, allow_optional=False,
 643                allow_metas=[]):
 644     global all_names
 645
 646     if value is None:
 647         return
 648
 649     # Check if array type for value is okay
 650     if isinstance(value, list):
 651         if not allow_array:
 652             raise QAPISemError(info, "%s cannot be an array" % source)
 653         if len(value) != 1 or not isinstance(value[0], str):
 654             raise QAPISemError(info,
 655                                "%s: array type must contain single type name" %
 656                                source)
 657         value = value[0]
 658
 659     # Check if type name for value is okay
 660     if isinstance(value, str):
 661         if value not in all_names:
 662             raise QAPISemError(info, "%s uses unknown type '%s'"
 663                                % (source, value))
 664         if not all_names[value] in allow_metas:
 665             raise QAPISemError(info, "%s cannot use %s type '%s'" %
 666                                (source, all_names[value], value))
 667         return
 668
 669     if not allow_dict:
 670         raise QAPISemError(info, "%s should be a type name" % source)
 671
 672     if not isinstance(value, OrderedDict):
 673         raise QAPISemError(info,
 674                            "%s should be a dictionary or type name" % source)
 675
 676     # value is a dictionary, check that each member is okay
 677     for (key, arg) in value.items():
 678         check_name(info, "Member of %s" % source, key,
 679                    allow_optional=allow_optional)
 680         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 681             raise QAPISemError(info, "Member of %s uses reserved name '%s'"
 682                                % (source, key))
 683         # Todo: allow dictionaries to represent default values of
 684         # an optional argument.
 685         check_type(info, "Member '%s' of %s" % (key, source), arg,
 686                    allow_array=True,
 687                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 688                                 'enum'])
 689
 690
 691 def check_command(expr, info):
 692     name = expr['command']
 693     boxed = expr.get('boxed', False)
 694
 695     args_meta = ['struct']
 696     if boxed:
 697         args_meta += ['union', 'alternate']
 698     check_type(info, "'data' for command '%s'" % name,
 699                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 700                allow_metas=args_meta)
 701     returns_meta = ['union', 'struct']
 702     if name in returns_whitelist:
 703         returns_meta += ['built-in', 'alternate', 'enum']
 704     check_type(info, "'returns' for command '%s'" % name,
 705                expr.get('returns'), allow_array=True,
 706                allow_optional=True, allow_metas=returns_meta)
 707
 708
 709 def check_event(expr, info):
 710     name = expr['event']
 711     boxed = expr.get('boxed', False)
 712
 713     meta = ['struct']
 714     if boxed:
 715         meta += ['union', 'alternate']
 716     check_type(info, "'data' for event '%s'" % name,
 717                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 718                allow_metas=meta)
 719
 720
 721 def check_union(expr, info):
 722     name = expr['union']
 723     base = expr.get('base')
 724     discriminator = expr.get('discriminator')
 725     members = expr['data']
 726
 727     # Two types of unions, determined by discriminator.
 728
 729     # With no discriminator it is a simple union.
 730     if discriminator is None:
 731         enum_define = None
 732         allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
 733         if base is not None:
 734             raise QAPISemError(info, "Simple union '%s' must not have a base" %
 735                                name)
 736
 737     # Else, it's a flat union.
 738     else:
 739         # The object must have a string or dictionary 'base'.
 740         check_type(info, "'base' for union '%s'" % name,
 741                    base, allow_dict=True, allow_optional=True,
 742                    allow_metas=['struct'])
 743         if not base:
 744             raise QAPISemError(info, "Flat union '%s' must have a base"
 745                                % name)
 746         base_members = find_base_members(base)
 747         assert base_members is not None
 748
 749         # The value of member 'discriminator' must name a non-optional
 750         # member of the base struct.
 751         check_name(info, "Discriminator of flat union '%s'" % name,
 752                    discriminator)
 753         discriminator_type = base_members.get(discriminator)
 754         if not discriminator_type:
 755             raise QAPISemError(info,
 756                                "Discriminator '%s' is not a member of base "
 757                                "struct '%s'"
 758                                % (discriminator, base))
 759         enum_define = enum_types.get(discriminator_type)
 760         allow_metas = ['struct']
 761         # Do not allow string discriminator
 762         if not enum_define:
 763             raise QAPISemError(info,
 764                                "Discriminator '%s' must be of enumeration "
 765                                "type" % discriminator)
 766
 767     # Check every branch; don't allow an empty union
 768     if len(members) == 0:
 769         raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
 770     for (key, value) in members.items():
 771         check_name(info, "Member of union '%s'" % name, key)
 772
 773         # Each value must name a known type
 774         check_type(info, "Member '%s' of union '%s'" % (key, name),
 775                    value, allow_array=not base, allow_metas=allow_metas)
 776
 777         # If the discriminator names an enum type, then all members
 778         # of 'data' must also be members of the enum type.
 779         if enum_define:
 780             if key not in enum_define['data']:
 781                 raise QAPISemError(info,
 782                                    "Discriminator value '%s' is not found in "
 783                                    "enum '%s'"
 784                                    % (key, enum_define['enum']))
 785
 786     # If discriminator is user-defined, ensure all values are covered
 787     if enum_define:
 788         for value in enum_define['data']:
 789             if value not in members.keys():
 790                 raise QAPISemError(info, "Union '%s' data missing '%s' branch"
 791                                    % (name, value))
 792
 793
 794 def check_alternate(expr, info):
 795     name = expr['alternate']
 796     members = expr['data']
 797     types_seen = {}
 798
 799     # Check every branch; require at least two branches
 800     if len(members) < 2:
 801         raise QAPISemError(info,
 802                            "Alternate '%s' should have at least two branches "
 803                            "in 'data'" % name)
 804     for (key, value) in members.items():
 805         check_name(info, "Member of alternate '%s'" % name, key)
 806
 807         # Ensure alternates have no type conflicts.
 808         check_type(info, "Member '%s' of alternate '%s'" % (key, name),
 809                    value,
 810                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 811         qtype = find_alternate_member_qtype(value)
 812         if not qtype:
 813             raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
 814                                "type '%s'" % (name, key, value))
 815         conflicting = set([qtype])
 816         if qtype == 'QTYPE_QSTRING':
 817             enum_expr = enum_types.get(value)
 818             if enum_expr:
 819                 for v in enum_expr['data']:
 820                     if v in ['on', 'off']:
 821                         conflicting.add('QTYPE_QBOOL')
 822                     if re.match(r'[-+0-9.]', v): # lazy, could be tightened
 823                         conflicting.add('QTYPE_QNUM')
 824             else:
 825                 conflicting.add('QTYPE_QNUM')
 826                 conflicting.add('QTYPE_QBOOL')
 827         for qt in conflicting:
 828             if qt in types_seen:
 829                 raise QAPISemError(info, "Alternate '%s' member '%s' can't "
 830                                    "be distinguished from member '%s'"
 831                                    % (name, key, types_seen[qt]))
 832             types_seen[qt] = key
 833
 834
 835 def check_enum(expr, info):
 836     name = expr['enum']
 837     members = expr.get('data')
 838     prefix = expr.get('prefix')
 839
 840     if not isinstance(members, list):
 841         raise QAPISemError(info,
 842                            "Enum '%s' requires an array for 'data'" % name)
 843     if prefix is not None and not isinstance(prefix, str):
 844         raise QAPISemError(info,
 845                            "Enum '%s' requires a string for 'prefix'" % name)
 846     for member in members:
 847         check_name(info, "Member of enum '%s'" % name, member,
 848                    enum_member=True)
 849
 850
 851 def check_struct(expr, info):
 852     name = expr['struct']
 853     members = expr['data']
 854
 855     check_type(info, "'data' for struct '%s'" % name, members,
 856                allow_dict=True, allow_optional=True)
 857     check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
 858                allow_metas=['struct'])
 859
 860
 861 def check_keys(expr_elem, meta, required, optional=[]):
 862     expr = expr_elem['expr']
 863     info = expr_elem['info']
 864     name = expr[meta]
 865     if not isinstance(name, str):
 866         raise QAPISemError(info, "'%s' key must have a string value" % meta)
 867     required = required + [meta]
 868     for (key, value) in expr.items():
 869         if key not in required and key not in optional:
 870             raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
 871                                % (key, meta, name))
 872         if (key == 'gen' or key == 'success-response') and value is not False:
 873             raise QAPISemError(info,
 874                                "'%s' of %s '%s' should only use false value"
 875                                % (key, meta, name))
 876         if key == 'boxed' and value is not True:
 877             raise QAPISemError(info,
 878                                "'%s' of %s '%s' should only use true value"
 879                                % (key, meta, name))
 880     for key in required:
 881         if key not in expr:
 882             raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
 883                                % (key, meta, name))
 884
 885
 886 def check_exprs(exprs):
 887     global all_names
 888
 889     # Populate name table with names of built-in types
 890     for builtin in builtin_types.keys():
 891         all_names[builtin] = 'built-in'
 892
 893     # Learn the types and check for valid expression keys
 894     for expr_elem in exprs:
 895         expr = expr_elem['expr']
 896         info = expr_elem['info']
 897         doc = expr_elem.get('doc')
 898
 899         if not doc and doc_required:
 900             raise QAPISemError(info,
 901                                "Expression missing documentation comment")
 902
 903         if 'enum' in expr:
 904             meta = 'enum'
 905             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 906             enum_types[expr[meta]] = expr
 907         elif 'union' in expr:
 908             meta = 'union'
 909             check_keys(expr_elem, 'union', ['data'],
 910                        ['base', 'discriminator'])
 911             union_types[expr[meta]] = expr
 912         elif 'alternate' in expr:
 913             meta = 'alternate'
 914             check_keys(expr_elem, 'alternate', ['data'])
 915         elif 'struct' in expr:
 916             meta = 'struct'
 917             check_keys(expr_elem, 'struct', ['data'], ['base'])
 918             struct_types[expr[meta]] = expr
 919         elif 'command' in expr:
 920             meta = 'command'
 921             check_keys(expr_elem, 'command', [],
 922                        ['data', 'returns', 'gen', 'success-response', 'boxed'])
 923         elif 'event' in expr:
 924             meta = 'event'
 925             check_keys(expr_elem, 'event', [], ['data', 'boxed'])
 926         else:
 927             raise QAPISemError(expr_elem['info'],
 928                                "Expression is missing metatype")
 929         name = expr[meta]
 930         add_name(name, info, meta)
 931         if doc and doc.symbol != name:
 932             raise QAPISemError(info, "Definition of '%s' follows documentation"
 933                                " for '%s'" % (name, doc.symbol))
 934
 935     # Try again for hidden UnionKind enum
 936     for expr_elem in exprs:
 937         expr = expr_elem['expr']
 938         if 'union' in expr and not discriminator_find_enum_define(expr):
 939             name = '%sKind' % expr['union']
 940         elif 'alternate' in expr:
 941             name = '%sKind' % expr['alternate']
 942         else:
 943             continue
 944         enum_types[name] = {'enum': name}
 945         add_name(name, info, 'enum', implicit=True)
 946
 947     # Validate that exprs make sense
 948     for expr_elem in exprs:
 949         expr = expr_elem['expr']
 950         info = expr_elem['info']
 951         doc = expr_elem.get('doc')
 952
 953         if 'enum' in expr:
 954             check_enum(expr, info)
 955         elif 'union' in expr:
 956             check_union(expr, info)
 957         elif 'alternate' in expr:
 958             check_alternate(expr, info)
 959         elif 'struct' in expr:
 960             check_struct(expr, info)
 961         elif 'command' in expr:
 962             check_command(expr, info)
 963         elif 'event' in expr:
 964             check_event(expr, info)
 965         else:
 966             assert False, 'unexpected meta type'
 967
 968         if doc:
 969             doc.check_expr(expr)
 970
 971     return exprs
 972
 973
 974 #
 975 # Schema compiler frontend
 976 #
 977
 978 class QAPISchemaEntity(object):
 979     def __init__(self, name, info, doc):
 980         assert isinstance(name, str)
 981         self.name = name
 982         # For explicitly defined entities, info points to the (explicit)
 983         # definition.  For builtins (and their arrays), info is None.
 984         # For implicitly defined entities, info points to a place that
 985         # triggered the implicit definition (there may be more than one
 986         # such place).
 987         self.info = info
 988         self.doc = doc
 989
 990     def c_name(self):
 991         return c_name(self.name)
 992
 993     def check(self, schema):
 994         pass
 995
 996     def is_implicit(self):
 997         return not self.info
 998
 999     def visit(self, visitor):
1000         pass
1001
1002
1003 class QAPISchemaVisitor(object):
1004     def visit_begin(self, schema):
1005         pass
1006
1007     def visit_end(self):
1008         pass
1009
1010     def visit_needed(self, entity):
1011         # Default to visiting everything
1012         return True
1013
1014     def visit_builtin_type(self, name, info, json_type):
1015         pass
1016
1017     def visit_enum_type(self, name, info, values, prefix):
1018         pass
1019
1020     def visit_array_type(self, name, info, element_type):
1021         pass
1022
1023     def visit_object_type(self, name, info, base, members, variants):
1024         pass
1025
1026     def visit_object_type_flat(self, name, info, members, variants):
1027         pass
1028
1029     def visit_alternate_type(self, name, info, variants):
1030         pass
1031
1032     def visit_command(self, name, info, arg_type, ret_type,
1033                       gen, success_response, boxed):
1034         pass
1035
1036     def visit_event(self, name, info, arg_type, boxed):
1037         pass
1038
1039
1040 class QAPISchemaType(QAPISchemaEntity):
1041     # Return the C type for common use.
1042     # For the types we commonly box, this is a pointer type.
1043     def c_type(self):
1044         pass
1045
1046     # Return the C type to be used in a parameter list.
1047     def c_param_type(self):
1048         return self.c_type()
1049
1050     # Return the C type to be used where we suppress boxing.
1051     def c_unboxed_type(self):
1052         return self.c_type()
1053
1054     def json_type(self):
1055         pass
1056
1057     def alternate_qtype(self):
1058         json2qtype = {
1059             'null':    'QTYPE_QNULL',
1060             'string':  'QTYPE_QSTRING',
1061             'number':  'QTYPE_QNUM',
1062             'int':     'QTYPE_QNUM',
1063             'boolean': 'QTYPE_QBOOL',
1064             'object':  'QTYPE_QDICT'
1065         }
1066         return json2qtype.get(self.json_type())
1067
1068     def doc_type(self):
1069         if self.is_implicit():
1070             return None
1071         return self.name
1072
1073
1074 class QAPISchemaBuiltinType(QAPISchemaType):
1075     def __init__(self, name, json_type, c_type):
1076         QAPISchemaType.__init__(self, name, None, None)
1077         assert not c_type or isinstance(c_type, str)
1078         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
1079                              'value')
1080         self._json_type_name = json_type
1081         self._c_type_name = c_type
1082
1083     def c_name(self):
1084         return self.name
1085
1086     def c_type(self):
1087         return self._c_type_name
1088
1089     def c_param_type(self):
1090         if self.name == 'str':
1091             return 'const ' + self._c_type_name
1092         return self._c_type_name
1093
1094     def json_type(self):
1095         return self._json_type_name
1096
1097     def doc_type(self):
1098         return self.json_type()
1099
1100     def visit(self, visitor):
1101         visitor.visit_builtin_type(self.name, self.info, self.json_type())
1102
1103
1104 class QAPISchemaEnumType(QAPISchemaType):
1105     def __init__(self, name, info, doc, values, prefix):
1106         QAPISchemaType.__init__(self, name, info, doc)
1107         for v in values:
1108             assert isinstance(v, QAPISchemaMember)
1109             v.set_owner(name)
1110         assert prefix is None or isinstance(prefix, str)
1111         self.values = values
1112         self.prefix = prefix
1113
1114     def check(self, schema):
1115         seen = {}
1116         for v in self.values:
1117             v.check_clash(self.info, seen)
1118             if self.doc:
1119                 self.doc.connect_member(v)
1120
1121     def is_implicit(self):
1122         # See QAPISchema._make_implicit_enum_type() and ._def_predefineds()
1123         return self.name.endswith('Kind') or self.name == 'QType'
1124
1125     def c_type(self):
1126         return c_name(self.name)
1127
1128     def member_names(self):
1129         return [v.name for v in self.values]
1130
1131     def json_type(self):
1132         return 'string'
1133
1134     def visit(self, visitor):
1135         visitor.visit_enum_type(self.name, self.info,
1136                                 self.member_names(), self.prefix)
1137
1138
1139 class QAPISchemaArrayType(QAPISchemaType):
1140     def __init__(self, name, info, element_type):
1141         QAPISchemaType.__init__(self, name, info, None)
1142         assert isinstance(element_type, str)
1143         self._element_type_name = element_type
1144         self.element_type = None
1145
1146     def check(self, schema):
1147         self.element_type = schema.lookup_type(self._element_type_name)
1148         assert self.element_type
1149
1150     def is_implicit(self):
1151         return True
1152
1153     def c_type(self):
1154         return c_name(self.name) + pointer_suffix
1155
1156     def json_type(self):
1157         return 'array'
1158
1159     def doc_type(self):
1160         elt_doc_type = self.element_type.doc_type()
1161         if not elt_doc_type:
1162             return None
1163         return 'array of ' + elt_doc_type
1164
1165     def visit(self, visitor):
1166         visitor.visit_array_type(self.name, self.info, self.element_type)
1167
1168
1169 class QAPISchemaObjectType(QAPISchemaType):
1170     def __init__(self, name, info, doc, base, local_members, variants):
1171         # struct has local_members, optional base, and no variants
1172         # flat union has base, variants, and no local_members
1173         # simple union has local_members, variants, and no base
1174         QAPISchemaType.__init__(self, name, info, doc)
1175         assert base is None or isinstance(base, str)
1176         for m in local_members:
1177             assert isinstance(m, QAPISchemaObjectTypeMember)
1178             m.set_owner(name)
1179         if variants is not None:
1180             assert isinstance(variants, QAPISchemaObjectTypeVariants)
1181             variants.set_owner(name)
1182         self._base_name = base
1183         self.base = None
1184         self.local_members = local_members
1185         self.variants = variants
1186         self.members = None
1187
1188     def check(self, schema):
1189         if self.members is False:               # check for cycles
1190             raise QAPISemError(self.info,
1191                                "Object %s contains itself" % self.name)
1192         if self.members:
1193             return
1194         self.members = False                    # mark as being checked
1195         seen = OrderedDict()
1196         if self._base_name:
1197             self.base = schema.lookup_type(self._base_name)
1198             assert isinstance(self.base, QAPISchemaObjectType)
1199             self.base.check(schema)
1200             self.base.check_clash(self.info, seen)
1201         for m in self.local_members:
1202             m.check(schema)
1203             m.check_clash(self.info, seen)
1204             if self.doc:
1205                 self.doc.connect_member(m)
1206         self.members = seen.values()
1207         if self.variants:
1208             self.variants.check(schema, seen)
1209             assert self.variants.tag_member in self.members
1210             self.variants.check_clash(self.info, seen)
1211         if self.doc:
1212             self.doc.check()
1213
1214     # Check that the members of this type do not cause duplicate JSON members,
1215     # and update seen to track the members seen so far. Report any errors
1216     # on behalf of info, which is not necessarily self.info
1217     def check_clash(self, info, seen):
1218         assert not self.variants       # not implemented
1219         for m in self.members:
1220             m.check_clash(info, seen)
1221
1222     def is_implicit(self):
1223         # See QAPISchema._make_implicit_object_type(), as well as
1224         # _def_predefineds()
1225         return self.name.startswith('q_')
1226
1227     def is_empty(self):
1228         assert self.members is not None
1229         return not self.members and not self.variants
1230
1231     def c_name(self):
1232         assert self.name != 'q_empty'
1233         return QAPISchemaType.c_name(self)
1234
1235     def c_type(self):
1236         assert not self.is_implicit()
1237         return c_name(self.name) + pointer_suffix
1238
1239     def c_unboxed_type(self):
1240         return c_name(self.name)
1241
1242     def json_type(self):
1243         return 'object'
1244
1245     def visit(self, visitor):
1246         visitor.visit_object_type(self.name, self.info,
1247                                   self.base, self.local_members, self.variants)
1248         visitor.visit_object_type_flat(self.name, self.info,
1249                                        self.members, self.variants)
1250
1251
1252 class QAPISchemaMember(object):
1253     role = 'member'
1254
1255     def __init__(self, name):
1256         assert isinstance(name, str)
1257         self.name = name
1258         self.owner = None
1259
1260     def set_owner(self, name):
1261         assert not self.owner
1262         self.owner = name
1263
1264     def check_clash(self, info, seen):
1265         cname = c_name(self.name)
1266         if cname.lower() != cname and self.owner not in name_case_whitelist:
1267             raise QAPISemError(info,
1268                                "%s should not use uppercase" % self.describe())
1269         if cname in seen:
1270             raise QAPISemError(info, "%s collides with %s" %
1271                                (self.describe(), seen[cname].describe()))
1272         seen[cname] = self
1273
1274     def _pretty_owner(self):
1275         owner = self.owner
1276         if owner.startswith('q_obj_'):
1277             # See QAPISchema._make_implicit_object_type() - reverse the
1278             # mapping there to create a nice human-readable description
1279             owner = owner[6:]
1280             if owner.endswith('-arg'):
1281                 return '(parameter of %s)' % owner[:-4]
1282             elif owner.endswith('-base'):
1283                 return '(base of %s)' % owner[:-5]
1284             else:
1285                 assert owner.endswith('-wrapper')
1286                 # Unreachable and not implemented
1287                 assert False
1288         if owner.endswith('Kind'):
1289             # See QAPISchema._make_implicit_enum_type()
1290             return '(branch of %s)' % owner[:-4]
1291         return '(%s of %s)' % (self.role, owner)
1292
1293     def describe(self):
1294         return "'%s' %s" % (self.name, self._pretty_owner())
1295
1296
1297 class QAPISchemaObjectTypeMember(QAPISchemaMember):
1298     def __init__(self, name, typ, optional):
1299         QAPISchemaMember.__init__(self, name)
1300         assert isinstance(typ, str)
1301         assert isinstance(optional, bool)
1302         self._type_name = typ
1303         self.type = None
1304         self.optional = optional
1305
1306     def check(self, schema):
1307         assert self.owner
1308         self.type = schema.lookup_type(self._type_name)
1309         assert self.type
1310
1311
1312 class QAPISchemaObjectTypeVariants(object):
1313     def __init__(self, tag_name, tag_member, variants):
1314         # Flat unions pass tag_name but not tag_member.
1315         # Simple unions and alternates pass tag_member but not tag_name.
1316         # After check(), tag_member is always set, and tag_name remains
1317         # a reliable witness of being used by a flat union.
1318         assert bool(tag_member) != bool(tag_name)
1319         assert (isinstance(tag_name, str) or
1320                 isinstance(tag_member, QAPISchemaObjectTypeMember))
1321         assert len(variants) > 0
1322         for v in variants:
1323             assert isinstance(v, QAPISchemaObjectTypeVariant)
1324         self._tag_name = tag_name
1325         self.tag_member = tag_member
1326         self.variants = variants
1327
1328     def set_owner(self, name):
1329         for v in self.variants:
1330             v.set_owner(name)
1331
1332     def check(self, schema, seen):
1333         if not self.tag_member:    # flat union
1334             self.tag_member = seen[c_name(self._tag_name)]
1335             assert self._tag_name == self.tag_member.name
1336         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
1337         for v in self.variants:
1338             v.check(schema)
1339             # Union names must match enum values; alternate names are
1340             # checked separately. Use 'seen' to tell the two apart.
1341             if seen:
1342                 assert v.name in self.tag_member.type.member_names()
1343                 assert isinstance(v.type, QAPISchemaObjectType)
1344                 v.type.check(schema)
1345
1346     def check_clash(self, info, seen):
1347         for v in self.variants:
1348             # Reset seen map for each variant, since qapi names from one
1349             # branch do not affect another branch
1350             assert isinstance(v.type, QAPISchemaObjectType)
1351             v.type.check_clash(info, dict(seen))
1352
1353
1354 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1355     role = 'branch'
1356
1357     def __init__(self, name, typ):
1358         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1359
1360
1361 class QAPISchemaAlternateType(QAPISchemaType):
1362     def __init__(self, name, info, doc, variants):
1363         QAPISchemaType.__init__(self, name, info, doc)
1364         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1365         assert variants.tag_member
1366         variants.set_owner(name)
1367         variants.tag_member.set_owner(self.name)
1368         self.variants = variants
1369
1370     def check(self, schema):
1371         self.variants.tag_member.check(schema)
1372         # Not calling self.variants.check_clash(), because there's nothing
1373         # to clash with
1374         self.variants.check(schema, {})
1375         # Alternate branch names have no relation to the tag enum values;
1376         # so we have to check for potential name collisions ourselves.
1377         seen = {}
1378         for v in self.variants.variants:
1379             v.check_clash(self.info, seen)
1380             if self.doc:
1381                 self.doc.connect_member(v)
1382         if self.doc:
1383             self.doc.check()
1384
1385     def c_type(self):
1386         return c_name(self.name) + pointer_suffix
1387
1388     def json_type(self):
1389         return 'value'
1390
1391     def visit(self, visitor):
1392         visitor.visit_alternate_type(self.name, self.info, self.variants)
1393
1394     def is_empty(self):
1395         return False
1396
1397
1398 class QAPISchemaCommand(QAPISchemaEntity):
1399     def __init__(self, name, info, doc, arg_type, ret_type,
1400                  gen, success_response, boxed):
1401         QAPISchemaEntity.__init__(self, name, info, doc)
1402         assert not arg_type or isinstance(arg_type, str)
1403         assert not ret_type or isinstance(ret_type, str)
1404         self._arg_type_name = arg_type
1405         self.arg_type = None
1406         self._ret_type_name = ret_type
1407         self.ret_type = None
1408         self.gen = gen
1409         self.success_response = success_response
1410         self.boxed = boxed
1411
1412     def check(self, schema):
1413         if self._arg_type_name:
1414             self.arg_type = schema.lookup_type(self._arg_type_name)
1415             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1416                     isinstance(self.arg_type, QAPISchemaAlternateType))
1417             self.arg_type.check(schema)
1418             if self.boxed:
1419                 if self.arg_type.is_empty():
1420                     raise QAPISemError(self.info,
1421                                        "Cannot use 'boxed' with empty type")
1422             else:
1423                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1424                 assert not self.arg_type.variants
1425         elif self.boxed:
1426             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1427         if self._ret_type_name:
1428             self.ret_type = schema.lookup_type(self._ret_type_name)
1429             assert isinstance(self.ret_type, QAPISchemaType)
1430
1431     def visit(self, visitor):
1432         visitor.visit_command(self.name, self.info,
1433                               self.arg_type, self.ret_type,
1434                               self.gen, self.success_response, self.boxed)
1435
1436
1437 class QAPISchemaEvent(QAPISchemaEntity):
1438     def __init__(self, name, info, doc, arg_type, boxed):
1439         QAPISchemaEntity.__init__(self, name, info, doc)
1440         assert not arg_type or isinstance(arg_type, str)
1441         self._arg_type_name = arg_type
1442         self.arg_type = None
1443         self.boxed = boxed
1444
1445     def check(self, schema):
1446         if self._arg_type_name:
1447             self.arg_type = schema.lookup_type(self._arg_type_name)
1448             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1449                     isinstance(self.arg_type, QAPISchemaAlternateType))
1450             self.arg_type.check(schema)
1451             if self.boxed:
1452                 if self.arg_type.is_empty():
1453                     raise QAPISemError(self.info,
1454                                        "Cannot use 'boxed' with empty type")
1455             else:
1456                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1457                 assert not self.arg_type.variants
1458         elif self.boxed:
1459             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1460
1461     def visit(self, visitor):
1462         visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
1463
1464
1465 class QAPISchema(object):
1466     def __init__(self, fname):
1467         try:
1468             parser = QAPISchemaParser(open(fname, 'r'))
1469             self.exprs = check_exprs(parser.exprs)
1470             self.docs = parser.docs
1471             self._entity_dict = {}
1472             self._predefining = True
1473             self._def_predefineds()
1474             self._predefining = False
1475             self._def_exprs()
1476             self.check()
1477         except QAPIError as err:
1478             print(err, file=sys.stderr)
1479             exit(1)
1480
1481     def _def_entity(self, ent):
1482         # Only the predefined types are allowed to not have info
1483         assert ent.info or self._predefining
1484         assert ent.name not in self._entity_dict
1485         self._entity_dict[ent.name] = ent
1486
1487     def lookup_entity(self, name, typ=None):
1488         ent = self._entity_dict.get(name)
1489         if typ and not isinstance(ent, typ):
1490             return None
1491         return ent
1492
1493     def lookup_type(self, name):
1494         return self.lookup_entity(name, QAPISchemaType)
1495
1496     def _def_builtin_type(self, name, json_type, c_type):
1497         self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
1498         # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
1499         # qapi-types.h from a single .c, all arrays of builtins must be
1500         # declared in the first file whether or not they are used.  Nicer
1501         # would be to use lazy instantiation, while figuring out how to
1502         # avoid compilation issues with multiple qapi-types.h.
1503         self._make_array_type(name, None)
1504
1505     def _def_predefineds(self):
1506         for t in [('str',    'string',  'char' + pointer_suffix),
1507                   ('number', 'number',  'double'),
1508                   ('int',    'int',     'int64_t'),
1509                   ('int8',   'int',     'int8_t'),
1510                   ('int16',  'int',     'int16_t'),
1511                   ('int32',  'int',     'int32_t'),
1512                   ('int64',  'int',     'int64_t'),
1513                   ('uint8',  'int',     'uint8_t'),
1514                   ('uint16', 'int',     'uint16_t'),
1515                   ('uint32', 'int',     'uint32_t'),
1516                   ('uint64', 'int',     'uint64_t'),
1517                   ('size',   'int',     'uint64_t'),
1518                   ('bool',   'boolean', 'bool'),
1519                   ('any',    'value',   'QObject' + pointer_suffix),
1520                   ('null',   'null',    'QNull' + pointer_suffix)]:
1521             self._def_builtin_type(*t)
1522         self.the_empty_object_type = QAPISchemaObjectType(
1523             'q_empty', None, None, None, [], None)
1524         self._def_entity(self.the_empty_object_type)
1525         qtype_values = self._make_enum_members(['none', 'qnull', 'qnum',
1526                                                 'qstring', 'qdict', 'qlist',
1527                                                 'qbool'])
1528         self._def_entity(QAPISchemaEnumType('QType', None, None,
1529                                             qtype_values, 'QTYPE'))
1530
1531     def _make_enum_members(self, values):
1532         return [QAPISchemaMember(v) for v in values]
1533
1534     def _make_implicit_enum_type(self, name, info, values):
1535         # See also QAPISchemaObjectTypeMember._pretty_owner()
1536         name = name + 'Kind'   # Use namespace reserved by add_name()
1537         self._def_entity(QAPISchemaEnumType(
1538             name, info, None, self._make_enum_members(values), None))
1539         return name
1540
1541     def _make_array_type(self, element_type, info):
1542         name = element_type + 'List'   # Use namespace reserved by add_name()
1543         if not self.lookup_type(name):
1544             self._def_entity(QAPISchemaArrayType(name, info, element_type))
1545         return name
1546
1547     def _make_implicit_object_type(self, name, info, doc, role, members):
1548         if not members:
1549             return None
1550         # See also QAPISchemaObjectTypeMember._pretty_owner()
1551         name = 'q_obj_%s-%s' % (name, role)
1552         if not self.lookup_entity(name, QAPISchemaObjectType):
1553             self._def_entity(QAPISchemaObjectType(name, info, doc, None,
1554                                                   members, None))
1555         return name
1556
1557     def _def_enum_type(self, expr, info, doc):
1558         name = expr['enum']
1559         data = expr['data']
1560         prefix = expr.get('prefix')
1561         self._def_entity(QAPISchemaEnumType(
1562             name, info, doc, self._make_enum_members(data), prefix))
1563
1564     def _make_member(self, name, typ, info):
1565         optional = False
1566         if name.startswith('*'):
1567             name = name[1:]
1568             optional = True
1569         if isinstance(typ, list):
1570             assert len(typ) == 1
1571             typ = self._make_array_type(typ[0], info)
1572         return QAPISchemaObjectTypeMember(name, typ, optional)
1573
1574     def _make_members(self, data, info):
1575         return [self._make_member(key, value, info)
1576                 for (key, value) in data.items()]
1577
1578     def _def_struct_type(self, expr, info, doc):
1579         name = expr['struct']
1580         base = expr.get('base')
1581         data = expr['data']
1582         self._def_entity(QAPISchemaObjectType(name, info, doc, base,
1583                                               self._make_members(data, info),
1584                                               None))
1585
1586     def _make_variant(self, case, typ):
1587         return QAPISchemaObjectTypeVariant(case, typ)
1588
1589     def _make_simple_variant(self, case, typ, info):
1590         if isinstance(typ, list):
1591             assert len(typ) == 1
1592             typ = self._make_array_type(typ[0], info)
1593         typ = self._make_implicit_object_type(
1594             typ, info, None, 'wrapper', [self._make_member('data', typ, info)])
1595         return QAPISchemaObjectTypeVariant(case, typ)
1596
1597     def _def_union_type(self, expr, info, doc):
1598         name = expr['union']
1599         data = expr['data']
1600         base = expr.get('base')
1601         tag_name = expr.get('discriminator')
1602         tag_member = None
1603         if isinstance(base, dict):
1604             base = (self._make_implicit_object_type(
1605                 name, info, doc, 'base', self._make_members(base, info)))
1606         if tag_name:
1607             variants = [self._make_variant(key, value)
1608                         for (key, value) in data.items()]
1609             members = []
1610         else:
1611             variants = [self._make_simple_variant(key, value, info)
1612                         for (key, value) in data.items()]
1613             typ = self._make_implicit_enum_type(name, info,
1614                                                 [v.name for v in variants])
1615             tag_member = QAPISchemaObjectTypeMember('type', typ, False)
1616             members = [tag_member]
1617         self._def_entity(
1618             QAPISchemaObjectType(name, info, doc, base, members,
1619                                  QAPISchemaObjectTypeVariants(tag_name,
1620                                                               tag_member,
1621                                                               variants)))
1622
1623     def _def_alternate_type(self, expr, info, doc):
1624         name = expr['alternate']
1625         data = expr['data']
1626         variants = [self._make_variant(key, value)
1627                     for (key, value) in data.items()]
1628         tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
1629         self._def_entity(
1630             QAPISchemaAlternateType(name, info, doc,
1631                                     QAPISchemaObjectTypeVariants(None,
1632                                                                  tag_member,
1633                                                                  variants)))
1634
1635     def _def_command(self, expr, info, doc):
1636         name = expr['command']
1637         data = expr.get('data')
1638         rets = expr.get('returns')
1639         gen = expr.get('gen', True)
1640         success_response = expr.get('success-response', True)
1641         boxed = expr.get('boxed', False)
1642         if isinstance(data, OrderedDict):
1643             data = self._make_implicit_object_type(
1644                 name, info, doc, 'arg', self._make_members(data, info))
1645         if isinstance(rets, list):
1646             assert len(rets) == 1
1647             rets = self._make_array_type(rets[0], info)
1648         self._def_entity(QAPISchemaCommand(name, info, doc, data, rets,
1649                                            gen, success_response, boxed))
1650
1651     def _def_event(self, expr, info, doc):
1652         name = expr['event']
1653         data = expr.get('data')
1654         boxed = expr.get('boxed', False)
1655         if isinstance(data, OrderedDict):
1656             data = self._make_implicit_object_type(
1657                 name, info, doc, 'arg', self._make_members(data, info))
1658         self._def_entity(QAPISchemaEvent(name, info, doc, data, boxed))
1659
1660     def _def_exprs(self):
1661         for expr_elem in self.exprs:
1662             expr = expr_elem['expr']
1663             info = expr_elem['info']
1664             doc = expr_elem.get('doc')
1665             if 'enum' in expr:
1666                 self._def_enum_type(expr, info, doc)
1667             elif 'struct' in expr:
1668                 self._def_struct_type(expr, info, doc)
1669             elif 'union' in expr:
1670                 self._def_union_type(expr, info, doc)
1671             elif 'alternate' in expr:
1672                 self._def_alternate_type(expr, info, doc)
1673             elif 'command' in expr:
1674                 self._def_command(expr, info, doc)
1675             elif 'event' in expr:
1676                 self._def_event(expr, info, doc)
1677             else:
1678                 assert False
1679
1680     def check(self):
1681         for ent in self._entity_dict.values():
1682             ent.check(self)
1683
1684     def visit(self, visitor):
1685         visitor.visit_begin(self)
1686         for (name, entity) in sorted(self._entity_dict.items()):
1687             if visitor.visit_needed(entity):
1688                 entity.visit(visitor)
1689         visitor.visit_end()
1690
1691
1692 #
1693 # Code generation helpers
1694 #
1695
1696 def camel_case(name):
1697     new_name = ''
1698     first = True
1699     for ch in name:
1700         if ch in ['_', '-']:
1701             first = True
1702         elif first:
1703             new_name += ch.upper()
1704             first = False
1705         else:
1706             new_name += ch.lower()
1707     return new_name
1708
1709
1710 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1711 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1712 # ENUM24_Name -> ENUM24_NAME
1713 def camel_to_upper(value):
1714     c_fun_str = c_name(value, False)
1715     if value.isupper():
1716         return c_fun_str
1717
1718     new_name = ''
1719     l = len(c_fun_str)
1720     for i in range(l):
1721         c = c_fun_str[i]
1722         # When c is upper and no '_' appears before, do more checks
1723         if c.isupper() and (i > 0) and c_fun_str[i - 1] != '_':
1724             if i < l - 1 and c_fun_str[i + 1].islower():
1725                 new_name += '_'
1726             elif c_fun_str[i - 1].isdigit():
1727                 new_name += '_'
1728         new_name += c
1729     return new_name.lstrip('_').upper()
1730
1731
1732 def c_enum_const(type_name, const_name, prefix=None):
1733     if prefix is not None:
1734         type_name = prefix
1735     return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
1736
1737 c_name_trans = string.maketrans('.-', '__')
1738
1739
1740 # Map @name to a valid C identifier.
1741 # If @protect, avoid returning certain ticklish identifiers (like
1742 # C keywords) by prepending 'q_'.
1743 #
1744 # Used for converting 'name' from a 'name':'type' qapi definition
1745 # into a generated struct member, as well as converting type names
1746 # into substrings of a generated C function name.
1747 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1748 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1749 def c_name(name, protect=True):
1750     # ANSI X3J11/88-090, 3.1.1
1751     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1752                      'default', 'do', 'double', 'else', 'enum', 'extern',
1753                      'float', 'for', 'goto', 'if', 'int', 'long', 'register',
1754                      'return', 'short', 'signed', 'sizeof', 'static',
1755                      'struct', 'switch', 'typedef', 'union', 'unsigned',
1756                      'void', 'volatile', 'while'])
1757     # ISO/IEC 9899:1999, 6.4.1
1758     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1759     # ISO/IEC 9899:2011, 6.4.1
1760     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
1761                      '_Noreturn', '_Static_assert', '_Thread_local'])
1762     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1763     # excluding _.*
1764     gcc_words = set(['asm', 'typeof'])
1765     # C++ ISO/IEC 14882:2003 2.11
1766     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1767                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1768                      'namespace', 'new', 'operator', 'private', 'protected',
1769                      'public', 'reinterpret_cast', 'static_cast', 'template',
1770                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1771                      'using', 'virtual', 'wchar_t',
1772                      # alternative representations
1773                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1774                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1775     # namespace pollution:
1776     polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
1777     name = name.translate(c_name_trans)
1778     if protect and (name in c89_words | c99_words | c11_words | gcc_words
1779                     | cpp_words | polluted_words):
1780         return 'q_' + name
1781     return name
1782
1783 eatspace = '\033EATSPACE.'
1784 pointer_suffix = ' *' + eatspace
1785
1786
1787 def genindent(count):
1788     ret = ''
1789     for _ in range(count):
1790         ret += ' '
1791     return ret
1792
1793 indent_level = 0
1794
1795
1796 def push_indent(indent_amount=4):
1797     global indent_level
1798     indent_level += indent_amount
1799
1800
1801 def pop_indent(indent_amount=4):
1802     global indent_level
1803     indent_level -= indent_amount
1804
1805
1806 # Generate @code with @kwds interpolated.
1807 # Obey indent_level, and strip eatspace.
1808 def cgen(code, **kwds):
1809     raw = code % kwds
1810     if indent_level:
1811         indent = genindent(indent_level)
1812         # re.subn() lacks flags support before Python 2.7, use re.compile()
1813         raw = re.subn(re.compile(r'^.', re.MULTILINE),
1814                       indent + r'\g<0>', raw)
1815         raw = raw[0]
1816     return re.sub(re.escape(eatspace) + r' *', '', raw)
1817
1818
1819 def mcgen(code, **kwds):
1820     if code[0] == '\n':
1821         code = code[1:]
1822     return cgen(code, **kwds)
1823
1824
1825 def guardname(filename):
1826     return c_name(filename, protect=False).upper()
1827
1828
1829 def guardstart(name):
1830     return mcgen('''
1831
1832 #ifndef %(name)s
1833 #define %(name)s
1834
1835 ''',
1836                  name=guardname(name))
1837
1838
1839 def guardend(name):
1840     return mcgen('''
1841
1842 #endif /* %(name)s */
1843
1844 ''',
1845                  name=guardname(name))
1846
1847
1848 def gen_enum_lookup(name, values, prefix=None):
1849     ret = mcgen('''
1850
1851 const QEnumLookup %(c_name)s_lookup = {
1852     .array = (const char *const[]) {
1853 ''',
1854                 c_name=c_name(name))
1855     for value in values:
1856         index = c_enum_const(name, value, prefix)
1857         ret += mcgen('''
1858         [%(index)s] = "%(value)s",
1859 ''',
1860                      index=index, value=value)
1861
1862     ret += mcgen('''
1863     },
1864     .size = %(max_index)s
1865 };
1866 ''',
1867                  max_index=c_enum_const(name, '_MAX', prefix))
1868     return ret
1869
1870
1871 def gen_enum(name, values, prefix=None):
1872     # append automatically generated _MAX value
1873     enum_values = values + ['_MAX']
1874
1875     ret = mcgen('''
1876
1877 typedef enum %(c_name)s {
1878 ''',
1879                 c_name=c_name(name))
1880
1881     i = 0
1882     for value in enum_values:
1883         ret += mcgen('''
1884     %(c_enum)s = %(i)d,
1885 ''',
1886                      c_enum=c_enum_const(name, value, prefix),
1887                      i=i)
1888         i += 1
1889
1890     ret += mcgen('''
1891 } %(c_name)s;
1892 ''',
1893                  c_name=c_name(name))
1894
1895     ret += mcgen('''
1896
1897 #define %(c_name)s_str(val) \\
1898     qapi_enum_lookup(&%(c_name)s_lookup, (val))
1899
1900 extern const QEnumLookup %(c_name)s_lookup;
1901 ''',
1902                  c_name=c_name(name))
1903     return ret
1904
1905
1906 def build_params(arg_type, boxed, extra):
1907     if not arg_type:
1908         assert not boxed
1909         return extra
1910     ret = ''
1911     sep = ''
1912     if boxed:
1913         ret += '%s arg' % arg_type.c_param_type()
1914         sep = ', '
1915     else:
1916         assert not arg_type.variants
1917         for memb in arg_type.members:
1918             ret += sep
1919             sep = ', '
1920             if memb.optional:
1921                 ret += 'bool has_%s, ' % c_name(memb.name)
1922             ret += '%s %s' % (memb.type.c_param_type(),
1923                               c_name(memb.name))
1924     if extra:
1925         ret += sep + extra
1926     return ret
1927
1928
1929 #
1930 # Common command line parsing
1931 #
1932
1933
1934 def parse_command_line(extra_options='', extra_long_options=[]):
1935
1936     try:
1937         opts, args = getopt.gnu_getopt(sys.argv[1:],
1938                                        'chp:o:' + extra_options,
1939                                        ['source', 'header', 'prefix=',
1940                                         'output-dir='] + extra_long_options)
1941     except getopt.GetoptError as err:
1942         print("%s: %s" % (sys.argv[0], str(err)), file=sys.stderr)
1943         sys.exit(1)
1944
1945     output_dir = ''
1946     prefix = ''
1947     do_c = False
1948     do_h = False
1949     extra_opts = []
1950
1951     for oa in opts:
1952         o, a = oa
1953         if o in ('-p', '--prefix'):
1954             match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1955             if match.end() != len(a):
1956                 print("%s: 'funny character '%s' in argument of --prefix" \
1957                       % (sys.argv[0], a[match.end()]), file=sys.stderr)
1958                 sys.exit(1)
1959             prefix = a
1960         elif o in ('-o', '--output-dir'):
1961             output_dir = a + '/'
1962         elif o in ('-c', '--source'):
1963             do_c = True
1964         elif o in ('-h', '--header'):
1965             do_h = True
1966         else:
1967             extra_opts.append(oa)
1968
1969     if not do_c and not do_h:
1970         do_c = True
1971         do_h = True
1972
1973     if len(args) != 1:
1974         print("%s: need exactly one argument" % sys.argv[0], file=sys.stderr)
1975         sys.exit(1)
1976     fname = args[0]
1977
1978     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
1979
1980 #
1981 # Generate output files with boilerplate
1982 #
1983
1984
1985 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
1986                 c_comment, h_comment):
1987     guard = guardname(prefix + h_file)
1988     c_file = output_dir + prefix + c_file
1989     h_file = output_dir + prefix + h_file
1990
1991     if output_dir:
1992         try:
1993             os.makedirs(output_dir)
1994         except os.error as e:
1995             if e.errno != errno.EEXIST:
1996                 raise
1997
1998     def maybe_open(really, name, opt):
1999         if really:
2000             return open(name, opt)
2001         else:
2002             return StringIO()
2003
2004     fdef = maybe_open(do_c, c_file, 'w')
2005     fdecl = maybe_open(do_h, h_file, 'w')
2006
2007     fdef.write(mcgen('''
2008 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2009 %(comment)s
2010 ''',
2011                      comment=c_comment))
2012
2013     fdecl.write(mcgen('''
2014 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2015 %(comment)s
2016 #ifndef %(guard)s
2017 #define %(guard)s
2018
2019 ''',
2020                       comment=h_comment, guard=guard))
2021
2022     return (fdef, fdecl)
2023
2024
2025 def close_output(fdef, fdecl):
2026     fdecl.write('''
2027 #endif
2028 ''')
2029     fdecl.close()
2030     fdef.close()