scripts/qapi/parser.py

   1 # -*- coding: utf-8 -*-
   2 #
   3 # QAPI schema parser
   4 #
   5 # Copyright IBM, Corp. 2011
   6 # Copyright (c) 2013-2019 Red Hat Inc.
   7 #
   8 # Authors:
   9 #  Anthony Liguori <aliguori@us.ibm.com>
  10 #  Markus Armbruster <armbru@redhat.com>
  11 #  Marc-André Lureau <marcandre.lureau@redhat.com>
  12 #  Kevin Wolf <kwolf@redhat.com>
  13 #
  14 # This work is licensed under the terms of the GNU GPL, version 2.
  15 # See the COPYING file in the top-level directory.
  16
  17 import os
  18 import re
  19 import sys
  20 from collections import OrderedDict
  21
  22 from qapi.error import QAPIParseError, QAPISemError
  23 from qapi.source import QAPISourceInfo
  24
  25
  26 class QAPISchemaParser(object):
  27
  28     def __init__(self, fname, previously_included=None, incl_info=None):
  29         previously_included = previously_included or set()
  30         previously_included.add(os.path.abspath(fname))
  31
  32         try:
  33             if sys.version_info[0] >= 3:
  34                 fp = open(fname, 'r', encoding='utf-8')
  35             else:
  36                 fp = open(fname, 'r')
  37             self.src = fp.read()
  38         except IOError as e:
  39             raise QAPISemError(incl_info or QAPISourceInfo(None, None, None),
  40                                "can't read %s file '%s': %s"
  41                                % ("include" if incl_info else "schema",
  42                                   fname,
  43                                   e.strerror))
  44
  45         if self.src == '' or self.src[-1] != '\n':
  46             self.src += '\n'
  47         self.cursor = 0
  48         self.info = QAPISourceInfo(fname, 1, incl_info)
  49         self.line_pos = 0
  50         self.exprs = []
  51         self.docs = []
  52         self.accept()
  53         cur_doc = None
  54
  55         while self.tok is not None:
  56             info = self.info
  57             if self.tok == '#':
  58                 self.reject_expr_doc(cur_doc)
  59                 cur_doc = self.get_doc(info)
  60                 self.docs.append(cur_doc)
  61                 continue
  62
  63             expr = self.get_expr(False)
  64             if 'include' in expr:
  65                 self.reject_expr_doc(cur_doc)
  66                 if len(expr) != 1:
  67                     raise QAPISemError(info, "invalid 'include' directive")
  68                 include = expr['include']
  69                 if not isinstance(include, str):
  70                     raise QAPISemError(info,
  71                                        "value of 'include' must be a string")
  72                 incl_fname = os.path.join(os.path.dirname(fname),
  73                                           include)
  74                 self.exprs.append({'expr': {'include': incl_fname},
  75                                    'info': info})
  76                 exprs_include = self._include(include, info, incl_fname,
  77                                               previously_included)
  78                 if exprs_include:
  79                     self.exprs.extend(exprs_include.exprs)
  80                     self.docs.extend(exprs_include.docs)
  81             elif "pragma" in expr:
  82                 self.reject_expr_doc(cur_doc)
  83                 if len(expr) != 1:
  84                     raise QAPISemError(info, "invalid 'pragma' directive")
  85                 pragma = expr['pragma']
  86                 if not isinstance(pragma, dict):
  87                     raise QAPISemError(
  88                         info, "value of 'pragma' must be an object")
  89                 for name, value in pragma.items():
  90                     self._pragma(name, value, info)
  91             else:
  92                 expr_elem = {'expr': expr,
  93                              'info': info}
  94                 if cur_doc:
  95                     if not cur_doc.symbol:
  96                         raise QAPISemError(
  97                             cur_doc.info, "definition documentation required")
  98                     expr_elem['doc'] = cur_doc
  99                 self.exprs.append(expr_elem)
 100             cur_doc = None
 101         self.reject_expr_doc(cur_doc)
 102
 103     @staticmethod
 104     def reject_expr_doc(doc):
 105         if doc and doc.symbol:
 106             raise QAPISemError(
 107                 doc.info,
 108                 "documentation for '%s' is not followed by the definition"
 109                 % doc.symbol)
 110
 111     def _include(self, include, info, incl_fname, previously_included):
 112         incl_abs_fname = os.path.abspath(incl_fname)
 113         # catch inclusion cycle
 114         inf = info
 115         while inf:
 116             if incl_abs_fname == os.path.abspath(inf.fname):
 117                 raise QAPISemError(info, "inclusion loop for %s" % include)
 118             inf = inf.parent
 119
 120         # skip multiple include of the same file
 121         if incl_abs_fname in previously_included:
 122             return None
 123
 124         return QAPISchemaParser(incl_fname, previously_included, info)
 125
 126     def _pragma(self, name, value, info):
 127         if name == 'doc-required':
 128             if not isinstance(value, bool):
 129                 raise QAPISemError(info,
 130                                    "pragma 'doc-required' must be boolean")
 131             info.pragma.doc_required = value
 132         elif name == 'returns-whitelist':
 133             if (not isinstance(value, list)
 134                     or any([not isinstance(elt, str) for elt in value])):
 135                 raise QAPISemError(
 136                     info,
 137                     "pragma returns-whitelist must be a list of strings")
 138             info.pragma.returns_whitelist = value
 139         elif name == 'name-case-whitelist':
 140             if (not isinstance(value, list)
 141                     or any([not isinstance(elt, str) for elt in value])):
 142                 raise QAPISemError(
 143                     info,
 144                     "pragma name-case-whitelist must be a list of strings")
 145             info.pragma.name_case_whitelist = value
 146         else:
 147             raise QAPISemError(info, "unknown pragma '%s'" % name)
 148
 149     def accept(self, skip_comment=True):
 150         while True:
 151             self.tok = self.src[self.cursor]
 152             self.pos = self.cursor
 153             self.cursor += 1
 154             self.val = None
 155
 156             if self.tok == '#':
 157                 if self.src[self.cursor] == '#':
 158                     # Start of doc comment
 159                     skip_comment = False
 160                 self.cursor = self.src.find('\n', self.cursor)
 161                 if not skip_comment:
 162                     self.val = self.src[self.pos:self.cursor]
 163                     return
 164             elif self.tok in '{}:,[]':
 165                 return
 166             elif self.tok == "'":
 167                 # Note: we accept only printable ASCII
 168                 string = ''
 169                 esc = False
 170                 while True:
 171                     ch = self.src[self.cursor]
 172                     self.cursor += 1
 173                     if ch == '\n':
 174                         raise QAPIParseError(self, "missing terminating \"'\"")
 175                     if esc:
 176                         # Note: we recognize only \\ because we have
 177                         # no use for funny characters in strings
 178                         if ch != '\\':
 179                             raise QAPIParseError(self,
 180                                                  "unknown escape \\%s" % ch)
 181                         esc = False
 182                     elif ch == '\\':
 183                         esc = True
 184                         continue
 185                     elif ch == "'":
 186                         self.val = string
 187                         return
 188                     if ord(ch) < 32 or ord(ch) >= 127:
 189                         raise QAPIParseError(
 190                             self, "funny character in string")
 191                     string += ch
 192             elif self.src.startswith('true', self.pos):
 193                 self.val = True
 194                 self.cursor += 3
 195                 return
 196             elif self.src.startswith('false', self.pos):
 197                 self.val = False
 198                 self.cursor += 4
 199                 return
 200             elif self.tok == '\n':
 201                 if self.cursor == len(self.src):
 202                     self.tok = None
 203                     return
 204                 self.info = self.info.next_line()
 205                 self.line_pos = self.cursor
 206             elif not self.tok.isspace():
 207                 # Show up to next structural, whitespace or quote
 208                 # character
 209                 match = re.match('[^[\\]{}:,\\s\'"]+',
 210                                  self.src[self.cursor-1:])
 211                 raise QAPIParseError(self, "stray '%s'" % match.group(0))
 212
 213     def get_members(self):
 214         expr = OrderedDict()
 215         if self.tok == '}':
 216             self.accept()
 217             return expr
 218         if self.tok != "'":
 219             raise QAPIParseError(self, "expected string or '}'")
 220         while True:
 221             key = self.val
 222             self.accept()
 223             if self.tok != ':':
 224                 raise QAPIParseError(self, "expected ':'")
 225             self.accept()
 226             if key in expr:
 227                 raise QAPIParseError(self, "duplicate key '%s'" % key)
 228             expr[key] = self.get_expr(True)
 229             if self.tok == '}':
 230                 self.accept()
 231                 return expr
 232             if self.tok != ',':
 233                 raise QAPIParseError(self, "expected ',' or '}'")
 234             self.accept()
 235             if self.tok != "'":
 236                 raise QAPIParseError(self, "expected string")
 237
 238     def get_values(self):
 239         expr = []
 240         if self.tok == ']':
 241             self.accept()
 242             return expr
 243         if self.tok not in "{['tfn":
 244             raise QAPIParseError(
 245                 self, "expected '{', '[', ']', string, boolean or 'null'")
 246         while True:
 247             expr.append(self.get_expr(True))
 248             if self.tok == ']':
 249                 self.accept()
 250                 return expr
 251             if self.tok != ',':
 252                 raise QAPIParseError(self, "expected ',' or ']'")
 253             self.accept()
 254
 255     def get_expr(self, nested):
 256         if self.tok != '{' and not nested:
 257             raise QAPIParseError(self, "expected '{'")
 258         if self.tok == '{':
 259             self.accept()
 260             expr = self.get_members()
 261         elif self.tok == '[':
 262             self.accept()
 263             expr = self.get_values()
 264         elif self.tok in "'tfn":
 265             expr = self.val
 266             self.accept()
 267         else:
 268             raise QAPIParseError(
 269                 self, "expected '{', '[', string, boolean or 'null'")
 270         return expr
 271
 272     def get_doc(self, info):
 273         if self.val != '##':
 274             raise QAPIParseError(
 275                 self, "junk after '##' at start of documentation comment")
 276
 277         doc = QAPIDoc(self, info)
 278         self.accept(False)
 279         while self.tok == '#':
 280             if self.val.startswith('##'):
 281                 # End of doc comment
 282                 if self.val != '##':
 283                     raise QAPIParseError(
 284                         self,
 285                         "junk after '##' at end of documentation comment")
 286                 doc.end_comment()
 287                 self.accept()
 288                 return doc
 289             else:
 290                 doc.append(self.val)
 291             self.accept(False)
 292
 293         raise QAPIParseError(self, "documentation comment must end with '##'")
 294
 295
 296 class QAPIDoc(object):
 297     """
 298     A documentation comment block, either definition or free-form
 299
 300     Definition documentation blocks consist of
 301
 302     * a body section: one line naming the definition, followed by an
 303       overview (any number of lines)
 304
 305     * argument sections: a description of each argument (for commands
 306       and events) or member (for structs, unions and alternates)
 307
 308     * features sections: a description of each feature flag
 309
 310     * additional (non-argument) sections, possibly tagged
 311
 312     Free-form documentation blocks consist only of a body section.
 313     """
 314
 315     class Section(object):
 316         def __init__(self, name=None):
 317             # optional section name (argument/member or section name)
 318             self.name = name
 319             # the list of lines for this section
 320             self.text = ''
 321
 322         def append(self, line):
 323             self.text += line.rstrip() + '\n'
 324
 325     class ArgSection(Section):
 326         def __init__(self, name):
 327             QAPIDoc.Section.__init__(self, name)
 328             self.member = None
 329
 330         def connect(self, member):
 331             self.member = member
 332
 333     def __init__(self, parser, info):
 334         # self._parser is used to report errors with QAPIParseError.  The
 335         # resulting error position depends on the state of the parser.
 336         # It happens to be the beginning of the comment.  More or less
 337         # servicable, but action at a distance.
 338         self._parser = parser
 339         self.info = info
 340         self.symbol = None
 341         self.body = QAPIDoc.Section()
 342         # dict mapping parameter name to ArgSection
 343         self.args = OrderedDict()
 344         self.features = OrderedDict()
 345         # a list of Section
 346         self.sections = []
 347         # the current section
 348         self._section = self.body
 349         self._append_line = self._append_body_line
 350
 351     def has_section(self, name):
 352         """Return True if we have a section with this name."""
 353         for i in self.sections:
 354             if i.name == name:
 355                 return True
 356         return False
 357
 358     def append(self, line):
 359         """
 360         Parse a comment line and add it to the documentation.
 361
 362         The way that the line is dealt with depends on which part of
 363         the documentation we're parsing right now:
 364         * The body section: ._append_line is ._append_body_line
 365         * An argument section: ._append_line is ._append_args_line
 366         * A features section: ._append_line is ._append_features_line
 367         * An additional section: ._append_line is ._append_various_line
 368         """
 369         line = line[1:]
 370         if not line:
 371             self._append_freeform(line)
 372             return
 373
 374         if line[0] != ' ':
 375             raise QAPIParseError(self._parser, "missing space after #")
 376         line = line[1:]
 377         self._append_line(line)
 378
 379     def end_comment(self):
 380         self._end_section()
 381
 382     @staticmethod
 383     def _is_section_tag(name):
 384         return name in ('Returns:', 'Since:',
 385                         # those are often singular or plural
 386                         'Note:', 'Notes:',
 387                         'Example:', 'Examples:',
 388                         'TODO:')
 389
 390     def _append_body_line(self, line):
 391         """
 392         Process a line of documentation text in the body section.
 393
 394         If this a symbol line and it is the section's first line, this
 395         is a definition documentation block for that symbol.
 396
 397         If it's a definition documentation block, another symbol line
 398         begins the argument section for the argument named by it, and
 399         a section tag begins an additional section.  Start that
 400         section and append the line to it.
 401
 402         Else, append the line to the current section.
 403         """
 404         name = line.split(' ', 1)[0]
 405         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 406         # recognized, and get silently treated as ordinary text
 407         if not self.symbol and not self.body.text and line.startswith('@'):
 408             if not line.endswith(':'):
 409                 raise QAPIParseError(self._parser, "line should end with ':'")
 410             self.symbol = line[1:-1]
 411             # FIXME invalid names other than the empty string aren't flagged
 412             if not self.symbol:
 413                 raise QAPIParseError(self._parser, "invalid name")
 414         elif self.symbol:
 415             # This is a definition documentation block
 416             if name.startswith('@') and name.endswith(':'):
 417                 self._append_line = self._append_args_line
 418                 self._append_args_line(line)
 419             elif line == 'Features:':
 420                 self._append_line = self._append_features_line
 421             elif self._is_section_tag(name):
 422                 self._append_line = self._append_various_line
 423                 self._append_various_line(line)
 424             else:
 425                 self._append_freeform(line.strip())
 426         else:
 427             # This is a free-form documentation block
 428             self._append_freeform(line.strip())
 429
 430     def _append_args_line(self, line):
 431         """
 432         Process a line of documentation text in an argument section.
 433
 434         A symbol line begins the next argument section, a section tag
 435         section or a non-indented line after a blank line begins an
 436         additional section.  Start that section and append the line to
 437         it.
 438
 439         Else, append the line to the current section.
 440
 441         """
 442         name = line.split(' ', 1)[0]
 443
 444         if name.startswith('@') and name.endswith(':'):
 445             line = line[len(name)+1:]
 446             self._start_args_section(name[1:-1])
 447         elif self._is_section_tag(name):
 448             self._append_line = self._append_various_line
 449             self._append_various_line(line)
 450             return
 451         elif (self._section.text.endswith('\n\n')
 452               and line and not line[0].isspace()):
 453             if line == 'Features:':
 454                 self._append_line = self._append_features_line
 455             else:
 456                 self._start_section()
 457                 self._append_line = self._append_various_line
 458                 self._append_various_line(line)
 459             return
 460
 461         self._append_freeform(line.strip())
 462
 463     def _append_features_line(self, line):
 464         name = line.split(' ', 1)[0]
 465
 466         if name.startswith('@') and name.endswith(':'):
 467             line = line[len(name)+1:]
 468             self._start_features_section(name[1:-1])
 469         elif self._is_section_tag(name):
 470             self._append_line = self._append_various_line
 471             self._append_various_line(line)
 472             return
 473         elif (self._section.text.endswith('\n\n')
 474               and line and not line[0].isspace()):
 475             self._start_section()
 476             self._append_line = self._append_various_line
 477             self._append_various_line(line)
 478             return
 479
 480         self._append_freeform(line.strip())
 481
 482     def _append_various_line(self, line):
 483         """
 484         Process a line of documentation text in an additional section.
 485
 486         A symbol line is an error.
 487
 488         A section tag begins an additional section.  Start that
 489         section and append the line to it.
 490
 491         Else, append the line to the current section.
 492         """
 493         name = line.split(' ', 1)[0]
 494
 495         if name.startswith('@') and name.endswith(':'):
 496             raise QAPIParseError(self._parser,
 497                                  "'%s' can't follow '%s' section"
 498                                  % (name, self.sections[0].name))
 499         elif self._is_section_tag(name):
 500             line = line[len(name)+1:]
 501             self._start_section(name[:-1])
 502
 503         if (not self._section.name or
 504                 not self._section.name.startswith('Example')):
 505             line = line.strip()
 506
 507         self._append_freeform(line)
 508
 509     def _start_symbol_section(self, symbols_dict, name):
 510         # FIXME invalid names other than the empty string aren't flagged
 511         if not name:
 512             raise QAPIParseError(self._parser, "invalid parameter name")
 513         if name in symbols_dict:
 514             raise QAPIParseError(self._parser,
 515                                  "'%s' parameter name duplicated" % name)
 516         assert not self.sections
 517         self._end_section()
 518         self._section = QAPIDoc.ArgSection(name)
 519         symbols_dict[name] = self._section
 520
 521     def _start_args_section(self, name):
 522         self._start_symbol_section(self.args, name)
 523
 524     def _start_features_section(self, name):
 525         self._start_symbol_section(self.features, name)
 526
 527     def _start_section(self, name=None):
 528         if name in ('Returns', 'Since') and self.has_section(name):
 529             raise QAPIParseError(self._parser,
 530                                  "duplicated '%s' section" % name)
 531         self._end_section()
 532         self._section = QAPIDoc.Section(name)
 533         self.sections.append(self._section)
 534
 535     def _end_section(self):
 536         if self._section:
 537             text = self._section.text = self._section.text.strip()
 538             if self._section.name and (not text or text.isspace()):
 539                 raise QAPIParseError(
 540                     self._parser,
 541                     "empty doc section '%s'" % self._section.name)
 542             self._section = None
 543
 544     def _append_freeform(self, line):
 545         match = re.match(r'(@\S+:)', line)
 546         if match:
 547             raise QAPIParseError(self._parser,
 548                                  "'%s' not allowed in free-form documentation"
 549                                  % match.group(1))
 550         self._section.append(line)
 551
 552     def connect_member(self, member):
 553         if member.name not in self.args:
 554             # Undocumented TODO outlaw
 555             self.args[member.name] = QAPIDoc.ArgSection(member.name)
 556         self.args[member.name].connect(member)
 557
 558     def connect_feature(self, feature):
 559         if feature.name not in self.features:
 560             raise QAPISemError(feature.info,
 561                                "feature '%s' lacks documentation"
 562                                % feature.name)
 563             self.features[feature.name] = QAPIDoc.ArgSection(feature.name)
 564         self.features[feature.name].connect(feature)
 565
 566     def check_expr(self, expr):
 567         if self.has_section('Returns') and 'command' not in expr:
 568             raise QAPISemError(self.info,
 569                                "'Returns:' is only valid for commands")
 570
 571     def check(self):
 572
 573         def check_args_section(args, info, what):
 574             bogus = [name for name, section in args.items()
 575                      if not section.member]
 576             if bogus:
 577                 raise QAPISemError(
 578                     self.info,
 579                     "documented member%s '%s' %s not exist"
 580                     % ("s" if len(bogus) > 1 else "",
 581                        "', '".join(bogus),
 582                        "do" if len(bogus) > 1 else "does"))
 583
 584         check_args_section(self.args, self.info, 'members')
 585         check_args_section(self.features, self.info, 'features')