scripts/qapi/parser.py

   1 # -*- coding: utf-8 -*-
   2 #
   3 # QAPI schema parser
   4 #
   5 # Copyright IBM, Corp. 2011
   6 # Copyright (c) 2013-2019 Red Hat Inc.
   7 #
   8 # Authors:
   9 #  Anthony Liguori <aliguori@us.ibm.com>
  10 #  Markus Armbruster <armbru@redhat.com>
  11 #  Marc-André Lureau <marcandre.lureau@redhat.com>
  12 #  Kevin Wolf <kwolf@redhat.com>
  13 #
  14 # This work is licensed under the terms of the GNU GPL, version 2.
  15 # See the COPYING file in the top-level directory.
  16
  17 from collections import OrderedDict
  18 import os
  19 import re
  20
  21 from .error import QAPIParseError, QAPISemError
  22 from .source import QAPISourceInfo
  23
  24
  25 class QAPISchemaParser:
  26
  27     def __init__(self, fname, previously_included=None, incl_info=None):
  28         previously_included = previously_included or set()
  29         previously_included.add(os.path.abspath(fname))
  30
  31         try:
  32             fp = open(fname, 'r', encoding='utf-8')
  33             self.src = fp.read()
  34         except IOError as e:
  35             raise QAPISemError(incl_info or QAPISourceInfo(None, None, None),
  36                                "can't read %s file '%s': %s"
  37                                % ("include" if incl_info else "schema",
  38                                   fname,
  39                                   e.strerror))
  40
  41         if self.src == '' or self.src[-1] != '\n':
  42             self.src += '\n'
  43         self.cursor = 0
  44         self.info = QAPISourceInfo(fname, 1, incl_info)
  45         self.line_pos = 0
  46         self.exprs = []
  47         self.docs = []
  48         self.accept()
  49         cur_doc = None
  50
  51         while self.tok is not None:
  52             info = self.info
  53             if self.tok == '#':
  54                 self.reject_expr_doc(cur_doc)
  55                 for cur_doc in self.get_doc(info):
  56                     self.docs.append(cur_doc)
  57                 continue
  58
  59             expr = self.get_expr(False)
  60             if 'include' in expr:
  61                 self.reject_expr_doc(cur_doc)
  62                 if len(expr) != 1:
  63                     raise QAPISemError(info, "invalid 'include' directive")
  64                 include = expr['include']
  65                 if not isinstance(include, str):
  66                     raise QAPISemError(info,
  67                                        "value of 'include' must be a string")
  68                 incl_fname = os.path.join(os.path.dirname(fname),
  69                                           include)
  70                 self.exprs.append({'expr': {'include': incl_fname},
  71                                    'info': info})
  72                 exprs_include = self._include(include, info, incl_fname,
  73                                               previously_included)
  74                 if exprs_include:
  75                     self.exprs.extend(exprs_include.exprs)
  76                     self.docs.extend(exprs_include.docs)
  77             elif "pragma" in expr:
  78                 self.reject_expr_doc(cur_doc)
  79                 if len(expr) != 1:
  80                     raise QAPISemError(info, "invalid 'pragma' directive")
  81                 pragma = expr['pragma']
  82                 if not isinstance(pragma, dict):
  83                     raise QAPISemError(
  84                         info, "value of 'pragma' must be an object")
  85                 for name, value in pragma.items():
  86                     self._pragma(name, value, info)
  87             else:
  88                 expr_elem = {'expr': expr,
  89                              'info': info}
  90                 if cur_doc:
  91                     if not cur_doc.symbol:
  92                         raise QAPISemError(
  93                             cur_doc.info, "definition documentation required")
  94                     expr_elem['doc'] = cur_doc
  95                 self.exprs.append(expr_elem)
  96             cur_doc = None
  97         self.reject_expr_doc(cur_doc)
  98
  99     @staticmethod
 100     def reject_expr_doc(doc):
 101         if doc and doc.symbol:
 102             raise QAPISemError(
 103                 doc.info,
 104                 "documentation for '%s' is not followed by the definition"
 105                 % doc.symbol)
 106
 107     def _include(self, include, info, incl_fname, previously_included):
 108         incl_abs_fname = os.path.abspath(incl_fname)
 109         # catch inclusion cycle
 110         inf = info
 111         while inf:
 112             if incl_abs_fname == os.path.abspath(inf.fname):
 113                 raise QAPISemError(info, "inclusion loop for %s" % include)
 114             inf = inf.parent
 115
 116         # skip multiple include of the same file
 117         if incl_abs_fname in previously_included:
 118             return None
 119
 120         return QAPISchemaParser(incl_fname, previously_included, info)
 121
 122     def _check_pragma_list_of_str(self, name, value, info):
 123         if (not isinstance(value, list)
 124                 or any([not isinstance(elt, str) for elt in value])):
 125             raise QAPISemError(
 126                 info,
 127                 "pragma %s must be a list of strings" % name)
 128
 129     def _pragma(self, name, value, info):
 130         if name == 'doc-required':
 131             if not isinstance(value, bool):
 132                 raise QAPISemError(info,
 133                                    "pragma 'doc-required' must be boolean")
 134             info.pragma.doc_required = value
 135         elif name == 'command-name-exceptions':
 136             self._check_pragma_list_of_str(name, value, info)
 137             info.pragma.command_name_exceptions = value
 138         elif name == 'command-returns-exceptions':
 139             self._check_pragma_list_of_str(name, value, info)
 140             info.pragma.command_returns_exceptions = value
 141         elif name == 'member-name-exceptions':
 142             self._check_pragma_list_of_str(name, value, info)
 143             info.pragma.member_name_exceptions = value
 144         else:
 145             raise QAPISemError(info, "unknown pragma '%s'" % name)
 146
 147     def accept(self, skip_comment=True):
 148         while True:
 149             self.tok = self.src[self.cursor]
 150             self.pos = self.cursor
 151             self.cursor += 1
 152             self.val = None
 153
 154             if self.tok == '#':
 155                 if self.src[self.cursor] == '#':
 156                     # Start of doc comment
 157                     skip_comment = False
 158                 self.cursor = self.src.find('\n', self.cursor)
 159                 if not skip_comment:
 160                     self.val = self.src[self.pos:self.cursor]
 161                     return
 162             elif self.tok in '{}:,[]':
 163                 return
 164             elif self.tok == "'":
 165                 # Note: we accept only printable ASCII
 166                 string = ''
 167                 esc = False
 168                 while True:
 169                     ch = self.src[self.cursor]
 170                     self.cursor += 1
 171                     if ch == '\n':
 172                         raise QAPIParseError(self, "missing terminating \"'\"")
 173                     if esc:
 174                         # Note: we recognize only \\ because we have
 175                         # no use for funny characters in strings
 176                         if ch != '\\':
 177                             raise QAPIParseError(self,
 178                                                  "unknown escape \\%s" % ch)
 179                         esc = False
 180                     elif ch == '\\':
 181                         esc = True
 182                         continue
 183                     elif ch == "'":
 184                         self.val = string
 185                         return
 186                     if ord(ch) < 32 or ord(ch) >= 127:
 187                         raise QAPIParseError(
 188                             self, "funny character in string")
 189                     string += ch
 190             elif self.src.startswith('true', self.pos):
 191                 self.val = True
 192                 self.cursor += 3
 193                 return
 194             elif self.src.startswith('false', self.pos):
 195                 self.val = False
 196                 self.cursor += 4
 197                 return
 198             elif self.tok == '\n':
 199                 if self.cursor == len(self.src):
 200                     self.tok = None
 201                     return
 202                 self.info = self.info.next_line()
 203                 self.line_pos = self.cursor
 204             elif not self.tok.isspace():
 205                 # Show up to next structural, whitespace or quote
 206                 # character
 207                 match = re.match('[^[\\]{}:,\\s\'"]+',
 208                                  self.src[self.cursor-1:])
 209                 raise QAPIParseError(self, "stray '%s'" % match.group(0))
 210
 211     def get_members(self):
 212         expr = OrderedDict()
 213         if self.tok == '}':
 214             self.accept()
 215             return expr
 216         if self.tok != "'":
 217             raise QAPIParseError(self, "expected string or '}'")
 218         while True:
 219             key = self.val
 220             self.accept()
 221             if self.tok != ':':
 222                 raise QAPIParseError(self, "expected ':'")
 223             self.accept()
 224             if key in expr:
 225                 raise QAPIParseError(self, "duplicate key '%s'" % key)
 226             expr[key] = self.get_expr(True)
 227             if self.tok == '}':
 228                 self.accept()
 229                 return expr
 230             if self.tok != ',':
 231                 raise QAPIParseError(self, "expected ',' or '}'")
 232             self.accept()
 233             if self.tok != "'":
 234                 raise QAPIParseError(self, "expected string")
 235
 236     def get_values(self):
 237         expr = []
 238         if self.tok == ']':
 239             self.accept()
 240             return expr
 241         if self.tok not in "{['tf":
 242             raise QAPIParseError(
 243                 self, "expected '{', '[', ']', string, or boolean")
 244         while True:
 245             expr.append(self.get_expr(True))
 246             if self.tok == ']':
 247                 self.accept()
 248                 return expr
 249             if self.tok != ',':
 250                 raise QAPIParseError(self, "expected ',' or ']'")
 251             self.accept()
 252
 253     def get_expr(self, nested):
 254         if self.tok != '{' and not nested:
 255             raise QAPIParseError(self, "expected '{'")
 256         if self.tok == '{':
 257             self.accept()
 258             expr = self.get_members()
 259         elif self.tok == '[':
 260             self.accept()
 261             expr = self.get_values()
 262         elif self.tok in "'tf":
 263             expr = self.val
 264             self.accept()
 265         else:
 266             raise QAPIParseError(
 267                 self, "expected '{', '[', string, or boolean")
 268         return expr
 269
 270     def get_doc(self, info):
 271         if self.val != '##':
 272             raise QAPIParseError(
 273                 self, "junk after '##' at start of documentation comment")
 274
 275         docs = []
 276         cur_doc = QAPIDoc(self, info)
 277         self.accept(False)
 278         while self.tok == '#':
 279             if self.val.startswith('##'):
 280                 # End of doc comment
 281                 if self.val != '##':
 282                     raise QAPIParseError(
 283                         self,
 284                         "junk after '##' at end of documentation comment")
 285                 cur_doc.end_comment()
 286                 docs.append(cur_doc)
 287                 self.accept()
 288                 return docs
 289             if self.val.startswith('# ='):
 290                 if cur_doc.symbol:
 291                     raise QAPIParseError(
 292                         self,
 293                         "unexpected '=' markup in definition documentation")
 294                 if cur_doc.body.text:
 295                     cur_doc.end_comment()
 296                     docs.append(cur_doc)
 297                     cur_doc = QAPIDoc(self, info)
 298             cur_doc.append(self.val)
 299             self.accept(False)
 300
 301         raise QAPIParseError(self, "documentation comment must end with '##'")
 302
 303
 304 class QAPIDoc:
 305     """
 306     A documentation comment block, either definition or free-form
 307
 308     Definition documentation blocks consist of
 309
 310     * a body section: one line naming the definition, followed by an
 311       overview (any number of lines)
 312
 313     * argument sections: a description of each argument (for commands
 314       and events) or member (for structs, unions and alternates)
 315
 316     * features sections: a description of each feature flag
 317
 318     * additional (non-argument) sections, possibly tagged
 319
 320     Free-form documentation blocks consist only of a body section.
 321     """
 322
 323     class Section:
 324         def __init__(self, parser, name=None, indent=0):
 325             # parser, for error messages about indentation
 326             self._parser = parser
 327             # optional section name (argument/member or section name)
 328             self.name = name
 329             self.text = ''
 330             # the expected indent level of the text of this section
 331             self._indent = indent
 332
 333         def append(self, line):
 334             # Strip leading spaces corresponding to the expected indent level
 335             # Blank lines are always OK.
 336             if line:
 337                 indent = re.match(r'\s*', line).end()
 338                 if indent < self._indent:
 339                     raise QAPIParseError(
 340                         self._parser,
 341                         "unexpected de-indent (expected at least %d spaces)" %
 342                         self._indent)
 343                 line = line[self._indent:]
 344
 345             self.text += line.rstrip() + '\n'
 346
 347     class ArgSection(Section):
 348         def __init__(self, parser, name, indent=0):
 349             super().__init__(parser, name, indent)
 350             self.member = None
 351
 352         def connect(self, member):
 353             self.member = member
 354
 355     def __init__(self, parser, info):
 356         # self._parser is used to report errors with QAPIParseError.  The
 357         # resulting error position depends on the state of the parser.
 358         # It happens to be the beginning of the comment.  More or less
 359         # servicable, but action at a distance.
 360         self._parser = parser
 361         self.info = info
 362         self.symbol = None
 363         self.body = QAPIDoc.Section(parser)
 364         # dict mapping parameter name to ArgSection
 365         self.args = OrderedDict()
 366         self.features = OrderedDict()
 367         # a list of Section
 368         self.sections = []
 369         # the current section
 370         self._section = self.body
 371         self._append_line = self._append_body_line
 372
 373     def has_section(self, name):
 374         """Return True if we have a section with this name."""
 375         for i in self.sections:
 376             if i.name == name:
 377                 return True
 378         return False
 379
 380     def append(self, line):
 381         """
 382         Parse a comment line and add it to the documentation.
 383
 384         The way that the line is dealt with depends on which part of
 385         the documentation we're parsing right now:
 386         * The body section: ._append_line is ._append_body_line
 387         * An argument section: ._append_line is ._append_args_line
 388         * A features section: ._append_line is ._append_features_line
 389         * An additional section: ._append_line is ._append_various_line
 390         """
 391         line = line[1:]
 392         if not line:
 393             self._append_freeform(line)
 394             return
 395
 396         if line[0] != ' ':
 397             raise QAPIParseError(self._parser, "missing space after #")
 398         line = line[1:]
 399         self._append_line(line)
 400
 401     def end_comment(self):
 402         self._end_section()
 403
 404     @staticmethod
 405     def _is_section_tag(name):
 406         return name in ('Returns:', 'Since:',
 407                         # those are often singular or plural
 408                         'Note:', 'Notes:',
 409                         'Example:', 'Examples:',
 410                         'TODO:')
 411
 412     def _append_body_line(self, line):
 413         """
 414         Process a line of documentation text in the body section.
 415
 416         If this a symbol line and it is the section's first line, this
 417         is a definition documentation block for that symbol.
 418
 419         If it's a definition documentation block, another symbol line
 420         begins the argument section for the argument named by it, and
 421         a section tag begins an additional section.  Start that
 422         section and append the line to it.
 423
 424         Else, append the line to the current section.
 425         """
 426         name = line.split(' ', 1)[0]
 427         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 428         # recognized, and get silently treated as ordinary text
 429         if not self.symbol and not self.body.text and line.startswith('@'):
 430             if not line.endswith(':'):
 431                 raise QAPIParseError(self._parser, "line should end with ':'")
 432             self.symbol = line[1:-1]
 433             # FIXME invalid names other than the empty string aren't flagged
 434             if not self.symbol:
 435                 raise QAPIParseError(self._parser, "invalid name")
 436         elif self.symbol:
 437             # This is a definition documentation block
 438             if name.startswith('@') and name.endswith(':'):
 439                 self._append_line = self._append_args_line
 440                 self._append_args_line(line)
 441             elif line == 'Features:':
 442                 self._append_line = self._append_features_line
 443             elif self._is_section_tag(name):
 444                 self._append_line = self._append_various_line
 445                 self._append_various_line(line)
 446             else:
 447                 self._append_freeform(line)
 448         else:
 449             # This is a free-form documentation block
 450             self._append_freeform(line)
 451
 452     def _append_args_line(self, line):
 453         """
 454         Process a line of documentation text in an argument section.
 455
 456         A symbol line begins the next argument section, a section tag
 457         section or a non-indented line after a blank line begins an
 458         additional section.  Start that section and append the line to
 459         it.
 460
 461         Else, append the line to the current section.
 462
 463         """
 464         name = line.split(' ', 1)[0]
 465
 466         if name.startswith('@') and name.endswith(':'):
 467             # If line is "@arg:   first line of description", find
 468             # the index of 'f', which is the indent we expect for any
 469             # following lines.  We then remove the leading "@arg:"
 470             # from line and replace it with spaces so that 'f' has the
 471             # same index as it did in the original line and can be
 472             # handled the same way we will handle following lines.
 473             indent = re.match(r'@\S*:\s*', line).end()
 474             line = line[indent:]
 475             if not line:
 476                 # Line was just the "@arg:" header; following lines
 477                 # are not indented
 478                 indent = 0
 479             else:
 480                 line = ' ' * indent + line
 481             self._start_args_section(name[1:-1], indent)
 482         elif self._is_section_tag(name):
 483             self._append_line = self._append_various_line
 484             self._append_various_line(line)
 485             return
 486         elif (self._section.text.endswith('\n\n')
 487               and line and not line[0].isspace()):
 488             if line == 'Features:':
 489                 self._append_line = self._append_features_line
 490             else:
 491                 self._start_section()
 492                 self._append_line = self._append_various_line
 493                 self._append_various_line(line)
 494             return
 495
 496         self._append_freeform(line)
 497
 498     def _append_features_line(self, line):
 499         name = line.split(' ', 1)[0]
 500
 501         if name.startswith('@') and name.endswith(':'):
 502             # If line is "@arg:   first line of description", find
 503             # the index of 'f', which is the indent we expect for any
 504             # following lines.  We then remove the leading "@arg:"
 505             # from line and replace it with spaces so that 'f' has the
 506             # same index as it did in the original line and can be
 507             # handled the same way we will handle following lines.
 508             indent = re.match(r'@\S*:\s*', line).end()
 509             line = line[indent:]
 510             if not line:
 511                 # Line was just the "@arg:" header; following lines
 512                 # are not indented
 513                 indent = 0
 514             else:
 515                 line = ' ' * indent + line
 516             self._start_features_section(name[1:-1], indent)
 517         elif self._is_section_tag(name):
 518             self._append_line = self._append_various_line
 519             self._append_various_line(line)
 520             return
 521         elif (self._section.text.endswith('\n\n')
 522               and line and not line[0].isspace()):
 523             self._start_section()
 524             self._append_line = self._append_various_line
 525             self._append_various_line(line)
 526             return
 527
 528         self._append_freeform(line)
 529
 530     def _append_various_line(self, line):
 531         """
 532         Process a line of documentation text in an additional section.
 533
 534         A symbol line is an error.
 535
 536         A section tag begins an additional section.  Start that
 537         section and append the line to it.
 538
 539         Else, append the line to the current section.
 540         """
 541         name = line.split(' ', 1)[0]
 542
 543         if name.startswith('@') and name.endswith(':'):
 544             raise QAPIParseError(self._parser,
 545                                  "'%s' can't follow '%s' section"
 546                                  % (name, self.sections[0].name))
 547         if self._is_section_tag(name):
 548             # If line is "Section:   first line of description", find
 549             # the index of 'f', which is the indent we expect for any
 550             # following lines.  We then remove the leading "Section:"
 551             # from line and replace it with spaces so that 'f' has the
 552             # same index as it did in the original line and can be
 553             # handled the same way we will handle following lines.
 554             indent = re.match(r'\S*:\s*', line).end()
 555             line = line[indent:]
 556             if not line:
 557                 # Line was just the "Section:" header; following lines
 558                 # are not indented
 559                 indent = 0
 560             else:
 561                 line = ' ' * indent + line
 562             self._start_section(name[:-1], indent)
 563
 564         self._append_freeform(line)
 565
 566     def _start_symbol_section(self, symbols_dict, name, indent):
 567         # FIXME invalid names other than the empty string aren't flagged
 568         if not name:
 569             raise QAPIParseError(self._parser, "invalid parameter name")
 570         if name in symbols_dict:
 571             raise QAPIParseError(self._parser,
 572                                  "'%s' parameter name duplicated" % name)
 573         assert not self.sections
 574         self._end_section()
 575         self._section = QAPIDoc.ArgSection(self._parser, name, indent)
 576         symbols_dict[name] = self._section
 577
 578     def _start_args_section(self, name, indent):
 579         self._start_symbol_section(self.args, name, indent)
 580
 581     def _start_features_section(self, name, indent):
 582         self._start_symbol_section(self.features, name, indent)
 583
 584     def _start_section(self, name=None, indent=0):
 585         if name in ('Returns', 'Since') and self.has_section(name):
 586             raise QAPIParseError(self._parser,
 587                                  "duplicated '%s' section" % name)
 588         self._end_section()
 589         self._section = QAPIDoc.Section(self._parser, name, indent)
 590         self.sections.append(self._section)
 591
 592     def _end_section(self):
 593         if self._section:
 594             text = self._section.text = self._section.text.strip()
 595             if self._section.name and (not text or text.isspace()):
 596                 raise QAPIParseError(
 597                     self._parser,
 598                     "empty doc section '%s'" % self._section.name)
 599             self._section = None
 600
 601     def _append_freeform(self, line):
 602         match = re.match(r'(@\S+:)', line)
 603         if match:
 604             raise QAPIParseError(self._parser,
 605                                  "'%s' not allowed in free-form documentation"
 606                                  % match.group(1))
 607         self._section.append(line)
 608
 609     def connect_member(self, member):
 610         if member.name not in self.args:
 611             # Undocumented TODO outlaw
 612             self.args[member.name] = QAPIDoc.ArgSection(self._parser,
 613                                                         member.name)
 614         self.args[member.name].connect(member)
 615
 616     def connect_feature(self, feature):
 617         if feature.name not in self.features:
 618             raise QAPISemError(feature.info,
 619                                "feature '%s' lacks documentation"
 620                                % feature.name)
 621         self.features[feature.name].connect(feature)
 622
 623     def check_expr(self, expr):
 624         if self.has_section('Returns') and 'command' not in expr:
 625             raise QAPISemError(self.info,
 626                                "'Returns:' is only valid for commands")
 627
 628     def check(self):
 629
 630         def check_args_section(args, info, what):
 631             bogus = [name for name, section in args.items()
 632                      if not section.member]
 633             if bogus:
 634                 raise QAPISemError(
 635                     self.info,
 636                     "documented member%s '%s' %s not exist"
 637                     % ("s" if len(bogus) > 1 else "",
 638                        "', '".join(bogus),
 639                        "do" if len(bogus) > 1 else "does"))
 640
 641         check_args_section(self.args, self.info, 'members')
 642         check_args_section(self.features, self.info, 'features')