scripts/qapi/parser.py

   1 # -*- coding: utf-8 -*-
   2 #
   3 # QAPI schema parser
   4 #
   5 # Copyright IBM, Corp. 2011
   6 # Copyright (c) 2013-2019 Red Hat Inc.
   7 #
   8 # Authors:
   9 #  Anthony Liguori <aliguori@us.ibm.com>
  10 #  Markus Armbruster <armbru@redhat.com>
  11 #  Marc-André Lureau <marcandre.lureau@redhat.com>
  12 #  Kevin Wolf <kwolf@redhat.com>
  13 #
  14 # This work is licensed under the terms of the GNU GPL, version 2.
  15 # See the COPYING file in the top-level directory.
  16
  17 from collections import OrderedDict
  18 import os
  19 import re
  20
  21 from .error import QAPISemError, QAPISourceError
  22 from .source import QAPISourceInfo
  23
  24
  25 class QAPIParseError(QAPISourceError):
  26     """Error class for all QAPI schema parsing errors."""
  27     def __init__(self, parser, msg):
  28         col = 1
  29         for ch in parser.src[parser.line_pos:parser.pos]:
  30             if ch == '\t':
  31                 col = (col + 7) % 8 + 1
  32             else:
  33                 col += 1
  34         super().__init__(parser.info, msg, col)
  35
  36
  37 class QAPISchemaParser:
  38
  39     def __init__(self, fname, previously_included=None, incl_info=None):
  40         previously_included = previously_included or set()
  41         previously_included.add(os.path.abspath(fname))
  42
  43         try:
  44             fp = open(fname, 'r', encoding='utf-8')
  45             self.src = fp.read()
  46         except IOError as e:
  47             raise QAPISemError(incl_info or QAPISourceInfo(None, None, None),
  48                                "can't read %s file '%s': %s"
  49                                % ("include" if incl_info else "schema",
  50                                   fname,
  51                                   e.strerror))
  52
  53         if self.src == '' or self.src[-1] != '\n':
  54             self.src += '\n'
  55         self.cursor = 0
  56         self.info = QAPISourceInfo(fname, 1, incl_info)
  57         self.line_pos = 0
  58         self.exprs = []
  59         self.docs = []
  60         self.accept()
  61         cur_doc = None
  62
  63         while self.tok is not None:
  64             info = self.info
  65             if self.tok == '#':
  66                 self.reject_expr_doc(cur_doc)
  67                 for cur_doc in self.get_doc(info):
  68                     self.docs.append(cur_doc)
  69                 continue
  70
  71             expr = self.get_expr(False)
  72             if 'include' in expr:
  73                 self.reject_expr_doc(cur_doc)
  74                 if len(expr) != 1:
  75                     raise QAPISemError(info, "invalid 'include' directive")
  76                 include = expr['include']
  77                 if not isinstance(include, str):
  78                     raise QAPISemError(info,
  79                                        "value of 'include' must be a string")
  80                 incl_fname = os.path.join(os.path.dirname(fname),
  81                                           include)
  82                 self.exprs.append({'expr': {'include': incl_fname},
  83                                    'info': info})
  84                 exprs_include = self._include(include, info, incl_fname,
  85                                               previously_included)
  86                 if exprs_include:
  87                     self.exprs.extend(exprs_include.exprs)
  88                     self.docs.extend(exprs_include.docs)
  89             elif "pragma" in expr:
  90                 self.reject_expr_doc(cur_doc)
  91                 if len(expr) != 1:
  92                     raise QAPISemError(info, "invalid 'pragma' directive")
  93                 pragma = expr['pragma']
  94                 if not isinstance(pragma, dict):
  95                     raise QAPISemError(
  96                         info, "value of 'pragma' must be an object")
  97                 for name, value in pragma.items():
  98                     self._pragma(name, value, info)
  99             else:
 100                 expr_elem = {'expr': expr,
 101                              'info': info}
 102                 if cur_doc:
 103                     if not cur_doc.symbol:
 104                         raise QAPISemError(
 105                             cur_doc.info, "definition documentation required")
 106                     expr_elem['doc'] = cur_doc
 107                 self.exprs.append(expr_elem)
 108             cur_doc = None
 109         self.reject_expr_doc(cur_doc)
 110
 111     @staticmethod
 112     def reject_expr_doc(doc):
 113         if doc and doc.symbol:
 114             raise QAPISemError(
 115                 doc.info,
 116                 "documentation for '%s' is not followed by the definition"
 117                 % doc.symbol)
 118
 119     def _include(self, include, info, incl_fname, previously_included):
 120         incl_abs_fname = os.path.abspath(incl_fname)
 121         # catch inclusion cycle
 122         inf = info
 123         while inf:
 124             if incl_abs_fname == os.path.abspath(inf.fname):
 125                 raise QAPISemError(info, "inclusion loop for %s" % include)
 126             inf = inf.parent
 127
 128         # skip multiple include of the same file
 129         if incl_abs_fname in previously_included:
 130             return None
 131
 132         return QAPISchemaParser(incl_fname, previously_included, info)
 133
 134     def _check_pragma_list_of_str(self, name, value, info):
 135         if (not isinstance(value, list)
 136                 or any([not isinstance(elt, str) for elt in value])):
 137             raise QAPISemError(
 138                 info,
 139                 "pragma %s must be a list of strings" % name)
 140
 141     def _pragma(self, name, value, info):
 142         if name == 'doc-required':
 143             if not isinstance(value, bool):
 144                 raise QAPISemError(info,
 145                                    "pragma 'doc-required' must be boolean")
 146             info.pragma.doc_required = value
 147         elif name == 'command-name-exceptions':
 148             self._check_pragma_list_of_str(name, value, info)
 149             info.pragma.command_name_exceptions = value
 150         elif name == 'command-returns-exceptions':
 151             self._check_pragma_list_of_str(name, value, info)
 152             info.pragma.command_returns_exceptions = value
 153         elif name == 'member-name-exceptions':
 154             self._check_pragma_list_of_str(name, value, info)
 155             info.pragma.member_name_exceptions = value
 156         else:
 157             raise QAPISemError(info, "unknown pragma '%s'" % name)
 158
 159     def accept(self, skip_comment=True):
 160         while True:
 161             self.tok = self.src[self.cursor]
 162             self.pos = self.cursor
 163             self.cursor += 1
 164             self.val = None
 165
 166             if self.tok == '#':
 167                 if self.src[self.cursor] == '#':
 168                     # Start of doc comment
 169                     skip_comment = False
 170                 self.cursor = self.src.find('\n', self.cursor)
 171                 if not skip_comment:
 172                     self.val = self.src[self.pos:self.cursor]
 173                     return
 174             elif self.tok in '{}:,[]':
 175                 return
 176             elif self.tok == "'":
 177                 # Note: we accept only printable ASCII
 178                 string = ''
 179                 esc = False
 180                 while True:
 181                     ch = self.src[self.cursor]
 182                     self.cursor += 1
 183                     if ch == '\n':
 184                         raise QAPIParseError(self, "missing terminating \"'\"")
 185                     if esc:
 186                         # Note: we recognize only \\ because we have
 187                         # no use for funny characters in strings
 188                         if ch != '\\':
 189                             raise QAPIParseError(self,
 190                                                  "unknown escape \\%s" % ch)
 191                         esc = False
 192                     elif ch == '\\':
 193                         esc = True
 194                         continue
 195                     elif ch == "'":
 196                         self.val = string
 197                         return
 198                     if ord(ch) < 32 or ord(ch) >= 127:
 199                         raise QAPIParseError(
 200                             self, "funny character in string")
 201                     string += ch
 202             elif self.src.startswith('true', self.pos):
 203                 self.val = True
 204                 self.cursor += 3
 205                 return
 206             elif self.src.startswith('false', self.pos):
 207                 self.val = False
 208                 self.cursor += 4
 209                 return
 210             elif self.tok == '\n':
 211                 if self.cursor == len(self.src):
 212                     self.tok = None
 213                     return
 214                 self.info = self.info.next_line()
 215                 self.line_pos = self.cursor
 216             elif not self.tok.isspace():
 217                 # Show up to next structural, whitespace or quote
 218                 # character
 219                 match = re.match('[^[\\]{}:,\\s\'"]+',
 220                                  self.src[self.cursor-1:])
 221                 raise QAPIParseError(self, "stray '%s'" % match.group(0))
 222
 223     def get_members(self):
 224         expr = OrderedDict()
 225         if self.tok == '}':
 226             self.accept()
 227             return expr
 228         if self.tok != "'":
 229             raise QAPIParseError(self, "expected string or '}'")
 230         while True:
 231             key = self.val
 232             self.accept()
 233             if self.tok != ':':
 234                 raise QAPIParseError(self, "expected ':'")
 235             self.accept()
 236             if key in expr:
 237                 raise QAPIParseError(self, "duplicate key '%s'" % key)
 238             expr[key] = self.get_expr(True)
 239             if self.tok == '}':
 240                 self.accept()
 241                 return expr
 242             if self.tok != ',':
 243                 raise QAPIParseError(self, "expected ',' or '}'")
 244             self.accept()
 245             if self.tok != "'":
 246                 raise QAPIParseError(self, "expected string")
 247
 248     def get_values(self):
 249         expr = []
 250         if self.tok == ']':
 251             self.accept()
 252             return expr
 253         if self.tok not in "{['tf":
 254             raise QAPIParseError(
 255                 self, "expected '{', '[', ']', string, or boolean")
 256         while True:
 257             expr.append(self.get_expr(True))
 258             if self.tok == ']':
 259                 self.accept()
 260                 return expr
 261             if self.tok != ',':
 262                 raise QAPIParseError(self, "expected ',' or ']'")
 263             self.accept()
 264
 265     def get_expr(self, nested):
 266         if self.tok != '{' and not nested:
 267             raise QAPIParseError(self, "expected '{'")
 268         if self.tok == '{':
 269             self.accept()
 270             expr = self.get_members()
 271         elif self.tok == '[':
 272             self.accept()
 273             expr = self.get_values()
 274         elif self.tok in "'tf":
 275             expr = self.val
 276             self.accept()
 277         else:
 278             raise QAPIParseError(
 279                 self, "expected '{', '[', string, or boolean")
 280         return expr
 281
 282     def get_doc(self, info):
 283         if self.val != '##':
 284             raise QAPIParseError(
 285                 self, "junk after '##' at start of documentation comment")
 286
 287         docs = []
 288         cur_doc = QAPIDoc(self, info)
 289         self.accept(False)
 290         while self.tok == '#':
 291             if self.val.startswith('##'):
 292                 # End of doc comment
 293                 if self.val != '##':
 294                     raise QAPIParseError(
 295                         self,
 296                         "junk after '##' at end of documentation comment")
 297                 cur_doc.end_comment()
 298                 docs.append(cur_doc)
 299                 self.accept()
 300                 return docs
 301             if self.val.startswith('# ='):
 302                 if cur_doc.symbol:
 303                     raise QAPIParseError(
 304                         self,
 305                         "unexpected '=' markup in definition documentation")
 306                 if cur_doc.body.text:
 307                     cur_doc.end_comment()
 308                     docs.append(cur_doc)
 309                     cur_doc = QAPIDoc(self, info)
 310             cur_doc.append(self.val)
 311             self.accept(False)
 312
 313         raise QAPIParseError(self, "documentation comment must end with '##'")
 314
 315
 316 class QAPIDoc:
 317     """
 318     A documentation comment block, either definition or free-form
 319
 320     Definition documentation blocks consist of
 321
 322     * a body section: one line naming the definition, followed by an
 323       overview (any number of lines)
 324
 325     * argument sections: a description of each argument (for commands
 326       and events) or member (for structs, unions and alternates)
 327
 328     * features sections: a description of each feature flag
 329
 330     * additional (non-argument) sections, possibly tagged
 331
 332     Free-form documentation blocks consist only of a body section.
 333     """
 334
 335     class Section:
 336         def __init__(self, parser, name=None, indent=0):
 337             # parser, for error messages about indentation
 338             self._parser = parser
 339             # optional section name (argument/member or section name)
 340             self.name = name
 341             self.text = ''
 342             # the expected indent level of the text of this section
 343             self._indent = indent
 344
 345         def append(self, line):
 346             # Strip leading spaces corresponding to the expected indent level
 347             # Blank lines are always OK.
 348             if line:
 349                 indent = re.match(r'\s*', line).end()
 350                 if indent < self._indent:
 351                     raise QAPIParseError(
 352                         self._parser,
 353                         "unexpected de-indent (expected at least %d spaces)" %
 354                         self._indent)
 355                 line = line[self._indent:]
 356
 357             self.text += line.rstrip() + '\n'
 358
 359     class ArgSection(Section):
 360         def __init__(self, parser, name, indent=0):
 361             super().__init__(parser, name, indent)
 362             self.member = None
 363
 364         def connect(self, member):
 365             self.member = member
 366
 367     def __init__(self, parser, info):
 368         # self._parser is used to report errors with QAPIParseError.  The
 369         # resulting error position depends on the state of the parser.
 370         # It happens to be the beginning of the comment.  More or less
 371         # servicable, but action at a distance.
 372         self._parser = parser
 373         self.info = info
 374         self.symbol = None
 375         self.body = QAPIDoc.Section(parser)
 376         # dict mapping parameter name to ArgSection
 377         self.args = OrderedDict()
 378         self.features = OrderedDict()
 379         # a list of Section
 380         self.sections = []
 381         # the current section
 382         self._section = self.body
 383         self._append_line = self._append_body_line
 384
 385     def has_section(self, name):
 386         """Return True if we have a section with this name."""
 387         for i in self.sections:
 388             if i.name == name:
 389                 return True
 390         return False
 391
 392     def append(self, line):
 393         """
 394         Parse a comment line and add it to the documentation.
 395
 396         The way that the line is dealt with depends on which part of
 397         the documentation we're parsing right now:
 398         * The body section: ._append_line is ._append_body_line
 399         * An argument section: ._append_line is ._append_args_line
 400         * A features section: ._append_line is ._append_features_line
 401         * An additional section: ._append_line is ._append_various_line
 402         """
 403         line = line[1:]
 404         if not line:
 405             self._append_freeform(line)
 406             return
 407
 408         if line[0] != ' ':
 409             raise QAPIParseError(self._parser, "missing space after #")
 410         line = line[1:]
 411         self._append_line(line)
 412
 413     def end_comment(self):
 414         self._end_section()
 415
 416     @staticmethod
 417     def _is_section_tag(name):
 418         return name in ('Returns:', 'Since:',
 419                         # those are often singular or plural
 420                         'Note:', 'Notes:',
 421                         'Example:', 'Examples:',
 422                         'TODO:')
 423
 424     def _append_body_line(self, line):
 425         """
 426         Process a line of documentation text in the body section.
 427
 428         If this a symbol line and it is the section's first line, this
 429         is a definition documentation block for that symbol.
 430
 431         If it's a definition documentation block, another symbol line
 432         begins the argument section for the argument named by it, and
 433         a section tag begins an additional section.  Start that
 434         section and append the line to it.
 435
 436         Else, append the line to the current section.
 437         """
 438         name = line.split(' ', 1)[0]
 439         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 440         # recognized, and get silently treated as ordinary text
 441         if not self.symbol and not self.body.text and line.startswith('@'):
 442             if not line.endswith(':'):
 443                 raise QAPIParseError(self._parser, "line should end with ':'")
 444             self.symbol = line[1:-1]
 445             # FIXME invalid names other than the empty string aren't flagged
 446             if not self.symbol:
 447                 raise QAPIParseError(self._parser, "invalid name")
 448         elif self.symbol:
 449             # This is a definition documentation block
 450             if name.startswith('@') and name.endswith(':'):
 451                 self._append_line = self._append_args_line
 452                 self._append_args_line(line)
 453             elif line == 'Features:':
 454                 self._append_line = self._append_features_line
 455             elif self._is_section_tag(name):
 456                 self._append_line = self._append_various_line
 457                 self._append_various_line(line)
 458             else:
 459                 self._append_freeform(line)
 460         else:
 461             # This is a free-form documentation block
 462             self._append_freeform(line)
 463
 464     def _append_args_line(self, line):
 465         """
 466         Process a line of documentation text in an argument section.
 467
 468         A symbol line begins the next argument section, a section tag
 469         section or a non-indented line after a blank line begins an
 470         additional section.  Start that section and append the line to
 471         it.
 472
 473         Else, append the line to the current section.
 474
 475         """
 476         name = line.split(' ', 1)[0]
 477
 478         if name.startswith('@') and name.endswith(':'):
 479             # If line is "@arg:   first line of description", find
 480             # the index of 'f', which is the indent we expect for any
 481             # following lines.  We then remove the leading "@arg:"
 482             # from line and replace it with spaces so that 'f' has the
 483             # same index as it did in the original line and can be
 484             # handled the same way we will handle following lines.
 485             indent = re.match(r'@\S*:\s*', line).end()
 486             line = line[indent:]
 487             if not line:
 488                 # Line was just the "@arg:" header; following lines
 489                 # are not indented
 490                 indent = 0
 491             else:
 492                 line = ' ' * indent + line
 493             self._start_args_section(name[1:-1], indent)
 494         elif self._is_section_tag(name):
 495             self._append_line = self._append_various_line
 496             self._append_various_line(line)
 497             return
 498         elif (self._section.text.endswith('\n\n')
 499               and line and not line[0].isspace()):
 500             if line == 'Features:':
 501                 self._append_line = self._append_features_line
 502             else:
 503                 self._start_section()
 504                 self._append_line = self._append_various_line
 505                 self._append_various_line(line)
 506             return
 507
 508         self._append_freeform(line)
 509
 510     def _append_features_line(self, line):
 511         name = line.split(' ', 1)[0]
 512
 513         if name.startswith('@') and name.endswith(':'):
 514             # If line is "@arg:   first line of description", find
 515             # the index of 'f', which is the indent we expect for any
 516             # following lines.  We then remove the leading "@arg:"
 517             # from line and replace it with spaces so that 'f' has the
 518             # same index as it did in the original line and can be
 519             # handled the same way we will handle following lines.
 520             indent = re.match(r'@\S*:\s*', line).end()
 521             line = line[indent:]
 522             if not line:
 523                 # Line was just the "@arg:" header; following lines
 524                 # are not indented
 525                 indent = 0
 526             else:
 527                 line = ' ' * indent + line
 528             self._start_features_section(name[1:-1], indent)
 529         elif self._is_section_tag(name):
 530             self._append_line = self._append_various_line
 531             self._append_various_line(line)
 532             return
 533         elif (self._section.text.endswith('\n\n')
 534               and line and not line[0].isspace()):
 535             self._start_section()
 536             self._append_line = self._append_various_line
 537             self._append_various_line(line)
 538             return
 539
 540         self._append_freeform(line)
 541
 542     def _append_various_line(self, line):
 543         """
 544         Process a line of documentation text in an additional section.
 545
 546         A symbol line is an error.
 547
 548         A section tag begins an additional section.  Start that
 549         section and append the line to it.
 550
 551         Else, append the line to the current section.
 552         """
 553         name = line.split(' ', 1)[0]
 554
 555         if name.startswith('@') and name.endswith(':'):
 556             raise QAPIParseError(self._parser,
 557                                  "'%s' can't follow '%s' section"
 558                                  % (name, self.sections[0].name))
 559         if self._is_section_tag(name):
 560             # If line is "Section:   first line of description", find
 561             # the index of 'f', which is the indent we expect for any
 562             # following lines.  We then remove the leading "Section:"
 563             # from line and replace it with spaces so that 'f' has the
 564             # same index as it did in the original line and can be
 565             # handled the same way we will handle following lines.
 566             indent = re.match(r'\S*:\s*', line).end()
 567             line = line[indent:]
 568             if not line:
 569                 # Line was just the "Section:" header; following lines
 570                 # are not indented
 571                 indent = 0
 572             else:
 573                 line = ' ' * indent + line
 574             self._start_section(name[:-1], indent)
 575
 576         self._append_freeform(line)
 577
 578     def _start_symbol_section(self, symbols_dict, name, indent):
 579         # FIXME invalid names other than the empty string aren't flagged
 580         if not name:
 581             raise QAPIParseError(self._parser, "invalid parameter name")
 582         if name in symbols_dict:
 583             raise QAPIParseError(self._parser,
 584                                  "'%s' parameter name duplicated" % name)
 585         assert not self.sections
 586         self._end_section()
 587         self._section = QAPIDoc.ArgSection(self._parser, name, indent)
 588         symbols_dict[name] = self._section
 589
 590     def _start_args_section(self, name, indent):
 591         self._start_symbol_section(self.args, name, indent)
 592
 593     def _start_features_section(self, name, indent):
 594         self._start_symbol_section(self.features, name, indent)
 595
 596     def _start_section(self, name=None, indent=0):
 597         if name in ('Returns', 'Since') and self.has_section(name):
 598             raise QAPIParseError(self._parser,
 599                                  "duplicated '%s' section" % name)
 600         self._end_section()
 601         self._section = QAPIDoc.Section(self._parser, name, indent)
 602         self.sections.append(self._section)
 603
 604     def _end_section(self):
 605         if self._section:
 606             text = self._section.text = self._section.text.strip()
 607             if self._section.name and (not text or text.isspace()):
 608                 raise QAPIParseError(
 609                     self._parser,
 610                     "empty doc section '%s'" % self._section.name)
 611             self._section = None
 612
 613     def _append_freeform(self, line):
 614         match = re.match(r'(@\S+:)', line)
 615         if match:
 616             raise QAPIParseError(self._parser,
 617                                  "'%s' not allowed in free-form documentation"
 618                                  % match.group(1))
 619         self._section.append(line)
 620
 621     def connect_member(self, member):
 622         if member.name not in self.args:
 623             # Undocumented TODO outlaw
 624             self.args[member.name] = QAPIDoc.ArgSection(self._parser,
 625                                                         member.name)
 626         self.args[member.name].connect(member)
 627
 628     def connect_feature(self, feature):
 629         if feature.name not in self.features:
 630             raise QAPISemError(feature.info,
 631                                "feature '%s' lacks documentation"
 632                                % feature.name)
 633         self.features[feature.name].connect(feature)
 634
 635     def check_expr(self, expr):
 636         if self.has_section('Returns') and 'command' not in expr:
 637             raise QAPISemError(self.info,
 638                                "'Returns:' is only valid for commands")
 639
 640     def check(self):
 641
 642         def check_args_section(args, info, what):
 643             bogus = [name for name, section in args.items()
 644                      if not section.member]
 645             if bogus:
 646                 raise QAPISemError(
 647                     self.info,
 648                     "documented member%s '%s' %s not exist"
 649                     % ("s" if len(bogus) > 1 else "",
 650                        "', '".join(bogus),
 651                        "do" if len(bogus) > 1 else "does"))
 652
 653         check_args_section(self.args, self.info, 'members')
 654         check_args_section(self.features, self.info, 'features')