scripts/qapi/parser.py

   1 # -*- coding: utf-8 -*-
   2 #
   3 # QAPI schema parser
   4 #
   5 # Copyright IBM, Corp. 2011
   6 # Copyright (c) 2013-2019 Red Hat Inc.
   7 #
   8 # Authors:
   9 #  Anthony Liguori <aliguori@us.ibm.com>
  10 #  Markus Armbruster <armbru@redhat.com>
  11 #  Marc-André Lureau <marcandre.lureau@redhat.com>
  12 #  Kevin Wolf <kwolf@redhat.com>
  13 #
  14 # This work is licensed under the terms of the GNU GPL, version 2.
  15 # See the COPYING file in the top-level directory.
  16
  17 from collections import OrderedDict
  18 import os
  19 import re
  20 from typing import (
  21     TYPE_CHECKING,
  22     Dict,
  23     List,
  24     Mapping,
  25     Optional,
  26     Set,
  27     Union,
  28 )
  29
  30 from .common import must_match
  31 from .error import QAPISemError, QAPISourceError
  32 from .source import QAPISourceInfo
  33
  34
  35 if TYPE_CHECKING:
  36     # pylint: disable=cyclic-import
  37     # TODO: Remove cycle. [schema -> expr -> parser -> schema]
  38     from .schema import QAPISchemaFeature, QAPISchemaMember
  39
  40
  41 # Return value alias for get_expr().
  42 _ExprValue = Union[List[object], Dict[str, object], str, bool]
  43
  44
  45 class QAPIExpression(Dict[str, object]):
  46     # pylint: disable=too-few-public-methods
  47     def __init__(self,
  48                  data: Mapping[str, object],
  49                  info: QAPISourceInfo,
  50                  doc: Optional['QAPIDoc'] = None):
  51         super().__init__(data)
  52         self.info = info
  53         self.doc: Optional['QAPIDoc'] = doc
  54
  55
  56 class QAPIParseError(QAPISourceError):
  57     """Error class for all QAPI schema parsing errors."""
  58     def __init__(self, parser: 'QAPISchemaParser', msg: str):
  59         col = 1
  60         for ch in parser.src[parser.line_pos:parser.pos]:
  61             if ch == '\t':
  62                 col = (col + 7) % 8 + 1
  63             else:
  64                 col += 1
  65         super().__init__(parser.info, msg, col)
  66
  67
  68 class QAPISchemaParser:
  69     """
  70     Parse QAPI schema source.
  71
  72     Parse a JSON-esque schema file and process directives.  See
  73     qapi-code-gen.txt section "Schema Syntax" for the exact syntax.
  74     Grammatical validation is handled later by `expr.check_exprs()`.
  75
  76     :param fname: Source file name.
  77     :param previously_included:
  78         The absolute names of previously included source files,
  79         if being invoked from another parser.
  80     :param incl_info:
  81        `QAPISourceInfo` belonging to the parent module.
  82        ``None`` implies this is the root module.
  83
  84     :ivar exprs: Resulting parsed expressions.
  85     :ivar docs: Resulting parsed documentation blocks.
  86
  87     :raise OSError: For problems reading the root schema document.
  88     :raise QAPIError: For errors in the schema source.
  89     """
  90     def __init__(self,
  91                  fname: str,
  92                  previously_included: Optional[Set[str]] = None,
  93                  incl_info: Optional[QAPISourceInfo] = None):
  94         self._fname = fname
  95         self._included = previously_included or set()
  96         self._included.add(os.path.abspath(self._fname))
  97         self.src = ''
  98
  99         # Lexer state (see `accept` for details):
 100         self.info = QAPISourceInfo(self._fname, incl_info)
 101         self.tok: Union[None, str] = None
 102         self.pos = 0
 103         self.cursor = 0
 104         self.val: Optional[Union[bool, str]] = None
 105         self.line_pos = 0
 106
 107         # Parser output:
 108         self.exprs: List[QAPIExpression] = []
 109         self.docs: List[QAPIDoc] = []
 110
 111         # Showtime!
 112         self._parse()
 113
 114     def _parse(self) -> None:
 115         """
 116         Parse the QAPI schema document.
 117
 118         :return: None.  Results are stored in ``.exprs`` and ``.docs``.
 119         """
 120         cur_doc = None
 121
 122         # May raise OSError; allow the caller to handle it.
 123         with open(self._fname, 'r', encoding='utf-8') as fp:
 124             self.src = fp.read()
 125         if self.src == '' or self.src[-1] != '\n':
 126             self.src += '\n'
 127
 128         # Prime the lexer:
 129         self.accept()
 130
 131         # Parse until done:
 132         while self.tok is not None:
 133             info = self.info
 134             if self.tok == '#':
 135                 self.reject_expr_doc(cur_doc)
 136                 for cur_doc in self.get_doc(info):
 137                     self.docs.append(cur_doc)
 138                 continue
 139
 140             expr = self.get_expr()
 141             if not isinstance(expr, dict):
 142                 raise QAPISemError(
 143                     info, "top-level expression must be an object")
 144
 145             if 'include' in expr:
 146                 self.reject_expr_doc(cur_doc)
 147                 if len(expr) != 1:
 148                     raise QAPISemError(info, "invalid 'include' directive")
 149                 include = expr['include']
 150                 if not isinstance(include, str):
 151                     raise QAPISemError(info,
 152                                        "value of 'include' must be a string")
 153                 incl_fname = os.path.join(os.path.dirname(self._fname),
 154                                           include)
 155                 self._add_expr(OrderedDict({'include': incl_fname}), info)
 156                 exprs_include = self._include(include, info, incl_fname,
 157                                               self._included)
 158                 if exprs_include:
 159                     self.exprs.extend(exprs_include.exprs)
 160                     self.docs.extend(exprs_include.docs)
 161             elif "pragma" in expr:
 162                 self.reject_expr_doc(cur_doc)
 163                 if len(expr) != 1:
 164                     raise QAPISemError(info, "invalid 'pragma' directive")
 165                 pragma = expr['pragma']
 166                 if not isinstance(pragma, dict):
 167                     raise QAPISemError(
 168                         info, "value of 'pragma' must be an object")
 169                 for name, value in pragma.items():
 170                     self._pragma(name, value, info)
 171             else:
 172                 if cur_doc and not cur_doc.symbol:
 173                     raise QAPISemError(
 174                         cur_doc.info, "definition documentation required")
 175                 self._add_expr(expr, info, cur_doc)
 176             cur_doc = None
 177         self.reject_expr_doc(cur_doc)
 178
 179     def _add_expr(self, expr: Mapping[str, object],
 180                   info: QAPISourceInfo,
 181                   doc: Optional['QAPIDoc'] = None) -> None:
 182         self.exprs.append(QAPIExpression(expr, info, doc))
 183
 184     @staticmethod
 185     def reject_expr_doc(doc: Optional['QAPIDoc']) -> None:
 186         if doc and doc.symbol:
 187             raise QAPISemError(
 188                 doc.info,
 189                 "documentation for '%s' is not followed by the definition"
 190                 % doc.symbol)
 191
 192     @staticmethod
 193     def _include(include: str,
 194                  info: QAPISourceInfo,
 195                  incl_fname: str,
 196                  previously_included: Set[str]
 197                  ) -> Optional['QAPISchemaParser']:
 198         incl_abs_fname = os.path.abspath(incl_fname)
 199         # catch inclusion cycle
 200         inf: Optional[QAPISourceInfo] = info
 201         while inf:
 202             if incl_abs_fname == os.path.abspath(inf.fname):
 203                 raise QAPISemError(info, "inclusion loop for %s" % include)
 204             inf = inf.parent
 205
 206         # skip multiple include of the same file
 207         if incl_abs_fname in previously_included:
 208             return None
 209
 210         try:
 211             return QAPISchemaParser(incl_fname, previously_included, info)
 212         except OSError as err:
 213             raise QAPISemError(
 214                 info,
 215                 f"can't read include file '{incl_fname}': {err.strerror}"
 216             ) from err
 217
 218     @staticmethod
 219     def _pragma(name: str, value: object, info: QAPISourceInfo) -> None:
 220
 221         def check_list_str(name: str, value: object) -> List[str]:
 222             if (not isinstance(value, list) or
 223                     any(not isinstance(elt, str) for elt in value)):
 224                 raise QAPISemError(
 225                     info,
 226                     "pragma %s must be a list of strings" % name)
 227             return value
 228
 229         pragma = info.pragma
 230
 231         if name == 'doc-required':
 232             if not isinstance(value, bool):
 233                 raise QAPISemError(info,
 234                                    "pragma 'doc-required' must be boolean")
 235             pragma.doc_required = value
 236         elif name == 'command-name-exceptions':
 237             pragma.command_name_exceptions = check_list_str(name, value)
 238         elif name == 'command-returns-exceptions':
 239             pragma.command_returns_exceptions = check_list_str(name, value)
 240         elif name == 'member-name-exceptions':
 241             pragma.member_name_exceptions = check_list_str(name, value)
 242         else:
 243             raise QAPISemError(info, "unknown pragma '%s'" % name)
 244
 245     def accept(self, skip_comment: bool = True) -> None:
 246         """
 247         Read and store the next token.
 248
 249         :param skip_comment:
 250             When false, return COMMENT tokens ("#").
 251             This is used when reading documentation blocks.
 252
 253         :return:
 254             None.  Several instance attributes are updated instead:
 255
 256             - ``.tok`` represents the token type.  See below for values.
 257             - ``.info`` describes the token's source location.
 258             - ``.val`` is the token's value, if any.  See below.
 259             - ``.pos`` is the buffer index of the first character of
 260               the token.
 261
 262         * Single-character tokens:
 263
 264             These are "{", "}", ":", ",", "[", and "]".
 265             ``.tok`` holds the single character and ``.val`` is None.
 266
 267         * Multi-character tokens:
 268
 269           * COMMENT:
 270
 271             This token is not normally returned by the lexer, but it can
 272             be when ``skip_comment`` is False.  ``.tok`` is "#", and
 273             ``.val`` is a string including all chars until end-of-line,
 274             including the "#" itself.
 275
 276           * STRING:
 277
 278             ``.tok`` is "'", the single quote.  ``.val`` contains the
 279             string, excluding the surrounding quotes.
 280
 281           * TRUE and FALSE:
 282
 283             ``.tok`` is either "t" or "f", ``.val`` will be the
 284             corresponding bool value.
 285
 286           * EOF:
 287
 288             ``.tok`` and ``.val`` will both be None at EOF.
 289         """
 290         while True:
 291             self.tok = self.src[self.cursor]
 292             self.pos = self.cursor
 293             self.cursor += 1
 294             self.val = None
 295
 296             if self.tok == '#':
 297                 if self.src[self.cursor] == '#':
 298                     # Start of doc comment
 299                     skip_comment = False
 300                 self.cursor = self.src.find('\n', self.cursor)
 301                 if not skip_comment:
 302                     self.val = self.src[self.pos:self.cursor]
 303                     return
 304             elif self.tok in '{}:,[]':
 305                 return
 306             elif self.tok == "'":
 307                 # Note: we accept only printable ASCII
 308                 string = ''
 309                 esc = False
 310                 while True:
 311                     ch = self.src[self.cursor]
 312                     self.cursor += 1
 313                     if ch == '\n':
 314                         raise QAPIParseError(self, "missing terminating \"'\"")
 315                     if esc:
 316                         # Note: we recognize only \\ because we have
 317                         # no use for funny characters in strings
 318                         if ch != '\\':
 319                             raise QAPIParseError(self,
 320                                                  "unknown escape \\%s" % ch)
 321                         esc = False
 322                     elif ch == '\\':
 323                         esc = True
 324                         continue
 325                     elif ch == "'":
 326                         self.val = string
 327                         return
 328                     if ord(ch) < 32 or ord(ch) >= 127:
 329                         raise QAPIParseError(
 330                             self, "funny character in string")
 331                     string += ch
 332             elif self.src.startswith('true', self.pos):
 333                 self.val = True
 334                 self.cursor += 3
 335                 return
 336             elif self.src.startswith('false', self.pos):
 337                 self.val = False
 338                 self.cursor += 4
 339                 return
 340             elif self.tok == '\n':
 341                 if self.cursor == len(self.src):
 342                     self.tok = None
 343                     return
 344                 self.info = self.info.next_line()
 345                 self.line_pos = self.cursor
 346             elif not self.tok.isspace():
 347                 # Show up to next structural, whitespace or quote
 348                 # character
 349                 match = must_match('[^[\\]{}:,\\s\'"]+',
 350                                    self.src[self.cursor-1:])
 351                 raise QAPIParseError(self, "stray '%s'" % match.group(0))
 352
 353     def get_members(self) -> Dict[str, object]:
 354         expr: Dict[str, object] = OrderedDict()
 355         if self.tok == '}':
 356             self.accept()
 357             return expr
 358         if self.tok != "'":
 359             raise QAPIParseError(self, "expected string or '}'")
 360         while True:
 361             key = self.val
 362             assert isinstance(key, str)  # Guaranteed by tok == "'"
 363
 364             self.accept()
 365             if self.tok != ':':
 366                 raise QAPIParseError(self, "expected ':'")
 367             self.accept()
 368             if key in expr:
 369                 raise QAPIParseError(self, "duplicate key '%s'" % key)
 370             expr[key] = self.get_expr()
 371             if self.tok == '}':
 372                 self.accept()
 373                 return expr
 374             if self.tok != ',':
 375                 raise QAPIParseError(self, "expected ',' or '}'")
 376             self.accept()
 377             if self.tok != "'":
 378                 raise QAPIParseError(self, "expected string")
 379
 380     def get_values(self) -> List[object]:
 381         expr: List[object] = []
 382         if self.tok == ']':
 383             self.accept()
 384             return expr
 385         if self.tok not in tuple("{['tf"):
 386             raise QAPIParseError(
 387                 self, "expected '{', '[', ']', string, or boolean")
 388         while True:
 389             expr.append(self.get_expr())
 390             if self.tok == ']':
 391                 self.accept()
 392                 return expr
 393             if self.tok != ',':
 394                 raise QAPIParseError(self, "expected ',' or ']'")
 395             self.accept()
 396
 397     def get_expr(self) -> _ExprValue:
 398         expr: _ExprValue
 399         if self.tok == '{':
 400             self.accept()
 401             expr = self.get_members()
 402         elif self.tok == '[':
 403             self.accept()
 404             expr = self.get_values()
 405         elif self.tok in tuple("'tf"):
 406             assert isinstance(self.val, (str, bool))
 407             expr = self.val
 408             self.accept()
 409         else:
 410             raise QAPIParseError(
 411                 self, "expected '{', '[', string, or boolean")
 412         return expr
 413
 414     def get_doc(self, info: QAPISourceInfo) -> List['QAPIDoc']:
 415         if self.val != '##':
 416             raise QAPIParseError(
 417                 self, "junk after '##' at start of documentation comment")
 418
 419         docs = []
 420         cur_doc = QAPIDoc(self, info)
 421         self.accept(False)
 422         while self.tok == '#':
 423             assert isinstance(self.val, str)
 424             if self.val.startswith('##'):
 425                 # End of doc comment
 426                 if self.val != '##':
 427                     raise QAPIParseError(
 428                         self,
 429                         "junk after '##' at end of documentation comment")
 430                 cur_doc.end_comment()
 431                 docs.append(cur_doc)
 432                 self.accept()
 433                 return docs
 434             if self.val.startswith('# ='):
 435                 if cur_doc.symbol:
 436                     raise QAPIParseError(
 437                         self,
 438                         "unexpected '=' markup in definition documentation")
 439                 if cur_doc.body.text:
 440                     cur_doc.end_comment()
 441                     docs.append(cur_doc)
 442                     cur_doc = QAPIDoc(self, info)
 443             cur_doc.append(self.val)
 444             self.accept(False)
 445
 446         raise QAPIParseError(self, "documentation comment must end with '##'")
 447
 448
 449 class QAPIDoc:
 450     """
 451     A documentation comment block, either definition or free-form
 452
 453     Definition documentation blocks consist of
 454
 455     * a body section: one line naming the definition, followed by an
 456       overview (any number of lines)
 457
 458     * argument sections: a description of each argument (for commands
 459       and events) or member (for structs, unions and alternates)
 460
 461     * features sections: a description of each feature flag
 462
 463     * additional (non-argument) sections, possibly tagged
 464
 465     Free-form documentation blocks consist only of a body section.
 466     """
 467
 468     class Section:
 469         # pylint: disable=too-few-public-methods
 470         def __init__(self, parser: QAPISchemaParser,
 471                      name: Optional[str] = None, indent: int = 0):
 472
 473             # parser, for error messages about indentation
 474             self._parser = parser
 475             # optional section name (argument/member or section name)
 476             self.name = name
 477             self.text = ''
 478             # the expected indent level of the text of this section
 479             self._indent = indent
 480
 481         def append(self, line: str) -> None:
 482             # Strip leading spaces corresponding to the expected indent level
 483             # Blank lines are always OK.
 484             if line:
 485                 indent = must_match(r'\s*', line).end()
 486                 if indent < self._indent:
 487                     raise QAPIParseError(
 488                         self._parser,
 489                         "unexpected de-indent (expected at least %d spaces)" %
 490                         self._indent)
 491                 line = line[self._indent:]
 492
 493             self.text += line.rstrip() + '\n'
 494
 495     class ArgSection(Section):
 496         def __init__(self, parser: QAPISchemaParser,
 497                      name: str, indent: int = 0):
 498             super().__init__(parser, name, indent)
 499             self.member: Optional['QAPISchemaMember'] = None
 500
 501         def connect(self, member: 'QAPISchemaMember') -> None:
 502             self.member = member
 503
 504     class NullSection(Section):
 505         """
 506         Immutable dummy section for use at the end of a doc block.
 507         """
 508         # pylint: disable=too-few-public-methods
 509         def append(self, line: str) -> None:
 510             assert False, "Text appended after end_comment() called."
 511
 512     def __init__(self, parser: QAPISchemaParser, info: QAPISourceInfo):
 513         # self._parser is used to report errors with QAPIParseError.  The
 514         # resulting error position depends on the state of the parser.
 515         # It happens to be the beginning of the comment.  More or less
 516         # servicable, but action at a distance.
 517         self._parser = parser
 518         self.info = info
 519         self.symbol: Optional[str] = None
 520         self.body = QAPIDoc.Section(parser)
 521         # dicts mapping parameter/feature names to their ArgSection
 522         self.args: Dict[str, QAPIDoc.ArgSection] = OrderedDict()
 523         self.features: Dict[str, QAPIDoc.ArgSection] = OrderedDict()
 524         self.sections: List[QAPIDoc.Section] = []
 525         # the current section
 526         self._section = self.body
 527         self._append_line = self._append_body_line
 528
 529     def has_section(self, name: str) -> bool:
 530         """Return True if we have a section with this name."""
 531         for i in self.sections:
 532             if i.name == name:
 533                 return True
 534         return False
 535
 536     def append(self, line: str) -> None:
 537         """
 538         Parse a comment line and add it to the documentation.
 539
 540         The way that the line is dealt with depends on which part of
 541         the documentation we're parsing right now:
 542         * The body section: ._append_line is ._append_body_line
 543         * An argument section: ._append_line is ._append_args_line
 544         * A features section: ._append_line is ._append_features_line
 545         * An additional section: ._append_line is ._append_various_line
 546         """
 547         line = line[1:]
 548         if not line:
 549             self._append_freeform(line)
 550             return
 551
 552         if line[0] != ' ':
 553             raise QAPIParseError(self._parser, "missing space after #")
 554         line = line[1:]
 555         self._append_line(line)
 556
 557     def end_comment(self) -> None:
 558         self._switch_section(QAPIDoc.NullSection(self._parser))
 559
 560     @staticmethod
 561     def _is_section_tag(name: str) -> bool:
 562         return name in ('Returns:', 'Since:',
 563                         # those are often singular or plural
 564                         'Note:', 'Notes:',
 565                         'Example:', 'Examples:',
 566                         'TODO:')
 567
 568     def _append_body_line(self, line: str) -> None:
 569         """
 570         Process a line of documentation text in the body section.
 571
 572         If this a symbol line and it is the section's first line, this
 573         is a definition documentation block for that symbol.
 574
 575         If it's a definition documentation block, another symbol line
 576         begins the argument section for the argument named by it, and
 577         a section tag begins an additional section.  Start that
 578         section and append the line to it.
 579
 580         Else, append the line to the current section.
 581         """
 582         name = line.split(' ', 1)[0]
 583         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 584         # recognized, and get silently treated as ordinary text
 585         if not self.symbol and not self.body.text and line.startswith('@'):
 586             if not line.endswith(':'):
 587                 raise QAPIParseError(self._parser, "line should end with ':'")
 588             self.symbol = line[1:-1]
 589             # Invalid names are not checked here, but the name provided MUST
 590             # match the following definition, which *is* validated in expr.py.
 591             if not self.symbol:
 592                 raise QAPIParseError(
 593                     self._parser, "name required after '@'")
 594         elif self.symbol:
 595             # This is a definition documentation block
 596             if name.startswith('@') and name.endswith(':'):
 597                 self._append_line = self._append_args_line
 598                 self._append_args_line(line)
 599             elif line == 'Features:':
 600                 self._append_line = self._append_features_line
 601             elif self._is_section_tag(name):
 602                 self._append_line = self._append_various_line
 603                 self._append_various_line(line)
 604             else:
 605                 self._append_freeform(line)
 606         else:
 607             # This is a free-form documentation block
 608             self._append_freeform(line)
 609
 610     def _append_args_line(self, line: str) -> None:
 611         """
 612         Process a line of documentation text in an argument section.
 613
 614         A symbol line begins the next argument section, a section tag
 615         section or a non-indented line after a blank line begins an
 616         additional section.  Start that section and append the line to
 617         it.
 618
 619         Else, append the line to the current section.
 620
 621         """
 622         name = line.split(' ', 1)[0]
 623
 624         if name.startswith('@') and name.endswith(':'):
 625             # If line is "@arg:   first line of description", find
 626             # the index of 'f', which is the indent we expect for any
 627             # following lines.  We then remove the leading "@arg:"
 628             # from line and replace it with spaces so that 'f' has the
 629             # same index as it did in the original line and can be
 630             # handled the same way we will handle following lines.
 631             indent = must_match(r'@\S*:\s*', line).end()
 632             line = line[indent:]
 633             if not line:
 634                 # Line was just the "@arg:" header; following lines
 635                 # are not indented
 636                 indent = 0
 637             else:
 638                 line = ' ' * indent + line
 639             self._start_args_section(name[1:-1], indent)
 640         elif self._is_section_tag(name):
 641             self._append_line = self._append_various_line
 642             self._append_various_line(line)
 643             return
 644         elif (self._section.text.endswith('\n\n')
 645               and line and not line[0].isspace()):
 646             if line == 'Features:':
 647                 self._append_line = self._append_features_line
 648             else:
 649                 self._start_section()
 650                 self._append_line = self._append_various_line
 651                 self._append_various_line(line)
 652             return
 653
 654         self._append_freeform(line)
 655
 656     def _append_features_line(self, line: str) -> None:
 657         name = line.split(' ', 1)[0]
 658
 659         if name.startswith('@') and name.endswith(':'):
 660             # If line is "@arg:   first line of description", find
 661             # the index of 'f', which is the indent we expect for any
 662             # following lines.  We then remove the leading "@arg:"
 663             # from line and replace it with spaces so that 'f' has the
 664             # same index as it did in the original line and can be
 665             # handled the same way we will handle following lines.
 666             indent = must_match(r'@\S*:\s*', line).end()
 667             line = line[indent:]
 668             if not line:
 669                 # Line was just the "@arg:" header; following lines
 670                 # are not indented
 671                 indent = 0
 672             else:
 673                 line = ' ' * indent + line
 674             self._start_features_section(name[1:-1], indent)
 675         elif self._is_section_tag(name):
 676             self._append_line = self._append_various_line
 677             self._append_various_line(line)
 678             return
 679         elif (self._section.text.endswith('\n\n')
 680               and line and not line[0].isspace()):
 681             self._start_section()
 682             self._append_line = self._append_various_line
 683             self._append_various_line(line)
 684             return
 685
 686         self._append_freeform(line)
 687
 688     def _append_various_line(self, line: str) -> None:
 689         """
 690         Process a line of documentation text in an additional section.
 691
 692         A symbol line is an error.
 693
 694         A section tag begins an additional section.  Start that
 695         section and append the line to it.
 696
 697         Else, append the line to the current section.
 698         """
 699         name = line.split(' ', 1)[0]
 700
 701         if name.startswith('@') and name.endswith(':'):
 702             raise QAPIParseError(self._parser,
 703                                  "'%s' can't follow '%s' section"
 704                                  % (name, self.sections[0].name))
 705         if self._is_section_tag(name):
 706             # If line is "Section:   first line of description", find
 707             # the index of 'f', which is the indent we expect for any
 708             # following lines.  We then remove the leading "Section:"
 709             # from line and replace it with spaces so that 'f' has the
 710             # same index as it did in the original line and can be
 711             # handled the same way we will handle following lines.
 712             indent = must_match(r'\S*:\s*', line).end()
 713             line = line[indent:]
 714             if not line:
 715                 # Line was just the "Section:" header; following lines
 716                 # are not indented
 717                 indent = 0
 718             else:
 719                 line = ' ' * indent + line
 720             self._start_section(name[:-1], indent)
 721
 722         self._append_freeform(line)
 723
 724     def _start_symbol_section(
 725             self,
 726             symbols_dict: Dict[str, 'QAPIDoc.ArgSection'],
 727             name: str,
 728             indent: int) -> None:
 729         # FIXME invalid names other than the empty string aren't flagged
 730         if not name:
 731             raise QAPIParseError(self._parser, "invalid parameter name")
 732         if name in symbols_dict:
 733             raise QAPIParseError(self._parser,
 734                                  "'%s' parameter name duplicated" % name)
 735         assert not self.sections
 736         new_section = QAPIDoc.ArgSection(self._parser, name, indent)
 737         self._switch_section(new_section)
 738         symbols_dict[name] = new_section
 739
 740     def _start_args_section(self, name: str, indent: int) -> None:
 741         self._start_symbol_section(self.args, name, indent)
 742
 743     def _start_features_section(self, name: str, indent: int) -> None:
 744         self._start_symbol_section(self.features, name, indent)
 745
 746     def _start_section(self, name: Optional[str] = None,
 747                        indent: int = 0) -> None:
 748         if name in ('Returns', 'Since') and self.has_section(name):
 749             raise QAPIParseError(self._parser,
 750                                  "duplicated '%s' section" % name)
 751         new_section = QAPIDoc.Section(self._parser, name, indent)
 752         self._switch_section(new_section)
 753         self.sections.append(new_section)
 754
 755     def _switch_section(self, new_section: 'QAPIDoc.Section') -> None:
 756         text = self._section.text = self._section.text.strip()
 757
 758         # Only the 'body' section is allowed to have an empty body.
 759         # All other sections, including anonymous ones, must have text.
 760         if self._section != self.body and not text:
 761             # We do not create anonymous sections unless there is
 762             # something to put in them; this is a parser bug.
 763             assert self._section.name
 764             raise QAPIParseError(
 765                 self._parser,
 766                 "empty doc section '%s'" % self._section.name)
 767
 768         self._section = new_section
 769
 770     def _append_freeform(self, line: str) -> None:
 771         match = re.match(r'(@\S+:)', line)
 772         if match:
 773             raise QAPIParseError(self._parser,
 774                                  "'%s' not allowed in free-form documentation"
 775                                  % match.group(1))
 776         self._section.append(line)
 777
 778     def connect_member(self, member: 'QAPISchemaMember') -> None:
 779         if member.name not in self.args:
 780             # Undocumented TODO outlaw
 781             self.args[member.name] = QAPIDoc.ArgSection(self._parser,
 782                                                         member.name)
 783         self.args[member.name].connect(member)
 784
 785     def connect_feature(self, feature: 'QAPISchemaFeature') -> None:
 786         if feature.name not in self.features:
 787             raise QAPISemError(feature.info,
 788                                "feature '%s' lacks documentation"
 789                                % feature.name)
 790         self.features[feature.name].connect(feature)
 791
 792     def check_expr(self, expr: QAPIExpression) -> None:
 793         if self.has_section('Returns') and 'command' not in expr:
 794             raise QAPISemError(self.info,
 795                                "'Returns:' is only valid for commands")
 796
 797     def check(self) -> None:
 798
 799         def check_args_section(
 800                 args: Dict[str, QAPIDoc.ArgSection], what: str
 801         ) -> None:
 802             bogus = [name for name, section in args.items()
 803                      if not section.member]
 804             if bogus:
 805                 raise QAPISemError(
 806                     self.info,
 807                     "documented %s%s '%s' %s not exist" % (
 808                         what,
 809                         "s" if len(bogus) > 1 else "",
 810                         "', '".join(bogus),
 811                         "do" if len(bogus) > 1 else "does"
 812                     ))
 813
 814         check_args_section(self.args, 'member')
 815         check_args_section(self.features, 'feature')