scripts/qapi/parser.py

   1 # -*- coding: utf-8 -*-
   2 #
   3 # QAPI schema parser
   4 #
   5 # Copyright IBM, Corp. 2011
   6 # Copyright (c) 2013-2019 Red Hat Inc.
   7 #
   8 # Authors:
   9 #  Anthony Liguori <aliguori@us.ibm.com>
  10 #  Markus Armbruster <armbru@redhat.com>
  11 #  Marc-André Lureau <marcandre.lureau@redhat.com>
  12 #  Kevin Wolf <kwolf@redhat.com>
  13 #
  14 # This work is licensed under the terms of the GNU GPL, version 2.
  15 # See the COPYING file in the top-level directory.
  16
  17 from collections import OrderedDict
  18 import os
  19 import re
  20 from typing import (
  21     TYPE_CHECKING,
  22     Dict,
  23     List,
  24     Optional,
  25     Set,
  26     Union,
  27 )
  28
  29 from .common import must_match
  30 from .error import QAPISemError, QAPISourceError
  31 from .source import QAPISourceInfo
  32
  33
  34 if TYPE_CHECKING:
  35     # pylint: disable=cyclic-import
  36     # TODO: Remove cycle. [schema -> expr -> parser -> schema]
  37     from .schema import QAPISchemaFeature, QAPISchemaMember
  38
  39
  40 #: Represents a single Top Level QAPI schema expression.
  41 TopLevelExpr = Dict[str, object]
  42
  43 # Return value alias for get_expr().
  44 _ExprValue = Union[List[object], Dict[str, object], str, bool]
  45
  46 # FIXME: Consolidate and centralize definitions for TopLevelExpr,
  47 # _ExprValue, _JSONValue, and _JSONObject; currently scattered across
  48 # several modules.
  49
  50
  51 class QAPIParseError(QAPISourceError):
  52     """Error class for all QAPI schema parsing errors."""
  53     def __init__(self, parser: 'QAPISchemaParser', msg: str):
  54         col = 1
  55         for ch in parser.src[parser.line_pos:parser.pos]:
  56             if ch == '\t':
  57                 col = (col + 7) % 8 + 1
  58             else:
  59                 col += 1
  60         super().__init__(parser.info, msg, col)
  61
  62
  63 class QAPISchemaParser:
  64     """
  65     Parse QAPI schema source.
  66
  67     Parse a JSON-esque schema file and process directives.  See
  68     qapi-code-gen.txt section "Schema Syntax" for the exact syntax.
  69     Grammatical validation is handled later by `expr.check_exprs()`.
  70
  71     :param fname: Source file name.
  72     :param previously_included:
  73         The absolute names of previously included source files,
  74         if being invoked from another parser.
  75     :param incl_info:
  76        `QAPISourceInfo` belonging to the parent module.
  77        ``None`` implies this is the root module.
  78
  79     :ivar exprs: Resulting parsed expressions.
  80     :ivar docs: Resulting parsed documentation blocks.
  81
  82     :raise OSError: For problems reading the root schema document.
  83     :raise QAPIError: For errors in the schema source.
  84     """
  85     def __init__(self,
  86                  fname: str,
  87                  previously_included: Optional[Set[str]] = None,
  88                  incl_info: Optional[QAPISourceInfo] = None):
  89         self._fname = fname
  90         self._included = previously_included or set()
  91         self._included.add(os.path.abspath(self._fname))
  92         self.src = ''
  93
  94         # Lexer state (see `accept` for details):
  95         self.info = QAPISourceInfo(self._fname, incl_info)
  96         self.tok: Union[None, str] = None
  97         self.pos = 0
  98         self.cursor = 0
  99         self.val: Optional[Union[bool, str]] = None
 100         self.line_pos = 0
 101
 102         # Parser output:
 103         self.exprs: List[Dict[str, object]] = []
 104         self.docs: List[QAPIDoc] = []
 105
 106         # Showtime!
 107         self._parse()
 108
 109     def _parse(self) -> None:
 110         """
 111         Parse the QAPI schema document.
 112
 113         :return: None.  Results are stored in ``.exprs`` and ``.docs``.
 114         """
 115         cur_doc = None
 116
 117         # May raise OSError; allow the caller to handle it.
 118         with open(self._fname, 'r', encoding='utf-8') as fp:
 119             self.src = fp.read()
 120         if self.src == '' or self.src[-1] != '\n':
 121             self.src += '\n'
 122
 123         # Prime the lexer:
 124         self.accept()
 125
 126         # Parse until done:
 127         while self.tok is not None:
 128             info = self.info
 129             if self.tok == '#':
 130                 self.reject_expr_doc(cur_doc)
 131                 for cur_doc in self.get_doc(info):
 132                     self.docs.append(cur_doc)
 133                 continue
 134
 135             expr = self.get_expr()
 136             if not isinstance(expr, dict):
 137                 raise QAPISemError(
 138                     info, "top-level expression must be an object")
 139
 140             if 'include' in expr:
 141                 self.reject_expr_doc(cur_doc)
 142                 if len(expr) != 1:
 143                     raise QAPISemError(info, "invalid 'include' directive")
 144                 include = expr['include']
 145                 if not isinstance(include, str):
 146                     raise QAPISemError(info,
 147                                        "value of 'include' must be a string")
 148                 incl_fname = os.path.join(os.path.dirname(self._fname),
 149                                           include)
 150                 self.exprs.append({'expr': {'include': incl_fname},
 151                                    'info': info})
 152                 exprs_include = self._include(include, info, incl_fname,
 153                                               self._included)
 154                 if exprs_include:
 155                     self.exprs.extend(exprs_include.exprs)
 156                     self.docs.extend(exprs_include.docs)
 157             elif "pragma" in expr:
 158                 self.reject_expr_doc(cur_doc)
 159                 if len(expr) != 1:
 160                     raise QAPISemError(info, "invalid 'pragma' directive")
 161                 pragma = expr['pragma']
 162                 if not isinstance(pragma, dict):
 163                     raise QAPISemError(
 164                         info, "value of 'pragma' must be an object")
 165                 for name, value in pragma.items():
 166                     self._pragma(name, value, info)
 167             else:
 168                 expr_elem = {'expr': expr,
 169                              'info': info}
 170                 if cur_doc:
 171                     if not cur_doc.symbol:
 172                         raise QAPISemError(
 173                             cur_doc.info, "definition documentation required")
 174                     expr_elem['doc'] = cur_doc
 175                 self.exprs.append(expr_elem)
 176             cur_doc = None
 177         self.reject_expr_doc(cur_doc)
 178
 179     @staticmethod
 180     def reject_expr_doc(doc: Optional['QAPIDoc']) -> None:
 181         if doc and doc.symbol:
 182             raise QAPISemError(
 183                 doc.info,
 184                 "documentation for '%s' is not followed by the definition"
 185                 % doc.symbol)
 186
 187     @staticmethod
 188     def _include(include: str,
 189                  info: QAPISourceInfo,
 190                  incl_fname: str,
 191                  previously_included: Set[str]
 192                  ) -> Optional['QAPISchemaParser']:
 193         incl_abs_fname = os.path.abspath(incl_fname)
 194         # catch inclusion cycle
 195         inf: Optional[QAPISourceInfo] = info
 196         while inf:
 197             if incl_abs_fname == os.path.abspath(inf.fname):
 198                 raise QAPISemError(info, "inclusion loop for %s" % include)
 199             inf = inf.parent
 200
 201         # skip multiple include of the same file
 202         if incl_abs_fname in previously_included:
 203             return None
 204
 205         try:
 206             return QAPISchemaParser(incl_fname, previously_included, info)
 207         except OSError as err:
 208             raise QAPISemError(
 209                 info,
 210                 f"can't read include file '{incl_fname}': {err.strerror}"
 211             ) from err
 212
 213     @staticmethod
 214     def _pragma(name: str, value: object, info: QAPISourceInfo) -> None:
 215
 216         def check_list_str(name: str, value: object) -> List[str]:
 217             if (not isinstance(value, list) or
 218                     any(not isinstance(elt, str) for elt in value)):
 219                 raise QAPISemError(
 220                     info,
 221                     "pragma %s must be a list of strings" % name)
 222             return value
 223
 224         pragma = info.pragma
 225
 226         if name == 'doc-required':
 227             if not isinstance(value, bool):
 228                 raise QAPISemError(info,
 229                                    "pragma 'doc-required' must be boolean")
 230             pragma.doc_required = value
 231         elif name == 'command-name-exceptions':
 232             pragma.command_name_exceptions = check_list_str(name, value)
 233         elif name == 'command-returns-exceptions':
 234             pragma.command_returns_exceptions = check_list_str(name, value)
 235         elif name == 'member-name-exceptions':
 236             pragma.member_name_exceptions = check_list_str(name, value)
 237         else:
 238             raise QAPISemError(info, "unknown pragma '%s'" % name)
 239
 240     def accept(self, skip_comment: bool = True) -> None:
 241         """
 242         Read and store the next token.
 243
 244         :param skip_comment:
 245             When false, return COMMENT tokens ("#").
 246             This is used when reading documentation blocks.
 247
 248         :return:
 249             None.  Several instance attributes are updated instead:
 250
 251             - ``.tok`` represents the token type.  See below for values.
 252             - ``.info`` describes the token's source location.
 253             - ``.val`` is the token's value, if any.  See below.
 254             - ``.pos`` is the buffer index of the first character of
 255               the token.
 256
 257         * Single-character tokens:
 258
 259             These are "{", "}", ":", ",", "[", and "]".
 260             ``.tok`` holds the single character and ``.val`` is None.
 261
 262         * Multi-character tokens:
 263
 264           * COMMENT:
 265
 266             This token is not normally returned by the lexer, but it can
 267             be when ``skip_comment`` is False.  ``.tok`` is "#", and
 268             ``.val`` is a string including all chars until end-of-line,
 269             including the "#" itself.
 270
 271           * STRING:
 272
 273             ``.tok`` is "'", the single quote.  ``.val`` contains the
 274             string, excluding the surrounding quotes.
 275
 276           * TRUE and FALSE:
 277
 278             ``.tok`` is either "t" or "f", ``.val`` will be the
 279             corresponding bool value.
 280
 281           * EOF:
 282
 283             ``.tok`` and ``.val`` will both be None at EOF.
 284         """
 285         while True:
 286             self.tok = self.src[self.cursor]
 287             self.pos = self.cursor
 288             self.cursor += 1
 289             self.val = None
 290
 291             if self.tok == '#':
 292                 if self.src[self.cursor] == '#':
 293                     # Start of doc comment
 294                     skip_comment = False
 295                 self.cursor = self.src.find('\n', self.cursor)
 296                 if not skip_comment:
 297                     self.val = self.src[self.pos:self.cursor]
 298                     return
 299             elif self.tok in '{}:,[]':
 300                 return
 301             elif self.tok == "'":
 302                 # Note: we accept only printable ASCII
 303                 string = ''
 304                 esc = False
 305                 while True:
 306                     ch = self.src[self.cursor]
 307                     self.cursor += 1
 308                     if ch == '\n':
 309                         raise QAPIParseError(self, "missing terminating \"'\"")
 310                     if esc:
 311                         # Note: we recognize only \\ because we have
 312                         # no use for funny characters in strings
 313                         if ch != '\\':
 314                             raise QAPIParseError(self,
 315                                                  "unknown escape \\%s" % ch)
 316                         esc = False
 317                     elif ch == '\\':
 318                         esc = True
 319                         continue
 320                     elif ch == "'":
 321                         self.val = string
 322                         return
 323                     if ord(ch) < 32 or ord(ch) >= 127:
 324                         raise QAPIParseError(
 325                             self, "funny character in string")
 326                     string += ch
 327             elif self.src.startswith('true', self.pos):
 328                 self.val = True
 329                 self.cursor += 3
 330                 return
 331             elif self.src.startswith('false', self.pos):
 332                 self.val = False
 333                 self.cursor += 4
 334                 return
 335             elif self.tok == '\n':
 336                 if self.cursor == len(self.src):
 337                     self.tok = None
 338                     return
 339                 self.info = self.info.next_line()
 340                 self.line_pos = self.cursor
 341             elif not self.tok.isspace():
 342                 # Show up to next structural, whitespace or quote
 343                 # character
 344                 match = must_match('[^[\\]{}:,\\s\'"]+',
 345                                    self.src[self.cursor-1:])
 346                 raise QAPIParseError(self, "stray '%s'" % match.group(0))
 347
 348     def get_members(self) -> Dict[str, object]:
 349         expr: Dict[str, object] = OrderedDict()
 350         if self.tok == '}':
 351             self.accept()
 352             return expr
 353         if self.tok != "'":
 354             raise QAPIParseError(self, "expected string or '}'")
 355         while True:
 356             key = self.val
 357             assert isinstance(key, str)  # Guaranteed by tok == "'"
 358
 359             self.accept()
 360             if self.tok != ':':
 361                 raise QAPIParseError(self, "expected ':'")
 362             self.accept()
 363             if key in expr:
 364                 raise QAPIParseError(self, "duplicate key '%s'" % key)
 365             expr[key] = self.get_expr()
 366             if self.tok == '}':
 367                 self.accept()
 368                 return expr
 369             if self.tok != ',':
 370                 raise QAPIParseError(self, "expected ',' or '}'")
 371             self.accept()
 372             if self.tok != "'":
 373                 raise QAPIParseError(self, "expected string")
 374
 375     def get_values(self) -> List[object]:
 376         expr: List[object] = []
 377         if self.tok == ']':
 378             self.accept()
 379             return expr
 380         if self.tok not in tuple("{['tf"):
 381             raise QAPIParseError(
 382                 self, "expected '{', '[', ']', string, or boolean")
 383         while True:
 384             expr.append(self.get_expr())
 385             if self.tok == ']':
 386                 self.accept()
 387                 return expr
 388             if self.tok != ',':
 389                 raise QAPIParseError(self, "expected ',' or ']'")
 390             self.accept()
 391
 392     def get_expr(self) -> _ExprValue:
 393         expr: _ExprValue
 394         if self.tok == '{':
 395             self.accept()
 396             expr = self.get_members()
 397         elif self.tok == '[':
 398             self.accept()
 399             expr = self.get_values()
 400         elif self.tok in tuple("'tf"):
 401             assert isinstance(self.val, (str, bool))
 402             expr = self.val
 403             self.accept()
 404         else:
 405             raise QAPIParseError(
 406                 self, "expected '{', '[', string, or boolean")
 407         return expr
 408
 409     def get_doc(self, info: QAPISourceInfo) -> List['QAPIDoc']:
 410         if self.val != '##':
 411             raise QAPIParseError(
 412                 self, "junk after '##' at start of documentation comment")
 413
 414         docs = []
 415         cur_doc = QAPIDoc(self, info)
 416         self.accept(False)
 417         while self.tok == '#':
 418             assert isinstance(self.val, str)
 419             if self.val.startswith('##'):
 420                 # End of doc comment
 421                 if self.val != '##':
 422                     raise QAPIParseError(
 423                         self,
 424                         "junk after '##' at end of documentation comment")
 425                 cur_doc.end_comment()
 426                 docs.append(cur_doc)
 427                 self.accept()
 428                 return docs
 429             if self.val.startswith('# ='):
 430                 if cur_doc.symbol:
 431                     raise QAPIParseError(
 432                         self,
 433                         "unexpected '=' markup in definition documentation")
 434                 if cur_doc.body.text:
 435                     cur_doc.end_comment()
 436                     docs.append(cur_doc)
 437                     cur_doc = QAPIDoc(self, info)
 438             cur_doc.append(self.val)
 439             self.accept(False)
 440
 441         raise QAPIParseError(self, "documentation comment must end with '##'")
 442
 443
 444 class QAPIDoc:
 445     """
 446     A documentation comment block, either definition or free-form
 447
 448     Definition documentation blocks consist of
 449
 450     * a body section: one line naming the definition, followed by an
 451       overview (any number of lines)
 452
 453     * argument sections: a description of each argument (for commands
 454       and events) or member (for structs, unions and alternates)
 455
 456     * features sections: a description of each feature flag
 457
 458     * additional (non-argument) sections, possibly tagged
 459
 460     Free-form documentation blocks consist only of a body section.
 461     """
 462
 463     class Section:
 464         # pylint: disable=too-few-public-methods
 465         def __init__(self, parser: QAPISchemaParser,
 466                      name: Optional[str] = None, indent: int = 0):
 467
 468             # parser, for error messages about indentation
 469             self._parser = parser
 470             # optional section name (argument/member or section name)
 471             self.name = name
 472             self.text = ''
 473             # the expected indent level of the text of this section
 474             self._indent = indent
 475
 476         def append(self, line: str) -> None:
 477             # Strip leading spaces corresponding to the expected indent level
 478             # Blank lines are always OK.
 479             if line:
 480                 indent = must_match(r'\s*', line).end()
 481                 if indent < self._indent:
 482                     raise QAPIParseError(
 483                         self._parser,
 484                         "unexpected de-indent (expected at least %d spaces)" %
 485                         self._indent)
 486                 line = line[self._indent:]
 487
 488             self.text += line.rstrip() + '\n'
 489
 490     class ArgSection(Section):
 491         def __init__(self, parser: QAPISchemaParser,
 492                      name: str, indent: int = 0):
 493             super().__init__(parser, name, indent)
 494             self.member: Optional['QAPISchemaMember'] = None
 495
 496         def connect(self, member: 'QAPISchemaMember') -> None:
 497             self.member = member
 498
 499     class NullSection(Section):
 500         """
 501         Immutable dummy section for use at the end of a doc block.
 502         """
 503         # pylint: disable=too-few-public-methods
 504         def append(self, line: str) -> None:
 505             assert False, "Text appended after end_comment() called."
 506
 507     def __init__(self, parser: QAPISchemaParser, info: QAPISourceInfo):
 508         # self._parser is used to report errors with QAPIParseError.  The
 509         # resulting error position depends on the state of the parser.
 510         # It happens to be the beginning of the comment.  More or less
 511         # servicable, but action at a distance.
 512         self._parser = parser
 513         self.info = info
 514         self.symbol: Optional[str] = None
 515         self.body = QAPIDoc.Section(parser)
 516         # dicts mapping parameter/feature names to their ArgSection
 517         self.args: Dict[str, QAPIDoc.ArgSection] = OrderedDict()
 518         self.features: Dict[str, QAPIDoc.ArgSection] = OrderedDict()
 519         self.sections: List[QAPIDoc.Section] = []
 520         # the current section
 521         self._section = self.body
 522         self._append_line = self._append_body_line
 523
 524     def has_section(self, name: str) -> bool:
 525         """Return True if we have a section with this name."""
 526         for i in self.sections:
 527             if i.name == name:
 528                 return True
 529         return False
 530
 531     def append(self, line: str) -> None:
 532         """
 533         Parse a comment line and add it to the documentation.
 534
 535         The way that the line is dealt with depends on which part of
 536         the documentation we're parsing right now:
 537         * The body section: ._append_line is ._append_body_line
 538         * An argument section: ._append_line is ._append_args_line
 539         * A features section: ._append_line is ._append_features_line
 540         * An additional section: ._append_line is ._append_various_line
 541         """
 542         line = line[1:]
 543         if not line:
 544             self._append_freeform(line)
 545             return
 546
 547         if line[0] != ' ':
 548             raise QAPIParseError(self._parser, "missing space after #")
 549         line = line[1:]
 550         self._append_line(line)
 551
 552     def end_comment(self) -> None:
 553         self._switch_section(QAPIDoc.NullSection(self._parser))
 554
 555     @staticmethod
 556     def _is_section_tag(name: str) -> bool:
 557         return name in ('Returns:', 'Since:',
 558                         # those are often singular or plural
 559                         'Note:', 'Notes:',
 560                         'Example:', 'Examples:',
 561                         'TODO:')
 562
 563     def _append_body_line(self, line: str) -> None:
 564         """
 565         Process a line of documentation text in the body section.
 566
 567         If this a symbol line and it is the section's first line, this
 568         is a definition documentation block for that symbol.
 569
 570         If it's a definition documentation block, another symbol line
 571         begins the argument section for the argument named by it, and
 572         a section tag begins an additional section.  Start that
 573         section and append the line to it.
 574
 575         Else, append the line to the current section.
 576         """
 577         name = line.split(' ', 1)[0]
 578         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 579         # recognized, and get silently treated as ordinary text
 580         if not self.symbol and not self.body.text and line.startswith('@'):
 581             if not line.endswith(':'):
 582                 raise QAPIParseError(self._parser, "line should end with ':'")
 583             self.symbol = line[1:-1]
 584             # Invalid names are not checked here, but the name provided MUST
 585             # match the following definition, which *is* validated in expr.py.
 586             if not self.symbol:
 587                 raise QAPIParseError(
 588                     self._parser, "name required after '@'")
 589         elif self.symbol:
 590             # This is a definition documentation block
 591             if name.startswith('@') and name.endswith(':'):
 592                 self._append_line = self._append_args_line
 593                 self._append_args_line(line)
 594             elif line == 'Features:':
 595                 self._append_line = self._append_features_line
 596             elif self._is_section_tag(name):
 597                 self._append_line = self._append_various_line
 598                 self._append_various_line(line)
 599             else:
 600                 self._append_freeform(line)
 601         else:
 602             # This is a free-form documentation block
 603             self._append_freeform(line)
 604
 605     def _append_args_line(self, line: str) -> None:
 606         """
 607         Process a line of documentation text in an argument section.
 608
 609         A symbol line begins the next argument section, a section tag
 610         section or a non-indented line after a blank line begins an
 611         additional section.  Start that section and append the line to
 612         it.
 613
 614         Else, append the line to the current section.
 615
 616         """
 617         name = line.split(' ', 1)[0]
 618
 619         if name.startswith('@') and name.endswith(':'):
 620             # If line is "@arg:   first line of description", find
 621             # the index of 'f', which is the indent we expect for any
 622             # following lines.  We then remove the leading "@arg:"
 623             # from line and replace it with spaces so that 'f' has the
 624             # same index as it did in the original line and can be
 625             # handled the same way we will handle following lines.
 626             indent = must_match(r'@\S*:\s*', line).end()
 627             line = line[indent:]
 628             if not line:
 629                 # Line was just the "@arg:" header; following lines
 630                 # are not indented
 631                 indent = 0
 632             else:
 633                 line = ' ' * indent + line
 634             self._start_args_section(name[1:-1], indent)
 635         elif self._is_section_tag(name):
 636             self._append_line = self._append_various_line
 637             self._append_various_line(line)
 638             return
 639         elif (self._section.text.endswith('\n\n')
 640               and line and not line[0].isspace()):
 641             if line == 'Features:':
 642                 self._append_line = self._append_features_line
 643             else:
 644                 self._start_section()
 645                 self._append_line = self._append_various_line
 646                 self._append_various_line(line)
 647             return
 648
 649         self._append_freeform(line)
 650
 651     def _append_features_line(self, line: str) -> None:
 652         name = line.split(' ', 1)[0]
 653
 654         if name.startswith('@') and name.endswith(':'):
 655             # If line is "@arg:   first line of description", find
 656             # the index of 'f', which is the indent we expect for any
 657             # following lines.  We then remove the leading "@arg:"
 658             # from line and replace it with spaces so that 'f' has the
 659             # same index as it did in the original line and can be
 660             # handled the same way we will handle following lines.
 661             indent = must_match(r'@\S*:\s*', line).end()
 662             line = line[indent:]
 663             if not line:
 664                 # Line was just the "@arg:" header; following lines
 665                 # are not indented
 666                 indent = 0
 667             else:
 668                 line = ' ' * indent + line
 669             self._start_features_section(name[1:-1], indent)
 670         elif self._is_section_tag(name):
 671             self._append_line = self._append_various_line
 672             self._append_various_line(line)
 673             return
 674         elif (self._section.text.endswith('\n\n')
 675               and line and not line[0].isspace()):
 676             self._start_section()
 677             self._append_line = self._append_various_line
 678             self._append_various_line(line)
 679             return
 680
 681         self._append_freeform(line)
 682
 683     def _append_various_line(self, line: str) -> None:
 684         """
 685         Process a line of documentation text in an additional section.
 686
 687         A symbol line is an error.
 688
 689         A section tag begins an additional section.  Start that
 690         section and append the line to it.
 691
 692         Else, append the line to the current section.
 693         """
 694         name = line.split(' ', 1)[0]
 695
 696         if name.startswith('@') and name.endswith(':'):
 697             raise QAPIParseError(self._parser,
 698                                  "'%s' can't follow '%s' section"
 699                                  % (name, self.sections[0].name))
 700         if self._is_section_tag(name):
 701             # If line is "Section:   first line of description", find
 702             # the index of 'f', which is the indent we expect for any
 703             # following lines.  We then remove the leading "Section:"
 704             # from line and replace it with spaces so that 'f' has the
 705             # same index as it did in the original line and can be
 706             # handled the same way we will handle following lines.
 707             indent = must_match(r'\S*:\s*', line).end()
 708             line = line[indent:]
 709             if not line:
 710                 # Line was just the "Section:" header; following lines
 711                 # are not indented
 712                 indent = 0
 713             else:
 714                 line = ' ' * indent + line
 715             self._start_section(name[:-1], indent)
 716
 717         self._append_freeform(line)
 718
 719     def _start_symbol_section(
 720             self,
 721             symbols_dict: Dict[str, 'QAPIDoc.ArgSection'],
 722             name: str,
 723             indent: int) -> None:
 724         # FIXME invalid names other than the empty string aren't flagged
 725         if not name:
 726             raise QAPIParseError(self._parser, "invalid parameter name")
 727         if name in symbols_dict:
 728             raise QAPIParseError(self._parser,
 729                                  "'%s' parameter name duplicated" % name)
 730         assert not self.sections
 731         new_section = QAPIDoc.ArgSection(self._parser, name, indent)
 732         self._switch_section(new_section)
 733         symbols_dict[name] = new_section
 734
 735     def _start_args_section(self, name: str, indent: int) -> None:
 736         self._start_symbol_section(self.args, name, indent)
 737
 738     def _start_features_section(self, name: str, indent: int) -> None:
 739         self._start_symbol_section(self.features, name, indent)
 740
 741     def _start_section(self, name: Optional[str] = None,
 742                        indent: int = 0) -> None:
 743         if name in ('Returns', 'Since') and self.has_section(name):
 744             raise QAPIParseError(self._parser,
 745                                  "duplicated '%s' section" % name)
 746         new_section = QAPIDoc.Section(self._parser, name, indent)
 747         self._switch_section(new_section)
 748         self.sections.append(new_section)
 749
 750     def _switch_section(self, new_section: 'QAPIDoc.Section') -> None:
 751         text = self._section.text = self._section.text.strip()
 752
 753         # Only the 'body' section is allowed to have an empty body.
 754         # All other sections, including anonymous ones, must have text.
 755         if self._section != self.body and not text:
 756             # We do not create anonymous sections unless there is
 757             # something to put in them; this is a parser bug.
 758             assert self._section.name
 759             raise QAPIParseError(
 760                 self._parser,
 761                 "empty doc section '%s'" % self._section.name)
 762
 763         self._section = new_section
 764
 765     def _append_freeform(self, line: str) -> None:
 766         match = re.match(r'(@\S+:)', line)
 767         if match:
 768             raise QAPIParseError(self._parser,
 769                                  "'%s' not allowed in free-form documentation"
 770                                  % match.group(1))
 771         self._section.append(line)
 772
 773     def connect_member(self, member: 'QAPISchemaMember') -> None:
 774         if member.name not in self.args:
 775             # Undocumented TODO outlaw
 776             self.args[member.name] = QAPIDoc.ArgSection(self._parser,
 777                                                         member.name)
 778         self.args[member.name].connect(member)
 779
 780     def connect_feature(self, feature: 'QAPISchemaFeature') -> None:
 781         if feature.name not in self.features:
 782             raise QAPISemError(feature.info,
 783                                "feature '%s' lacks documentation"
 784                                % feature.name)
 785         self.features[feature.name].connect(feature)
 786
 787     def check_expr(self, expr: TopLevelExpr) -> None:
 788         if self.has_section('Returns') and 'command' not in expr:
 789             raise QAPISemError(self.info,
 790                                "'Returns:' is only valid for commands")
 791
 792     def check(self) -> None:
 793
 794         def check_args_section(
 795                 args: Dict[str, QAPIDoc.ArgSection], what: str
 796         ) -> None:
 797             bogus = [name for name, section in args.items()
 798                      if not section.member]
 799             if bogus:
 800                 raise QAPISemError(
 801                     self.info,
 802                     "documented %s%s '%s' %s not exist" % (
 803                         what,
 804                         "s" if len(bogus) > 1 else "",
 805                         "', '".join(bogus),
 806                         "do" if len(bogus) > 1 else "does"
 807                     ))
 808
 809         check_args_section(self.args, 'member')
 810         check_args_section(self.features, 'feature')