qapi: Rewrite doc comment parser
[qemu/armbru.git] / scripts / qapi / parser.py
blob73ff15043049289035341bc5deac8b084b4aeb9c
1 # -*- coding: utf-8 -*-
3 # QAPI schema parser
5 # Copyright IBM, Corp. 2011
6 # Copyright (c) 2013-2019 Red Hat Inc.
8 # Authors:
9 # Anthony Liguori <aliguori@us.ibm.com>
10 # Markus Armbruster <armbru@redhat.com>
11 # Marc-André Lureau <marcandre.lureau@redhat.com>
12 # Kevin Wolf <kwolf@redhat.com>
14 # This work is licensed under the terms of the GNU GPL, version 2.
15 # See the COPYING file in the top-level directory.
17 from collections import OrderedDict
18 import os
19 import re
20 from typing import (
21 TYPE_CHECKING,
22 Dict,
23 List,
24 Mapping,
25 Match,
26 Optional,
27 Set,
28 Union,
31 from .common import must_match
32 from .error import QAPISemError, QAPISourceError
33 from .source import QAPISourceInfo
36 if TYPE_CHECKING:
37 # pylint: disable=cyclic-import
38 # TODO: Remove cycle. [schema -> expr -> parser -> schema]
39 from .schema import QAPISchemaFeature, QAPISchemaMember
42 # Return value alias for get_expr().
43 _ExprValue = Union[List[object], Dict[str, object], str, bool]
46 class QAPIExpression(Dict[str, object]):
47 # pylint: disable=too-few-public-methods
48 def __init__(self,
49 data: Mapping[str, object],
50 info: QAPISourceInfo,
51 doc: Optional['QAPIDoc'] = None):
52 super().__init__(data)
53 self.info = info
54 self.doc: Optional['QAPIDoc'] = doc
57 class QAPIParseError(QAPISourceError):
58 """Error class for all QAPI schema parsing errors."""
59 def __init__(self, parser: 'QAPISchemaParser', msg: str):
60 col = 1
61 for ch in parser.src[parser.line_pos:parser.pos]:
62 if ch == '\t':
63 col = (col + 7) % 8 + 1
64 else:
65 col += 1
66 super().__init__(parser.info, msg, col)
69 class QAPISchemaParser:
70 """
71 Parse QAPI schema source.
73 Parse a JSON-esque schema file and process directives. See
74 qapi-code-gen.rst section "Schema Syntax" for the exact syntax.
75 Grammatical validation is handled later by `expr.check_exprs()`.
77 :param fname: Source file name.
78 :param previously_included:
79 The absolute names of previously included source files,
80 if being invoked from another parser.
81 :param incl_info:
82 `QAPISourceInfo` belonging to the parent module.
83 ``None`` implies this is the root module.
85 :ivar exprs: Resulting parsed expressions.
86 :ivar docs: Resulting parsed documentation blocks.
88 :raise OSError: For problems reading the root schema document.
89 :raise QAPIError: For errors in the schema source.
90 """
91 def __init__(self,
92 fname: str,
93 previously_included: Optional[Set[str]] = None,
94 incl_info: Optional[QAPISourceInfo] = None):
95 self._fname = fname
96 self._included = previously_included or set()
97 self._included.add(os.path.abspath(self._fname))
98 self.src = ''
100 # Lexer state (see `accept` for details):
101 self.info = QAPISourceInfo(self._fname, incl_info)
102 self.tok: Union[None, str] = None
103 self.pos = 0
104 self.cursor = 0
105 self.val: Optional[Union[bool, str]] = None
106 self.line_pos = 0
108 # Parser output:
109 self.exprs: List[QAPIExpression] = []
110 self.docs: List[QAPIDoc] = []
112 # Showtime!
113 self._parse()
115 def _parse(self) -> None:
117 Parse the QAPI schema document.
119 :return: None. Results are stored in ``.exprs`` and ``.docs``.
121 cur_doc = None
123 # May raise OSError; allow the caller to handle it.
124 with open(self._fname, 'r', encoding='utf-8') as fp:
125 self.src = fp.read()
126 if self.src == '' or self.src[-1] != '\n':
127 self.src += '\n'
129 # Prime the lexer:
130 self.accept()
132 # Parse until done:
133 while self.tok is not None:
134 info = self.info
135 if self.tok == '#':
136 self.reject_expr_doc(cur_doc)
137 cur_doc = self.get_doc()
138 self.docs.append(cur_doc)
139 continue
141 expr = self.get_expr()
142 if not isinstance(expr, dict):
143 raise QAPISemError(
144 info, "top-level expression must be an object")
146 if 'include' in expr:
147 self.reject_expr_doc(cur_doc)
148 if len(expr) != 1:
149 raise QAPISemError(info, "invalid 'include' directive")
150 include = expr['include']
151 if not isinstance(include, str):
152 raise QAPISemError(info,
153 "value of 'include' must be a string")
154 incl_fname = os.path.join(os.path.dirname(self._fname),
155 include)
156 self._add_expr(OrderedDict({'include': incl_fname}), info)
157 exprs_include = self._include(include, info, incl_fname,
158 self._included)
159 if exprs_include:
160 self.exprs.extend(exprs_include.exprs)
161 self.docs.extend(exprs_include.docs)
162 elif "pragma" in expr:
163 self.reject_expr_doc(cur_doc)
164 if len(expr) != 1:
165 raise QAPISemError(info, "invalid 'pragma' directive")
166 pragma = expr['pragma']
167 if not isinstance(pragma, dict):
168 raise QAPISemError(
169 info, "value of 'pragma' must be an object")
170 for name, value in pragma.items():
171 self._pragma(name, value, info)
172 else:
173 if cur_doc and not cur_doc.symbol:
174 raise QAPISemError(
175 cur_doc.info, "definition documentation required")
176 self._add_expr(expr, info, cur_doc)
177 cur_doc = None
178 self.reject_expr_doc(cur_doc)
180 def _add_expr(self, expr: Mapping[str, object],
181 info: QAPISourceInfo,
182 doc: Optional['QAPIDoc'] = None) -> None:
183 self.exprs.append(QAPIExpression(expr, info, doc))
185 @staticmethod
186 def reject_expr_doc(doc: Optional['QAPIDoc']) -> None:
187 if doc and doc.symbol:
188 raise QAPISemError(
189 doc.info,
190 "documentation for '%s' is not followed by the definition"
191 % doc.symbol)
193 @staticmethod
194 def _include(include: str,
195 info: QAPISourceInfo,
196 incl_fname: str,
197 previously_included: Set[str]
198 ) -> Optional['QAPISchemaParser']:
199 incl_abs_fname = os.path.abspath(incl_fname)
200 # catch inclusion cycle
201 inf: Optional[QAPISourceInfo] = info
202 while inf:
203 if incl_abs_fname == os.path.abspath(inf.fname):
204 raise QAPISemError(info, "inclusion loop for %s" % include)
205 inf = inf.parent
207 # skip multiple include of the same file
208 if incl_abs_fname in previously_included:
209 return None
211 try:
212 return QAPISchemaParser(incl_fname, previously_included, info)
213 except OSError as err:
214 raise QAPISemError(
215 info,
216 f"can't read include file '{incl_fname}': {err.strerror}"
217 ) from err
219 @staticmethod
220 def _pragma(name: str, value: object, info: QAPISourceInfo) -> None:
222 def check_list_str(name: str, value: object) -> List[str]:
223 if (not isinstance(value, list) or
224 any(not isinstance(elt, str) for elt in value)):
225 raise QAPISemError(
226 info,
227 "pragma %s must be a list of strings" % name)
228 return value
230 pragma = info.pragma
232 if name == 'doc-required':
233 if not isinstance(value, bool):
234 raise QAPISemError(info,
235 "pragma 'doc-required' must be boolean")
236 pragma.doc_required = value
237 elif name == 'command-name-exceptions':
238 pragma.command_name_exceptions = check_list_str(name, value)
239 elif name == 'command-returns-exceptions':
240 pragma.command_returns_exceptions = check_list_str(name, value)
241 elif name == 'documentation-exceptions':
242 pragma.documentation_exceptions = check_list_str(name, value)
243 elif name == 'member-name-exceptions':
244 pragma.member_name_exceptions = check_list_str(name, value)
245 else:
246 raise QAPISemError(info, "unknown pragma '%s'" % name)
248 def accept(self, skip_comment: bool = True) -> None:
250 Read and store the next token.
252 :param skip_comment:
253 When false, return COMMENT tokens ("#").
254 This is used when reading documentation blocks.
256 :return:
257 None. Several instance attributes are updated instead:
259 - ``.tok`` represents the token type. See below for values.
260 - ``.info`` describes the token's source location.
261 - ``.val`` is the token's value, if any. See below.
262 - ``.pos`` is the buffer index of the first character of
263 the token.
265 * Single-character tokens:
267 These are "{", "}", ":", ",", "[", and "]".
268 ``.tok`` holds the single character and ``.val`` is None.
270 * Multi-character tokens:
272 * COMMENT:
274 This token is not normally returned by the lexer, but it can
275 be when ``skip_comment`` is False. ``.tok`` is "#", and
276 ``.val`` is a string including all chars until end-of-line,
277 including the "#" itself.
279 * STRING:
281 ``.tok`` is "'", the single quote. ``.val`` contains the
282 string, excluding the surrounding quotes.
284 * TRUE and FALSE:
286 ``.tok`` is either "t" or "f", ``.val`` will be the
287 corresponding bool value.
289 * EOF:
291 ``.tok`` and ``.val`` will both be None at EOF.
293 while True:
294 self.tok = self.src[self.cursor]
295 self.pos = self.cursor
296 self.cursor += 1
297 self.val = None
299 if self.tok == '#':
300 if self.src[self.cursor] == '#':
301 # Start of doc comment
302 skip_comment = False
303 self.cursor = self.src.find('\n', self.cursor)
304 if not skip_comment:
305 self.val = self.src[self.pos:self.cursor]
306 return
307 elif self.tok in '{}:,[]':
308 return
309 elif self.tok == "'":
310 # Note: we accept only printable ASCII
311 string = ''
312 esc = False
313 while True:
314 ch = self.src[self.cursor]
315 self.cursor += 1
316 if ch == '\n':
317 raise QAPIParseError(self, "missing terminating \"'\"")
318 if esc:
319 # Note: we recognize only \\ because we have
320 # no use for funny characters in strings
321 if ch != '\\':
322 raise QAPIParseError(self,
323 "unknown escape \\%s" % ch)
324 esc = False
325 elif ch == '\\':
326 esc = True
327 continue
328 elif ch == "'":
329 self.val = string
330 return
331 if ord(ch) < 32 or ord(ch) >= 127:
332 raise QAPIParseError(
333 self, "funny character in string")
334 string += ch
335 elif self.src.startswith('true', self.pos):
336 self.val = True
337 self.cursor += 3
338 return
339 elif self.src.startswith('false', self.pos):
340 self.val = False
341 self.cursor += 4
342 return
343 elif self.tok == '\n':
344 if self.cursor == len(self.src):
345 self.tok = None
346 return
347 self.info = self.info.next_line()
348 self.line_pos = self.cursor
349 elif not self.tok.isspace():
350 # Show up to next structural, whitespace or quote
351 # character
352 match = must_match('[^[\\]{}:,\\s\']+',
353 self.src[self.cursor-1:])
354 raise QAPIParseError(self, "stray '%s'" % match.group(0))
356 def get_members(self) -> Dict[str, object]:
357 expr: Dict[str, object] = OrderedDict()
358 if self.tok == '}':
359 self.accept()
360 return expr
361 if self.tok != "'":
362 raise QAPIParseError(self, "expected string or '}'")
363 while True:
364 key = self.val
365 assert isinstance(key, str) # Guaranteed by tok == "'"
367 self.accept()
368 if self.tok != ':':
369 raise QAPIParseError(self, "expected ':'")
370 self.accept()
371 if key in expr:
372 raise QAPIParseError(self, "duplicate key '%s'" % key)
373 expr[key] = self.get_expr()
374 if self.tok == '}':
375 self.accept()
376 return expr
377 if self.tok != ',':
378 raise QAPIParseError(self, "expected ',' or '}'")
379 self.accept()
380 if self.tok != "'":
381 raise QAPIParseError(self, "expected string")
383 def get_values(self) -> List[object]:
384 expr: List[object] = []
385 if self.tok == ']':
386 self.accept()
387 return expr
388 if self.tok not in tuple("{['tf"):
389 raise QAPIParseError(
390 self, "expected '{', '[', ']', string, or boolean")
391 while True:
392 expr.append(self.get_expr())
393 if self.tok == ']':
394 self.accept()
395 return expr
396 if self.tok != ',':
397 raise QAPIParseError(self, "expected ',' or ']'")
398 self.accept()
400 def get_expr(self) -> _ExprValue:
401 expr: _ExprValue
402 if self.tok == '{':
403 self.accept()
404 expr = self.get_members()
405 elif self.tok == '[':
406 self.accept()
407 expr = self.get_values()
408 elif self.tok in tuple("'tf"):
409 assert isinstance(self.val, (str, bool))
410 expr = self.val
411 self.accept()
412 else:
413 raise QAPIParseError(
414 self, "expected '{', '[', string, or boolean")
415 return expr
417 def get_doc_line(self) -> Optional[str]:
418 if self.tok != '#':
419 raise QAPIParseError(
420 self, "documentation comment must end with '##'")
421 assert isinstance(self.val, str)
422 if self.val.startswith('##'):
423 # End of doc comment
424 if self.val != '##':
425 raise QAPIParseError(
426 self, "junk after '##' at end of documentation comment")
427 return None
428 if self.val == '#':
429 return ''
430 if self.val[1] != ' ':
431 raise QAPIParseError(self, "missing space after #")
432 return self.val[2:].rstrip()
434 @staticmethod
435 def _match_at_name_colon(string: str) -> Optional[Match[str]]:
436 return re.match(r'@([^:]*): *', string)
438 def get_doc_indented(self, doc: 'QAPIDoc') -> Optional[str]:
439 self.accept(False)
440 line = self.get_doc_line()
441 while line == '':
442 doc.append_line(line)
443 self.accept(False)
444 line = self.get_doc_line()
445 if line is None:
446 return line
447 indent = must_match(r'\s*', line).end()
448 if not indent:
449 return line
450 doc.append_line(line[indent:])
451 prev_line_blank = False
452 while True:
453 self.accept(False)
454 line = self.get_doc_line()
455 if line is None:
456 return line
457 if self._match_at_name_colon(line):
458 return line
459 cur_indent = must_match(r'\s*', line).end()
460 if line != '' and cur_indent < indent:
461 if prev_line_blank:
462 return line
463 raise QAPIParseError(
464 self,
465 "unexpected de-indent (expected at least %d spaces)" %
466 indent)
467 doc.append_line(line[indent:])
468 prev_line_blank = True
470 def get_doc_paragraph(self, doc: 'QAPIDoc') -> Optional[str]:
471 while True:
472 self.accept(False)
473 line = self.get_doc_line()
474 if line is None:
475 return line
476 if line == '':
477 return line
478 doc.append_line(line)
480 def get_doc(self) -> 'QAPIDoc':
481 if self.val != '##':
482 raise QAPIParseError(
483 self, "junk after '##' at start of documentation comment")
484 info = self.info
485 self.accept(False)
486 line = self.get_doc_line()
487 if line is not None and line.startswith('@'):
488 # Definition documentation
489 if not line.endswith(':'):
490 raise QAPIParseError(self, "line should end with ':'")
491 # Invalid names are not checked here, but the name
492 # provided *must* match the following definition,
493 # which *is* validated in expr.py.
494 symbol = line[1:-1]
495 if not symbol:
496 raise QAPIParseError(self, "name required after '@'")
497 doc = QAPIDoc(self, info, symbol)
498 self.accept(False)
499 line = self.get_doc_line()
500 no_more_args = False
502 while line is not None:
503 # Blank lines
504 while line == '':
505 self.accept(False)
506 line = self.get_doc_line()
507 if line is None:
508 break
509 # Non-blank line, first of a section
510 if line == 'Features:' and not doc.features:
511 self.accept(False)
512 line = self.get_doc_line()
513 while line == '':
514 self.accept(False)
515 line = self.get_doc_line()
516 while (line is not None
517 and (match := self._match_at_name_colon(line))):
518 doc.new_feature(match.group(1))
519 text = line[match.end():]
520 if text:
521 doc.append_line(text)
522 line = self.get_doc_indented(doc)
523 no_more_args = True
524 elif match := self._match_at_name_colon(line):
525 # description
526 if no_more_args:
527 raise QAPIParseError(
528 self,
529 "description of '@%s:' follows a section"
530 % match.group(1))
531 while (line is not None
532 and (match := self._match_at_name_colon(line))):
533 doc.new_argument(match.group(1))
534 text = line[match.end():]
535 if text:
536 doc.append_line(text)
537 line = self.get_doc_indented(doc)
538 no_more_args = True
539 elif match := re.match(
540 r'(Returns|Since|Notes?|Examples?|TODO): *',
541 line):
542 # tagged section
543 doc.new_tagged_section(match.group(1))
544 text = line[match.end():]
545 if text:
546 doc.append_line(text)
547 line = self.get_doc_indented(doc)
548 no_more_args = True
549 elif line.startswith('='):
550 raise QAPIParseError(
551 self,
552 "unexpected '=' markup in definition documentation")
553 else:
554 # tag-less paragraph
555 doc.ensure_untagged_section()
556 doc.append_line(line)
557 line = self.get_doc_paragraph(doc)
558 else:
559 # Free-form documentation
560 doc = QAPIDoc(self, info)
561 doc.ensure_untagged_section()
562 first = True
563 while line is not None:
564 if match := self._match_at_name_colon(line):
565 raise QAPIParseError(
566 self,
567 "'@%s:' not allowed in free-form documentation"
568 % match.group(1))
569 if line.startswith('='):
570 if not first:
571 raise QAPIParseError(
572 self,
573 "'=' heading must come first in a comment block")
574 doc.append_line(line)
575 self.accept(False)
576 line = self.get_doc_line()
577 first = False
579 self.accept(False)
580 doc.end()
581 return doc
584 class QAPIDoc:
586 A documentation comment block, either definition or free-form
588 Definition documentation blocks consist of
590 * a body section: one line naming the definition, followed by an
591 overview (any number of lines)
593 * argument sections: a description of each argument (for commands
594 and events) or member (for structs, unions and alternates)
596 * features sections: a description of each feature flag
598 * additional (non-argument) sections, possibly tagged
600 Free-form documentation blocks consist only of a body section.
603 class Section:
604 def __init__(self, parser: QAPISchemaParser,
605 tag: Optional[str] = None):
606 # section source info, i.e. where it begins
607 self.info = parser.info
608 # parser, for error messages about indentation
609 self._parser = parser
610 # section tag, if any ('Returns', '@name', ...)
611 self.tag = tag
612 # section text without tag
613 self.text = ''
615 def append_line(self, line: str) -> None:
616 self.text += line + '\n'
618 class ArgSection(Section):
619 def __init__(self, parser: QAPISchemaParser,
620 tag: str):
621 super().__init__(parser, tag)
622 self.member: Optional['QAPISchemaMember'] = None
624 def connect(self, member: 'QAPISchemaMember') -> None:
625 self.member = member
627 def __init__(self, parser: QAPISchemaParser, info: QAPISourceInfo,
628 symbol: Optional[str] = None):
629 # self._parser is used to report errors with QAPIParseError. The
630 # resulting error position depends on the state of the parser.
631 # It happens to be the beginning of the comment. More or less
632 # servicable, but action at a distance.
633 self._parser = parser
634 # info points to the doc comment block's first line
635 self.info = info
636 # definition doc's symbol, None for free-form doc
637 self.symbol: Optional[str] = symbol
638 # the sections in textual order
639 self.all_sections: List[QAPIDoc.Section] = [QAPIDoc.Section(parser)]
640 # the body section
641 self.body: Optional[QAPIDoc.Section] = self.all_sections[0]
642 # dicts mapping parameter/feature names to their description
643 self.args: Dict[str, QAPIDoc.ArgSection] = {}
644 self.features: Dict[str, QAPIDoc.ArgSection] = {}
645 # sections other than .body, .args, .features
646 self.sections: List[QAPIDoc.Section] = []
648 def end(self) -> None:
649 for section in self.all_sections:
650 section.text = section.text.strip('\n')
651 if section.tag is not None and section.text == '':
652 raise QAPISemError(
653 section.info, "text required after '%s:'" % section.tag)
655 def ensure_untagged_section(self) -> None:
656 if self.all_sections and not self.all_sections[-1].tag:
657 # extend current section
658 self.all_sections[-1].text += '\n'
659 return
660 # start new section
661 section = self.Section(self._parser)
662 self.sections.append(section)
663 self.all_sections.append(section)
665 def new_tagged_section(self, tag: str) -> None:
666 if tag in ('Returns', 'Since'):
667 for section in self.all_sections:
668 if isinstance(section, self.ArgSection):
669 continue
670 if section.tag == tag:
671 raise QAPIParseError(
672 self._parser, "duplicated '%s' section" % tag)
673 section = self.Section(self._parser, tag)
674 self.sections.append(section)
675 self.all_sections.append(section)
677 def _new_description(self, name: str,
678 desc: Dict[str, ArgSection]) -> None:
679 if not name:
680 raise QAPIParseError(self._parser, "invalid parameter name")
681 if name in desc:
682 raise QAPIParseError(self._parser,
683 "'%s' parameter name duplicated" % name)
684 section = self.ArgSection(self._parser, '@' + name)
685 self.all_sections.append(section)
686 desc[name] = section
688 def new_argument(self, name: str) -> None:
689 self._new_description(name, self.args)
691 def new_feature(self, name: str) -> None:
692 self._new_description(name, self.features)
694 def append_line(self, line: str) -> None:
695 self.all_sections[-1].append_line(line)
697 def connect_member(self, member: 'QAPISchemaMember') -> None:
698 if member.name not in self.args:
699 if self.symbol not in member.info.pragma.documentation_exceptions:
700 raise QAPISemError(member.info,
701 "%s '%s' lacks documentation"
702 % (member.role, member.name))
703 self.args[member.name] = QAPIDoc.ArgSection(
704 self._parser, '@' + member.name)
705 self.args[member.name].connect(member)
707 def connect_feature(self, feature: 'QAPISchemaFeature') -> None:
708 if feature.name not in self.features:
709 raise QAPISemError(feature.info,
710 "feature '%s' lacks documentation"
711 % feature.name)
712 self.features[feature.name].connect(feature)
714 def check_expr(self, expr: QAPIExpression) -> None:
715 if 'command' not in expr:
716 sec = next((sec for sec in self.sections
717 if sec.tag == 'Returns'),
718 None)
719 if sec:
720 raise QAPISemError(sec.info,
721 "'Returns:' is only valid for commands")
723 def check(self) -> None:
725 def check_args_section(
726 args: Dict[str, QAPIDoc.ArgSection], what: str
727 ) -> None:
728 bogus = [name for name, section in args.items()
729 if not section.member]
730 if bogus:
731 raise QAPISemError(
732 args[bogus[0]].info,
733 "documented %s%s '%s' %s not exist" % (
734 what,
735 "s" if len(bogus) > 1 else "",
736 "', '".join(bogus),
737 "do" if len(bogus) > 1 else "does"
740 check_args_section(self.args, 'member')
741 check_args_section(self.features, 'feature')