1 # -*- coding: utf-8 -*-
5 # Copyright IBM, Corp. 2011
6 # Copyright (c) 2013-2019 Red Hat Inc.
9 # Anthony Liguori <aliguori@us.ibm.com>
10 # Markus Armbruster <armbru@redhat.com>
11 # Marc-André Lureau <marcandre.lureau@redhat.com>
12 # Kevin Wolf <kwolf@redhat.com>
14 # This work is licensed under the terms of the GNU GPL, version 2.
15 # See the COPYING file in the top-level directory.
17 from collections
import OrderedDict
28 from .common
import must_match
29 from .error
import QAPISemError
, QAPISourceError
30 from .source
import QAPISourceInfo
33 # Return value alias for get_expr().
34 _ExprValue
= Union
[List
[object], Dict
[str, object], str, bool]
37 class QAPIParseError(QAPISourceError
):
38 """Error class for all QAPI schema parsing errors."""
39 def __init__(self
, parser
: 'QAPISchemaParser', msg
: str):
41 for ch
in parser
.src
[parser
.line_pos
:parser
.pos
]:
43 col
= (col
+ 7) % 8 + 1
46 super().__init
__(parser
.info
, msg
, col
)
49 class QAPISchemaParser
:
51 Parse QAPI schema source.
53 Parse a JSON-esque schema file and process directives. See
54 qapi-code-gen.txt section "Schema Syntax" for the exact syntax.
55 Grammatical validation is handled later by `expr.check_exprs()`.
57 :param fname: Source file name.
58 :param previously_included:
59 The absolute names of previously included source files,
60 if being invoked from another parser.
62 `QAPISourceInfo` belonging to the parent module.
63 ``None`` implies this is the root module.
65 :ivar exprs: Resulting parsed expressions.
66 :ivar docs: Resulting parsed documentation blocks.
68 :raise OSError: For problems reading the root schema document.
69 :raise QAPIError: For errors in the schema source.
73 previously_included
: Optional
[Set
[str]] = None,
74 incl_info
: Optional
[QAPISourceInfo
] = None):
76 self
._included
= previously_included
or set()
77 self
._included
.add(os
.path
.abspath(self
._fname
))
80 # Lexer state (see `accept` for details):
81 self
.info
= QAPISourceInfo(self
._fname
, incl_info
)
82 self
.tok
: Union
[None, str] = None
85 self
.val
: Optional
[Union
[bool, str]] = None
89 self
.exprs
: List
[Dict
[str, object]] = []
90 self
.docs
: List
[QAPIDoc
] = []
95 def _parse(self
) -> None:
97 Parse the QAPI schema document.
99 :return: None. Results are stored in ``.exprs`` and ``.docs``.
103 # May raise OSError; allow the caller to handle it.
104 with
open(self
._fname
, 'r', encoding
='utf-8') as fp
:
106 if self
.src
== '' or self
.src
[-1] != '\n':
113 while self
.tok
is not None:
116 self
.reject_expr_doc(cur_doc
)
117 for cur_doc
in self
.get_doc(info
):
118 self
.docs
.append(cur_doc
)
121 expr
= self
.get_expr()
122 if not isinstance(expr
, dict):
124 info
, "top-level expression must be an object")
126 if 'include' in expr
:
127 self
.reject_expr_doc(cur_doc
)
129 raise QAPISemError(info
, "invalid 'include' directive")
130 include
= expr
['include']
131 if not isinstance(include
, str):
132 raise QAPISemError(info
,
133 "value of 'include' must be a string")
134 incl_fname
= os
.path
.join(os
.path
.dirname(self
._fname
),
136 self
.exprs
.append({'expr': {'include': incl_fname
},
138 exprs_include
= self
._include
(include
, info
, incl_fname
,
141 self
.exprs
.extend(exprs_include
.exprs
)
142 self
.docs
.extend(exprs_include
.docs
)
143 elif "pragma" in expr
:
144 self
.reject_expr_doc(cur_doc
)
146 raise QAPISemError(info
, "invalid 'pragma' directive")
147 pragma
= expr
['pragma']
148 if not isinstance(pragma
, dict):
150 info
, "value of 'pragma' must be an object")
151 for name
, value
in pragma
.items():
152 self
._pragma
(name
, value
, info
)
154 expr_elem
= {'expr': expr
,
157 if not cur_doc
.symbol
:
159 cur_doc
.info
, "definition documentation required")
160 expr_elem
['doc'] = cur_doc
161 self
.exprs
.append(expr_elem
)
163 self
.reject_expr_doc(cur_doc
)
166 def reject_expr_doc(doc
: Optional
['QAPIDoc']) -> None:
167 if doc
and doc
.symbol
:
170 "documentation for '%s' is not followed by the definition"
174 def _include(include
: str,
175 info
: QAPISourceInfo
,
177 previously_included
: Set
[str]
178 ) -> Optional
['QAPISchemaParser']:
179 incl_abs_fname
= os
.path
.abspath(incl_fname
)
180 # catch inclusion cycle
181 inf
: Optional
[QAPISourceInfo
] = info
183 if incl_abs_fname
== os
.path
.abspath(inf
.fname
):
184 raise QAPISemError(info
, "inclusion loop for %s" % include
)
187 # skip multiple include of the same file
188 if incl_abs_fname
in previously_included
:
192 return QAPISchemaParser(incl_fname
, previously_included
, info
)
193 except OSError as err
:
196 f
"can't read include file '{incl_fname}': {err.strerror}"
200 def _pragma(name
: str, value
: object, info
: QAPISourceInfo
) -> None:
202 def check_list_str(name
: str, value
: object) -> List
[str]:
203 if (not isinstance(value
, list) or
204 any(not isinstance(elt
, str) for elt
in value
)):
207 "pragma %s must be a list of strings" % name
)
212 if name
== 'doc-required':
213 if not isinstance(value
, bool):
214 raise QAPISemError(info
,
215 "pragma 'doc-required' must be boolean")
216 pragma
.doc_required
= value
217 elif name
== 'command-name-exceptions':
218 pragma
.command_name_exceptions
= check_list_str(name
, value
)
219 elif name
== 'command-returns-exceptions':
220 pragma
.command_returns_exceptions
= check_list_str(name
, value
)
221 elif name
== 'member-name-exceptions':
222 pragma
.member_name_exceptions
= check_list_str(name
, value
)
224 raise QAPISemError(info
, "unknown pragma '%s'" % name
)
226 def accept(self
, skip_comment
: bool = True) -> None:
228 Read and store the next token.
231 When false, return COMMENT tokens ("#").
232 This is used when reading documentation blocks.
235 None. Several instance attributes are updated instead:
237 - ``.tok`` represents the token type. See below for values.
238 - ``.info`` describes the token's source location.
239 - ``.val`` is the token's value, if any. See below.
240 - ``.pos`` is the buffer index of the first character of
243 * Single-character tokens:
245 These are "{", "}", ":", ",", "[", and "]".
246 ``.tok`` holds the single character and ``.val`` is None.
248 * Multi-character tokens:
252 This token is not normally returned by the lexer, but it can
253 be when ``skip_comment`` is False. ``.tok`` is "#", and
254 ``.val`` is a string including all chars until end-of-line,
255 including the "#" itself.
259 ``.tok`` is "'", the single quote. ``.val`` contains the
260 string, excluding the surrounding quotes.
264 ``.tok`` is either "t" or "f", ``.val`` will be the
265 corresponding bool value.
269 ``.tok`` and ``.val`` will both be None at EOF.
272 self
.tok
= self
.src
[self
.cursor
]
273 self
.pos
= self
.cursor
278 if self
.src
[self
.cursor
] == '#':
279 # Start of doc comment
281 self
.cursor
= self
.src
.find('\n', self
.cursor
)
283 self
.val
= self
.src
[self
.pos
:self
.cursor
]
285 elif self
.tok
in '{}:,[]':
287 elif self
.tok
== "'":
288 # Note: we accept only printable ASCII
292 ch
= self
.src
[self
.cursor
]
295 raise QAPIParseError(self
, "missing terminating \"'\"")
297 # Note: we recognize only \\ because we have
298 # no use for funny characters in strings
300 raise QAPIParseError(self
,
301 "unknown escape \\%s" % ch
)
309 if ord(ch
) < 32 or ord(ch
) >= 127:
310 raise QAPIParseError(
311 self
, "funny character in string")
313 elif self
.src
.startswith('true', self
.pos
):
317 elif self
.src
.startswith('false', self
.pos
):
321 elif self
.tok
== '\n':
322 if self
.cursor
== len(self
.src
):
325 self
.info
= self
.info
.next_line()
326 self
.line_pos
= self
.cursor
327 elif not self
.tok
.isspace():
328 # Show up to next structural, whitespace or quote
330 match
= must_match('[^[\\]{}:,\\s\'"]+',
331 self
.src
[self
.cursor
-1:])
332 raise QAPIParseError(self
, "stray '%s'" % match
.group(0))
334 def get_members(self
) -> Dict
[str, object]:
335 expr
: Dict
[str, object] = OrderedDict()
340 raise QAPIParseError(self
, "expected string or '}'")
343 assert isinstance(key
, str) # Guaranteed by tok == "'"
347 raise QAPIParseError(self
, "expected ':'")
350 raise QAPIParseError(self
, "duplicate key '%s'" % key
)
351 expr
[key
] = self
.get_expr()
356 raise QAPIParseError(self
, "expected ',' or '}'")
359 raise QAPIParseError(self
, "expected string")
361 def get_values(self
) -> List
[object]:
362 expr
: List
[object] = []
366 if self
.tok
not in tuple("{['tf"):
367 raise QAPIParseError(
368 self
, "expected '{', '[', ']', string, or boolean")
370 expr
.append(self
.get_expr())
375 raise QAPIParseError(self
, "expected ',' or ']'")
378 def get_expr(self
) -> _ExprValue
:
382 expr
= self
.get_members()
383 elif self
.tok
== '[':
385 expr
= self
.get_values()
386 elif self
.tok
in tuple("'tf"):
387 assert isinstance(self
.val
, (str, bool))
391 raise QAPIParseError(
392 self
, "expected '{', '[', string, or boolean")
395 def get_doc(self
, info
: QAPISourceInfo
) -> List
['QAPIDoc']:
397 raise QAPIParseError(
398 self
, "junk after '##' at start of documentation comment")
401 cur_doc
= QAPIDoc(self
, info
)
403 while self
.tok
== '#':
404 assert isinstance(self
.val
, str)
405 if self
.val
.startswith('##'):
408 raise QAPIParseError(
410 "junk after '##' at end of documentation comment")
411 cur_doc
.end_comment()
415 if self
.val
.startswith('# ='):
417 raise QAPIParseError(
419 "unexpected '=' markup in definition documentation")
420 if cur_doc
.body
.text
:
421 cur_doc
.end_comment()
423 cur_doc
= QAPIDoc(self
, info
)
424 cur_doc
.append(self
.val
)
427 raise QAPIParseError(self
, "documentation comment must end with '##'")
432 A documentation comment block, either definition or free-form
434 Definition documentation blocks consist of
436 * a body section: one line naming the definition, followed by an
437 overview (any number of lines)
439 * argument sections: a description of each argument (for commands
440 and events) or member (for structs, unions and alternates)
442 * features sections: a description of each feature flag
444 * additional (non-argument) sections, possibly tagged
446 Free-form documentation blocks consist only of a body section.
450 def __init__(self
, parser
, name
=None, indent
=0):
451 # parser, for error messages about indentation
452 self
._parser
= parser
453 # optional section name (argument/member or section name)
456 # the expected indent level of the text of this section
457 self
._indent
= indent
459 def append(self
, line
):
460 # Strip leading spaces corresponding to the expected indent level
461 # Blank lines are always OK.
463 indent
= must_match(r
'\s*', line
).end()
464 if indent
< self
._indent
:
465 raise QAPIParseError(
467 "unexpected de-indent (expected at least %d spaces)" %
469 line
= line
[self
._indent
:]
471 self
.text
+= line
.rstrip() + '\n'
473 class ArgSection(Section
):
474 def __init__(self
, parser
, name
, indent
=0):
475 super().__init
__(parser
, name
, indent
)
478 def connect(self
, member
):
481 def __init__(self
, parser
, info
):
482 # self._parser is used to report errors with QAPIParseError. The
483 # resulting error position depends on the state of the parser.
484 # It happens to be the beginning of the comment. More or less
485 # servicable, but action at a distance.
486 self
._parser
= parser
489 self
.body
= QAPIDoc
.Section(parser
)
490 # dict mapping parameter name to ArgSection
491 self
.args
= OrderedDict()
492 self
.features
= OrderedDict()
495 # the current section
496 self
._section
= self
.body
497 self
._append
_line
= self
._append
_body
_line
499 def has_section(self
, name
):
500 """Return True if we have a section with this name."""
501 for i
in self
.sections
:
506 def append(self
, line
):
508 Parse a comment line and add it to the documentation.
510 The way that the line is dealt with depends on which part of
511 the documentation we're parsing right now:
512 * The body section: ._append_line is ._append_body_line
513 * An argument section: ._append_line is ._append_args_line
514 * A features section: ._append_line is ._append_features_line
515 * An additional section: ._append_line is ._append_various_line
519 self
._append
_freeform
(line
)
523 raise QAPIParseError(self
._parser
, "missing space after #")
525 self
._append
_line
(line
)
527 def end_comment(self
):
531 def _is_section_tag(name
):
532 return name
in ('Returns:', 'Since:',
533 # those are often singular or plural
535 'Example:', 'Examples:',
538 def _append_body_line(self
, line
):
540 Process a line of documentation text in the body section.
542 If this a symbol line and it is the section's first line, this
543 is a definition documentation block for that symbol.
545 If it's a definition documentation block, another symbol line
546 begins the argument section for the argument named by it, and
547 a section tag begins an additional section. Start that
548 section and append the line to it.
550 Else, append the line to the current section.
552 name
= line
.split(' ', 1)[0]
553 # FIXME not nice: things like '# @foo:' and '# @foo: ' aren't
554 # recognized, and get silently treated as ordinary text
555 if not self
.symbol
and not self
.body
.text
and line
.startswith('@'):
556 if not line
.endswith(':'):
557 raise QAPIParseError(self
._parser
, "line should end with ':'")
558 self
.symbol
= line
[1:-1]
559 # FIXME invalid names other than the empty string aren't flagged
561 raise QAPIParseError(self
._parser
, "invalid name")
563 # This is a definition documentation block
564 if name
.startswith('@') and name
.endswith(':'):
565 self
._append
_line
= self
._append
_args
_line
566 self
._append
_args
_line
(line
)
567 elif line
== 'Features:':
568 self
._append
_line
= self
._append
_features
_line
569 elif self
._is
_section
_tag
(name
):
570 self
._append
_line
= self
._append
_various
_line
571 self
._append
_various
_line
(line
)
573 self
._append
_freeform
(line
)
575 # This is a free-form documentation block
576 self
._append
_freeform
(line
)
578 def _append_args_line(self
, line
):
580 Process a line of documentation text in an argument section.
582 A symbol line begins the next argument section, a section tag
583 section or a non-indented line after a blank line begins an
584 additional section. Start that section and append the line to
587 Else, append the line to the current section.
590 name
= line
.split(' ', 1)[0]
592 if name
.startswith('@') and name
.endswith(':'):
593 # If line is "@arg: first line of description", find
594 # the index of 'f', which is the indent we expect for any
595 # following lines. We then remove the leading "@arg:"
596 # from line and replace it with spaces so that 'f' has the
597 # same index as it did in the original line and can be
598 # handled the same way we will handle following lines.
599 indent
= must_match(r
'@\S*:\s*', line
).end()
602 # Line was just the "@arg:" header; following lines
606 line
= ' ' * indent
+ line
607 self
._start
_args
_section
(name
[1:-1], indent
)
608 elif self
._is
_section
_tag
(name
):
609 self
._append
_line
= self
._append
_various
_line
610 self
._append
_various
_line
(line
)
612 elif (self
._section
.text
.endswith('\n\n')
613 and line
and not line
[0].isspace()):
614 if line
== 'Features:':
615 self
._append
_line
= self
._append
_features
_line
617 self
._start
_section
()
618 self
._append
_line
= self
._append
_various
_line
619 self
._append
_various
_line
(line
)
622 self
._append
_freeform
(line
)
624 def _append_features_line(self
, line
):
625 name
= line
.split(' ', 1)[0]
627 if name
.startswith('@') and name
.endswith(':'):
628 # If line is "@arg: first line of description", find
629 # the index of 'f', which is the indent we expect for any
630 # following lines. We then remove the leading "@arg:"
631 # from line and replace it with spaces so that 'f' has the
632 # same index as it did in the original line and can be
633 # handled the same way we will handle following lines.
634 indent
= must_match(r
'@\S*:\s*', line
).end()
637 # Line was just the "@arg:" header; following lines
641 line
= ' ' * indent
+ line
642 self
._start
_features
_section
(name
[1:-1], indent
)
643 elif self
._is
_section
_tag
(name
):
644 self
._append
_line
= self
._append
_various
_line
645 self
._append
_various
_line
(line
)
647 elif (self
._section
.text
.endswith('\n\n')
648 and line
and not line
[0].isspace()):
649 self
._start
_section
()
650 self
._append
_line
= self
._append
_various
_line
651 self
._append
_various
_line
(line
)
654 self
._append
_freeform
(line
)
656 def _append_various_line(self
, line
):
658 Process a line of documentation text in an additional section.
660 A symbol line is an error.
662 A section tag begins an additional section. Start that
663 section and append the line to it.
665 Else, append the line to the current section.
667 name
= line
.split(' ', 1)[0]
669 if name
.startswith('@') and name
.endswith(':'):
670 raise QAPIParseError(self
._parser
,
671 "'%s' can't follow '%s' section"
672 % (name
, self
.sections
[0].name
))
673 if self
._is
_section
_tag
(name
):
674 # If line is "Section: first line of description", find
675 # the index of 'f', which is the indent we expect for any
676 # following lines. We then remove the leading "Section:"
677 # from line and replace it with spaces so that 'f' has the
678 # same index as it did in the original line and can be
679 # handled the same way we will handle following lines.
680 indent
= must_match(r
'\S*:\s*', line
).end()
683 # Line was just the "Section:" header; following lines
687 line
= ' ' * indent
+ line
688 self
._start
_section
(name
[:-1], indent
)
690 self
._append
_freeform
(line
)
692 def _start_symbol_section(self
, symbols_dict
, name
, indent
):
693 # FIXME invalid names other than the empty string aren't flagged
695 raise QAPIParseError(self
._parser
, "invalid parameter name")
696 if name
in symbols_dict
:
697 raise QAPIParseError(self
._parser
,
698 "'%s' parameter name duplicated" % name
)
699 assert not self
.sections
701 self
._section
= QAPIDoc
.ArgSection(self
._parser
, name
, indent
)
702 symbols_dict
[name
] = self
._section
704 def _start_args_section(self
, name
, indent
):
705 self
._start
_symbol
_section
(self
.args
, name
, indent
)
707 def _start_features_section(self
, name
, indent
):
708 self
._start
_symbol
_section
(self
.features
, name
, indent
)
710 def _start_section(self
, name
=None, indent
=0):
711 if name
in ('Returns', 'Since') and self
.has_section(name
):
712 raise QAPIParseError(self
._parser
,
713 "duplicated '%s' section" % name
)
715 self
._section
= QAPIDoc
.Section(self
._parser
, name
, indent
)
716 self
.sections
.append(self
._section
)
718 def _end_section(self
):
720 text
= self
._section
.text
= self
._section
.text
.strip()
721 if self
._section
.name
and (not text
or text
.isspace()):
722 raise QAPIParseError(
724 "empty doc section '%s'" % self
._section
.name
)
727 def _append_freeform(self
, line
):
728 match
= re
.match(r
'(@\S+:)', line
)
730 raise QAPIParseError(self
._parser
,
731 "'%s' not allowed in free-form documentation"
733 self
._section
.append(line
)
735 def connect_member(self
, member
):
736 if member
.name
not in self
.args
:
737 # Undocumented TODO outlaw
738 self
.args
[member
.name
] = QAPIDoc
.ArgSection(self
._parser
,
740 self
.args
[member
.name
].connect(member
)
742 def connect_feature(self
, feature
):
743 if feature
.name
not in self
.features
:
744 raise QAPISemError(feature
.info
,
745 "feature '%s' lacks documentation"
747 self
.features
[feature
.name
].connect(feature
)
749 def check_expr(self
, expr
):
750 if self
.has_section('Returns') and 'command' not in expr
:
751 raise QAPISemError(self
.info
,
752 "'Returns:' is only valid for commands")
756 def check_args_section(args
, info
, what
):
757 bogus
= [name
for name
, section
in args
.items()
758 if not section
.member
]
762 "documented member%s '%s' %s not exist"
763 % ("s" if len(bogus
) > 1 else "",
765 "do" if len(bogus
) > 1 else "does"))
767 check_args_section(self
.args
, self
.info
, 'members')
768 check_args_section(self
.features
, self
.info
, 'features')