1 # -*- coding: utf-8 -*-
5 # Copyright IBM, Corp. 2011
6 # Copyright (c) 2013-2019 Red Hat Inc.
9 # Anthony Liguori <aliguori@us.ibm.com>
10 # Markus Armbruster <armbru@redhat.com>
11 # Marc-André Lureau <marcandre.lureau@redhat.com>
12 # Kevin Wolf <kwolf@redhat.com>
14 # This work is licensed under the terms of the GNU GPL, version 2.
15 # See the COPYING file in the top-level directory.
17 from collections
import OrderedDict
32 from .common
import must_match
33 from .error
import QAPISemError
, QAPISourceError
34 from .source
import QAPISourceInfo
38 # pylint: disable=cyclic-import
39 # TODO: Remove cycle. [schema -> expr -> parser -> schema]
40 from .schema
import QAPISchemaFeature
, QAPISchemaMember
43 # Return value alias for get_expr().
44 _ExprValue
= Union
[List
[object], Dict
[str, object], str, bool]
47 class QAPIExpression(Dict
[str, Any
]):
48 # pylint: disable=too-few-public-methods
50 data
: Mapping
[str, object],
52 doc
: Optional
['QAPIDoc'] = None):
53 super().__init
__(data
)
55 self
.doc
: Optional
['QAPIDoc'] = doc
58 class QAPIParseError(QAPISourceError
):
59 """Error class for all QAPI schema parsing errors."""
60 def __init__(self
, parser
: 'QAPISchemaParser', msg
: str):
62 for ch
in parser
.src
[parser
.line_pos
:parser
.pos
]:
64 col
= (col
+ 7) % 8 + 1
67 super().__init
__(parser
.info
, msg
, col
)
70 class QAPISchemaParser
:
72 Parse QAPI schema source.
74 Parse a JSON-esque schema file and process directives. See
75 qapi-code-gen.rst section "Schema Syntax" for the exact syntax.
76 Grammatical validation is handled later by `expr.check_exprs()`.
78 :param fname: Source file name.
79 :param previously_included:
80 The absolute names of previously included source files,
81 if being invoked from another parser.
83 `QAPISourceInfo` belonging to the parent module.
84 ``None`` implies this is the root module.
86 :ivar exprs: Resulting parsed expressions.
87 :ivar docs: Resulting parsed documentation blocks.
89 :raise OSError: For problems reading the root schema document.
90 :raise QAPIError: For errors in the schema source.
94 previously_included
: Optional
[Set
[str]] = None,
95 incl_info
: Optional
[QAPISourceInfo
] = None):
97 self
._included
= previously_included
or set()
98 self
._included
.add(os
.path
.abspath(self
._fname
))
101 # Lexer state (see `accept` for details):
102 self
.info
= QAPISourceInfo(self
._fname
, incl_info
)
103 self
.tok
: Union
[None, str] = None
106 self
.val
: Optional
[Union
[bool, str]] = None
110 self
.exprs
: List
[QAPIExpression
] = []
111 self
.docs
: List
[QAPIDoc
] = []
116 def _parse(self
) -> None:
118 Parse the QAPI schema document.
120 :return: None. Results are stored in ``.exprs`` and ``.docs``.
124 # May raise OSError; allow the caller to handle it.
125 with
open(self
._fname
, 'r', encoding
='utf-8') as fp
:
127 if self
.src
== '' or self
.src
[-1] != '\n':
134 while self
.tok
is not None:
137 self
.reject_expr_doc(cur_doc
)
138 cur_doc
= self
.get_doc()
139 self
.docs
.append(cur_doc
)
142 expr
= self
.get_expr()
143 if not isinstance(expr
, dict):
145 info
, "top-level expression must be an object")
147 if 'include' in expr
:
148 self
.reject_expr_doc(cur_doc
)
150 raise QAPISemError(info
, "invalid 'include' directive")
151 include
= expr
['include']
152 if not isinstance(include
, str):
153 raise QAPISemError(info
,
154 "value of 'include' must be a string")
155 incl_fname
= os
.path
.join(os
.path
.dirname(self
._fname
),
157 self
._add
_expr
(OrderedDict({'include': incl_fname
}), info
)
158 exprs_include
= self
._include
(include
, info
, incl_fname
,
161 self
.exprs
.extend(exprs_include
.exprs
)
162 self
.docs
.extend(exprs_include
.docs
)
163 elif "pragma" in expr
:
164 self
.reject_expr_doc(cur_doc
)
166 raise QAPISemError(info
, "invalid 'pragma' directive")
167 pragma
= expr
['pragma']
168 if not isinstance(pragma
, dict):
170 info
, "value of 'pragma' must be an object")
171 for name
, value
in pragma
.items():
172 self
._pragma
(name
, value
, info
)
174 if cur_doc
and not cur_doc
.symbol
:
176 cur_doc
.info
, "definition documentation required")
177 self
._add
_expr
(expr
, info
, cur_doc
)
179 self
.reject_expr_doc(cur_doc
)
181 def _add_expr(self
, expr
: Mapping
[str, object],
182 info
: QAPISourceInfo
,
183 doc
: Optional
['QAPIDoc'] = None) -> None:
184 self
.exprs
.append(QAPIExpression(expr
, info
, doc
))
187 def reject_expr_doc(doc
: Optional
['QAPIDoc']) -> None:
188 if doc
and doc
.symbol
:
191 "documentation for '%s' is not followed by the definition"
195 def _include(include
: str,
196 info
: QAPISourceInfo
,
198 previously_included
: Set
[str]
199 ) -> Optional
['QAPISchemaParser']:
200 incl_abs_fname
= os
.path
.abspath(incl_fname
)
201 # catch inclusion cycle
202 inf
: Optional
[QAPISourceInfo
] = info
204 if incl_abs_fname
== os
.path
.abspath(inf
.fname
):
205 raise QAPISemError(info
, "inclusion loop for %s" % include
)
208 # skip multiple include of the same file
209 if incl_abs_fname
in previously_included
:
213 return QAPISchemaParser(incl_fname
, previously_included
, info
)
214 except OSError as err
:
217 f
"can't read include file '{incl_fname}': {err.strerror}"
221 def _pragma(name
: str, value
: object, info
: QAPISourceInfo
) -> None:
223 def check_list_str(name
: str, value
: object) -> List
[str]:
224 if (not isinstance(value
, list) or
225 any(not isinstance(elt
, str) for elt
in value
)):
228 "pragma %s must be a list of strings" % name
)
233 if name
== 'doc-required':
234 if not isinstance(value
, bool):
235 raise QAPISemError(info
,
236 "pragma 'doc-required' must be boolean")
237 pragma
.doc_required
= value
238 elif name
== 'command-name-exceptions':
239 pragma
.command_name_exceptions
= check_list_str(name
, value
)
240 elif name
== 'command-returns-exceptions':
241 pragma
.command_returns_exceptions
= check_list_str(name
, value
)
242 elif name
== 'documentation-exceptions':
243 pragma
.documentation_exceptions
= check_list_str(name
, value
)
244 elif name
== 'member-name-exceptions':
245 pragma
.member_name_exceptions
= check_list_str(name
, value
)
247 raise QAPISemError(info
, "unknown pragma '%s'" % name
)
249 def accept(self
, skip_comment
: bool = True) -> None:
251 Read and store the next token.
254 When false, return COMMENT tokens ("#").
255 This is used when reading documentation blocks.
258 None. Several instance attributes are updated instead:
260 - ``.tok`` represents the token type. See below for values.
261 - ``.info`` describes the token's source location.
262 - ``.val`` is the token's value, if any. See below.
263 - ``.pos`` is the buffer index of the first character of
266 * Single-character tokens:
268 These are "{", "}", ":", ",", "[", and "]".
269 ``.tok`` holds the single character and ``.val`` is None.
271 * Multi-character tokens:
275 This token is not normally returned by the lexer, but it can
276 be when ``skip_comment`` is False. ``.tok`` is "#", and
277 ``.val`` is a string including all chars until end-of-line,
278 including the "#" itself.
282 ``.tok`` is "'", the single quote. ``.val`` contains the
283 string, excluding the surrounding quotes.
287 ``.tok`` is either "t" or "f", ``.val`` will be the
288 corresponding bool value.
292 ``.tok`` and ``.val`` will both be None at EOF.
295 self
.tok
= self
.src
[self
.cursor
]
296 self
.pos
= self
.cursor
301 if self
.src
[self
.cursor
] == '#':
302 # Start of doc comment
304 self
.cursor
= self
.src
.find('\n', self
.cursor
)
306 self
.val
= self
.src
[self
.pos
:self
.cursor
]
308 elif self
.tok
in '{}:,[]':
310 elif self
.tok
== "'":
311 # Note: we accept only printable ASCII
315 ch
= self
.src
[self
.cursor
]
318 raise QAPIParseError(self
, "missing terminating \"'\"")
320 # Note: we recognize only \\ because we have
321 # no use for funny characters in strings
323 raise QAPIParseError(self
,
324 "unknown escape \\%s" % ch
)
332 if ord(ch
) < 32 or ord(ch
) >= 127:
333 raise QAPIParseError(
334 self
, "funny character in string")
336 elif self
.src
.startswith('true', self
.pos
):
340 elif self
.src
.startswith('false', self
.pos
):
344 elif self
.tok
== '\n':
345 if self
.cursor
== len(self
.src
):
348 self
.info
= self
.info
.next_line()
349 self
.line_pos
= self
.cursor
350 elif not self
.tok
.isspace():
351 # Show up to next structural, whitespace or quote
353 match
= must_match('[^[\\]{}:,\\s\']+',
354 self
.src
[self
.cursor
-1:])
355 raise QAPIParseError(self
, "stray '%s'" % match
.group(0))
357 def get_members(self
) -> Dict
[str, object]:
358 expr
: Dict
[str, object] = OrderedDict()
363 raise QAPIParseError(self
, "expected string or '}'")
366 assert isinstance(key
, str) # Guaranteed by tok == "'"
370 raise QAPIParseError(self
, "expected ':'")
373 raise QAPIParseError(self
, "duplicate key '%s'" % key
)
374 expr
[key
] = self
.get_expr()
379 raise QAPIParseError(self
, "expected ',' or '}'")
382 raise QAPIParseError(self
, "expected string")
384 def get_values(self
) -> List
[object]:
385 expr
: List
[object] = []
389 if self
.tok
not in tuple("{['tf"):
390 raise QAPIParseError(
391 self
, "expected '{', '[', ']', string, or boolean")
393 expr
.append(self
.get_expr())
398 raise QAPIParseError(self
, "expected ',' or ']'")
401 def get_expr(self
) -> _ExprValue
:
405 expr
= self
.get_members()
406 elif self
.tok
== '[':
408 expr
= self
.get_values()
409 elif self
.tok
in tuple("'tf"):
410 assert isinstance(self
.val
, (str, bool))
414 raise QAPIParseError(
415 self
, "expected '{', '[', string, or boolean")
418 def get_doc_line(self
) -> Optional
[str]:
420 raise QAPIParseError(
421 self
, "documentation comment must end with '##'")
422 assert isinstance(self
.val
, str)
423 if self
.val
.startswith('##'):
426 raise QAPIParseError(
427 self
, "junk after '##' at end of documentation comment")
431 if self
.val
[1] != ' ':
432 raise QAPIParseError(self
, "missing space after #")
433 return self
.val
[2:].rstrip()
436 def _match_at_name_colon(string
: str) -> Optional
[Match
[str]]:
437 return re
.match(r
'@([^:]*): *', string
)
439 def get_doc_indented(self
, doc
: 'QAPIDoc') -> Optional
[str]:
441 line
= self
.get_doc_line()
443 doc
.append_line(line
)
445 line
= self
.get_doc_line()
448 indent
= must_match(r
'\s*', line
).end()
451 doc
.append_line(line
)
452 prev_line_blank
= False
455 line
= self
.get_doc_line()
458 if self
._match
_at
_name
_colon
(line
):
460 cur_indent
= must_match(r
'\s*', line
).end()
461 if line
!= '' and cur_indent
< indent
:
464 raise QAPIParseError(
466 "unexpected de-indent (expected at least %d spaces)" %
468 doc
.append_line(line
)
469 prev_line_blank
= True
471 def get_doc_paragraph(self
, doc
: 'QAPIDoc') -> Optional
[str]:
474 line
= self
.get_doc_line()
479 doc
.append_line(line
)
481 def get_doc(self
) -> 'QAPIDoc':
483 raise QAPIParseError(
484 self
, "junk after '##' at start of documentation comment")
487 line
= self
.get_doc_line()
488 if line
is not None and line
.startswith('@'):
489 # Definition documentation
490 if not line
.endswith(':'):
491 raise QAPIParseError(self
, "line should end with ':'")
492 # Invalid names are not checked here, but the name
493 # provided *must* match the following definition,
494 # which *is* validated in expr.py.
497 raise QAPIParseError(self
, "name required after '@'")
498 doc
= QAPIDoc(info
, symbol
)
500 line
= self
.get_doc_line()
503 while line
is not None:
507 line
= self
.get_doc_line()
510 # Non-blank line, first of a section
511 if line
== 'Features:':
513 raise QAPIParseError(
514 self
, "duplicated 'Features:' line")
516 line
= self
.get_doc_line()
519 line
= self
.get_doc_line()
520 while (line
is not None
521 and (match
:= self
._match
_at
_name
_colon
(line
))):
522 doc
.new_feature(self
.info
, match
.group(1))
523 text
= line
[match
.end():]
525 doc
.append_line(text
)
526 line
= self
.get_doc_indented(doc
)
528 raise QAPIParseError(
529 self
, 'feature descriptions expected')
531 elif match
:= self
._match
_at
_name
_colon
(line
):
534 raise QAPIParseError(
536 "description of '@%s:' follows a section"
538 while (line
is not None
539 and (match
:= self
._match
_at
_name
_colon
(line
))):
540 doc
.new_argument(self
.info
, match
.group(1))
541 text
= line
[match
.end():]
543 doc
.append_line(text
)
544 line
= self
.get_doc_indented(doc
)
546 elif match
:= re
.match(
547 r
'(Returns|Errors|Since|Notes?|Examples?|TODO)'
553 # Note: "sections" with two colons are left alone as
554 # rST markup and not interpreted as a section heading.
556 # TODO: Remove these errors sometime in 2025 or so
557 # after we've fully transitioned to the new qapidoc
560 # See commit message for more markup suggestions O:-)
561 if 'Note' in match
.group(1):
563 f
"The '{match.group(1)}' section is no longer "
564 "supported. Please use rST's '.. note::' or "
565 "'.. admonition:: notes' directives, or another "
566 "suitable admonition instead."
568 raise QAPIParseError(self
, emsg
)
570 if 'Example' in match
.group(1):
572 f
"The '{match.group(1)}' section is no longer "
573 "supported. Please use the '.. qmp-example::' "
574 "directive, or other suitable markup instead."
576 raise QAPIParseError(self
, emsg
)
578 doc
.new_tagged_section(self
.info
, match
.group(1))
579 text
= line
[match
.end():]
581 doc
.append_line(text
)
582 line
= self
.get_doc_indented(doc
)
584 elif line
.startswith('='):
585 raise QAPIParseError(
587 "unexpected '=' markup in definition documentation")
590 doc
.ensure_untagged_section(self
.info
)
591 doc
.append_line(line
)
592 line
= self
.get_doc_paragraph(doc
)
594 # Free-form documentation
596 doc
.ensure_untagged_section(self
.info
)
598 while line
is not None:
599 if match
:= self
._match
_at
_name
_colon
(line
):
600 raise QAPIParseError(
602 "'@%s:' not allowed in free-form documentation"
604 if line
.startswith('='):
606 raise QAPIParseError(
608 "'=' heading must come first in a comment block")
609 doc
.append_line(line
)
611 line
= self
.get_doc_line()
621 A documentation comment block, either definition or free-form
623 Definition documentation blocks consist of
625 * a body section: one line naming the definition, followed by an
626 overview (any number of lines)
628 * argument sections: a description of each argument (for commands
629 and events) or member (for structs, unions and alternates)
631 * features sections: a description of each feature flag
633 * additional (non-argument) sections, possibly tagged
635 Free-form documentation blocks consist only of a body section.
639 # pylint: disable=too-few-public-methods
640 def __init__(self
, info
: QAPISourceInfo
,
641 tag
: Optional
[str] = None):
642 # section source info, i.e. where it begins
644 # section tag, if any ('Returns', '@name', ...)
646 # section text without tag
649 def append_line(self
, line
: str) -> None:
650 self
.text
+= line
+ '\n'
652 class ArgSection(Section
):
653 def __init__(self
, info
: QAPISourceInfo
, tag
: str):
654 super().__init
__(info
, tag
)
655 self
.member
: Optional
['QAPISchemaMember'] = None
657 def connect(self
, member
: 'QAPISchemaMember') -> None:
660 def __init__(self
, info
: QAPISourceInfo
, symbol
: Optional
[str] = None):
661 # info points to the doc comment block's first line
663 # definition doc's symbol, None for free-form doc
664 self
.symbol
: Optional
[str] = symbol
665 # the sections in textual order
666 self
.all_sections
: List
[QAPIDoc
.Section
] = [QAPIDoc
.Section(info
)]
668 self
.body
: Optional
[QAPIDoc
.Section
] = self
.all_sections
[0]
669 # dicts mapping parameter/feature names to their description
670 self
.args
: Dict
[str, QAPIDoc
.ArgSection
] = {}
671 self
.features
: Dict
[str, QAPIDoc
.ArgSection
] = {}
672 # a command's "Returns" and "Errors" section
673 self
.returns
: Optional
[QAPIDoc
.Section
] = None
674 self
.errors
: Optional
[QAPIDoc
.Section
] = None
676 self
.since
: Optional
[QAPIDoc
.Section
] = None
677 # sections other than .body, .args, .features
678 self
.sections
: List
[QAPIDoc
.Section
] = []
680 def end(self
) -> None:
681 for section
in self
.all_sections
:
682 section
.text
= section
.text
.strip('\n')
683 if section
.tag
is not None and section
.text
== '':
685 section
.info
, "text required after '%s:'" % section
.tag
)
687 def ensure_untagged_section(self
, info
: QAPISourceInfo
) -> None:
688 if self
.all_sections
and not self
.all_sections
[-1].tag
:
689 # extend current section
690 self
.all_sections
[-1].text
+= '\n'
693 section
= self
.Section(info
)
694 self
.sections
.append(section
)
695 self
.all_sections
.append(section
)
697 def new_tagged_section(self
, info
: QAPISourceInfo
, tag
: str) -> None:
698 section
= self
.Section(info
, tag
)
702 info
, "duplicated '%s' section" % tag
)
703 self
.returns
= section
704 elif tag
== 'Errors':
707 info
, "duplicated '%s' section" % tag
)
708 self
.errors
= section
712 info
, "duplicated '%s' section" % tag
)
714 self
.sections
.append(section
)
715 self
.all_sections
.append(section
)
717 def _new_description(self
, info
: QAPISourceInfo
, name
: str,
718 desc
: Dict
[str, ArgSection
]) -> None:
720 raise QAPISemError(info
, "invalid parameter name")
722 raise QAPISemError(info
, "'%s' parameter name duplicated" % name
)
723 section
= self
.ArgSection(info
, '@' + name
)
724 self
.all_sections
.append(section
)
727 def new_argument(self
, info
: QAPISourceInfo
, name
: str) -> None:
728 self
._new
_description
(info
, name
, self
.args
)
730 def new_feature(self
, info
: QAPISourceInfo
, name
: str) -> None:
731 self
._new
_description
(info
, name
, self
.features
)
733 def append_line(self
, line
: str) -> None:
734 self
.all_sections
[-1].append_line(line
)
736 def connect_member(self
, member
: 'QAPISchemaMember') -> None:
737 if member
.name
not in self
.args
:
739 if self
.symbol
not in member
.info
.pragma
.documentation_exceptions
:
740 raise QAPISemError(member
.info
,
741 "%s '%s' lacks documentation"
742 % (member
.role
, member
.name
))
743 self
.args
[member
.name
] = QAPIDoc
.ArgSection(
744 self
.info
, '@' + member
.name
)
745 self
.args
[member
.name
].connect(member
)
747 def connect_feature(self
, feature
: 'QAPISchemaFeature') -> None:
748 if feature
.name
not in self
.features
:
749 raise QAPISemError(feature
.info
,
750 "feature '%s' lacks documentation"
752 self
.features
[feature
.name
].connect(feature
)
754 def check_expr(self
, expr
: QAPIExpression
) -> None:
755 if 'command' in expr
:
756 if self
.returns
and 'returns' not in expr
:
759 "'Returns' section, but command doesn't return anything")
764 "'Returns' section is only valid for commands")
768 "'Errors' section is only valid for commands")
770 def check(self
) -> None:
772 def check_args_section(
773 args
: Dict
[str, QAPIDoc
.ArgSection
], what
: str
775 bogus
= [name
for name
, section
in args
.items()
776 if not section
.member
]
780 "documented %s%s '%s' %s not exist" % (
782 "s" if len(bogus
) > 1 else "",
784 "do" if len(bogus
) > 1 else "does"
787 check_args_section(self
.args
, 'member')
788 check_args_section(self
.features
, 'feature')