1 # -*- coding: utf-8 -*-
5 # Copyright IBM, Corp. 2011
6 # Copyright (c) 2013-2019 Red Hat Inc.
9 # Anthony Liguori <aliguori@us.ibm.com>
10 # Markus Armbruster <armbru@redhat.com>
11 # Marc-André Lureau <marcandre.lureau@redhat.com>
12 # Kevin Wolf <kwolf@redhat.com>
14 # This work is licensed under the terms of the GNU GPL, version 2.
15 # See the COPYING file in the top-level directory.
17 from collections
import OrderedDict
31 from .common
import must_match
32 from .error
import QAPISemError
, QAPISourceError
33 from .source
import QAPISourceInfo
37 # pylint: disable=cyclic-import
38 # TODO: Remove cycle. [schema -> expr -> parser -> schema]
39 from .schema
import QAPISchemaFeature
, QAPISchemaMember
42 # Return value alias for get_expr().
43 _ExprValue
= Union
[List
[object], Dict
[str, object], str, bool]
46 class QAPIExpression(Dict
[str, object]):
47 # pylint: disable=too-few-public-methods
49 data
: Mapping
[str, object],
51 doc
: Optional
['QAPIDoc'] = None):
52 super().__init
__(data
)
54 self
.doc
: Optional
['QAPIDoc'] = doc
57 class QAPIParseError(QAPISourceError
):
58 """Error class for all QAPI schema parsing errors."""
59 def __init__(self
, parser
: 'QAPISchemaParser', msg
: str):
61 for ch
in parser
.src
[parser
.line_pos
:parser
.pos
]:
63 col
= (col
+ 7) % 8 + 1
66 super().__init
__(parser
.info
, msg
, col
)
69 class QAPISchemaParser
:
71 Parse QAPI schema source.
73 Parse a JSON-esque schema file and process directives. See
74 qapi-code-gen.rst section "Schema Syntax" for the exact syntax.
75 Grammatical validation is handled later by `expr.check_exprs()`.
77 :param fname: Source file name.
78 :param previously_included:
79 The absolute names of previously included source files,
80 if being invoked from another parser.
82 `QAPISourceInfo` belonging to the parent module.
83 ``None`` implies this is the root module.
85 :ivar exprs: Resulting parsed expressions.
86 :ivar docs: Resulting parsed documentation blocks.
88 :raise OSError: For problems reading the root schema document.
89 :raise QAPIError: For errors in the schema source.
93 previously_included
: Optional
[Set
[str]] = None,
94 incl_info
: Optional
[QAPISourceInfo
] = None):
96 self
._included
= previously_included
or set()
97 self
._included
.add(os
.path
.abspath(self
._fname
))
100 # Lexer state (see `accept` for details):
101 self
.info
= QAPISourceInfo(self
._fname
, incl_info
)
102 self
.tok
: Union
[None, str] = None
105 self
.val
: Optional
[Union
[bool, str]] = None
109 self
.exprs
: List
[QAPIExpression
] = []
110 self
.docs
: List
[QAPIDoc
] = []
115 def _parse(self
) -> None:
117 Parse the QAPI schema document.
119 :return: None. Results are stored in ``.exprs`` and ``.docs``.
123 # May raise OSError; allow the caller to handle it.
124 with
open(self
._fname
, 'r', encoding
='utf-8') as fp
:
126 if self
.src
== '' or self
.src
[-1] != '\n':
133 while self
.tok
is not None:
136 self
.reject_expr_doc(cur_doc
)
137 cur_doc
= self
.get_doc()
138 self
.docs
.append(cur_doc
)
141 expr
= self
.get_expr()
142 if not isinstance(expr
, dict):
144 info
, "top-level expression must be an object")
146 if 'include' in expr
:
147 self
.reject_expr_doc(cur_doc
)
149 raise QAPISemError(info
, "invalid 'include' directive")
150 include
= expr
['include']
151 if not isinstance(include
, str):
152 raise QAPISemError(info
,
153 "value of 'include' must be a string")
154 incl_fname
= os
.path
.join(os
.path
.dirname(self
._fname
),
156 self
._add
_expr
(OrderedDict({'include': incl_fname
}), info
)
157 exprs_include
= self
._include
(include
, info
, incl_fname
,
160 self
.exprs
.extend(exprs_include
.exprs
)
161 self
.docs
.extend(exprs_include
.docs
)
162 elif "pragma" in expr
:
163 self
.reject_expr_doc(cur_doc
)
165 raise QAPISemError(info
, "invalid 'pragma' directive")
166 pragma
= expr
['pragma']
167 if not isinstance(pragma
, dict):
169 info
, "value of 'pragma' must be an object")
170 for name
, value
in pragma
.items():
171 self
._pragma
(name
, value
, info
)
173 if cur_doc
and not cur_doc
.symbol
:
175 cur_doc
.info
, "definition documentation required")
176 self
._add
_expr
(expr
, info
, cur_doc
)
178 self
.reject_expr_doc(cur_doc
)
180 def _add_expr(self
, expr
: Mapping
[str, object],
181 info
: QAPISourceInfo
,
182 doc
: Optional
['QAPIDoc'] = None) -> None:
183 self
.exprs
.append(QAPIExpression(expr
, info
, doc
))
186 def reject_expr_doc(doc
: Optional
['QAPIDoc']) -> None:
187 if doc
and doc
.symbol
:
190 "documentation for '%s' is not followed by the definition"
194 def _include(include
: str,
195 info
: QAPISourceInfo
,
197 previously_included
: Set
[str]
198 ) -> Optional
['QAPISchemaParser']:
199 incl_abs_fname
= os
.path
.abspath(incl_fname
)
200 # catch inclusion cycle
201 inf
: Optional
[QAPISourceInfo
] = info
203 if incl_abs_fname
== os
.path
.abspath(inf
.fname
):
204 raise QAPISemError(info
, "inclusion loop for %s" % include
)
207 # skip multiple include of the same file
208 if incl_abs_fname
in previously_included
:
212 return QAPISchemaParser(incl_fname
, previously_included
, info
)
213 except OSError as err
:
216 f
"can't read include file '{incl_fname}': {err.strerror}"
220 def _pragma(name
: str, value
: object, info
: QAPISourceInfo
) -> None:
222 def check_list_str(name
: str, value
: object) -> List
[str]:
223 if (not isinstance(value
, list) or
224 any(not isinstance(elt
, str) for elt
in value
)):
227 "pragma %s must be a list of strings" % name
)
232 if name
== 'doc-required':
233 if not isinstance(value
, bool):
234 raise QAPISemError(info
,
235 "pragma 'doc-required' must be boolean")
236 pragma
.doc_required
= value
237 elif name
== 'command-name-exceptions':
238 pragma
.command_name_exceptions
= check_list_str(name
, value
)
239 elif name
== 'command-returns-exceptions':
240 pragma
.command_returns_exceptions
= check_list_str(name
, value
)
241 elif name
== 'documentation-exceptions':
242 pragma
.documentation_exceptions
= check_list_str(name
, value
)
243 elif name
== 'member-name-exceptions':
244 pragma
.member_name_exceptions
= check_list_str(name
, value
)
246 raise QAPISemError(info
, "unknown pragma '%s'" % name
)
248 def accept(self
, skip_comment
: bool = True) -> None:
250 Read and store the next token.
253 When false, return COMMENT tokens ("#").
254 This is used when reading documentation blocks.
257 None. Several instance attributes are updated instead:
259 - ``.tok`` represents the token type. See below for values.
260 - ``.info`` describes the token's source location.
261 - ``.val`` is the token's value, if any. See below.
262 - ``.pos`` is the buffer index of the first character of
265 * Single-character tokens:
267 These are "{", "}", ":", ",", "[", and "]".
268 ``.tok`` holds the single character and ``.val`` is None.
270 * Multi-character tokens:
274 This token is not normally returned by the lexer, but it can
275 be when ``skip_comment`` is False. ``.tok`` is "#", and
276 ``.val`` is a string including all chars until end-of-line,
277 including the "#" itself.
281 ``.tok`` is "'", the single quote. ``.val`` contains the
282 string, excluding the surrounding quotes.
286 ``.tok`` is either "t" or "f", ``.val`` will be the
287 corresponding bool value.
291 ``.tok`` and ``.val`` will both be None at EOF.
294 self
.tok
= self
.src
[self
.cursor
]
295 self
.pos
= self
.cursor
300 if self
.src
[self
.cursor
] == '#':
301 # Start of doc comment
303 self
.cursor
= self
.src
.find('\n', self
.cursor
)
305 self
.val
= self
.src
[self
.pos
:self
.cursor
]
307 elif self
.tok
in '{}:,[]':
309 elif self
.tok
== "'":
310 # Note: we accept only printable ASCII
314 ch
= self
.src
[self
.cursor
]
317 raise QAPIParseError(self
, "missing terminating \"'\"")
319 # Note: we recognize only \\ because we have
320 # no use for funny characters in strings
322 raise QAPIParseError(self
,
323 "unknown escape \\%s" % ch
)
331 if ord(ch
) < 32 or ord(ch
) >= 127:
332 raise QAPIParseError(
333 self
, "funny character in string")
335 elif self
.src
.startswith('true', self
.pos
):
339 elif self
.src
.startswith('false', self
.pos
):
343 elif self
.tok
== '\n':
344 if self
.cursor
== len(self
.src
):
347 self
.info
= self
.info
.next_line()
348 self
.line_pos
= self
.cursor
349 elif not self
.tok
.isspace():
350 # Show up to next structural, whitespace or quote
352 match
= must_match('[^[\\]{}:,\\s\']+',
353 self
.src
[self
.cursor
-1:])
354 raise QAPIParseError(self
, "stray '%s'" % match
.group(0))
356 def get_members(self
) -> Dict
[str, object]:
357 expr
: Dict
[str, object] = OrderedDict()
362 raise QAPIParseError(self
, "expected string or '}'")
365 assert isinstance(key
, str) # Guaranteed by tok == "'"
369 raise QAPIParseError(self
, "expected ':'")
372 raise QAPIParseError(self
, "duplicate key '%s'" % key
)
373 expr
[key
] = self
.get_expr()
378 raise QAPIParseError(self
, "expected ',' or '}'")
381 raise QAPIParseError(self
, "expected string")
383 def get_values(self
) -> List
[object]:
384 expr
: List
[object] = []
388 if self
.tok
not in tuple("{['tf"):
389 raise QAPIParseError(
390 self
, "expected '{', '[', ']', string, or boolean")
392 expr
.append(self
.get_expr())
397 raise QAPIParseError(self
, "expected ',' or ']'")
400 def get_expr(self
) -> _ExprValue
:
404 expr
= self
.get_members()
405 elif self
.tok
== '[':
407 expr
= self
.get_values()
408 elif self
.tok
in tuple("'tf"):
409 assert isinstance(self
.val
, (str, bool))
413 raise QAPIParseError(
414 self
, "expected '{', '[', string, or boolean")
417 def get_doc_line(self
) -> Optional
[str]:
419 raise QAPIParseError(
420 self
, "documentation comment must end with '##'")
421 assert isinstance(self
.val
, str)
422 if self
.val
.startswith('##'):
425 raise QAPIParseError(
426 self
, "junk after '##' at end of documentation comment")
430 if self
.val
[1] != ' ':
431 raise QAPIParseError(self
, "missing space after #")
432 return self
.val
[2:].rstrip()
435 def _match_at_name_colon(string
: str) -> Optional
[Match
[str]]:
436 return re
.match(r
'@([^:]*): *', string
)
438 def get_doc_indented(self
, doc
: 'QAPIDoc') -> Optional
[str]:
440 line
= self
.get_doc_line()
442 doc
.append_line(line
)
444 line
= self
.get_doc_line()
447 indent
= must_match(r
'\s*', line
).end()
450 doc
.append_line(line
[indent
:])
451 prev_line_blank
= False
454 line
= self
.get_doc_line()
457 if self
._match
_at
_name
_colon
(line
):
459 cur_indent
= must_match(r
'\s*', line
).end()
460 if line
!= '' and cur_indent
< indent
:
463 raise QAPIParseError(
465 "unexpected de-indent (expected at least %d spaces)" %
467 doc
.append_line(line
[indent
:])
468 prev_line_blank
= True
470 def get_doc_paragraph(self
, doc
: 'QAPIDoc') -> Optional
[str]:
473 line
= self
.get_doc_line()
478 doc
.append_line(line
)
480 def get_doc(self
) -> 'QAPIDoc':
482 raise QAPIParseError(
483 self
, "junk after '##' at start of documentation comment")
486 line
= self
.get_doc_line()
487 if line
is not None and line
.startswith('@'):
488 # Definition documentation
489 if not line
.endswith(':'):
490 raise QAPIParseError(self
, "line should end with ':'")
491 # Invalid names are not checked here, but the name
492 # provided *must* match the following definition,
493 # which *is* validated in expr.py.
496 raise QAPIParseError(self
, "name required after '@'")
497 doc
= QAPIDoc(info
, symbol
)
499 line
= self
.get_doc_line()
502 while line
is not None:
506 line
= self
.get_doc_line()
509 # Non-blank line, first of a section
510 if line
== 'Features:':
512 raise QAPIParseError(
513 self
, "duplicated 'Features:' line")
515 line
= self
.get_doc_line()
518 line
= self
.get_doc_line()
519 while (line
is not None
520 and (match
:= self
._match
_at
_name
_colon
(line
))):
521 doc
.new_feature(self
.info
, match
.group(1))
522 text
= line
[match
.end():]
524 doc
.append_line(text
)
525 line
= self
.get_doc_indented(doc
)
527 raise QAPIParseError(
528 self
, 'feature descriptions expected')
530 elif match
:= self
._match
_at
_name
_colon
(line
):
533 raise QAPIParseError(
535 "description of '@%s:' follows a section"
537 while (line
is not None
538 and (match
:= self
._match
_at
_name
_colon
(line
))):
539 doc
.new_argument(self
.info
, match
.group(1))
540 text
= line
[match
.end():]
542 doc
.append_line(text
)
543 line
= self
.get_doc_indented(doc
)
545 elif match
:= re
.match(
546 r
'(Returns|Since|Notes?|Examples?|TODO): *',
549 doc
.new_tagged_section(self
.info
, match
.group(1))
550 text
= line
[match
.end():]
552 doc
.append_line(text
)
553 line
= self
.get_doc_indented(doc
)
555 elif line
.startswith('='):
556 raise QAPIParseError(
558 "unexpected '=' markup in definition documentation")
561 doc
.ensure_untagged_section(self
.info
)
562 doc
.append_line(line
)
563 line
= self
.get_doc_paragraph(doc
)
565 # Free-form documentation
567 doc
.ensure_untagged_section(self
.info
)
569 while line
is not None:
570 if match
:= self
._match
_at
_name
_colon
(line
):
571 raise QAPIParseError(
573 "'@%s:' not allowed in free-form documentation"
575 if line
.startswith('='):
577 raise QAPIParseError(
579 "'=' heading must come first in a comment block")
580 doc
.append_line(line
)
582 line
= self
.get_doc_line()
592 A documentation comment block, either definition or free-form
594 Definition documentation blocks consist of
596 * a body section: one line naming the definition, followed by an
597 overview (any number of lines)
599 * argument sections: a description of each argument (for commands
600 and events) or member (for structs, unions and alternates)
602 * features sections: a description of each feature flag
604 * additional (non-argument) sections, possibly tagged
606 Free-form documentation blocks consist only of a body section.
610 def __init__(self
, info
: QAPISourceInfo
,
611 tag
: Optional
[str] = None):
612 # section source info, i.e. where it begins
614 # section tag, if any ('Returns', '@name', ...)
616 # section text without tag
619 def append_line(self
, line
: str) -> None:
620 self
.text
+= line
+ '\n'
622 class ArgSection(Section
):
623 def __init__(self
, info
: QAPISourceInfo
, tag
: str):
624 super().__init
__(info
, tag
)
625 self
.member
: Optional
['QAPISchemaMember'] = None
627 def connect(self
, member
: 'QAPISchemaMember') -> None:
630 def __init__(self
, info
: QAPISourceInfo
, symbol
: Optional
[str] = None):
631 # info points to the doc comment block's first line
633 # definition doc's symbol, None for free-form doc
634 self
.symbol
: Optional
[str] = symbol
635 # the sections in textual order
636 self
.all_sections
: List
[QAPIDoc
.Section
] = [QAPIDoc
.Section(info
)]
638 self
.body
: Optional
[QAPIDoc
.Section
] = self
.all_sections
[0]
639 # dicts mapping parameter/feature names to their description
640 self
.args
: Dict
[str, QAPIDoc
.ArgSection
] = {}
641 self
.features
: Dict
[str, QAPIDoc
.ArgSection
] = {}
642 # a command's "Returns" section
643 self
.returns
: Optional
[QAPIDoc
.Section
] = None
645 self
.since
: Optional
[QAPIDoc
.Section
] = None
646 # sections other than .body, .args, .features
647 self
.sections
: List
[QAPIDoc
.Section
] = []
649 def end(self
) -> None:
650 for section
in self
.all_sections
:
651 section
.text
= section
.text
.strip('\n')
652 if section
.tag
is not None and section
.text
== '':
654 section
.info
, "text required after '%s:'" % section
.tag
)
656 def ensure_untagged_section(self
, info
: QAPISourceInfo
) -> None:
657 if self
.all_sections
and not self
.all_sections
[-1].tag
:
658 # extend current section
659 self
.all_sections
[-1].text
+= '\n'
662 section
= self
.Section(info
)
663 self
.sections
.append(section
)
664 self
.all_sections
.append(section
)
666 def new_tagged_section(self
, info
: QAPISourceInfo
, tag
: str) -> None:
667 section
= self
.Section(info
, tag
)
671 info
, "duplicated '%s' section" % tag
)
672 self
.returns
= section
676 info
, "duplicated '%s' section" % tag
)
678 self
.sections
.append(section
)
679 self
.all_sections
.append(section
)
681 def _new_description(self
, info
: QAPISourceInfo
, name
: str,
682 desc
: Dict
[str, ArgSection
]) -> None:
684 raise QAPISemError(info
, "invalid parameter name")
686 raise QAPISemError(info
, "'%s' parameter name duplicated" % name
)
687 section
= self
.ArgSection(info
, '@' + name
)
688 self
.all_sections
.append(section
)
691 def new_argument(self
, info
: QAPISourceInfo
, name
: str) -> None:
692 self
._new
_description
(info
, name
, self
.args
)
694 def new_feature(self
, info
: QAPISourceInfo
, name
: str) -> None:
695 self
._new
_description
(info
, name
, self
.features
)
697 def append_line(self
, line
: str) -> None:
698 self
.all_sections
[-1].append_line(line
)
700 def connect_member(self
, member
: 'QAPISchemaMember') -> None:
701 if member
.name
not in self
.args
:
702 if self
.symbol
not in member
.info
.pragma
.documentation_exceptions
:
703 raise QAPISemError(member
.info
,
704 "%s '%s' lacks documentation"
705 % (member
.role
, member
.name
))
706 self
.args
[member
.name
] = QAPIDoc
.ArgSection(
707 self
.info
, '@' + member
.name
)
708 self
.args
[member
.name
].connect(member
)
710 def connect_feature(self
, feature
: 'QAPISchemaFeature') -> None:
711 if feature
.name
not in self
.features
:
712 raise QAPISemError(feature
.info
,
713 "feature '%s' lacks documentation"
715 self
.features
[feature
.name
].connect(feature
)
717 def check_expr(self
, expr
: QAPIExpression
) -> None:
718 if self
.returns
and 'command' not in expr
:
721 "'Returns' section is only valid for commands")
723 def check(self
) -> None:
725 def check_args_section(
726 args
: Dict
[str, QAPIDoc
.ArgSection
], what
: str
728 bogus
= [name
for name
, section
in args
.items()
729 if not section
.member
]
733 "documented %s%s '%s' %s not exist" % (
735 "s" if len(bogus
) > 1 else "",
737 "do" if len(bogus
) > 1 else "does"
740 check_args_section(self
.args
, 'member')
741 check_args_section(self
.features
, 'feature')