1 # -*- coding: utf-8 -*-
5 # Copyright IBM, Corp. 2011
6 # Copyright (c) 2013-2019 Red Hat Inc.
9 # Anthony Liguori <aliguori@us.ibm.com>
10 # Markus Armbruster <armbru@redhat.com>
11 # Marc-André Lureau <marcandre.lureau@redhat.com>
12 # Kevin Wolf <kwolf@redhat.com>
14 # This work is licensed under the terms of the GNU GPL, version 2.
15 # See the COPYING file in the top-level directory.
17 from collections
import OrderedDict
21 from .error
import QAPIParseError
, QAPISemError
22 from .source
import QAPISourceInfo
25 class QAPISchemaParser
:
27 def __init__(self
, fname
, previously_included
=None, incl_info
=None):
28 previously_included
= previously_included
or set()
29 previously_included
.add(os
.path
.abspath(fname
))
32 fp
= open(fname
, 'r', encoding
='utf-8')
35 raise QAPISemError(incl_info
or QAPISourceInfo(None, None, None),
36 "can't read %s file '%s': %s"
37 % ("include" if incl_info
else "schema",
41 if self
.src
== '' or self
.src
[-1] != '\n':
44 self
.info
= QAPISourceInfo(fname
, 1, incl_info
)
51 while self
.tok
is not None:
54 self
.reject_expr_doc(cur_doc
)
55 for cur_doc
in self
.get_doc(info
):
56 self
.docs
.append(cur_doc
)
59 expr
= self
.get_expr(False)
61 self
.reject_expr_doc(cur_doc
)
63 raise QAPISemError(info
, "invalid 'include' directive")
64 include
= expr
['include']
65 if not isinstance(include
, str):
66 raise QAPISemError(info
,
67 "value of 'include' must be a string")
68 incl_fname
= os
.path
.join(os
.path
.dirname(fname
),
70 self
.exprs
.append({'expr': {'include': incl_fname
},
72 exprs_include
= self
._include
(include
, info
, incl_fname
,
75 self
.exprs
.extend(exprs_include
.exprs
)
76 self
.docs
.extend(exprs_include
.docs
)
77 elif "pragma" in expr
:
78 self
.reject_expr_doc(cur_doc
)
80 raise QAPISemError(info
, "invalid 'pragma' directive")
81 pragma
= expr
['pragma']
82 if not isinstance(pragma
, dict):
84 info
, "value of 'pragma' must be an object")
85 for name
, value
in pragma
.items():
86 self
._pragma
(name
, value
, info
)
88 expr_elem
= {'expr': expr
,
91 if not cur_doc
.symbol
:
93 cur_doc
.info
, "definition documentation required")
94 expr_elem
['doc'] = cur_doc
95 self
.exprs
.append(expr_elem
)
97 self
.reject_expr_doc(cur_doc
)
100 def reject_expr_doc(doc
):
101 if doc
and doc
.symbol
:
104 "documentation for '%s' is not followed by the definition"
107 def _include(self
, include
, info
, incl_fname
, previously_included
):
108 incl_abs_fname
= os
.path
.abspath(incl_fname
)
109 # catch inclusion cycle
112 if incl_abs_fname
== os
.path
.abspath(inf
.fname
):
113 raise QAPISemError(info
, "inclusion loop for %s" % include
)
116 # skip multiple include of the same file
117 if incl_abs_fname
in previously_included
:
120 return QAPISchemaParser(incl_fname
, previously_included
, info
)
122 def _pragma(self
, name
, value
, info
):
123 if name
== 'doc-required':
124 if not isinstance(value
, bool):
125 raise QAPISemError(info
,
126 "pragma 'doc-required' must be boolean")
127 info
.pragma
.doc_required
= value
128 elif name
== 'returns-whitelist':
129 if (not isinstance(value
, list)
130 or any([not isinstance(elt
, str) for elt
in value
])):
133 "pragma returns-whitelist must be a list of strings")
134 info
.pragma
.returns_whitelist
= value
135 elif name
== 'name-case-whitelist':
136 if (not isinstance(value
, list)
137 or any([not isinstance(elt
, str) for elt
in value
])):
140 "pragma name-case-whitelist must be a list of strings")
141 info
.pragma
.name_case_whitelist
= value
143 raise QAPISemError(info
, "unknown pragma '%s'" % name
)
145 def accept(self
, skip_comment
=True):
147 self
.tok
= self
.src
[self
.cursor
]
148 self
.pos
= self
.cursor
153 if self
.src
[self
.cursor
] == '#':
154 # Start of doc comment
156 self
.cursor
= self
.src
.find('\n', self
.cursor
)
158 self
.val
= self
.src
[self
.pos
:self
.cursor
]
160 elif self
.tok
in '{}:,[]':
162 elif self
.tok
== "'":
163 # Note: we accept only printable ASCII
167 ch
= self
.src
[self
.cursor
]
170 raise QAPIParseError(self
, "missing terminating \"'\"")
172 # Note: we recognize only \\ because we have
173 # no use for funny characters in strings
175 raise QAPIParseError(self
,
176 "unknown escape \\%s" % ch
)
184 if ord(ch
) < 32 or ord(ch
) >= 127:
185 raise QAPIParseError(
186 self
, "funny character in string")
188 elif self
.src
.startswith('true', self
.pos
):
192 elif self
.src
.startswith('false', self
.pos
):
196 elif self
.tok
== '\n':
197 if self
.cursor
== len(self
.src
):
200 self
.info
= self
.info
.next_line()
201 self
.line_pos
= self
.cursor
202 elif not self
.tok
.isspace():
203 # Show up to next structural, whitespace or quote
205 match
= re
.match('[^[\\]{}:,\\s\'"]+',
206 self
.src
[self
.cursor
-1:])
207 raise QAPIParseError(self
, "stray '%s'" % match
.group(0))
209 def get_members(self
):
215 raise QAPIParseError(self
, "expected string or '}'")
220 raise QAPIParseError(self
, "expected ':'")
223 raise QAPIParseError(self
, "duplicate key '%s'" % key
)
224 expr
[key
] = self
.get_expr(True)
229 raise QAPIParseError(self
, "expected ',' or '}'")
232 raise QAPIParseError(self
, "expected string")
234 def get_values(self
):
239 if self
.tok
not in "{['tf":
240 raise QAPIParseError(
241 self
, "expected '{', '[', ']', string, or boolean")
243 expr
.append(self
.get_expr(True))
248 raise QAPIParseError(self
, "expected ',' or ']'")
251 def get_expr(self
, nested
):
252 if self
.tok
!= '{' and not nested
:
253 raise QAPIParseError(self
, "expected '{'")
256 expr
= self
.get_members()
257 elif self
.tok
== '[':
259 expr
= self
.get_values()
260 elif self
.tok
in "'tf":
264 raise QAPIParseError(
265 self
, "expected '{', '[', string, or boolean")
268 def get_doc(self
, info
):
270 raise QAPIParseError(
271 self
, "junk after '##' at start of documentation comment")
274 cur_doc
= QAPIDoc(self
, info
)
276 while self
.tok
== '#':
277 if self
.val
.startswith('##'):
280 raise QAPIParseError(
282 "junk after '##' at end of documentation comment")
283 cur_doc
.end_comment()
287 if self
.val
.startswith('# ='):
289 raise QAPIParseError(
291 "unexpected '=' markup in definition documentation")
292 if cur_doc
.body
.text
:
293 cur_doc
.end_comment()
295 cur_doc
= QAPIDoc(self
, info
)
296 cur_doc
.append(self
.val
)
299 raise QAPIParseError(self
, "documentation comment must end with '##'")
304 A documentation comment block, either definition or free-form
306 Definition documentation blocks consist of
308 * a body section: one line naming the definition, followed by an
309 overview (any number of lines)
311 * argument sections: a description of each argument (for commands
312 and events) or member (for structs, unions and alternates)
314 * features sections: a description of each feature flag
316 * additional (non-argument) sections, possibly tagged
318 Free-form documentation blocks consist only of a body section.
322 def __init__(self
, parser
, name
=None, indent
=0):
323 # parser, for error messages about indentation
324 self
._parser
= parser
325 # optional section name (argument/member or section name)
328 # the expected indent level of the text of this section
329 self
._indent
= indent
331 def append(self
, line
):
332 # Strip leading spaces corresponding to the expected indent level
333 # Blank lines are always OK.
335 indent
= re
.match(r
'\s*', line
).end()
336 if indent
< self
._indent
:
337 raise QAPIParseError(
339 "unexpected de-indent (expected at least %d spaces)" %
341 line
= line
[self
._indent
:]
343 self
.text
+= line
.rstrip() + '\n'
345 class ArgSection(Section
):
346 def __init__(self
, parser
, name
, indent
=0):
347 super().__init
__(parser
, name
, indent
)
350 def connect(self
, member
):
353 def __init__(self
, parser
, info
):
354 # self._parser is used to report errors with QAPIParseError. The
355 # resulting error position depends on the state of the parser.
356 # It happens to be the beginning of the comment. More or less
357 # servicable, but action at a distance.
358 self
._parser
= parser
361 self
.body
= QAPIDoc
.Section(parser
)
362 # dict mapping parameter name to ArgSection
363 self
.args
= OrderedDict()
364 self
.features
= OrderedDict()
367 # the current section
368 self
._section
= self
.body
369 self
._append
_line
= self
._append
_body
_line
371 def has_section(self
, name
):
372 """Return True if we have a section with this name."""
373 for i
in self
.sections
:
378 def append(self
, line
):
380 Parse a comment line and add it to the documentation.
382 The way that the line is dealt with depends on which part of
383 the documentation we're parsing right now:
384 * The body section: ._append_line is ._append_body_line
385 * An argument section: ._append_line is ._append_args_line
386 * A features section: ._append_line is ._append_features_line
387 * An additional section: ._append_line is ._append_various_line
391 self
._append
_freeform
(line
)
395 raise QAPIParseError(self
._parser
, "missing space after #")
397 self
._append
_line
(line
)
399 def end_comment(self
):
403 def _is_section_tag(name
):
404 return name
in ('Returns:', 'Since:',
405 # those are often singular or plural
407 'Example:', 'Examples:',
410 def _append_body_line(self
, line
):
412 Process a line of documentation text in the body section.
414 If this a symbol line and it is the section's first line, this
415 is a definition documentation block for that symbol.
417 If it's a definition documentation block, another symbol line
418 begins the argument section for the argument named by it, and
419 a section tag begins an additional section. Start that
420 section and append the line to it.
422 Else, append the line to the current section.
424 name
= line
.split(' ', 1)[0]
425 # FIXME not nice: things like '# @foo:' and '# @foo: ' aren't
426 # recognized, and get silently treated as ordinary text
427 if not self
.symbol
and not self
.body
.text
and line
.startswith('@'):
428 if not line
.endswith(':'):
429 raise QAPIParseError(self
._parser
, "line should end with ':'")
430 self
.symbol
= line
[1:-1]
431 # FIXME invalid names other than the empty string aren't flagged
433 raise QAPIParseError(self
._parser
, "invalid name")
435 # This is a definition documentation block
436 if name
.startswith('@') and name
.endswith(':'):
437 self
._append
_line
= self
._append
_args
_line
438 self
._append
_args
_line
(line
)
439 elif line
== 'Features:':
440 self
._append
_line
= self
._append
_features
_line
441 elif self
._is
_section
_tag
(name
):
442 self
._append
_line
= self
._append
_various
_line
443 self
._append
_various
_line
(line
)
445 self
._append
_freeform
(line
)
447 # This is a free-form documentation block
448 self
._append
_freeform
(line
)
450 def _append_args_line(self
, line
):
452 Process a line of documentation text in an argument section.
454 A symbol line begins the next argument section, a section tag
455 section or a non-indented line after a blank line begins an
456 additional section. Start that section and append the line to
459 Else, append the line to the current section.
462 name
= line
.split(' ', 1)[0]
464 if name
.startswith('@') and name
.endswith(':'):
465 # If line is "@arg: first line of description", find
466 # the index of 'f', which is the indent we expect for any
467 # following lines. We then remove the leading "@arg:"
468 # from line and replace it with spaces so that 'f' has the
469 # same index as it did in the original line and can be
470 # handled the same way we will handle following lines.
471 indent
= re
.match(r
'@\S*:\s*', line
).end()
474 # Line was just the "@arg:" header; following lines
478 line
= ' ' * indent
+ line
479 self
._start
_args
_section
(name
[1:-1], indent
)
480 elif self
._is
_section
_tag
(name
):
481 self
._append
_line
= self
._append
_various
_line
482 self
._append
_various
_line
(line
)
484 elif (self
._section
.text
.endswith('\n\n')
485 and line
and not line
[0].isspace()):
486 if line
== 'Features:':
487 self
._append
_line
= self
._append
_features
_line
489 self
._start
_section
()
490 self
._append
_line
= self
._append
_various
_line
491 self
._append
_various
_line
(line
)
494 self
._append
_freeform
(line
)
496 def _append_features_line(self
, line
):
497 name
= line
.split(' ', 1)[0]
499 if name
.startswith('@') and name
.endswith(':'):
500 # If line is "@arg: first line of description", find
501 # the index of 'f', which is the indent we expect for any
502 # following lines. We then remove the leading "@arg:"
503 # from line and replace it with spaces so that 'f' has the
504 # same index as it did in the original line and can be
505 # handled the same way we will handle following lines.
506 indent
= re
.match(r
'@\S*:\s*', line
).end()
509 # Line was just the "@arg:" header; following lines
513 line
= ' ' * indent
+ line
514 self
._start
_features
_section
(name
[1:-1], indent
)
515 elif self
._is
_section
_tag
(name
):
516 self
._append
_line
= self
._append
_various
_line
517 self
._append
_various
_line
(line
)
519 elif (self
._section
.text
.endswith('\n\n')
520 and line
and not line
[0].isspace()):
521 self
._start
_section
()
522 self
._append
_line
= self
._append
_various
_line
523 self
._append
_various
_line
(line
)
526 self
._append
_freeform
(line
)
528 def _append_various_line(self
, line
):
530 Process a line of documentation text in an additional section.
532 A symbol line is an error.
534 A section tag begins an additional section. Start that
535 section and append the line to it.
537 Else, append the line to the current section.
539 name
= line
.split(' ', 1)[0]
541 if name
.startswith('@') and name
.endswith(':'):
542 raise QAPIParseError(self
._parser
,
543 "'%s' can't follow '%s' section"
544 % (name
, self
.sections
[0].name
))
545 if self
._is
_section
_tag
(name
):
546 # If line is "Section: first line of description", find
547 # the index of 'f', which is the indent we expect for any
548 # following lines. We then remove the leading "Section:"
549 # from line and replace it with spaces so that 'f' has the
550 # same index as it did in the original line and can be
551 # handled the same way we will handle following lines.
552 indent
= re
.match(r
'\S*:\s*', line
).end()
555 # Line was just the "Section:" header; following lines
559 line
= ' ' * indent
+ line
560 self
._start
_section
(name
[:-1], indent
)
562 self
._append
_freeform
(line
)
564 def _start_symbol_section(self
, symbols_dict
, name
, indent
):
565 # FIXME invalid names other than the empty string aren't flagged
567 raise QAPIParseError(self
._parser
, "invalid parameter name")
568 if name
in symbols_dict
:
569 raise QAPIParseError(self
._parser
,
570 "'%s' parameter name duplicated" % name
)
571 assert not self
.sections
573 self
._section
= QAPIDoc
.ArgSection(self
._parser
, name
, indent
)
574 symbols_dict
[name
] = self
._section
576 def _start_args_section(self
, name
, indent
):
577 self
._start
_symbol
_section
(self
.args
, name
, indent
)
579 def _start_features_section(self
, name
, indent
):
580 self
._start
_symbol
_section
(self
.features
, name
, indent
)
582 def _start_section(self
, name
=None, indent
=0):
583 if name
in ('Returns', 'Since') and self
.has_section(name
):
584 raise QAPIParseError(self
._parser
,
585 "duplicated '%s' section" % name
)
587 self
._section
= QAPIDoc
.Section(self
._parser
, name
, indent
)
588 self
.sections
.append(self
._section
)
590 def _end_section(self
):
592 text
= self
._section
.text
= self
._section
.text
.strip()
593 if self
._section
.name
and (not text
or text
.isspace()):
594 raise QAPIParseError(
596 "empty doc section '%s'" % self
._section
.name
)
599 def _append_freeform(self
, line
):
600 match
= re
.match(r
'(@\S+:)', line
)
602 raise QAPIParseError(self
._parser
,
603 "'%s' not allowed in free-form documentation"
605 self
._section
.append(line
)
607 def connect_member(self
, member
):
608 if member
.name
not in self
.args
:
609 # Undocumented TODO outlaw
610 self
.args
[member
.name
] = QAPIDoc
.ArgSection(self
._parser
,
612 self
.args
[member
.name
].connect(member
)
614 def connect_feature(self
, feature
):
615 if feature
.name
not in self
.features
:
616 raise QAPISemError(feature
.info
,
617 "feature '%s' lacks documentation"
619 self
.features
[feature
.name
].connect(feature
)
621 def check_expr(self
, expr
):
622 if self
.has_section('Returns') and 'command' not in expr
:
623 raise QAPISemError(self
.info
,
624 "'Returns:' is only valid for commands")
628 def check_args_section(args
, info
, what
):
629 bogus
= [name
for name
, section
in args
.items()
630 if not section
.member
]
634 "documented member%s '%s' %s not exist"
635 % ("s" if len(bogus
) > 1 else "",
637 "do" if len(bogus
) > 1 else "does"))
639 check_args_section(self
.args
, self
.info
, 'members')
640 check_args_section(self
.features
, self
.info
, 'features')