qapi: Check feature documentation against the schema
[qemu/armbru.git] / scripts / qapi / parser.py
blob342792e4103c351268b10fbb0624d0db7ade5834
1 # -*- coding: utf-8 -*-
3 # QAPI schema parser
5 # Copyright IBM, Corp. 2011
6 # Copyright (c) 2013-2019 Red Hat Inc.
8 # Authors:
9 # Anthony Liguori <aliguori@us.ibm.com>
10 # Markus Armbruster <armbru@redhat.com>
11 # Marc-André Lureau <marcandre.lureau@redhat.com>
12 # Kevin Wolf <kwolf@redhat.com>
14 # This work is licensed under the terms of the GNU GPL, version 2.
15 # See the COPYING file in the top-level directory.
17 import os
18 import re
19 import sys
20 from collections import OrderedDict
22 from qapi.error import QAPIParseError, QAPISemError
23 from qapi.source import QAPISourceInfo
26 class QAPISchemaParser(object):
28 def __init__(self, fname, previously_included=None, incl_info=None):
29 previously_included = previously_included or set()
30 previously_included.add(os.path.abspath(fname))
32 try:
33 if sys.version_info[0] >= 3:
34 fp = open(fname, 'r', encoding='utf-8')
35 else:
36 fp = open(fname, 'r')
37 self.src = fp.read()
38 except IOError as e:
39 raise QAPISemError(incl_info or QAPISourceInfo(None, None, None),
40 "can't read %s file '%s': %s"
41 % ("include" if incl_info else "schema",
42 fname,
43 e.strerror))
45 if self.src == '' or self.src[-1] != '\n':
46 self.src += '\n'
47 self.cursor = 0
48 self.info = QAPISourceInfo(fname, 1, incl_info)
49 self.line_pos = 0
50 self.exprs = []
51 self.docs = []
52 self.accept()
53 cur_doc = None
55 while self.tok is not None:
56 info = self.info
57 if self.tok == '#':
58 self.reject_expr_doc(cur_doc)
59 cur_doc = self.get_doc(info)
60 self.docs.append(cur_doc)
61 continue
63 expr = self.get_expr(False)
64 if 'include' in expr:
65 self.reject_expr_doc(cur_doc)
66 if len(expr) != 1:
67 raise QAPISemError(info, "invalid 'include' directive")
68 include = expr['include']
69 if not isinstance(include, str):
70 raise QAPISemError(info,
71 "value of 'include' must be a string")
72 incl_fname = os.path.join(os.path.dirname(fname),
73 include)
74 self.exprs.append({'expr': {'include': incl_fname},
75 'info': info})
76 exprs_include = self._include(include, info, incl_fname,
77 previously_included)
78 if exprs_include:
79 self.exprs.extend(exprs_include.exprs)
80 self.docs.extend(exprs_include.docs)
81 elif "pragma" in expr:
82 self.reject_expr_doc(cur_doc)
83 if len(expr) != 1:
84 raise QAPISemError(info, "invalid 'pragma' directive")
85 pragma = expr['pragma']
86 if not isinstance(pragma, dict):
87 raise QAPISemError(
88 info, "value of 'pragma' must be an object")
89 for name, value in pragma.items():
90 self._pragma(name, value, info)
91 else:
92 expr_elem = {'expr': expr,
93 'info': info}
94 if cur_doc:
95 if not cur_doc.symbol:
96 raise QAPISemError(
97 cur_doc.info, "definition documentation required")
98 expr_elem['doc'] = cur_doc
99 self.exprs.append(expr_elem)
100 cur_doc = None
101 self.reject_expr_doc(cur_doc)
103 @staticmethod
104 def reject_expr_doc(doc):
105 if doc and doc.symbol:
106 raise QAPISemError(
107 doc.info,
108 "documentation for '%s' is not followed by the definition"
109 % doc.symbol)
111 def _include(self, include, info, incl_fname, previously_included):
112 incl_abs_fname = os.path.abspath(incl_fname)
113 # catch inclusion cycle
114 inf = info
115 while inf:
116 if incl_abs_fname == os.path.abspath(inf.fname):
117 raise QAPISemError(info, "inclusion loop for %s" % include)
118 inf = inf.parent
120 # skip multiple include of the same file
121 if incl_abs_fname in previously_included:
122 return None
124 return QAPISchemaParser(incl_fname, previously_included, info)
126 def _pragma(self, name, value, info):
127 if name == 'doc-required':
128 if not isinstance(value, bool):
129 raise QAPISemError(info,
130 "pragma 'doc-required' must be boolean")
131 info.pragma.doc_required = value
132 elif name == 'returns-whitelist':
133 if (not isinstance(value, list)
134 or any([not isinstance(elt, str) for elt in value])):
135 raise QAPISemError(
136 info,
137 "pragma returns-whitelist must be a list of strings")
138 info.pragma.returns_whitelist = value
139 elif name == 'name-case-whitelist':
140 if (not isinstance(value, list)
141 or any([not isinstance(elt, str) for elt in value])):
142 raise QAPISemError(
143 info,
144 "pragma name-case-whitelist must be a list of strings")
145 info.pragma.name_case_whitelist = value
146 else:
147 raise QAPISemError(info, "unknown pragma '%s'" % name)
149 def accept(self, skip_comment=True):
150 while True:
151 self.tok = self.src[self.cursor]
152 self.pos = self.cursor
153 self.cursor += 1
154 self.val = None
156 if self.tok == '#':
157 if self.src[self.cursor] == '#':
158 # Start of doc comment
159 skip_comment = False
160 self.cursor = self.src.find('\n', self.cursor)
161 if not skip_comment:
162 self.val = self.src[self.pos:self.cursor]
163 return
164 elif self.tok in '{}:,[]':
165 return
166 elif self.tok == "'":
167 # Note: we accept only printable ASCII
168 string = ''
169 esc = False
170 while True:
171 ch = self.src[self.cursor]
172 self.cursor += 1
173 if ch == '\n':
174 raise QAPIParseError(self, "missing terminating \"'\"")
175 if esc:
176 # Note: we recognize only \\ because we have
177 # no use for funny characters in strings
178 if ch != '\\':
179 raise QAPIParseError(self,
180 "unknown escape \\%s" % ch)
181 esc = False
182 elif ch == '\\':
183 esc = True
184 continue
185 elif ch == "'":
186 self.val = string
187 return
188 if ord(ch) < 32 or ord(ch) >= 127:
189 raise QAPIParseError(
190 self, "funny character in string")
191 string += ch
192 elif self.src.startswith('true', self.pos):
193 self.val = True
194 self.cursor += 3
195 return
196 elif self.src.startswith('false', self.pos):
197 self.val = False
198 self.cursor += 4
199 return
200 elif self.tok == '\n':
201 if self.cursor == len(self.src):
202 self.tok = None
203 return
204 self.info = self.info.next_line()
205 self.line_pos = self.cursor
206 elif not self.tok.isspace():
207 # Show up to next structural, whitespace or quote
208 # character
209 match = re.match('[^[\\]{}:,\\s\'"]+',
210 self.src[self.cursor-1:])
211 raise QAPIParseError(self, "stray '%s'" % match.group(0))
213 def get_members(self):
214 expr = OrderedDict()
215 if self.tok == '}':
216 self.accept()
217 return expr
218 if self.tok != "'":
219 raise QAPIParseError(self, "expected string or '}'")
220 while True:
221 key = self.val
222 self.accept()
223 if self.tok != ':':
224 raise QAPIParseError(self, "expected ':'")
225 self.accept()
226 if key in expr:
227 raise QAPIParseError(self, "duplicate key '%s'" % key)
228 expr[key] = self.get_expr(True)
229 if self.tok == '}':
230 self.accept()
231 return expr
232 if self.tok != ',':
233 raise QAPIParseError(self, "expected ',' or '}'")
234 self.accept()
235 if self.tok != "'":
236 raise QAPIParseError(self, "expected string")
238 def get_values(self):
239 expr = []
240 if self.tok == ']':
241 self.accept()
242 return expr
243 if self.tok not in "{['tfn":
244 raise QAPIParseError(
245 self, "expected '{', '[', ']', string, boolean or 'null'")
246 while True:
247 expr.append(self.get_expr(True))
248 if self.tok == ']':
249 self.accept()
250 return expr
251 if self.tok != ',':
252 raise QAPIParseError(self, "expected ',' or ']'")
253 self.accept()
255 def get_expr(self, nested):
256 if self.tok != '{' and not nested:
257 raise QAPIParseError(self, "expected '{'")
258 if self.tok == '{':
259 self.accept()
260 expr = self.get_members()
261 elif self.tok == '[':
262 self.accept()
263 expr = self.get_values()
264 elif self.tok in "'tfn":
265 expr = self.val
266 self.accept()
267 else:
268 raise QAPIParseError(
269 self, "expected '{', '[', string, boolean or 'null'")
270 return expr
272 def get_doc(self, info):
273 if self.val != '##':
274 raise QAPIParseError(
275 self, "junk after '##' at start of documentation comment")
277 doc = QAPIDoc(self, info)
278 self.accept(False)
279 while self.tok == '#':
280 if self.val.startswith('##'):
281 # End of doc comment
282 if self.val != '##':
283 raise QAPIParseError(
284 self,
285 "junk after '##' at end of documentation comment")
286 doc.end_comment()
287 self.accept()
288 return doc
289 else:
290 doc.append(self.val)
291 self.accept(False)
293 raise QAPIParseError(self, "documentation comment must end with '##'")
296 class QAPIDoc(object):
298 A documentation comment block, either definition or free-form
300 Definition documentation blocks consist of
302 * a body section: one line naming the definition, followed by an
303 overview (any number of lines)
305 * argument sections: a description of each argument (for commands
306 and events) or member (for structs, unions and alternates)
308 * features sections: a description of each feature flag
310 * additional (non-argument) sections, possibly tagged
312 Free-form documentation blocks consist only of a body section.
315 class Section(object):
316 def __init__(self, name=None):
317 # optional section name (argument/member or section name)
318 self.name = name
319 # the list of lines for this section
320 self.text = ''
322 def append(self, line):
323 self.text += line.rstrip() + '\n'
325 class ArgSection(Section):
326 def __init__(self, name):
327 QAPIDoc.Section.__init__(self, name)
328 self.member = None
330 def connect(self, member):
331 self.member = member
333 def __init__(self, parser, info):
334 # self._parser is used to report errors with QAPIParseError. The
335 # resulting error position depends on the state of the parser.
336 # It happens to be the beginning of the comment. More or less
337 # servicable, but action at a distance.
338 self._parser = parser
339 self.info = info
340 self.symbol = None
341 self.body = QAPIDoc.Section()
342 # dict mapping parameter name to ArgSection
343 self.args = OrderedDict()
344 self.features = OrderedDict()
345 # a list of Section
346 self.sections = []
347 # the current section
348 self._section = self.body
349 self._append_line = self._append_body_line
351 def has_section(self, name):
352 """Return True if we have a section with this name."""
353 for i in self.sections:
354 if i.name == name:
355 return True
356 return False
358 def append(self, line):
360 Parse a comment line and add it to the documentation.
362 The way that the line is dealt with depends on which part of
363 the documentation we're parsing right now:
364 * The body section: ._append_line is ._append_body_line
365 * An argument section: ._append_line is ._append_args_line
366 * A features section: ._append_line is ._append_features_line
367 * An additional section: ._append_line is ._append_various_line
369 line = line[1:]
370 if not line:
371 self._append_freeform(line)
372 return
374 if line[0] != ' ':
375 raise QAPIParseError(self._parser, "missing space after #")
376 line = line[1:]
377 self._append_line(line)
379 def end_comment(self):
380 self._end_section()
382 @staticmethod
383 def _is_section_tag(name):
384 return name in ('Returns:', 'Since:',
385 # those are often singular or plural
386 'Note:', 'Notes:',
387 'Example:', 'Examples:',
388 'TODO:')
390 def _append_body_line(self, line):
392 Process a line of documentation text in the body section.
394 If this a symbol line and it is the section's first line, this
395 is a definition documentation block for that symbol.
397 If it's a definition documentation block, another symbol line
398 begins the argument section for the argument named by it, and
399 a section tag begins an additional section. Start that
400 section and append the line to it.
402 Else, append the line to the current section.
404 name = line.split(' ', 1)[0]
405 # FIXME not nice: things like '# @foo:' and '# @foo: ' aren't
406 # recognized, and get silently treated as ordinary text
407 if not self.symbol and not self.body.text and line.startswith('@'):
408 if not line.endswith(':'):
409 raise QAPIParseError(self._parser, "line should end with ':'")
410 self.symbol = line[1:-1]
411 # FIXME invalid names other than the empty string aren't flagged
412 if not self.symbol:
413 raise QAPIParseError(self._parser, "invalid name")
414 elif self.symbol:
415 # This is a definition documentation block
416 if name.startswith('@') and name.endswith(':'):
417 self._append_line = self._append_args_line
418 self._append_args_line(line)
419 elif line == 'Features:':
420 self._append_line = self._append_features_line
421 elif self._is_section_tag(name):
422 self._append_line = self._append_various_line
423 self._append_various_line(line)
424 else:
425 self._append_freeform(line.strip())
426 else:
427 # This is a free-form documentation block
428 self._append_freeform(line.strip())
430 def _append_args_line(self, line):
432 Process a line of documentation text in an argument section.
434 A symbol line begins the next argument section, a section tag
435 section or a non-indented line after a blank line begins an
436 additional section. Start that section and append the line to
439 Else, append the line to the current section.
442 name = line.split(' ', 1)[0]
444 if name.startswith('@') and name.endswith(':'):
445 line = line[len(name)+1:]
446 self._start_args_section(name[1:-1])
447 elif self._is_section_tag(name):
448 self._append_line = self._append_various_line
449 self._append_various_line(line)
450 return
451 elif (self._section.text.endswith('\n\n')
452 and line and not line[0].isspace()):
453 if line == 'Features:':
454 self._append_line = self._append_features_line
455 else:
456 self._start_section()
457 self._append_line = self._append_various_line
458 self._append_various_line(line)
459 return
461 self._append_freeform(line.strip())
463 def _append_features_line(self, line):
464 name = line.split(' ', 1)[0]
466 if name.startswith('@') and name.endswith(':'):
467 line = line[len(name)+1:]
468 self._start_features_section(name[1:-1])
469 elif self._is_section_tag(name):
470 self._append_line = self._append_various_line
471 self._append_various_line(line)
472 return
473 elif (self._section.text.endswith('\n\n')
474 and line and not line[0].isspace()):
475 self._start_section()
476 self._append_line = self._append_various_line
477 self._append_various_line(line)
478 return
480 self._append_freeform(line.strip())
482 def _append_various_line(self, line):
484 Process a line of documentation text in an additional section.
486 A symbol line is an error.
488 A section tag begins an additional section. Start that
489 section and append the line to it.
491 Else, append the line to the current section.
493 name = line.split(' ', 1)[0]
495 if name.startswith('@') and name.endswith(':'):
496 raise QAPIParseError(self._parser,
497 "'%s' can't follow '%s' section"
498 % (name, self.sections[0].name))
499 elif self._is_section_tag(name):
500 line = line[len(name)+1:]
501 self._start_section(name[:-1])
503 if (not self._section.name or
504 not self._section.name.startswith('Example')):
505 line = line.strip()
507 self._append_freeform(line)
509 def _start_symbol_section(self, symbols_dict, name):
510 # FIXME invalid names other than the empty string aren't flagged
511 if not name:
512 raise QAPIParseError(self._parser, "invalid parameter name")
513 if name in symbols_dict:
514 raise QAPIParseError(self._parser,
515 "'%s' parameter name duplicated" % name)
516 assert not self.sections
517 self._end_section()
518 self._section = QAPIDoc.ArgSection(name)
519 symbols_dict[name] = self._section
521 def _start_args_section(self, name):
522 self._start_symbol_section(self.args, name)
524 def _start_features_section(self, name):
525 self._start_symbol_section(self.features, name)
527 def _start_section(self, name=None):
528 if name in ('Returns', 'Since') and self.has_section(name):
529 raise QAPIParseError(self._parser,
530 "duplicated '%s' section" % name)
531 self._end_section()
532 self._section = QAPIDoc.Section(name)
533 self.sections.append(self._section)
535 def _end_section(self):
536 if self._section:
537 text = self._section.text = self._section.text.strip()
538 if self._section.name and (not text or text.isspace()):
539 raise QAPIParseError(
540 self._parser,
541 "empty doc section '%s'" % self._section.name)
542 self._section = None
544 def _append_freeform(self, line):
545 match = re.match(r'(@\S+:)', line)
546 if match:
547 raise QAPIParseError(self._parser,
548 "'%s' not allowed in free-form documentation"
549 % match.group(1))
550 self._section.append(line)
552 def connect_member(self, member):
553 if member.name not in self.args:
554 # Undocumented TODO outlaw
555 self.args[member.name] = QAPIDoc.ArgSection(member.name)
556 self.args[member.name].connect(member)
558 def connect_feature(self, feature):
559 if feature.name not in self.features:
560 raise QAPISemError(feature.info,
561 "feature '%s' lacks documentation"
562 % feature.name)
563 self.features[feature.name] = QAPIDoc.ArgSection(feature.name)
564 self.features[feature.name].connect(feature)
566 def check_expr(self, expr):
567 if self.has_section('Returns') and 'command' not in expr:
568 raise QAPISemError(self.info,
569 "'Returns:' is only valid for commands")
571 def check(self):
573 def check_args_section(args, info, what):
574 bogus = [name for name, section in args.items()
575 if not section.member]
576 if bogus:
577 raise QAPISemError(
578 self.info,
579 "documented member%s '%s' %s not exist"
580 % ("s" if len(bogus) > 1 else "",
581 "', '".join(bogus),
582 "do" if len(bogus) > 1 else "does"))
584 check_args_section(self.args, self.info, 'members')
585 check_args_section(self.features, self.info, 'features')