1 # -*- coding: utf-8 -*-
3 # Copyright IBM, Corp. 2011
4 # Copyright (c) 2013-2021 Red Hat Inc.
7 # Anthony Liguori <aliguori@us.ibm.com>
8 # Markus Armbruster <armbru@redhat.com>
9 # Eric Blake <eblake@redhat.com>
10 # Marc-André Lureau <marcandre.lureau@redhat.com>
11 # John Snow <jsnow@redhat.com>
13 # This work is licensed under the terms of the GNU GPL, version 2.
14 # See the COPYING file in the top-level directory.
17 Normalize and validate (context-free) QAPI schema expression structures.
19 `QAPISchemaParser` parses a QAPI schema into abstract syntax trees
20 consisting of dict, list, str, bool, and int nodes. This module ensures
21 that these nested structures have the correct type(s) and key(s) where
22 appropriate for the QAPI context-free grammar.
24 The QAPI schema expression language allows for certain syntactic sugar;
25 this module also handles the normalization process of these nested
28 See `check_exprs` for the main entry point.
30 See `schema.QAPISchema` for processing into native Python data
31 structures and contextual semantic validation.
45 from .common
import c_name
46 from .error
import QAPISemError
47 from .parser
import QAPIDoc
48 from .source
import QAPISourceInfo
51 # Deserialized JSON objects as returned by the parser.
52 # The values of this mapping are not necessary to exhaustively type
53 # here (and also not practical as long as mypy lacks recursive
54 # types), because the purpose of this module is to interrogate that
56 _JSONObject
= Dict
[str, object]
59 # See check_name_str(), below.
60 valid_name
= re
.compile(r
'(__[a-z0-9.-]+_)?'
62 r
'([a-z][a-z0-9_-]*)$', re
.IGNORECASE
)
65 def check_name_is_str(name
: object,
69 Ensure that ``name`` is a ``str``.
71 :raise QAPISemError: When ``name`` fails validation.
73 if not isinstance(name
, str):
74 raise QAPISemError(info
, "%s requires a string name" % source
)
77 def check_name_str(name
: str, info
: QAPISourceInfo
, source
: str) -> str:
79 Ensure that ``name`` is a valid QAPI name.
81 A valid name consists of ASCII letters, digits, ``-``, and ``_``,
82 starting with a letter. It may be prefixed by a downstream prefix
83 of the form __RFQDN_, or the experimental prefix ``x-``. If both
84 prefixes are present, the __RFDQN_ prefix goes first.
86 A valid name cannot start with ``q_``, which is reserved.
88 :param name: Name to check.
89 :param info: QAPI schema source file information.
90 :param source: Error string describing what ``name`` belongs to.
92 :raise QAPISemError: When ``name`` fails validation.
93 :return: The stem of the valid name, with no prefixes.
95 # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
96 # and 'q_obj_*' implicit type names.
97 match
= valid_name
.match(name
)
98 if not match
or c_name(name
, False).startswith('q_'):
99 raise QAPISemError(info
, "%s has an invalid name" % source
)
100 return match
.group(3)
103 def check_name_upper(name
: str, info
: QAPISourceInfo
, source
: str) -> None:
105 Ensure that ``name`` is a valid event name.
107 This means it must be a valid QAPI name as checked by
108 `check_name_str()`, but where the stem prohibits lowercase
109 characters and ``-``.
111 :param name: Name to check.
112 :param info: QAPI schema source file information.
113 :param source: Error string describing what ``name`` belongs to.
115 :raise QAPISemError: When ``name`` fails validation.
117 stem
= check_name_str(name
, info
, source
)
118 if re
.search(r
'[a-z-]', stem
):
120 info
, "name of %s must not use lowercase or '-'" % source
)
123 def check_name_lower(name
: str, info
: QAPISourceInfo
, source
: str,
124 permit_upper
: bool = False,
125 permit_underscore
: bool = False) -> None:
127 Ensure that ``name`` is a valid command or member name.
129 This means it must be a valid QAPI name as checked by
130 `check_name_str()`, but where the stem prohibits uppercase
131 characters and ``_``.
133 :param name: Name to check.
134 :param info: QAPI schema source file information.
135 :param source: Error string describing what ``name`` belongs to.
136 :param permit_upper: Additionally permit uppercase.
137 :param permit_underscore: Additionally permit ``_``.
139 :raise QAPISemError: When ``name`` fails validation.
141 stem
= check_name_str(name
, info
, source
)
142 if ((not permit_upper
and re
.search(r
'[A-Z]', stem
))
143 or (not permit_underscore
and '_' in stem
)):
145 info
, "name of %s must not use uppercase or '_'" % source
)
148 def check_name_camel(name
: str, info
: QAPISourceInfo
, source
: str) -> None:
150 Ensure that ``name`` is a valid user-defined type name.
152 This means it must be a valid QAPI name as checked by
153 `check_name_str()`, but where the stem must be in CamelCase.
155 :param name: Name to check.
156 :param info: QAPI schema source file information.
157 :param source: Error string describing what ``name`` belongs to.
159 :raise QAPISemError: When ``name`` fails validation.
161 stem
= check_name_str(name
, info
, source
)
162 if not re
.match(r
'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem
):
163 raise QAPISemError(info
, "name of %s must use CamelCase" % source
)
166 def check_defn_name_str(name
: str, info
: QAPISourceInfo
, meta
: str) -> None:
168 Ensure that ``name`` is a valid definition name.
170 Based on the value of ``meta``, this means that:
171 - 'event' names adhere to `check_name_upper()`.
172 - 'command' names adhere to `check_name_lower()`.
173 - Else, meta is a type, and must pass `check_name_camel()`.
174 These names must not end with ``List``.
176 :param name: Name to check.
177 :param info: QAPI schema source file information.
178 :param meta: Meta-type name of the QAPI expression.
180 :raise QAPISemError: When ``name`` fails validation.
183 check_name_upper(name
, info
, meta
)
184 elif meta
== 'command':
187 permit_underscore
=name
in info
.pragma
.command_name_exceptions
)
189 check_name_camel(name
, info
, meta
)
190 if name
.endswith('List'):
192 info
, "%s name should not end in 'List'" % meta
)
195 def check_keys(value
: _JSONObject
,
196 info
: QAPISourceInfo
,
198 required
: Collection
[str],
199 optional
: Collection
[str]) -> None:
201 Ensure that a dict has a specific set of keys.
203 :param value: The dict to check.
204 :param info: QAPI schema source file information.
205 :param source: Error string describing this ``value``.
206 :param required: Keys that *must* be present.
207 :param optional: Keys that *may* be present.
209 :raise QAPISemError: When unknown keys are present.
212 def pprint(elems
: Iterable
[str]) -> str:
213 return ', '.join("'" + e
+ "'" for e
in sorted(elems
))
215 missing
= set(required
) - set(value
)
220 % (source
, 's' if len(missing
) > 1 else '',
222 allowed
= set(required
) |
set(optional
)
223 unknown
= set(value
) - allowed
227 "%s has unknown key%s %s\nValid keys are %s."
228 % (source
, 's' if len(unknown
) > 1 else '',
229 pprint(unknown
), pprint(allowed
)))
232 def check_flags(expr
: _JSONObject
, info
: QAPISourceInfo
) -> None:
234 Ensure flag members (if present) have valid values.
236 :param expr: The expression to validate.
237 :param info: QAPI schema source file information.
240 When certain flags have an invalid value, or when
241 incompatible flags are present.
243 for key
in ('gen', 'success-response'):
244 if key
in expr
and expr
[key
] is not False:
246 info
, "flag '%s' may only use false value" % key
)
247 for key
in ('boxed', 'allow-oob', 'allow-preconfig', 'coroutine'):
248 if key
in expr
and expr
[key
] is not True:
250 info
, "flag '%s' may only use true value" % key
)
251 if 'allow-oob' in expr
and 'coroutine' in expr
:
252 # This is not necessarily a fundamental incompatibility, but
253 # we don't have a use case and the desired semantics isn't
254 # obvious. The simplest solution is to forbid it until we get
256 raise QAPISemError(info
, "flags 'allow-oob' and 'coroutine' "
260 def check_if(expr
: _JSONObject
, info
: QAPISourceInfo
, source
: str) -> None:
262 Validate the ``if`` member of an object.
264 The ``if`` member may be either a ``str`` or a dict.
266 :param expr: The expression containing the ``if`` member to validate.
267 :param info: QAPI schema source file information.
268 :param source: Error string describing ``expr``.
271 When the "if" member fails validation, or when there are no
272 non-empty conditions.
276 def _check_if(cond
: Union
[str, object]) -> None:
277 if isinstance(cond
, str):
278 if not re
.fullmatch(r
'[A-Z][A-Z0-9_]*', cond
):
281 "'if' condition '%s' of %s is not a valid identifier"
285 if not isinstance(cond
, dict):
288 "'if' condition of %s must be a string or an object" % source
)
289 check_keys(cond
, info
, "'if' condition of %s" % source
, [],
290 ["all", "any", "not"])
294 "'if' condition of %s has conflicting keys" % source
)
297 _check_if(cond
['not'])
299 _check_infix('all', cond
['all'])
301 _check_infix('any', cond
['any'])
303 def _check_infix(operator
: str, operands
: object) -> None:
304 if not isinstance(operands
, list):
307 "'%s' condition of %s must be an array"
308 % (operator
, source
))
311 info
, "'if' condition [] of %s is useless" % source
)
312 for operand
in operands
:
315 ifcond
= expr
.get('if')
322 def normalize_members(members
: object) -> None:
324 Normalize a "members" value.
326 If ``members`` is a dict, for every value in that dict, if that
327 value is not itself already a dict, normalize it to
331 :sugared: ``Dict[str, Union[str, TypeRef]]``
332 :canonical: ``Dict[str, TypeRef]``
334 :param members: The members value to normalize.
336 :return: None, ``members`` is normalized in-place as needed.
338 if isinstance(members
, dict):
339 for key
, arg
in members
.items():
340 if isinstance(arg
, dict):
342 members
[key
] = {'type': arg
}
345 def check_type(value
: Optional
[object],
346 info
: QAPISourceInfo
,
348 allow_array
: bool = False,
349 allow_dict
: Union
[bool, str] = False) -> None:
351 Normalize and validate the QAPI type of ``value``.
353 Python types of ``str`` or ``None`` are always allowed.
355 :param value: The value to check.
356 :param info: QAPI schema source file information.
357 :param source: Error string describing this ``value``.
359 Allow a ``List[str]`` of length 1, which indicates an array of
360 the type named by the list element.
362 Allow a dict. Its members can be struct type members or union
363 branches. When the value of ``allow_dict`` is in pragma
364 ``member-name-exceptions``, the dict's keys may violate the
365 member naming rules. The dict members are normalized in place.
367 :raise QAPISemError: When ``value`` fails validation.
368 :return: None, ``value`` is normalized in-place as needed.
374 if isinstance(value
, str):
378 if isinstance(value
, list):
380 raise QAPISemError(info
, "%s cannot be an array" % source
)
381 if len(value
) != 1 or not isinstance(value
[0], str):
382 raise QAPISemError(info
,
383 "%s: array type must contain single type name" %
390 raise QAPISemError(info
, "%s should be a type name" % source
)
392 if not isinstance(value
, dict):
393 raise QAPISemError(info
,
394 "%s should be an object or type name" % source
)
397 if isinstance(allow_dict
, str):
398 permissive
= allow_dict
in info
.pragma
.member_name_exceptions
400 # value is a dictionary, check that each member is okay
401 for (key
, arg
) in value
.items():
402 key_source
= "%s member '%s'" % (source
, key
)
403 if key
.startswith('*'):
405 check_name_lower(key
, info
, key_source
,
406 permit_upper
=permissive
,
407 permit_underscore
=permissive
)
408 if c_name(key
, False) == 'u' or c_name(key
, False).startswith('has_'):
409 raise QAPISemError(info
, "%s uses reserved name" % key_source
)
410 check_keys(arg
, info
, key_source
, ['type'], ['if', 'features'])
411 check_if(arg
, info
, key_source
)
412 check_features(arg
.get('features'), info
)
413 check_type(arg
['type'], info
, key_source
, allow_array
=True)
416 def check_features(features
: Optional
[object],
417 info
: QAPISourceInfo
) -> None:
419 Normalize and validate the ``features`` member.
421 ``features`` may be a ``list`` of either ``str`` or ``dict``.
422 Any ``str`` element will be normalized to ``{'name': element}``.
425 :sugared: ``List[Union[str, Feature]]``
426 :canonical: ``List[Feature]``
428 :param features: The features member value to validate.
429 :param info: QAPI schema source file information.
431 :raise QAPISemError: When ``features`` fails validation.
432 :return: None, ``features`` is normalized in-place as needed.
436 if not isinstance(features
, list):
437 raise QAPISemError(info
, "'features' must be an array")
438 features
[:] = [f
if isinstance(f
, dict) else {'name': f
}
440 for feat
in features
:
441 source
= "'features' member"
442 assert isinstance(feat
, dict)
443 check_keys(feat
, info
, source
, ['name'], ['if'])
444 check_name_is_str(feat
['name'], info
, source
)
445 source
= "%s '%s'" % (source
, feat
['name'])
446 check_name_str(feat
['name'], info
, source
)
447 check_if(feat
, info
, source
)
450 def check_enum(expr
: _JSONObject
, info
: QAPISourceInfo
) -> None:
452 Normalize and validate this expression as an ``enum`` definition.
454 :param expr: The expression to validate.
455 :param info: QAPI schema source file information.
457 :raise QAPISemError: When ``expr`` is not a valid ``enum``.
458 :return: None, ``expr`` is normalized in-place as needed.
461 members
= expr
['data']
462 prefix
= expr
.get('prefix')
464 if not isinstance(members
, list):
465 raise QAPISemError(info
, "'data' must be an array")
466 if prefix
is not None and not isinstance(prefix
, str):
467 raise QAPISemError(info
, "'prefix' must be a string")
469 permissive
= name
in info
.pragma
.member_name_exceptions
471 members
[:] = [m
if isinstance(m
, dict) else {'name': m
}
473 for member
in members
:
474 source
= "'data' member"
475 check_keys(member
, info
, source
, ['name'], ['if', 'features'])
476 member_name
= member
['name']
477 check_name_is_str(member_name
, info
, source
)
478 source
= "%s '%s'" % (source
, member_name
)
479 # Enum members may start with a digit
480 if member_name
[0].isdigit():
481 member_name
= 'd' + member_name
# Hack: hide the digit
482 check_name_lower(member_name
, info
, source
,
483 permit_upper
=permissive
,
484 permit_underscore
=permissive
)
485 check_if(member
, info
, source
)
486 check_features(member
.get('features'), info
)
489 def check_struct(expr
: _JSONObject
, info
: QAPISourceInfo
) -> None:
491 Normalize and validate this expression as a ``struct`` definition.
493 :param expr: The expression to validate.
494 :param info: QAPI schema source file information.
496 :raise QAPISemError: When ``expr`` is not a valid ``struct``.
497 :return: None, ``expr`` is normalized in-place as needed.
499 name
= cast(str, expr
['struct']) # Checked in check_exprs
500 members
= expr
['data']
502 check_type(members
, info
, "'data'", allow_dict
=name
)
503 check_type(expr
.get('base'), info
, "'base'")
506 def check_union(expr
: _JSONObject
, info
: QAPISourceInfo
) -> None:
508 Normalize and validate this expression as a ``union`` definition.
510 :param expr: The expression to validate.
511 :param info: QAPI schema source file information.
513 :raise QAPISemError: when ``expr`` is not a valid ``union``.
514 :return: None, ``expr`` is normalized in-place as needed.
516 name
= cast(str, expr
['union']) # Checked in check_exprs
518 discriminator
= expr
['discriminator']
519 members
= expr
['data']
521 check_type(base
, info
, "'base'", allow_dict
=name
)
522 check_name_is_str(discriminator
, info
, "'discriminator'")
524 if not isinstance(members
, dict):
525 raise QAPISemError(info
, "'data' must be an object")
527 for (key
, value
) in members
.items():
528 source
= "'data' member '%s'" % key
529 check_keys(value
, info
, source
, ['type'], ['if'])
530 check_if(value
, info
, source
)
531 check_type(value
['type'], info
, source
, allow_array
=not base
)
534 def check_alternate(expr
: _JSONObject
, info
: QAPISourceInfo
) -> None:
536 Normalize and validate this expression as an ``alternate`` definition.
538 :param expr: The expression to validate.
539 :param info: QAPI schema source file information.
541 :raise QAPISemError: When ``expr`` is not a valid ``alternate``.
542 :return: None, ``expr`` is normalized in-place as needed.
544 members
= expr
['data']
547 raise QAPISemError(info
, "'data' must not be empty")
549 if not isinstance(members
, dict):
550 raise QAPISemError(info
, "'data' must be an object")
552 for (key
, value
) in members
.items():
553 source
= "'data' member '%s'" % key
554 check_name_lower(key
, info
, source
)
555 check_keys(value
, info
, source
, ['type'], ['if'])
556 check_if(value
, info
, source
)
557 check_type(value
['type'], info
, source
)
560 def check_command(expr
: _JSONObject
, info
: QAPISourceInfo
) -> None:
562 Normalize and validate this expression as a ``command`` definition.
564 :param expr: The expression to validate.
565 :param info: QAPI schema source file information.
567 :raise QAPISemError: When ``expr`` is not a valid ``command``.
568 :return: None, ``expr`` is normalized in-place as needed.
570 args
= expr
.get('data')
571 rets
= expr
.get('returns')
572 boxed
= expr
.get('boxed', False)
574 if boxed
and args
is None:
575 raise QAPISemError(info
, "'boxed': true requires 'data'")
576 check_type(args
, info
, "'data'", allow_dict
=not boxed
)
577 check_type(rets
, info
, "'returns'", allow_array
=True)
580 def check_event(expr
: _JSONObject
, info
: QAPISourceInfo
) -> None:
582 Normalize and validate this expression as an ``event`` definition.
584 :param expr: The expression to validate.
585 :param info: QAPI schema source file information.
587 :raise QAPISemError: When ``expr`` is not a valid ``event``.
588 :return: None, ``expr`` is normalized in-place as needed.
590 args
= expr
.get('data')
591 boxed
= expr
.get('boxed', False)
593 if boxed
and args
is None:
594 raise QAPISemError(info
, "'boxed': true requires 'data'")
595 check_type(args
, info
, "'data'", allow_dict
=not boxed
)
598 def check_exprs(exprs
: List
[_JSONObject
]) -> List
[_JSONObject
]:
600 Validate and normalize a list of parsed QAPI schema expressions.
602 This function accepts a list of expressions and metadata as returned
603 by the parser. It destructively normalizes the expressions in-place.
605 :param exprs: The list of expressions to normalize and validate.
607 :raise QAPISemError: When any expression fails validation.
608 :return: The same list of expressions (now modified).
610 for expr_elem
in exprs
:
612 assert isinstance(expr_elem
['expr'], dict)
613 for key
in expr_elem
['expr'].keys():
614 assert isinstance(key
, str)
615 expr
: _JSONObject
= expr_elem
['expr']
618 assert isinstance(expr_elem
['info'], QAPISourceInfo
)
619 info
: QAPISourceInfo
= expr_elem
['info']
622 tmp
= expr_elem
.get('doc')
623 assert tmp
is None or isinstance(tmp
, QAPIDoc
)
624 doc
: Optional
[QAPIDoc
] = tmp
626 if 'include' in expr
:
629 metas
= expr
.keys() & {'enum', 'struct', 'union', 'alternate',
634 "expression must have exactly one key"
635 " 'enum', 'struct', 'union', 'alternate',"
636 " 'command', 'event'")
639 check_name_is_str(expr
[meta
], info
, "'%s'" % meta
)
640 name
= cast(str, expr
[meta
])
641 info
.set_defn(meta
, name
)
642 check_defn_name_str(name
, info
, meta
)
645 if doc
.symbol
!= name
:
647 info
, "documentation comment is for '%s'" % doc
.symbol
)
649 elif info
.pragma
.doc_required
:
650 raise QAPISemError(info
,
651 "documentation comment required")
654 check_keys(expr
, info
, meta
,
655 ['enum', 'data'], ['if', 'features', 'prefix'])
656 check_enum(expr
, info
)
657 elif meta
== 'union':
658 check_keys(expr
, info
, meta
,
659 ['union', 'base', 'discriminator', 'data'],
661 normalize_members(expr
.get('base'))
662 normalize_members(expr
['data'])
663 check_union(expr
, info
)
664 elif meta
== 'alternate':
665 check_keys(expr
, info
, meta
,
666 ['alternate', 'data'], ['if', 'features'])
667 normalize_members(expr
['data'])
668 check_alternate(expr
, info
)
669 elif meta
== 'struct':
670 check_keys(expr
, info
, meta
,
671 ['struct', 'data'], ['base', 'if', 'features'])
672 normalize_members(expr
['data'])
673 check_struct(expr
, info
)
674 elif meta
== 'command':
675 check_keys(expr
, info
, meta
,
677 ['data', 'returns', 'boxed', 'if', 'features',
678 'gen', 'success-response', 'allow-oob',
679 'allow-preconfig', 'coroutine'])
680 normalize_members(expr
.get('data'))
681 check_command(expr
, info
)
682 elif meta
== 'event':
683 check_keys(expr
, info
, meta
,
684 ['event'], ['data', 'boxed', 'if', 'features'])
685 normalize_members(expr
.get('data'))
686 check_event(expr
, info
)
688 assert False, 'unexpected meta type'
690 check_if(expr
, info
, meta
)
691 check_features(expr
.get('features'), info
)
692 check_flags(expr
, info
)