scripts/qapi/expr.py

   1 # -*- coding: utf-8 -*-
   2 #
   3 # Copyright IBM, Corp. 2011
   4 # Copyright (c) 2013-2021 Red Hat Inc.
   5 #
   6 # Authors:
   7 #  Anthony Liguori <aliguori@us.ibm.com>
   8 #  Markus Armbruster <armbru@redhat.com>
   9 #  Eric Blake <eblake@redhat.com>
  10 #  Marc-André Lureau <marcandre.lureau@redhat.com>
  11 #  John Snow <jsnow@redhat.com>
  12 #
  13 # This work is licensed under the terms of the GNU GPL, version 2.
  14 # See the COPYING file in the top-level directory.
  15
  16 """
  17 Normalize and validate (context-free) QAPI schema expression structures.
  18
  19 `QAPISchemaParser` parses a QAPI schema into abstract syntax trees
  20 consisting of dict, list, str, bool, and int nodes.  This module ensures
  21 that these nested structures have the correct type(s) and key(s) where
  22 appropriate for the QAPI context-free grammar.
  23
  24 The QAPI schema expression language allows for certain syntactic sugar;
  25 this module also handles the normalization process of these nested
  26 structures.
  27
  28 See `check_exprs` for the main entry point.
  29
  30 See `schema.QAPISchema` for processing into native Python data
  31 structures and contextual semantic validation.
  32 """
  33
  34 import re
  35 from typing import (
  36     Dict,
  37     Iterable,
  38     List,
  39     Optional,
  40     Union,
  41     cast,
  42 )
  43
  44 from .common import c_name
  45 from .error import QAPISemError
  46 from .parser import QAPIExpression
  47 from .source import QAPISourceInfo
  48
  49
  50 # See check_name_str(), below.
  51 valid_name = re.compile(r'(__[a-z0-9.-]+_)?'
  52                         r'(x-)?'
  53                         r'([a-z][a-z0-9_-]*)$', re.IGNORECASE)
  54
  55
  56 def check_name_is_str(name: object,
  57                       info: QAPISourceInfo,
  58                       source: str) -> None:
  59     """
  60     Ensure that ``name`` is a ``str``.
  61
  62     :raise QAPISemError: When ``name`` fails validation.
  63     """
  64     if not isinstance(name, str):
  65         raise QAPISemError(info, "%s requires a string name" % source)
  66
  67
  68 def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str:
  69     """
  70     Ensure that ``name`` is a valid QAPI name.
  71
  72     A valid name consists of ASCII letters, digits, ``-``, and ``_``,
  73     starting with a letter.  It may be prefixed by a downstream prefix
  74     of the form __RFQDN_, or the experimental prefix ``x-``.  If both
  75     prefixes are present, the __RFDQN_ prefix goes first.
  76
  77     A valid name cannot start with ``q_``, which is reserved.
  78
  79     :param name: Name to check.
  80     :param info: QAPI schema source file information.
  81     :param source: Error string describing what ``name`` belongs to.
  82
  83     :raise QAPISemError: When ``name`` fails validation.
  84     :return: The stem of the valid name, with no prefixes.
  85     """
  86     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
  87     # and 'q_obj_*' implicit type names.
  88     match = valid_name.match(name)
  89     if not match or c_name(name, False).startswith('q_'):
  90         raise QAPISemError(info, "%s has an invalid name" % source)
  91     return match.group(3)
  92
  93
  94 def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None:
  95     """
  96     Ensure that ``name`` is a valid event name.
  97
  98     This means it must be a valid QAPI name as checked by
  99     `check_name_str()`, but where the stem prohibits lowercase
 100     characters and ``-``.
 101
 102     :param name: Name to check.
 103     :param info: QAPI schema source file information.
 104     :param source: Error string describing what ``name`` belongs to.
 105
 106     :raise QAPISemError: When ``name`` fails validation.
 107     """
 108     stem = check_name_str(name, info, source)
 109     if re.search(r'[a-z-]', stem):
 110         raise QAPISemError(
 111             info, "name of %s must not use lowercase or '-'" % source)
 112
 113
 114 def check_name_lower(name: str, info: QAPISourceInfo, source: str,
 115                      permit_upper: bool = False,
 116                      permit_underscore: bool = False) -> None:
 117     """
 118     Ensure that ``name`` is a valid command or member name.
 119
 120     This means it must be a valid QAPI name as checked by
 121     `check_name_str()`, but where the stem prohibits uppercase
 122     characters and ``_``.
 123
 124     :param name: Name to check.
 125     :param info: QAPI schema source file information.
 126     :param source: Error string describing what ``name`` belongs to.
 127     :param permit_upper: Additionally permit uppercase.
 128     :param permit_underscore: Additionally permit ``_``.
 129
 130     :raise QAPISemError: When ``name`` fails validation.
 131     """
 132     stem = check_name_str(name, info, source)
 133     if ((not permit_upper and re.search(r'[A-Z]', stem))
 134             or (not permit_underscore and '_' in stem)):
 135         raise QAPISemError(
 136             info, "name of %s must not use uppercase or '_'" % source)
 137
 138
 139 def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None:
 140     """
 141     Ensure that ``name`` is a valid user-defined type name.
 142
 143     This means it must be a valid QAPI name as checked by
 144     `check_name_str()`, but where the stem must be in CamelCase.
 145
 146     :param name: Name to check.
 147     :param info: QAPI schema source file information.
 148     :param source: Error string describing what ``name`` belongs to.
 149
 150     :raise QAPISemError: When ``name`` fails validation.
 151     """
 152     stem = check_name_str(name, info, source)
 153     if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem):
 154         raise QAPISemError(info, "name of %s must use CamelCase" % source)
 155
 156
 157 def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None:
 158     """
 159     Ensure that ``name`` is a valid definition name.
 160
 161     Based on the value of ``meta``, this means that:
 162       - 'event' names adhere to `check_name_upper()`.
 163       - 'command' names adhere to `check_name_lower()`.
 164       - Else, meta is a type, and must pass `check_name_camel()`.
 165         These names must not end with ``List``.
 166
 167     :param name: Name to check.
 168     :param info: QAPI schema source file information.
 169     :param meta: Meta-type name of the QAPI expression.
 170
 171     :raise QAPISemError: When ``name`` fails validation.
 172     """
 173     if meta == 'event':
 174         check_name_upper(name, info, meta)
 175     elif meta == 'command':
 176         check_name_lower(
 177             name, info, meta,
 178             permit_underscore=name in info.pragma.command_name_exceptions)
 179     else:
 180         check_name_camel(name, info, meta)
 181         if name.endswith('List'):
 182             raise QAPISemError(
 183                 info, "%s name should not end in 'List'" % meta)
 184
 185
 186 def check_keys(value: Dict[str, object],
 187                info: QAPISourceInfo,
 188                source: str,
 189                required: List[str],
 190                optional: List[str]) -> None:
 191     """
 192     Ensure that a dict has a specific set of keys.
 193
 194     :param value: The dict to check.
 195     :param info: QAPI schema source file information.
 196     :param source: Error string describing this ``value``.
 197     :param required: Keys that *must* be present.
 198     :param optional: Keys that *may* be present.
 199
 200     :raise QAPISemError: When unknown keys are present.
 201     """
 202
 203     def pprint(elems: Iterable[str]) -> str:
 204         return ', '.join("'" + e + "'" for e in sorted(elems))
 205
 206     missing = set(required) - set(value)
 207     if missing:
 208         raise QAPISemError(
 209             info,
 210             "%s misses key%s %s"
 211             % (source, 's' if len(missing) > 1 else '',
 212                pprint(missing)))
 213     allowed = set(required) | set(optional)
 214     unknown = set(value) - allowed
 215     if unknown:
 216         raise QAPISemError(
 217             info,
 218             "%s has unknown key%s %s\nValid keys are %s."
 219             % (source, 's' if len(unknown) > 1 else '',
 220                pprint(unknown), pprint(allowed)))
 221
 222
 223 def check_flags(expr: QAPIExpression) -> None:
 224     """
 225     Ensure flag members (if present) have valid values.
 226
 227     :param expr: The expression to validate.
 228
 229     :raise QAPISemError:
 230         When certain flags have an invalid value, or when
 231         incompatible flags are present.
 232     """
 233     for key in ('gen', 'success-response'):
 234         if key in expr and expr[key] is not False:
 235             raise QAPISemError(
 236                 expr.info, "flag '%s' may only use false value" % key)
 237     for key in ('boxed', 'allow-oob', 'allow-preconfig', 'coroutine'):
 238         if key in expr and expr[key] is not True:
 239             raise QAPISemError(
 240                 expr.info, "flag '%s' may only use true value" % key)
 241     if 'allow-oob' in expr and 'coroutine' in expr:
 242         # This is not necessarily a fundamental incompatibility, but
 243         # we don't have a use case and the desired semantics isn't
 244         # obvious.  The simplest solution is to forbid it until we get
 245         # a use case for it.
 246         raise QAPISemError(
 247             expr.info, "flags 'allow-oob' and 'coroutine' are incompatible")
 248
 249
 250 def check_if(expr: Dict[str, object],
 251              info: QAPISourceInfo, source: str) -> None:
 252     """
 253     Validate the ``if`` member of an object.
 254
 255     The ``if`` member may be either a ``str`` or a dict.
 256
 257     :param expr: The expression containing the ``if`` member to validate.
 258     :param info: QAPI schema source file information.
 259     :param source: Error string describing ``expr``.
 260
 261     :raise QAPISemError:
 262         When the "if" member fails validation, or when there are no
 263         non-empty conditions.
 264     :return: None
 265     """
 266
 267     def _check_if(cond: Union[str, object]) -> None:
 268         if isinstance(cond, str):
 269             if not re.fullmatch(r'[A-Z][A-Z0-9_]*', cond):
 270                 raise QAPISemError(
 271                     info,
 272                     "'if' condition '%s' of %s is not a valid identifier"
 273                     % (cond, source))
 274             return
 275
 276         if not isinstance(cond, dict):
 277             raise QAPISemError(
 278                 info,
 279                 "'if' condition of %s must be a string or an object" % source)
 280         check_keys(cond, info, "'if' condition of %s" % source, [],
 281                    ["all", "any", "not"])
 282         if len(cond) != 1:
 283             raise QAPISemError(
 284                 info,
 285                 "'if' condition of %s has conflicting keys" % source)
 286
 287         if 'not' in cond:
 288             _check_if(cond['not'])
 289         elif 'all' in cond:
 290             _check_infix('all', cond['all'])
 291         else:
 292             _check_infix('any', cond['any'])
 293
 294     def _check_infix(operator: str, operands: object) -> None:
 295         if not isinstance(operands, list):
 296             raise QAPISemError(
 297                 info,
 298                 "'%s' condition of %s must be an array"
 299                 % (operator, source))
 300         if not operands:
 301             raise QAPISemError(
 302                 info, "'if' condition [] of %s is useless" % source)
 303         for operand in operands:
 304             _check_if(operand)
 305
 306     ifcond = expr.get('if')
 307     if ifcond is None:
 308         return
 309
 310     _check_if(ifcond)
 311
 312
 313 def normalize_members(members: object) -> None:
 314     """
 315     Normalize a "members" value.
 316
 317     If ``members`` is a dict, for every value in that dict, if that
 318     value is not itself already a dict, normalize it to
 319     ``{'type': value}``.
 320
 321     :forms:
 322       :sugared: ``Dict[str, Union[str, TypeRef]]``
 323       :canonical: ``Dict[str, TypeRef]``
 324
 325     :param members: The members value to normalize.
 326
 327     :return: None, ``members`` is normalized in-place as needed.
 328     """
 329     if isinstance(members, dict):
 330         for key, arg in members.items():
 331             if isinstance(arg, dict):
 332                 continue
 333             members[key] = {'type': arg}
 334
 335
 336 def check_type_name(value: Optional[object],
 337                     info: QAPISourceInfo, source: str) -> None:
 338     if value is not None and not isinstance(value, str):
 339         raise QAPISemError(info, "%s should be a type name" % source)
 340
 341
 342 def check_type_name_or_array(value: Optional[object],
 343                              info: QAPISourceInfo, source: str) -> None:
 344     if value is None or isinstance(value, str):
 345         return
 346
 347     if not isinstance(value, list):
 348         raise QAPISemError(info,
 349                            "%s should be a type name or array" % source)
 350
 351     if len(value) != 1 or not isinstance(value[0], str):
 352         raise QAPISemError(info,
 353                            "%s: array type must contain single type name" %
 354                            source)
 355
 356
 357 def check_type_implicit(value: Optional[object],
 358                         info: QAPISourceInfo, source: str,
 359                         parent_name: Optional[str]) -> None:
 360     """
 361     Normalize and validate an optional implicit struct type.
 362
 363     Accept ``None`` or a ``dict`` defining an implicit struct type.
 364     The latter is normalized in place.
 365
 366     :param value: The value to check.
 367     :param info: QAPI schema source file information.
 368     :param source: Error string describing this ``value``.
 369     :param parent_name:
 370         When the value of ``parent_name`` is in pragma
 371         ``member-name-exceptions``, an implicit struct type may
 372         violate the member naming rules.
 373
 374     :raise QAPISemError: When ``value`` fails validation.
 375     :return: None
 376     """
 377     if value is None:
 378         return
 379
 380     if not isinstance(value, dict):
 381         raise QAPISemError(info,
 382                            "%s should be an object or type name" % source)
 383
 384     permissive = parent_name in info.pragma.member_name_exceptions
 385
 386     for (key, arg) in value.items():
 387         key_source = "%s member '%s'" % (source, key)
 388         if key.startswith('*'):
 389             key = key[1:]
 390         check_name_lower(key, info, key_source,
 391                          permit_upper=permissive,
 392                          permit_underscore=permissive)
 393         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 394             raise QAPISemError(info, "%s uses reserved name" % key_source)
 395         check_keys(arg, info, key_source, ['type'], ['if', 'features'])
 396         check_if(arg, info, key_source)
 397         check_features(arg.get('features'), info)
 398         check_type_name_or_array(arg['type'], info, key_source)
 399
 400
 401 def check_type_name_or_implicit(value: Optional[object],
 402                                 info: QAPISourceInfo, source: str,
 403                                 parent_name: Optional[str]) -> None:
 404     if value is None or isinstance(value, str):
 405         return
 406
 407     check_type_implicit(value, info, source, parent_name)
 408
 409
 410 def check_features(features: Optional[object],
 411                    info: QAPISourceInfo) -> None:
 412     """
 413     Normalize and validate the ``features`` member.
 414
 415     ``features`` may be a ``list`` of either ``str`` or ``dict``.
 416     Any ``str`` element will be normalized to ``{'name': element}``.
 417
 418     :forms:
 419       :sugared: ``List[Union[str, Feature]]``
 420       :canonical: ``List[Feature]``
 421
 422     :param features: The features member value to validate.
 423     :param info: QAPI schema source file information.
 424
 425     :raise QAPISemError: When ``features`` fails validation.
 426     :return: None, ``features`` is normalized in-place as needed.
 427     """
 428     if features is None:
 429         return
 430     if not isinstance(features, list):
 431         raise QAPISemError(info, "'features' must be an array")
 432     features[:] = [f if isinstance(f, dict) else {'name': f}
 433                    for f in features]
 434     for feat in features:
 435         source = "'features' member"
 436         assert isinstance(feat, dict)
 437         check_keys(feat, info, source, ['name'], ['if'])
 438         check_name_is_str(feat['name'], info, source)
 439         source = "%s '%s'" % (source, feat['name'])
 440         check_name_lower(feat['name'], info, source)
 441         check_if(feat, info, source)
 442
 443
 444 def check_enum(expr: QAPIExpression) -> None:
 445     """
 446     Normalize and validate this expression as an ``enum`` definition.
 447
 448     :param expr: The expression to validate.
 449
 450     :raise QAPISemError: When ``expr`` is not a valid ``enum``.
 451     :return: None, ``expr`` is normalized in-place as needed.
 452     """
 453     name = expr['enum']
 454     members = expr['data']
 455     prefix = expr.get('prefix')
 456     info = expr.info
 457
 458     if not isinstance(members, list):
 459         raise QAPISemError(info, "'data' must be an array")
 460     if prefix is not None and not isinstance(prefix, str):
 461         raise QAPISemError(info, "'prefix' must be a string")
 462
 463     permissive = name in info.pragma.member_name_exceptions
 464
 465     members[:] = [m if isinstance(m, dict) else {'name': m}
 466                   for m in members]
 467     for member in members:
 468         source = "'data' member"
 469         check_keys(member, info, source, ['name'], ['if', 'features'])
 470         member_name = member['name']
 471         check_name_is_str(member_name, info, source)
 472         source = "%s '%s'" % (source, member_name)
 473         # Enum members may start with a digit
 474         if member_name[0].isdigit():
 475             member_name = 'd' + member_name  # Hack: hide the digit
 476         check_name_lower(member_name, info, source,
 477                          permit_upper=permissive,
 478                          permit_underscore=permissive)
 479         check_if(member, info, source)
 480         check_features(member.get('features'), info)
 481
 482
 483 def check_struct(expr: QAPIExpression) -> None:
 484     """
 485     Normalize and validate this expression as a ``struct`` definition.
 486
 487     :param expr: The expression to validate.
 488
 489     :raise QAPISemError: When ``expr`` is not a valid ``struct``.
 490     :return: None, ``expr`` is normalized in-place as needed.
 491     """
 492     name = cast(str, expr['struct'])  # Checked in check_exprs
 493     members = expr['data']
 494
 495     check_type_implicit(members, expr.info, "'data'", name)
 496     check_type_name(expr.get('base'), expr.info, "'base'")
 497
 498
 499 def check_union(expr: QAPIExpression) -> None:
 500     """
 501     Normalize and validate this expression as a ``union`` definition.
 502
 503     :param expr: The expression to validate.
 504
 505     :raise QAPISemError: when ``expr`` is not a valid ``union``.
 506     :return: None, ``expr`` is normalized in-place as needed.
 507     """
 508     name = cast(str, expr['union'])  # Checked in check_exprs
 509     base = expr['base']
 510     discriminator = expr['discriminator']
 511     members = expr['data']
 512     info = expr.info
 513
 514     check_type_name_or_implicit(base, info, "'base'", name)
 515     check_name_is_str(discriminator, info, "'discriminator'")
 516
 517     if not isinstance(members, dict):
 518         raise QAPISemError(info, "'data' must be an object")
 519
 520     for (key, value) in members.items():
 521         source = "'data' member '%s'" % key
 522         check_keys(value, info, source, ['type'], ['if'])
 523         check_if(value, info, source)
 524         check_type_name(value['type'], info, source)
 525
 526
 527 def check_alternate(expr: QAPIExpression) -> None:
 528     """
 529     Normalize and validate this expression as an ``alternate`` definition.
 530
 531     :param expr: The expression to validate.
 532
 533     :raise QAPISemError: When ``expr`` is not a valid ``alternate``.
 534     :return: None, ``expr`` is normalized in-place as needed.
 535     """
 536     members = expr['data']
 537     info = expr.info
 538
 539     if not members:
 540         raise QAPISemError(info, "'data' must not be empty")
 541
 542     if not isinstance(members, dict):
 543         raise QAPISemError(info, "'data' must be an object")
 544
 545     for (key, value) in members.items():
 546         source = "'data' member '%s'" % key
 547         check_name_lower(key, info, source)
 548         check_keys(value, info, source, ['type'], ['if'])
 549         check_if(value, info, source)
 550         check_type_name_or_array(value['type'], info, source)
 551
 552
 553 def check_command(expr: QAPIExpression) -> None:
 554     """
 555     Normalize and validate this expression as a ``command`` definition.
 556
 557     :param expr: The expression to validate.
 558
 559     :raise QAPISemError: When ``expr`` is not a valid ``command``.
 560     :return: None, ``expr`` is normalized in-place as needed.
 561     """
 562     args = expr.get('data')
 563     rets = expr.get('returns')
 564     boxed = expr.get('boxed', False)
 565
 566     if boxed:
 567         if args is None:
 568             raise QAPISemError(expr.info, "'boxed': true requires 'data'")
 569         check_type_name(args, expr.info, "'data'")
 570     else:
 571         check_type_name_or_implicit(args, expr.info, "'data'", None)
 572     check_type_name_or_array(rets, expr.info, "'returns'")
 573
 574
 575 def check_event(expr: QAPIExpression) -> None:
 576     """
 577     Normalize and validate this expression as an ``event`` definition.
 578
 579     :param expr: The expression to validate.
 580
 581     :raise QAPISemError: When ``expr`` is not a valid ``event``.
 582     :return: None, ``expr`` is normalized in-place as needed.
 583     """
 584     args = expr.get('data')
 585     boxed = expr.get('boxed', False)
 586
 587     if boxed:
 588         if args is None:
 589             raise QAPISemError(expr.info, "'boxed': true requires 'data'")
 590         check_type_name(args, expr.info, "'data'")
 591     else:
 592         check_type_name_or_implicit(args, expr.info, "'data'", None)
 593
 594
 595 def check_exprs(exprs: List[QAPIExpression]) -> List[QAPIExpression]:
 596     """
 597     Validate and normalize a list of parsed QAPI schema expressions.
 598
 599     This function accepts a list of expressions and metadata as returned
 600     by the parser.  It destructively normalizes the expressions in-place.
 601
 602     :param exprs: The list of expressions to normalize and validate.
 603
 604     :raise QAPISemError: When any expression fails validation.
 605     :return: The same list of expressions (now modified).
 606     """
 607     for expr in exprs:
 608         info = expr.info
 609         doc = expr.doc
 610
 611         if 'include' in expr:
 612             continue
 613
 614         metas = expr.keys() & {'enum', 'struct', 'union', 'alternate',
 615                                'command', 'event'}
 616         if len(metas) != 1:
 617             raise QAPISemError(
 618                 info,
 619                 "expression must have exactly one key"
 620                 " 'enum', 'struct', 'union', 'alternate',"
 621                 " 'command', 'event'")
 622         meta = metas.pop()
 623
 624         check_name_is_str(expr[meta], info, "'%s'" % meta)
 625         name = cast(str, expr[meta])
 626         info.set_defn(meta, name)
 627         check_defn_name_str(name, info, meta)
 628
 629         if doc:
 630             if doc.symbol != name:
 631                 raise QAPISemError(
 632                     info, "documentation comment is for '%s'" % doc.symbol)
 633             doc.check_expr(expr)
 634         elif info.pragma.doc_required:
 635             raise QAPISemError(info,
 636                                "documentation comment required")
 637
 638         if meta == 'enum':
 639             check_keys(expr, info, meta,
 640                        ['enum', 'data'], ['if', 'features', 'prefix'])
 641             check_enum(expr)
 642         elif meta == 'union':
 643             check_keys(expr, info, meta,
 644                        ['union', 'base', 'discriminator', 'data'],
 645                        ['if', 'features'])
 646             normalize_members(expr.get('base'))
 647             normalize_members(expr['data'])
 648             check_union(expr)
 649         elif meta == 'alternate':
 650             check_keys(expr, info, meta,
 651                        ['alternate', 'data'], ['if', 'features'])
 652             normalize_members(expr['data'])
 653             check_alternate(expr)
 654         elif meta == 'struct':
 655             check_keys(expr, info, meta,
 656                        ['struct', 'data'], ['base', 'if', 'features'])
 657             normalize_members(expr['data'])
 658             check_struct(expr)
 659         elif meta == 'command':
 660             check_keys(expr, info, meta,
 661                        ['command'],
 662                        ['data', 'returns', 'boxed', 'if', 'features',
 663                         'gen', 'success-response', 'allow-oob',
 664                         'allow-preconfig', 'coroutine'])
 665             normalize_members(expr.get('data'))
 666             check_command(expr)
 667         elif meta == 'event':
 668             check_keys(expr, info, meta,
 669                        ['event'], ['data', 'boxed', 'if', 'features'])
 670             normalize_members(expr.get('data'))
 671             check_event(expr)
 672         else:
 673             assert False, 'unexpected meta type'
 674
 675         check_if(expr, info, meta)
 676         check_features(expr.get('features'), info)
 677         check_flags(expr)
 678
 679     return exprs