scripts/qapi/introspect.py

   1 """
   2 QAPI introspection generator
   3
   4 Copyright (C) 2015-2021 Red Hat, Inc.
   5
   6 Authors:
   7  Markus Armbruster <armbru@redhat.com>
   8  John Snow <jsnow@redhat.com>
   9
  10 This work is licensed under the terms of the GNU GPL, version 2.
  11 See the COPYING file in the top-level directory.
  12 """
  13
  14 from typing import (
  15     Any,
  16     Dict,
  17     Generic,
  18     Iterable,
  19     List,
  20     Optional,
  21     Sequence,
  22     Tuple,
  23     TypeVar,
  24     Union,
  25 )
  26
  27 from .common import (
  28     c_name,
  29     gen_endif,
  30     gen_if,
  31     mcgen,
  32 )
  33 from .gen import QAPISchemaMonolithicCVisitor
  34 from .schema import (
  35     QAPISchema,
  36     QAPISchemaArrayType,
  37     QAPISchemaBuiltinType,
  38     QAPISchemaEntity,
  39     QAPISchemaEnumMember,
  40     QAPISchemaFeature,
  41     QAPISchemaObjectType,
  42     QAPISchemaObjectTypeMember,
  43     QAPISchemaType,
  44     QAPISchemaVariant,
  45     QAPISchemaVariants,
  46 )
  47 from .source import QAPISourceInfo
  48
  49
  50 # This module constructs a tree data structure that is used to
  51 # generate the introspection information for QEMU. It is shaped
  52 # like a JSON value.
  53 #
  54 # A complexity over JSON is that our values may or may not be annotated.
  55 #
  56 # Un-annotated values may be:
  57 #     Scalar: str, bool, None.
  58 #     Non-scalar: List, Dict
  59 # _value = Union[str, bool, None, Dict[str, JSONValue], List[JSONValue]]
  60 #
  61 # With optional annotations, the type of all values is:
  62 # JSONValue = Union[_Value, Annotated[_Value]]
  63 #
  64 # Sadly, mypy does not support recursive types; so the _Stub alias is used to
  65 # mark the imprecision in the type model where we'd otherwise use JSONValue.
  66 _Stub = Any
  67 _Scalar = Union[str, bool, None]
  68 _NonScalar = Union[Dict[str, _Stub], List[_Stub]]
  69 _Value = Union[_Scalar, _NonScalar]
  70 JSONValue = Union[_Value, 'Annotated[_Value]']
  71
  72 # These types are based on structures defined in QEMU's schema, so we
  73 # lack precise types for them here. Python 3.6 does not offer
  74 # TypedDict constructs, so they are broadly typed here as simple
  75 # Python Dicts.
  76 SchemaInfo = Dict[str, object]
  77 SchemaInfoObject = Dict[str, object]
  78 SchemaInfoObjectVariant = Dict[str, object]
  79 SchemaInfoObjectMember = Dict[str, object]
  80 SchemaInfoCommand = Dict[str, object]
  81
  82
  83 _ValueT = TypeVar('_ValueT', bound=_Value)
  84
  85
  86 class Annotated(Generic[_ValueT]):
  87     """
  88     Annotated generally contains a SchemaInfo-like type (as a dict),
  89     But it also used to wrap comments/ifconds around scalar leaf values,
  90     for the benefit of features and enums.
  91     """
  92     # TODO: Remove after Python 3.7 adds @dataclass:
  93     # pylint: disable=too-few-public-methods
  94     def __init__(self, value: _ValueT, ifcond: Iterable[str],
  95                  comment: Optional[str] = None):
  96         self.value = value
  97         self.comment: Optional[str] = comment
  98         self.ifcond: Tuple[str, ...] = tuple(ifcond)
  99
 100
 101 def _tree_to_qlit(obj: JSONValue,
 102                   level: int = 0,
 103                   dict_value: bool = False) -> str:
 104     """
 105     Convert the type tree into a QLIT C string, recursively.
 106
 107     :param obj: The value to convert.
 108                 This value may not be Annotated when dict_value is True.
 109     :param level: The indentation level for this particular value.
 110     :param dict_value: True when the value being processed belongs to a
 111                        dict key; which suppresses the output indent.
 112     """
 113
 114     def indent(level: int) -> str:
 115         return level * 4 * ' '
 116
 117     if isinstance(obj, Annotated):
 118         # NB: _tree_to_qlit is called recursively on the values of a
 119         # key:value pair; those values can't be decorated with
 120         # comments or conditionals.
 121         msg = "dict values cannot have attached comments or if-conditionals."
 122         assert not dict_value, msg
 123
 124         ret = ''
 125         if obj.comment:
 126             ret += indent(level) + f"/* {obj.comment} */\n"
 127         if obj.ifcond:
 128             ret += gen_if(obj.ifcond)
 129         ret += _tree_to_qlit(obj.value, level)
 130         if obj.ifcond:
 131             ret += '\n' + gen_endif(obj.ifcond)
 132         return ret
 133
 134     ret = ''
 135     if not dict_value:
 136         ret += indent(level)
 137
 138     # Scalars:
 139     if obj is None:
 140         ret += 'QLIT_QNULL'
 141     elif isinstance(obj, str):
 142         ret += f"QLIT_QSTR({to_c_string(obj)})"
 143     elif isinstance(obj, bool):
 144         ret += f"QLIT_QBOOL({str(obj).lower()})"
 145
 146     # Non-scalars:
 147     elif isinstance(obj, list):
 148         ret += 'QLIT_QLIST(((QLitObject[]) {\n'
 149         for value in obj:
 150             ret += _tree_to_qlit(value, level + 1).strip('\n') + '\n'
 151         ret += indent(level + 1) + '{}\n'
 152         ret += indent(level) + '}))'
 153     elif isinstance(obj, dict):
 154         ret += 'QLIT_QDICT(((QLitDictEntry[]) {\n'
 155         for key, value in sorted(obj.items()):
 156             ret += indent(level + 1) + "{{ {:s}, {:s} }},\n".format(
 157                 to_c_string(key),
 158                 _tree_to_qlit(value, level + 1, dict_value=True)
 159             )
 160         ret += indent(level + 1) + '{}\n'
 161         ret += indent(level) + '}))'
 162     else:
 163         raise NotImplementedError(
 164             f"type '{type(obj).__name__}' not implemented"
 165         )
 166
 167     if level > 0:
 168         ret += ','
 169     return ret
 170
 171
 172 def to_c_string(string: str) -> str:
 173     return '"' + string.replace('\\', r'\\').replace('"', r'\"') + '"'
 174
 175
 176 class QAPISchemaGenIntrospectVisitor(QAPISchemaMonolithicCVisitor):
 177
 178     def __init__(self, prefix: str, unmask: bool):
 179         super().__init__(
 180             prefix, 'qapi-introspect',
 181             ' * QAPI/QMP schema introspection', __doc__)
 182         self._unmask = unmask
 183         self._schema: Optional[QAPISchema] = None
 184         self._trees: List[Annotated[SchemaInfo]] = []
 185         self._used_types: List[QAPISchemaType] = []
 186         self._name_map: Dict[str, str] = {}
 187         self._genc.add(mcgen('''
 188 #include "qemu/osdep.h"
 189 #include "%(prefix)sqapi-introspect.h"
 190
 191 ''',
 192                              prefix=prefix))
 193
 194     def visit_begin(self, schema: QAPISchema) -> None:
 195         self._schema = schema
 196
 197     def visit_end(self) -> None:
 198         # visit the types that are actually used
 199         for typ in self._used_types:
 200             typ.visit(self)
 201         # generate C
 202         name = c_name(self._prefix, protect=False) + 'qmp_schema_qlit'
 203         self._genh.add(mcgen('''
 204 #include "qapi/qmp/qlit.h"
 205
 206 extern const QLitObject %(c_name)s;
 207 ''',
 208                              c_name=c_name(name)))
 209         self._genc.add(mcgen('''
 210 const QLitObject %(c_name)s = %(c_string)s;
 211 ''',
 212                              c_name=c_name(name),
 213                              c_string=_tree_to_qlit(self._trees)))
 214         self._schema = None
 215         self._trees = []
 216         self._used_types = []
 217         self._name_map = {}
 218
 219     def visit_needed(self, entity: QAPISchemaEntity) -> bool:
 220         # Ignore types on first pass; visit_end() will pick up used types
 221         return not isinstance(entity, QAPISchemaType)
 222
 223     def _name(self, name: str) -> str:
 224         if self._unmask:
 225             return name
 226         if name not in self._name_map:
 227             self._name_map[name] = '%d' % len(self._name_map)
 228         return self._name_map[name]
 229
 230     def _use_type(self, typ: QAPISchemaType) -> str:
 231         assert self._schema is not None
 232
 233         # Map the various integer types to plain int
 234         if typ.json_type() == 'int':
 235             typ = self._schema.lookup_type('int')
 236         elif (isinstance(typ, QAPISchemaArrayType) and
 237               typ.element_type.json_type() == 'int'):
 238             typ = self._schema.lookup_type('intList')
 239         # Add type to work queue if new
 240         if typ not in self._used_types:
 241             self._used_types.append(typ)
 242         # Clients should examine commands and events, not types.  Hide
 243         # type names as integers to reduce the temptation.  Also, it
 244         # saves a few characters on the wire.
 245         if isinstance(typ, QAPISchemaBuiltinType):
 246             return typ.name
 247         if isinstance(typ, QAPISchemaArrayType):
 248             return '[' + self._use_type(typ.element_type) + ']'
 249         return self._name(typ.name)
 250
 251     @staticmethod
 252     def _gen_features(features: Sequence[QAPISchemaFeature]
 253                       ) -> List[Annotated[str]]:
 254         return [Annotated(f.name, f.ifcond) for f in features]
 255
 256     def _gen_tree(self, name: str, mtype: str, obj: Dict[str, object],
 257                   ifcond: Sequence[str] = (),
 258                   features: Sequence[QAPISchemaFeature] = ()) -> None:
 259         """
 260         Build and append a SchemaInfo object to self._trees.
 261
 262         :param name: The SchemaInfo's name.
 263         :param mtype: The SchemaInfo's meta-type.
 264         :param obj: Additional SchemaInfo members, as appropriate for
 265                     the meta-type.
 266         :param ifcond: Conditionals to apply to the SchemaInfo.
 267         :param features: The SchemaInfo's features.
 268                          Will be omitted from the output if empty.
 269         """
 270         comment: Optional[str] = None
 271         if mtype not in ('command', 'event', 'builtin', 'array'):
 272             if not self._unmask:
 273                 # Output a comment to make it easy to map masked names
 274                 # back to the source when reading the generated output.
 275                 comment = f'"{self._name(name)}" = {name}'
 276             name = self._name(name)
 277         obj['name'] = name
 278         obj['meta-type'] = mtype
 279         if features:
 280             obj['features'] = self._gen_features(features)
 281         self._trees.append(Annotated(obj, ifcond, comment))
 282
 283     def _gen_member(self, member: QAPISchemaObjectTypeMember
 284                     ) -> Annotated[SchemaInfoObjectMember]:
 285         obj: SchemaInfoObjectMember = {
 286             'name': member.name,
 287             'type': self._use_type(member.type)
 288         }
 289         if member.optional:
 290             obj['default'] = None
 291         if member.features:
 292             obj['features'] = self._gen_features(member.features)
 293         return Annotated(obj, member.ifcond)
 294
 295     def _gen_variant(self, variant: QAPISchemaVariant
 296                      ) -> Annotated[SchemaInfoObjectVariant]:
 297         obj: SchemaInfoObjectVariant = {
 298             'case': variant.name,
 299             'type': self._use_type(variant.type)
 300         }
 301         return Annotated(obj, variant.ifcond)
 302
 303     def visit_builtin_type(self, name: str, info: Optional[QAPISourceInfo],
 304                            json_type: str) -> None:
 305         self._gen_tree(name, 'builtin', {'json-type': json_type})
 306
 307     def visit_enum_type(self, name: str, info: Optional[QAPISourceInfo],
 308                         ifcond: Sequence[str],
 309                         features: List[QAPISchemaFeature],
 310                         members: List[QAPISchemaEnumMember],
 311                         prefix: Optional[str]) -> None:
 312         self._gen_tree(
 313             name, 'enum',
 314             {'values': [Annotated(m.name, m.ifcond) for m in members]},
 315             ifcond, features
 316         )
 317
 318     def visit_array_type(self, name: str, info: Optional[QAPISourceInfo],
 319                          ifcond: Sequence[str],
 320                          element_type: QAPISchemaType) -> None:
 321         element = self._use_type(element_type)
 322         self._gen_tree('[' + element + ']', 'array', {'element-type': element},
 323                        ifcond)
 324
 325     def visit_object_type_flat(self, name: str, info: Optional[QAPISourceInfo],
 326                                ifcond: Sequence[str],
 327                                features: List[QAPISchemaFeature],
 328                                members: List[QAPISchemaObjectTypeMember],
 329                                variants: Optional[QAPISchemaVariants]) -> None:
 330         obj: SchemaInfoObject = {
 331             'members': [self._gen_member(m) for m in members]
 332         }
 333         if variants:
 334             obj['tag'] = variants.tag_member.name
 335             obj['variants'] = [self._gen_variant(v) for v in variants.variants]
 336         self._gen_tree(name, 'object', obj, ifcond, features)
 337
 338     def visit_alternate_type(self, name: str, info: Optional[QAPISourceInfo],
 339                              ifcond: Sequence[str],
 340                              features: List[QAPISchemaFeature],
 341                              variants: QAPISchemaVariants) -> None:
 342         self._gen_tree(
 343             name, 'alternate',
 344             {'members': [Annotated({'type': self._use_type(m.type)},
 345                                    m.ifcond)
 346                          for m in variants.variants]},
 347             ifcond, features
 348         )
 349
 350     def visit_command(self, name: str, info: Optional[QAPISourceInfo],
 351                       ifcond: Sequence[str],
 352                       features: List[QAPISchemaFeature],
 353                       arg_type: Optional[QAPISchemaObjectType],
 354                       ret_type: Optional[QAPISchemaType], gen: bool,
 355                       success_response: bool, boxed: bool, allow_oob: bool,
 356                       allow_preconfig: bool, coroutine: bool) -> None:
 357         assert self._schema is not None
 358
 359         arg_type = arg_type or self._schema.the_empty_object_type
 360         ret_type = ret_type or self._schema.the_empty_object_type
 361         obj: SchemaInfoCommand = {
 362             'arg-type': self._use_type(arg_type),
 363             'ret-type': self._use_type(ret_type)
 364         }
 365         if allow_oob:
 366             obj['allow-oob'] = allow_oob
 367         self._gen_tree(name, 'command', obj, ifcond, features)
 368
 369     def visit_event(self, name: str, info: Optional[QAPISourceInfo],
 370                     ifcond: Sequence[str], features: List[QAPISchemaFeature],
 371                     arg_type: Optional[QAPISchemaObjectType],
 372                     boxed: bool) -> None:
 373         assert self._schema is not None
 374
 375         arg_type = arg_type or self._schema.the_empty_object_type
 376         self._gen_tree(name, 'event', {'arg-type': self._use_type(arg_type)},
 377                        ifcond, features)
 378
 379
 380 def gen_introspect(schema: QAPISchema, output_dir: str, prefix: str,
 381                    opt_unmask: bool) -> None:
 382     vis = QAPISchemaGenIntrospectVisitor(prefix, opt_unmask)
 383     schema.visit(vis)
 384     vis.write(output_dir)