scripts/qapi/introspect.py

   1 """
   2 QAPI introspection generator
   3
   4 Copyright (C) 2015-2021 Red Hat, Inc.
   5
   6 Authors:
   7  Markus Armbruster <armbru@redhat.com>
   8  John Snow <jsnow@redhat.com>
   9
  10 This work is licensed under the terms of the GNU GPL, version 2.
  11 See the COPYING file in the top-level directory.
  12 """
  13
  14 from typing import (
  15     Any,
  16     Dict,
  17     Generic,
  18     List,
  19     Optional,
  20     Sequence,
  21     TypeVar,
  22     Union,
  23 )
  24
  25 from .common import c_name, mcgen
  26 from .gen import QAPISchemaMonolithicCVisitor
  27 from .schema import (
  28     QAPISchema,
  29     QAPISchemaArrayType,
  30     QAPISchemaBuiltinType,
  31     QAPISchemaEntity,
  32     QAPISchemaEnumMember,
  33     QAPISchemaFeature,
  34     QAPISchemaIfCond,
  35     QAPISchemaObjectType,
  36     QAPISchemaObjectTypeMember,
  37     QAPISchemaType,
  38     QAPISchemaVariant,
  39     QAPISchemaVariants,
  40 )
  41 from .source import QAPISourceInfo
  42
  43
  44 # This module constructs a tree data structure that is used to
  45 # generate the introspection information for QEMU. It is shaped
  46 # like a JSON value.
  47 #
  48 # A complexity over JSON is that our values may or may not be annotated.
  49 #
  50 # Un-annotated values may be:
  51 #     Scalar: str, bool, None.
  52 #     Non-scalar: List, Dict
  53 # _value = Union[str, bool, None, Dict[str, JSONValue], List[JSONValue]]
  54 #
  55 # With optional annotations, the type of all values is:
  56 # JSONValue = Union[_Value, Annotated[_Value]]
  57 #
  58 # Sadly, mypy does not support recursive types; so the _Stub alias is used to
  59 # mark the imprecision in the type model where we'd otherwise use JSONValue.
  60 _Stub = Any
  61 _Scalar = Union[str, bool, None]
  62 _NonScalar = Union[Dict[str, _Stub], List[_Stub]]
  63 _Value = Union[_Scalar, _NonScalar]
  64 JSONValue = Union[_Value, 'Annotated[_Value]']
  65
  66 # These types are based on structures defined in QEMU's schema, so we
  67 # lack precise types for them here. Python 3.6 does not offer
  68 # TypedDict constructs, so they are broadly typed here as simple
  69 # Python Dicts.
  70 SchemaInfo = Dict[str, object]
  71 SchemaInfoEnumMember = Dict[str, object]
  72 SchemaInfoObject = Dict[str, object]
  73 SchemaInfoObjectVariant = Dict[str, object]
  74 SchemaInfoObjectMember = Dict[str, object]
  75 SchemaInfoCommand = Dict[str, object]
  76
  77
  78 _ValueT = TypeVar('_ValueT', bound=_Value)
  79
  80
  81 class Annotated(Generic[_ValueT]):
  82     """
  83     Annotated generally contains a SchemaInfo-like type (as a dict),
  84     But it also used to wrap comments/ifconds around scalar leaf values,
  85     for the benefit of features and enums.
  86     """
  87     # TODO: Remove after Python 3.7 adds @dataclass:
  88     # pylint: disable=too-few-public-methods
  89     def __init__(self, value: _ValueT, ifcond: QAPISchemaIfCond,
  90                  comment: Optional[str] = None):
  91         self.value = value
  92         self.comment: Optional[str] = comment
  93         self.ifcond = ifcond
  94
  95
  96 def _tree_to_qlit(obj: JSONValue,
  97                   level: int = 0,
  98                   dict_value: bool = False) -> str:
  99     """
 100     Convert the type tree into a QLIT C string, recursively.
 101
 102     :param obj: The value to convert.
 103                 This value may not be Annotated when dict_value is True.
 104     :param level: The indentation level for this particular value.
 105     :param dict_value: True when the value being processed belongs to a
 106                        dict key; which suppresses the output indent.
 107     """
 108
 109     def indent(level: int) -> str:
 110         return level * 4 * ' '
 111
 112     if isinstance(obj, Annotated):
 113         # NB: _tree_to_qlit is called recursively on the values of a
 114         # key:value pair; those values can't be decorated with
 115         # comments or conditionals.
 116         msg = "dict values cannot have attached comments or if-conditionals."
 117         assert not dict_value, msg
 118
 119         ret = ''
 120         if obj.comment:
 121             ret += indent(level) + f"/* {obj.comment} */\n"
 122         if obj.ifcond.is_present():
 123             ret += obj.ifcond.gen_if()
 124         ret += _tree_to_qlit(obj.value, level)
 125         if obj.ifcond.is_present():
 126             ret += '\n' + obj.ifcond.gen_endif()
 127         return ret
 128
 129     ret = ''
 130     if not dict_value:
 131         ret += indent(level)
 132
 133     # Scalars:
 134     if obj is None:
 135         ret += 'QLIT_QNULL'
 136     elif isinstance(obj, str):
 137         ret += f"QLIT_QSTR({to_c_string(obj)})"
 138     elif isinstance(obj, bool):
 139         ret += f"QLIT_QBOOL({str(obj).lower()})"
 140
 141     # Non-scalars:
 142     elif isinstance(obj, list):
 143         ret += 'QLIT_QLIST(((QLitObject[]) {\n'
 144         for value in obj:
 145             ret += _tree_to_qlit(value, level + 1).strip('\n') + '\n'
 146         ret += indent(level + 1) + '{}\n'
 147         ret += indent(level) + '}))'
 148     elif isinstance(obj, dict):
 149         ret += 'QLIT_QDICT(((QLitDictEntry[]) {\n'
 150         for key, value in sorted(obj.items()):
 151             ret += indent(level + 1) + "{{ {:s}, {:s} }},\n".format(
 152                 to_c_string(key),
 153                 _tree_to_qlit(value, level + 1, dict_value=True)
 154             )
 155         ret += indent(level + 1) + '{}\n'
 156         ret += indent(level) + '}))'
 157     else:
 158         raise NotImplementedError(
 159             f"type '{type(obj).__name__}' not implemented"
 160         )
 161
 162     if level > 0:
 163         ret += ','
 164     return ret
 165
 166
 167 def to_c_string(string: str) -> str:
 168     return '"' + string.replace('\\', r'\\').replace('"', r'\"') + '"'
 169
 170
 171 class QAPISchemaGenIntrospectVisitor(QAPISchemaMonolithicCVisitor):
 172
 173     def __init__(self, prefix: str, unmask: bool):
 174         super().__init__(
 175             prefix, 'qapi-introspect',
 176             ' * QAPI/QMP schema introspection', __doc__)
 177         self._unmask = unmask
 178         self._schema: Optional[QAPISchema] = None
 179         self._trees: List[Annotated[SchemaInfo]] = []
 180         self._used_types: List[QAPISchemaType] = []
 181         self._name_map: Dict[str, str] = {}
 182         self._genc.add(mcgen('''
 183 #include "qemu/osdep.h"
 184 #include "%(prefix)sqapi-introspect.h"
 185
 186 ''',
 187                              prefix=prefix))
 188
 189     def visit_begin(self, schema: QAPISchema) -> None:
 190         self._schema = schema
 191
 192     def visit_end(self) -> None:
 193         # visit the types that are actually used
 194         for typ in self._used_types:
 195             typ.visit(self)
 196         # generate C
 197         name = c_name(self._prefix, protect=False) + 'qmp_schema_qlit'
 198         self._genh.add(mcgen('''
 199 #include "qapi/qmp/qlit.h"
 200
 201 extern const QLitObject %(c_name)s;
 202 ''',
 203                              c_name=c_name(name)))
 204         self._genc.add(mcgen('''
 205 const QLitObject %(c_name)s = %(c_string)s;
 206 ''',
 207                              c_name=c_name(name),
 208                              c_string=_tree_to_qlit(self._trees)))
 209         self._schema = None
 210         self._trees = []
 211         self._used_types = []
 212         self._name_map = {}
 213
 214     def visit_needed(self, entity: QAPISchemaEntity) -> bool:
 215         # Ignore types on first pass; visit_end() will pick up used types
 216         return not isinstance(entity, QAPISchemaType)
 217
 218     def _name(self, name: str) -> str:
 219         if self._unmask:
 220             return name
 221         if name not in self._name_map:
 222             self._name_map[name] = '%d' % len(self._name_map)
 223         return self._name_map[name]
 224
 225     def _use_type(self, typ: QAPISchemaType) -> str:
 226         assert self._schema is not None
 227
 228         # Map the various integer types to plain int
 229         if typ.json_type() == 'int':
 230             typ = self._schema.lookup_type('int')
 231         elif (isinstance(typ, QAPISchemaArrayType) and
 232               typ.element_type.json_type() == 'int'):
 233             typ = self._schema.lookup_type('intList')
 234         # Add type to work queue if new
 235         if typ not in self._used_types:
 236             self._used_types.append(typ)
 237         # Clients should examine commands and events, not types.  Hide
 238         # type names as integers to reduce the temptation.  Also, it
 239         # saves a few characters on the wire.
 240         if isinstance(typ, QAPISchemaBuiltinType):
 241             return typ.name
 242         if isinstance(typ, QAPISchemaArrayType):
 243             return '[' + self._use_type(typ.element_type) + ']'
 244         return self._name(typ.name)
 245
 246     @staticmethod
 247     def _gen_features(features: Sequence[QAPISchemaFeature]
 248                       ) -> List[Annotated[str]]:
 249         return [Annotated(f.name, f.ifcond) for f in features]
 250
 251     def _gen_tree(self, name: str, mtype: str, obj: Dict[str, object],
 252                   ifcond: QAPISchemaIfCond = QAPISchemaIfCond(),
 253                   features: Sequence[QAPISchemaFeature] = ()) -> None:
 254         """
 255         Build and append a SchemaInfo object to self._trees.
 256
 257         :param name: The SchemaInfo's name.
 258         :param mtype: The SchemaInfo's meta-type.
 259         :param obj: Additional SchemaInfo members, as appropriate for
 260                     the meta-type.
 261         :param ifcond: Conditionals to apply to the SchemaInfo.
 262         :param features: The SchemaInfo's features.
 263                          Will be omitted from the output if empty.
 264         """
 265         comment: Optional[str] = None
 266         if mtype not in ('command', 'event', 'builtin', 'array'):
 267             if not self._unmask:
 268                 # Output a comment to make it easy to map masked names
 269                 # back to the source when reading the generated output.
 270                 comment = f'"{self._name(name)}" = {name}'
 271             name = self._name(name)
 272         obj['name'] = name
 273         obj['meta-type'] = mtype
 274         if features:
 275             obj['features'] = self._gen_features(features)
 276         self._trees.append(Annotated(obj, ifcond, comment))
 277
 278     def _gen_enum_member(self, member: QAPISchemaEnumMember
 279                          ) -> Annotated[SchemaInfoEnumMember]:
 280         obj: SchemaInfoEnumMember = {
 281             'name': member.name,
 282         }
 283         if member.features:
 284             obj['features'] = self._gen_features(member.features)
 285         return Annotated(obj, member.ifcond)
 286
 287     def _gen_object_member(self, member: QAPISchemaObjectTypeMember
 288                            ) -> Annotated[SchemaInfoObjectMember]:
 289         obj: SchemaInfoObjectMember = {
 290             'name': member.name,
 291             'type': self._use_type(member.type)
 292         }
 293         if member.optional:
 294             obj['default'] = None
 295         if member.features:
 296             obj['features'] = self._gen_features(member.features)
 297         return Annotated(obj, member.ifcond)
 298
 299     def _gen_variant(self, variant: QAPISchemaVariant
 300                      ) -> Annotated[SchemaInfoObjectVariant]:
 301         obj: SchemaInfoObjectVariant = {
 302             'case': variant.name,
 303             'type': self._use_type(variant.type)
 304         }
 305         return Annotated(obj, variant.ifcond)
 306
 307     def visit_builtin_type(self, name: str, info: Optional[QAPISourceInfo],
 308                            json_type: str) -> None:
 309         self._gen_tree(name, 'builtin', {'json-type': json_type})
 310
 311     def visit_enum_type(self, name: str, info: Optional[QAPISourceInfo],
 312                         ifcond: QAPISchemaIfCond,
 313                         features: List[QAPISchemaFeature],
 314                         members: List[QAPISchemaEnumMember],
 315                         prefix: Optional[str]) -> None:
 316         self._gen_tree(
 317             name, 'enum',
 318             {'members': [self._gen_enum_member(m) for m in members],
 319              'values': [Annotated(m.name, m.ifcond) for m in members]},
 320             ifcond, features
 321         )
 322
 323     def visit_array_type(self, name: str, info: Optional[QAPISourceInfo],
 324                          ifcond: QAPISchemaIfCond,
 325                          element_type: QAPISchemaType) -> None:
 326         element = self._use_type(element_type)
 327         self._gen_tree('[' + element + ']', 'array', {'element-type': element},
 328                        ifcond)
 329
 330     def visit_object_type_flat(self, name: str, info: Optional[QAPISourceInfo],
 331                                ifcond: QAPISchemaIfCond,
 332                                features: List[QAPISchemaFeature],
 333                                members: List[QAPISchemaObjectTypeMember],
 334                                variants: Optional[QAPISchemaVariants]) -> None:
 335         obj: SchemaInfoObject = {
 336             'members': [self._gen_object_member(m) for m in members]
 337         }
 338         if variants:
 339             obj['tag'] = variants.tag_member.name
 340             obj['variants'] = [self._gen_variant(v) for v in variants.variants]
 341         self._gen_tree(name, 'object', obj, ifcond, features)
 342
 343     def visit_alternate_type(self, name: str, info: Optional[QAPISourceInfo],
 344                              ifcond: QAPISchemaIfCond,
 345                              features: List[QAPISchemaFeature],
 346                              variants: QAPISchemaVariants) -> None:
 347         self._gen_tree(
 348             name, 'alternate',
 349             {'members': [Annotated({'type': self._use_type(m.type)},
 350                                    m.ifcond)
 351                          for m in variants.variants]},
 352             ifcond, features
 353         )
 354
 355     def visit_command(self, name: str, info: Optional[QAPISourceInfo],
 356                       ifcond: QAPISchemaIfCond,
 357                       features: List[QAPISchemaFeature],
 358                       arg_type: Optional[QAPISchemaObjectType],
 359                       ret_type: Optional[QAPISchemaType], gen: bool,
 360                       success_response: bool, boxed: bool, allow_oob: bool,
 361                       allow_preconfig: bool, coroutine: bool) -> None:
 362         assert self._schema is not None
 363
 364         arg_type = arg_type or self._schema.the_empty_object_type
 365         ret_type = ret_type or self._schema.the_empty_object_type
 366         obj: SchemaInfoCommand = {
 367             'arg-type': self._use_type(arg_type),
 368             'ret-type': self._use_type(ret_type)
 369         }
 370         if allow_oob:
 371             obj['allow-oob'] = allow_oob
 372         self._gen_tree(name, 'command', obj, ifcond, features)
 373
 374     def visit_event(self, name: str, info: Optional[QAPISourceInfo],
 375                     ifcond: QAPISchemaIfCond,
 376                     features: List[QAPISchemaFeature],
 377                     arg_type: Optional[QAPISchemaObjectType],
 378                     boxed: bool) -> None:
 379         assert self._schema is not None
 380
 381         arg_type = arg_type or self._schema.the_empty_object_type
 382         self._gen_tree(name, 'event', {'arg-type': self._use_type(arg_type)},
 383                        ifcond, features)
 384
 385
 386 def gen_introspect(schema: QAPISchema, output_dir: str, prefix: str,
 387                    opt_unmask: bool) -> None:
 388     vis = QAPISchemaGenIntrospectVisitor(prefix, opt_unmask)
 389     schema.visit(vis)
 390     vis.write(output_dir)