scripts/codeconverter/codeconverter/qom_macros.py

   1 # Copyright (C) 2020 Red Hat Inc.
   2 #
   3 # Authors:
   4 #  Eduardo Habkost <ehabkost@redhat.com>
   5 #
   6 # This work is licensed under the terms of the GNU GPL, version 2.  See
   7 # the COPYING file in the top-level directory.
   8 import re
   9 from itertools import chain
  10 from typing import *
  11
  12 from .regexps import *
  13 from .patching import *
  14 from .utils import *
  15
  16 import logging
  17 logger = logging.getLogger(__name__)
  18 DBG = logger.debug
  19 INFO = logger.info
  20 WARN = logger.warning
  21
  22 # simple expressions:
  23
  24 RE_CONSTANT = OR(RE_STRING, RE_NUMBER)
  25
  26 class DefineDirective(FileMatch):
  27     """Match any #define directive"""
  28     regexp = S(r'^[ \t]*#[ \t]*define', CPP_SPACE, NAMED('name', RE_IDENTIFIER), r'\b')
  29
  30 class ExpressionDefine(FileMatch):
  31     """Simple #define preprocessor directive for an expression"""
  32     regexp = S(r'^[ \t]*#[ \t]*define', CPP_SPACE, NAMED('name', RE_IDENTIFIER),
  33                CPP_SPACE, NAMED('value', RE_EXPRESSION), r'[ \t]*\n')
  34
  35     def provided_identifiers(self) -> Iterable[RequiredIdentifier]:
  36         yield RequiredIdentifier('constant', self.group('name'))
  37
  38 class ConstantDefine(ExpressionDefine):
  39     """Simple #define preprocessor directive for a number or string constant"""
  40     regexp = S(r'^[ \t]*#[ \t]*define', CPP_SPACE, NAMED('name', RE_IDENTIFIER),
  41                CPP_SPACE, NAMED('value', RE_CONSTANT), r'[ \t]*\n')
  42
  43
  44 class TypeIdentifiers(NamedTuple):
  45     """Type names found in type declarations"""
  46     # TYPE_MYDEVICE
  47     typename: Optional[str]
  48     # MYDEVICE
  49     uppercase: Optional[str] = None
  50     # MyDevice
  51     instancetype: Optional[str] = None
  52     # MyDeviceClass
  53     classtype: Optional[str] = None
  54     # my_device
  55     lowercase: Optional[str] = None
  56
  57     def allfields(self):
  58         return tuple(getattr(self, f) for f in self._fields)
  59
  60     def merge(self, other: 'TypeIdentifiers') -> Optional['TypeIdentifiers']:
  61         """Check if identifiers match, return new identifier with complete list"""
  62         if any(not opt_compare(a, b) for a,b in zip(self, other)):
  63             return None
  64         return TypeIdentifiers(*(merge(a, b) for a,b in zip(self, other)))
  65
  66     def __str__(self) -> str:
  67         values = ((f, getattr(self, f)) for f in self._fields)
  68         s = ', '.join('%s=%s' % (f,v) for f,v in values if v is not None)
  69         return f'{s}'
  70
  71     def check_consistency(self) -> List[str]:
  72         """Check if identifiers are consistent with each other,
  73         return list of problems (or empty list if everything seems consistent)
  74         """
  75         r = []
  76         if self.typename is None:
  77             r.append("typename (TYPE_MYDEVICE) is unavailable")
  78
  79         if self.uppercase is None:
  80             r.append("uppercase name is unavailable")
  81
  82         if (self.instancetype is not None
  83             and self.classtype is not None
  84             and self.classtype != f'{self.instancetype}Class'):
  85                 r.append("class typedef %s doesn't match instance typedef %s" %
  86                          (self.classtype, self.instancetype))
  87
  88         if (self.uppercase is not None
  89             and self.typename is not None
  90             and f'TYPE_{self.uppercase}' != self.typename):
  91             r.append("uppercase name (%s) doesn't match type name (%s)" %
  92                      (self.uppercase, self.typename))
  93
  94         return r
  95
  96 class TypedefMatch(FileMatch):
  97     """typedef declaration"""
  98     def provided_identifiers(self) -> Iterable[RequiredIdentifier]:
  99         yield RequiredIdentifier('type', self.group('name'))
 100
 101 class SimpleTypedefMatch(TypedefMatch):
 102     """Simple typedef declaration
 103     (no replacement rules)"""
 104     regexp = S(r'^[ \t]*typedef', SP,
 105                NAMED('typedef_type', RE_TYPE), SP,
 106                NAMED('name', RE_IDENTIFIER), r'\s*;[ \t]*\n')
 107
 108 RE_MACRO_DEFINE = S(r'^[ \t]*#\s*define\s+', NAMED('name', RE_IDENTIFIER),
 109                     r'\s*\(\s*', RE_IDENTIFIER, r'\s*\)', CPP_SPACE)
 110
 111 RE_STRUCT_ATTRIBUTE = r'QEMU_PACKED'
 112
 113 # This doesn't parse the struct definitions completely, it just assumes
 114 # the closing brackets are going to be in an unindented line:
 115 RE_FULL_STRUCT = S('struct', SP, M(RE_IDENTIFIER, n='?', name='structname'), SP,
 116                    NAMED('body', r'{\n',
 117                          # acceptable inside the struct body:
 118                          # - lines starting with space or tab
 119                          # - empty lines
 120                          # - preprocessor directives
 121                          # - comments
 122                          OR(r'[ \t][^\n]*\n',
 123                             r'#[^\n]*\n',
 124                             r'\n',
 125                             S(r'[ \t]*', RE_COMMENT, r'[ \t]*\n'),
 126                             repeat='*?'),
 127                          r'}', M(RE_STRUCT_ATTRIBUTE, SP, n='*')))
 128 RE_STRUCT_TYPEDEF = S(r'^[ \t]*typedef', SP, RE_FULL_STRUCT, SP,
 129                       NAMED('name', RE_IDENTIFIER), r'\s*;[ \t]*\n')
 130
 131 class FullStructTypedefMatch(TypedefMatch):
 132     """typedef struct [SomeStruct] { ...} SomeType
 133     Will be replaced by separate struct declaration + typedef
 134     """
 135     regexp = RE_STRUCT_TYPEDEF
 136
 137     def make_structname(self) -> str:
 138         """Make struct name for struct+typedef split"""
 139         name = self.group('structname')
 140         if not name:
 141             name = self.name
 142         return name
 143
 144     def strip_typedef(self) -> Patch:
 145         """generate patch that will strip typedef from the struct declartion
 146
 147         The caller is responsible for readding the typedef somewhere else.
 148         """
 149         name = self.make_structname()
 150         body = self.group('body')
 151         return self.make_patch(f'struct {name} {body};\n')
 152
 153     def make_simple_typedef(self) -> str:
 154         structname = self.make_structname()
 155         name = self.name
 156         return f'typedef struct {structname} {name};\n'
 157
 158     def move_typedef(self, position) -> Iterator[Patch]:
 159         """Generate patches to move typedef elsewhere"""
 160         yield self.strip_typedef()
 161         yield Patch(position, position, self.make_simple_typedef())
 162
 163     def split_typedef(self) -> Iterator[Patch]:
 164         """Split into struct definition + typedef in-place"""
 165         yield self.strip_typedef()
 166         yield self.append(self.make_simple_typedef())
 167
 168 class StructTypedefSplit(FullStructTypedefMatch):
 169     """split struct+typedef declaration"""
 170     def gen_patches(self) -> Iterator[Patch]:
 171         if self.group('structname'):
 172             yield from self.split_typedef()
 173
 174 class DuplicatedTypedefs(SimpleTypedefMatch):
 175     """Delete ALL duplicate typedefs (unsafe)"""
 176     def gen_patches(self) -> Iterable[Patch]:
 177         other_td = [td for td in chain(self.file.matches_of_type(SimpleTypedefMatch),
 178                                        self.file.matches_of_type(FullStructTypedefMatch))
 179                     if td.name == self.name]
 180         DBG("other_td: %r", other_td)
 181         if any(td.start() < self.start() for td in other_td):
 182             # patch only if handling the first typedef
 183             return
 184         for td in other_td:
 185             if isinstance(td, SimpleTypedefMatch):
 186                 DBG("other td: %r", td.match.groupdict())
 187                 if td.group('typedef_type') != self.group('typedef_type'):
 188                     yield td.make_removal_patch()
 189             elif isinstance(td, FullStructTypedefMatch):
 190                 DBG("other td: %r", td.match.groupdict())
 191                 if self.group('typedef_type') == 'struct '+td.group('structname'):
 192                     yield td.strip_typedef()
 193
 194 class QOMDuplicatedTypedefs(DuplicatedTypedefs):
 195     """Delete duplicate typedefs if used by QOM type"""
 196     def gen_patches(self) -> Iterable[Patch]:
 197         qom_macros = [TypeCheckMacro, DeclareInstanceChecker, DeclareClassCheckers, DeclareObjCheckers]
 198         qom_matches = chain(*(self.file.matches_of_type(t) for t in qom_macros))
 199         in_use = any(RequiredIdentifier('type', self.name) in m.required_identifiers()
 200                      for m in qom_matches)
 201         if in_use:
 202             yield from DuplicatedTypedefs.gen_patches(self)
 203
 204 class QOMStructTypedefSplit(FullStructTypedefMatch):
 205     """split struct+typedef declaration if used by QOM type"""
 206     def gen_patches(self) -> Iterator[Patch]:
 207         qom_macros = [TypeCheckMacro, DeclareInstanceChecker, DeclareClassCheckers, DeclareObjCheckers]
 208         qom_matches = chain(*(self.file.matches_of_type(t) for t in qom_macros))
 209         in_use = any(RequiredIdentifier('type', self.name) in m.required_identifiers()
 210                      for m in qom_matches)
 211         if in_use:
 212             yield from self.split_typedef()
 213
 214 def typedefs(file: FileInfo) -> Iterable[TypedefMatch]:
 215     return (cast(TypedefMatch, m)
 216             for m in chain(file.matches_of_type(SimpleTypedefMatch),
 217                            file.matches_of_type(FullStructTypedefMatch)))
 218
 219 def find_typedef(f: FileInfo, name: Optional[str]) -> Optional[TypedefMatch]:
 220     if not name:
 221         return None
 222     for td in typedefs(f):
 223         if td.name == name:
 224             return td
 225     return None
 226
 227 CHECKER_MACROS = ['OBJECT_CHECK', 'OBJECT_CLASS_CHECK', 'OBJECT_GET_CLASS']
 228 CheckerMacroName = Literal['OBJECT_CHECK', 'OBJECT_CLASS_CHECK', 'OBJECT_GET_CLASS']
 229
 230 RE_CHECK_MACRO = \
 231     S(RE_MACRO_DEFINE,
 232       OR(*CHECKER_MACROS, name='checker'),
 233       M(r'\s*\(\s*', OR(NAMED('typedefname', RE_IDENTIFIER), RE_TYPE, name='c_type'), r'\s*,', CPP_SPACE,
 234         OPTIONAL_PARS(RE_IDENTIFIER), r',', CPP_SPACE,
 235         NAMED('qom_typename', RE_IDENTIFIER), r'\s*\)\n',
 236         n='?', name='check_args'))
 237
 238 EXPECTED_CHECKER_SUFFIXES: List[Tuple[CheckerMacroName, str]] = [
 239     ('OBJECT_GET_CLASS', '_GET_CLASS'),
 240     ('OBJECT_CLASS_CHECK', '_CLASS'),
 241 ]
 242
 243 class TypeCheckMacro(FileMatch):
 244     """OBJECT_CHECK/OBJECT_CLASS_CHECK/OBJECT_GET_CLASS macro definitions
 245     Will be replaced by DECLARE_*_CHECKERS macro
 246     """
 247     regexp = RE_CHECK_MACRO
 248
 249     @property
 250     def checker(self) -> CheckerMacroName:
 251         """Name of checker macro being used"""
 252         return self.group('checker') # type: ignore
 253
 254     @property
 255     def typedefname(self) -> Optional[str]:
 256         return self.group('typedefname')
 257
 258     def find_typedef(self) -> Optional[TypedefMatch]:
 259         return find_typedef(self.file, self.typedefname)
 260
 261     def sanity_check(self) -> None:
 262         DBG("groups: %r", self.match.groups())
 263         if not self.group('check_args'):
 264             self.warn("type check macro not parsed completely: %s", self.name)
 265             return
 266         DBG("type identifiers: %r", self.type_identifiers)
 267         if self.typedefname and self.find_typedef() is None:
 268             self.warn("typedef used by %s not found", self.name)
 269
 270     def find_matching_macros(self) -> List['TypeCheckMacro']:
 271         """Find other check macros that generate the same macro names
 272
 273         The returned list will always be sorted.
 274         """
 275         my_ids = self.type_identifiers
 276         assert my_ids
 277         return [m for m in self.file.matches_of_type(TypeCheckMacro)
 278                 if m.type_identifiers is not None
 279                    and my_ids.uppercase is not None
 280                    and (my_ids.uppercase == m.type_identifiers.uppercase
 281                         or my_ids.typename == m.type_identifiers.typename)]
 282
 283     def merge_ids(self, matches: List['TypeCheckMacro']) -> Optional[TypeIdentifiers]:
 284         """Try to merge info about type identifiers from all matches in a list"""
 285         if not matches:
 286             return None
 287         r = matches[0].type_identifiers
 288         if r is None:
 289             return None
 290         for m in matches[1:]:
 291             assert m.type_identifiers
 292             new = r.merge(m.type_identifiers)
 293             if new is None:
 294                 self.warn("macro %s identifiers (%s) don't match macro %s (%s)",
 295                           matches[0].name, r, m.name, m.type_identifiers)
 296                 return None
 297             r = new
 298         return r
 299
 300     def required_identifiers(self) -> Iterable[RequiredIdentifier]:
 301         yield RequiredIdentifier('include', '"qom/object.h"')
 302         if self.type_identifiers is None:
 303             return
 304         # to make sure typedefs will be moved above all related macros,
 305         # return dependencies from all of them, not just this match
 306         for m in self.find_matching_macros():
 307             yield RequiredIdentifier('type', m.group('c_type'))
 308             yield RequiredIdentifier('constant', m.group('qom_typename'))
 309
 310     @property
 311     def type_identifiers(self) -> Optional[TypeIdentifiers]:
 312         """Extract type identifier information from match"""
 313         typename = self.group('qom_typename')
 314         c_type = self.group('c_type')
 315         if not typename or not c_type:
 316             return None
 317         typedef = self.group('typedefname')
 318         classtype = None
 319         instancetype = None
 320         uppercase = None
 321         expected_suffix = dict(EXPECTED_CHECKER_SUFFIXES).get(self.checker)
 322
 323         # here the available data depends on the checker macro being called:
 324         # - we need to remove the suffix from the macro name
 325         # - depending on the macro type, we know the class type name, or
 326         #   the instance type name
 327         if self.checker in ('OBJECT_GET_CLASS', 'OBJECT_CLASS_CHECK'):
 328             classtype = c_type
 329         elif self.checker == 'OBJECT_CHECK':
 330             instancetype = c_type
 331             uppercase = self.name
 332         else:
 333             assert False
 334         if expected_suffix and self.name.endswith(expected_suffix):
 335             uppercase = self.name[:-len(expected_suffix)]
 336         return TypeIdentifiers(typename=typename, classtype=classtype,
 337                                instancetype=instancetype, uppercase=uppercase)
 338
 339     def gen_patches(self) -> Iterable[Patch]:
 340         # the implementation is a bit tricky because we need to group
 341         # macros dealing with the same type into a single declaration
 342         if self.type_identifiers is None:
 343             self.warn("couldn't extract type information from macro %s", self.name)
 344             return
 345
 346         if self.name == 'INTERFACE_CLASS':
 347             # INTERFACE_CLASS is special and won't be patched
 348             return
 349
 350         for checker,suffix in EXPECTED_CHECKER_SUFFIXES:
 351             if self.name.endswith(suffix):
 352                 if self.checker != checker:
 353                     self.warn("macro %s is using macro %s instead of %s", self.name, self.checker, checker)
 354                     return
 355                 break
 356
 357         matches = self.find_matching_macros()
 358         DBG("found %d matching macros: %s", len(matches), ' '.join(m.name for m in matches))
 359         # we will generate patches only when processing the first macro:
 360         if matches[0].start != self.start:
 361             DBG("skipping %s (will patch when handling %s)", self.name, matches[0].name)
 362             return
 363
 364
 365         ids = self.merge_ids(matches)
 366         if ids is None:
 367             DBG("type identifier mismatch, won't patch %s", self.name)
 368             return
 369
 370         if not ids.uppercase:
 371             self.warn("macro %s doesn't follow the expected name pattern", self.name)
 372             return
 373         if not ids.typename:
 374             self.warn("macro %s: couldn't extract type name", self.name)
 375             return
 376
 377         #issues = ids.check_consistency()
 378         #if issues:
 379         #    for i in issues:
 380         #        self.warn("inconsistent identifiers: %s", i)
 381
 382         names = [n for n in (ids.instancetype, ids.classtype, ids.uppercase, ids.typename)
 383                  if n is not None]
 384         if len(set(names)) != len(names):
 385             self.warn("duplicate names used by macro: %r", ids)
 386             return
 387
 388         assert ids.classtype or ids.instancetype
 389         assert ids.typename
 390         assert ids.uppercase
 391         if ids.classtype and ids.instancetype:
 392             new_decl = (f'DECLARE_OBJ_CHECKERS({ids.instancetype}, {ids.classtype},\n'
 393                         f'                     {ids.uppercase}, {ids.typename})\n')
 394         elif ids.classtype:
 395             new_decl = (f'DECLARE_CLASS_CHECKERS({ids.classtype}, {ids.uppercase},\n'
 396                         f'                       {ids.typename})\n')
 397         elif ids.instancetype:
 398             new_decl = (f'DECLARE_INSTANCE_CHECKER({ids.instancetype}, {ids.uppercase},\n'
 399                         f'                         {ids.typename})\n')
 400         else:
 401             assert False
 402
 403         # we need to ensure the typedefs are already available
 404         issues = []
 405         for t in [ids.instancetype, ids.classtype]:
 406             if not t:
 407                 continue
 408             if re.fullmatch(RE_STRUCT_TYPE, t):
 409                 self.info("type %s is not a typedef", t)
 410                 continue
 411             td = find_typedef(self.file, t)
 412             #if not td and self.allfiles.find_file('include/qemu/typedefs.h'):
 413             #
 414             if not td:
 415                 # it is OK if the typedef is in typedefs.h
 416                 f = self.allfiles.find_file('include/qemu/typedefs.h')
 417                 if f and find_typedef(f, t):
 418                     self.info("typedef %s found in typedefs.h", t)
 419                     continue
 420
 421                 issues.append("couldn't find typedef %s" % (t))
 422             elif td.start() > self.start():
 423                 issues.append("typedef %s need to be moved earlier in the file" % (td.name))
 424
 425         for issue in issues:
 426             self.warn(issue)
 427
 428         if issues and not self.file.force:
 429             return
 430
 431         # delete all matching macros and add new declaration:
 432         for m in matches:
 433             yield m.make_patch('')
 434         for issue in issues:
 435             yield self.prepend("/* FIXME: %s */\n" % (issue))
 436         yield self.append(new_decl)
 437
 438 class InterfaceCheckMacro(FileMatch):
 439     """Type checking macro using INTERFACE_CHECK
 440     Will be replaced by DECLARE_INTERFACE_CHECKER
 441     """
 442     regexp = S(RE_MACRO_DEFINE,
 443                'INTERFACE_CHECK',
 444                r'\s*\(\s*', OR(NAMED('instancetype', RE_IDENTIFIER), RE_TYPE, name='c_type'),
 445                r'\s*,', CPP_SPACE,
 446                OPTIONAL_PARS(RE_IDENTIFIER), r',', CPP_SPACE,
 447                NAMED('qom_typename', RE_IDENTIFIER), r'\s*\)\n')
 448
 449     def required_identifiers(self) -> Iterable[RequiredIdentifier]:
 450         yield RequiredIdentifier('include', '"qom/object.h"')
 451         yield RequiredIdentifier('type', self.group('instancetype'))
 452         yield RequiredIdentifier('constant', self.group('qom_typename'))
 453
 454     def gen_patches(self) -> Iterable[Patch]:
 455         if self.file.filename_matches('qom/object.h'):
 456             self.debug("skipping object.h")
 457             return
 458
 459         typename = self.group('qom_typename')
 460         uppercase = self.name
 461         instancetype = self.group('instancetype')
 462         c = f"DECLARE_INTERFACE_CHECKER({instancetype}, {uppercase},\n"+\
 463             f"                          {typename})\n"
 464         yield self.make_patch(c)
 465
 466
 467 class TypeDeclaration(FileMatch):
 468     """Parent class to all type declarations"""
 469     @property
 470     def instancetype(self) -> Optional[str]:
 471         return self.getgroup('instancetype')
 472
 473     @property
 474     def classtype(self) -> Optional[str]:
 475         return self.getgroup('classtype')
 476
 477     @property
 478     def typename(self) -> Optional[str]:
 479         return self.getgroup('typename')
 480
 481 class TypeCheckerDeclaration(TypeDeclaration):
 482     """Parent class to all type checker declarations"""
 483     @property
 484     def typename(self) -> str:
 485         return self.group('typename')
 486
 487     @property
 488     def uppercase(self) -> str:
 489         return self.group('uppercase')
 490
 491 class DeclareInstanceChecker(TypeCheckerDeclaration):
 492     """DECLARE_INSTANCE_CHECKER use"""
 493     #TODO: replace lonely DECLARE_INSTANCE_CHECKER with DECLARE_OBJ_CHECKERS
 494     #      if all types are found.
 495     #      This will require looking up the correct class type in the TypeInfo
 496     #      structs in another file
 497     regexp = S(r'^[ \t]*DECLARE_INSTANCE_CHECKER\s*\(\s*',
 498                NAMED('instancetype', RE_TYPE), r'\s*,\s*',
 499                NAMED('uppercase', RE_IDENTIFIER), r'\s*,\s*',
 500                OR(RE_IDENTIFIER, RE_STRING, RE_MACRO_CONCAT, RE_FUN_CALL, name='typename'), SP,
 501                r'\)[ \t]*;?[ \t]*\n')
 502
 503     def required_identifiers(self) -> Iterable[RequiredIdentifier]:
 504         yield RequiredIdentifier('include', '"qom/object.h"')
 505         yield RequiredIdentifier('constant', self.group('typename'))
 506         yield RequiredIdentifier('type', self.group('instancetype'))
 507
 508 class DeclareInterfaceChecker(TypeCheckerDeclaration):
 509     """DECLARE_INTERFACE_CHECKER use"""
 510     regexp = S(r'^[ \t]*DECLARE_INTERFACE_CHECKER\s*\(\s*',
 511                NAMED('instancetype', RE_TYPE), r'\s*,\s*',
 512                NAMED('uppercase', RE_IDENTIFIER), r'\s*,\s*',
 513                OR(RE_IDENTIFIER, RE_STRING, RE_MACRO_CONCAT, RE_FUN_CALL, name='typename'), SP,
 514                r'\)[ \t]*;?[ \t]*\n')
 515
 516     def required_identifiers(self) -> Iterable[RequiredIdentifier]:
 517         yield RequiredIdentifier('include', '"qom/object.h"')
 518         yield RequiredIdentifier('constant', self.group('typename'))
 519         yield RequiredIdentifier('type', self.group('instancetype'))
 520
 521 class DeclareInstanceType(TypeDeclaration):
 522     """DECLARE_INSTANCE_TYPE use"""
 523     regexp = S(r'^[ \t]*DECLARE_INSTANCE_TYPE\s*\(\s*',
 524                NAMED('uppercase', RE_IDENTIFIER), r'\s*,\s*',
 525                NAMED('instancetype', RE_TYPE), SP,
 526                r'\)[ \t]*;?[ \t]*\n')
 527
 528     def required_identifiers(self) -> Iterable[RequiredIdentifier]:
 529         yield RequiredIdentifier('include', '"qom/object.h"')
 530         yield RequiredIdentifier('type', self.group('instancetype'))
 531
 532 class DeclareClassType(TypeDeclaration):
 533     """DECLARE_CLASS_TYPE use"""
 534     regexp = S(r'^[ \t]*DECLARE_CLASS_TYPE\s*\(\s*',
 535                NAMED('uppercase', RE_IDENTIFIER), r'\s*,\s*',
 536                NAMED('classtype', RE_TYPE), SP,
 537                r'\)[ \t]*;?[ \t]*\n')
 538
 539     def required_identifiers(self) -> Iterable[RequiredIdentifier]:
 540         yield RequiredIdentifier('include', '"qom/object.h"')
 541         yield RequiredIdentifier('type', self.group('classtype'))
 542
 543
 544
 545 class DeclareClassCheckers(TypeCheckerDeclaration):
 546     """DECLARE_CLASS_CHECKER use"""
 547     regexp = S(r'^[ \t]*DECLARE_CLASS_CHECKERS\s*\(\s*',
 548                NAMED('classtype', RE_TYPE), r'\s*,\s*',
 549                NAMED('uppercase', RE_IDENTIFIER), r'\s*,\s*',
 550                OR(RE_IDENTIFIER, RE_STRING, RE_MACRO_CONCAT, RE_FUN_CALL, name='typename'), SP,
 551                r'\)[ \t]*;?[ \t]*\n')
 552
 553     def required_identifiers(self) -> Iterable[RequiredIdentifier]:
 554         yield RequiredIdentifier('include', '"qom/object.h"')
 555         yield RequiredIdentifier('constant', self.group('typename'))
 556         yield RequiredIdentifier('type', self.group('classtype'))
 557
 558 class DeclareObjCheckers(TypeCheckerDeclaration):
 559     """DECLARE_OBJ_CHECKERS use"""
 560     #TODO: detect when OBJECT_DECLARE_SIMPLE_TYPE can be used
 561     regexp = S(r'^[ \t]*DECLARE_OBJ_CHECKERS\s*\(\s*',
 562                NAMED('instancetype', RE_TYPE), r'\s*,\s*',
 563                NAMED('classtype', RE_TYPE), r'\s*,\s*',
 564                NAMED('uppercase', RE_IDENTIFIER), r'\s*,\s*',
 565                OR(RE_IDENTIFIER, RE_STRING, RE_MACRO_CONCAT, RE_FUN_CALL, name='typename'), SP,
 566                r'\)[ \t]*;?[ \t]*\n')
 567
 568     def required_identifiers(self) -> Iterable[RequiredIdentifier]:
 569         yield RequiredIdentifier('include', '"qom/object.h"')
 570         yield RequiredIdentifier('constant', self.group('typename'))
 571         yield RequiredIdentifier('type', self.group('classtype'))
 572         yield RequiredIdentifier('type', self.group('instancetype'))
 573
 574 class TypeDeclarationFixup(FileMatch):
 575     """Common base class for code that will look at a set of type declarations"""
 576     regexp = RE_FILE_BEGIN
 577     def gen_patches(self) -> Iterable[Patch]:
 578         if self.file.filename_matches('qom/object.h'):
 579             self.debug("skipping object.h")
 580             return
 581
 582         # group checkers by uppercase name:
 583         decl_types: List[Type[TypeDeclaration]] = [DeclareInstanceChecker, DeclareInstanceType,
 584                                                    DeclareClassCheckers, DeclareClassType,
 585                                                    DeclareObjCheckers]
 586         checker_dict: Dict[str, List[TypeDeclaration]] = {}
 587         for t in decl_types:
 588             for m in self.file.matches_of_type(t):
 589                 checker_dict.setdefault(m.group('uppercase'), []).append(m)
 590         self.debug("checker_dict: %r", checker_dict)
 591         for uppercase,checkers in checker_dict.items():
 592             fields = ('instancetype', 'classtype', 'uppercase', 'typename')
 593             fvalues = dict((field, set(getattr(m, field) for m in checkers
 594                                        if getattr(m, field, None) is not None))
 595                             for field in fields)
 596             for field,values in fvalues.items():
 597                 if len(values) > 1:
 598                     for c in checkers:
 599                         c.warn("%s mismatch (%s)", field, ' '.join(values))
 600                     return
 601
 602             field_dict = dict((f, v.pop() if v else None) for f,v in fvalues.items())
 603             yield from self.gen_patches_for_type(uppercase, checkers, field_dict)
 604
 605     def find_conflicts(self, uppercase: str, checkers: List[TypeDeclaration]) -> bool:
 606         """Look for conflicting declarations that would make it unsafe to add new ones"""
 607         conflicting: List[FileMatch] = []
 608         # conflicts in the same file:
 609         conflicting.extend(chain(self.file.find_matches(DefineDirective, uppercase),
 610                                  self.file.find_matches(DeclareInterfaceChecker, uppercase, 'uppercase'),
 611                                  self.file.find_matches(DeclareClassType, uppercase, 'uppercase'),
 612                                  self.file.find_matches(DeclareInstanceType, uppercase, 'uppercase')))
 613
 614         # conflicts in another file:
 615         conflicting.extend(o for o in chain(self.allfiles.find_matches(DeclareInstanceChecker, uppercase, 'uppercase'),
 616                                             self.allfiles.find_matches(DeclareClassCheckers, uppercase, 'uppercase'),
 617                                             self.allfiles.find_matches(DeclareInterfaceChecker, uppercase, 'uppercase'),
 618                                             self.allfiles.find_matches(DefineDirective, uppercase))
 619                            if o is not None and o.file != self.file
 620                                # if both are .c files, there's no conflict at all:
 621                                and not (o.file.filename.suffix == '.c' and
 622                                        self.file.filename.suffix == '.c'))
 623
 624         if conflicting:
 625             for c in checkers:
 626                 c.warn("skipping due to conflicting %s macro", uppercase)
 627             for o in conflicting:
 628                 if o is None:
 629                     continue
 630                 o.warn("conflicting %s macro is here", uppercase)
 631             return True
 632
 633         return False
 634
 635     def gen_patches_for_type(self, uppercase: str,
 636                              checkers: List[TypeDeclaration],
 637                              fields: Dict[str, Optional[str]]) -> Iterable[Patch]:
 638         """Should be reimplemented by subclasses"""
 639         return
 640         yield
 641
 642 class DeclareVoidTypes(TypeDeclarationFixup):
 643     """Add DECLARE_*_TYPE(..., void) when there's no declared type"""
 644     regexp = RE_FILE_BEGIN
 645     def gen_patches_for_type(self, uppercase: str,
 646                              checkers: List[TypeDeclaration],
 647                              fields: Dict[str, Optional[str]]) -> Iterable[Patch]:
 648         if self.find_conflicts(uppercase, checkers):
 649             return
 650
 651         #_,last_checker = max((m.start(), m) for m in checkers)
 652         _,first_checker = min((m.start(), m) for m in checkers)
 653
 654         if not any(m.instancetype for m in checkers):
 655             yield first_checker.prepend(f'DECLARE_INSTANCE_TYPE({uppercase}, void)\n')
 656         if not any(m.classtype for m in checkers):
 657             yield first_checker.prepend(f'DECLARE_CLASS_TYPE({uppercase}, void)\n')
 658
 659         #if not all(len(v) == 1 for v in fvalues.values()):
 660         #    return
 661         #
 662         #final_values = dict((field, values.pop())
 663         #                    for field,values in fvalues.items())
 664         #s = (f"DECLARE_OBJ_CHECKERS({final_values['instancetype']}, {final_values['classtype']},\n"+
 665         #        f"                     {final_values['uppercase']}, {final_values['typename']})\n")
 666         #for c in checkers:
 667         #    yield c.make_removal_patch()
 668         #yield last_checker.append(s)
 669
 670
 671 class AddDeclareTypeName(TypeDeclarationFixup):
 672     """Add DECLARE_TYPE_NAME declarations if necessary"""
 673     def gen_patches_for_type(self, uppercase: str,
 674                              checkers: List[TypeDeclaration],
 675                              fields: Dict[str, Optional[str]]) -> Iterable[Patch]:
 676         typename = fields.get('typename')
 677         if typename is None:
 678             self.warn("typename unavailable")
 679             return
 680         if typename == f'TYPE_{uppercase}':
 681             self.info("already using TYPE_%s as type name", uppercase)
 682             return
 683         if self.file.find_match(DeclareTypeName, uppercase, 'uppercase'):
 684             self.info("type name for %s already declared", uppercase)
 685             return
 686         _,first_checker = min((m.start(), m) for m in checkers)
 687         s = f'DECLARE_TYPE_NAME({uppercase}, {typename})\n'
 688         yield first_checker.prepend(s)
 689
 690 class TrivialClassStruct(FileMatch):
 691     """Trivial class struct"""
 692     regexp = S(r'^[ \t]*struct\s*', NAMED('name', RE_IDENTIFIER),
 693                r'\s*{\s*', NAMED('parent_struct', RE_IDENTIFIER), r'\s*parent(_class)?\s*;\s*};\n')
 694
 695 class DeclareTypeName(FileMatch):
 696     """DECLARE_TYPE_NAME usage"""
 697     regexp = S(r'^[ \t]*DECLARE_TYPE_NAME\s*\(',
 698                NAMED('uppercase', RE_IDENTIFIER), r'\s*,\s*',
 699                OR(RE_IDENTIFIER, RE_STRING, RE_MACRO_CONCAT, RE_FUN_CALL, name='typename'),
 700                r'\s*\);?[ \t]*\n')
 701
 702 class ObjectDeclareType(TypeCheckerDeclaration):
 703     """OBJECT_DECLARE_TYPE usage
 704     Will be replaced with OBJECT_DECLARE_SIMPLE_TYPE if possible
 705     """
 706     regexp = S(r'^[ \t]*OBJECT_DECLARE_TYPE\s*\(',
 707                NAMED('instancetype', RE_TYPE), r'\s*,\s*',
 708                NAMED('classtype', RE_TYPE), r'\s*,\s*',
 709                NAMED('uppercase', RE_IDENTIFIER), SP,
 710                r'\)[ \t]*;?[ \t]*\n')
 711
 712     def gen_patches(self):
 713         DBG("groups: %r", self.match.groupdict())
 714         trivial_struct = self.file.find_match(TrivialClassStruct, self.group('classtype'))
 715         if trivial_struct:
 716             d = self.match.groupdict().copy()
 717             d['parent_struct'] = trivial_struct.group("parent_struct")
 718             yield trivial_struct.make_removal_patch()
 719             c = ("OBJECT_DECLARE_SIMPLE_TYPE(%(instancetype)s, %(lowercase)s,\n"
 720                  "                           %(uppercase)s, %(parent_struct)s)\n" % d)
 721             yield self.make_patch(c)
 722
 723 class ObjectDeclareSimpleType(TypeCheckerDeclaration):
 724     """OBJECT_DECLARE_SIMPLE_TYPE usage"""
 725     regexp = S(r'^[ \t]*OBJECT_DECLARE_SIMPLE_TYPE\s*\(',
 726                NAMED('instancetype', RE_TYPE), r'\s*,\s*',
 727                NAMED('uppercase', RE_IDENTIFIER), SP,
 728                r'\)[ \t]*;?[ \t]*\n')
 729
 730 class OldStyleObjectDeclareSimpleType(TypeCheckerDeclaration):
 731     """OBJECT_DECLARE_SIMPLE_TYPE usage (old API)"""
 732     regexp = S(r'^[ \t]*OBJECT_DECLARE_SIMPLE_TYPE\s*\(',
 733                NAMED('instancetype', RE_TYPE), r'\s*,\s*',
 734                NAMED('lowercase', RE_IDENTIFIER), r'\s*,\s*',
 735                NAMED('uppercase', RE_IDENTIFIER), r'\s*,\s*',
 736                NAMED('parent_classtype', RE_TYPE), SP,
 737                r'\)[ \t]*;?[ \t]*\n')
 738
 739     @property
 740     def classtype(self) -> Optional[str]:
 741         instancetype = self.instancetype
 742         assert instancetype
 743         return f"{instancetype}Class"
 744
 745 def find_typename_uppercase(files: FileList, typename: str) -> Optional[str]:
 746     """Try to find what's the right MODULE_OBJ_NAME for a given type name"""
 747     decl = files.find_match(DeclareTypeName, name=typename, group='typename')
 748     if decl:
 749         return decl.group('uppercase')
 750     if typename.startswith('TYPE_'):
 751         return typename[len('TYPE_'):]
 752     return None
 753
 754 def find_type_checkers(files:FileList, name:str, group:str='uppercase') -> Iterable[TypeCheckerDeclaration]:
 755     """Find usage of DECLARE*CHECKER macro"""
 756     c: Type[TypeCheckerDeclaration]
 757     for c in (DeclareInstanceChecker, DeclareClassCheckers, DeclareObjCheckers, ObjectDeclareType, ObjectDeclareSimpleType):
 758         yield from files.find_matches(c, name=name, group=group)
 759
 760 class Include(FileMatch):
 761     """#include directive"""
 762     regexp = RE_INCLUDE
 763     def provided_identifiers(self) -> Iterable[RequiredIdentifier]:
 764         yield RequiredIdentifier('include', self.group('includepath'))
 765
 766 class InitialIncludes(FileMatch):
 767     """Initial #include block"""
 768     regexp = S(RE_FILE_BEGIN,
 769                M(SP, RE_COMMENTS,
 770                  r'^[ \t]*#[ \t]*ifndef[ \t]+', RE_IDENTIFIER, r'[ \t]*\n',
 771                  n='?', name='ifndef_block'),
 772                M(SP, RE_COMMENTS,
 773                  OR(RE_INCLUDE, RE_SIMPLEDEFINE),
 774                  n='*', name='includes'))
 775
 776 class SymbolUserList(NamedTuple):
 777     definitions: List[FileMatch]
 778     users: List[FileMatch]
 779
 780 class MoveSymbols(FileMatch):
 781     """Handle missing symbols
 782     - Move typedefs and defines when necessary
 783     - Add missing #include lines when necessary
 784     """
 785     regexp = RE_FILE_BEGIN
 786
 787     def gen_patches(self) -> Iterator[Patch]:
 788         if self.file.filename_matches('qom/object.h'):
 789             self.debug("skipping object.h")
 790             return
 791
 792         index: Dict[RequiredIdentifier, SymbolUserList] = {}
 793         definition_classes = [SimpleTypedefMatch, FullStructTypedefMatch, ConstantDefine, Include]
 794         user_classes = [TypeCheckMacro, DeclareObjCheckers, DeclareInstanceChecker, DeclareClassCheckers, InterfaceCheckMacro]
 795
 796         # first we scan for all symbol definitions and usage:
 797         for dc in definition_classes:
 798             defs = self.file.matches_of_type(dc)
 799             for d in defs:
 800                 DBG("scanning %r", d)
 801                 for i in d.provided_identifiers():
 802                     index.setdefault(i, SymbolUserList([], [])).definitions.append(d)
 803         DBG("index: %r", list(index.keys()))
 804         for uc in user_classes:
 805             users = self.file.matches_of_type(uc)
 806             for u in users:
 807                 for i in u.required_identifiers():
 808                     index.setdefault(i, SymbolUserList([], [])).users.append(u)
 809
 810         # validate all symbols:
 811         for i,ul in index.items():
 812             if not ul.users:
 813                 # unused symbol
 814                 continue
 815
 816             # symbol not defined
 817             if len(ul.definitions) == 0:
 818                 if i.type == 'include':
 819                    includes, = self.file.matches_of_type(InitialIncludes)
 820                    #FIXME: don't do this if we're already inside qom/object.h
 821                    yield includes.append(f'#include {i.name}\n')
 822                 else:
 823                     u.warn("definition of %s %s not found in file", i.type, i.name)
 824                 continue
 825
 826             # symbol defined twice:
 827             if len(ul.definitions) > 1:
 828                 ul.definitions[1].warn("%s defined twice", i.name)
 829                 ul.definitions[0].warn("previously defined here")
 830                 continue
 831
 832             # symbol defined.  check if all users are after its definition:
 833             assert len(ul.definitions) == 1
 834             definition = ul.definitions[0]
 835             DBG("handling repositioning of %r", definition)
 836             earliest = min(ul.users, key=lambda u: u.start())
 837             if earliest.start() > definition.start():
 838                 DBG("%r is OK", definition)
 839                 continue
 840
 841             DBG("%r needs to be moved", definition)
 842             if isinstance(definition, SimpleTypedefMatch) \
 843                or isinstance(definition, ConstantDefine):
 844                 # simple typedef or define can be moved directly:
 845                 yield definition.make_removal_patch()
 846                 yield earliest.prepend(definition.group(0))
 847             elif isinstance(definition, FullStructTypedefMatch) \
 848                  and definition.group('structname'):
 849                 # full struct typedef is more complex: we need to remove
 850                 # the typedef
 851                 yield from definition.move_typedef(earliest.start())
 852             else:
 853                 definition.warn("definition of %s %s needs to be moved earlier in the file", i.type, i.name)
 854                 earliest.warn("definition of %s %s is used here", i.type, i.name)
 855
 856
 857 class EmptyPreprocessorConditional(FileMatch):
 858     """Delete empty preprocessor conditionals"""
 859     regexp = r'^[ \t]*#(if|ifdef)[ \t].*\n+[ \t]*#endif[ \t]*\n'
 860     def gen_patches(self) -> Iterable[Patch]:
 861         yield self.make_removal_patch()