source/tools/entity/checkrefs.py

   1 #!/usr/bin/env python3
   2 from argparse import ArgumentParser
   3 from io import BytesIO
   4 from json import load, loads
   5 from pathlib import Path
   6 from re import split, match
   7 from struct import unpack, calcsize
   8 from os.path import sep, exists, basename
   9 from xml.etree import ElementTree
  10 import sys
  11 from scriptlib import SimulTemplateEntity, find_files
  12 from logging import WARNING, getLogger, StreamHandler, INFO, Formatter, Filter
  13
  14 class SingleLevelFilter(Filter):
  15     def __init__(self, passlevel, reject):
  16         self.passlevel = passlevel
  17         self.reject = reject
  18
  19     def filter(self, record):
  20         if self.reject:
  21             return (record.levelno != self.passlevel)
  22         else:
  23             return (record.levelno == self.passlevel)
  24
  25 class CheckRefs:
  26     def __init__(self):
  27         # list of relative root file:str
  28         self.files = []
  29         # list of relative file:str
  30         self.roots = []
  31         # list of tuple (parent_file:str, dep_file:str)
  32         self.deps = []
  33         self.vfs_root = Path(__file__).resolve().parents[3] / 'binaries' / 'data' / 'mods'
  34         self.supportedTextureFormats = ('dds', 'png')
  35         self.supportedMeshesFormats = ('pmd', 'dae')
  36         self.supportedAnimationFormats = ('psa', 'dae')
  37         self.supportedAudioFormats = ('ogg')
  38         self.mods = []
  39         self.__init_logger
  40
  41     @property
  42     def __init_logger(self):
  43         logger = getLogger(__name__)
  44         logger.setLevel(INFO)
  45         # create a console handler, seems nicer to Windows and for future uses
  46         ch = StreamHandler(sys.stdout)
  47         ch.setLevel(INFO)
  48         ch.setFormatter(Formatter('%(levelname)s - %(message)s'))
  49         f1 = SingleLevelFilter(INFO, False)
  50         ch.addFilter(f1)
  51         logger.addHandler(ch)
  52         errorch = StreamHandler(sys.stderr)
  53         errorch.setLevel(WARNING)
  54         errorch.setFormatter(Formatter('%(levelname)s - %(message)s'))
  55         logger.addHandler(errorch)
  56         self.logger = logger
  57
  58     def main(self):
  59         ap = ArgumentParser(description="Checks the game files for missing dependencies, unused files,"
  60                             " and for file integrity.")
  61         ap.add_argument('-u', '--check-unused', action='store_true',
  62                         help="check for all the unused files in the given mods and their dependencies."
  63                         " Implies --check-map-xml. Currently yields a lot of false positives.")
  64         ap.add_argument('-x', '--check-map-xml', action='store_true',
  65                         help="check maps for missing actor and templates.")
  66         ap.add_argument('-a', '--validate-actors', action='store_true',
  67                         help="run the validator.py script to check if the actors files have extra or missing textures."
  68                         " This currently only works for the public mod.")
  69         ap.add_argument('-t', '--validate-templates', action='store_true',
  70                         help="run the validator.py script to check if the xml files match their (.rng) grammar file.")
  71         ap.add_argument('-m', '--mods', metavar="MOD", dest='mods', nargs='+', default=['public'],
  72                         help="specify which mods to check. Default to public.")
  73         args = ap.parse_args()
  74         # force check_map_xml if check_unused is used to avoid false positives.
  75         args.check_map_xml |= args.check_unused
  76         # ordered uniq mods (dict maintains ordered keys from python 3.6)
  77         self.mods = list(dict.fromkeys([*args.mods, *self.get_mod_dependencies(*args.mods), 'mod']).keys())
  78         self.logger.info(f"Checking {'|'.join(args.mods)}'s integrity.")
  79         self.logger.info(f"The following mods will be loaded: {'|'.join(self.mods)}.")
  80         if args.check_map_xml:
  81             self.add_maps_xml()
  82         self.add_maps_pmp()
  83         self.add_entities()
  84         self.add_actors()
  85         self.add_variants()
  86         self.add_art()
  87         self.add_materials()
  88         self.add_particles()
  89         self.add_soundgroups()
  90         self.add_audio()
  91         self.add_gui_xml()
  92         self.add_gui_data()
  93         self.add_civs()
  94         self.add_rms()
  95         self.add_techs()
  96         self.add_terrains()
  97         self.add_auras()
  98         self.add_tips()
  99         self.check_deps()
 100         if args.check_unused:
 101             self.check_unused()
 102         if args.validate_templates:
 103             sys.path.append("../xmlvalidator/")
 104             from validate_grammar import RelaxNGValidator
 105             validate = RelaxNGValidator(self.vfs_root, self.mods)
 106             validate.run()
 107         if args.validate_actors:
 108             sys.path.append("../xmlvalidator/")
 109             from validator import Validator
 110             validator = Validator(self.vfs_root, self.mods)
 111             validator.run()
 112
 113     def get_mod_dependencies(self, *mods):
 114         modjsondeps = []
 115         for mod in mods:
 116             mod_json_path = self.vfs_root / mod / 'mod.json'
 117             if not exists(mod_json_path):
 118                 continue
 119
 120             with open(mod_json_path, encoding='utf-8') as f:
 121                 modjson = load(f)
 122             # 0ad's folder isn't named like the mod.
 123             modjsondeps.extend(['public' if '0ad' in dep else dep for dep in modjson.get('dependencies', [])])
 124         return modjsondeps
 125
 126     def vfs_to_relative_to_mods(self, vfs_path):
 127         for dep in self.mods:
 128             fn = Path(dep) / vfs_path
 129             if (self.vfs_root / fn).exists():
 130                 return fn
 131         return None
 132
 133     def vfs_to_physical(self, vfs_path):
 134         fn = self.vfs_to_relative_to_mods(vfs_path)
 135         return self.vfs_root / fn
 136
 137     def find_files(self, vfs_path, *ext_list):
 138         return find_files(self.vfs_root, self.mods, vfs_path, *ext_list)
 139
 140     def add_maps_xml(self):
 141         self.logger.info("Loading maps XML...")
 142         mapfiles = self.find_files('maps/scenarios', 'xml')
 143         mapfiles.extend(self.find_files('maps/skirmishes', 'xml'))
 144         mapfiles.extend(self.find_files('maps/tutorials', 'xml'))
 145         actor_prefix = 'actor|'
 146         resource_prefix = 'resource|'
 147         for (fp, ffp) in sorted(mapfiles):
 148             self.files.append(str(fp))
 149             self.roots.append(str(fp))
 150             et_map = ElementTree.parse(ffp).getroot()
 151             entities = et_map.find('Entities')
 152             used = {entity.find('Template').text.strip() for entity in entities.findall('Entity')} if entities is not None else {}
 153             for template in used:
 154                 if template.startswith(actor_prefix):
 155                     self.deps.append((str(fp), f'art/actors/{template[len(actor_prefix):]}'))
 156                 elif template.startswith(resource_prefix):
 157                     self.deps.append((str(fp), f'simulation/templates/{template[len(resource_prefix):]}.xml'))
 158                 else:
 159                     self.deps.append((str(fp), f'simulation/templates/{template}.xml'))
 160             # Map previews
 161             settings = loads(et_map.find('ScriptSettings').text)
 162             if settings.get('Preview', None):
 163                 self.deps.append((str(fp), f'art/textures/ui/session/icons/mappreview/{settings["Preview"]}'))
 164
 165     def add_maps_pmp(self):
 166         self.logger.info("Loading maps PMP...")
 167         # Need to generate terrain texture filename=>relative path lookup first
 168         terrains = dict()
 169         for (fp, ffp) in self.find_files('art/terrains', 'xml'):
 170             name = fp.stem
 171             # ignore terrains.xml
 172             if name != 'terrains':
 173                 if name in terrains:
 174                     self.logger.warning(f"Duplicate terrain name '{name}' (from '{terrains[name]}' and '{ffp}')")
 175                 terrains[name] = str(fp)
 176         mapfiles = self.find_files('maps/scenarios', 'pmp')
 177         mapfiles.extend(self.find_files('maps/skirmishes', 'pmp'))
 178         for (fp, ffp) in sorted(mapfiles):
 179             self.files.append(str(fp))
 180             self.roots.append(str(fp))
 181             with open(ffp, 'rb') as f:
 182                 expected_header = b'PSMP'
 183                 header = f.read(len(expected_header))
 184                 if header != expected_header:
 185                     raise ValueError(f"Invalid PMP header {header} in '{ffp}'")
 186                 int_fmt = '<L'  # little endian long int
 187                 int_len = calcsize(int_fmt)
 188                 version, = unpack(int_fmt, f.read(int_len))
 189                 if version != 7:
 190                     raise ValueError(f"Invalid PMP version ({version}) in '{ffp}'")
 191                 datasize, = unpack(int_fmt, f.read(int_len))
 192                 mapsize, = unpack(int_fmt, f.read(int_len))
 193                 f.seek(2 * (mapsize * 16 + 1) * (mapsize * 16 + 1), 1)  # skip heightmap
 194                 numtexs, = unpack(int_fmt, f.read(int_len))
 195                 for i in range(numtexs):
 196                     length, = unpack(int_fmt, f.read(int_len))
 197                     terrain_name = f.read(length).decode('ascii')  # suppose ascii encoding
 198                     self.deps.append((str(fp), terrains.get(terrain_name, f'art/terrains/(unknown)/{terrain_name}')))
 199
 200     def add_entities(self):
 201         self.logger.info("Loading entities...")
 202         simul_templates_path = Path('simulation/templates')
 203         # TODO: We might want to get computed templates through the RL interface instead of computing the values ourselves.
 204         simul_template_entity = SimulTemplateEntity(self.vfs_root, self.logger)
 205         for (fp, _) in sorted(self.find_files(simul_templates_path, 'xml')):
 206             self.files.append(str(fp))
 207             entity = simul_template_entity.load_inherited(simul_templates_path, str(fp.relative_to(simul_templates_path)), self.mods)
 208             if entity.get('parent'):
 209                 for parent in entity.get('parent').split('|'):
 210                     self.deps.append((str(fp), str(simul_templates_path / (parent + '.xml'))))
 211             if not str(fp).startswith('template_'):
 212                 self.roots.append(str(fp))
 213                 if entity and entity.find('VisualActor') is not None and entity.find('VisualActor').find('Actor') is not None:
 214                     if entity.find('Identity'):
 215                         phenotype_tag = entity.find('Identity').find('Phenotype')
 216                         phenotypes = split(r'\s', phenotype_tag.text if phenotype_tag is not None and phenotype_tag.text else 'default')
 217                         actor = entity.find('VisualActor').find('Actor')
 218                         if '{phenotype}' in actor.text:
 219                             for phenotype in phenotypes:
 220                                 # See simulation2/components/CCmpVisualActor.cpp and Identity.js for explanation.
 221                                 actor_path = actor.text.replace('{phenotype}', phenotype)
 222                                 self.deps.append((str(fp), f'art/actors/{actor_path}'))
 223                         else:
 224                             actor_path = actor.text
 225                             self.deps.append((str(fp), f'art/actors/{actor_path}'))
 226                         foundation_actor = entity.find('VisualActor').find('FoundationActor')
 227                         if foundation_actor:
 228                             self.deps.append((str(fp), f'art/actors/{foundation_actor.text}'))
 229                 if entity.find('Sound'):
 230                     phenotype_tag = entity.find('Identity').find('Phenotype')
 231                     phenotypes = split(r'\s', phenotype_tag.text if phenotype_tag is not None and phenotype_tag.text else 'default')
 232                     lang_tag = entity.find('Identity').find('Lang')
 233                     lang = lang_tag.text if lang_tag is not None and lang_tag.text else 'greek'
 234                     sound_groups = entity.find('Sound').find('SoundGroups')
 235                     for sound_group in sound_groups:
 236                         if sound_group.text and sound_group.text.strip():
 237                             if '{phenotype}' in sound_group.text:
 238                                 for phenotype in phenotypes:
 239                                     # see simulation/components/Sound.js and Identity.js for explanation
 240                                     sound_path = sound_group.text.replace('{phenotype}', phenotype).replace('{lang}', lang)
 241                                     self.deps.append((str(fp), f'audio/{sound_path}'))
 242                             else:
 243                                 sound_path = sound_group.text.replace('{lang}', lang)
 244                                 self.deps.append((str(fp), f'audio/{sound_path}'))
 245                 if entity.find('Identity') is not None:
 246                     icon = entity.find('Identity').find('Icon')
 247                     if icon is not None and icon.text:
 248                         if entity.find('Formation') is not None:
 249                             self.deps.append((str(fp), f'art/textures/ui/session/icons/{icon.text}'))
 250                         else:
 251                             self.deps.append((str(fp), f'art/textures/ui/session/portraits/{icon.text}'))
 252                 if entity.find('Heal') is not None and entity.find('Heal').find('RangeOverlay') is not None:
 253                     range_overlay = entity.find('Heal').find('RangeOverlay')
 254                     for tag in ('LineTexture', 'LineTextureMask'):
 255                         elem = range_overlay.find(tag)
 256                         if elem is not None and elem.text:
 257                             self.deps.append((str(fp), f'art/textures/selection/{elem.text}'))
 258                 if entity.find('Selectable') is not None and entity.find('Selectable').find('Overlay') is not None \
 259                         and entity.find('Selectable').find('Overlay').find('Texture') is not None:
 260                     texture = entity.find('Selectable').find('Overlay').find('Texture')
 261                     for tag in ('MainTexture', 'MainTextureMask'):
 262                         elem = texture.find(tag)
 263                         if elem is not None and elem.text:
 264                             self.deps.append((str(fp), f'art/textures/selection/{elem.text}'))
 265                 if entity.find('Formation') is not None:
 266                     icon = entity.find('Formation').find('Icon')
 267                     if icon is not None and icon.text:
 268                         self.deps.append((str(fp), f'art/textures/ui/session/icons/{icon.text}'))
 269
 270     def append_variant_dependencies(self, variant, fp):
 271         variant_file = variant.get('file')
 272         mesh = variant.find('mesh')
 273         particles = variant.find('particles')
 274         texture_files = [tex.get('file') for tex in variant.find('textures').findall('texture')] \
 275             if variant.find('textures') is not None else []
 276         prop_actors = [prop.get('actor') for prop in variant.find('props').findall('prop')] \
 277             if variant.find('props') is not None else []
 278         animation_files = [anim.get('file') for anim in variant.find('animations').findall('animation')] \
 279             if variant.find('animations') is not None else []
 280         if variant_file:
 281             self.deps.append((str(fp), f'art/variants/{variant_file}'))
 282         if mesh is not None and mesh.text:
 283             self.deps.append((str(fp), f'art/meshes/{mesh.text}'))
 284         if particles is not None and particles.get('file'):
 285             self.deps.append((str(fp), f'art/particles/{particles.get("file")}'))
 286         for texture_file in [x for x in texture_files if x]:
 287             self.deps.append((str(fp), f'art/textures/skins/{texture_file}'))
 288         for prop_actor in [x for x in prop_actors if x]:
 289             self.deps.append((str(fp), f'art/actors/{prop_actor}'))
 290         for animation_file in [x for x in animation_files if x]:
 291             self.deps.append((str(fp), f'art/animation/{animation_file}'))
 292
 293     def append_actor_dependencies(self, actor, fp):
 294         for group in actor.findall('group'):
 295             for variant in group.findall('variant'):
 296                 self.append_variant_dependencies(variant, fp)
 297         material = actor.find('material')
 298         if material is not None and material.text:
 299             self.deps.append((str(fp), f'art/materials/{material.text}'))
 300
 301     def add_actors(self):
 302         self.logger.info("Loading actors...")
 303         for (fp, ffp) in sorted(self.find_files('art/actors', 'xml')):
 304             self.files.append(str(fp))
 305             self.roots.append(str(fp))
 306             root = ElementTree.parse(ffp).getroot()
 307             if root.tag == 'actor':
 308                 self.append_actor_dependencies(root, fp)
 309
 310             # model has lods
 311             elif root.tag == 'qualitylevels':
 312                 qualitylevels = root
 313                 for actor in qualitylevels.findall('actor'):
 314                     self.append_actor_dependencies(actor, fp)
 315                 for actor in qualitylevels.findall('inline'):
 316                     self.append_actor_dependencies(actor, fp)
 317
 318     def add_variants(self):
 319         self.logger.info("Loading variants...")
 320         for (fp, ffp) in sorted(self.find_files('art/variants', 'xml')):
 321             self.files.append(str(fp))
 322             self.roots.append(str(fp))
 323             variant = ElementTree.parse(ffp).getroot()
 324             self.append_variant_dependencies(variant, fp)
 325
 326     def add_art(self):
 327         self.logger.info("Loading art files...")
 328         self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/textures/particles', *self.supportedTextureFormats)])
 329         self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/textures/terrain', *self.supportedTextureFormats)])
 330         self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/textures/skins', *self.supportedTextureFormats)])
 331         self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/meshes', *self.supportedMeshesFormats)])
 332         self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/animation', *self.supportedAnimationFormats)])
 333
 334
 335     def add_materials(self):
 336         self.logger.info("Loading materials...")
 337         for (fp, ffp) in sorted(self.find_files('art/materials', 'xml')):
 338             self.files.append(str(fp))
 339             material_elem = ElementTree.parse(ffp).getroot()
 340             for alternative in material_elem.findall('alternative'):
 341                 material = alternative.get('material')
 342                 if material:
 343                     self.deps.append((str(fp), f'art/materials/{material}'))
 344
 345     def add_particles(self):
 346         self.logger.info("Loading particles...")
 347         for (fp, ffp) in sorted(self.find_files('art/particles', 'xml')):
 348             self.files.append(str(fp))
 349             self.roots.append(str(fp))
 350             particle = ElementTree.parse(ffp).getroot()
 351             texture = particle.find('texture')
 352             if texture:
 353                 self.deps.append((str(fp), texture.text))
 354
 355     def add_soundgroups(self):
 356         self.logger.info("Loading sound groups...")
 357         for (fp, ffp) in sorted(self.find_files('audio', 'xml')):
 358             self.files.append(str(fp))
 359             self.roots.append(str(fp))
 360             sound_group = ElementTree.parse(ffp).getroot()
 361             path = sound_group.find('Path').text.rstrip('/')
 362             for sound in sound_group.findall('Sound'):
 363                 self.deps.append((str(fp), f'{path}/{sound.text}'))
 364
 365     def add_audio(self):
 366         self.logger.info("Loading audio files...")
 367         self.files.extend([str(fp) for (fp, ffp) in self.find_files('audio/', self.supportedAudioFormats)])
 368
 369
 370     def add_gui_object_repeat(self, obj, fp):
 371         for repeat in obj.findall('repeat'):
 372             for sub_obj in repeat.findall('object'):
 373                 # TODO: look at sprites, styles, etc
 374                 self.add_gui_object_include(sub_obj, fp)
 375             for sub_obj in repeat.findall('objects'):
 376                 # TODO: look at sprites, styles, etc
 377                 self.add_gui_object_include(sub_obj, fp)
 378
 379             self.add_gui_object_include(repeat, fp)
 380
 381     def add_gui_object_include(self, obj, fp):
 382         for include in obj.findall('include'):
 383             included_file = include.get('file')
 384             if included_file:
 385                 self.deps.append((str(fp), f'{included_file}'))
 386
 387     def add_gui_object(self, parent, fp):
 388         if parent is None:
 389             return
 390
 391         for obj in parent.findall('object'):
 392             # TODO: look at sprites, styles, etc
 393             self.add_gui_object_repeat(obj, fp)
 394             self.add_gui_object_include(obj, fp)
 395             self.add_gui_object(obj, fp)
 396         for obj in parent.findall('objects'):
 397             # TODO: look at sprites, styles, etc
 398             self.add_gui_object_repeat(obj, fp)
 399             self.add_gui_object_include(obj, fp)
 400             self.add_gui_object(obj, fp)
 401
 402
 403     def add_gui_xml(self):
 404         self.logger.info("Loading GUI XML...")
 405         for (fp, ffp) in sorted(self.find_files('gui', 'xml')):
 406             self.files.append(str(fp))
 407             # GUI page definitions are assumed to be named page_[something].xml and alone in that.
 408             if match(r".*[\\\/]page(_[^.\/\\]+)?\.xml$", str(fp)):
 409                 self.roots.append(str(fp))
 410                 root_xml = ElementTree.parse(ffp).getroot()
 411                 for include in root_xml.findall('include'):
 412                     # If including an entire directory, find all the *.xml files
 413                     if include.text.endswith('/'):
 414                         self.deps.extend([(str(fp), str(sub_fp)) for (sub_fp, sub_ffp) in self.find_files(f'gui/{include.text}', 'xml')])
 415                     else:
 416                         self.deps.append((str(fp), f'gui/{include.text}'))
 417             else:
 418                 xml = ElementTree.parse(ffp)
 419                 root_xml = xml.getroot()
 420                 name = root_xml.tag
 421                 self.roots.append(str(fp))
 422                 if name in ('objects', 'object'):
 423                     for script in root_xml.findall('script'):
 424                         if script.get('file'):
 425                             self.deps.append((str(fp), script.get('file')))
 426                         if script.get('directory'):
 427                             # If including an entire directory, find all the *.js files
 428                             self.deps.extend([(str(fp), str(sub_fp)) for (sub_fp, sub_ffp) in self.find_files(script.get('directory'), 'js')])
 429                     self.add_gui_object(root_xml, fp)
 430                 elif name == 'setup':
 431                     # TODO: look at sprites, styles, etc
 432                     pass
 433                 elif name == 'styles':
 434                     for style in root_xml.findall('style'):
 435                         if(style.get('sound_opened')):
 436                             self.deps.append((str(fp), f"{style.get('sound_opened')}"))
 437                         if(style.get('sound_closed')):
 438                             self.deps.append((str(fp), f"{style.get('sound_closed')}"))
 439                         if(style.get('sound_selected')):
 440                             self.deps.append((str(fp), f"{style.get('sound_selected')}"))
 441                         if(style.get('sound_disabled')):
 442                             self.deps.append((str(fp), f"{style.get('sound_disabled')}"))
 443                     # TODO: look at sprites, styles, etc
 444                     pass
 445                 elif name == 'sprites':
 446                     for sprite in root_xml.findall('sprite'):
 447                         for image in sprite.findall('image'):
 448                             if image.get('texture'):
 449                                 self.deps.append((str(fp), f"art/textures/ui/{image.get('texture')}"))
 450                 else:
 451                     bio = BytesIO()
 452                     xml.write(bio)
 453                     bio.seek(0)
 454                     raise ValueError(f"Unexpected GUI XML root element '{name}':\n{bio.read().decode('ascii')}")
 455
 456     def add_gui_data(self):
 457         self.logger.info("Loading GUI data...")
 458         self.files.extend([str(fp) for (fp, ffp) in self.find_files('gui', 'js')])
 459         self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/textures/ui', *self.supportedTextureFormats)])
 460         self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/textures/selection', *self.supportedTextureFormats)])
 461
 462     def add_civs(self):
 463         self.logger.info("Loading civs...")
 464         for (fp, ffp) in sorted(self.find_files('simulation/data/civs', 'json')):
 465             self.files.append(str(fp))
 466             self.roots.append(str(fp))
 467             with open(ffp, encoding='utf-8') as f:
 468                 civ = load(f)
 469             for music in civ.get('Music', []):
 470                 self.deps.append((str(fp), f"audio/music/{music['File']}"))
 471
 472     def add_tips(self):
 473         self.logger.info("Loading tips...")
 474         for (fp, ffp) in sorted(self.find_files('gui/text/tips', 'txt')):
 475             relative_path = str(fp)
 476             self.files.append(relative_path)
 477             self.roots.append(relative_path)
 478             self.deps.append((relative_path, f"art/textures/ui/loading/tips/{basename(relative_path).split('.')[0]}.png"))
 479
 480
 481     def add_rms(self):
 482         self.logger.info("Loading random maps...")
 483         self.files.extend([str(fp) for (fp, ffp) in self.find_files('maps/random', 'js')])
 484         for (fp, ffp) in sorted(self.find_files('maps/random', 'json')):
 485             if str(fp).startswith('maps/random/rmbiome'):
 486                 continue
 487             self.files.append(str(fp))
 488             self.roots.append(str(fp))
 489             with open(ffp, encoding='utf-8') as f:
 490                 randmap = load(f)
 491             settings = randmap.get('settings', {})
 492             if settings.get('Script', None):
 493                 self.deps.append((str(fp), f"maps/random/{settings['Script']}"))
 494             # Map previews
 495             if settings.get('Preview', None):
 496                 self.deps.append((str(fp), f'art/textures/ui/session/icons/mappreview/{settings["Preview"]}'))
 497
 498     def add_techs(self):
 499         self.logger.info("Loading techs...")
 500         for (fp, ffp) in sorted(self.find_files('simulation/data/technologies', 'json')):
 501             self.files.append(str(fp))
 502             self.roots.append(str(fp))
 503             with open(ffp, encoding='utf-8') as f:
 504                 tech = load(f)
 505             if tech.get('icon', None):
 506                 self.deps.append((str(fp), f"art/textures/ui/session/portraits/technologies/{tech['icon']}"))
 507             if tech.get('supersedes', None):
 508                 self.deps.append((str(fp), f"simulation/data/technologies/{tech['supersedes']}.json"))
 509
 510     def add_terrains(self):
 511         self.logger.info("Loading terrains...")
 512         for (fp, ffp) in sorted(self.find_files('art/terrains', 'xml')):
 513             # ignore terrains.xml
 514             if str(fp).endswith('terrains.xml'):
 515                 continue
 516             self.files.append(str(fp))
 517             self.roots.append(str(fp))
 518             terrain = ElementTree.parse(ffp).getroot()
 519             for texture in terrain.find('textures').findall('texture'):
 520                 if texture.get('file'):
 521                     self.deps.append((str(fp), f"art/textures/terrain/{texture.get('file')}"))
 522             if terrain.find('material') is not None:
 523                 material = terrain.find('material').text
 524                 self.deps.append((str(fp), f"art/materials/{material}"))
 525
 526     def add_auras(self):
 527         self.logger.info("Loading auras...")
 528         for (fp, ffp) in sorted(self.find_files('simulation/data/auras', 'json')):
 529             self.files.append(str(fp))
 530             self.roots.append(str(fp))
 531             with open(ffp, encoding='utf-8') as f:
 532                 aura = load(f)
 533             if aura.get('overlayIcon', None):
 534                 self.deps.append((str(fp), aura['overlayIcon']))
 535             range_overlay = aura.get('rangeOverlay', {})
 536             for prop in ('lineTexture', 'lineTextureMask'):
 537                 if range_overlay.get(prop, None):
 538                     self.deps.append((str(fp), f"art/textures/selection/{range_overlay[prop]}"))
 539
 540     def check_deps(self):
 541         self.logger.info("Looking for missing files...")
 542         uniq_files = set(self.files)
 543         uniq_files = [r.replace(sep, '/') for r in uniq_files]
 544         lower_case_files = {f.lower(): f for f in uniq_files}
 545         reverse_deps = dict()
 546         for parent, dep in self.deps:
 547             if sep != '/':
 548                 parent = parent.replace(sep, '/')
 549                 dep = dep.replace(sep, '/')
 550             if dep not in reverse_deps:
 551                 reverse_deps[dep] = {parent}
 552             else:
 553                 reverse_deps[dep].add(parent)
 554
 555         for dep in sorted(reverse_deps.keys()):
 556             if "simulation/templates" in dep and (
 557                     dep.replace("templates/", "template/special/filter/") in uniq_files or
 558                     dep.replace("templates/", "template/mixins/") in uniq_files
 559                 ):
 560                 continue
 561
 562             if dep in uniq_files:
 563                 continue
 564
 565             callers = [str(self.vfs_to_relative_to_mods(ref)) for ref in reverse_deps[dep]]
 566             self.logger.warning(f"Missing file '{dep}' referenced by: {', '.join(sorted(callers))}")
 567             if dep.lower() in lower_case_files:
 568                 self.logger.warning(f"### Case-insensitive match (found '{lower_case_files[dep.lower()]}')")
 569
 570     def check_unused(self):
 571         self.logger.info("Looking for unused files...")
 572         deps = dict()
 573         for parent, dep in self.deps:
 574             if sep != '/':
 575                 parent = parent.replace(sep, '/')
 576                 dep = dep.replace(sep, '/')
 577
 578             if parent not in deps:
 579                 deps[parent] = {dep}
 580             else:
 581                 deps[parent].add(dep)
 582
 583         uniq_files = set(self.files)
 584         uniq_files = [r.replace(sep, '/') for r in uniq_files]
 585         reachable = list(set(self.roots))
 586         reachable = [r.replace(sep, '/') for r in reachable]
 587         while True:
 588             new_reachable = []
 589             for r in reachable:
 590                 new_reachable.extend([x for x in deps.get(r, {}) if x not in reachable])
 591             if new_reachable:
 592                 reachable.extend(new_reachable)
 593             else:
 594                 break
 595
 596         for f in sorted(uniq_files):
 597             if any((
 598                 f in reachable,
 599                 'art/terrains/' in f,
 600                 'maps/random/' in f,
 601             )):
 602                 continue
 603             self.logger.warning(f"Unused file '{str(self.vfs_to_relative_to_mods(f))}'")
 604
 605
 606 if __name__ == '__main__':
 607     check_ref = CheckRefs()
 608     check_ref.main()