first-diff.py

   1 #!/usr/bin/env python3
   2 import sys
   3 import os.path
   4
   5 lang = None
   6
   7 # TODO: -S argument for shifted ROMs
   8 args = []
   9 for arg in sys.argv[1:]:
  10     if arg == '-j':
  11         lang = 'jp'
  12     elif arg == '-u':
  13         lang = 'us'
  14     elif arg == '-e':
  15         lang = 'eu'
  16     elif arg == '-s':
  17         lang = 'sh'
  18     else:
  19         args.append(arg)
  20
  21 if lang is None:
  22     lang = 'us'
  23     best = 0
  24     for path in ['build/us/sm64.us.z64', 'build/jp/sm64.jp.z64', 'build/eu/sm64.eu.z64', 'build/sh/sm64.sh.z64']:
  25         try:
  26             mtime = os.path.getmtime(path)
  27             if mtime > best:
  28                 best = mtime
  29                 lang = path.split('/')[1]
  30         except Exception:
  31             pass
  32     print("Assuming language " + lang)
  33
  34 baseimg = 'baserom.' + lang + '.z64'
  35 basemap = 'sm64.' + lang + '.map'
  36
  37 myimg = 'build/' + lang + '/sm64.' + lang + '.z64'
  38 mymap = 'build/' + lang + '/sm64.' + lang + '.map'
  39
  40 if os.path.isfile('expected/' + mymap):
  41     basemap = 'expected/' + mymap
  42
  43 required_files = [baseimg, myimg, mymap]
  44 if any(not os.path.isfile(path) for path in required_files):
  45     print(', '.join(required_files[:-1]) + " and " + required_files[-1] + " must exist.")
  46     exit(1)
  47
  48 mybin = open(myimg, 'rb').read()
  49 basebin = open(baseimg, 'rb').read()
  50
  51 if len(mybin) != len(basebin):
  52     print("Modified ROM has different size...")
  53     exit(1)
  54
  55 if mybin == basebin:
  56     print("No differences!")
  57     exit(0)
  58
  59 def search_map(rom_addr):
  60     ram_offset = None
  61     last_ram = 0
  62     last_rom = 0
  63     last_fn = '<start of rom>'
  64     last_file = '<no file>'
  65     prev_line = ''
  66     with open(mymap) as f:
  67         for line in f:
  68             if 'load address' in line:
  69                 # Example: ".boot           0x0000000004000000     0x1000 load address 0x0000000000000000"
  70                 if 'noload' in line or 'noload' in prev_line:
  71                     ram_offset = None
  72                     continue
  73                 ram = int(line[16:16+18], 0)
  74                 rom = int(line[59:59+18], 0)
  75                 ram_offset = ram - rom
  76                 continue
  77             prev_line = line
  78
  79             if ram_offset is None or '=' in line or '*fill*' in line or ' 0x' not in line:
  80                 continue
  81             ram = int(line[16:16+18], 0)
  82             rom = ram - ram_offset
  83             fn = line.split()[-1]
  84             if '0x' in fn:
  85                 ram_offset = None
  86                 continue
  87             if rom > rom_addr or (rom_addr & 0x80000000 and ram > rom_addr):
  88                 return 'in {} (ram 0x{:08x}, rom 0x{:x}, {})'.format(last_fn, last_ram, last_rom, last_file)
  89             last_ram = ram
  90             last_rom = rom
  91             last_fn = fn
  92             if '/' in fn:
  93                 last_file = fn
  94     return 'at end of rom?'
  95
  96 def parse_map(fname):
  97     ram_offset = None
  98     cur_file = '<no file>'
  99     syms = {}
 100     prev_sym = None
 101     prev_line = ''
 102     with open(fname) as f:
 103         for line in f:
 104             if 'load address' in line:
 105                 if 'noload' in line or 'noload' in prev_line:
 106                     ram_offset = None
 107                     continue
 108                 ram = int(line[16:16+18], 0)
 109                 rom = int(line[59:59+18], 0)
 110                 ram_offset = ram - rom
 111                 continue
 112             prev_line = line
 113
 114             if ram_offset is None or '=' in line or '*fill*' in line or ' 0x' not in line:
 115                 continue
 116             ram = int(line[16:16+18], 0)
 117             rom = ram - ram_offset
 118             fn = line.split()[-1]
 119             if '0x' in fn:
 120                 ram_offset = None
 121             elif '/' in fn:
 122                 cur_file = fn
 123             else:
 124                 syms[fn] = (rom, cur_file, prev_sym, ram)
 125                 prev_sym = fn
 126     return syms
 127
 128 def map_diff():
 129     map1 = parse_map(mymap)
 130     map2 = parse_map(basemap)
 131     min_ram = None
 132     found = None
 133     for sym, addr in map1.items():
 134         if sym not in map2:
 135             continue
 136         if addr[0] != map2[sym][0]:
 137             if min_ram is None or addr[0] < min_ram:
 138                 min_ram = addr[0]
 139                 found = (sym, addr[1], addr[2])
 140     if min_ram is None:
 141         return False
 142     else:
 143         print()
 144         print("Map appears to have shifted just before {} ({}) -- in {}?".format(found[0], found[1], found[2]))
 145         if found[2] is not None and found[2] not in map2:
 146             print()
 147             print("(Base map file {} out of date due to renamed symbols, so result may be imprecise.)".format(basemap))
 148         return True
 149
 150 def hexbytes(bs):
 151     return ":".join("{:02x}".format(c) for c in bs)
 152
 153 # For convenience, allow `./first-diff.py <ROM addr | RAM addr | function name>`
 154 # to do a symbol <-> address lookup. This should really be split out into a
 155 # separate script...
 156 if args:
 157     try:
 158         addr = int(args[0], 0)
 159         print(args[0], "is", search_map(addr))
 160     except ValueError:
 161         m = parse_map(mymap)
 162         try:
 163             print(args[0], "is at position", hex(m[args[0]][0]), "in ROM,", hex(m[args[0]][3]), "in RAM")
 164         except KeyError:
 165             print("function", args[0], "not found")
 166     exit()
 167
 168 found_instr_diff = None
 169 diffs = 0
 170 shift_cap = 1000
 171 for i in range(24, len(mybin), 4):
 172     # (mybin[i:i+4] != basebin[i:i+4], but that's slightly slower in CPython...)
 173     if diffs <= shift_cap and (mybin[i] != basebin[i] or mybin[i+1] != basebin[i+1] or mybin[i+2] != basebin[i+2] or mybin[i+3] != basebin[i+3]):
 174         if diffs == 0:
 175             print("First difference at ROM addr " + hex(i) + ", " + search_map(i))
 176             print("Bytes:", hexbytes(mybin[i:i+4]), 'vs', hexbytes(basebin[i:i+4]))
 177         diffs += 1
 178     if found_instr_diff is None and mybin[i] >> 2 != basebin[i] >> 2:
 179         found_instr_diff = i
 180 if diffs == 0:
 181     print("No differences!")
 182     exit()
 183 definite_shift = (diffs > shift_cap)
 184 if not definite_shift:
 185     print(str(diffs) + " differing word(s).")
 186
 187 if diffs > 100:
 188     if found_instr_diff is not None:
 189         i = found_instr_diff
 190         print("First instruction difference at ROM addr " + hex(i) + ", " + search_map(i))
 191         print("Bytes:", hexbytes(mybin[i:i+4]), 'vs', hexbytes(basebin[i:i+4]))
 192     if lang == 'sh':
 193         print("Shifted ROM, as expected.")
 194     else:
 195         if not os.path.isfile(basemap):
 196             if definite_shift:
 197                 print("Tons of differences, must be a shifted ROM.")
 198             print("To find ROM shifts, copy a clean .map file to " + basemap + " and rerun this script.")
 199             exit()
 200
 201         if not map_diff():
 202             print("No ROM shift{}.".format(" (!?)" if definite_shift else ""))