test.py

   1 #!/usr/bin/env python
   2 # Automated script to run the pspautotests test suite in PPSSPP.
   3
   4 import sys
   5 import os
   6 import subprocess
   7 import threading
   8 import glob
   9
  10
  11 PPSSPP_EXECUTABLES = [
  12   # Windows
  13   "Windows\\Debug\\PPSSPPHeadless.exe",
  14   "Windows\\Release\\PPSSPPHeadless.exe",
  15   "Windows\\x64\\Debug\\PPSSPPHeadless.exe",
  16   "Windows\\x64\\Release\\PPSSPPHeadless.exe",
  17   # Mac
  18   "build*/Debug/PPSSPPHeadless",
  19   "build*/Release/PPSSPPHeadless",
  20   "build*/RelWithDebInfo/PPSSPPHeadless",
  21   "build*/MinSizeRel/PPSSPPHeadless",
  22   # Linux
  23   "build*/PPSSPPHeadless",
  24   "./PPSSPPHeadless"
  25 ]
  26
  27 PPSSPP_EXE = None
  28 TEST_ROOT = "pspautotests/tests/"
  29 teamcity_mode = False
  30 TIMEOUT = 5
  31
  32 class Command(object):
  33   def __init__(self, cmd, data = None):
  34     self.cmd = cmd
  35     self.data = data
  36     self.process = None
  37     self.output = None
  38     self.timeout = False
  39
  40   def run(self, timeout):
  41     def target():
  42       self.process = subprocess.Popen(self.cmd, bufsize=1, stdin=subprocess.PIPE, stdout=sys.stdout, stderr=subprocess.STDOUT)
  43       self.process.stdin.write(self.data)
  44       self.process.stdin.close()
  45       self.process.communicate()
  46
  47     thread = threading.Thread(target=target)
  48     thread.start()
  49
  50     thread.join(timeout)
  51     if thread.isAlive():
  52       self.timeout = True
  53       if sys.version_info < (2, 6):
  54         os.kill(self.process.pid, signal.SIGKILL)
  55       else:
  56         self.process.terminate()
  57       thread.join()
  58
  59 # Test names are the C files without the .c extension.
  60 # These have worked and should keep working always - regression tests.
  61 tests_good = [
  62   "cpu/cpu_alu/cpu_alu",
  63   "cpu/vfpu/colors",
  64   "cpu/vfpu/convert",
  65   "cpu/vfpu/gum",
  66   "cpu/vfpu/matrix",
  67   "cpu/vfpu/prefixes",
  68   "cpu/vfpu/vector",
  69   "cpu/icache/icache",
  70   "cpu/lsu/lsu",
  71   "cpu/fpu/fpu",
  72
  73   "audio/atrac/ids",
  74   "audio/atrac/setdata",
  75   "audio/mp3/mp3test",
  76   "audio/sascore/sascore",
  77   "audio/sascore/adsrcurve",
  78   "audio/sascore/getheight",
  79   "audio/sascore/keyoff",
  80   "audio/sascore/keyon",
  81   "audio/sascore/noise",
  82   "audio/sascore/outputmode",
  83   "audio/sascore/pause",
  84   "audio/sascore/pcm",
  85   "audio/sascore/pitch",
  86   "audio/sascore/vag",
  87   "ctrl/ctrl",
  88   "ctrl/idle/idle",
  89   "ctrl/sampling/sampling",
  90   "ctrl/sampling2/sampling2",
  91   "ctrl/vblank",
  92   "display/display",
  93   "display/vblankmulti",
  94   "dmac/dmactest",
  95   "font/charimagerect",
  96   "font/find",
  97   "font/fontinfo",
  98   "font/fontinfobyindex",
  99   "font/fontlist",
 100   "font/optimum",
 101   "font/shadowimagerect",
 102   "gpu/callbacks/ge_callbacks",
 103   "gpu/commands/blocktransfer",
 104   "gpu/ge/context",
 105   "gpu/ge/edram",
 106   "gpu/ge/queue",
 107   "hash/hash",
 108   "hle/check_not_used_uids",
 109   "intr/intr",
 110   "intr/suspended",
 111   "intr/vblank/vblank",
 112   "io/cwd/cwd",
 113   "loader/bss/bss",
 114   "malloc/malloc",
 115   "misc/dcache",
 116   "misc/deadbeef",
 117   "misc/libc",
 118   "misc/sdkver",
 119   "misc/testgp",
 120   "misc/timeconv",
 121   "mstick/mstick",
 122   "rtc/rtc",
 123   "string/string",
 124   "sysmem/freesize",
 125   "sysmem/memblock",
 126   "sysmem/sysmem",
 127   "threads/alarm/alarm",
 128   "threads/alarm/cancel/cancel",
 129   "threads/alarm/refer/refer",
 130   "threads/alarm/set/set",
 131   "threads/callbacks/callbacks",
 132   "threads/callbacks/check",
 133   "threads/callbacks/create",
 134   "threads/callbacks/delete",
 135   "threads/callbacks/exit",
 136   "threads/callbacks/refer",
 137   "threads/events/events",
 138   "threads/events/cancel/cancel",
 139   "threads/events/clear/clear",
 140   "threads/events/create/create",
 141   "threads/events/delete/delete",
 142   "threads/events/poll/poll",
 143   "threads/events/refer/refer",
 144   "threads/events/set/set",
 145   "threads/events/wait/wait",
 146   "threads/fpl/fpl",
 147   "threads/fpl/allocate",
 148   "threads/fpl/cancel",
 149   "threads/fpl/create",
 150   "threads/fpl/delete",
 151   "threads/fpl/free",
 152   "threads/fpl/refer",
 153   "threads/fpl/priority",
 154   "threads/fpl/tryallocate",
 155   "threads/k0/k0",
 156   "threads/lwmutex/create",
 157   "threads/lwmutex/delete",
 158   "threads/lwmutex/lock",
 159   "threads/lwmutex/priority",
 160   "threads/lwmutex/refer",
 161   "threads/lwmutex/try",
 162   "threads/lwmutex/try600",
 163   "threads/lwmutex/unlock",
 164   "threads/mbx/mbx",
 165   "threads/mbx/cancel/cancel",
 166   "threads/mbx/create/create",
 167   "threads/mbx/delete/delete",
 168   "threads/mbx/poll/poll",
 169   "threads/mbx/priority/priority",
 170   "threads/mbx/receive/receive",
 171   "threads/mbx/refer/refer",
 172   "threads/mbx/send/send",
 173   "threads/msgpipe/msgpipe",
 174   "threads/msgpipe/cancel",
 175   "threads/msgpipe/create",
 176   "threads/msgpipe/data",
 177   "threads/msgpipe/delete",
 178   "threads/msgpipe/receive",
 179   "threads/msgpipe/refer",
 180   "threads/msgpipe/send",
 181   "threads/msgpipe/tryreceive",
 182   "threads/msgpipe/trysend",
 183   "threads/mutex/cancel",
 184   "threads/mutex/create",
 185   "threads/mutex/delete",
 186   "threads/mutex/lock",
 187   "threads/mutex/mutex",
 188   "threads/mutex/priority",
 189   "threads/mutex/refer",
 190   "threads/mutex/try",
 191   "threads/mutex/unlock",
 192   "threads/semaphores/semaphores",
 193   "threads/semaphores/cancel",
 194   "threads/semaphores/create",
 195   "threads/semaphores/delete",
 196   "threads/semaphores/fifo",
 197   "threads/semaphores/poll",
 198   "threads/semaphores/priority",
 199   "threads/semaphores/refer",
 200   "threads/semaphores/signal",
 201   "threads/semaphores/wait",
 202   "threads/threads/change",
 203   "threads/threads/exitstatus",
 204   "threads/threads/extend",
 205   "threads/threads/refer",
 206   "threads/threads/release",
 207   "threads/threads/rotate",
 208   "threads/threads/stackfree",
 209   "threads/threads/start",
 210   "threads/threads/suspend",
 211   "threads/threads/threadend",
 212   "threads/threads/threadmanidlist",
 213   "threads/threads/threadmanidtype",
 214   "threads/threads/threads",
 215   "threads/wakeup/wakeup",
 216   "threads/vpl/allocate",
 217   "threads/vpl/cancel",
 218   "threads/vpl/delete",
 219   "threads/vpl/fifo",
 220   "threads/vpl/free",
 221   "threads/vpl/order",
 222   "threads/vpl/priority",
 223   "threads/vpl/refer",
 224   "threads/vpl/try",
 225   "threads/vpl/vpl",
 226   "threads/vtimers/vtimer",
 227   "threads/vtimers/cancelhandler",
 228   "threads/vtimers/create",
 229   "threads/vtimers/delete",
 230   "threads/vtimers/getbase",
 231   "threads/vtimers/gettime",
 232   "threads/vtimers/interrupt",
 233   "threads/vtimers/refer",
 234   "threads/vtimers/sethandler",
 235   "threads/vtimers/settime",
 236   "threads/vtimers/start",
 237   "threads/vtimers/stop",
 238   "utility/savedata/autosave",
 239   "utility/savedata/filelist",
 240   "utility/savedata/makedata",
 241   "utility/systemparam/systemparam",
 242   "power/power",
 243   "power/volatile/lock",
 244   "power/volatile/trylock",
 245   "power/volatile/unlock",
 246   "umd/register",
 247   "umd/callbacks/umd",
 248   "io/directory/directory",
 249   "video/mpeg/ringbuffer/construct",
 250   "video/mpeg/ringbuffer/destruct",
 251   "video/mpeg/ringbuffer/memsize",
 252   "video/mpeg/ringbuffer/packnum",
 253   "video/psmfplayer/getvideodata",
 254 ]
 255
 256 tests_next = [
 257 # These are the next tests up for fixing. These run by default.
 258   "cpu/cpu_alu/cpu_branch",
 259   "cpu/fpu/fcr",
 260   "audio/atrac/atractest",
 261   "audio/atrac/decode",
 262   "audio/atrac/resetting",
 263   "audio/sceaudio/datalen",
 264   "audio/sceaudio/output",
 265   "audio/sceaudio/reserve",
 266   "audio/sascore/setadsr",
 267   "display/hcount",
 268   "intr/waits",
 269   "threads/callbacks/cancel",
 270   "threads/callbacks/count",
 271   "threads/callbacks/notify",
 272   "threads/scheduling/dispatch",
 273   "threads/scheduling/scheduling",
 274   "threads/threads/create",
 275   "threads/threads/terminate",
 276   "threads/vpl/create",
 277   "utility/savedata/getsize",
 278   "utility/savedata/idlist",
 279   "utility/savedata/sizes",
 280   "utility/msgdialog/abort",
 281   "utility/msgdialog/dialog",
 282   "gpu/commands/basic",
 283   "gpu/commands/blend",
 284   "gpu/commands/material",
 285   "gpu/complex/complex",
 286   "gpu/displaylist/state",
 287   "gpu/ge/break",
 288   "gpu/ge/get",
 289   "gpu/reflection/reflection",
 290   "gpu/rendertarget/rendertarget",
 291   "gpu/signals/continue",
 292   "gpu/signals/jumps",
 293   "gpu/signals/pause",
 294   "gpu/signals/simple",
 295   "gpu/signals/suspend",
 296   "gpu/signals/sync",
 297   "gpu/simple/simple",
 298   "gpu/triangle/triangle",
 299   "font/fonttest",
 300   "font/altcharcode",
 301   "font/charglyphimage",
 302   "font/charglyphimageclip",
 303   "font/charinfo",
 304   "font/newlib",
 305   "font/open",
 306   "font/openfile",
 307   "font/openmem",
 308   "font/resolution",
 309   "font/shadowglyphimage",
 310   "font/shadowglyphimageclip",
 311   "font/shadowinfo",
 312   "io/file/file",
 313   "io/file/rename",
 314   "io/io/io",
 315   "io/iodrv/iodrv",
 316   # Doesn't work on a PSP for security reasons, hangs in PPSSPP currently.
 317   # Commented out to make tests run much faster.
 318   #"modules/loadexec/loader",
 319   "net/http/http",
 320   "net/primary/ether",
 321   "power/cpu",
 322   "power/freq",
 323   "rtc/arithmetic",
 324   "rtc/convert",
 325   "rtc/lookup",
 326   "sysmem/partition",
 327   "umd/io/umd_io",
 328   "umd/raw_access/raw_access",
 329   "umd/wait/wait",
 330   "video/mpeg/basic",
 331   "video/mpeg/ringbuffer/avail",
 332   "video/pmf/pmf",
 333   "video/pmf_simple/pmf_simple",
 334   "video/psmfplayer/basic",
 335 ]
 336
 337
 338 # These are the tests we ignore (not important, or impossible to run)
 339 tests_ignored = [
 340   "kirk/kirk",
 341   "me/me",
 342 ]
 343
 344
 345
 346 def init():
 347   global PPSSPP_EXE, TEST_ROOT
 348   if not os.path.exists("pspautotests"):
 349     if os.path.exists(os.path.dirname(__file__) + "/pspautotests"):
 350       TEST_ROOT = os.path.dirname(__file__) + "/pspautotests/tests/";
 351     else:
 352       print("Please run git submodule init; git submodule update;")
 353       sys.exit(1)
 354
 355   if not os.path.exists(TEST_ROOT + "cpu/cpu_alu/cpu_alu.prx"):
 356     print("Please install the pspsdk and run make in common/ and in all the tests")
 357     print("(checked for existence of cpu/cpu_alu/cpu_alu.prx)")
 358     sys.exit(1)
 359
 360   possible_exes = [glob.glob(f) for f in PPSSPP_EXECUTABLES]
 361   possible_exes = [x for sublist in possible_exes for x in sublist]
 362   existing = filter(os.path.exists, possible_exes)
 363   if existing:
 364     PPSSPP_EXE = max((os.path.getmtime(f), f) for f in existing)[1]
 365   else:
 366     PPSSPP_EXE = None
 367
 368   if not PPSSPP_EXE:
 369     print("PPSSPPHeadless executable missing, please build one.")
 370     sys.exit(1)
 371
 372 def tcprint(arg):
 373   global teamcity_mode
 374   if teamcity_mode:
 375     print(arg)
 376
 377 def run_tests(test_list, args):
 378   global PPSSPP_EXE, TIMEOUT
 379   tests_passed = []
 380   tests_failed = []
 381
 382   test_filenames = []
 383   for test in test_list:
 384     # Try prx first
 385     elf_filename = TEST_ROOT + test + ".prx"
 386     if not os.path.exists(elf_filename):
 387       print("WARNING: no prx, trying elf")
 388       elf_filename = TEST_ROOT + test + ".elf"
 389
 390     test_filenames.append(elf_filename)
 391
 392   if len(test_filenames):
 393     # TODO: Maybe --compare should detect --graphics?
 394     cmdline = [PPSSPP_EXE, '--root', TEST_ROOT + '../', '--compare', '--timeout=' + str(TIMEOUT), '@-']
 395     cmdline.extend([i for i in args if i not in ['-g', '-m']])
 396
 397     c = Command(cmdline, '\n'.join(test_filenames))
 398     c.run(TIMEOUT * len(test_filenames))
 399
 400     print("Ran " + PPSSPP_EXE)
 401
 402
 403 def main():
 404   global teamcity_mode
 405   init()
 406   tests = []
 407   args = []
 408   for arg in sys.argv[1:]:
 409     if arg == '--teamcity':
 410       teamcity_mode = True
 411       args.append(arg)
 412     elif arg[0] == '-':
 413       args.append(arg)
 414     else:
 415       tests.append(arg)
 416
 417   if not tests:
 418     if '-g' in args:
 419       tests = tests_good
 420     else:
 421       tests = tests_next + tests_good
 422   elif '-m' in args and '-g' in args:
 423     tests = [i for i in tests_good if i.startswith(tests[0])]
 424   elif '-m' in args:
 425     tests = [i for i in tests_next + tests_good if i.startswith(tests[0])]
 426
 427   run_tests(tests, args)
 428
 429 main()