hphp/test/run

   1 #!/usr/bin/env php
   2 <?php
   3 /**
   4 * Run the test suites in various configurations.
   5 */
   6
   7 function usage() {
   8   global $argv;
   9   return "usage: $argv[0] [-m jit|interp] [-r] <test/directories>";
  10 }
  11
  12 function help() {
  13   global $argv;
  14   $ztestexample = 'test/zend/good/*/*z*.php'; // sep. for syntax highlighting
  15   $help = <<<EOT
  16
  17
  18 This is the hhvm test-suite runner.  For more detailed documentation,
  19 see hphp/test/README.md.
  20
  21 The test argument may be a path to a php test file, a directory name, or
  22 one of a few pre-defined suite names that this script knows about.
  23
  24 If you work with hhvm a lot, you might consider a bash alias:
  25
  26    alias ht="path/to/fbcode/hphp/test/run"
  27
  28 Examples:
  29
  30   # Quick tests in JIT mode:
  31   % $argv[0] test/quick
  32
  33   # Slow tests in interp mode:
  34   % $argv[0] -m interp test/slow
  35
  36   # Slow closure tests in JIT mode:
  37   % $argv[0] test/slow/closure
  38
  39   # Slow closure tests in JIT mode with RepoAuthoritative:
  40   % $argv[0] -r test/slow/closure
  41
  42   # Slow array tests, in RepoAuthoritative:
  43   % $argv[0] -r test/slow/array
  44
  45   # Zend tests with a "z" in their name:
  46   % $argv[0] $ztestexample
  47
  48   # Quick tests in JIT mode with some extra runtime options:
  49   % $argv[0] test/quick -a '-vEval.JitMaxTranslations=120 -vEval.HHIRJumpOpts=0'
  50
  51 EOT;
  52   return usage().$help;
  53 }
  54
  55 function error($message) {
  56   print "$message\n";
  57   exit(1);
  58 }
  59
  60 function hphp_home() {
  61   return realpath(__DIR__.'/../..');
  62 }
  63
  64 function idx($array, $key, $default = null) {
  65   return isset($array[$key]) ? $array[$key] : $default;
  66 }
  67
  68 function idx_file($array, $key, $default = null) {
  69   $file = is_file(idx($array, $key)) ? realpath($array[$key]) : $default;
  70   if (!is_file($file)) {
  71     error("$file doesn't exist");
  72   }
  73   return rel_path($file);
  74 }
  75
  76 function bin_root() {
  77   $dir = hphp_home() . '/' . idx($_ENV, 'FBMAKE_BIN_ROOT', '_bin');
  78   return is_dir($dir) ?
  79     $dir :      # fbmake
  80     hphp_home() # github
  81   ;
  82 }
  83
  84 function verify_hhbc() {
  85   return idx($_ENV, 'VERIFY_HHBC', bin_root().'/verify.hhbc');
  86 }
  87
  88 function read_file($file) {
  89   return file_exists($file) ?
  90          preg_replace('/\s+/', ' ', (file_get_contents($file))) : "";
  91 }
  92
  93 // http://stackoverflow.com/questions/2637945/
  94 function rel_path($to) {
  95     $from     = explode('/', getcwd().'/');
  96     $to       = explode('/', $to);
  97     $relPath  = $to;
  98
  99     foreach($from as $depth => $dir) {
 100         // find first non-matching dir
 101         if($dir === $to[$depth]) {
 102             // ignore this directory
 103             array_shift($relPath);
 104         } else {
 105             // get number of remaining dirs to $from
 106             $remaining = count($from) - $depth;
 107             if($remaining > 1) {
 108                 // add traversals up to first matching dir
 109                 $padLength = (count($relPath) + $remaining - 1) * -1;
 110                 $relPath = array_pad($relPath, $padLength, '..');
 111                 break;
 112             } else {
 113                 $relPath[0] = './' . $relPath[0];
 114             }
 115         }
 116     }
 117     return implode('/', $relPath);
 118 }
 119
 120 function get_options($argv) {
 121   $parameters = array(
 122     'repo' => 'r',
 123     'mode:' => 'm:',
 124     'server' => '',
 125     'help' => 'h',
 126     'verbose' => 'v',
 127     'fbmake' => '',
 128     'threads:' => '',
 129     'args:' => 'a:',
 130   );
 131   $options = array();
 132   $files = array();
 133   for ($i = 1; $i < count($argv); $i++) {
 134     $arg = $argv[$i];
 135     $found = false;
 136     if ($arg && $arg[0] == '-') {
 137       foreach ($parameters as $long => $short) {
 138         if ($arg == '-'.str_replace(':', '', $short) ||
 139             $arg == '--'.str_replace(':', '', $long)) {
 140           if (substr($long, -1, 1) == ':') {
 141             $value = $argv[++$i];
 142           } else {
 143             $value = true;
 144           }
 145           $options[str_replace(':', '', $long)] = $value;
 146           $found = true;
 147           break;
 148         }
 149       }
 150     }
 151     if (!$found && $arg) {
 152       $files[] = $arg;
 153     }
 154   }
 155   return array($options, $files);
 156 }
 157
 158 /*
 159  * We support some 'special' file names, that just know where the test
 160  * suites are, to avoid typing 'hphp/test/foo'.
 161  */
 162 function map_convenience_filename($file) {
 163   $mappage = array(
 164     'quick'    => 'hphp/test/quick',
 165     'slow'     => 'hphp/test/slow',
 166     'zend'     => 'hphp/test/zend/good',
 167     'zend_bad' => 'hphp/test/zend/bad',
 168     'facebook' => 'hphp/facebook/test',
 169   );
 170
 171   $m = null;
 172   if (!preg_match('/([^\/]*)/', $file, $m) ||
 173       !isset($mappage[$m[1]])) {
 174     return $file;
 175   }
 176   return hphp_home().'/'.$mappage[$m[1]];
 177 }
 178
 179 function find_tests($files) {
 180   if (!$files) {
 181     $files = array('quick');
 182   }
 183   if ($files == array('all')) {
 184     $files = array('quick', 'slow', 'zend');
 185   }
 186   foreach ($files as &$file) {
 187     $file = map_convenience_filename($file);
 188     if (!@stat($file)) {
 189       error("Not valid file or directory: '$file'");
 190     }
 191     $file = preg_replace(',//+,', '/', realpath($file));
 192     $file = preg_replace(',^'.getcwd().'/,', '', $file);
 193   }
 194   $files = implode(' ', $files);
 195   $tests = explode("\n", shell_exec(
 196       "find $files -name '*.php' -o -name '*.hhas'"
 197   ));
 198   if (!$tests) {
 199     error(usage());
 200   }
 201   asort($tests);
 202   return array_filter($tests);
 203 }
 204
 205 function find_config($test, $name) {
 206   return find_config_for_dir(dirname($test), $name);
 207 }
 208
 209 function find_config_for_dir($dir, $name) {
 210   while ($dir && stat($dir)) {
 211     $config = "$dir/$name";
 212     if (is_file($config)) {
 213       return $config;
 214     }
 215     $dir = substr($dir, 0, strrpos($dir, '/'));
 216   }
 217   return __DIR__.'/'.$name;
 218 }
 219
 220 function find_debug_config($test, $name) {
 221   $debug_config = find_config_for_dir(dirname($test), $name);
 222   if (is_file($debug_config)) {
 223     return "-m debug --debug-config ".$debug_config;
 224   }
 225   return "";
 226 }
 227
 228 function mode_cmd($options) {
 229   $repo_args = "-vRepo.Local.Mode=-- -vRepo.Central.Path=".verify_hhbc();
 230   $jit_args = "$repo_args -vEval.Jit=true ".
 231               "-vEval.JitEnableRenameFunction=true";
 232   $mode = idx($options, 'mode');
 233   switch ($mode) {
 234     case '':
 235     case 'jit':
 236       return "$jit_args";
 237     case 'interp':
 238       return "$repo_args -vEval.Jit=0";
 239     default:
 240       error("-m must be one of jit | interp. Got: '$mode'");
 241   }
 242 }
 243
 244 function extra_args($options) {
 245   return idx($options, 'args', '');
 246 }
 247
 248 function hhvm_cmd($options, $test) {
 249   $cmd = implode(" ", array(
 250     idx_file($_ENV, 'HHVM_BIN', bin_root().'/hphp/hhvm/hhvm'),
 251     '--config',
 252     find_config($test, 'config.hdf'),
 253     find_debug_config($test, 'hphpd.hdf'),
 254     mode_cmd($options),
 255     '-vEval.EnableArgsInBacktraces=true',
 256     read_file("$test.opts"),
 257     extra_args($options),
 258     '--file',
 259     $test
 260   ));
 261   if (file_exists("$test.in")) {
 262     $cmd .= " <$test.in";
 263   }
 264   return $cmd;
 265 }
 266
 267 function hphp_cmd($options, $test) {
 268   return implode(" ", array(
 269     idx_file($_ENV, 'HPHP_BIN', bin_root().'/hphp/hhvm/hphp'),
 270     '--config',
 271     find_config($test, 'hphp_config.hdf'),
 272     read_file("$test.hphp_opts"),
 273     "-thhbc -l0 -k1 -o $test.repo $test",
 274   ));
 275 }
 276
 277 class Status {
 278   private static $results = array();
 279   private static $mode = 0;
 280
 281   const MODE_NORMAL = 0;
 282   const MODE_VERBOSE = 1;
 283   const MODE_FBMAKE = 2;
 284
 285   public static function setMode($mode) {
 286     self::$mode = $mode;
 287   }
 288
 289   public static function pass($test) {
 290     array_push(self::$results, array('name' => $test, 'status' => 'passed'));
 291     switch (self::$mode) {
 292       case self::MODE_NORMAL:
 293         if (self::hasColor()) {
 294           print "\033[1;32m.\033[0m";
 295         } else {
 296           print '.';
 297         }
 298         break;
 299       case self::MODE_VERBOSE:
 300         if (self::hasColor()) {
 301           print "$test \033[1;32mpassed\033[0m\n";
 302         } else {
 303           print "$test passed";
 304         }
 305         break;
 306       case self::MODE_FBMAKE:
 307         self::sayFBMake($test, 'passed');
 308         break;
 309     }
 310   }
 311
 312   public static function fail($test) {
 313     array_push(self::$results, array(
 314         'name' => $test,
 315         'status' => 'failed',
 316         'details' => @file_get_contents("$test.diff")
 317     ));
 318     switch (self::$mode) {
 319       case self::MODE_NORMAL:
 320         $diff = @file_get_contents($test.'.diff');
 321         if (self::hasColor()) {
 322           print "\n\033[0;31m$test\033[0m\n$diff";
 323         } else {
 324           print "\nFAILED: $test\n$diff";
 325         }
 326         break;
 327       case self::MODE_VERBOSE:
 328         if (self::hasColor()) {
 329           print "$test \033[0;31mFAILED\033[0m\n";
 330         } else {
 331           print "$test FAILED\n";
 332         }
 333         break;
 334       case self::MODE_FBMAKE:
 335         self::sayFBMake($test, 'failed');
 336         break;
 337     }
 338   }
 339
 340   private static function sayFBMake($test, $status) {
 341     $start = array('op' => 'start', 'test' => $test);
 342     $end = array('op' => 'test_done', 'test' => $test, 'status' => $status);
 343     if ($status == 'failed') {
 344       $end['details'] = @file_get_contents("$test.diff");
 345     }
 346     self::say($start, $end);
 347   }
 348
 349   public static function getResults() {
 350     return self::$results;
 351   }
 352
 353   /** Output is in the format expected by JsonTestRunner. */
 354   public static function say(/* ... */) {
 355     $data = array_map(function($row) {
 356       return json_encode($row, JSON_UNESCAPED_SLASHES) . "\n";
 357     }, func_get_args());
 358     fwrite(STDERR, implode("", $data));
 359   }
 360
 361   private static function hasColor() {
 362     return posix_isatty(STDOUT);
 363   }
 364 }
 365
 366 function run($options, $tests, $bad_test_file) {
 367   if (isset($options['verbose'])) {
 368     Status::setMode(Status::MODE_VERBOSE);
 369   }
 370   if (isset($options['fbmake'])) {
 371     Status::setMode(Status::MODE_FBMAKE);
 372   }
 373   foreach ($tests as $test) {
 374     $status = run_test($options, $test);
 375     if ($status) {
 376       Status::pass($test);
 377     } else {
 378       Status::fail($test);
 379     }
 380   }
 381   file_put_contents($bad_test_file, json_encode(Status::getResults()));
 382   foreach (Status::getResults() as $result) {
 383     if ($result['status'] == 'failed') {
 384       return 1;
 385     }
 386   }
 387   return 0;
 388 }
 389
 390 function run_test($options, $test) {
 391   $hhvm = hhvm_cmd($options, $test);
 392   $output = "";
 393   if (isset($options['repo'])) {
 394     if (strpos($test, '.hhas') !== false ||
 395         strpos($hhvm, '-m debug') != false || is_file($test.'.norepo')) {
 396       # We don't have a way to skip, I guess run non-repo?
 397     } else {
 398       unlink("$test.repo/hhvm.hhbc");
 399       $hphp = hphp_cmd($options, $test);
 400       $output .= shell_exec("$hphp 2>&1");
 401       $hhvm .= " -vRepo.Authoritative=true ".
 402                "-vRepo.Central.Path=$test.repo/hhvm.hhbc";
 403     }
 404   }
 405
 406   $descriptorspec = array(
 407      0 => array("pipe", "r"),
 408      1 => array("pipe", "w"),
 409      2 => array("pipe", "w"),
 410   );
 411   $process = proc_open("$hhvm 2>&1", $descriptorspec, $pipes);
 412   if (!is_resource($process)) {
 413     file_put_contents("$test.diff", "Couldn't invoke $hhvm");
 414     return false;
 415   }
 416
 417   fclose($pipes[0]);
 418   $output .= stream_get_contents($pipes[1]);
 419   file_put_contents("$test.out", $output);
 420   fclose($pipes[1]);
 421
 422   // hhvm redirects errors to stdout, so anything on stderr is really bad
 423   $stderr = stream_get_contents($pipes[2]);
 424   if ($stderr) {
 425     file_put_contents(
 426       "$test.diff",
 427       "Test failed because the process wrote on stderr:\n$stderr"
 428     );
 429     return false;
 430   }
 431   fclose($pipes[2]);
 432   proc_close($process);
 433
 434   // Needed for testing non-hhvm binaries that don't actually run the code
 435   // e.g. util/parser/test/parse_tester.cpp
 436   if ($output == "FORCE PASS") {
 437     return true;
 438   }
 439
 440   if (file_exists("$test.expect")) {
 441     $diff_cmds = "--text -u";
 442     exec("diff --text -u $test.expect $test.out > $test.diff 2>&1",
 443          $_, $status);
 444     // unix 0 == success
 445     return !$status;
 446
 447   } else if (file_exists("$test.expectf")) {
 448     $wanted_re = file_get_contents("$test.expectf");
 449
 450     // do preg_quote, but miss out any %r delimited sections
 451     $temp = "";
 452     $r = "%r";
 453     $startOffset = 0;
 454     $length = strlen($wanted_re);
 455     while($startOffset < $length) {
 456       $start = strpos($wanted_re, $r, $startOffset);
 457       if ($start !== false) {
 458         // we have found a start tag
 459         $end = strpos($wanted_re, $r, $start+2);
 460         if ($end === false) {
 461           // unbalanced tag, ignore it.
 462           $end = $start = $length;
 463         }
 464       } else {
 465         // no more %r sections
 466         $start = $end = $length;
 467       }
 468       // quote a non re portion of the string
 469       $temp = $temp.preg_quote(substr($wanted_re, $startOffset,
 470                                       ($start - $startOffset)),  '/');
 471       // add the re unquoted.
 472       if ($end > $start) {
 473         $temp = $temp.'('.substr($wanted_re, $start+2, ($end - $start-2)).')';
 474       }
 475       $startOffset = $end + 2;
 476     }
 477     $wanted_re = $temp;
 478
 479     $wanted_re = str_replace(
 480       array('%binary_string_optional%'),
 481       'string',
 482       $wanted_re
 483     );
 484     $wanted_re = str_replace(
 485       array('%unicode_string_optional%'),
 486       'string',
 487       $wanted_re
 488     );
 489     $wanted_re = str_replace(
 490       array('%unicode\|string%', '%string\|unicode%'),
 491       'string',
 492       $wanted_re
 493     );
 494     $wanted_re = str_replace(
 495       array('%u\|b%', '%b\|u%'),
 496       '',
 497       $wanted_re
 498     );
 499     // Stick to basics
 500     $wanted_re = str_replace('%e', '\\' . DIRECTORY_SEPARATOR, $wanted_re);
 501     $wanted_re = str_replace('%s', '[^\r\n]+', $wanted_re);
 502     $wanted_re = str_replace('%S', '[^\r\n]*', $wanted_re);
 503     $wanted_re = str_replace('%a', '.+', $wanted_re);
 504     $wanted_re = str_replace('%A', '.*', $wanted_re);
 505     $wanted_re = str_replace('%w', '\s*', $wanted_re);
 506     $wanted_re = str_replace('%i', '[+-]?\d+', $wanted_re);
 507     $wanted_re = str_replace('%d', '\d+', $wanted_re);
 508     $wanted_re = str_replace('%x', '[0-9a-fA-F]+', $wanted_re);
 509     $wanted_re = str_replace('%f', '[+-]?\.?\d+\.?\d*(?:[Ee][+-]?\d+)?',
 510                              $wanted_re);
 511     $wanted_re = str_replace('%c', '.', $wanted_re);
 512     // %f allows two points "-.0.0" but that is the best *simple* expression
 513
 514     # a poor man's aide for debugging
 515     shell_exec("diff --text -u $test.expectf $test.out > $test.diff 2>&1");
 516
 517     return preg_match("/^$wanted_re\$/s", $output);
 518
 519   } else if (file_exists("$test.expectregex")) {
 520     $wanted_re = file_get_contents("$test.expectregex");
 521
 522     # a poor man's aide for debugging
 523     shell_exec("diff --text -u $test.expectregex $test.out > $test.diff 2>&1");
 524
 525     return preg_match("/^$wanted_re\$/s", $output);
 526   }
 527 }
 528
 529 function num_cpus() {
 530   $data = file('/proc/stat');
 531   $cores = 0;
 532   foreach($data as $line) {
 533     if (preg_match('/^cpu[0-9]/', $line)) {
 534       $cores++;
 535     }
 536   }
 537   return $cores;
 538 }
 539
 540 function main($argv) {
 541
 542   ini_set('pcre.backtrack_limit', PHP_INT_MAX);
 543
 544   list($options, $files) = get_options($argv);
 545   if (isset($options['help'])) {
 546     error(help());
 547   }
 548   $tests = find_tests($files);
 549
 550   $threads = min(count($tests), idx($options, 'threads', num_cpus() + 1));
 551
 552   if (!isset($options['fbmake'])) {
 553     print "Running ".count($tests)." tests in $threads threads\n";
 554   }
 555
 556   # Try to construct the buckets so the test results are ready in approximately
 557   # alphabetical order
 558   $test_buckets = array();
 559   $i = 0;
 560   foreach ($tests as $test) {
 561     $test_buckets[$i][] = $test;
 562     $i = ($i + 1) % $threads;
 563   }
 564
 565   # Spawn off worker threads
 566   $children = array();
 567   # A poor man's shared memory
 568   $bad_test_files = array();
 569   for ($i = 0; $i < $threads; $i++) {
 570     $bad_test_file = tempnam('/tmp', 'test-run-');
 571     $bad_test_files[] = $bad_test_file;
 572     $pid = pcntl_fork();
 573     if ($pid == -1) {
 574       error('could not fork');
 575     } else if ($pid) {
 576       $children[] = $pid;
 577     } else {
 578       exit(run($options, $test_buckets[$i], $bad_test_file));
 579     }
 580   }
 581
 582   # Wait for the kids
 583   $return_value = 0;
 584   foreach ($children as $child) {
 585     pcntl_waitpid($child, $status);
 586     $return_value |= pcntl_wexitstatus($status);
 587   }
 588
 589   $results = array();
 590   foreach ($bad_test_files as $bad_test_file) {
 591     $json = json_decode(file_get_contents($bad_test_file), true);
 592     if (!is_array($json)) {
 593       error(
 594         "A test thread didn't send json to the controller. WTF. ".
 595         "If your code isn't horribly broken, please complain loudly"
 596       );
 597     }
 598     $results = array_merge($results, $json);
 599   }
 600   if (isset($options['fbmake'])) {
 601     Status::say(array('op' => 'all_done', 'results' => $results));
 602   } else {
 603     if (!$return_value) {
 604       print "\nAll tests passed.\n\n".<<<SHIP
 605               |    |    |
 606              )_)  )_)  )_)
 607             )___))___))___)\
 608            )____)____)_____)\\
 609          _____|____|____|____\\\__
 610 ---------\      SHIP IT      /---------
 611   ^^^^^ ^^^^^^^^^^^^^^^^^^^^^
 612     ^^^^      ^^^^     ^^^    ^^
 613          ^^^^      ^^^
 614 SHIP
 615 ."\n";
 616     } else {
 617       $failed = array();
 618       foreach ($results as $result) {
 619         if ($result['status'] == 'failed') {
 620           $failed[] = $result['name'];
 621         }
 622       }
 623       asort($failed);
 624       $header_start = "\n\033[0;33m";
 625       $header_end = "\033[0m\n";
 626       print "\n".count($failed)." tests failed\n";
 627
 628       print $header_start."See the diffs:".$header_end.
 629         implode("\n", array_map(
 630           function($test) { return 'cat '.$test.'.diff'; },
 631         $failed))."\n";
 632
 633       print $header_start."Run these by hand:".$header_end;
 634
 635       foreach ($failed as $test) {
 636         $command = hhvm_cmd($options, $test);
 637         if (isset($options['repo'])) {
 638           $command .= " -vRepo.Authoritative=true ";
 639           $command = str_replace(verify_hhbc(), "$test.repo/hhvm.hhbc",
 640                                  $command);
 641           $command = hphp_cmd($options, $test)."\n".$command."\n";
 642         }
 643         print "$command\n";
 644       }
 645
 646       print $header_start."Re-run just the failing tests:".$header_end.
 647             "$argv[0] ".implode(' ', $failed)."\n";
 648     }
 649   }
 650
 651   return $return_value;
 652 }
 653
 654 exit(main($argv));