hphp/test/run

   1 #!/usr/bin/env php
   2 <?php
   3 /**
   4 * Run the test suites in various configurations.
   5 */
   6
   7 function usage() {
   8   global $argv;
   9   return "usage: $argv[0] [-m jit|interp] [-r] <test/directories>";
  10 }
  11
  12 function help() {
  13   global $argv;
  14   $ztestexample = 'test/zend/good/*/*z*.php'; // sep. for syntax highlighting
  15   $help = <<<EOT
  16
  17
  18 This is the hhvm test-suite runner.  For more detailed documentation,
  19 see hphp/test/README.md.
  20
  21 The test argument may be a path to a php test file, a directory name, or
  22 one of a few pre-defined suite names that this script knows about.
  23
  24 If you work with hhvm a lot, you might consider a bash alias:
  25
  26    alias ht="path/to/fbcode/hphp/test/run"
  27
  28 Examples:
  29
  30   # Quick tests in JIT mode:
  31   % $argv[0] test/quick
  32
  33   # Slow tests in interp mode:
  34   % $argv[0] -m interp test/slow
  35
  36   # Slow closure tests in JIT mode:
  37   % $argv[0] test/slow/closure
  38
  39   # Slow closure tests in JIT mode with RepoAuthoritative:
  40   % $argv[0] -r test/slow/closure
  41
  42   # Slow array tests, in RepoAuthoritative:
  43   % $argv[0] -r test/slow/array
  44
  45   # Zend tests with a "z" in their name:
  46   % $argv[0] $ztestexample
  47 EOT;
  48   return usage().$help;
  49 }
  50
  51 function error($message) {
  52   print "$message\n";
  53   exit(1);
  54 }
  55
  56 function hphp_home() {
  57   return realpath(__DIR__.'/../..');
  58 }
  59
  60 function idx($array, $key, $default = null) {
  61   return isset($array[$key]) ? $array[$key] : $default;
  62 }
  63
  64 function idx_file($array, $key, $default = null) {
  65   $file = is_file(idx($array, $key)) ? realpath($array[$key]) : $default;
  66   if (!is_file($file)) {
  67     error("$file doesn't exist");
  68   }
  69   return rel_path($file);
  70 }
  71
  72 function bin_root() {
  73   $dir = hphp_home() . '/' . idx($_ENV, 'FBMAKE_BIN_ROOT', '_bin');
  74   return is_dir($dir) ?
  75     $dir :      # fbmake
  76     hphp_home() # github
  77   ;
  78 }
  79
  80 function verify_hhbc() {
  81   return idx($_ENV, 'VERIFY_HHBC', bin_root().'/verify.hhbc');
  82 }
  83
  84 function read_file($file) {
  85   return file_exists($file) ? preg_replace('/\s+/', ' ', (file_get_contents($file))) : "";
  86 }
  87
  88 // http://stackoverflow.com/questions/2637945/
  89 function rel_path($to) {
  90     $from     = explode('/', getcwd().'/');
  91     $to       = explode('/', $to);
  92     $relPath  = $to;
  93
  94     foreach($from as $depth => $dir) {
  95         // find first non-matching dir
  96         if($dir === $to[$depth]) {
  97             // ignore this directory
  98             array_shift($relPath);
  99         } else {
 100             // get number of remaining dirs to $from
 101             $remaining = count($from) - $depth;
 102             if($remaining > 1) {
 103                 // add traversals up to first matching dir
 104                 $padLength = (count($relPath) + $remaining - 1) * -1;
 105                 $relPath = array_pad($relPath, $padLength, '..');
 106                 break;
 107             } else {
 108                 $relPath[0] = './' . $relPath[0];
 109             }
 110         }
 111     }
 112     return implode('/', $relPath);
 113 }
 114
 115 function get_options($argv) {
 116   $parameters = array(
 117     'repo' => 'r',
 118     'mode:' => 'm:',
 119     'server' => '',
 120     'help' => 'h',
 121     'verbose' => 'v',
 122     'fbmake' => '',
 123     'threads:' => '',
 124   );
 125   $options = array();
 126   $files = array();
 127   for ($i = 1; $i < count($argv); $i++) {
 128     $arg = $argv[$i];
 129     $found = false;
 130     if ($arg && $arg[0] == '-') {
 131       foreach ($parameters as $long => $short) {
 132         if ($arg == '-'.str_replace(':', '', $short) ||
 133             $arg == '--'.str_replace(':', '', $long)) {
 134           if (substr($long, -1, 1) == ':') {
 135             $value = $argv[++$i];
 136           } else {
 137             $value = true;
 138           }
 139           $options[str_replace(':', '', $long)] = $value;
 140           $found = true;
 141           break;
 142         }
 143       }
 144     }
 145     if (!$found && $arg) {
 146       $files[] = $arg;
 147     }
 148   }
 149   return array($options, $files);
 150 }
 151
 152 /*
 153  * We support some 'special' file names, that just know where the test
 154  * suites are, to avoid typing 'hphp/test/foo'.
 155  */
 156 function map_convenience_filename($file) {
 157   $mappage = array(
 158     'quick'    => 'hphp/test/quick',
 159     'slow'     => 'hphp/test/slow',
 160     'zend'     => 'hphp/test/zend/good',
 161     'zend_bad' => 'hphp/test/zend/bad',
 162     'facebook' => 'hphp/facebook/test',
 163   );
 164
 165   $m = null;
 166   if (!preg_match('/([^\/]*)/', $file, $m) ||
 167       !isset($mappage[$m[1]])) {
 168     return $file;
 169   }
 170   return hphp_home().'/'.$mappage[$m[1]];
 171 }
 172
 173 function find_tests($files) {
 174   if (!$files) {
 175     $files = array('quick');
 176   }
 177   if ($files == array('all')) {
 178     $files = array('quick', 'slow', 'zend');
 179   }
 180   foreach ($files as &$file) {
 181     $file = map_convenience_filename($file);
 182     if (!@stat($file)) {
 183       error("Not valid file or directory: '$file'");
 184     }
 185     $file = preg_replace(',//+,', '/', realpath($file));
 186     $file = preg_replace(',^'.getcwd().'/,', '', $file);
 187   }
 188   $files = implode(' ', $files);
 189   $tests = explode("\n", shell_exec("find $files -name '*.php' -o -name '*.hhas'"));
 190   if (!$tests) {
 191     error(usage());
 192   }
 193   asort($tests);
 194   return array_filter($tests);
 195 }
 196
 197 function find_config($test, $name) {
 198   return find_config_for_dir(dirname($test), $name);
 199 }
 200
 201 function find_config_for_dir($dir, $name) {
 202   while ($dir && stat($dir)) {
 203     $config = "$dir/$name";
 204     if (is_file($config)) {
 205       return $config;
 206     }
 207     $dir = substr($dir, 0, strrpos($dir, '/'));
 208   }
 209   return __DIR__.'/'.$name;
 210 }
 211
 212 function find_debug_config($test, $name) {
 213   $debug_config = find_config_for_dir(dirname($test), $name);
 214   if (is_file($debug_config)) {
 215     return "-m debug --debug-config ".$debug_config;
 216   }
 217   return "";
 218 }
 219
 220 function mode_cmd($options) {
 221   $repo_args = "-v Repo.Local.Mode=-- -v Repo.Central.Path=".verify_hhbc();
 222   $jit_args = "$repo_args -v Eval.Jit=true -v Eval.JitEnableRenameFunction=true";
 223   $mode = idx($options, 'mode');
 224   switch ($mode) {
 225     case '':
 226     case 'jit':
 227       return "$jit_args";
 228     case 'interp':
 229       return "$repo_args -vEval.Jit=0";
 230     default:
 231       error("-m must be one of jit | interp. Got: '$mode'");
 232   }
 233 }
 234
 235 function hhvm_cmd($options, $test) {
 236   $cmd = implode(" ", array(
 237     idx_file($_ENV, 'HHVM_BIN', bin_root().'/hphp/hhvm/hhvm'),
 238     '--config',
 239     find_config($test, 'config.hdf'),
 240     find_debug_config($test, 'hphpd.hdf'),
 241     mode_cmd($options),
 242     '-v Eval.EnableArgsInBacktraces=true',
 243     read_file("$test.opts"),
 244     '--file',
 245     $test
 246   ));
 247   if (file_exists("$test.in")) {
 248     $cmd .= " <$test.in";
 249   }
 250   return $cmd;
 251 }
 252
 253 function hphp_cmd($options, $test) {
 254   return implode(" ", array(
 255     idx_file($_ENV, 'HPHP_BIN', bin_root().'/hphp/hhvm/hphp'),
 256     '--config',
 257     find_config($test, 'hphp_config.hdf'),
 258     read_file("$test.hphp_opts"),
 259     "-thhbc -l0 -k1 -o $test.repo $test",
 260   ));
 261 }
 262
 263 class Status {
 264   private static $results = array();
 265   private static $mode = 0;
 266
 267   const MODE_NORMAL = 0;
 268   const MODE_VERBOSE = 1;
 269   const MODE_FBMAKE = 2;
 270
 271   public static function setMode($mode) {
 272     self::$mode = $mode;
 273   }
 274
 275   public static function pass($test) {
 276     array_push(self::$results, array('name' => $test, 'status' => 'passed'));
 277     switch (self::$mode) {
 278       case self::MODE_NORMAL:
 279         if (self::hasColor()) {
 280           print "\033[1;32m.\033[0m";
 281         } else {
 282           print '.';
 283         }
 284         break;
 285       case self::MODE_VERBOSE:
 286         if (self::hasColor()) {
 287           print "$test \033[1;32mpassed\033[0m\n";
 288         } else {
 289           print "$test passed";
 290         }
 291         break;
 292       case self::MODE_FBMAKE:
 293         self::sayFBMake($test, 'passed');
 294         break;
 295     }
 296   }
 297
 298   public static function fail($test) {
 299     array_push(self::$results, array(
 300         'name' => $test,
 301         'status' => 'failed',
 302         'details' => @file_get_contents("$test.diff")
 303     ));
 304     switch (self::$mode) {
 305       case self::MODE_NORMAL:
 306         $diff = @file_get_contents($test.'.diff');
 307         if (self::hasColor()) {
 308           print "\n\033[0;31m$test\033[0m\n$diff";
 309         } else {
 310           print "\nFAILED: $test\n$diff";
 311         }
 312         break;
 313       case self::MODE_VERBOSE:
 314         if (self::hasColor()) {
 315           print "$test \033[0;31mFAILED\033[0m\n";
 316         } else {
 317           print "$test FAILED\n";
 318         }
 319         break;
 320       case self::MODE_FBMAKE:
 321         self::sayFBMake($test, 'failed');
 322         break;
 323     }
 324   }
 325
 326   private static function sayFBMake($test, $status) {
 327     $start = array('op' => 'start', 'test' => $test);
 328     $end = array('op' => 'test_done', 'test' => $test, 'status' => $status);
 329     if ($status == 'failed') {
 330       $end['details'] = @file_get_contents("$test.diff");
 331     }
 332     self::say($start, $end);
 333   }
 334
 335   public static function getResults() {
 336     return self::$results;
 337   }
 338
 339   /** Output is in the format expected by JsonTestRunner. */
 340   public static function say(/* ... */) {
 341     $data = array_map(function($row) {
 342       return json_encode($row, JSON_UNESCAPED_SLASHES) . "\n";
 343     }, func_get_args());
 344     fwrite(STDERR, implode("", $data));
 345   }
 346
 347   private static function hasColor() {
 348     return posix_isatty(STDOUT);
 349   }
 350 }
 351
 352 function run($options, $tests, $bad_test_file) {
 353   if (isset($options['verbose'])) {
 354     Status::setMode(Status::MODE_VERBOSE);
 355   }
 356   if (isset($options['fbmake'])) {
 357     Status::setMode(Status::MODE_FBMAKE);
 358   }
 359   foreach ($tests as $test) {
 360     $status = run_test($options, $test);
 361     if ($status) {
 362       Status::pass($test);
 363     } else {
 364       Status::fail($test);
 365     }
 366   }
 367   file_put_contents($bad_test_file, json_encode(Status::getResults()));
 368   foreach (Status::getResults() as $result) {
 369     if ($result['status'] == 'failed') {
 370       return 1;
 371     }
 372   }
 373   return 0;
 374 }
 375
 376 function run_test($options, $test) {
 377   $hhvm = hhvm_cmd($options, $test);
 378   $output = "";
 379   if (isset($options['repo'])) {
 380     if (strpos($test, '.hhas') !== false || strpos($hhvm, '-m debug') != false || is_file($test.'.norepo')) {
 381       # We don't have a way to skip, I guess run non-repo?
 382     } else {
 383       unlink("$test.repo/hhvm.hhbc");
 384       $hphp = hphp_cmd($options, $test);
 385       $output .= shell_exec("$hphp 2>&1");
 386       $hhvm .= " -v Repo.Authoritative=true -v Repo.Central.Path=$test.repo/hhvm.hhbc";
 387     }
 388   }
 389
 390   $descriptorspec = array(
 391      0 => array("pipe", "r"),
 392      1 => array("pipe", "w"),
 393      2 => array("pipe", "w"),
 394   );
 395   $process = proc_open("$hhvm 2>&1", $descriptorspec, $pipes);
 396   if (!is_resource($process)) {
 397     file_put_contents("$test.diff", "Couldn't invoke $hhvm");
 398     return false;
 399   }
 400
 401   fclose($pipes[0]);
 402   $output .= stream_get_contents($pipes[1]);
 403   file_put_contents("$test.out", $output);
 404   fclose($pipes[1]);
 405
 406   // hhvm redirects errors to stdout, so anything on stderr is really bad
 407   $stderr = stream_get_contents($pipes[2]);
 408   if ($stderr) {
 409     file_put_contents(
 410       "$test.diff",
 411       "Test failed because the process wrote on stderr:\n$stderr"
 412     );
 413     return false;
 414   }
 415   fclose($pipes[2]);
 416   proc_close($process);
 417
 418   // Needed for testing non-hhvm binaries that don't actually run the code
 419   // e.g. util/parser/test/parse_tester.cpp
 420   if ($output == "FORCE PASS") {
 421     return true;
 422   }
 423
 424   if (file_exists("$test.expect")) {
 425     $diff_cmds = "--text -u";
 426     exec("diff --text -u $test.expect $test.out > $test.diff 2>&1", $_, $status);
 427     // unix 0 == success
 428     return !$status;
 429
 430   } else if (file_exists("$test.expectf")) {
 431     $wanted_re = file_get_contents("$test.expectf");
 432
 433     // do preg_quote, but miss out any %r delimited sections
 434     $temp = "";
 435     $r = "%r";
 436     $startOffset = 0;
 437     $length = strlen($wanted_re);
 438     while($startOffset < $length) {
 439       $start = strpos($wanted_re, $r, $startOffset);
 440       if ($start !== false) {
 441         // we have found a start tag
 442         $end = strpos($wanted_re, $r, $start+2);
 443         if ($end === false) {
 444           // unbalanced tag, ignore it.
 445           $end = $start = $length;
 446         }
 447       } else {
 448         // no more %r sections
 449         $start = $end = $length;
 450       }
 451       // quote a non re portion of the string
 452       $temp = $temp . preg_quote(substr($wanted_re, $startOffset, ($start - $startOffset)),  '/');
 453       // add the re unquoted.
 454       if ($end > $start) {
 455         $temp = $temp . '(' . substr($wanted_re, $start+2, ($end - $start-2)). ')';
 456       }
 457       $startOffset = $end + 2;
 458     }
 459     $wanted_re = $temp;
 460
 461     $wanted_re = str_replace(
 462       array('%binary_string_optional%'),
 463       'string',
 464       $wanted_re
 465     );
 466     $wanted_re = str_replace(
 467       array('%unicode_string_optional%'),
 468       'string',
 469       $wanted_re
 470     );
 471     $wanted_re = str_replace(
 472       array('%unicode\|string%', '%string\|unicode%'),
 473       'string',
 474       $wanted_re
 475     );
 476     $wanted_re = str_replace(
 477       array('%u\|b%', '%b\|u%'),
 478       '',
 479       $wanted_re
 480     );
 481     // Stick to basics
 482     $wanted_re = str_replace('%e', '\\' . DIRECTORY_SEPARATOR, $wanted_re);
 483     $wanted_re = str_replace('%s', '[^\r\n]+', $wanted_re);
 484     $wanted_re = str_replace('%S', '[^\r\n]*', $wanted_re);
 485     $wanted_re = str_replace('%a', '.+', $wanted_re);
 486     $wanted_re = str_replace('%A', '.*', $wanted_re);
 487     $wanted_re = str_replace('%w', '\s*', $wanted_re);
 488     $wanted_re = str_replace('%i', '[+-]?\d+', $wanted_re);
 489     $wanted_re = str_replace('%d', '\d+', $wanted_re);
 490     $wanted_re = str_replace('%x', '[0-9a-fA-F]+', $wanted_re);
 491     $wanted_re = str_replace('%f', '[+-]?\.?\d+\.?\d*(?:[Ee][+-]?\d+)?', $wanted_re);
 492     $wanted_re = str_replace('%c', '.', $wanted_re);
 493     // %f allows two points "-.0.0" but that is the best *simple* expression
 494
 495     # a poor man's aide for debugging
 496     shell_exec("diff --text -u $test.expectf $test.out > $test.diff 2>&1");
 497
 498     return preg_match("/^$wanted_re\$/s", $output);
 499
 500   } else if (file_exists("$test.expectregex")) {
 501     $wanted_re = file_get_contents("$test.expectregex");
 502
 503     # a poor man's aide for debugging
 504     shell_exec("diff --text -u $test.expectregex $test.out > $test.diff 2>&1");
 505
 506     return preg_match("/^$wanted_re\$/s", $output);
 507   }
 508 }
 509
 510 function num_cpus() {
 511   $data = file('/proc/stat');
 512   $cores = 0;
 513   foreach($data as $line) {
 514     if (preg_match('/^cpu[0-9]/', $line)) {
 515       $cores++;
 516     }
 517   }
 518   return $cores;
 519 }
 520
 521 function main($argv) {
 522
 523   ini_set('pcre.backtrack_limit', PHP_INT_MAX);
 524
 525   list($options, $files) = get_options($argv);
 526   if (isset($options['help'])) {
 527     error(help());
 528   }
 529   $tests = find_tests($files);
 530
 531   $threads = min(count($tests), idx($options, 'threads', num_cpus() + 1));
 532
 533   if (!isset($options['fbmake'])) {
 534     print "Running ".count($tests)." tests in $threads threads\n";
 535   }
 536
 537   # Try to construct the buckets so the test results are ready in approximately
 538   # alphabetical order
 539   $test_buckets = array();
 540   $i = 0;
 541   foreach ($tests as $test) {
 542     $test_buckets[$i][] = $test;
 543     $i = ($i + 1) % $threads;
 544   }
 545
 546   # Spawn off worker threads
 547   $children = array();
 548   # A poor man's shared memory
 549   $bad_test_files = array();
 550   for ($i = 0; $i < $threads; $i++) {
 551     $bad_test_file = tempnam('/tmp', 'test-run-');
 552     $bad_test_files[] = $bad_test_file;
 553     $pid = pcntl_fork();
 554     if ($pid == -1) {
 555       error('could not fork');
 556     } else if ($pid) {
 557       $children[] = $pid;
 558     } else {
 559       exit(run($options, $test_buckets[$i], $bad_test_file));
 560     }
 561   }
 562
 563   # Wait for the kids
 564   $return_value = 0;
 565   foreach ($children as $child) {
 566     pcntl_waitpid($child, $status);
 567     $return_value |= pcntl_wexitstatus($status);
 568   }
 569
 570   $results = array();
 571   foreach ($bad_test_files as $bad_test_file) {
 572     $json = json_decode(file_get_contents($bad_test_file), true);
 573     if (!is_array($json)) {
 574       error(
 575         "A test thread didn't send json to the controller. WTF. ".
 576         "If your code isn't horribly broken, please complain loudly"
 577       );
 578     }
 579     $results = array_merge($results, $json);
 580   }
 581   if (isset($options['fbmake'])) {
 582     Status::say(array('op' => 'all_done', 'results' => $results));
 583   } else {
 584     if (!$return_value) {
 585       print "\nAll tests passed.\n\n".<<<SHIP
 586               |    |    |
 587              )_)  )_)  )_)
 588             )___))___))___)\
 589            )____)____)_____)\\
 590          _____|____|____|____\\\__
 591 ---------\      SHIP IT      /---------
 592   ^^^^^ ^^^^^^^^^^^^^^^^^^^^^
 593     ^^^^      ^^^^     ^^^    ^^
 594          ^^^^      ^^^
 595 SHIP
 596 ."\n";
 597     } else {
 598       $failed = array();
 599       foreach ($results as $result) {
 600         if ($result['status'] == 'failed') {
 601           $failed[] = $result['name'];
 602         }
 603       }
 604       asort($failed);
 605       $header_start = "\n\033[0;33m";
 606       $header_end = "\033[0m\n";
 607       print "\n".count($failed)." tests failed\n";
 608
 609       print $header_start."See the diffs:".$header_end.
 610         implode("\n", array_map(
 611           function($test) { return 'cat '.$test.'.diff'; },
 612         $failed))."\n";
 613
 614       print $header_start."Run these by hand:".$header_end;
 615
 616       foreach ($failed as $test) {
 617         $command = hhvm_cmd($options, $test);
 618         if (isset($options['repo'])) {
 619           $command .= " -v Repo.Authoritative=true ";
 620           $command = str_replace(verify_hhbc(), "$test.repo/hhvm.hhbc", $command);
 621           $command = hphp_cmd($options, $test)."\n".$command."\n";
 622         }
 623         print "$command\n";
 624       }
 625
 626       print $header_start."Re-run just the failing tests:".$header_end.
 627             "$argv[0] ".implode(' ', $failed)."\n";
 628     }
 629   }
 630
 631   return $return_value;
 632 }
 633
 634 exit(main($argv));