4 * Run the test suites in various configurations.
7 function is_testing_dso_extension() {
8 // detecting if we're running outside of the hhvm codebase.
9 return !is_file(__DIR__
. "/../../hphp/test/run");
12 function get_expect_file_and_type($test, $options) {
13 // .typechecker files are for typechecker (hh_server --check) test runs.
15 if (isset($options['typechecker'])) {
16 $types = array('typechecker.expect', 'typechecker.expectf');
18 $types = array('expect', 'hhvm.expect', 'expectf', 'hhvm.expectf',
21 if (isset($options['repo'])) {
22 foreach ($types as $type) {
23 $fname = "$test.$type-repo";
24 if (file_exists($fname)) {
25 return array($fname, $type);
29 foreach ($types as $type) {
30 $fname = "$test.$type";
31 if (file_exists($fname)) {
32 return array($fname, $type);
35 return array(null, null);
40 return "usage: $argv[0] [-m jit|interp] [-r] <test/directories>";
43 function suppress_opts_for_codegen_test($test) {
44 $opts_file = $test . ".opts";
45 if (!file_exists($opts_file)) {
48 $content = file_get_contents($opts_file);
49 return preg_match("/Eval\.DisableHphpcOpts=0/i", $content) !== 1;
54 $ztestexample = 'test/zend/good/*/*z*.php'; // sep. for syntax highlighting.
58 This is the hhvm test-suite runner. For more detailed documentation,
59 see hphp/test/README.md.
61 The test argument may be a path to a php test file, a directory name, or
62 one of a few pre-defined suite names that this script knows about.
64 If you work with hhvm a lot, you might consider a bash alias:
66 alias ht="path/to/hphp/test/run"
70 # Quick tests in JIT mode:
73 # Slow tests in interp mode:
74 % $argv[0] -m interp test/slow
76 # PHP specificaion tests in JIT mode:
79 # Slow closure tests in JIT mode:
80 % $argv[0] test/slow/closure
82 # Slow closure tests in JIT mode with RepoAuthoritative:
83 % $argv[0] -r test/slow/closure
85 # Slow array tests, in RepoAuthoritative:
86 % $argv[0] -r test/slow/array
88 # Zend tests with a "z" in their name:
89 % $argv[0] $ztestexample
91 # Quick tests in JIT mode with some extra runtime options:
92 % $argv[0] test/quick -a '-vEval.JitMaxTranslations=120 -vEval.HHIRRefcountOpts=0'
94 # All quick tests except debugger
95 % $argv[0] -e debugger test/quick
97 # All tests except those containing a string of 3 digits
98 % $argv[0] -E '/\d{3}/' all
100 # All tests whose name containing pdo_mysql
101 % $argv[0] -i pdo_mysql -m jit -r zend
103 # Print all the standard tests
104 % $argv[0] --list-tests
106 # Use a specific HHVM binary
107 % $argv[0] -b ~/code/hhvm/hphp/hhvm/hhvm
108 % $argv[0] --hhvm-binary-path ~/code/hhvm/hphp/hhvm/hhvm
110 # Use relocation to run tests in the same thread. e.g, 6 times in the same thread,
111 # where the 3 specifies a random relocation for the 3rd request and the test is
113 % $argv[0] --relocate 3 test/quick/silencer.php
115 # Run the Hack typechecker against quick typechecker.expect[f] files
116 # Could explcitly use quick here too
117 # $argv[0] --typechecker
119 # Run the Hack typechecker against typechecker.expect[f] files in the slow
121 # $argv[0] --typechecker slow
123 # Run the Hack typechecker against the typechecker.expect[f] file in this test
124 # $argv[0] --typechecker test/slow/test_runner_typechecker_mode/basic.php
126 # Use a specific typechecker binary
127 # $argv[0] --hhserver-binary-path ~/code/hhvm/hphp/hack/bin/hh_server --typechecker .
130 return usage().$help;
133 function error($message) {
138 // If a user-supplied path is provided, let's make sure we have a valid
140 function check_executable($path, $typechecker) {
141 $type = $typechecker ?
"HH_SERVER" : "HHVM";
142 $rpath = realpath($path);
143 $msg = "Provided ".$type." executable (".$path.") is not a file.\n"
144 . "If using ".$type."_BIN, make sure that is set correctly.";
145 if (!is_file($rpath)) {
149 exec($rpath . " --help", $output);
150 $str = implode($output);
151 $msg = "Provided file (".$rpath.") is not a/an ".$type." executable.\n"
152 . "If using ".$type."_BIN, make sure that is set correctly.";
153 if (strpos($str, "Usage") !== 0) {
158 function hhvm_binary_routes() {
160 "fbbuild" => "/_bin/hphp/hhvm",
161 "buck" => "/buck-out/gen/hphp/hhvm/hhvm",
162 "cmake" => "/hphp/hhvm"
165 $env_root = getenv("FBMAKE_BIN_ROOT");
166 if ($env_root !== false) {
167 $routes["fbbuild"] = "/" . $env_root . "/hphp/hhvm";
172 function hh_server_binary_routes() {
174 "fbbuild" => "/_bin/hphp/hack/src",
175 "buck" => "/buck-out/gen/hphp/hack/src/hh_server",
176 "cmake" => "/hphp/hack/bin"
178 $env_root = getenv("FBMAKE_BIN_ROOT");
179 if ($env_root !== false) {
180 $routes["fbbuild"] = "/" . $env_root . "/hphp/hack/src";
185 function hh_codegen_binary_routes() {
187 "buck" => "/buck-out/gen/hphp/hack/src/hh_single_compile",
188 "cmake" => "/hphp/hack/bin"
192 function hh_semdiff_binary_routes() {
194 "buck" => "/buck-out/bin/hphp/hack/src/hhbc/semdiff/semdiff",
195 "cmake" => "/hphp/hack/bin"
199 // For Facebook: We have several build systems, and we can use any of them in
200 // the same code repo. If multiple binaries exist, we want the onus to be on
201 // the user to specify a particular one because before we chose the fbmake one
202 // by default and that could cause unexpected results.
203 function check_for_multiple_default_binaries($typechecker) {
204 // Env var we use in testing that'll pick which build system to use.
205 if (getenv("FBCODE_BUILD_TOOL") !== false) {
210 $routes = $typechecker ?
hh_server_binary_routes() : hhvm_binary_routes();
211 $binary = $typechecker ?
"hh_server" : "hhvm";
214 foreach ($routes as $_ => $path) {
215 $abs_path = $home . $path . "/" . $binary;
216 if (file_exists($abs_path)) {
217 $found[] = $abs_path;
221 if (count($found) <= 1) {
225 $path_option = $typechecker ?
"--hhserver-binary-path" : "--hhvm-binary-path";
227 $msg = "Multiple binaries exist in this repo. \n";
228 foreach ($found as $bin) {
229 $msg .= " - " . $bin . "\n";
231 $msg .= "Are you in fbcode? If so, remove a binary \n"
232 . "or use the " . $path_option . " option to the test runner. \n"
235 $msg .= "--typechecker";
237 $msg .= " " . $path_option . " /path/to/binary slow\n";
241 function hphp_home() {
242 if (is_testing_dso_extension()) {
243 return realpath(__DIR__
);
245 return realpath(__DIR__
.'/../..');
248 function idx($array, $key, $default = null) {
249 return isset($array[$key]) ?
$array[$key] : $default;
252 function hhvm_path() {
254 if (getenv("HHVM_BIN") !== false) {
255 $file = realpath(getenv("HHVM_BIN"));
257 $file = bin_root().'/hhvm';
260 if (!is_file($file)) {
261 if (is_testing_dso_extension()) {
262 exec("which hhvm", $output);
263 if (isset($output[0]) && $output[0]) {
266 error("You need to specify hhvm bin with env HHVM_BIN");
269 error("$file doesn't exist. Did you forget to build first?");
271 return rel_path($file);
274 function hh_codegen_cmd($options, $config_file = null, $disable_hphpc_opts = true) {
275 $cmd = hh_codegen_path();
276 if (isset($config_file)) {
277 $cmd .= ' -c "'.$config_file.'"';
279 $cmd .= ' -v Hack.Compiler.SourceMapping=1 ';
280 if (isset($options['compare-hh-codegen']) && $disable_hphpc_opts) {
281 $cmd .= '-v Eval.DisableHphpcOpts=1';
283 if (isset($options['hackc'])) {
290 function hh_semdiff_cmd($options) {
291 $cmd = hh_semdiff_path();
292 if (isset($options['hackc'])) $cmd .= ' --daemon';
293 $cmd .= ' --verbose 1';
297 function bin_root() {
298 if (getenv("HHVM_BIN") !== false) {
299 return dirname(realpath(getenv("HHVM_BIN")));
303 $env_tool = getenv("FBCODE_BUILD_TOOL");
304 $routes = hhvm_binary_routes();
306 if ($env_tool !== false) {
307 return $home . $routes[$env_tool];
310 foreach ($routes as $_ => $path) {
311 $dir = $home . $path;
317 return $home . $routes["cmake"];
320 function hh_server_path() {
322 if (getenv("HH_SERVER_BIN") !== false) {
323 $file = realpath(getenv("HH_SERVER_BIN"));
325 $file = hh_server_bin_root().'/hh_server';
327 if (!is_file($file)) {
328 error("$file doesn't exist. Did you forget to build first?");
330 return rel_path($file);
333 function hh_server_bin_root() {
334 if (getenv("HH_SERVER_BIN") !== false) {
335 return dirname(realpath(getenv("HH_SERVER_BIN")));
339 $env_tool = getenv("FBCODE_BUILD_TOOL");
340 $routes = hh_server_binary_routes();
342 if ($env_tool !== false) {
343 return $home . $routes[$env_tool];
346 foreach ($routes as $_ => $path) {
347 $dir = $home . $path;
353 return $home . $routes["cmake"];
356 function hh_codegen_path() {
358 if (getenv("HH_CODEGEN_BIN") !== false) {
359 $file = realpath(getenv("HH_CODEGEN_BIN"));
361 $file = hh_codegen_bin_root().'/hh_single_compile.opt';
363 if (!is_file($file)) {
364 error("$file doesn't exist. Did you forget to build first?");
366 return rel_path($file);
369 function hh_semdiff_path() {
371 if (getenv("HH_SEMDIFF_BIN") !== false) {
372 $file = realpath(getenv("HH_SEMDIFF_BIN"));
374 $file = hh_semdiff_bin_root().'/semdiff.opt';
376 if (!is_file($file)) {
377 error("$file doesn't exist. Did you forget to build first?");
379 return rel_path($file);
382 function hh_codegen_bin_root() {
384 $env_tool = getenv("FBCODE_BUILD_TOOL");
385 $routes = hh_codegen_binary_routes();
387 if ($env_tool !== false) {
388 return $home . $routes[$env_tool];
391 foreach ($routes as $_ => $path) {
392 $dir = $home . $path;
398 return $home . $routes["cmake"];
401 function hh_semdiff_bin_root() {
403 $env_tool = getenv("FBCODE_BUILD_TOOL");
404 $routes = hh_semdiff_binary_routes();
406 if ($env_tool !== false) {
407 return $home . $routes[$env_tool];
410 foreach ($routes as $_ => $path) {
411 $dir = $home . $path;
417 return $home . $routes["cmake"];
420 function verify_hhbc() {
421 if (getenv("VERIFY_HHBC") !== false) {
422 return getenv($env_hhbc);
424 return bin_root().'/verify.hhbc';
427 function read_opts_file($file) {
428 if (!file_exists($file)) {
432 $fp = fopen($file, "r");
435 while ($line = fgets($fp)) {
436 // Compress out white space.
437 $line = preg_replace('/\s+/', ' ', $line);
439 // Discard simple line oriented ; and # comments to end of line
440 // Comments at end of line (after payload) are not allowed.
441 $line = preg_replace('/^ *;.*$/', ' ', $line);
442 $line = preg_replace('/^ *#.*$/', ' ', $line);
444 // Substitute in the directory name
445 $line = str_replace('__DIR__', dirname($file), $line);
453 // http://stackoverflow.com/questions/2637945/
454 function rel_path($to) {
455 $from = explode('/', getcwd().'/');
456 $to = explode('/', $to);
459 foreach ($from as $depth => $dir) {
460 // find first non-matching dir.
461 if ($dir === $to[$depth]) {
462 // ignore this directory.
463 array_shift($relPath);
465 // get number of remaining dirs to $from.
466 $remaining = count($from) - $depth;
467 if ($remaining > 1) {
468 // add traversals up to first matching dir.
469 $padLength = (count($relPath) +
$remaining - 1) * -1;
470 $relPath = array_pad($relPath, $padLength, '..');
473 $relPath[0] = './' . $relPath[0];
477 return implode('/', $relPath);
480 function get_options($argv) {
484 'exclude-pattern:' => 'E:',
485 'exclude-recorded-failures:' => 'x:',
487 'include-pattern:' => 'I:',
501 'failure-file:' => '',
504 'hhas-round-trip' => '',
512 'hhvm-binary-path:' => 'b:',
514 'hhserver-binary-path:' => '',
515 'compare-hh-codegen' => '',
517 'run-hh-codegen' => '',
518 'record-failures:' => '',
527 * '-' argument causes all future arguments to be treated as filenames, even
528 * if they would otherwise match a valid option. Otherwise, arguments starting
529 * with '-' MUST match a valid option.
533 for ($i = 1; $i < count($argv); $i++
) {
536 if (strlen($arg) == 0) {
538 } else if ($force_file) {
540 } else if ($arg === '-') {
542 } else if ($arg[0] === '-') {
545 foreach ($parameters as $long => $short) {
546 if ($arg == '-'.str_replace(':', '', $short) ||
547 $arg == '--'.str_replace(':', '', $long)) {
548 if (substr($long, -1, 1) == ':') {
549 $value = $argv[++
$i];
553 $options[str_replace(':', '', $long)] = $value;
560 error(sprintf("Invalid argument: '%s'\nSee $argv[0] --help", $arg));
567 if (isset($options['repo']) && isset($options['hhas-round-trip'])) {
568 echo "repo and hhas-round-trip are mutually exclusive options\n";
572 if (isset($options['relocate']) && isset($options['recycle-tc'])) {
573 echo "relocate and recycle-tc are mutually exclusive options\n";
577 if (isset($options['hhbbc2'])) $options['repo'] = true;
579 return array($options, $files);
583 * Return the path to $test relative to __DIR__, or false if __DIR__ does not
586 function canonical_path_from_base($test, $base) {
587 $full = realpath($test);
588 if (substr($full, 0, strlen($base)) === $base) {
589 return substr($full, strlen($base) +
1);
591 $dirstat = stat($base);
592 if (!is_array($dirstat)) return false;
593 for ($p = dirname($full); $p && $p !== "/"; $p = dirname($p)) {
595 if (!is_array($s)) continue;
596 if ($s['ino'] === $dirstat['ino'] && $s['dev'] === $dirstat['dev']) {
597 return substr($full, strlen($p) +
1);
603 function canonical_path($test) {
604 $attempt = canonical_path_from_base($test,__DIR__
);
605 if ($attempt === false) {
606 return canonical_path_from_base($test, hphp_home());
613 * We support some 'special' file names, that just know where the test
614 * suites are, to avoid typing 'hphp/test/foo'.
616 function find_test_files($file) {
618 'quick' => 'hphp/test/quick',
619 'slow' => 'hphp/test/slow',
620 'spec' => 'hphp/test/spec',
621 'debugger' => 'hphp/test/server/debugger/tests',
622 'http' => 'hphp/test/server/http/tests',
623 'fastcgi' => 'hphp/test/server/fastcgi/tests',
624 'zend' => 'hphp/test/zend/good',
625 'facebook' => 'hphp/facebook/test',
627 // Subsets of zend tests.
628 'zend_ext' => 'hphp/test/zend/good/ext',
629 'zend_ext_am' => 'hphp/test/zend/good/ext/[a-m]*',
630 'zend_ext_nz' => 'hphp/test/zend/good/ext/[n-z]*',
631 'zend_Zend' => 'hphp/test/zend/good/Zend',
632 'zend_tests' => 'hphp/test/zend/good/tests',
635 if (isset($mappage[$file])) {
636 $matches = glob(hphp_home().'/'.$mappage[$file]);
637 if (count($matches) == 0) {
639 "Convenience test name '%s' is recognized but does not match any test ".
640 "files (pattern = '%s', hphp_home = '%s')",
641 $file, $mappage[$file], hphp_home()));
645 return array($file, );
649 // Some tests have to be run together in the same test bucket, serially, one
650 // after other in order to avoid races and other collisions.
651 function serial_only_tests($tests) {
652 if (is_testing_dso_extension()) {
655 // Add a <testname>.php.serial file to make your test run in the serial
657 $serial_tests = array_filter(
660 return file_exists($test . '.serial');
663 return $serial_tests;
666 function find_tests($files, array $options = null) {
668 $files = array('quick');
670 if ($files == array('all')) {
671 $files = array('quick', 'slow', 'spec', 'zend', 'fastcgi');
672 if (is_dir(hphp_home() . '/hphp/facebook/test')) {
673 $files[] = 'facebook';
677 foreach ($files as $file) {
678 $ft = array_merge($ft, find_test_files($file));
681 foreach ($files as &$file) {
683 error("Not valid file or directory: '$file'");
685 $file = preg_replace(',//+,', '/', realpath($file));
686 $file = preg_replace(',^'.getcwd().'/,', '', $file);
688 $files = array_map('escapeshellarg', $files);
689 $files = implode(' ', $files);
690 if (isset($options['typechecker'])) {
691 $tests = explode("\n", shell_exec(
692 "find $files -name '*.php' -o -name '*.php.type-errors'"
694 // The above will get all the php files. Now filter out only the ones
695 // that have a .hhconfig associated with it.
696 $tests = array_filter(
699 return (file_exists($test . '.typechecker.expect') ||
700 file_exists($test . '.typechecker.expectf')) &&
701 file_exists($test . '.hhconfig');
704 } else if (isset($options['compare-hh-codegen'])) {
705 $tests = explode("\n", shell_exec(
706 "find $files -name '*.php'"
709 $tests = explode("\n", shell_exec(
710 "find $files -name '*.php' -o -name '*.php.type-errors' " .
711 "-o -name '*.hhas' | grep -v round_trip.hhas"
715 error("Could not find any tests associated with your options.\n" .
716 "Make sure your test path is correct and that you have " .
717 "the right expect files for the tests you are trying to run.\n" .
721 $tests = array_filter($tests);
722 if (!empty($options['exclude'])) {
723 $exclude = $options['exclude'];
724 $tests = array_filter($tests, function($test) use ($exclude) {
725 return (false === strpos($test, $exclude));
728 if (!empty($options['exclude-pattern'])) {
729 $exclude = $options['exclude-pattern'];
730 $tests = array_filter($tests, function($test) use ($exclude) {
731 return !preg_match($exclude, $test);
734 if (!empty($options['exclude-recorded-failures'])) {
735 $exclude_file = $options['exclude-recorded-failures'];
736 $exclude = file($exclude_file, FILE_IGNORE_NEW_LINES
);
737 $tests = array_filter($tests, function($test) use ($exclude) {
738 return (false === in_array(canonical_path($test), $exclude));
741 if (!empty($options['include'])) {
742 $include = $options['include'];
743 $tests = array_filter($tests, function($test) use ($include) {
744 return (false !== strpos($test, $include));
747 if (!empty($options['include-pattern'])) {
748 $include = $options['include-pattern'];
749 $tests = array_filter($tests, function($test) use ($include) {
750 return preg_match($include, $test);
756 function list_tests($files, $options) {
758 $mode = idx($options, 'mode', '');
765 $args[] = '-m ' . $mode;
768 throw new Exception("Unsupported mode for listing tests: ".$mode);
771 if (isset($options['hhbbc2'])) {
772 $args[] = '--hhbbc2';
773 } else if (isset($options['repo'])) {
777 if (isset($options['relocate'])) {
778 $args[] = '--relocate';
779 $args[] = $options['relocate'];
782 if (isset($options['recycle-tc'])) {
783 $args[] = '--recycle-tc';
784 $args[] = $options['recycle-tc'];
787 foreach (find_tests($files, $options) as $test) {
788 print Status
::jsonEncode(array(
789 'args' => implode(' ', $args),
795 function find_test_ext($test, $ext, $configName='config') {
796 if (is_file("{$test}.{$ext}")) {
797 return "{$test}.{$ext}";
799 return find_file_for_dir(dirname($test), "{$configName}.{$ext}");
802 function find_file_for_dir($dir, $name) {
803 // Handle the case where the $dir might come in as '.' because you
804 // are running the test runner on a file from the same directory as
805 // the test e.g., './mytest.php'. dirname() will give you the '.' when
806 // you actually have a lot of path to traverse upwards like
807 // /home/you/code/tests/mytest.php. Use realpath() to get that.
808 $dir = realpath($dir);
809 while ($dir !== '/' && is_dir($dir)) {
810 $file = "$dir/$name";
811 if (is_file($file)) {
814 $dir = dirname($dir);
816 $file = __DIR__
.'/'.$name;
817 if (file_exists($file)) {
823 function find_debug_config($test, $name) {
824 $debug_config = find_file_for_dir(dirname($test), $name);
825 if ($debug_config !== null) {
826 return "-m debug --debug-config ".$debug_config;
831 function mode_cmd($options) {
833 if (!isset($options['repo'])) {
834 // Set the non-repo-mode shared repo.
835 // When in repo mode, we set our own central path.
836 $repo_args = "-vRepo.Local.Mode=-- -vRepo.Central.Path=".verify_hhbc();
838 $jit_args = "$repo_args -vEval.Jit=true";
839 $mode = idx($options, 'mode', '');
847 ' -vEval.JitPGORegionSelector=hottrace'.
848 ' -vEval.JitPGOHotOnly=0';
850 return "$repo_args -vEval.Jit=0";
852 return array("$repo_args -vEval.Jit=0", $jit_args);
854 error("-m must be one of jit | pgo | interp | interp,jit. Got: '$mode'");
858 function extra_args($options) {
859 return idx($options, 'args', '');
862 function hhvm_cmd_impl() {
863 $args = func_get_args();
864 $options = array_shift($args);
865 $disable_hphpc_opts = array_shift($args);
866 $config = array_shift($args);
868 $modes = (array)mode_cmd($options);
870 foreach ($modes as $mode) {
875 '-vEval.EnableArgsInBacktraces=true',
876 '-vEval.EnableIntrinsicsExtension=true',
878 isset($options['arm']) ?
'-vEval.SimulateARM=1' : '',
879 isset($options['wholecfg']) ?
'-vEval.JitPGORegionSelector=wholecfg' : '',
881 // load/store counters don't work on Ivy Bridge so disable for tests
882 '-vEval.ProfileHWEnable=false',
884 extra_args($options),
887 if (isset($options['hackc'])) {
888 $args[] = '-vEval.HackCompilerCommand="'.hh_codegen_cmd($options).'"';
889 $args[] = '-vEval.HackCompilerDefault=true';
892 if (isset($options['relocate'])) {
893 $args[] = '--count='.($options['relocate'] * 2);
894 $args[] = '-vEval.JitAHotSize=6000000';
895 $args[] = '-vEval.HotFuncCount=0';
896 $args[] = '-vEval.PerfRelocate='.$options['relocate'];
899 if (isset($options['recycle-tc'])) {
900 $args[] = '--count='.$options['recycle-tc'];
901 $args[] = '-vEval.StressUnitCacheFreq=1';
902 $args[] = '-vEval.EnableReusableTC=true';
905 if (isset($options['run-hh-codegen']) ||
906 isset($options['hhas-round-trip'])) {
907 $args[] = '-vEval.AllowHhas=1';
908 $args[] = '-vEval.LoadFilepathFromUnitCache=1';
911 if (isset($options['compare-hh-codegen'])) {
912 if ($disable_hphpc_opts) {
913 $args[] = '-vEval.DisableHphpcOpts=1';
915 $args[] = '-vEval.DisassemblerSourceMapping=1';
916 $args[] = '-vEval.CreateInOutWrapperFunctions=0';
918 // TODO(paulbiss): support these
919 $args[] = '-vEval.DisassemblerPropDocComments=0';
920 $args[] = '-vEval.JitEnableRenameFunction=0';
921 $args[] = '-vEval.DisassemblerDocComments=0';
924 if (!isset($options['cores'])) {
925 $args[] = '-vResourceLimit.CoreFileSize=0';
927 $cmds[] = implode(' ', array_merge($args, $extra_args));
929 if (count($cmds) != 1) return $cmds;
933 // Return the command and the env to run it in.
934 function hhvm_cmd($options, $test, $test_run = null, $is_temp_file = false) {
935 if ($test_run === null) {
938 // hdf support is only temporary until we fully migrate to ini
939 // Discourage broad use.
940 $hdf_suffix = ".use.for.ini.migration.testing.only.hdf";
941 $hdf = file_exists($test.$hdf_suffix)
942 ?
'-c ' . $test . $hdf_suffix
944 $disable_hphpc_opts = suppress_opts_for_codegen_test($test);
945 $cmds = hhvm_cmd_impl(
948 find_test_ext($test, 'ini'),
950 find_debug_config($test, 'hphpd.ini'),
951 read_opts_file(find_test_ext($test, 'opts')),
953 escapeshellarg($test_run),
954 $is_temp_file ?
" --temp-file" : ""
959 if (file_exists($test.'.verify')) {
960 $cmd .= " -m verify";
963 if (isset($options['cli-server']) && can_run_server_test($test)) {
964 $config = find_file_for_dir(dirname($test), 'config.ini');
965 $socket = $options['servers']['configs'][$config]['cli-socket'];
966 $cmd .= ' -vEval.UseRemoteUnixServer=only';
967 $cmd .= ' -vEval.UnixServerPath='.$socket;
970 // Special support for tests that require a path to the current
971 // test directory for things like prepend_file and append_file
973 if (file_exists($test.'.ini')) {
974 $contents = file_get_contents($test.'.ini');
975 if (strpos($contents, '{PWD}') !== false) {
976 $test_ini = tempnam('/tmp', $test).'.ini';
977 file_put_contents($test_ini,
978 str_replace('{PWD}', dirname($test), $contents));
979 $cmd .= " -c $test_ini";
983 $contents = file_get_contents($test.$hdf_suffix);
984 if (strpos($contents, '{PWD}') !== false) {
985 $test_hdf = tempnam('/tmp', $test).$hdf_suffix;
986 file_put_contents($test_hdf,
987 str_replace('{PWD}', dirname($test), $contents));
988 $cmd .= " -c $test_hdf";
992 if (isset($options['repo'])) {
993 $program = isset($options['hackc']) ?
"hackc" : "hhvm";
994 $hhbbc_repo = "\"$test.repo/$program.hhbbc\"";
995 $cmd .= ' -vRepo.Authoritative=true -vRepo.Commit=0';
996 $cmd .= " -vRepo.Central.Path=$hhbbc_repo";
999 // Command line arguments
1000 $cli_args = find_test_ext($test, 'cli_args');
1001 if ($cli_args !== null) {
1002 $cmd .= " -- " . trim(file_get_contents($cli_args));
1006 $extra_env = array();
1008 // Apply the --env option
1009 if (isset($options['env'])) {
1010 $extra_env = array_merge($extra_env,
1011 explode(",", $options['env']));
1014 // If there's an <test name>.env file then inject the contents of that into
1015 // the test environment.
1016 $env_file = find_test_ext($test, 'env');
1017 if ($env_file !== null) {
1018 $extra_env = array_merge($extra_env,
1019 explode("\n", trim(file_get_contents($env_file))));
1023 foreach ($extra_env as $arg) {
1024 $i = strpos($arg, '=');
1026 $key = substr($arg, 0, $i);
1027 $val = substr($arg, $i +
1);
1035 $in = find_test_ext($test, 'in');
1037 $cmd .= ' < ' . escapeshellarg($in);
1038 // If we're piping the input into the command then setup a simple
1039 // dumb terminal so hhvm doesn't try to control it and pollute the
1040 // output with control characters, which could change depending on
1041 // a wide variety of terminal settings.
1042 $env["TERM"] = "dumb";
1045 if (is_array($cmds)) {
1046 foreach ($cmds as &$c) {
1051 $cmd = $cmds . $cmd;
1054 return array($cmd, $env);
1057 function hphp_cmd($options, $test, $timeout_prefix, $hackc) {
1058 $extra_args = preg_replace("/-v\s*/", "-vRuntime.", extra_args($options));
1064 $hh_single_compile = hh_codegen_path();
1065 $hackc_args = implode(" ", array(
1066 "-vRuntime.Eval.HackCompilerReset=1",
1067 "-vRuntime.Eval.HackCompilerDefault=true",
1068 "-vRuntime.Eval.HackCompilerInheritConfig=true",
1069 "-vRuntime.Eval.HackCompilerCommand=\"${hh_single_compile} -v Hack.Compiler.SourceMapping=1 --daemon --stop-logging-stats --dump-symbol-refs\""
1073 return implode(" ", array(
1074 "HHVM_DISABLE_HHBBC=1",
1079 find_test_ext($test, 'ini', 'hphp_config'),
1080 read_opts_file("$test.hphp_opts"),
1081 '-vRuntime.Eval.EnableIntrinsicsExtension=true',
1082 '--nofork=1 -thhbc -l1 -k1',
1083 "-o \"$test.repo\" --program $program.hhbc \"$test\"",
1089 function hhbbc_cmd($options, $test, $timeout_prefix, $program) {
1090 return implode(" ", array(
1095 '--parallel-num-threads=1',
1096 read_opts_file("$test.hhbbc_opts"),
1097 "-o \"$test.repo/$program.hhbbc\" \"$test.repo/$program.hhbc\"",
1101 function hh_server_cmd($options, $test) {
1102 // In order to run hh_server --check on only one file, we copy all of the
1103 // files associated with the test to a temporary directory, rename the
1104 // basename($test_file).hhconfig file to just .hhconfig and set the command
1106 $temp_dir = '/tmp/hh-test-runner-'.bin2hex(random_bytes(16));
1108 foreach (glob($test . '*') as $test_file) {
1109 copy($test_file, $temp_dir . '/' . basename($test_file));
1110 if (strpos($test_file, '.hhconfig') !== false) {
1112 $temp_dir . '/' . basename($test) . '.hhconfig',
1113 $temp_dir . '/.hhconfig'
1115 } else if (strpos($test_file, '.type-errors') !== false) {
1116 // In order to actually run hh_server --check successfully, all files
1117 // named *.php.type-errors have to be renamed *.php
1119 $temp_dir . '/' . basename($test_file),
1120 $temp_dir . '/' . str_replace('.type-errors', '', basename($test_file))
1124 // Just copy all the .php.inc files, even if they are not related since
1125 // unrelated ones will be ignored anyway. This just makes it easier to
1126 // start with instead of doing a search inside the test file for requires
1127 // and includes and extracting it.
1128 foreach (glob(dirname($test) . "/*.inc.php") as $inc_file) {
1129 copy($inc_file, $temp_dir . '/' . basename($inc_file));
1131 $cmd = hh_server_path() . ' --check ' . $temp_dir;
1132 return array($cmd, ' ', $temp_dir);
1136 private static $results = array();
1137 private static $mode = 0;
1139 private static $use_color = false;
1141 private static $queue = null;
1142 private static $killed = false;
1145 private static $overall_start_time = 0;
1146 private static $overall_end_time = 0;
1148 private static $tempdir = "";
1150 const MODE_NORMAL
= 0;
1151 const MODE_VERBOSE
= 1;
1152 const MODE_FBMAKE
= 2;
1153 const MODE_TESTPILOT
= 3;
1154 const MODE_RECORD_FAILURES
= 4;
1156 const MSG_STARTED
= 7;
1157 const MSG_FINISHED
= 1;
1158 const MSG_TEST_PASS
= 2;
1159 const MSG_TEST_FAIL
= 4;
1160 const MSG_TEST_SKIP
= 5;
1161 const MSG_SERVER_RESTARTED
= 6;
1168 const PASS_SERVER
= 0;
1169 const SKIP_SERVER
= 1;
1172 private static function getTempDir() {
1173 self
::$tempdir = sys_get_temp_dir();
1174 // Apparently some systems might not put the trailing slash
1175 if (substr(self
::$tempdir, -1) !== "/") {
1176 self
::$tempdir .= "/";
1178 self
::$tempdir .= getmypid().'-'.rand();
1179 mkdir(self
::$tempdir);
1182 // Since we run the tests in forked processes, state is not shared
1183 // So we cannot keep a static variable adding individual test times.
1184 // But we can put the times files and add the values later.
1185 public static function setTestTime($time) {
1186 file_put_contents(tempnam(self
::$tempdir, "trun"), $time);
1189 // The total time running the tests if they were run serially.
1190 public static function addTestTimesSerial() {
1192 $files = scandir(self
::$tempdir);
1193 foreach ($files as $file) {
1194 if (strpos($file, 'trun') === 0) {
1195 $time +
= floatval(file_get_contents(self
::$tempdir . "/" . $file));
1196 unlink(self
::$tempdir . "/" . $file);
1202 public static function setMode($mode) {
1203 self
::$mode = $mode;
1206 public static function setUseColor($use) {
1207 self
::$use_color = $use;
1210 public static function getMode() {
1214 public static function getOverallStartTime() {
1215 return self
::$overall_start_time;
1218 public static function getOverallEndTime() {
1219 return self
::$overall_end_time;
1222 public static function started() {
1224 self
::send(self
::MSG_STARTED
, null);
1225 self
::$overall_start_time = microtime(true);
1228 public static function finished() {
1229 self
::$overall_end_time = microtime(true);
1230 self
::send(self
::MSG_FINISHED
, null);
1233 public static function killQueue() {
1234 if (!self
::$killed) {
1235 msg_remove_queue(self
::$queue);
1236 self
::$queue = null;
1237 self
::$killed = true;
1241 public static function pass($test, $detail, $time, $stime, $etime) {
1242 array_push(self
::$results, array('name' => $test,
1243 'status' => 'passed',
1244 'start_time' => $stime,
1245 'end_time' => $etime,
1247 $how = $detail === 'pass-server' ? self
::PASS_SERVER
:
1248 ($detail === 'skip-server' ? self
::SKIP_SERVER
: self
::PASS_CLI
);
1249 self
::send(self
::MSG_TEST_PASS
, array($test, $how, $time, $stime, $etime));
1252 public static function skip($test, $reason, $time, $stime, $etime) {
1253 if (self
::getMode() === self
::MODE_FBMAKE
) {
1254 /* Intentionally supress skips */
1255 } elseif (self
::getMode() === self
::MODE_TESTPILOT
) {
1256 /* testpilot needs a positive response for every test run, report
1257 * that this test isn't relevant so it can silently drop. */
1258 array_push(self
::$results, array('name' => $test,
1259 'status' => 'not_relevant',
1260 'start_time' => $stime,
1261 'end_time' => $etime,
1264 array_push(self
::$results, array('name' => $test,
1265 'status' => 'skipped',
1266 'start_time' => $stime,
1267 'end_time' => $etime,
1270 self
::send(self
::MSG_TEST_SKIP
,
1271 array($test, $reason, $time, $stime, $etime));
1274 public static function fail($test, $time, $stime, $etime) {
1275 array_push(self
::$results, array(
1277 'status' => 'failed',
1278 'details' => self
::utf8Sanitize(@file_get_contents
("$test.diff")),
1279 'start_time' => $stime,
1280 'end_time' => $etime,
1283 self
::send(self
::MSG_TEST_FAIL
, array($test, $time, $stime, $etime));
1286 public static function serverRestarted() {
1287 self
::send(self
::MSG_SERVER_RESTARTED
, null);
1290 private static function send($type, $msg) {
1291 if (self
::$killed) {
1294 msg_send(self
::getQueue(), $type, $msg);
1298 * Takes a variable number of string arguments. If color output is enabled
1299 * and any one of the arguments is preceded by an integer (see the color
1300 * constants above), that argument will be given the indicated color.
1302 public static function sayColor() {
1303 $args = func_get_args();
1304 while (count($args)) {
1306 $str = array_shift($args);
1307 if (is_integer($str)) {
1309 if (self
::$use_color) {
1310 print "\033[0;${color}m";
1312 $str = array_shift($args);
1317 if (self
::$use_color && !is_null($color)) {
1323 public static function sayFBMake($test, $status, $stime, $etime) {
1324 $start = array('op' => 'start', 'test' => $test);
1325 $end = array('op' => 'test_done', 'test' => $test, 'status' => $status,
1326 'start_time' => $stime, 'end_time' => $etime);
1327 if ($status == 'failed') {
1328 $end['details'] = self
::utf8Sanitize(@file_get_contents
("$test.diff"));
1330 self
::say($start, $end);
1333 public static function getResults() {
1334 return self
::$results;
1337 /** Output is in the format expected by JsonTestRunner. */
1338 public static function say(/* ... */) {
1339 $data = array_map(function($row) {
1340 return self
::jsonEncode($row) . "\n";
1341 }, func_get_args());
1342 fwrite(STDERR
, implode("", $data));
1345 public static function hasCursorControl() {
1346 // for runs on hudson-ci.org (aka jenkins).
1347 if (getenv("HUDSON_URL")) {
1350 // for runs on travis-ci.org
1351 if (getenv("TRAVIS")) {
1354 $stty = self
::getSTTY();
1358 return strpos($stty, 'erase = <undef>') === false;
1361 public static function getSTTY() {
1362 $descriptorspec = array(1 => array("pipe", "w"), 2 => array("pipe", "w"));
1363 $process = proc_open(
1364 'stty -a', $descriptorspec, $pipes, null, null,
1365 array('suppress_errors' => true)
1367 $stty = stream_get_contents($pipes[1]);
1368 proc_close($process);
1372 public static function utf8Sanitize($str) {
1373 if (!is_string($str)) {
1374 // We sometimes get called with the
1375 // return value of file_get_contents()
1376 // when fgc() has failed.
1380 return UConverter
::transcode($str, 'UTF-8', 'UTF-8');
1383 public static function jsonEncode($data) {
1384 // JSON_UNESCAPED_SLASHES is Zend 5.4+.
1385 if (defined("JSON_UNESCAPED_SLASHES")) {
1386 return json_encode($data, JSON_UNESCAPED_SLASHES
);
1389 $json = json_encode($data);
1390 return str_replace('\\/', '/', $json);
1393 public static function getQueue() {
1394 if (!self
::$queue) {
1395 self
::$queue = msg_get_queue(self
::$key);
1397 return self
::$queue;
1401 function clean_intermediate_files($test, $options) {
1402 if (isset($options['no-clean'])) {
1405 $exts = array('out', 'diff', 'repo', 'round_trip.hhas',
1406 'hhcodegen_output.hhas', 'hhcodegen_messages', 'hhcodegen_config.json');
1407 foreach ($exts as $ext) {
1408 $file = "$test.$ext";
1409 if (file_exists($file)) {
1410 if (is_dir($file)) {
1411 foreach(new RecursiveIteratorIterator(new
1412 RecursiveDirectoryIterator($file, FilesystemIterator
::SKIP_DOTS
),
1413 RecursiveIteratorIterator
::CHILD_FIRST
) as $path) {
1415 ?
rmdir($path->getPathname())
1416 : unlink($path->getPathname());
1426 function run($options, $tests, $bad_test_file) {
1427 foreach ($tests as $test) {
1429 $time = microtime(true);
1430 $status = run_and_lock_test($options, $test);
1431 $time = microtime(true) - $time;
1433 Status
::setTestTime($time);
1434 if ($status === 'skip') {
1435 Status
::skip($test, null, $time, $stime, $etime);
1436 clean_intermediate_files($test, $options);
1437 } else if ($status === 'skip-norepo') {
1438 Status
::skip($test, 'norepo', $time, $stime, $etime);
1439 clean_intermediate_files($test, $options);
1440 } else if ($status === 'skip-verify') {
1441 Status
::skip($test, 'verify', $time, $stime, $etime);
1442 clean_intermediate_files($test, $options);
1443 } else if ($status === 'skip-onlyrepo') {
1444 Status
::skip($test, 'onlyrepo', $time, $stime, $etime);
1445 clean_intermediate_files($test, $options);
1446 } else if ($status === 'skip-nosemdiff') {
1447 Status
::skip($test, 'no-semdiff', $time, $stime, $etime);
1448 clean_intermediate_files($test, $options);
1449 } else if ($status) {
1450 Status
::pass($test, $status, $time, $stime, $etime);
1451 clean_intermediate_files($test, $options);
1453 Status
::fail($test, $time, $stime, $etime);
1456 file_put_contents($bad_test_file, json_encode(Status
::getResults()));
1457 foreach (Status
::getResults() as $result) {
1458 if ($result['status'] == 'failed') {
1465 function is_hack_file($options, $test) {
1466 if (substr($test, -3) === '.hh') return true;
1468 $file = fopen($test, 'r');
1469 if ($file === false) return false;
1471 // Skip lines that are a shebang or whitespace.
1472 while (($line = fgets($file)) !== false) {
1473 $line = trim($line);
1474 if ($line === '' ||
substr($line, 0, 2) === '#!') continue;
1475 // Allow partial and strict, but don't count decl files as Hack code
1476 if ($line === '<?hh' ||
$line === '<?hh //strict') return true;
1484 function skip_test($options, $test) {
1485 if (isset($options['hack-only']) &&
1486 substr($test, -5) !== '.hhas' &&
1487 !is_hack_file($options, $test)) {
1491 $skipif_test = find_test_ext($test, 'skipif');
1492 if (!$skipif_test) {
1496 // For now, run the .skipif in non-repo since building a repo for it is hard.
1497 $options_without_repo = $options;
1498 unset($options_without_repo['repo']);
1500 list($hhvm, $_) = hhvm_cmd($options_without_repo, $test, $skipif_test);
1501 if (is_array($hhvm)) $hhvm=$hhvm[0];
1503 $descriptorspec = array(
1504 0 => array("pipe", "r"),
1505 1 => array("pipe", "w"),
1506 2 => array("pipe", "w"),
1509 $process = proc_open("$hhvm $test 2>&1", $descriptorspec, $pipes);
1510 if (!is_resource($process)) {
1511 // This is weird. We can't run HHVM but we probably shouldn't skip the test
1512 // since on a broken build everything will show up as skipped and give you a
1518 $output = stream_get_contents($pipes[1]);
1520 proc_close($process);
1522 // The standard php5 .skipif semantics is if the .skipif outputs ANYTHING
1523 // then it should be skipped. This is a poor design, but I'll just add a
1524 // small blacklist of things that are really bad if they are output so we
1525 // surface the errors in the tests themselves.
1526 if (stripos($output, 'segmentation fault') !== false) {
1530 return strlen($output) != 0;
1533 function comp_line($l1, $l2, $is_reg) {
1535 return preg_match('/^'. $l1 . '$/s', $l2);
1537 return !strcmp($l1, $l2);
1541 function count_array_diff($ar1, $ar2, $is_reg, $w, $idx1, $idx2, $cnt1, $cnt2,
1545 while ($idx1 < $cnt1 && $idx2 < $cnt2 && comp_line($ar1[$idx1], $ar2[$idx2],
1556 for ($ofs1 = $idx1 +
1; $ofs1 < $cnt1 && $st-- > 0; $ofs1++
) {
1557 $eq = @count_array_diff
($ar1, $ar2, $is_reg, $w, $ofs1, $idx2, $cnt1,
1568 for ($ofs2 = $idx2 +
1; $ofs2 < $cnt2 && $st-- > 0; $ofs2++
) {
1569 $eq = @count_array_diff
($ar1, $ar2, $is_reg, $w, $idx1, $ofs2, $cnt1, $cnt2, $st);
1577 } else if ($eq2 > 0) {
1585 function generate_array_diff($ar1, $ar2, $is_reg, $w) {
1586 $idx1 = 0; $ofs1 = 0; $cnt1 = @count
($ar1);
1587 $idx2 = 0; $ofs2 = 0; $cnt2 = @count
($ar2);
1592 while ($idx1 < $cnt1 && $idx2 < $cnt2) {
1594 if (comp_line($ar1[$idx1], $ar2[$idx2], $is_reg)) {
1600 $c1 = @count_array_diff
($ar1, $ar2, $is_reg, $w, $idx1+
1, $idx2, $cnt1,
1602 $c2 = @count_array_diff
($ar1, $ar2, $is_reg, $w, $idx1, $idx2+
1, $cnt1,
1606 $old1[$idx1] = sprintf("%03d- ", $idx1+
1) . $w[$idx1++
];
1608 } else if ($c2 > 0) {
1609 $old2[$idx2] = sprintf("%03d+ ", $idx2+
1) . $ar2[$idx2++
];
1612 $old1[$idx1] = sprintf("%03d- ", $idx1+
1) . $w[$idx1++
];
1613 $old2[$idx2] = sprintf("%03d+ ", $idx2+
1) . $ar2[$idx2++
];
1618 reset($old1); $k1 = key($old1); $l1 = -2;
1619 reset($old2); $k2 = key($old2); $l2 = -2;
1621 while ($k1 !== null ||
$k2 !== null) {
1623 if ($k1 == $l1 +
1 ||
$k2 === null) {
1625 $diff[] = current($old1);
1626 $k1 = next($old1) ?
key($old1) : null;
1627 } else if ($k2 == $l2 +
1 ||
$k1 === null) {
1629 $diff[] = current($old2);
1630 $k2 = next($old2) ?
key($old2) : null;
1631 } else if ($k1 < $k2) {
1633 $diff[] = current($old1);
1634 $k1 = next($old1) ?
key($old1) : null;
1637 $diff[] = current($old2);
1638 $k2 = next($old2) ?
key($old2) : null;
1642 while ($idx1 < $cnt1) {
1643 $diff[] = sprintf("%03d- ", $idx1 +
1) . $w[$idx1++
];
1646 while ($idx2 < $cnt2) {
1647 $diff[] = sprintf("%03d+ ", $idx2 +
1) . $ar2[$idx2++
];
1653 function generate_diff($wanted, $wanted_re, $output)
1655 $w = explode("\n", $wanted);
1656 $o = explode("\n", $output);
1657 if (is_null($wanted_re)) {
1660 if (preg_match('/^\((.*)\)\{(\d+)\}$/s', $wanted_re, $m)) {
1661 $t = explode("\n", $m[1]);
1664 for ($i = 0; $i < $m[2]; $i++
) {
1665 foreach ($t as $v) {
1668 foreach ($w as $v) {
1672 $w = $wanted === $wanted_re ?
$r : $w2;
1674 $r = explode("\n", $wanted_re);
1677 $diff = generate_array_diff($r, $o, !is_null($wanted_re), $w);
1679 return implode("\r\n", $diff);
1682 function dump_repo_to_hhas_files($test, $program, $hhas_target) {
1683 $repo = "$test.repo/$program.hhbc";
1684 $dump_cmd = implode(" ", array(
1686 '-vRepo.Authoritative=true',
1687 '-vRepo.Commit=false',
1688 "-vRepo.Central.Path=$repo",
1689 "-vRepo.Local.Path=$repo",
1690 '-vEval.DumpHhas=true'));
1691 system("$dump_cmd $test 2> /dev/null > $hhas_target", $ret);
1692 return $ret === 0 ?
$hhas_target : false;
1695 function dump_hhas_cmd($hhvm_cmd, $test) {
1697 ' -- ', ' -m dumphhas -vEval.AllowHhas=1 -- ', $hhvm_cmd
1699 if ($cmd == $hhvm_cmd) $cmd .= " -m dumphhas -vEval.AllowHhas=1";
1703 function dump_hhas_to_temp($hhvm_cmd, $test) {
1704 $temp_file = $test . '.round_trip.hhas';
1705 if (isset($options['repo'])) {
1706 return dump_repo_to_hhas_files($test, "hhvm", $temp_file);
1709 $cmd = dump_hhas_cmd($hhvm_cmd, $test);
1710 system("$cmd > $temp_file 2> /dev/null", $ret);
1711 return $ret === 0 ?
$temp_file : false;
1714 function dump_hh_codegen($options, $test, $test_config) {
1715 $temp_file = $test.'.hhcodegen_output.hhas';
1716 if (isset($options['repo'])) {
1717 return dump_repo_to_hhas_files($test, "hackc", $temp_file);
1719 $disable_hphpc_opts = suppress_opts_for_codegen_test($test);
1720 $msgs_file = $test.'.hhcodegen_messages';
1722 $test_config_file = $test.'.hhcodegen_config.json';
1723 file_put_contents($test_config_file, $test_config);
1724 $cmd = hh_codegen_cmd($options, $test_config_file, $disable_hphpc_opts);
1727 $cmd = hh_codegen_cmd($options, null, $disable_hphpc_opts);
1729 system("/usr/bin/timeout 300 $cmd $test > $temp_file 2> $msgs_file", $ret);
1730 return $ret === 0 ?
$temp_file : false;
1733 function semdiff_output($options, $test) {
1734 $temp_file = $test.'.diff';
1735 $msgs_file = $test.'.semdiff_messages';
1736 $hhcodegen_output = $test.'.hhcodegen_output.hhas';
1737 $hhvm_output = $test.'.round_trip.hhas';
1738 $cmd = hh_semdiff_cmd($options);
1740 system("/usr/bin/timeout 300 $cmd $hhcodegen_output $hhvm_output > $temp_file 2> $msgs_file", $ret);
1741 return $ret === 0 ?
$temp_file : false;
1744 function strip_hhas_file($file) {
1745 $h = fopen($file, "r");
1747 while (($s = fgets($h)) !== false) {
1748 if (strpos($s, "HackCNYI") !== false) {
1752 if ($s === "" ||
$s[0] === "#" ||
substr($s, 0, 10) === ".filepath "
1753 ||
substr($s, 0, 7) === ".strict") {
1759 file_put_contents($file, $buf);
1762 function contains_nyi_line($file) {
1763 $h = fopen($file, "r");
1764 while (($s = fgets($h)) !== false) {
1765 if (strpos($s, "HackCNYI") !== false) {
1774 const ASM_ERROR
= " String \"Assembler Error: ";
1775 function is_assembler_fail($file) {
1776 $h = fopen($file, "r");
1777 while (($s = fgets($h)) !== false) {
1778 if (substr($s, 0, strlen(ASM_ERROR
)) === ASM_ERROR
) {
1787 const HHAS_EXT
= '.hhas';
1788 function can_run_server_test($test) {
1790 !is_file("$test.noserver") &&
1791 !find_test_ext($test, 'opts') &&
1792 !is_file("$test.ini") &&
1793 !is_file("$test.onlyrepo") &&
1794 strpos($test, 'quick/debugger') === false &&
1795 strpos($test, 'quick/xenon') === false &&
1796 strpos($test, 'slow/streams/') === false &&
1797 strpos($test, 'slow/ext_mongo/') === false &&
1798 strpos($test, 'slow/ext_oauth/') === false &&
1799 strpos($test, 'slow/ext_yaml/') === false &&
1800 strpos($test, 'slow/debugger/') === false &&
1801 strpos($test, 'slow/type_profiler/debugger/') === false &&
1802 strpos($test, 'zend/good/ext/standard/tests/array/') === false &&
1803 strpos($test, 'zend/good/ext/ftp') === false &&
1804 strrpos($test, HHAS_EXT
) !== (strlen($test) - strlen(HHAS_EXT
))
1808 const SERVER_TIMEOUT
= 45;
1809 function run_config_server($options, $test) {
1810 if (!isset($options['server']) ||
!can_run_server_test($test)) {
1814 $config = find_file_for_dir(dirname($test), 'config.ini');
1815 $port = $options['servers']['configs'][$config]['port'];
1816 $ch = curl_init("localhost:$port/$test");
1817 curl_setopt($ch, CURLOPT_RETURNTRANSFER
, true);
1818 curl_setopt($ch, CURLOPT_TIMEOUT
, SERVER_TIMEOUT
);
1819 curl_setopt($ch, CURLOPT_BINARYTRANSFER
, true);
1820 $output = curl_exec($ch);
1821 if ($output === false) {
1822 // The server probably crashed so fall back to cli to determine if this was
1823 // the test that caused the crash. Our parent process will see that the
1824 // server died and restart it.
1825 if (getenv('HHVM_TEST_SERVER_LOG')) {
1826 printf("Curl failed: %d\n", curl_errno($ch));
1831 $output = trim($output);
1833 return array($output, '');
1836 function run_config_cli($options, $test, $cmd, $cmd_env) {
1837 if (isset($options['log']) && !isset($options['typechecker'])) {
1838 $cmd_env['TRACE'] = 'printir:1';
1839 $cmd_env['HPHP_TRACE_FILE'] = $test . '.log';
1842 $descriptorspec = array(
1843 0 => array("pipe", "r"),
1844 1 => array("pipe", "w"),
1845 2 => array("pipe", "w"),
1848 if (isset($options['typechecker'])) {
1849 $process = proc_open(
1850 "$cmd 2>/dev/null", $descriptorspec, $pipes, null, $cmd_env
1853 $process = proc_open("$cmd 2>&1", $descriptorspec, $pipes, null, $cmd_env);
1855 if (!is_resource($process)) {
1856 file_put_contents("$test.diff", "Couldn't invoke $cmd");
1861 $output = stream_get_contents($pipes[1]);
1862 $output = trim($output);
1863 $stderr = stream_get_contents($pipes[2]);
1866 proc_close($process);
1868 return array($output, $stderr);
1871 function replace_object_ids($str, $replacement) {
1872 return preg_replace(
1873 '/(object\([a-zA-Z_\\\\]+\)#)\d+/', '\1'.$replacement, $str
1877 function run_config_post($outputs, $test, $options) {
1878 $output = $outputs[0];
1879 $stderr = $outputs[1];
1880 file_put_contents("$test.out", $output);
1882 // hhvm redirects errors to stdout, so anything on stderr is really bad.
1886 "Test failed because the process wrote on stderr:\n$stderr"
1891 // Needed for testing non-hhvm binaries that don't actually run the code
1892 // e.g. parser/test/parse_tester.cpp.
1893 if ($output == "FORCE PASS") {
1899 if (isset($options['relocate'])) {
1900 $repeats = $options['relocate'] * 2;
1903 if (isset($options['recycle-tc'])) {
1904 $repeats = $options['recycle-tc'];
1907 list($file, $type) = get_expect_file_and_type($test, $options);
1908 if ($file === null ||
$type === null) {
1910 "$test.diff", "No $test.expect, $test.expectf, " .
1911 "$test.hhvm.expect, $test.hhvm.expectf, " .
1912 "$test.typechecker.expect, $test.typechecker.expectf, " .
1913 "nor $test.expectregex. If $test is meant to be included by other ".
1914 "tests, use a different file extension.\n"
1919 $is_tc = isset($options['typechecker']);
1920 if ((!$is_tc && ($type === 'expect' ||
$type === 'hhvm.expect')) ||
1921 ($is_tc && $type === 'typechecker.expect')) {
1922 $wanted = trim(file_get_contents($file));
1923 if (isset($options['ignore-oids'])) {
1924 $output = replace_object_ids($output, 'n');
1925 $wanted = replace_object_ids($wanted, 'n');
1929 $passed = !strcmp($output, $wanted);
1931 file_put_contents("$test.diff", generate_diff($wanted, null, $output));
1935 $wanted_re = preg_quote($wanted);
1936 } else if ((!$is_tc && ($type === 'expectf' ||
$type === 'hhvm.expectf')) ||
1937 ($is_tc && $type === 'typechecker.expectf')) {
1938 $wanted = trim(file_get_contents($file));
1939 if (isset($options['ignore-oids'])) {
1940 $wanted = replace_object_ids($wanted, '%d');
1942 $wanted_re = $wanted;
1944 // do preg_quote, but miss out any %r delimited sections.
1948 $length = strlen($wanted_re);
1949 while ($startOffset < $length) {
1950 $start = strpos($wanted_re, $r, $startOffset);
1951 if ($start !== false) {
1952 // we have found a start tag.
1953 $end = strpos($wanted_re, $r, $start+
2);
1954 if ($end === false) {
1955 // unbalanced tag, ignore it.
1956 $end = $start = $length;
1959 // no more %r sections.
1960 $start = $end = $length;
1962 // quote a non re portion of the string.
1963 $temp = $temp.preg_quote(substr($wanted_re, $startOffset,
1964 ($start - $startOffset)), '/');
1965 // add the re unquoted.
1966 if ($end > $start) {
1967 $temp = $temp.'('.substr($wanted_re, $start+
2, ($end - $start-2)).')';
1969 $startOffset = $end +
2;
1973 $wanted_re = str_replace(
1974 array('%binary_string_optional%'),
1978 $wanted_re = str_replace(
1979 array('%unicode_string_optional%'),
1983 $wanted_re = str_replace(
1984 array('%unicode\|string%', '%string\|unicode%'),
1988 $wanted_re = str_replace(
1989 array('%u\|b%', '%b\|u%'),
1994 $wanted_re = str_replace('%e', '\\' . DIRECTORY_SEPARATOR
, $wanted_re);
1995 $wanted_re = str_replace('%s', '[^\r\n]+', $wanted_re);
1996 $wanted_re = str_replace('%S', '[^\r\n]*', $wanted_re);
1997 $wanted_re = str_replace('%a', '.+', $wanted_re);
1998 $wanted_re = str_replace('%A', '.*', $wanted_re);
1999 $wanted_re = str_replace('%w', '\s*', $wanted_re);
2000 $wanted_re = str_replace('%i', '[+-]?\d+', $wanted_re);
2001 $wanted_re = str_replace('%d', '\d+', $wanted_re);
2002 $wanted_re = str_replace('%x', '[0-9a-fA-F]+', $wanted_re);
2003 // %f allows two points "-.0.0" but that is the best *simple* expression.
2004 $wanted_re = str_replace('%f', '[+-]?\.?\d+\.?\d*(?:[Ee][+-]?\d+)?',
2006 $wanted_re = str_replace('%c', '.', $wanted_re);
2008 $wanted_re = str_replace('%%', '%%?', $wanted_re);
2010 // Normalize newlines.
2011 $wanted_re = preg_replace("/(\r\n?|\n)/", "\n", $wanted_re);
2012 $output = preg_replace("/(\r\n?|\n)/", "\n", $output);
2013 } else if (!$is_tc && $type === 'expectregex') {
2014 $wanted_re = trim(file_get_contents($file));
2016 throw new Exception("Unsupported expect file type: ".$type);
2020 $wanted_re = "($wanted_re\s*)".'{'.$repeats.'}';
2022 if (!isset($wanted)) $wanted = $wanted_re;
2023 $passed = @preg_match
("/^$wanted_re\$/s", $output);
2024 if ($passed === false && $repeats) {
2025 // $repeats can cause the regex to become too big, and fail
2030 $diff = generate_diff($wanted_re, $wanted_re, $output);
2031 if ($passed === false && $diff === "") {
2032 // the preg match failed, probably because the regex was too complex,
2033 // but since the line by line diff came up empty, we're fine
2036 file_put_contents("$test.diff", $diff);
2042 function timeout_prefix() {
2043 if (is_executable('/usr/bin/timeout')) {
2044 return '/usr/bin/timeout 300 ';
2046 return __DIR__
.'/../tools/timeout.sh -t 300 ';
2050 function run_one_config($options, $test, $cmd, $cmd_env) {
2051 if (is_array($cmd)) {
2053 foreach ($cmd as $c) {
2054 $result = run_one_config($options, $test, $c, $cmd_env);
2055 if (!$result) return $result;
2059 $cmd = timeout_prefix() . $cmd;
2060 $outputs = run_config_cli($options, $test, $cmd, $cmd_env);
2061 if ($outputs === false) return false;
2062 return run_config_post($outputs, $test, $options);
2065 function run_and_lock_test($options, $test) {
2066 $lock = fopen($test, 'r');
2067 if (!flock($lock, LOCK_EX
)) return false;
2068 if (isset($options['typechecker'])) {
2069 $result = run_typechecker_test($options, $test);
2071 $result = run_test($options, $test);
2073 if (!flock($lock, LOCK_UN
)) return false;
2078 function run_typechecker_test($options, $test) {
2079 if (skip_test($options, $test)) return 'skip';
2080 if (!file_exists($test . ".hhconfig")) return 'skip';
2081 list($hh_server, $hh_server_env, $temp_dir) = hh_server_cmd($options, $test);
2082 $result = run_one_config($options, $test, $hh_server, $hh_server_env);
2083 // Remove the temporary directory.
2084 if (!isset($options['no-clean'])) {
2085 shell_exec('rm -rf ' . $temp_dir);
2090 function get_hhvm_ini_values($test, $options) {
2091 // prepare script to setting values from HHVM
2092 $get_ini_file = tempnam(sys_get_temp_dir(), "get_ini");
2093 $tmp_file = tempnam(sys_get_temp_dir(), "output");
2094 // write JSON encoded values from the script
2095 // instead of just 'echo' to make sure result
2096 // is not mixed with some another writes to stdout
2097 file_put_contents($get_ini_file, "<?php\nfile_put_contents('$tmp_file', json_encode(ini_get_all()));");
2098 list($hhvm_get_ini_cmd, $_) = hhvm_cmd($options, $test, $get_ini_file, true);
2099 system("$hhvm_get_ini_cmd 1> /dev/null 2> /dev/null");
2100 $test_config = file_get_contents($tmp_file);
2102 return $test_config;
2105 function run_test($options, $test) {
2106 if (skip_test($options, $test)) return 'skip';
2108 // Skip pure hhas tests if we're running the Hack compiler
2109 if (isset($options['hackc'])) {
2110 if (substr($test, -5) === ".hhas") return 'skip';
2113 // Skip any tests tht would require HPHPC for HackC-only features
2114 $only_hackc = file_exists($test.'.onlyhackc');
2116 if (!isset($options['hackc']) ||
isset($options['compare-hh-codegen'])) {
2117 return 'skip-onlyhackc';
2121 $test_ext = pathinfo($test, PATHINFO_EXTENSION
);
2122 list($hhvm, $hhvm_env) = hhvm_cmd($options, $test);
2123 if ((isset($options['relocate']) ||
isset($options['recycle-tc'])) &&
2124 preg_grep('/ --count[ =].* --count[ =]/', (array)$hhvm)) {
2128 if(file_exists($test . ".verify") && (isset($options['relocate']) ||
2129 isset($options['recycle-tc']) ||
2130 isset($options['repo']))) {
2131 return 'skip-verify';
2134 if (isset($options['repo'])) {
2135 if (preg_grep('/-m debug/', (array)$hhvm) ||
file_exists($test.'.norepo')) {
2136 return 'skip-norepo';
2139 $hphp_hhvm_repo = "$test.repo/hhvm.hhbc";
2140 $hhbbc_hhvm_repo = "$test.repo/hhvm.hhbbc";
2141 $hphp_hackc_repo = "$test.repo/hackc.hhbc";
2142 $hhbbc_hackc_repo = "$test.repo/hackc.hhbbc";
2143 shell_exec("rm -f \"$hphp_hhvm_repo\" \"$hhbbc_hhvm_repo\" \"$hphp_hackc_repo\" \"$hhbbc_hackc_repo\" ");
2147 if (isset($options['hackc']) ||
2148 isset($options['compare-hh-codegen']) ||
2149 isset($options['run-hh-codegen'])) {
2150 $hphp = hphp_cmd($options, $test, timeout_prefix(), true);
2151 shell_exec("$hphp 2>&1");
2152 if (!isset($options['compare-hh-codegen'])) {
2153 $hhbbc = hhbbc_cmd($options, $test, timeout_prefix(), "hackc");
2154 shell_exec("$hhbbc 2>&1");
2159 if ((!isset($options['hackc']) &&
2160 !isset($options['run-hh-codegen'])) ||
2161 isset($options['compare-hh-codegen'])) {
2162 $hphp = hphp_cmd($options, $test, timeout_prefix(), false);
2163 shell_exec("$hphp 2>&1");
2164 if (!isset($options['compare-hh-codegen'])) {
2165 $hhbbc = hhbbc_cmd($options, $test, timeout_prefix(), "hhvm");
2166 shell_exec("$hhbbc 2>&1");
2170 if (isset($hhbbc)) {
2171 if (isset($options['hhbbc2'])) {
2172 $hhas_temp1 = dump_hhas_to_temp($hhvm, "$test.before");
2173 shell_exec("mv $hhbbc_hhvm_repo $hphp_hhvm_repo");
2174 shell_exec("$hhbbc 2>&1");
2175 $hhas_temp2 = dump_hhas_to_temp($hhvm, "$test.after");
2176 if ($hhas_temp1 === false ||
$hhas_temp2 === false) {
2177 if ($hhas_temp1 != $hhas_temp2) {
2178 shell_exec("cat $hhas_temp1$hhas_temp2 > $test.diff");
2182 $diff = shell_exec("diff $hhas_temp1 $hhas_temp2 | wc -l");
2183 if (trim($diff) != '0') {
2184 shell_exec("diff $hhas_temp1 $hhas_temp2 > $test.diff");
2190 return run_one_config($options, $test, $hhvm, $hhvm_env);
2194 if (file_exists($test.'.onlyrepo')) {
2195 return 'skip-onlyrepo';
2198 if (isset($options['hhas-round-trip'])) {
2199 if (substr($test, -5) === ".hhas") return 'skip';
2200 $hhas_temp = dump_hhas_to_temp($hhvm, $test);
2201 if ($hhas_temp === false) {
2202 $err = "system failed: ".dump_hhas_cmd($hhvm, $test)."\n";
2203 file_put_contents("$test.diff", $err);
2206 list($hhvm, $hhvm_env) = hhvm_cmd($options, $test, $hhas_temp);
2209 // For tests that should run HackC pull ini values from HHVM
2211 // In repo mode, HackC will inherit configs from the hhvm process that invokes
2213 $test_config = null;
2214 if ((isset($options['run-hh-codegen']) ||
2215 isset($options['compare-hh-codegen'])) &&
2216 !isset($options['repo'])
2218 $test_config = get_hhvm_ini_values($test, $options);
2221 // Run the Hack code generator
2222 if (isset($options['run-hh-codegen'])) {
2223 if (substr($test, -5) === ".hhas") return 'skip';
2225 $hhas_temp = dump_hh_codegen($options, $test, $test_config);
2226 if (!$hhas_temp ||
file_get_contents($hhas_temp) === "") {
2227 file_put_contents($test.'.diff', "CODEGEN FAILED: NO OUTPUT");
2230 if (contains_nyi_line($hhas_temp) === true) {
2231 file_put_contents($test.'.diff', "CODEGEN FAILED: NYI");
2234 list($hhvm, $hhvm_env) = hhvm_cmd($options, $hhas_temp);
2237 // Compare the output of Hack code gen with that of HHVM
2238 if (isset($options['compare-hh-codegen'])) {
2239 $diff = $test.'.diff';
2241 if (substr($test, -5) === ".hhas") return 'skip';
2242 $hhas = dump_hhas_to_temp($hhvm, $test);
2243 if (!$hhas) return 'skip';
2244 $hhcg = dump_hh_codegen($options, $test, $test_config);
2245 if (!$hhcg ||
file_get_contents($hhcg) === "") {
2246 file_put_contents($diff, "HACK CODEGEN FAILED");
2249 if (contains_nyi_line($hhcg) === true) {
2250 file_put_contents($test.'.diff', "CODEGEN FAILED: NYI");
2254 if (!isset($options['no-semdiff'])) {
2255 if (file_exists($test.'.nosemdiff')) {
2256 return 'skip-nosemdiff';
2258 // Run semantic diff on outputs first
2259 $semdiff = semdiff_output($options, $test);
2260 // If it fails completely (e.g. timeout or exception) then
2263 // Now we look for "distance = 0" in the file
2264 // If we find it we "succeed" and don't drop through to vanilla diff
2265 $h = fopen($semdiff, "r");
2266 if (($s = fgets($h)) !== false) {
2267 if (strtolower(substr($s, 0, 12)) === "distance = 0") {
2270 unlink($test . '.semdiff_messages');
2274 return false; // semdiff ran OK, returned non-zero distance, report as test failing
2276 fclose($h); // couldn't read a string, so something went wrong with semdiff
2277 rename($semdiff, $test.".semdiff"); // preserve the output as it may be useful?
2280 // so either semdiff went wrong, or we have no-semdiff set. So we try ordinary diffing
2281 strip_hhas_file($hhas);
2282 strip_hhas_file($hhcg);
2284 system("diff -B $hhcg $hhas > $diff", $ret);
2286 // If identical, don't bother dropping through to roundtrip stage
2292 // If it failed, do a round-trip through HHVM and try again
2293 list($hhvm2, $hhvm2_env) = hhvm_cmd($options, $hhcg);
2294 $hhcg2 = dump_hhas_to_temp($hhvm2, $hhcg);
2298 file_get_contents($hhcg2) === "" ||
2299 is_assembler_fail($hhcg2)
2305 strip_hhas_file($hhcg2);
2307 system("diff $hhcg2 $hhas > $diff", $ret);
2309 if ($rv) unlink($diff);
2315 if ($outputs = run_config_server($options, $test)) {
2316 return run_config_post($outputs, $test, $options) ?
'pass-server'
2317 : (run_one_config($options, $test, $hhvm, $hhvm_env) ?
'skip-server'
2320 return run_one_config($options, $test, $hhvm, $hhvm_env);
2323 function num_cpus() {
2326 $data = file('/proc/stat');
2328 foreach($data as $line) {
2329 if (preg_match('/^cpu[0-9]/', $line)) {
2336 return exec('sysctl -n hw.ncpu');
2338 return 2; // default when we don't know how to detect.
2341 function make_header($str) {
2342 return "\n\033[0;33m".$str."\033[0m\n";
2345 function print_commands($tests, $options) {
2346 print make_header("Run these by hand:");
2348 foreach ($tests as $test) {
2349 if (isset($options['typechecker'])) {
2350 list($command, $_, ) = hh_server_cmd($options, $test);
2352 list($command, $_) = hhvm_cmd($options, $test);
2354 if (!isset($options['repo'])) {
2355 foreach ((array)$command as $c) {
2361 // How to run it with hhbbc:
2362 $hhbbc_cmds = hphp_cmd($options, $test, '', isset($options['hackc']))."\n";
2363 $program = isset($options['hackc']) ?
"hackc" : "hhvm";
2364 $hhbbc_cmd = hhbbc_cmd($options, $test, '', $program)."\n";
2365 $hhbbc_cmds .= $hhbbc_cmd;
2366 if (isset($options['hhbbc2'])) {
2367 foreach ((array)$command as $c) {
2368 $hhbbc_cmds .= $c." -m dumphhas > $test.before.round_trip.hhas\n";
2370 $hhbbc_cmds .= "mv $test.repo/$program.hhbbc $test.repo/$program.hhbc\n";
2371 $hhbbc_cmds .= $hhbbc_cmd;
2372 foreach ((array)$command as $c) {
2373 $hhbbc_cmds .= $c." -m dumphhas > $test.after.round_trip.hhas\n";
2376 "diff $test.before.round_trip.hhas $test.after.round_trip.hhas\n";
2378 foreach ((array)$command as $c) {
2379 $hhbbc_cmds .= $c."\n";
2381 print "$hhbbc_cmds\n";
2385 function msg_loop($num_tests, $queue) {
2392 Status
::getMode() === Status
::MODE_NORMAL ||
2393 Status
::getMode() === Status
::MODE_RECORD_FAILURES
2395 Status
::hasCursorControl();
2398 $stty = strtolower(Status
::getSTTY());
2399 preg_match_all("/columns ([0-9]+);/", $stty, $output);
2400 if (!isset($output[1][0])) {
2401 // because BSD has to be different
2402 preg_match_all("/([0-9]+) columns;/", $stty, $output);
2404 if (!isset($output[1][0])) {
2405 $do_progress = false;
2407 $cols = $output[1][0];
2412 if (!msg_receive($queue, 0, $type, 1024, $message)) {
2413 error("msg_receive failed");
2417 case Status
::MSG_STARTED
:
2420 case Status
::MSG_FINISHED
:
2423 case Status
::MSG_SERVER_RESTARTED
:
2424 switch (Status
::getMode()) {
2425 case Status
::MODE_NORMAL
:
2426 if (!Status
::hasCursorControl()) {
2427 Status
::sayColor(Status
::RED
, 'x');
2431 case Status
::MODE_VERBOSE
:
2432 Status
::sayColor("$test ", Status
::YELLOW
, "failed",
2433 " to talk to server\n");
2436 case Status
::MODE_FBMAKE
:
2439 case Status
::MODE_TESTPILOT
:
2442 case Status
::MODE_RECORD_FAILURES
:
2446 case Status
::MSG_TEST_PASS
:
2448 list($test, $how, $time, $stime, $etime) = $message;
2449 switch (Status
::getMode()) {
2450 case Status
::MODE_NORMAL
:
2451 if (!Status
::hasCursorControl()) {
2452 if ($how == Status
::SKIP_SERVER
) {
2453 Status
::sayColor(Status
::RED
, '.');
2455 Status
::sayColor(Status
::GREEN
,
2456 $how == Status
::PASS_SERVER ?
',' : '.');
2461 case Status
::MODE_VERBOSE
:
2462 Status
::sayColor("$test ", Status
::GREEN
,
2463 sprintf("passed (%.2fs)\n", $time));
2466 case Status
::MODE_FBMAKE
:
2467 Status
::sayFBMake($test, 'passed', $stime, $etime);
2470 case Status
::MODE_TESTPILOT
:
2471 Status
::sayFBMake($test, 'passed', $stime, $etime);
2474 case Status
::MODE_RECORD_FAILURES
:
2479 case Status
::MSG_TEST_SKIP
:
2481 list($test, $reason, $time, $stime, $etime) = $message;
2483 switch (Status
::getMode()) {
2484 case Status
::MODE_NORMAL
:
2485 if (!Status
::hasCursorControl()) {
2486 Status
::sayColor(Status
::YELLOW
, 's');
2490 case Status
::MODE_VERBOSE
:
2491 Status
::sayColor("$test ", Status
::YELLOW
, "skipped");
2493 if ($reason !== null) {
2494 Status
::sayColor(" - $reason");
2496 Status
::sayColor(sprintf(" (%.2fs)\n", $time));
2499 case Status
::MODE_FBMAKE
:
2500 /* Intentionally discard result */
2503 case Status
::MODE_TESTPILOT
:
2504 Status
::sayFBMake($test, 'not_relevant', $stime, $etime);
2507 case Status
::MODE_RECORD_FAILURES
:
2512 case Status
::MSG_TEST_FAIL
:
2514 list($test, $time, $stime, $etime) = $message;
2515 switch (Status
::getMode()) {
2516 case Status
::MODE_NORMAL
:
2517 if (Status
::hasCursorControl()) {
2518 print "\033[2K\033[1G";
2520 $diff = (string)@file_get_contents
($test.'.diff');
2521 Status
::sayColor(Status
::RED
, "\nFAILED",
2522 ": $test\n$diff\n");
2525 case Status
::MODE_VERBOSE
:
2526 Status
::sayColor("$test ", Status
::RED
,
2527 sprintf("FAILED (%.2fs)\n", $time));
2530 case Status
::MODE_FBMAKE
:
2531 Status
::sayFBMake($test, 'failed', $stime, $etime);
2534 case Status
::MODE_TESTPILOT
:
2535 Status
::sayFBMake($test, 'failed', $stime, $etime);
2538 case Status
::MODE_RECORD_FAILURES
:
2544 error("Unknown message $type");
2548 $total_run = ($skipped +
$failed +
$passed);
2549 $bar_cols = ($cols - 45);
2551 $passed_ticks = round($bar_cols * ($passed / $num_tests));
2552 $skipped_ticks = round($bar_cols * ($skipped / $num_tests));
2553 $failed_ticks = round($bar_cols * ($failed / $num_tests));
2555 $fill = $bar_cols - ($passed_ticks +
$skipped_ticks +
$failed_ticks);
2556 if ($fill < 0) $fill = 0;
2558 $fill = str_repeat('-', $fill);
2560 $passed_ticks = str_repeat('#', $passed_ticks);
2561 $skipped_ticks = str_repeat('#', $skipped_ticks);
2562 $failed_ticks = str_repeat('#', $failed_ticks);
2564 print "\033[2K\033[1G[".
2565 "\033[0;32m$passed_ticks".
2566 "\033[33m$skipped_ticks".
2567 "\033[31m$failed_ticks".
2568 "\033[0m$fill] ($total_run/$num_tests) ".
2569 "($skipped skipped, $failed failed)";
2574 print "\033[2K\033[1G";
2576 print "$skipped tests \033[1;33mskipped\033[0m\n";
2581 function print_success($tests, $results, $options) {
2582 // We didn't run any tests, not even skipped. Clowntown!
2584 print "\nCLOWNTOWN: No tests!\n";
2585 if (empty($options['no-fun'])) {
2601 /* Emacs' syntax highlighting gets confused by that clown and this comment
2602 * resets whatever state got messed up. */
2606 foreach ($results as $result) {
2607 // The result here will either be skipped or passed (since failed is
2608 // handled in print_failure.
2609 if ($result['status'] == 'passed') {
2614 // We just had skipped tests
2616 print "\nSKIP-ALOO: Only skipped tests!\n";
2617 if (empty($options['no-fun'])) {
2641 /* Emacs' syntax highlighting may get confused by the skipper and this
2642 * rcomment esets whatever state got messed up. */
2645 print "\nAll tests passed.\n";
2646 if (empty($options['no-fun'])) {
2652 _____|____|____|____\\\__
2653 ---------\ SHIP IT
/---------
2654 ^^^^^ ^^^^^^^^^^^^^^^^^^^^^
2660 if (isset($options['verbose'])) {
2661 print_commands($tests, $options);
2665 function print_failure($argv, $results, $options) {
2668 foreach ($results as $result) {
2669 if ($result['status'] === 'failed') {
2670 $failed[] = $result['name'];
2672 if ($result['status'] === 'passed') {
2673 $passed[] = $result['name'];
2677 print "\n".count($failed)." tests failed\n";
2678 if (empty($options['no-fun'])) {
2679 print "(╯°□°)╯︵ ┻━┻\n";
2680 // TODO: Google indicates that this is some old emoji-thing relating to
2681 // table flipping... Maybe replace to stop other people spending time
2682 // trying to decipher it?
2685 print make_header("See the diffs:").
2686 implode("\n", array_map(
2687 function($test) { return 'cat '.$test.'.diff'; },
2690 $failing_tests_file = !empty($options['failure-file'])
2691 ?
$options['failure-file']
2692 : tempnam('/tmp', 'test-failures');
2693 file_put_contents($failing_tests_file, implode("\n", $failed)."\n");
2694 print make_header('For xargs, list of failures is available using:').
2695 'cat '.$failing_tests_file."\n";
2697 if (!empty($passed)) {
2698 $passing_tests_file = !empty($options['success-file'])
2699 ?
$options['success-file']
2700 : tempnam('/tmp', 'tests-passed');
2701 file_put_contents($passing_tests_file, implode("\n", $passed)."\n");
2702 print make_header('For xargs, list of passed tests is available using:').
2703 'cat '.$passing_tests_file."\n";
2706 print_commands($failed, $options);
2708 $rerun = make_header("Re-run just the failing tests:") . $argv[0];
2709 foreach ($options as $option => $value) {
2710 if ($option === "servers") continue;
2711 if ($option === "threads") {
2712 // e.g., For a small number of failed tests, we don't need max threads.
2713 $rerun .= " --" . $option . ' ' . get_num_threads($options, $failed);
2714 } else if ($option === "typechecker" ||
2715 $option === "verbose" ||
2716 $option === "repo" ||
2717 $option === "server" ||
2718 $option === "cli-server" ||
2719 $option === "compare-hh-codegen" ||
2720 $option === "no-semdiff") {
2721 // The escapeshellarg($value) of these is 1, but there is no real value
2722 // associated with these options.
2723 $rerun .= " --" . $option;
2725 $rerun .= " --" . $option . ' ' . escapeshellarg($value);
2728 $rerun .= ' ' . implode(' ', array_map('escapeshellarg', $failed)) . "\n";
2732 function port_is_listening($port) {
2733 $socket = socket_create(AF_INET
, SOCK_STREAM
, SOL_TCP
);
2734 return @socket_connect
($socket, 'localhost', $port);
2737 function find_open_port() {
2738 for ($i = 0; $i < 50; ++
$i) {
2739 $port = rand(1024, 65535);
2740 if (!port_is_listening($port)) return $port;
2743 error("Couldn't find an open port");
2746 function start_server_proc($options, $config, $port) {
2747 if (isset($options['cli-server'])) {
2748 $cli_sock = tempnam(sys_get_temp_dir(), 'hhvm-cli-');
2750 // still want to test that an unwritable socket works...
2751 $cli_sock = '/var/run/hhvm-cli.sock';
2753 $threads = $options['threads'];
2754 $thread_option = isset($options['cli-server'])
2755 ?
'-vEval.UnixServerWorkers='.$threads
2756 : '-vServer.ThreadCount='.$threads;
2757 $command = hhvm_cmd_impl(
2759 true, /*$disable_hphpc_opts*/
2762 "-vServer.Port=$port",
2763 "-vServer.Type=proxygen",
2764 "-vAdminServer.Port=0",
2766 '-vServer.ExitOnBindFail=1',
2767 '-vServer.RequestTimeoutSeconds='.SERVER_TIMEOUT
,
2768 '-vPageletServer.ThreadCount=0',
2769 '-vLog.UseRequestLog=1',
2770 '-vLog.File=/dev/null',
2772 // The server will unlink the temp file
2773 '-vEval.UnixServerPath='.$cli_sock,
2775 // This ensures we actually jit everything:
2776 '-vEval.JitRequireWriteLease=1',
2778 // The default test config uses a small TC but we'll be running thousands
2779 // of tests against the same process:
2780 '-vEval.JitASize=100000000',
2781 '-vEval.JitGlobalDataSize=32000000',
2783 // load/store counters don't work on Ivy Bridge so disable for tests
2784 '-vEval.ProfileHWEnable=false'
2786 if (is_array($command)) {
2787 error("Can't run multi-mode tests in server mode");
2789 if (getenv('HHVM_TEST_SERVER_LOG')) {
2790 echo "Starting server '$command'\n";
2793 $descriptors = array(
2794 0 => array('file', '/dev/null', 'r'),
2795 1 => array('file', '/dev/null', 'w'),
2796 2 => array('file', '/dev/null', 'w'),
2799 $proc = proc_open($command, $descriptors, $dummy);
2801 error("Failed to start server process");
2803 $status = proc_get_status($proc);
2804 $status['proc'] = $proc;
2805 $status['port'] = $port;
2806 $status['config'] = $config;
2807 $status['cli-socket'] = $cli_sock;
2812 * For each config file in $configs, start up a server on a randomly-determined
2813 * port. Return value is an array mapping pids and config files to arrays of
2814 * information about the server.
2816 function start_servers($options, $configs) {
2817 $starting = array();
2818 foreach ($configs as $config) {
2819 $starting[] = start_server_proc($options, $config, find_open_port());
2822 $start_time = microtime(true);
2823 $servers = array('pids' => array(), 'configs' => array());
2825 // Wait for all servers to come up.
2826 while (count($starting) > 0) {
2827 $still_starting = array();
2829 foreach ($starting as $server) {
2830 $new_status = proc_get_status($server['proc']);
2832 if (!$new_status['running']) {
2833 if ($new_status['exitcode'] === 0) {
2834 error("Server exited prematurely but without error");
2837 // We lost a race. Try another port.
2838 if (getenv('HHVM_TEST_SERVER_LOG')) {
2839 echo "\n\nLost connection race on port $port. Trying another.\n\n";
2842 start_server_proc($options, $server['config'], find_open_port());
2843 } else if (!port_is_listening($server['port'])) {
2844 $still_starting[] = $server;
2846 $servers['pids'][$server['pid']] =& $server;
2847 $servers['configs'][$server['config']] =& $server;
2852 $starting = $still_starting;
2854 if (microtime(true) - $start_time > $max_time) {
2855 error("Servers took more than $max_time seconds to come up");
2858 // Take a short nap and try again.
2862 $elapsed = microtime(true) - $start_time;
2863 printf("Started %d servers in %.1f seconds\n\n", count($configs), $elapsed);
2867 function drain_queue($queue) {
2868 while (@msg_receive
($queue, 0, $type, 1024, $message, true,
2869 MSG_IPC_NOWAIT | MSG_NOERROR
));
2872 function get_num_threads($options, $tests) {
2873 $cpus = isset($options['server']) ||
isset($options['cli-server'])
2874 ?
num_cpus() * 2 : num_cpus();
2875 return min(count($tests), idx($options, 'threads', $cpus));
2878 function runner_precheck() {
2879 // basic checking for runner.
2880 if (empty($_SERVER) ||
empty($_ENV)) {
2881 echo "Warning: \$_SERVER/\$_ENV variables not available, please check \n" .
2882 "your ini setting: variables_order, it should have both 'E' and 'S'\n";
2886 function main($argv) {
2889 ini_set('pcre.backtrack_limit', PHP_INT_MAX
);
2891 list($options, $files) = get_options($argv);
2892 if (isset($options['help'])) {
2895 if (isset($options['list-tests'])) {
2896 error(list_tests($files, $options));
2899 $tests = find_tests($files, $options);
2900 if (isset($options['shuffle'])) {
2904 if (isset($options['repo']) && isset($options['typechecker'])) {
2905 error("Repo mode and typechecker mode are not compatible");
2908 if (isset($options['hhvm-binary-path']) &&
2909 isset($options['typechecker'])) {
2910 error("Did you mean to set the hh_server binary path instead?");
2913 if (isset($options['hhserver-binary-path']) &&
2914 !isset($options['typechecker'])) {
2915 error("hh_server binary path set, but not --typechecker");
2918 if (isset($options['hhvm-binary-path']) &&
2919 isset($options['hhserver-binary-path'])) {
2920 error("Need to choose one of the two binaries to run");
2924 $typechecker = false;
2925 if (isset($options['hhvm-binary-path'])) {
2926 check_executable($options['hhvm-binary-path'], false);
2927 $binary_path = realpath($options['hhvm-binary-path']);
2928 putenv("HHVM_BIN=" . $binary_path);
2929 } else if (isset($options['hhserver-binary-path'])) {
2930 check_executable($options['hhserver-binary-path'], true);
2931 $binary_path = realpath($options['hhserver-binary-path']);
2932 $typechecker = true;
2933 putenv("HH_SERVER_BIN=" . $binary_path);
2934 } else if (isset($options['typechecker'])) {
2935 $typechecker = true;
2938 // Explicit path given by --hhvm-binary-path or --hhserver-binary-path
2939 // takes priority (see above)
2940 // Then, if an HHVM_BIN or HH_SERVER env var exists, and the file it
2941 // points to exists, that trumps any default hhvm / typechecker executable
2943 if ($binary_path === "") {
2944 if (!$typechecker) {
2945 if (getenv("HHVM_BIN") !== false) {
2946 $binary_path = realpath(getenv("HHVM_BIN"));
2947 check_executable($binary_path, false);
2949 check_for_multiple_default_binaries(false);
2950 $binary_path = hhvm_path();
2953 if (getenv("HH_SERVER_BIN") !== false) {
2954 $binary_path = realpath(getenv("HH_SERVER_BIN"));
2955 check_executable($binary_path, true);
2957 check_for_multiple_default_binaries(true);
2958 $binary_path = hh_server_path();
2963 if (isset($options['verbose'])) {
2964 print "You are using the binary located at: " . $binary_path . "\n";
2967 $options['threads'] = get_num_threads($options, $tests);
2970 if (isset($options['server']) ||
isset($options['cli-server'])) {
2971 if (isset($options['server']) && isset($options['cli-server'])) {
2972 error("Server mode and CLI Server mode are mutually exclusive");
2974 if (isset($options['repo']) ||
isset($options['typechecker'])) {
2975 error("Server mode repo tests are not supported");
2979 /* We need to start up a separate server process for each config file
2981 foreach ($tests as $test) {
2982 if (!can_run_server_test($test)) continue;
2983 $config = find_file_for_dir(dirname($test), 'config.ini');
2985 error("Couldn't find config file for $test");
2987 $configs[$config] = $config;
2991 if (count($configs) > $max_configs) {
2992 error("More than $max_configs unique config files will be needed to run ".
2993 "the tests you specified. They may not be a good fit for server ".
2997 $servers = $options['servers'] = start_servers($options, $configs);
3000 // Try to construct the buckets so the test results are ready in
3001 // approximately alphabetical order.
3002 $test_buckets = array();
3005 // Get the serial tests to be in their own bucket later.
3006 $serial_tests = serial_only_tests($tests);
3008 // If we have no serial tests, we can use the maximum number of allowed
3009 // threads for the test running. If we have some, we save one thread for
3010 // the serial bucket.
3011 $parallel_threads = count($serial_tests) > 0
3012 ?
$options['threads'] - 1
3013 : $options['threads'];
3015 foreach ($tests as $test) {
3016 if (!in_array($test, $serial_tests)) {
3017 $test_buckets[$i][] = $test;
3018 $i = ($i +
1) %
$parallel_threads;
3022 if (count($serial_tests) > 0) {
3023 // The last bucket is serial.
3024 // If the number of parallel tests didn't equal the actual number of
3025 // parallel threads because the number of serial tests reduced it enough,
3026 // then our next bucket is just $i; otherwise it is final available
3027 // thread. For example, we have 13 total tests which initially gave us
3028 // 13 parallel threads, but then we find out 3 are serial, so the parallel
3029 // tests would only fill 9 parallel buckets (< 12). The next one would be
3030 // 10 for the 3 serial. Now if we have 40 total tests which gave us 32
3031 // parallel threads and 4 serial tests, then all of possible parallel
3032 // buckets (31) would be filled regardless; so the serial bucket is what
3033 // would have been the last parallel thread (32).
3034 // $i got bumped to the next bucket at the end of the parallel test loop
3035 // above, so no $i++ here.
3036 $i = count($tests) - count($serial_tests) < $parallel_threads
3037 ?
$i // we didn't fill all the parallel buckets, so use next one in line
3038 : $options['threads'] - 1; // all parallel filled; last thread; 0 indexed
3039 foreach ($serial_tests as $test) {
3040 $test_buckets[$i][] = $test;
3044 // If our total number of test buckets didn't overflow back to 0 above
3045 // when we % against the number of threads (because we didn't have that
3046 // many tests for this run), then just set the threads to how many
3047 // buckets we actually have to make calculations below correct.
3048 if (count($test_buckets) < $options['threads']) {
3049 $options['threads'] = count($test_buckets);
3052 // Remember that the serial tests are also in the tests array too,
3053 // so they are part of the total count.
3054 if (!isset($options['fbmake']) && !isset($options['testpilot'])) {
3055 print "Running ".count($tests)." tests in ".
3056 $options['threads']." threads (" . count($serial_tests) .
3060 if (isset($options['verbose'])) {
3061 Status
::setMode(Status
::MODE_VERBOSE
);
3063 if (isset($options['fbmake'])) {
3064 Status
::setMode(Status
::MODE_FBMAKE
);
3066 if (isset($options['testpilot'])) {
3067 Status
::setMode(Status
::MODE_TESTPILOT
);
3069 if (isset($options['record-failures'])) {
3070 Status
::setMode(Status
::MODE_RECORD_FAILURES
);
3072 Status
::setUseColor(isset($options['color']) ?
true : posix_isatty(STDOUT
));
3074 Status
::$key = rand();
3075 $queue = Status
::getQueue();
3076 drain_queue($queue);
3079 // Spawn off worker threads.
3080 $children = array();
3081 // A poor man's shared memory.
3082 $bad_test_files = array();
3083 for ($i = 0; $i < $options['threads']; $i++
) {
3084 $bad_test_file = tempnam('/tmp', 'test-run-');
3085 $bad_test_files[] = $bad_test_file;
3086 $pid = pcntl_fork();
3088 error('could not fork');
3090 $children[$pid] = $pid;
3092 exit(run($options, $test_buckets[$i], $bad_test_file));
3096 // Fork off a child to receive messages and print status, and have the parent
3097 // wait for all children to exit.
3098 $printer_pid = pcntl_fork();
3099 if ($printer_pid == -1) {
3100 error("failed to fork");
3101 } else if ($printer_pid == 0) {
3102 msg_loop(count($tests), $queue);
3106 // In case we exit in a crazy way, have the parent blow up the queue.
3107 // Do this here so no children inherit this.
3108 $kill_queue = function() { Status
::killQueue(); };
3109 register_shutdown_function($kill_queue);
3110 pcntl_signal(SIGTERM
, $kill_queue);
3111 pcntl_signal(SIGINT
, $kill_queue);
3114 while (count($children) && $printer_pid != 0) {
3115 $pid = pcntl_wait($status);
3116 if (!pcntl_wifexited($status) && !pcntl_wifsignaled($status)) {
3117 error("Unexpected exit status from child");
3120 if ($pid == $printer_pid) {
3121 // We should be finishing up soon.
3123 } else if (isset($servers['pids'][$pid])) {
3124 // A server crashed. Restart it.
3125 if (getenv('HHVM_TEST_SERVER_LOG')) {
3126 echo "\nServer $pid crashed. Restarting.\n";
3128 Status
::serverRestarted();
3129 $server =& $servers['pids'][$pid];
3130 $server = start_server_proc($options, $server['config'], $server['port']);
3132 // Unset the old $pid entry and insert the new one.
3133 unset($servers['pids'][$pid]);
3134 $servers['pids'][$server['pid']] =& $server;
3136 } elseif (isset($children[$pid])) {
3137 unset($children[$pid]);
3138 $return_value |
= pcntl_wexitstatus($status);
3139 } // Else, ignorable signal
3146 foreach ($servers['pids'] as $server) {
3147 proc_terminate($server['proc']);
3148 proc_close($server['proc']);
3153 foreach ($bad_test_files as $bad_test_file) {
3154 $json = json_decode(file_get_contents($bad_test_file), true);
3155 if (!is_array($json)) {
3157 "\nNo JSON output was received from a test thread. ".
3158 "Either you killed it, or it might be a bug in the test script."
3161 $results = array_merge($results, $json);
3162 unlink($bad_test_file);
3165 if (isset($options['record-failures'])) {
3166 $fail_file = $options['record-failures'];
3167 $failed_tests = array();
3168 $prev_failing = array();
3169 if (file_exists($fail_file)) {
3170 $prev_failing = explode("\n", file_get_contents($fail_file));
3175 foreach ($results as $r) {
3176 if (!isset($r['name']) ||
!isset($r['status'])) continue;
3177 $test = canonical_path($r['name']);
3178 $status = $r['status'];
3179 if ($status === 'passed' && in_array($test, $prev_failing)) {
3183 if ($status !== 'failed') continue;
3184 if (!in_array($test, $prev_failing)) $new_fails++
;
3185 $failed_tests[] = $test;
3188 "Recording %d tests as failing.\n".
3189 "There are %d new failing tests, and %d new passing tests.\n",
3190 count($failed_tests), $new_fails, $new_passes
3192 sort($failed_tests);
3193 file_put_contents($fail_file, implode("\n", $failed_tests));
3194 } else if (isset($options['fbmake']) ||
isset($options['testpilot'])) {
3195 Status
::say(array('op' => 'all_done', 'results' => $results));
3196 } else if (!$return_value) {
3197 print_success($tests, $results, $options);
3199 print_failure($argv, $results, $options);
3202 if (!isset($options['fbmake'])) {
3203 Status
::sayColor("\nTotal time for all executed tests as run: ",
3206 Status
::getOverallEndTime() -
3207 Status
::getOverallStartTime()));
3208 Status
::sayColor("Total time for all executed tests if run serially: ",
3211 Status
::addTestTimesSerial()));
3214 return $return_value;