hphp/tools/import_zend_test.py

   1 #!/usr/bin/env python
   2
   3 """
   4 Copies all the Zend tests to a temporary directory, runs them in interp mode,
   5 then copies the good ones to test/zend/good and the bad ones to test/zend/bad.
   6 """
   7
   8 import argparse
   9 import glob
  10 import json
  11 import os
  12 import re
  13 import shutil
  14 import subprocess
  15 import sys
  16
  17 bad_tests = (
  18     # SESSION is bused on husdon
  19     'unset_cv05.php',
  20     'unset_cv06.php',
  21
  22     # unpredictable numbers - we need param matching
  23     'bug29971.php',
  24     'bug35143.php',
  25     'gettimeofday_basic.php',
  26     'localtime_basic.php',
  27     'time_basic.php',
  28     'posix_getpgid_basic.php',
  29     'posix_getpgid_basic.php',
  30     'posix_getpgrp_basic.php',
  31     'posix_getpid_basic.php',
  32     'posix_getppid_basic.php',
  33     'posix_getsid_basic.php',
  34     'posix_times_basic.php',
  35     'socket_getsockname.php',
  36     'fileinode_variation1.php',
  37     'filestat.php',
  38     'fstat_basic.php',
  39     'fstat.php',
  40     'touch_basic.php',
  41     'openssl_random_pseudo_bytes.php',
  42     'pcntl_fork_basic.php',
  43     'posix_times.php',
  44     'array_diff_assoc_variation6.php',
  45     'bug39322.php',
  46     'getmypid_basic.php',
  47     'getrusage_basic.php',
  48
  49     # not implemented extensions
  50     'phar', # this appears in filenames
  51
  52     # works in interp but not others
  53     'bug25922.php',
  54     'bug34064.php',
  55     'objects_029.php',
  56     'objects_030.php',
  57
  58     # line number is inconsistent on stack overflow
  59     'bug41633_3.php',
  60
  61     # broken in Jenkins
  62     'bug35239.php',
  63     'bug54265.php',
  64 )
  65
  66 no_import = (
  67     # these hang forever
  68     '005a.phpt',
  69     'array_012.phpt',
  70     'array_pad_variation2.phpt',
  71     'bug27508.phpt',
  72     'gzgetc_basic.phpt',
  73     'gzgets_basic.phpt',
  74     'observer_003.phpt',
  75     'observer_004.phpt',
  76     'observer_005.phpt',
  77     'observer_006.phpt',
  78     'observer_009.phpt',
  79     'sleep_error.phpt',
  80     'socket_select-wrongparams-1.phpt',
  81     'test010.phpt',
  82     'usleep_error.phpt',
  83
  84     # segfaults
  85     'bz2/tests/004.phpt',
  86     'date/tests/bug50055.phpt',
  87     'operators/divide_basiclong_64bit.phpt',
  88     'operators/modulus_basiclong_64bit.phpt',
  89     'lang/bug21820.phpt',
  90     'lang/func_get_arg.003.phpt',
  91     'lang/func_num_args.003.phpt',
  92     'lang/func_get_args.003.phpt',
  93     'func/010.phpt',
  94     'Zend/tests/020.phpt',
  95     'Zend/tests/bug35239.phpt',
  96     'Zend/tests/bug54265.phpt',
  97     'Zend/tests/bug55705.phpt',
  98     'Zend/tests/callable_type_hint_001.phpt',
  99     'Zend/tests/callable_type_hint_003.phpt',
 100     'Zend/tests/jump13.phpt',
 101     'Zend/tests/heredoc_005.phpt',
 102     'gd/tests/crafted_gd2.phpt',
 103     'pcntl/tests/pcntl_exec.phpt',
 104     'pcntl/tests/pcntl_exec_2.phpt',
 105     'pcntl/tests/pcntl_exec_3.phpt',
 106     'session/tests/bug61728.phpt',
 107     'session/tests/session_module_name_variation2.phpt',
 108
 109     # not imported yet, but will be
 110     '/ext/gd',
 111     '/ext/standard',
 112     '/ext/mysql',
 113     '/ext/pdo_mysql',
 114     '/ext/pdo_sqlite',
 115     '/ext/pgsql',
 116     '/ext/posix',
 117     '/ext/soap',
 118     '/ext/spl',
 119     '/ext/sqlite3',
 120     '/ext/xml',
 121     '/ext/xmlreader',
 122     '/ext/xmlwriter',
 123     '/ext/zlib',
 124
 125     # not implemented extensions
 126     '/sapi',
 127     '/ext/calendar',
 128     '/ext/com_dotnet',
 129     '/ext/dba',
 130     '/ext/dom',
 131     '/ext/enchant',
 132     '/ext/ereg',
 133     '/ext/fileinfo',
 134     '/ext/filter',
 135     '/ext/ftp',
 136     '/ext/gettext',
 137     '/ext/gmp',
 138     '/ext/interbase',
 139     '/ext/mssql',
 140     '/ext/mysqli',
 141     '/ext/mysqlnd',
 142     '/ext/oci8',
 143     '/ext/odbc',
 144     '/ext/pdo_dblib',
 145     '/ext/pdo_firebird',
 146     '/ext/pdo_odbc',
 147     '/ext/pdo_pgsql',
 148     '/ext/pdo_oci',
 149     '/ext/phar',
 150     '/ext/pspell',
 151     '/ext/readline',
 152     '/ext/recode',
 153     '/ext/reflection',
 154     '/ext/shmop',
 155     '/ext/skeleton',
 156     '/ext/snmp',
 157     '/ext/sybase_ct',
 158     '/ext/sysvmsg',
 159     '/ext/sysvsem',
 160     '/ext/sysvshm',
 161     '/ext/tidy',
 162     '/ext/tokenizer',
 163     '/ext/wddx',
 164     '/ext/xmlrpc',
 165     '/ext/xsl',
 166     '/ext/zip',
 167 )
 168
 169 # Random other files that zend wants
 170 other_files = (
 171     'curl_testdata1.txt',
 172     'curl_testdata2.txt',
 173     'autoload_root.p5c',
 174     'autoload_derived.p5c',
 175     'autoload_implements.p5c',
 176     'autoload_interface.p5c',
 177     'constants_basic_003.inc',
 178     'interface_optional_arg_003.inc',
 179     '015.inc',
 180     '016.inc',
 181     '023-2.inc',
 182     'inc.inc',
 183     'inc_throw.inc',
 184     'bug54804.inc',
 185     'nowdoc.inc',
 186     'ns_022.inc',
 187     'ns_027.inc',
 188     'ns_028.inc',
 189     'ns_065.inc',
 190     'ns_066.inc',
 191     'ns_067.inc',
 192     'unset.inc',
 193     '/ext-exif/bug48378.jpeg',
 194     '/ext-gd/Tuffy.ttf',
 195     '/ext-gd/bug37346.gif',
 196     '/ext-gd/bug38112.gif',
 197     '/ext-gd/bug43121.gif',
 198     '/ext-gd/conv_test.gif',
 199     '/ext-gd/conv_test.jpeg',
 200     '/ext-gd/conv_test.png',
 201     '/ext-gd/conv_test.xbm',
 202     '/ext-gd/php.gif',
 203     '/ext-gd/src.gd2',
 204     '/ext-gd/src.wbmp',
 205     '/ext-gd/test8859.ttf',
 206     '/ext-gd/test_gif.gif',
 207     '/ext-intl/ut_common.inc',
 208     '/ext-ldap/connect.inc',
 209     '/ext-mbstring/common.inc',
 210     '/ext-mcrypt/vectors.txt',
 211     '/ext-openssl/005_crt.txt',
 212     '/ext-openssl/bug37820cert.pem',
 213     '/ext-openssl/bug37820key.pem',
 214     '/ext-openssl/bug39217cert1.txt',
 215     '/ext-openssl/bug39217cert2.txt',
 216     '/ext-openssl/cert.crt',
 217     '/ext-openssl/openssl.cnf',
 218     '/ext-openssl/private.key',
 219     '/ext-openssl/public.key',
 220     '/ext-session/save_handler.inc',
 221     '/ext-simplexml/bug24392.xml',
 222     '/tests/quicktester.inc',
 223 )
 224
 225 errors = (
 226     # generic inconsistencies
 227     ('Variable passed to ([^\s]+)\(\) is not an array or object', 'Invalid operand type was used: expecting an array'),
 228     ('bcdiv\(\): ', ''),
 229     ('bcsqrt\(\): ', ''),
 230     ('bcpowmod\(\): ', ''),
 231 )
 232
 233 parser = argparse.ArgumentParser()
 234 parser.add_argument(
 235     "-z",
 236     "--zend_path",
 237     type=str,
 238     help="zend path to import tests from."
 239 )
 240 parser.add_argument(
 241     "-o",
 242     "--only",
 243     type=str,
 244     action='append',
 245     help="only import tests whose path matches this regex."
 246 )
 247 parser.add_argument(
 248     "--dirty",
 249     action='store_true',
 250     help="leave around test/zend/all directory."
 251 )
 252 parser.add_argument(
 253     "-v",
 254     "--verbose",
 255     action='store_true',
 256     help="print out extra stuff."
 257 )
 258 args = parser.parse_args()
 259
 260
 261 def mkdir_p(path):
 262     try:
 263         os.makedirs(path)
 264     except OSError as exc: # Python >2.5
 265         pass
 266
 267 def walk(filename, source):
 268     dest_filename = os.path.basename(filename).replace('.phpt', '.php')
 269     source_dir = source.lower().replace('/tests', '').replace('/', '-')
 270
 271     cur_dir = os.path.dirname(__file__)
 272     dest_subdir = os.path.join(cur_dir, '../test/zend/all', source_dir)
 273     mkdir_p(dest_subdir)
 274     full_dest_filename = os.path.join(dest_subdir, dest_filename)
 275
 276     if not '.phpt' in filename:
 277         shutil.copyfile(filename, full_dest_filename)
 278         return
 279
 280     print "Importing %s" % filename
 281
 282     def split(pattern, str):
 283         return re.split(r'\n\s*--'+pattern+'--\s*\n', str, 1)
 284
 285     def parse_headers(zend):
 286         sections = {}
 287         cur_header = None
 288         for line in zend.split('\n'):
 289             header = re.match('--([_A-Z]+)--', line)
 290             if header:
 291                 cur_header = header.group(1)
 292                 sections[cur_header] = []
 293             else:
 294                 sections[cur_header].append(line)
 295         return sections
 296
 297     sections = parse_headers(file(filename).read())
 298     for i in sections.keys():
 299         sections[i] = '\n'.join(sections[i])
 300
 301     unsupported_sections = ('INI', 'POST_RAW')
 302     for name in unsupported_sections:
 303         if sections.has_key(name):
 304             print "Unsupported test with section --%s--: " % name, filename
 305             return
 306
 307     if not sections.has_key('FILE'):
 308         print "Malformed test, no --FILE--: ", filename
 309         return
 310
 311     for key in ('EXPECT', 'EXPECTF', 'EXPECTREGEX'):
 312         if sections.has_key(key):
 313             exp = sections[key]
 314
 315             # tests are really inconsistent about whitespace
 316             exp = re.sub(r'(\r\n|\r|\n)', '\n', exp.strip())
 317
 318             exp = exp.replace('in %s on', 'in %s/%s/%s on' % ('hphp/test/zend/all', source_dir, dest_filename))
 319
 320             # PHP puts a newline in that we don't
 321             exp = exp.replace('\n\nFatal error:', '\nFatal error:')
 322             exp = exp.replace('\n\nWarning:', '\nWarning:')
 323             exp = exp.replace('\n\nNotice:', '\nNotice:')
 324
 325             match_rest_of_line = '%a'
 326             if key == 'EXPECTREGEX':
 327                 match_rest_of_line = '.+'
 328
 329             exp = re.sub(r'Fatal\\? error\\?:.*', 'HipHop Fatal error: '+match_rest_of_line, exp)
 330             exp = re.sub(r'Warning\\?:.*', 'HipHop Warning: '+match_rest_of_line, exp)
 331             exp = re.sub(r'Notice\\?:.*', 'HipHop Notice: '+match_rest_of_line, exp)
 332
 333             for error in errors:
 334                 exp = re.sub(error[0], error[1], exp)
 335
 336             sections[key] = exp
 337
 338     if sections.has_key('EXPECT'):
 339         exp = sections['EXPECT']
 340         # we use %a for error messages so always write expectf
 341         file(full_dest_filename+'.expectf', 'w').write(exp)
 342     elif sections.has_key('EXPECTREGEX'):
 343         exp = sections['EXPECTREGEX']
 344         file(full_dest_filename+'.expectregex', 'w').write(exp)
 345     elif sections.has_key('EXPECTF'):
 346         exp = sections['EXPECTF']
 347         file(full_dest_filename+'.expectf', 'w').write(exp)
 348     else:
 349         print "Malformed test, no --EXPECT-- or --EXPECTF-- or --EXPECTREGEX--: ", filename
 350         return
 351
 352     test = sections['FILE']
 353
 354     if sections.has_key('POST'):
 355         test = test.replace(
 356             '<?php',
 357             '<?php\nparse_str("' + sections['POST'] + '", $_POST);\n'
 358         )
 359     if sections.has_key('GET'):
 360         test = test.replace(
 361             '<?php',
 362             '<?php\nparse_str("' + sections['GET'] + '", $_GET);\n'
 363         )
 364     if sections.has_key('COOKIE'):
 365         test = test.replace(
 366             '<?php',
 367             '<?php\n$_COOKIE = http_parse_cookie("' + sections['COOKIE'] + '");\n'
 368         )
 369     if sections.has_key('ENV'):
 370         for line in sections['ENV'].split('\n'):
 371             boom = line.split('=')
 372             if len(boom) == 2 and boom[0] and boom[1]:
 373                 test = test.replace(
 374                     '<?php',
 375                     '<?php\n$_ENV[%s] = %s;\n' % (boom[0], boom[1])
 376                 )
 377     if sections.has_key('CLEAN'):
 378         test += sections['CLEAN']
 379
 380     if 'bug60771.php' in full_dest_filename:
 381         test = test.replace("?>", "unlink('test.php');\n?>")
 382     if 'bug44805.php' in full_dest_filename:
 383         test = test.replace("1)) {\n\tunlink($file2", "2)) {\n\tunlink($file2")
 384     if 'bug24054.php' in full_dest_filename:
 385         test = test.replace("quicktester.inc", "tests/quicktester.inc")
 386     if 'ext-bz2/with_strings.php' in full_dest_filename:
 387         test = test.replace("../..", "")
 388     if 'ext-bz2/bug51997.php' in full_dest_filename:
 389         test = test.replace("testfile.bz2", "bug51997.bz2")
 390     if 'ext-bz2/with_files.php' in full_dest_filename:
 391         test = test.replace("testfile.bz2", "with_files.bz2")
 392
 393     file(full_dest_filename, 'w').write(test)
 394
 395 if args.zend_path:
 396     test_dirs = (('Zend/tests'), ('tests'), ('sapi'), ('ext'))
 397     def should_import(filename):
 398         for bad in no_import:
 399             if bad in filename:
 400                 return False
 401         return True
 402
 403     for source in test_dirs:
 404         for root, dirs, files in os.walk(os.path.join(args.zend_path, source)):
 405             for filename in files:
 406                 full_file = os.path.join(root, filename)
 407
 408                 def matches(regexes):
 409                     if not regexes:
 410                         return True
 411                     for regex in regexes:
 412                         if re.search(regex, full_file):
 413                             return True
 414                     return False
 415
 416                 if matches(args.only) and should_import(full_file):
 417                     walk(full_file, root.replace(args.zend_path, ''))
 418
 419 if not os.path.isdir('test/zend/all'):
 420     if args.zend_path:
 421         print "No test/zend/all. Maybe no tests were imported?"
 422         sys.exit(0)
 423     else:
 424         print "Running all tests from test/zend/bad"
 425         shutil.copytree('test/zend/bad', 'test/zend/all')
 426 else:
 427     print "Running all tests from zend/all"
 428
 429 stdout = subprocess.Popen(
 430     [
 431         'tools/verify_to_json.php',
 432         'test/run',
 433         'test/zend/all',
 434         'interp',
 435         '',
 436         '../_bin',
 437     ],
 438     stdout=subprocess.PIPE,
 439     stderr=subprocess.STDOUT
 440 ).communicate()[0]
 441
 442 # segfaults also print on stderr
 443 stdout = re.sub('\nsh: line 1:.*', '', stdout)
 444 # fbmake, you are crazy
 445 print stdout
 446 results = json.loads('['+stdout.strip().replace("\n", ",\n")+']')[-1]['results']
 447
 448 if args.verbose:
 449     print results
 450
 451 for test in results:
 452     filename = test['name']
 453     good_file = filename.replace('all', 'good', 1)
 454     bad_file = filename.replace('all', 'bad', 1)
 455     mkdir_p(os.path.dirname(good_file))
 456     mkdir_p(os.path.dirname(bad_file))
 457
 458     good = (test['status'] == 'passed')
 459     for test in bad_tests:
 460         if test in filename:
 461             good = False
 462
 463     if good:
 464         dest_file = good_file
 465         delete_file = bad_file
 466         subpath = 'good'
 467     else:
 468         dest_file = bad_file
 469         delete_file = good_file
 470         subpath = 'bad'
 471
 472     exps = glob.glob(filename+'.expect*')
 473     if not exps:
 474         # this file is probably generated while running tests :(
 475         continue
 476
 477     source_file_exp = exps[0]
 478     _, dest_ext = os.path.splitext(source_file_exp)
 479     os.rename(filename, dest_file)
 480     file(dest_file+dest_ext, 'w').write(
 481         file(source_file_exp).read().replace('/all', '/' + subpath)
 482     )
 483     os.unlink(source_file_exp)
 484     for f in glob.glob(delete_file+"*"):
 485         os.unlink(f)
 486
 487 # extra random files needed for tests...
 488 for root, dirs, files in os.walk('test/zend/all'):
 489     for filename in files:
 490         filename = os.path.join(root, filename)
 491
 492         for name in other_files:
 493             if name in filename:
 494                 dest = filename.replace('all', 'good', 1)
 495                 dir = os.path.dirname(dest)
 496                 mkdir_p(dir)
 497                 shutil.copyfile(filename, dest)
 498
 499 if not args.dirty:
 500     shutil.rmtree('test/zend/all')