hphp/hack/test/verify.py

   1 #!/usr/bin/env python3
   2
   3 import argparse
   4 import os.path
   5 import os
   6 import subprocess
   7 import sys
   8 import difflib
   9 import shlex
  10 from collections import namedtuple
  11 from concurrent.futures import ThreadPoolExecutor
  12
  13 max_workers = 48
  14 verbose = False
  15 dump_on_failure = False
  16
  17
  18 Failure = namedtuple('Failure', ['fname', 'expected', 'output'])
  19
  20 """
  21 Per-test flags passed to test executable. Expected to be in a file with
  22 same name as test, but with .flags extension.
  23 """
  24
  25 def get_test_flags(f):
  26     prefix, _ext = os.path.splitext(f)
  27     path = prefix + '.flags'
  28
  29     if not os.path.isfile(path):
  30         return []
  31     with open(path) as f:
  32         return shlex.split(f.read().strip())
  33
  34
  35 def run_test_program(files, program, expect_ext, get_flags, use_stdin):
  36     """
  37     Run the program and return a list of Failures.
  38     """
  39     def run(f):
  40         test_dir, test_name = os.path.split(f)
  41         flags = get_flags(test_dir)
  42         test_flags = get_test_flags(f)
  43         cmd = [program]
  44         if not use_stdin:
  45             cmd.append(test_name)
  46         cmd += flags + test_flags
  47         if verbose:
  48             print('Executing', ' '.join(cmd))
  49         try:
  50             def go(stdin=None):
  51                 return subprocess.check_output(
  52                     cmd, stderr=subprocess.STDOUT, cwd=test_dir,
  53                     universal_newlines=True, stdin=stdin)
  54             if use_stdin:
  55                 with open(f) as stdin:
  56                     output = go(stdin)
  57             else:
  58                 output = go()
  59         except subprocess.CalledProcessError as e:
  60             # we don't care about nonzero exit codes... for instance, type
  61             # errors cause hh_single_type_check to produce them
  62             output = e.output
  63         return check_result(f, expect_ext, output)
  64
  65     executor = ThreadPoolExecutor(max_workers=max_workers)
  66     futures = [executor.submit(run, f) for f in files]
  67
  68     results = [f.result() for f in futures]
  69     return [r for r in results if r is not None]
  70
  71 def filter_ocaml_stacktrace(text):
  72     """take a string and remove all the lines that look like
  73     they're part of an OCaml stacktrace"""
  74     assert isinstance(text, str)
  75     it = text.splitlines()
  76     out = []
  77     for x in it:
  78         drop_line = (
  79             x.lstrip().startswith("Called") or
  80             x.lstrip().startswith("Raised")
  81         )
  82         if drop_line:
  83             pass
  84         else:
  85             out.append(x)
  86     # force trailing newline
  87     return "\n".join(out) + "\n"
  88
  89 def check_result(fname, expect_exp, out):
  90     try:
  91         with open(fname + expect_exp, 'rt') as fexp:
  92             exp = fexp.read()
  93     except FileNotFoundError:
  94         exp = ''
  95     if exp != out and exp != filter_ocaml_stacktrace(out):
  96         return Failure(fname=fname, expected=exp, output=out)
  97
  98 def record_failures(failures, out_ext):
  99     for failure in failures:
 100         outfile = failure.fname + out_ext
 101         with open(outfile, 'wb') as f:
 102             f.write(bytes(failure.output, 'UTF-8'))
 103
 104 def dump_failures(failures):
 105     for f in failures:
 106         expected = f.expected
 107         actual = f.output
 108         diff = difflib.ndiff(
 109             expected.splitlines(1),
 110             actual.splitlines(1))
 111         print("Details for the failed test %s:" % f.fname)
 112         print("\n>>>>>  Expected output  >>>>>>\n")
 113         print(expected)
 114         print("\n=====   Actual output   ======\n")
 115         print(actual)
 116         print("\n<<<<< End Actual output <<<<<<<\n")
 117         print("\n>>>>>       Diff        >>>>>>>\n")
 118         print(''.join(diff))
 119         print("\n<<<<<     End Diff      <<<<<<<\n")
 120
 121 def get_hh_flags(test_dir):
 122     path = os.path.join(test_dir, 'HH_FLAGS')
 123     if not os.path.isfile(path):
 124         if verbose:
 125             print("No HH_FLAGS file found")
 126         return []
 127     with open(path) as f:
 128         return shlex.split(f.read().strip())
 129
 130 def files_with_ext(files, ext):
 131     """
 132     Returns the set of filenames in :files that end in :ext
 133     """
 134     result = set()
 135     for f in files:
 136         prefix, suffix = os.path.splitext(f)
 137         if suffix == ext:
 138             result.add(prefix)
 139     return result
 140
 141 def list_test_files(root, disabled_ext, test_ext):
 142     if os.path.isfile(root):
 143         if root.endswith(test_ext):
 144             return [root]
 145         else:
 146             return []
 147     elif os.path.isdir(root):
 148         result = []
 149         children = os.listdir(root)
 150         disabled = files_with_ext(children, disabled_ext)
 151         for child in children:
 152             if child != 'disabled' and child not in disabled:
 153                 result.extend(
 154                     list_test_files(
 155                         os.path.join(root, child),
 156                         disabled_ext,
 157                         test_ext))
 158         return result
 159     elif os.path.islink(root):
 160         # Some editors create broken symlinks as part of their locking scheme,
 161         # so ignore those.
 162         return []
 163     else:
 164         raise Exception('Could not find test file or directory at %s' %
 165             args.test_path)
 166
 167 if __name__ == '__main__':
 168     parser = argparse.ArgumentParser()
 169     parser.add_argument(
 170             'test_path',
 171             help='A file or a directory. ')
 172     parser.add_argument('--program', type=os.path.abspath)
 173     parser.add_argument('--out-extension', type=str, default='.out')
 174     parser.add_argument('--expect-extension', type=str, default='.exp')
 175     parser.add_argument('--in-extension', type=str, default='.php')
 176     parser.add_argument('--disabled-extension', type=str,
 177             default='.no_typecheck')
 178     parser.add_argument('--verbose', action='store_true')
 179     parser.add_argument('--max-workers', type=int, default='48')
 180     parser.add_argument('--diff', action='store_true',
 181                        help='On test failure, show the content of the files and a diff')
 182     parser.add_argument('--flags', nargs=argparse.REMAINDER)
 183     parser.add_argument('--stdin', action='store_true',
 184                         help='Pass test input file via stdin')
 185     parser.epilog = "Unless --flags is passed as an argument, "\
 186                     "%s looks for a file named HH_FLAGS in the same directory" \
 187                     " as the test files it is executing. If found, the " \
 188                     "contents will be passed as arguments to " \
 189                     "<program>." % parser.prog
 190     args = parser.parse_args()
 191
 192     max_workers = args.max_workers
 193     verbose = args.verbose
 194     dump_on_failure = args.diff
 195
 196     if not os.path.isfile(args.program):
 197         raise Exception('Could not find program at %s' % args.program)
 198
 199     files = list_test_files(
 200         args.test_path,
 201         args.disabled_extension,
 202         args.in_extension)
 203
 204     if len(files) == 0:
 205         raise Exception(
 206             'Could not find any files to test in ' + args.test_path)
 207
 208     flags_cache = {}
 209
 210     def get_flags(test_dir):
 211         if args.flags is not None:
 212             flags = args.flags
 213         else:
 214             if test_dir not in flags_cache:
 215                 flags_cache[test_dir] = get_hh_flags(test_dir)
 216             flags = flags_cache[test_dir]
 217         hacksperimental_file = os.path.join(test_dir, '.hacksperimental')
 218         if os.path.isfile(hacksperimental_file):
 219             flags += ["--hacksperimental"]
 220         return flags
 221
 222     failures = run_test_program(
 223         files, args.program, args.expect_extension, get_flags, args.stdin)
 224     total = len(files)
 225     if failures == []:
 226         print("All %d tests passed!\n" % total)
 227     else:
 228         record_failures(failures, args.out_extension)
 229         fnames = [failure.fname for failure in failures]
 230         print("To review the failures, use the following command: ")
 231         print("OUT_EXT=%s EXP_EXT=%s ./hphp/hack/test/review.sh %s" %
 232                 (args.out_extension, args.expect_extension, " ".join(fnames)))
 233         if dump_on_failure:
 234             dump_failures(failures)
 235         print("Failed %d out of %d tests." % (len(failures), total))
 236         sys.exit(1)