hphp/hack/test/verify.py

   1 #!/usr/bin/env python3
   2
   3 import argparse
   4 import os.path
   5 import os
   6 import subprocess
   7 import sys
   8 import difflib
   9 import shlex
  10 from collections import namedtuple
  11 from concurrent.futures import ThreadPoolExecutor
  12
  13 max_workers = 48
  14 verbose = False
  15 dump_on_failure = False
  16
  17
  18 Failure = namedtuple('Failure', ['fname', 'expected', 'output'])
  19
  20 """
  21 Per-test flags passed to test executable. Expected to be in a file with
  22 same name as test, but with .flags extension.
  23 """
  24
  25 def get_test_flags(f):
  26     prefix, _ext = os.path.splitext(f)
  27     path = prefix + '.flags'
  28
  29     if not os.path.isfile(path):
  30         return []
  31     with open(path) as f:
  32         return shlex.split(f.read().strip())
  33
  34
  35 def run_test_program(files, program, expect_ext, get_flags, use_stdin):
  36     """
  37     Run the program and return a list of Failures.
  38     """
  39     def run(f):
  40         test_dir, test_name = os.path.split(f)
  41         flags = get_flags(test_dir)
  42         test_flags = get_test_flags(f)
  43         cmd = [program]
  44         if not use_stdin:
  45             cmd.append(test_name)
  46         cmd += flags + test_flags
  47         if verbose:
  48             print('Executing', ' '.join(cmd))
  49         try:
  50             def go(stdin=None):
  51                 return subprocess.check_output(
  52                     cmd, stderr=subprocess.STDOUT, cwd=test_dir,
  53                     universal_newlines=True, stdin=stdin)
  54             if use_stdin:
  55                 with open(f) as stdin:
  56                     output = go(stdin)
  57             else:
  58                 output = go()
  59         except subprocess.CalledProcessError as e:
  60             # we don't care about nonzero exit codes... for instance, type
  61             # errors cause hh_single_type_check to produce them
  62             output = e.output
  63         return check_result(f, expect_ext, output)
  64
  65     executor = ThreadPoolExecutor(max_workers=max_workers)
  66     futures = [executor.submit(run, f) for f in files]
  67
  68     results = [f.result() for f in futures]
  69     return [r for r in results if r is not None]
  70
  71 def check_result(fname, expect_exp, out):
  72     try:
  73         with open(fname + expect_exp, 'rt') as fexp:
  74             exp = fexp.read()
  75     except FileNotFoundError:
  76         exp = ''
  77     if exp != out:
  78         return Failure(fname=fname, expected=exp, output=out)
  79
  80 def record_failures(failures, out_ext):
  81     for failure in failures:
  82         outfile = failure.fname + out_ext
  83         with open(outfile, 'wb') as f:
  84             f.write(bytes(failure.output, 'UTF-8'))
  85
  86 def dump_failures(failures):
  87     for f in failures:
  88         expected = f.expected
  89         actual = f.output
  90         diff = difflib.ndiff(
  91             expected.splitlines(1),
  92             actual.splitlines(1))
  93         print("Details for the failed test %s:" % f.fname)
  94         print("\n>>>>>  Expected output  >>>>>>\n")
  95         print(expected)
  96         print("\n=====   Actual output   ======\n")
  97         print(actual)
  98         print("\n<<<<< End Actual output <<<<<<<\n")
  99         print("\n>>>>>       Diff        >>>>>>>\n")
 100         print(''.join(diff))
 101         print("\n<<<<<     End Diff      <<<<<<<\n")
 102
 103 def get_hh_flags(test_dir):
 104     path = os.path.join(test_dir, 'HH_FLAGS')
 105     if not os.path.isfile(path):
 106         if verbose:
 107             print("No HH_FLAGS file found")
 108         return []
 109     with open(path) as f:
 110         return shlex.split(f.read().strip())
 111
 112 def files_with_ext(files, ext):
 113     """
 114     Returns the set of filenames in :files that end in :ext
 115     """
 116     result = set()
 117     for f in files:
 118         prefix, suffix = os.path.splitext(f)
 119         if suffix == ext:
 120             result.add(prefix)
 121     return result
 122
 123 def list_test_files(root, disabled_ext, test_ext):
 124     if os.path.isfile(root):
 125         if root.endswith(test_ext):
 126             return [root]
 127         else:
 128             return []
 129     elif os.path.isdir(root):
 130         result = []
 131         children = os.listdir(root)
 132         disabled = files_with_ext(children, disabled_ext)
 133         for child in children:
 134             if child != 'disabled' and child not in disabled:
 135                 result.extend(
 136                     list_test_files(
 137                         os.path.join(root, child),
 138                         disabled_ext,
 139                         test_ext))
 140         return result
 141     elif os.path.islink(root):
 142         # Some editors create broken symlinks as part of their locking scheme,
 143         # so ignore those.
 144         return []
 145     else:
 146         raise Exception('Could not find test file or directory at %s' %
 147             args.test_path)
 148
 149 if __name__ == '__main__':
 150     parser = argparse.ArgumentParser()
 151     parser.add_argument(
 152             'test_path',
 153             help='A file or a directory. ')
 154     parser.add_argument('--program', type=os.path.abspath)
 155     parser.add_argument('--out-extension', type=str, default='.out')
 156     parser.add_argument('--expect-extension', type=str, default='.exp')
 157     parser.add_argument('--in-extension', type=str, default='.php')
 158     parser.add_argument('--disabled-extension', type=str,
 159             default='.no_typecheck')
 160     parser.add_argument('--verbose', action='store_true')
 161     parser.add_argument('--max-workers', type=int, default='48')
 162     parser.add_argument('--diff', action='store_true',
 163                        help='On test failure, show the content of the files and a diff')
 164     parser.add_argument('--flags', nargs=argparse.REMAINDER)
 165     parser.add_argument('--stdin', action='store_true',
 166                         help='Pass test input file via stdin')
 167     parser.epilog = "Unless --flags is passed as an argument, "\
 168                     "%s looks for a file named HH_FLAGS in the same directory" \
 169                     " as the test files it is executing. If found, the " \
 170                     "contents will be passed as arguments to " \
 171                     "<program>." % parser.prog
 172     args = parser.parse_args()
 173
 174     max_workers = args.max_workers
 175     verbose = args.verbose
 176     dump_on_failure = args.diff
 177
 178     if not os.path.isfile(args.program):
 179         raise Exception('Could not find program at %s' % args.program)
 180
 181     files = list_test_files(
 182         args.test_path,
 183         args.disabled_extension,
 184         args.in_extension)
 185
 186     if len(files) == 0:
 187         raise Exception(
 188             'Could not find any files to test in ' + args.test_path)
 189
 190     flags_cache = {}
 191
 192     def get_flags(test_dir):
 193         if args.flags is not None:
 194             return args.flags
 195         else:
 196             if test_dir not in flags_cache:
 197                 flags_cache[test_dir] = get_hh_flags(test_dir)
 198             return flags_cache[test_dir]
 199
 200     failures = run_test_program(
 201         files, args.program, args.expect_extension, get_flags, args.stdin)
 202     total = len(files)
 203     if failures == []:
 204         print("All %d tests passed!\n" % total)
 205     else:
 206         record_failures(failures, args.out_extension)
 207         fnames = [failure.fname for failure in failures]
 208         print("To review the failures, use the following command: ")
 209         print("OUT_EXT=%s EXP_EXT=%s ./hphp/hack/test/review.sh %s" %
 210                 (args.out_extension, args.expect_extension, " ".join(fnames)))
 211         if dump_on_failure:
 212             dump_failures(failures)
 213         print("Failed %d out of %d tests." % (len(failures), total))
 214         sys.exit(1)