hphp/hack/test/verify.py

   1 #!/usr/bin/env python3
   2 # pyre-strict
   3
   4 import argparse
   5 import difflib
   6 import os
   7 import os.path
   8 import re
   9 import shlex
  10 import subprocess
  11 import sys
  12 from concurrent.futures import ThreadPoolExecutor
  13 from dataclasses import dataclass
  14 from enum import Enum
  15 from typing import Callable, Dict, List, Optional, Tuple
  16
  17 from hphp.hack.test.parse_errors import Error, parse_errors, sprint_errors
  18
  19 DEFAULT_OUT_EXT = ".out"
  20 DEFAULT_EXP_EXT = ".exp"
  21
  22 flags_pessimise_unsupported = [
  23     "--complex-coercion",
  24     "--enable-higher-kinded-types",
  25     "--enable-global-access-check",
  26 ]
  27 max_workers = 48
  28 verbose = False
  29 dump_on_failure = False
  30 batch_size = 500
  31
  32
  33 @dataclass
  34 class TestCase:
  35     file_path: str
  36     input: Optional[str]
  37     expected: str
  38
  39
  40 @dataclass
  41 class Result:
  42     test_case: TestCase
  43     output: str
  44     is_failure: bool
  45
  46
  47 class VerifyPessimisationOptions(Enum):
  48     no = "no"
  49     all = "all"
  50     added = "added"
  51     removed = "removed"
  52     full = "full"
  53
  54     def __str__(self) -> str:
  55         return self.value
  56
  57
  58 """
  59 Per-test flags passed to test executable. Expected to be in a file with
  60 same name as test, but with .flags extension.
  61 """
  62
  63
  64 def compare_errors_by_line_no(
  65     errors_exp: List[Error], errors_out: List[Error]
  66 ) -> Tuple[List[Error], List[Error]]:
  67     i_out = 0
  68     i_exp = 0
  69     len_out = len(errors_out)
  70     len_exp = len(errors_exp)
  71     errors_in_out_not_in_exp = []
  72     errors_in_exp_not_in_out = []
  73     while i_out < len_out and i_exp < len_exp:
  74         err_out = errors_out[i_out]
  75         err_exp = errors_exp[i_exp]
  76         l_out = err_out.message.position.line
  77         l_exp = err_exp.message.position.line
  78         if l_out < l_exp:
  79             errors_in_out_not_in_exp.append(err_out)
  80             i_out += 1
  81         elif l_exp < l_out:
  82             errors_in_exp_not_in_out.append(err_exp)
  83             i_exp += 1
  84         else:
  85             i_out += 1
  86             i_exp += 1
  87     if i_out >= len_out:
  88         for i in range(i_exp, len_exp):
  89             errors_in_exp_not_in_out.append(errors_exp[i])
  90     elif i_exp >= len_exp:
  91         for i in range(i_out, len_out):
  92             errors_in_out_not_in_exp.append(errors_out[i])
  93     return (errors_in_exp_not_in_out, errors_in_out_not_in_exp)
  94
  95
  96 def compare_output_files_error_lines_only(
  97     file_out: str, file_exp: str
  98 ) -> Tuple[bool, str]:
  99     out = ""
 100     failed = False
 101     try:
 102         errors_out = parse_errors(file_out)
 103         errors_exp = parse_errors(file_exp)
 104         (
 105             errors_in_exp_not_in_out,
 106             errors_in_out_not_in_exp,
 107         ) = compare_errors_by_line_no(errors_out=errors_out, errors_exp=errors_exp)
 108
 109         failed = bool(errors_in_exp_not_in_out) or bool(errors_in_out_not_in_exp)
 110         if errors_in_exp_not_in_out:
 111             out += f"""\033[93mExpected errors which were not produced:\033[0m
 112 {sprint_errors(errors_in_exp_not_in_out)}
 113 """
 114         if errors_in_out_not_in_exp:
 115             out += f"""\033[93mProduced errors which were not expected:\033[0m
 116 {sprint_errors(errors_in_out_not_in_exp)}
 117 """
 118     except IOError as e:
 119         out = f"Warning: {e}"
 120     return (failed, out)
 121
 122
 123 def check_output_error_lines_only(
 124     test: str, out_ext: str = DEFAULT_OUT_EXT, exp_ext: str = DEFAULT_EXP_EXT
 125 ) -> Tuple[bool, str]:
 126     file_out = test + out_ext
 127     file_exp = test + exp_ext
 128     return compare_output_files_error_lines_only(file_out=file_out, file_exp=file_exp)
 129
 130
 131 def get_test_flags(path: str) -> List[str]:
 132     prefix, _ext = os.path.splitext(path)
 133     path = prefix + ".flags"
 134
 135     if not os.path.isfile(path):
 136         return []
 137     with open(path) as file:
 138         return shlex.split(file.read().strip())
 139
 140
 141 def check_output(
 142     case: TestCase,
 143     out_extension: str,
 144     default_expect_regex: Optional[str],
 145     ignore_error_text: bool,
 146     only_compare_error_lines: bool,
 147     normalize_paths: bool,
 148     verify_pessimisation: VerifyPessimisationOptions,
 149     check_expected_included_in_actual: bool,
 150 ) -> Result:
 151     if only_compare_error_lines:
 152         (failed, out) = check_output_error_lines_only(case.file_path)
 153         return Result(test_case=case, output=out, is_failure=failed)
 154     else:
 155         out_path = (
 156             case.file_path + out_extension
 157             if verify_pessimisation == VerifyPessimisationOptions.no
 158             or verify_pessimisation == VerifyPessimisationOptions.full
 159             or out_extension == ".pess.out"
 160             else case.file_path + ".pess" + out_extension
 161         )
 162         try:
 163             with open(out_path, "r") as f:
 164                 output: str = f.read()
 165         except FileNotFoundError:
 166             out_path = os.path.realpath(out_path)
 167             output = "Output file " + out_path + " was not found!"
 168         return check_result(
 169             case,
 170             default_expect_regex,
 171             ignore_error_text,
 172             normalize_paths,
 173             verify_pessimisation,
 174             check_expected_included_in_actual=check_expected_included_in_actual,
 175             out=output,
 176         )
 177
 178
 179 def debug_cmd(cwd: str, cmd: List[str]) -> None:
 180     if verbose:
 181         print("From directory", os.path.realpath(cwd))
 182         print("Executing", " ".join(cmd))
 183         print()
 184
 185
 186 def run_batch_tests(
 187     test_cases: List[TestCase],
 188     program: str,
 189     default_expect_regex: Optional[str],
 190     ignore_error_text: bool,
 191     no_stderr: bool,
 192     force_color: bool,
 193     mode_flag: List[str],
 194     get_flags: Callable[[str], List[str]],
 195     out_extension: str,
 196     normalize_paths: bool,
 197     verify_pessimisation: VerifyPessimisationOptions,
 198     check_expected_included_in_actual: bool,
 199     only_compare_error_lines: bool = False,
 200 ) -> List[Result]:
 201     """
 202     Run the program with batches of files and return a list of results.
 203     """
 204     # Each directory needs to be in a separate batch because flags are different
 205     # for each directory.
 206     # Compile a list of directories to test cases, and then
 207     dirs_to_files: Dict[str, List[TestCase]] = {}
 208     for case in test_cases:
 209         test_dir = os.path.dirname(case.file_path)
 210         dirs_to_files.setdefault(test_dir, []).append(case)
 211
 212     # run a list of test cases.
 213     # The contract here is that the program will write to
 214     # filename.out_extension for each file, and we read that
 215     # for the output.
 216     # Remark: if verify_pessimisation is set, then the result
 217     # is in filename.pess.out_extension.
 218     def run(test_cases: List[TestCase]) -> List[Result]:
 219         if not test_cases:
 220             raise AssertionError()
 221         first_test = test_cases[0]
 222         test_dir = os.path.dirname(first_test.file_path)
 223         flags = get_flags(test_dir)
 224         test_flags = get_test_flags(first_test.file_path)
 225         if verify_pessimisation != VerifyPessimisationOptions.no:
 226             path = os.path.join(test_dir, "NO_PESS")
 227             if os.path.isfile(path):
 228                 return []
 229             for flag in flags_pessimise_unsupported:
 230                 if flag in flags:
 231                     return []
 232         cmd = [program]
 233         cmd += mode_flag
 234         cmd += ["--batch-files", "--out-extension", out_extension]
 235         cmd += flags + test_flags
 236         cmd += [os.path.basename(case.file_path) for case in test_cases]
 237         debug_cmd(test_dir, cmd)
 238         env = os.environ.copy()
 239         env["FORCE_ERROR_COLOR"] = "true" if force_color else "false"
 240         try:
 241             return_code = subprocess.call(
 242                 cmd,
 243                 stderr=None if no_stderr else subprocess.STDOUT,
 244                 cwd=test_dir,
 245                 universal_newlines=True,
 246                 env=env,
 247             )
 248         except subprocess.CalledProcessError:
 249             # we don't care about nonzero exit codes... for instance, type
 250             # errors cause hh_single_type_check to produce them
 251             return_code = None
 252         if return_code == -11:
 253             print(
 254                 "Segmentation fault while running the following command "
 255                 + "from directory "
 256                 + os.path.realpath(test_dir)
 257             )
 258             print(" ".join(cmd))
 259             print()
 260         results = []
 261         for case in test_cases:
 262             result = check_output(
 263                 case,
 264                 out_extension=out_extension,
 265                 default_expect_regex=default_expect_regex,
 266                 ignore_error_text=ignore_error_text,
 267                 only_compare_error_lines=only_compare_error_lines,
 268                 normalize_paths=normalize_paths,
 269                 verify_pessimisation=verify_pessimisation,
 270                 check_expected_included_in_actual=check_expected_included_in_actual,
 271             )
 272             results.append(result)
 273         return results
 274
 275     # Create a list of batched cases.
 276     all_batched_cases: List[List[TestCase]] = []
 277
 278     # For each directory, we split all the test cases
 279     # into chunks of batch_size. Then each of these lists
 280     # is a separate job for each thread in the threadpool.
 281     for cases in dirs_to_files.values():
 282         batched_cases: List[List[TestCase]] = [
 283             cases[i : i + batch_size] for i in range(0, len(cases), batch_size)
 284         ]
 285         all_batched_cases += batched_cases
 286
 287     executor = ThreadPoolExecutor(max_workers=max_workers)
 288     futures = [executor.submit(run, test_batch) for test_batch in all_batched_cases]
 289
 290     results = [future.result() for future in futures]
 291     # Flatten the list
 292     return [item for sublist in results for item in sublist]
 293
 294
 295 def run_test_program(
 296     test_cases: List[TestCase],
 297     program: str,
 298     default_expect_regex: Optional[str],
 299     ignore_error_text: bool,
 300     no_stderr: bool,
 301     force_color: bool,
 302     normalize_paths: bool,
 303     mode_flag: List[str],
 304     get_flags: Callable[[str], List[str]],
 305     verify_pessimisation: VerifyPessimisationOptions,
 306     check_expected_included_in_actual: bool,
 307     timeout: Optional[float] = None,
 308     filter_glog_failures: bool = False,
 309 ) -> List[Result]:
 310     """
 311     Run the program and return a list of results.
 312     """
 313
 314     def run(test_case: TestCase) -> Result:
 315         test_dir, test_name = os.path.split(test_case.file_path)
 316         flags = get_flags(test_dir)
 317         test_flags = get_test_flags(test_case.file_path)
 318         cmd = [program]
 319         cmd += mode_flag
 320         if test_case.input is None:
 321             cmd.append(test_name)
 322         cmd += flags + test_flags
 323         debug_cmd(test_dir, cmd)
 324         env = os.environ.copy()
 325         env["FORCE_ERROR_COLOR"] = "true" if force_color else "false"
 326         try:
 327             output = subprocess.check_output(
 328                 cmd,
 329                 stderr=None if no_stderr else subprocess.STDOUT,
 330                 cwd=test_dir,
 331                 universal_newlines=True,
 332                 input=test_case.input,
 333                 timeout=timeout,
 334                 errors="replace",
 335                 env=env,
 336             )
 337         except subprocess.TimeoutExpired as e:
 338             output = "Timed out. " + str(e.output)
 339         except subprocess.CalledProcessError as e:
 340             if e.returncode == 126:
 341                 # unable to execute, typically due in buck2 to "Text file is busy" because someone still has a write handle open to it.
 342                 # https://www.internalfb.com/intern/qa/312685/text-file-is-busy---test-is-run-before-fclose-on-e
 343                 # Ugly workaround for now: just skip it
 344                 # This should be removed once T107518211 is closed.
 345                 output = "Timed out. " + str(e.output)
 346             else:
 347                 # we don't care about nonzero exit codes... for instance, type
 348                 # errors cause hh_single_type_check to produce them
 349                 output = str(e.output)
 350         if filter_glog_failures:
 351             glog_message_re = r"COULD NOT CREATE A LOGGINGFILE .+\!|Could not create logging file: No such file or directory\n"
 352             output = re.sub(glog_message_re, r"", output)
 353         return check_result(
 354             test_case,
 355             default_expect_regex,
 356             ignore_error_text,
 357             normalize_paths,
 358             verify_pessimisation,
 359             check_expected_included_in_actual=check_expected_included_in_actual,
 360             out=output,
 361         )
 362
 363     executor = ThreadPoolExecutor(max_workers=max_workers)
 364     futures = [executor.submit(run, test_case) for test_case in test_cases]
 365
 366     return [future.result() for future in futures]
 367
 368
 369 def filter_ocaml_stacktrace(text: str) -> str:
 370     """take a string and remove all the lines that look like
 371     they're part of an OCaml stacktrace"""
 372     assert isinstance(text, str)
 373     it = text.splitlines()
 374     out = []
 375     for x in it:
 376         drop_line = x.lstrip().startswith("Called") or x.lstrip().startswith("Raised")
 377         if drop_line:
 378             pass
 379         else:
 380             out.append(x)
 381     return "\n".join(out)
 382
 383
 384 def filter_cwd(text: str) -> str:
 385     """take a string and remove all instances of the current cwd"""
 386     assert isinstance(text, str)
 387     # Get the absolute path of the current working directory
 388     cwd = os.path.abspath(os.getcwd())
 389     # Normalize the path separators to work on different operating systems
 390     cwd = cwd.replace("\\", "/")
 391     return text.replace(cwd, "/")
 392
 393
 394 def filter_temp_hhi_path(text: str) -> str:
 395     """The .hhi files are stored in a temporary directory whose name
 396     changes every time. Normalise it.
 397
 398     /tmp/ASjh5RoWbb/builtins_fb.hhi -> /tmp/hhi_dir/builtins_fb.hhi
 399
 400     """
 401     return re.sub(
 402         r"/tmp/[^/]*/([a-zA-Z0-9_]+\.hhi)",
 403         "/tmp/hhi_dir/\\1",
 404         text,
 405     )
 406
 407
 408 def strip_pess_suffix(text: str) -> str:
 409     return re.sub(
 410         r"_pess.php",
 411         ".php",
 412         text,
 413     )
 414
 415
 416 def compare_expected(
 417     expected: str, out: str, verify_pessimisation: VerifyPessimisationOptions
 418 ) -> bool:
 419     if (
 420         verify_pessimisation == VerifyPessimisationOptions.no
 421         or verify_pessimisation == VerifyPessimisationOptions.all
 422         or verify_pessimisation == VerifyPessimisationOptions.full
 423     ):
 424         if expected == "No errors" or out == "No errors":
 425             return expected == out
 426         else:
 427             return True
 428     elif verify_pessimisation == VerifyPessimisationOptions.added:
 429         return not (expected == "No errors" and out != "No errors")
 430     elif verify_pessimisation == VerifyPessimisationOptions.removed:
 431         return not (expected != "No errors" and out == "No errors")
 432     else:
 433         raise Exception(
 434             "Cannot happen: verify_pessimisation option %s" % verify_pessimisation
 435         )
 436
 437
 438 # Strip leading and trailing whitespace from every line
 439 def strip_lines(text: str) -> str:
 440     return "\n".join(line.strip() for line in text.splitlines())
 441
 442
 443 def check_result(
 444     test_case: TestCase,
 445     default_expect_regex: Optional[str],
 446     ignore_error_messages: bool,
 447     normalize_paths: bool,
 448     verify_pessimisation: VerifyPessimisationOptions,
 449     check_expected_included_in_actual: bool,
 450     out: str,
 451 ) -> Result:
 452     if check_expected_included_in_actual:
 453         return check_included(test_case, out)
 454     else:
 455         return check_expected_equal_actual(
 456             test_case,
 457             default_expect_regex,
 458             ignore_error_messages,
 459             normalize_paths,
 460             verify_pessimisation,
 461             out,
 462         )
 463
 464
 465 def check_expected_equal_actual(
 466     test_case: TestCase,
 467     default_expect_regex: Optional[str],
 468     ignore_error_messages: bool,
 469     normalize_paths: bool,
 470     verify_pessimisation: VerifyPessimisationOptions,
 471     out: str,
 472 ) -> Result:
 473     """
 474     Check that the output of the test in :out corresponds to the expected
 475     output, or if a :default_expect_regex is provided,
 476     check that the output in :out contains the provided regex.
 477     """
 478     expected = filter_temp_hhi_path(strip_lines(test_case.expected))
 479     normalized_out = (
 480         filter_temp_hhi_path(strip_lines(out))
 481         if verify_pessimisation == "no"
 482         else strip_pess_suffix(filter_temp_hhi_path(strip_lines(out)))
 483     )
 484     out = filter_cwd(normalized_out) if normalize_paths else out
 485     is_ok = (
 486         expected == normalized_out
 487         or (
 488             (
 489                 ignore_error_messages
 490                 or verify_pessimisation != VerifyPessimisationOptions.no
 491                 and verify_pessimisation != VerifyPessimisationOptions.full
 492             )
 493             and compare_expected(expected, normalized_out, verify_pessimisation)
 494         )
 495         or expected == filter_ocaml_stacktrace(normalized_out)
 496         or (
 497             default_expect_regex is not None
 498             and re.search(default_expect_regex, normalized_out) is not None
 499             and expected == ""
 500         )
 501         or (normalize_paths and (expected == out))
 502     )
 503     return Result(test_case=test_case, output=out, is_failure=not is_ok)
 504
 505
 506 def check_included(test_case: TestCase, output: str) -> Result:
 507     elts = set(output.splitlines())
 508     expected_elts = set(test_case.expected.splitlines())
 509     is_failure = False
 510     for expected_elt in expected_elts:
 511         if expected_elt not in elts:
 512             is_failure = True
 513             break
 514     output = ""
 515     if is_failure:
 516         for elt in expected_elts.intersection(elts):
 517             output += elt + "\n"
 518     return Result(test_case, output, is_failure)
 519
 520
 521 def record_results(results: List[Result], out_ext: str) -> None:
 522     for result in results:
 523         outfile = result.test_case.file_path + out_ext
 524         with open(outfile, "wb") as f:
 525             f.write(bytes(result.output, "UTF-8"))
 526
 527
 528 def find_in_ancestors_rec(dir: str, path: str) -> str:
 529     if path == "" or os.path.dirname(path) == path:
 530         raise Exception("Could not find directory %s in ancestors." % dir)
 531     if os.path.basename(path) == dir:
 532         return path
 533     return find_in_ancestors_rec(dir, os.path.dirname(path))
 534
 535
 536 def find_in_ancestors(dir: str, path: str) -> str:
 537     try:
 538         return find_in_ancestors_rec(dir, path)
 539     except Exception:
 540         raise Exception("Could not find directory %s in ancestors of %s." % (dir, path))
 541
 542
 543 def get_exp_out_dirs(test_file: str) -> Tuple[str, str]:
 544     if (
 545         os.environ.get("HACK_BUILD_ROOT") is not None
 546         and os.environ.get("HACK_SOURCE_ROOT") is not None
 547     ):
 548         exp_dir = os.environ["HACK_SOURCE_ROOT"]
 549         out_dir = os.environ["HACK_BUILD_ROOT"]
 550     else:
 551         fbcode = find_in_ancestors("fbcode", test_file)
 552         exp_dir = os.path.join(fbcode, "hphp", "hack")
 553         out_dir = os.path.dirname(find_in_ancestors("test", test_file))
 554     return exp_dir, out_dir
 555
 556
 557 def report_failures(
 558     total: int,
 559     failures: List[Result],
 560     out_extension: str,
 561     expect_extension: str,
 562     fallback_expect_extension: Optional[str],
 563     verify_pessimisation: VerifyPessimisationOptions = VerifyPessimisationOptions.no,
 564     no_copy: bool = False,
 565     only_compare_error_lines: bool = False,
 566 ) -> None:
 567     if only_compare_error_lines:
 568         for failure in failures:
 569             print(f"\033[95m{failure.test_case.file_path}\033[0m")
 570             print(failure.output)
 571             print()
 572     elif failures != []:
 573         record_results(failures, out_extension)
 574         if dump_on_failure:
 575             dump_failures(failures)
 576         fnames = [failure.test_case.file_path for failure in failures]
 577         print("To review the failures, use the following command: ")
 578
 579         first_test_file = os.path.realpath(failures[0].test_case.file_path)
 580         out_dir: str  # for Pyre
 581         (exp_dir, out_dir) = get_exp_out_dirs(first_test_file)
 582
 583         # Get a full path to 'review.sh' so this command be run
 584         # regardless of your current directory.
 585         review_script = os.path.join(
 586             os.path.dirname(os.path.realpath(__file__)), "review.sh"
 587         )
 588         if not os.path.isfile(review_script):
 589             review_script = "./hphp/hack/test/review.sh"
 590
 591         def fname_map_var(f: str) -> str:
 592             return "hphp/hack/" + os.path.relpath(f, out_dir)
 593
 594         env_vars = []
 595         if out_extension != DEFAULT_OUT_EXT:
 596             env_vars.append("OUT_EXT=%s" % out_extension)
 597         if expect_extension != DEFAULT_EXP_EXT:
 598             env_vars.append("EXP_EXT=%s" % expect_extension)
 599         if fallback_expect_extension is not None:
 600             env_vars.append("FALLBACK_EXP_EXT=%s " % fallback_expect_extension)
 601         if verify_pessimisation != VerifyPessimisationOptions.no:
 602             env_vars.append("VERIFY_PESSIMISATION=true")
 603         if no_copy:
 604             env_vars.append("UPDATE=never")
 605
 606         env_vars.extend(["SOURCE_ROOT=%s" % exp_dir, "OUTPUT_ROOT=%s" % out_dir])
 607
 608         print(
 609             "%s %s %s"
 610             % (
 611                 " ".join(env_vars),
 612                 review_script,
 613                 " ".join(map(fname_map_var, fnames)),
 614             )
 615         )
 616
 617         # If more than 75% of files have changed, we're probably doing
 618         # a transformation to all the .exp files.
 619         if len(fnames) >= (0.75 * total):
 620             print(
 621                 "\nJust want to update all the %s files? Use UPDATE=always with the above command."
 622                 % expect_extension
 623             )
 624
 625
 626 def dump_failures(failures: List[Result]) -> None:
 627     for f in failures:
 628         expected = f.test_case.expected
 629         actual = f.output
 630         diff = difflib.ndiff(expected.splitlines(True), actual.splitlines(True))
 631         print("Details for the failed test %s:" % f.test_case.file_path)
 632         print("\n>>>>>  Expected output  >>>>>>\n")
 633         print(expected)
 634         print("\n=====   Actual output   ======\n")
 635         print(actual)
 636         print("\n<<<<< End Actual output <<<<<<<\n")
 637         print("\n>>>>>       Diff        >>>>>>>\n")
 638         print("".join(diff))
 639         print("\n<<<<<     End Diff      <<<<<<<\n")
 640
 641
 642 def get_hh_flags(test_dir: str) -> List[str]:
 643     path = os.path.join(test_dir, "HH_FLAGS")
 644     if not os.path.isfile(path):
 645         if verbose:
 646             print("No HH_FLAGS file found")
 647         return []
 648     with open(path) as f:
 649         return shlex.split(f.read())
 650
 651
 652 def files_with_ext(files: List[str], ext: str) -> List[str]:
 653     """
 654     Returns the set of filenames in :files that end in :ext
 655     """
 656     filtered_files: List[str] = []
 657     for file in files:
 658         prefix, suffix = os.path.splitext(file)
 659         if suffix == ext:
 660             filtered_files.append(prefix)
 661     return filtered_files
 662
 663
 664 def list_test_files(
 665     root: str, disabled_ext: str, test_ext: str, include_directories: bool
 666 ) -> List[str]:
 667     if os.path.isfile(root):
 668         if root.endswith(test_ext):
 669             return [root]
 670         else:
 671             return []
 672     elif os.path.isdir(root):
 673         result: List[str] = []
 674         if include_directories and root.endswith(test_ext):
 675             result.append(root)
 676         children = os.listdir(root)
 677         disabled = files_with_ext(children, disabled_ext)
 678         for child in children:
 679             if child != "disabled" and child not in disabled:
 680                 result.extend(
 681                     list_test_files(
 682                         os.path.join(root, child),
 683                         disabled_ext,
 684                         test_ext,
 685                         include_directories,
 686                     )
 687                 )
 688         return result
 689     elif os.path.islink(root):
 690         # Some editors create broken symlinks as part of their locking scheme,
 691         # so ignore those.
 692         return []
 693     else:
 694         raise Exception("Could not find test file or directory at %s" % root)
 695
 696
 697 def get_content_(file_path: str, ext: str) -> str:
 698     with open(file_path + ext, "r") as fexp:
 699         return fexp.read()
 700
 701
 702 def get_content(
 703     file_path: str, ext: str = "", fallback_ext: Optional[str] = None
 704 ) -> str:
 705     try:
 706         return get_content_(file_path, ext)
 707     except FileNotFoundError:
 708         if fallback_ext is not None:
 709             try:
 710                 return get_content_(file_path, fallback_ext)
 711             except FileNotFoundError:
 712                 return ""
 713         else:
 714             return ""
 715
 716
 717 def run_tests(
 718     files: List[str],
 719     expected_extension: str,
 720     fallback_expect_extension: Optional[str],
 721     out_extension: str,
 722     use_stdin: str,
 723     program: str,
 724     default_expect_regex: Optional[str],
 725     batch_mode: str,
 726     ignore_error_text: bool,
 727     no_stderr: bool,
 728     force_color: bool,
 729     normalize_paths: bool,
 730     verify_pessimisation: VerifyPessimisationOptions,
 731     mode_flag: List[str],
 732     get_flags: Callable[[str], List[str]],
 733     check_expected_included_in_actual: bool,
 734     timeout: Optional[float] = None,
 735     only_compare_error_lines: bool = False,
 736     filter_glog_failures: bool = False,
 737 ) -> List[Result]:
 738
 739     # for each file, create a test case
 740     test_cases = [
 741         TestCase(
 742             file_path=file,
 743             expected=get_content(file, expected_extension, fallback_expect_extension),
 744             input=get_content(file) if use_stdin else None,
 745         )
 746         for file in files
 747     ]
 748     if batch_mode:
 749         results = run_batch_tests(
 750             test_cases,
 751             program,
 752             default_expect_regex,
 753             ignore_error_text,
 754             no_stderr,
 755             force_color,
 756             mode_flag,
 757             get_flags,
 758             out_extension,
 759             normalize_paths,
 760             verify_pessimisation,
 761             check_expected_included_in_actual,
 762             only_compare_error_lines,
 763         )
 764     else:
 765         results = run_test_program(
 766             test_cases,
 767             program,
 768             default_expect_regex,
 769             ignore_error_text,
 770             no_stderr,
 771             force_color,
 772             normalize_paths,
 773             mode_flag,
 774             get_flags,
 775             verify_pessimisation,
 776             check_expected_included_in_actual=check_expected_included_in_actual,
 777             timeout=timeout,
 778             filter_glog_failures=filter_glog_failures,
 779         )
 780
 781     failures = [result for result in results if result.is_failure]
 782
 783     num_results = len(results)
 784     if failures == []:
 785         print(
 786             "All tests in the suite passed! "
 787             "The number of tests that ran: %d\n" % num_results
 788         )
 789     else:
 790         print("The number of tests that failed: %d/%d\n" % (len(failures), num_results))
 791         report_failures(
 792             num_results,
 793             failures,
 794             args.out_extension,
 795             args.expect_extension,
 796             args.fallback_expect_extension,
 797             verify_pessimisation=verify_pessimisation,
 798             only_compare_error_lines=only_compare_error_lines,
 799         )
 800         sys.exit(1)  # this exit code fails the suite and lets Buck know
 801
 802     return results
 803
 804
 805 def run_idempotence_tests(
 806     results: List[Result],
 807     expected_extension: str,
 808     out_extension: str,
 809     program: str,
 810     default_expect_regex: Optional[str],
 811     mode_flag: List[str],
 812     get_flags: Callable[[str], List[str]],
 813     normalize_paths: bool,
 814     check_expected_included_in_actual: bool,
 815 ) -> None:
 816     idempotence_test_cases = [
 817         TestCase(
 818             file_path=result.test_case.file_path,
 819             expected=result.test_case.expected,
 820             input=result.output,
 821         )
 822         for result in results
 823     ]
 824
 825     idempotence_results = run_test_program(
 826         idempotence_test_cases,
 827         program,
 828         default_expect_regex,
 829         False,
 830         False,
 831         False,
 832         normalize_paths,
 833         mode_flag,
 834         get_flags,
 835         VerifyPessimisationOptions.no,
 836         check_expected_included_in_actual=check_expected_included_in_actual,
 837     )
 838
 839     num_idempotence_results = len(idempotence_results)
 840
 841     idempotence_failures = [
 842         result for result in idempotence_results if result.is_failure
 843     ]
 844
 845     if idempotence_failures == []:
 846         print(
 847             "All idempotence tests in the suite passed! The number of "
 848             "idempotence tests that ran: %d\n" % num_idempotence_results
 849         )
 850     else:
 851         print(
 852             "The number of idempotence tests that failed: %d/%d\n"
 853             % (len(idempotence_failures), num_idempotence_results)
 854         )
 855         report_failures(
 856             num_idempotence_results,
 857             idempotence_failures,
 858             out_extension + out_extension,  # e.g., *.out.out
 859             expected_extension,
 860             None,
 861             no_copy=True,
 862         )
 863         sys.exit(1)  # this exit code fails the suite and lets Buck know
 864
 865
 866 def get_flags_cache(args_flags: List[str]) -> Callable[[str], List[str]]:
 867     flags_cache: Dict[str, List[str]] = {}
 868
 869     def get_flags(test_dir: str) -> List[str]:
 870         if test_dir not in flags_cache:
 871             flags_cache[test_dir] = get_hh_flags(test_dir)
 872         flags = flags_cache[test_dir]
 873         if args_flags is not None:
 874             flags = flags + args_flags
 875         return flags
 876
 877     return get_flags
 878
 879
 880 def get_flags_dummy(args_flags: List[str]) -> Callable[[str], List[str]]:
 881     def get_flags(_: str) -> List[str]:
 882         return args_flags
 883
 884     return get_flags
 885
 886
 887 def main() -> None:
 888     global max_workers, dump_on_failure, verbose, args
 889     parser = argparse.ArgumentParser()
 890     parser.add_argument("test_path", help="A file or a directory. ")
 891     parser.add_argument("--program", type=os.path.abspath)
 892     parser.add_argument("--out-extension", type=str, default=DEFAULT_OUT_EXT)
 893     parser.add_argument("--expect-extension", type=str, default=DEFAULT_EXP_EXT)
 894     parser.add_argument("--fallback-expect-extension", type=str)
 895     parser.add_argument("--default-expect-regex", type=str)
 896     parser.add_argument("--in-extension", type=str, default=".php")
 897     parser.add_argument("--include-directories", action="store_true")
 898     parser.add_argument("--disabled-extension", type=str, default=".no_typecheck")
 899     parser.add_argument("--verbose", action="store_true")
 900     parser.add_argument(
 901         "--idempotence",
 902         action="store_true",
 903         help="Verify that the output passed to the program "
 904         "as input results in the same output.",
 905     )
 906     parser.add_argument("--max-workers", type=int, default="48")
 907     parser.add_argument(
 908         "--diff",
 909         action="store_true",
 910         help="On test failure, show the content of " "the files and a diff",
 911     )
 912     parser.add_argument("--mode-flag", type=str)
 913     parser.add_argument("--flags", nargs=argparse.REMAINDER)
 914     parser.add_argument(
 915         "--stdin", action="store_true", help="Pass test input file via stdin"
 916     )
 917     parser.add_argument(
 918         "--no-stderr",
 919         action="store_true",
 920         help="Do not include stderr output in the output file",
 921     )
 922     parser.add_argument(
 923         "--batch", action="store_true", help="Run tests in batches to the test program"
 924     )
 925     parser.add_argument(
 926         "--ignore-error-text",
 927         action="store_true",
 928         help="Do not compare error text when verifying output",
 929     )
 930     parser.add_argument(
 931         "--only-compare-error-lines",
 932         action="store_true",
 933         help="Does not care about exact expected error message, "
 934         "but only compare the error line numbers.",
 935     )
 936     parser.add_argument(
 937         "--check-expected-included-in-actual",
 938         action="store_true",
 939         help="Check that the set of lines in the expected file is included in the set"
 940         " of lines in the output",
 941     )
 942     parser.add_argument(
 943         "--timeout",
 944         type=int,
 945         help="Timeout in seconds for each test, in non-batch mode.",
 946     )
 947     parser.add_argument(
 948         "--force-color",
 949         action="store_true",
 950         help="Set the FORCE_ERROR_COLOR environment variable, "
 951         "which causes the test output to retain terminal escape codes.",
 952     )
 953     parser.add_argument(
 954         "--no-hh-flags", action="store_true", help="Do not read HH_FLAGS files"
 955     )
 956     parser.add_argument(
 957         "--verify-pessimisation",
 958         type=VerifyPessimisationOptions,
 959         choices=list(VerifyPessimisationOptions),
 960         default=VerifyPessimisationOptions.no,
 961         help="Experimental test suite for hh_pessimisation",
 962     )
 963     parser.add_argument(
 964         "--normalize-paths",
 965         action="store_true",
 966         help="Match on normalized file paths in error messages",
 967     )
 968     parser.add_argument(
 969         "--filter-glog-failures",
 970         action="store_true",
 971         help='Filters out glog messages of the "COULD NOT CREATE A LOGGINGFILE" form.',
 972     )
 973     parser.epilog = (
 974         "%s looks for a file named HH_FLAGS in the same directory"
 975         " as the test files it is executing. If found, the "
 976         "contents will be passed as arguments to "
 977         "<program> in addition to any arguments "
 978         "specified by --flags" % parser.prog
 979     )
 980     args = parser.parse_args()
 981
 982     max_workers = args.max_workers
 983     verbose = args.verbose
 984
 985     dump_on_failure = args.diff
 986     if os.getenv("SANDCASTLE") is not None:
 987         dump_on_failure = True
 988
 989     if not os.path.isfile(args.program):
 990         raise Exception("Could not find program at %s" % args.program)
 991
 992     # 'args.test_path' is a path relative to the current working
 993     # directory. buck1 runs this test from fbsource/fbocde, buck2 runs
 994     # it from fbsource.
 995     if os.path.basename(os.getcwd()) != "fbsource":
 996
 997         # If running under buck1 then we are in fbcode, if running
 998         # under dune then some ancestor directory of fbcode. These two
 999         # cases are handled by the logic of this script and
1000         # 'review.sh' and there are no adjustments to make.
1001         pass
1002     else:
1003
1004         # The buck2 case has us running in fbsource. This puts us in
1005         # fbcode.
1006         os.chdir("fbcode")
1007
1008     files: List[str] = list_test_files(
1009         args.test_path,
1010         args.disabled_extension,
1011         args.in_extension,
1012         args.include_directories,
1013     )
1014
1015     if len(files) == 0:
1016         raise Exception("Could not find any files to test in " + args.test_path)
1017
1018     mode_flag: List[str] = [] if args.mode_flag is None else [args.mode_flag]
1019     get_flags: Callable[[str], List[str]] = (
1020         get_flags_dummy(args.flags) if args.no_hh_flags else get_flags_cache(args.flags)
1021     )
1022
1023     results: List[Result] = run_tests(
1024         files,
1025         args.expect_extension,
1026         args.fallback_expect_extension,
1027         args.out_extension,
1028         args.stdin,
1029         args.program,
1030         args.default_expect_regex,
1031         args.batch,
1032         args.ignore_error_text,
1033         args.no_stderr,
1034         args.force_color,
1035         args.normalize_paths,
1036         args.verify_pessimisation,
1037         mode_flag,
1038         get_flags,
1039         check_expected_included_in_actual=args.check_expected_included_in_actual,
1040         timeout=args.timeout,
1041         only_compare_error_lines=args.only_compare_error_lines,
1042         filter_glog_failures=args.filter_glog_failures,
1043     )
1044
1045     # Doesn't make sense to check failures for idempotence
1046     successes: List[Result] = [result for result in results if not result.is_failure]
1047
1048     if args.idempotence and successes:
1049         run_idempotence_tests(
1050             successes,
1051             args.expect_extension,
1052             args.out_extension,
1053             args.program,
1054             args.default_expect_regex,
1055             mode_flag,
1056             get_flags,
1057             args.normalize_paths,
1058             check_expected_included_in_actual=args.check_expected_included_in_actual,
1059         )
1060
1061
1062 args: argparse.Namespace
1063
1064
1065 if __name__ == "__main__":
1066     main()