test/functional/test_runner.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2014-2016 The Bitcoin Core developers
   3 # Distributed under the MIT software license, see the accompanying
   4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5 """Run regression test suite.
   6
   7 This module calls down into individual test cases via subprocess. It will
   8 forward all unrecognized arguments onto the individual test scripts.
   9
  10 Functional tests are disabled on Windows by default. Use --force to run them anyway.
  11
  12 For a description of arguments recognized by test scripts, see
  13 `test/functional/test_framework/test_framework.py:BitcoinTestFramework.main`.
  14
  15 """
  16
  17 import argparse
  18 import configparser
  19 import datetime
  20 import os
  21 import time
  22 import shutil
  23 import signal
  24 import sys
  25 import subprocess
  26 import tempfile
  27 import re
  28 import logging
  29
  30 # Formatting. Default colors to empty strings.
  31 BOLD, BLUE, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "")
  32 try:
  33     # Make sure python thinks it can write unicode to its stdout
  34     "\u2713".encode("utf_8").decode(sys.stdout.encoding)
  35     TICK = "✓ "
  36     CROSS = "✖ "
  37     CIRCLE = "○ "
  38 except UnicodeDecodeError:
  39     TICK = "P "
  40     CROSS = "x "
  41     CIRCLE = "o "
  42
  43 if os.name == 'posix':
  44     # primitive formatting on supported
  45     # terminal via ANSI escape sequences:
  46     BOLD = ('\033[0m', '\033[1m')
  47     BLUE = ('\033[0m', '\033[0;34m')
  48     RED = ('\033[0m', '\033[0;31m')
  49     GREY = ('\033[0m', '\033[1;30m')
  50
  51 TEST_EXIT_PASSED = 0
  52 TEST_EXIT_SKIPPED = 77
  53
  54 BASE_SCRIPTS= [
  55     # Scripts that are run by the travis build process.
  56     # Longest test should go first, to favor running tests in parallel
  57     'wallet-hd.py',
  58     'walletbackup.py',
  59     # vv Tests less than 5m vv
  60     'p2p-fullblocktest.py',
  61     'fundrawtransaction.py',
  62     'p2p-compactblocks.py',
  63     'segwit.py',
  64     # vv Tests less than 2m vv
  65     'wallet.py',
  66     'wallet-accounts.py',
  67     'p2p-segwit.py',
  68     'wallet-dump.py',
  69     'listtransactions.py',
  70     # vv Tests less than 60s vv
  71     'sendheaders.py',
  72     'zapwallettxes.py',
  73     'importmulti.py',
  74     'mempool_limit.py',
  75     'merkle_blocks.py',
  76     'receivedby.py',
  77     'abandonconflict.py',
  78     'bip68-112-113-p2p.py',
  79     'rawtransactions.py',
  80     'reindex.py',
  81     # vv Tests less than 30s vv
  82     'keypool-topup.py',
  83     'zmq_test.py',
  84     'mempool_resurrect_test.py',
  85     'txn_doublespend.py --mineblock',
  86     'txn_clone.py',
  87     'getchaintips.py',
  88     'rest.py',
  89     'mempool_spendcoinbase.py',
  90     'mempool_reorg.py',
  91     'mempool_persist.py',
  92     'multiwallet.py',
  93     'httpbasics.py',
  94     'multi_rpc.py',
  95     'proxy_test.py',
  96     'signrawtransactions.py',
  97     'disconnect_ban.py',
  98     'decodescript.py',
  99     'blockchain.py',
 100     'disablewallet.py',
 101     'net.py',
 102     'keypool.py',
 103     'p2p-mempool.py',
 104     'prioritise_transaction.py',
 105     'invalidblockrequest.py',
 106     'invalidtxrequest.py',
 107     'p2p-versionbits-warning.py',
 108     'preciousblock.py',
 109     'importprunedfunds.py',
 110     'signmessages.py',
 111     'nulldummy.py',
 112     'import-rescan.py',
 113     'mining.py',
 114     'bumpfee.py',
 115     'rpcnamedargs.py',
 116     'listsinceblock.py',
 117     'p2p-leaktests.py',
 118     'wallet-encryption.py',
 119     'bipdersig-p2p.py',
 120     'bip65-cltv-p2p.py',
 121     'uptime.py',
 122     'resendwallettransactions.py',
 123 ]
 124
 125 EXTENDED_SCRIPTS = [
 126     # These tests are not run by the travis build process.
 127     # Longest test should go first, to favor running tests in parallel
 128     'pruning.py',
 129     # vv Tests less than 20m vv
 130     'smartfees.py',
 131     # vv Tests less than 5m vv
 132     'maxuploadtarget.py',
 133     'mempool_packages.py',
 134     'dbcrash.py',
 135     # vv Tests less than 2m vv
 136     'bip68-sequence.py',
 137     'getblocktemplate_longpoll.py',
 138     'p2p-timeouts.py',
 139     # vv Tests less than 60s vv
 140     'bip9-softforks.py',
 141     'p2p-feefilter.py',
 142     'rpcbind_test.py',
 143     # vv Tests less than 30s vv
 144     'assumevalid.py',
 145     'example_test.py',
 146     'txn_doublespend.py',
 147     'txn_clone.py --mineblock',
 148     'forknotify.py',
 149     'invalidateblock.py',
 150     'p2p-acceptblock.py',
 151     'replace-by-fee.py',
 152 ]
 153
 154 # Place EXTENDED_SCRIPTS first since it has the 3 longest running tests
 155 ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS
 156
 157 NON_SCRIPTS = [
 158     # These are python files that live in the functional tests directory, but are not test scripts.
 159     "combine_logs.py",
 160     "create_cache.py",
 161     "test_runner.py",
 162 ]
 163
 164 def main():
 165     # Parse arguments and pass through unrecognised args
 166     parser = argparse.ArgumentParser(add_help=False,
 167                                      usage='%(prog)s [test_runner.py options] [script options] [scripts]',
 168                                      description=__doc__,
 169                                      epilog='''
 170     Help text and arguments for individual test script:''',
 171                                      formatter_class=argparse.RawTextHelpFormatter)
 172     parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface')
 173     parser.add_argument('--exclude', '-x', help='specify a comma-separated-list of scripts to exclude.')
 174     parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests')
 175     parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).')
 176     parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit')
 177     parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.')
 178     parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.')
 179     parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs')
 180     parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs")
 181     args, unknown_args = parser.parse_known_args()
 182
 183     # args to be passed on always start with two dashes; tests are the remaining unknown args
 184     tests = [arg for arg in unknown_args if arg[:2] != "--"]
 185     passon_args = [arg for arg in unknown_args if arg[:2] == "--"]
 186
 187     # Read config generated by configure.
 188     config = configparser.ConfigParser()
 189     configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
 190     config.read_file(open(configfile))
 191
 192     passon_args.append("--configfile=%s" % configfile)
 193
 194     # Set up logging
 195     logging_level = logging.INFO if args.quiet else logging.DEBUG
 196     logging.basicConfig(format='%(message)s', level=logging_level)
 197
 198     # Create base test directory
 199     tmpdir = "%s/bitcoin_test_runner_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
 200     os.makedirs(tmpdir)
 201
 202     logging.debug("Temporary test directory at %s" % tmpdir)
 203
 204     enable_wallet = config["components"].getboolean("ENABLE_WALLET")
 205     enable_utils = config["components"].getboolean("ENABLE_UTILS")
 206     enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND")
 207
 208     if config["environment"]["EXEEXT"] == ".exe" and not args.force:
 209         # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
 210         # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
 211         print("Tests currently disabled on Windows by default. Use --force option to enable")
 212         sys.exit(0)
 213
 214     if not (enable_wallet and enable_utils and enable_bitcoind):
 215         print("No functional tests to run. Wallet, utils, and bitcoind must all be enabled")
 216         print("Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make")
 217         sys.exit(0)
 218
 219     # Build list of tests
 220     if tests:
 221         # Individual tests have been specified. Run specified tests that exist
 222         # in the ALL_SCRIPTS list. Accept the name with or without .py extension.
 223         tests = [re.sub("\.py$", "", t) + ".py" for t in tests]
 224         test_list = []
 225         for t in tests:
 226             if t in ALL_SCRIPTS:
 227                 test_list.append(t)
 228             else:
 229                 print("{}WARNING!{} Test '{}' not found in full test list.".format(BOLD[1], BOLD[0], t))
 230     else:
 231         # No individual tests have been specified.
 232         # Run all base tests, and optionally run extended tests.
 233         test_list = BASE_SCRIPTS
 234         if args.extended:
 235             # place the EXTENDED_SCRIPTS first since the three longest ones
 236             # are there and the list is shorter
 237             test_list = EXTENDED_SCRIPTS + test_list
 238
 239     # Remove the test cases that the user has explicitly asked to exclude.
 240     if args.exclude:
 241         tests_excl = [re.sub("\.py$", "", t) + ".py" for t in args.exclude.split(',')]
 242         for exclude_test in tests_excl:
 243             if exclude_test in test_list:
 244                 test_list.remove(exclude_test)
 245             else:
 246                 print("{}WARNING!{} Test '{}' not found in current test list.".format(BOLD[1], BOLD[0], exclude_test))
 247
 248     if not test_list:
 249         print("No valid test scripts specified. Check that your test is in one "
 250               "of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests")
 251         sys.exit(0)
 252
 253     if args.help:
 254         # Print help for test_runner.py, then print help of the first script (with args removed) and exit.
 255         parser.print_help()
 256         subprocess.check_call([(config["environment"]["SRCDIR"] + '/test/functional/' + test_list[0].split()[0])] + ['-h'])
 257         sys.exit(0)
 258
 259     check_script_list(config["environment"]["SRCDIR"])
 260
 261     if not args.keepcache:
 262         shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True)
 263
 264     run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args)
 265
 266 def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=[]):
 267     # Warn if bitcoind is already running (unix only)
 268     try:
 269         if subprocess.check_output(["pidof", "bitcoind"]) is not None:
 270             print("%sWARNING!%s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0]))
 271     except (OSError, subprocess.SubprocessError):
 272         pass
 273
 274     # Warn if there is a cache directory
 275     cache_dir = "%s/test/cache" % build_dir
 276     if os.path.isdir(cache_dir):
 277         print("%sWARNING!%s There is a cache directory here: %s. If tests fail unexpectedly, try deleting the cache directory." % (BOLD[1], BOLD[0], cache_dir))
 278
 279     #Set env vars
 280     if "BITCOIND" not in os.environ:
 281         os.environ["BITCOIND"] = build_dir + '/src/bitcoind' + exeext
 282
 283     tests_dir = src_dir + '/test/functional/'
 284
 285     flags = ["--srcdir={}/src".format(build_dir)] + args
 286     flags.append("--cachedir=%s" % cache_dir)
 287
 288     if enable_coverage:
 289         coverage = RPCCoverage()
 290         flags.append(coverage.flag)
 291         logging.debug("Initializing coverage directory at %s" % coverage.dir)
 292     else:
 293         coverage = None
 294
 295     if len(test_list) > 1 and jobs > 1:
 296         # Populate cache
 297         subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir])
 298
 299     #Run Tests
 300     job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags)
 301     time0 = time.time()
 302     test_results = []
 303
 304     max_len_name = len(max(test_list, key=len))
 305
 306     for _ in range(len(test_list)):
 307         test_result, stdout, stderr = job_queue.get_next()
 308         test_results.append(test_result)
 309
 310         if test_result.status == "Passed":
 311             logging.debug("\n%s%s%s passed, Duration: %s s" % (BOLD[1], test_result.name, BOLD[0], test_result.time))
 312         elif test_result.status == "Skipped":
 313             logging.debug("\n%s%s%s skipped" % (BOLD[1], test_result.name, BOLD[0]))
 314         else:
 315             print("\n%s%s%s failed, Duration: %s s\n" % (BOLD[1], test_result.name, BOLD[0], test_result.time))
 316             print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n')
 317             print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n')
 318
 319     print_results(test_results, max_len_name, (int(time.time() - time0)))
 320
 321     if coverage:
 322         coverage.report_rpc_coverage()
 323
 324         logging.debug("Cleaning up coverage data")
 325         coverage.cleanup()
 326
 327     # Clear up the temp directory if all subdirectories are gone
 328     if not os.listdir(tmpdir):
 329         os.rmdir(tmpdir)
 330
 331     all_passed = all(map(lambda test_result: test_result.was_successful, test_results))
 332
 333     sys.exit(not all_passed)
 334
 335 def print_results(test_results, max_len_name, runtime):
 336     results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS   ", "DURATION") + BOLD[0]
 337
 338     test_results.sort(key=lambda result: result.name.lower())
 339     all_passed = True
 340     time_sum = 0
 341
 342     for test_result in test_results:
 343         all_passed = all_passed and test_result.was_successful
 344         time_sum += test_result.time
 345         test_result.padding = max_len_name
 346         results += str(test_result)
 347
 348     status = TICK + "Passed" if all_passed else CROSS + "Failed"
 349     results += BOLD[1] + "\n%s | %s | %s s (accumulated) \n" % ("ALL".ljust(max_len_name), status.ljust(9), time_sum) + BOLD[0]
 350     results += "Runtime: %s s\n" % (runtime)
 351     print(results)
 352
 353 class TestHandler:
 354     """
 355     Trigger the test scripts passed in via the list.
 356     """
 357
 358     def __init__(self, num_tests_parallel, tests_dir, tmpdir, test_list=None, flags=None):
 359         assert(num_tests_parallel >= 1)
 360         self.num_jobs = num_tests_parallel
 361         self.tests_dir = tests_dir
 362         self.tmpdir = tmpdir
 363         self.test_list = test_list
 364         self.flags = flags
 365         self.num_running = 0
 366         # In case there is a graveyard of zombie bitcoinds, we can apply a
 367         # pseudorandom offset to hopefully jump over them.
 368         # (625 is PORT_RANGE/MAX_NODES)
 369         self.portseed_offset = int(time.time() * 1000) % 625
 370         self.jobs = []
 371
 372     def get_next(self):
 373         while self.num_running < self.num_jobs and self.test_list:
 374             # Add tests
 375             self.num_running += 1
 376             t = self.test_list.pop(0)
 377             portseed = len(self.test_list) + self.portseed_offset
 378             portseed_arg = ["--portseed={}".format(portseed)]
 379             log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16)
 380             log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16)
 381             test_argv = t.split()
 382             tmpdir = ["--tmpdir=%s/%s_%s" % (self.tmpdir, re.sub(".py$", "", test_argv[0]), portseed)]
 383             self.jobs.append((t,
 384                               time.time(),
 385                               subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir,
 386                                                universal_newlines=True,
 387                                                stdout=log_stdout,
 388                                                stderr=log_stderr),
 389                               log_stdout,
 390                               log_stderr))
 391         if not self.jobs:
 392             raise IndexError('pop from empty list')
 393         while True:
 394             # Return first proc that finishes
 395             time.sleep(.5)
 396             for j in self.jobs:
 397                 (name, time0, proc, log_out, log_err) = j
 398                 if os.getenv('TRAVIS') == 'true' and int(time.time() - time0) > 20 * 60:
 399                     # In travis, timeout individual tests after 20 minutes (to stop tests hanging and not
 400                     # providing useful output.
 401                     proc.send_signal(signal.SIGINT)
 402                 if proc.poll() is not None:
 403                     log_out.seek(0), log_err.seek(0)
 404                     [stdout, stderr] = [l.read().decode('utf-8') for l in (log_out, log_err)]
 405                     log_out.close(), log_err.close()
 406                     if proc.returncode == TEST_EXIT_PASSED and stderr == "":
 407                         status = "Passed"
 408                     elif proc.returncode == TEST_EXIT_SKIPPED:
 409                         status = "Skipped"
 410                     else:
 411                         status = "Failed"
 412                     self.num_running -= 1
 413                     self.jobs.remove(j)
 414
 415                     return TestResult(name, status, int(time.time() - time0)), stdout, stderr
 416             print('.', end='', flush=True)
 417
 418 class TestResult():
 419     def __init__(self, name, status, time):
 420         self.name = name
 421         self.status = status
 422         self.time = time
 423         self.padding = 0
 424
 425     def __repr__(self):
 426         if self.status == "Passed":
 427             color = BLUE
 428             glyph = TICK
 429         elif self.status == "Failed":
 430             color = RED
 431             glyph = CROSS
 432         elif self.status == "Skipped":
 433             color = GREY
 434             glyph = CIRCLE
 435
 436         return color[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0]
 437
 438     @property
 439     def was_successful(self):
 440         return self.status != "Failed"
 441
 442
 443 def check_script_list(src_dir):
 444     """Check scripts directory.
 445
 446     Check that there are no scripts in the functional tests directory which are
 447     not being run by pull-tester.py."""
 448     script_dir = src_dir + '/test/functional/'
 449     python_files = set([t for t in os.listdir(script_dir) if t[-3:] == ".py"])
 450     missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS)))
 451     if len(missed_tests) != 0:
 452         print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests)))
 453         if os.getenv('TRAVIS') == 'true':
 454             # On travis this warning is an error to prevent merging incomplete commits into master
 455             sys.exit(1)
 456
 457 class RPCCoverage(object):
 458     """
 459     Coverage reporting utilities for test_runner.
 460
 461     Coverage calculation works by having each test script subprocess write
 462     coverage files into a particular directory. These files contain the RPC
 463     commands invoked during testing, as well as a complete listing of RPC
 464     commands per `bitcoin-cli help` (`rpc_interface.txt`).
 465
 466     After all tests complete, the commands run are combined and diff'd against
 467     the complete list to calculate uncovered RPC commands.
 468
 469     See also: test/functional/test_framework/coverage.py
 470
 471     """
 472     def __init__(self):
 473         self.dir = tempfile.mkdtemp(prefix="coverage")
 474         self.flag = '--coveragedir=%s' % self.dir
 475
 476     def report_rpc_coverage(self):
 477         """
 478         Print out RPC commands that were unexercised by tests.
 479
 480         """
 481         uncovered = self._get_uncovered_rpc_commands()
 482
 483         if uncovered:
 484             print("Uncovered RPC commands:")
 485             print("".join(("  - %s\n" % i) for i in sorted(uncovered)))
 486         else:
 487             print("All RPC commands covered.")
 488
 489     def cleanup(self):
 490         return shutil.rmtree(self.dir)
 491
 492     def _get_uncovered_rpc_commands(self):
 493         """
 494         Return a set of currently untested RPC commands.
 495
 496         """
 497         # This is shared from `test/functional/test-framework/coverage.py`
 498         reference_filename = 'rpc_interface.txt'
 499         coverage_file_prefix = 'coverage.'
 500
 501         coverage_ref_filename = os.path.join(self.dir, reference_filename)
 502         coverage_filenames = set()
 503         all_cmds = set()
 504         covered_cmds = set()
 505
 506         if not os.path.isfile(coverage_ref_filename):
 507             raise RuntimeError("No coverage reference found")
 508
 509         with open(coverage_ref_filename, 'r') as f:
 510             all_cmds.update([i.strip() for i in f.readlines()])
 511
 512         for root, dirs, files in os.walk(self.dir):
 513             for filename in files:
 514                 if filename.startswith(coverage_file_prefix):
 515                     coverage_filenames.add(os.path.join(root, filename))
 516
 517         for filename in coverage_filenames:
 518             with open(filename, 'r') as f:
 519                 covered_cmds.update([i.strip() for i in f.readlines()])
 520
 521         return all_cmds - covered_cmds
 522
 523
 524 if __name__ == '__main__':
 525     main()