test/functional/test_runner.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2014-2016 The Bitcoin Core developers
   3 # Distributed under the MIT software license, see the accompanying
   4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5 """Run regression test suite.
   6
   7 This module calls down into individual test cases via subprocess. It will
   8 forward all unrecognized arguments onto the individual test scripts.
   9
  10 Functional tests are disabled on Windows by default. Use --force to run them anyway.
  11
  12 For a description of arguments recognized by test scripts, see
  13 `test/functional/test_framework/test_framework.py:BitcoinTestFramework.main`.
  14
  15 """
  16
  17 import argparse
  18 import configparser
  19 import datetime
  20 import os
  21 import time
  22 import shutil
  23 import signal
  24 import sys
  25 import subprocess
  26 import tempfile
  27 import re
  28 import logging
  29
  30 # Formatting. Default colors to empty strings.
  31 BOLD, BLUE, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "")
  32 try:
  33     # Make sure python thinks it can write unicode to its stdout
  34     "\u2713".encode("utf_8").decode(sys.stdout.encoding)
  35     TICK = "✓ "
  36     CROSS = "✖ "
  37     CIRCLE = "○ "
  38 except UnicodeDecodeError:
  39     TICK = "P "
  40     CROSS = "x "
  41     CIRCLE = "o "
  42
  43 if os.name == 'posix':
  44     # primitive formatting on supported
  45     # terminal via ANSI escape sequences:
  46     BOLD = ('\033[0m', '\033[1m')
  47     BLUE = ('\033[0m', '\033[0;34m')
  48     RED = ('\033[0m', '\033[0;31m')
  49     GREY = ('\033[0m', '\033[1;30m')
  50
  51 TEST_EXIT_PASSED = 0
  52 TEST_EXIT_SKIPPED = 77
  53
  54 BASE_SCRIPTS= [
  55     # Scripts that are run by the travis build process.
  56     # Longest test should go first, to favor running tests in parallel
  57     'wallet-hd.py',
  58     'walletbackup.py',
  59     # vv Tests less than 5m vv
  60     'p2p-fullblocktest.py',
  61     'fundrawtransaction.py',
  62     'p2p-compactblocks.py',
  63     'segwit.py',
  64     # vv Tests less than 2m vv
  65     'wallet.py',
  66     'multiwallet.py',
  67     'wallet-accounts.py',
  68     'p2p-segwit.py',
  69     'wallet-dump.py',
  70     'listtransactions.py',
  71     # vv Tests less than 60s vv
  72     'sendheaders.py',
  73     'zapwallettxes.py',
  74     'importmulti.py',
  75     'mempool_limit.py',
  76     'merkle_blocks.py',
  77     'receivedby.py',
  78     'abandonconflict.py',
  79     'bip68-112-113-p2p.py',
  80     'rawtransactions.py',
  81     'reindex.py',
  82     # vv Tests less than 30s vv
  83     'zmq_test.py',
  84     'mempool_resurrect_test.py',
  85     'txn_doublespend.py --mineblock',
  86     'txn_clone.py',
  87     'getchaintips.py',
  88     'rest.py',
  89     'mempool_spendcoinbase.py',
  90     'mempool_reorg.py',
  91     'mempool_persist.py',
  92     'multiwallet.py',
  93     'httpbasics.py',
  94     'multi_rpc.py',
  95     'proxy_test.py',
  96     'signrawtransactions.py',
  97     'disconnect_ban.py',
  98     'decodescript.py',
  99     'blockchain.py',
 100     'disablewallet.py',
 101     'net.py',
 102     'keypool.py',
 103     'p2p-mempool.py',
 104     'prioritise_transaction.py',
 105     'invalidblockrequest.py',
 106     'invalidtxrequest.py',
 107     'p2p-versionbits-warning.py',
 108     'preciousblock.py',
 109     'importprunedfunds.py',
 110     'signmessages.py',
 111     'nulldummy.py',
 112     'import-rescan.py',
 113     'mining.py',
 114     'bumpfee.py',
 115     'rpcnamedargs.py',
 116     'listsinceblock.py',
 117     'p2p-leaktests.py',
 118     'wallet-encryption.py',
 119     'uptime.py',
 120 ]
 121
 122 EXTENDED_SCRIPTS = [
 123     # These tests are not run by the travis build process.
 124     # Longest test should go first, to favor running tests in parallel
 125     'pruning.py',
 126     # vv Tests less than 20m vv
 127     'smartfees.py',
 128     # vv Tests less than 5m vv
 129     'maxuploadtarget.py',
 130     'mempool_packages.py',
 131     'dbcrash.py',
 132     # vv Tests less than 2m vv
 133     'bip68-sequence.py',
 134     'getblocktemplate_longpoll.py',
 135     'p2p-timeouts.py',
 136     # vv Tests less than 60s vv
 137     'bip9-softforks.py',
 138     'p2p-feefilter.py',
 139     'rpcbind_test.py',
 140     # vv Tests less than 30s vv
 141     'assumevalid.py',
 142     'bip65-cltv.py',
 143     'bip65-cltv-p2p.py',
 144     'bipdersig-p2p.py',
 145     'bipdersig.py',
 146     'example_test.py',
 147     'txn_doublespend.py',
 148     'txn_clone.py --mineblock',
 149     'forknotify.py',
 150     'invalidateblock.py',
 151     'p2p-acceptblock.py',
 152     'replace-by-fee.py',
 153 ]
 154
 155 # Place EXTENDED_SCRIPTS first since it has the 3 longest running tests
 156 ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS
 157
 158 NON_SCRIPTS = [
 159     # These are python files that live in the functional tests directory, but are not test scripts.
 160     "combine_logs.py",
 161     "create_cache.py",
 162     "test_runner.py",
 163 ]
 164
 165 def main():
 166     # Parse arguments and pass through unrecognised args
 167     parser = argparse.ArgumentParser(add_help=False,
 168                                      usage='%(prog)s [test_runner.py options] [script options] [scripts]',
 169                                      description=__doc__,
 170                                      epilog='''
 171     Help text and arguments for individual test script:''',
 172                                      formatter_class=argparse.RawTextHelpFormatter)
 173     parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface')
 174     parser.add_argument('--exclude', '-x', help='specify a comma-seperated-list of scripts to exclude.')
 175     parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests')
 176     parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).')
 177     parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit')
 178     parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.')
 179     parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.')
 180     parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs')
 181     parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs")
 182     args, unknown_args = parser.parse_known_args()
 183
 184     # args to be passed on always start with two dashes; tests are the remaining unknown args
 185     tests = [arg for arg in unknown_args if arg[:2] != "--"]
 186     passon_args = [arg for arg in unknown_args if arg[:2] == "--"]
 187
 188     # Read config generated by configure.
 189     config = configparser.ConfigParser()
 190     configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
 191     config.read_file(open(configfile))
 192
 193     passon_args.append("--configfile=%s" % configfile)
 194
 195     # Set up logging
 196     logging_level = logging.INFO if args.quiet else logging.DEBUG
 197     logging.basicConfig(format='%(message)s', level=logging_level)
 198
 199     # Create base test directory
 200     tmpdir = "%s/bitcoin_test_runner_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
 201     os.makedirs(tmpdir)
 202
 203     logging.debug("Temporary test directory at %s" % tmpdir)
 204
 205     enable_wallet = config["components"].getboolean("ENABLE_WALLET")
 206     enable_utils = config["components"].getboolean("ENABLE_UTILS")
 207     enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND")
 208
 209     if config["environment"]["EXEEXT"] == ".exe" and not args.force:
 210         # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
 211         # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
 212         print("Tests currently disabled on Windows by default. Use --force option to enable")
 213         sys.exit(0)
 214
 215     if not (enable_wallet and enable_utils and enable_bitcoind):
 216         print("No functional tests to run. Wallet, utils, and bitcoind must all be enabled")
 217         print("Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make")
 218         sys.exit(0)
 219
 220     # Build list of tests
 221     if tests:
 222         # Individual tests have been specified. Run specified tests that exist
 223         # in the ALL_SCRIPTS list. Accept the name with or without .py extension.
 224         tests = [re.sub("\.py$", "", t) + ".py" for t in tests]
 225         test_list = []
 226         for t in tests:
 227             if t in ALL_SCRIPTS:
 228                 test_list.append(t)
 229             else:
 230                 print("{}WARNING!{} Test '{}' not found in full test list.".format(BOLD[1], BOLD[0], t))
 231     else:
 232         # No individual tests have been specified.
 233         # Run all base tests, and optionally run extended tests.
 234         test_list = BASE_SCRIPTS
 235         if args.extended:
 236             # place the EXTENDED_SCRIPTS first since the three longest ones
 237             # are there and the list is shorter
 238             test_list = EXTENDED_SCRIPTS + test_list
 239
 240     # Remove the test cases that the user has explicitly asked to exclude.
 241     if args.exclude:
 242         tests_excl = [re.sub("\.py$", "", t) + ".py" for t in args.exclude.split(',')]
 243         for exclude_test in tests_excl:
 244             if exclude_test in test_list:
 245                 test_list.remove(exclude_test)
 246             else:
 247                 print("{}WARNING!{} Test '{}' not found in current test list.".format(BOLD[1], BOLD[0], exclude_test))
 248
 249     if not test_list:
 250         print("No valid test scripts specified. Check that your test is in one "
 251               "of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests")
 252         sys.exit(0)
 253
 254     if args.help:
 255         # Print help for test_runner.py, then print help of the first script (with args removed) and exit.
 256         parser.print_help()
 257         subprocess.check_call([(config["environment"]["SRCDIR"] + '/test/functional/' + test_list[0].split()[0])] + ['-h'])
 258         sys.exit(0)
 259
 260     check_script_list(config["environment"]["SRCDIR"])
 261
 262     if not args.keepcache:
 263         shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True)
 264
 265     run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args)
 266
 267 def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=[]):
 268     # Warn if bitcoind is already running (unix only)
 269     try:
 270         if subprocess.check_output(["pidof", "bitcoind"]) is not None:
 271             print("%sWARNING!%s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0]))
 272     except (OSError, subprocess.SubprocessError):
 273         pass
 274
 275     # Warn if there is a cache directory
 276     cache_dir = "%s/test/cache" % build_dir
 277     if os.path.isdir(cache_dir):
 278         print("%sWARNING!%s There is a cache directory here: %s. If tests fail unexpectedly, try deleting the cache directory." % (BOLD[1], BOLD[0], cache_dir))
 279
 280     #Set env vars
 281     if "BITCOIND" not in os.environ:
 282         os.environ["BITCOIND"] = build_dir + '/src/bitcoind' + exeext
 283
 284     tests_dir = src_dir + '/test/functional/'
 285
 286     flags = ["--srcdir={}/src".format(build_dir)] + args
 287     flags.append("--cachedir=%s" % cache_dir)
 288
 289     if enable_coverage:
 290         coverage = RPCCoverage()
 291         flags.append(coverage.flag)
 292         logging.debug("Initializing coverage directory at %s" % coverage.dir)
 293     else:
 294         coverage = None
 295
 296     if len(test_list) > 1 and jobs > 1:
 297         # Populate cache
 298         subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir])
 299
 300     #Run Tests
 301     job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags)
 302     time0 = time.time()
 303     test_results = []
 304
 305     max_len_name = len(max(test_list, key=len))
 306
 307     for _ in range(len(test_list)):
 308         test_result, stdout, stderr = job_queue.get_next()
 309         test_results.append(test_result)
 310
 311         if test_result.status == "Passed":
 312             logging.debug("\n%s%s%s passed, Duration: %s s" % (BOLD[1], test_result.name, BOLD[0], test_result.time))
 313         elif test_result.status == "Skipped":
 314             logging.debug("\n%s%s%s skipped" % (BOLD[1], test_result.name, BOLD[0]))
 315         else:
 316             print("\n%s%s%s failed, Duration: %s s\n" % (BOLD[1], test_result.name, BOLD[0], test_result.time))
 317             print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n')
 318             print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n')
 319
 320     print_results(test_results, max_len_name, (int(time.time() - time0)))
 321
 322     if coverage:
 323         coverage.report_rpc_coverage()
 324
 325         logging.debug("Cleaning up coverage data")
 326         coverage.cleanup()
 327
 328     # Clear up the temp directory if all subdirectories are gone
 329     if not os.listdir(tmpdir):
 330         os.rmdir(tmpdir)
 331
 332     all_passed = all(map(lambda test_result: test_result.was_successful, test_results))
 333
 334     sys.exit(not all_passed)
 335
 336 def print_results(test_results, max_len_name, runtime):
 337     results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS   ", "DURATION") + BOLD[0]
 338
 339     test_results.sort(key=lambda result: result.name.lower())
 340     all_passed = True
 341     time_sum = 0
 342
 343     for test_result in test_results:
 344         all_passed = all_passed and test_result.was_successful
 345         time_sum += test_result.time
 346         test_result.padding = max_len_name
 347         results += str(test_result)
 348
 349     status = TICK + "Passed" if all_passed else CROSS + "Failed"
 350     results += BOLD[1] + "\n%s | %s | %s s (accumulated) \n" % ("ALL".ljust(max_len_name), status.ljust(9), time_sum) + BOLD[0]
 351     results += "Runtime: %s s\n" % (runtime)
 352     print(results)
 353
 354 class TestHandler:
 355     """
 356     Trigger the testscrips passed in via the list.
 357     """
 358
 359     def __init__(self, num_tests_parallel, tests_dir, tmpdir, test_list=None, flags=None):
 360         assert(num_tests_parallel >= 1)
 361         self.num_jobs = num_tests_parallel
 362         self.tests_dir = tests_dir
 363         self.tmpdir = tmpdir
 364         self.test_list = test_list
 365         self.flags = flags
 366         self.num_running = 0
 367         # In case there is a graveyard of zombie bitcoinds, we can apply a
 368         # pseudorandom offset to hopefully jump over them.
 369         # (625 is PORT_RANGE/MAX_NODES)
 370         self.portseed_offset = int(time.time() * 1000) % 625
 371         self.jobs = []
 372
 373     def get_next(self):
 374         while self.num_running < self.num_jobs and self.test_list:
 375             # Add tests
 376             self.num_running += 1
 377             t = self.test_list.pop(0)
 378             portseed = len(self.test_list) + self.portseed_offset
 379             portseed_arg = ["--portseed={}".format(portseed)]
 380             log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16)
 381             log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16)
 382             test_argv = t.split()
 383             tmpdir = ["--tmpdir=%s/%s_%s" % (self.tmpdir, re.sub(".py$", "", test_argv[0]), portseed)]
 384             self.jobs.append((t,
 385                               time.time(),
 386                               subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir,
 387                                                universal_newlines=True,
 388                                                stdout=log_stdout,
 389                                                stderr=log_stderr),
 390                               log_stdout,
 391                               log_stderr))
 392         if not self.jobs:
 393             raise IndexError('pop from empty list')
 394         while True:
 395             # Return first proc that finishes
 396             time.sleep(.5)
 397             for j in self.jobs:
 398                 (name, time0, proc, log_out, log_err) = j
 399                 if os.getenv('TRAVIS') == 'true' and int(time.time() - time0) > 20 * 60:
 400                     # In travis, timeout individual tests after 20 minutes (to stop tests hanging and not
 401                     # providing useful output.
 402                     proc.send_signal(signal.SIGINT)
 403                 if proc.poll() is not None:
 404                     log_out.seek(0), log_err.seek(0)
 405                     [stdout, stderr] = [l.read().decode('utf-8') for l in (log_out, log_err)]
 406                     log_out.close(), log_err.close()
 407                     if proc.returncode == TEST_EXIT_PASSED and stderr == "":
 408                         status = "Passed"
 409                     elif proc.returncode == TEST_EXIT_SKIPPED:
 410                         status = "Skipped"
 411                     else:
 412                         status = "Failed"
 413                     self.num_running -= 1
 414                     self.jobs.remove(j)
 415
 416                     return TestResult(name, status, int(time.time() - time0)), stdout, stderr
 417             print('.', end='', flush=True)
 418
 419 class TestResult():
 420     def __init__(self, name, status, time):
 421         self.name = name
 422         self.status = status
 423         self.time = time
 424         self.padding = 0
 425
 426     def __repr__(self):
 427         if self.status == "Passed":
 428             color = BLUE
 429             glyph = TICK
 430         elif self.status == "Failed":
 431             color = RED
 432             glyph = CROSS
 433         elif self.status == "Skipped":
 434             color = GREY
 435             glyph = CIRCLE
 436
 437         return color[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0]
 438
 439     @property
 440     def was_successful(self):
 441         return self.status != "Failed"
 442
 443
 444 def check_script_list(src_dir):
 445     """Check scripts directory.
 446
 447     Check that there are no scripts in the functional tests directory which are
 448     not being run by pull-tester.py."""
 449     script_dir = src_dir + '/test/functional/'
 450     python_files = set([t for t in os.listdir(script_dir) if t[-3:] == ".py"])
 451     missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS)))
 452     if len(missed_tests) != 0:
 453         print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests)))
 454         if os.getenv('TRAVIS') == 'true':
 455             # On travis this warning is an error to prevent merging incomplete commits into master
 456             sys.exit(1)
 457
 458 class RPCCoverage(object):
 459     """
 460     Coverage reporting utilities for test_runner.
 461
 462     Coverage calculation works by having each test script subprocess write
 463     coverage files into a particular directory. These files contain the RPC
 464     commands invoked during testing, as well as a complete listing of RPC
 465     commands per `bitcoin-cli help` (`rpc_interface.txt`).
 466
 467     After all tests complete, the commands run are combined and diff'd against
 468     the complete list to calculate uncovered RPC commands.
 469
 470     See also: test/functional/test_framework/coverage.py
 471
 472     """
 473     def __init__(self):
 474         self.dir = tempfile.mkdtemp(prefix="coverage")
 475         self.flag = '--coveragedir=%s' % self.dir
 476
 477     def report_rpc_coverage(self):
 478         """
 479         Print out RPC commands that were unexercised by tests.
 480
 481         """
 482         uncovered = self._get_uncovered_rpc_commands()
 483
 484         if uncovered:
 485             print("Uncovered RPC commands:")
 486             print("".join(("  - %s\n" % i) for i in sorted(uncovered)))
 487         else:
 488             print("All RPC commands covered.")
 489
 490     def cleanup(self):
 491         return shutil.rmtree(self.dir)
 492
 493     def _get_uncovered_rpc_commands(self):
 494         """
 495         Return a set of currently untested RPC commands.
 496
 497         """
 498         # This is shared from `test/functional/test-framework/coverage.py`
 499         reference_filename = 'rpc_interface.txt'
 500         coverage_file_prefix = 'coverage.'
 501
 502         coverage_ref_filename = os.path.join(self.dir, reference_filename)
 503         coverage_filenames = set()
 504         all_cmds = set()
 505         covered_cmds = set()
 506
 507         if not os.path.isfile(coverage_ref_filename):
 508             raise RuntimeError("No coverage reference found")
 509
 510         with open(coverage_ref_filename, 'r') as f:
 511             all_cmds.update([i.strip() for i in f.readlines()])
 512
 513         for root, dirs, files in os.walk(self.dir):
 514             for filename in files:
 515                 if filename.startswith(coverage_file_prefix):
 516                     coverage_filenames.add(os.path.join(root, filename))
 517
 518         for filename in coverage_filenames:
 519             with open(filename, 'r') as f:
 520                 covered_cmds.update([i.strip() for i in f.readlines()])
 521
 522         return all_cmds - covered_cmds
 523
 524
 525 if __name__ == '__main__':
 526     main()