test/functional/test_runner.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2014-2016 The Bitcoin Core developers
   3 # Distributed under the MIT software license, see the accompanying
   4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5 """Run regression test suite.
   6
   7 This module calls down into individual test cases via subprocess. It will
   8 forward all unrecognized arguments onto the individual test scripts.
   9
  10 Functional tests are disabled on Windows by default. Use --force to run them anyway.
  11
  12 For a description of arguments recognized by test scripts, see
  13 `test/functional/test_framework/test_framework.py:BitcoinTestFramework.main`.
  14
  15 """
  16
  17 import argparse
  18 import configparser
  19 import os
  20 import time
  21 import shutil
  22 import sys
  23 import subprocess
  24 import tempfile
  25 import re
  26 import logging
  27
  28 # Formatting. Default colors to empty strings.
  29 BOLD, BLUE, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "")
  30 try:
  31     # Make sure python thinks it can write unicode to its stdout
  32     "\u2713".encode("utf_8").decode(sys.stdout.encoding)
  33     TICK = "✓ "
  34     CROSS = "✖ "
  35     CIRCLE = "○ "
  36 except UnicodeDecodeError:
  37     TICK = "P "
  38     CROSS = "x "
  39     CIRCLE = "o "
  40
  41 if os.name == 'posix':
  42     # primitive formatting on supported
  43     # terminal via ANSI escape sequences:
  44     BOLD = ('\033[0m', '\033[1m')
  45     BLUE = ('\033[0m', '\033[0;34m')
  46     RED = ('\033[0m', '\033[0;31m')
  47     GREY = ('\033[0m', '\033[1;30m')
  48
  49 TEST_EXIT_PASSED = 0
  50 TEST_EXIT_SKIPPED = 77
  51
  52 BASE_SCRIPTS= [
  53     # Scripts that are run by the travis build process.
  54     # Longest test should go first, to favor running tests in parallel
  55     'wallet-hd.py',
  56     'walletbackup.py',
  57     # vv Tests less than 5m vv
  58     'p2p-fullblocktest.py',
  59     'fundrawtransaction.py',
  60     'p2p-compactblocks.py',
  61     'segwit.py',
  62     # vv Tests less than 2m vv
  63     'wallet.py',
  64     'wallet-accounts.py',
  65     'p2p-segwit.py',
  66     'wallet-dump.py',
  67     'listtransactions.py',
  68     # vv Tests less than 60s vv
  69     'sendheaders.py',
  70     'zapwallettxes.py',
  71     'importmulti.py',
  72     'mempool_limit.py',
  73     'merkle_blocks.py',
  74     'receivedby.py',
  75     'abandonconflict.py',
  76     'bip68-112-113-p2p.py',
  77     'rawtransactions.py',
  78     'reindex.py',
  79     # vv Tests less than 30s vv
  80     "zmq_test.py",
  81     'mempool_resurrect_test.py',
  82     'txn_doublespend.py --mineblock',
  83     'txn_clone.py',
  84     'getchaintips.py',
  85     'rest.py',
  86     'mempool_spendcoinbase.py',
  87     'mempool_reorg.py',
  88     'mempool_persist.py',
  89     'httpbasics.py',
  90     'multi_rpc.py',
  91     'proxy_test.py',
  92     'signrawtransactions.py',
  93     'disconnect_ban.py',
  94     'decodescript.py',
  95     'blockchain.py',
  96     'disablewallet.py',
  97     'net.py',
  98     'keypool.py',
  99     'p2p-mempool.py',
 100     'prioritise_transaction.py',
 101     'invalidblockrequest.py',
 102     'invalidtxrequest.py',
 103     'p2p-versionbits-warning.py',
 104     'preciousblock.py',
 105     'importprunedfunds.py',
 106     'signmessages.py',
 107     'nulldummy.py',
 108     'import-rescan.py',
 109     'bumpfee.py',
 110     'rpcnamedargs.py',
 111     'listsinceblock.py',
 112     'p2p-leaktests.py',
 113 ]
 114
 115 EXTENDED_SCRIPTS = [
 116     # These tests are not run by the travis build process.
 117     # Longest test should go first, to favor running tests in parallel
 118     'pruning.py',
 119     # vv Tests less than 20m vv
 120     'smartfees.py',
 121     # vv Tests less than 5m vv
 122     'maxuploadtarget.py',
 123     'mempool_packages.py',
 124     # vv Tests less than 2m vv
 125     'bip68-sequence.py',
 126     'getblocktemplate_longpoll.py',
 127     'p2p-timeouts.py',
 128     # vv Tests less than 60s vv
 129     'bip9-softforks.py',
 130     'p2p-feefilter.py',
 131     'rpcbind_test.py',
 132     # vv Tests less than 30s vv
 133     'assumevalid.py',
 134     'bip65-cltv.py',
 135     'bip65-cltv-p2p.py',
 136     'bipdersig-p2p.py',
 137     'bipdersig.py',
 138     'getblocktemplate_proposals.py',
 139     'txn_doublespend.py',
 140     'txn_clone.py --mineblock',
 141     'forknotify.py',
 142     'invalidateblock.py',
 143     'p2p-acceptblock.py',
 144     'replace-by-fee.py',
 145 ]
 146
 147 # Place EXTENDED_SCRIPTS first since it has the 3 longest running tests
 148 ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS
 149
 150 NON_SCRIPTS = [
 151     # These are python files that live in the functional tests directory, but are not test scripts.
 152     "combine_logs.py",
 153     "create_cache.py",
 154     "test_runner.py",
 155 ]
 156
 157 def main():
 158     # Parse arguments and pass through unrecognised args
 159     parser = argparse.ArgumentParser(add_help=False,
 160                                      usage='%(prog)s [test_runner.py options] [script options] [scripts]',
 161                                      description=__doc__,
 162                                      epilog='''
 163     Help text and arguments for individual test script:''',
 164                                      formatter_class=argparse.RawTextHelpFormatter)
 165     parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface')
 166     parser.add_argument('--exclude', '-x', help='specify a comma-seperated-list of scripts to exclude. Do not include the .py extension in the name.')
 167     parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests')
 168     parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).')
 169     parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit')
 170     parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.')
 171     parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.')
 172     parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs')
 173     args, unknown_args = parser.parse_known_args()
 174
 175     # Create a set to store arguments and create the passon string
 176     tests = set(arg for arg in unknown_args if arg[:2] != "--")
 177     passon_args = [arg for arg in unknown_args if arg[:2] == "--"]
 178
 179     # Read config generated by configure.
 180     config = configparser.ConfigParser()
 181     configfile = os.path.abspath(os.path.dirname(__file__)) + "/config.ini"
 182     config.read_file(open(configfile))
 183
 184     passon_args.append("--configfile=%s" % configfile)
 185
 186     # Set up logging
 187     logging_level = logging.INFO if args.quiet else logging.DEBUG
 188     logging.basicConfig(format='%(message)s', level=logging_level)
 189
 190     enable_wallet = config["components"].getboolean("ENABLE_WALLET")
 191     enable_utils = config["components"].getboolean("ENABLE_UTILS")
 192     enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND")
 193
 194     if config["environment"]["EXEEXT"] == ".exe" and not args.force:
 195         # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
 196         # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
 197         print("Tests currently disabled on Windows by default. Use --force option to enable")
 198         sys.exit(0)
 199
 200     if not (enable_wallet and enable_utils and enable_bitcoind):
 201         print("No functional tests to run. Wallet, utils, and bitcoind must all be enabled")
 202         print("Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make")
 203         sys.exit(0)
 204
 205     # Build list of tests
 206     if tests:
 207         # Individual tests have been specified. Run specified tests that exist
 208         # in the ALL_SCRIPTS list. Accept the name with or without .py extension.
 209         test_list = [t for t in ALL_SCRIPTS if
 210                 (t in tests or re.sub(".py$", "", t) in tests)]
 211     else:
 212         # No individual tests have been specified.
 213         # Run all base tests, and optionally run extended tests.
 214         test_list = BASE_SCRIPTS
 215         if args.extended:
 216             # place the EXTENDED_SCRIPTS first since the three longest ones
 217             # are there and the list is shorter
 218             test_list = EXTENDED_SCRIPTS + test_list
 219
 220     # Remove the test cases that the user has explicitly asked to exclude.
 221     if args.exclude:
 222         for exclude_test in args.exclude.split(','):
 223             if exclude_test + ".py" in test_list:
 224                 test_list.remove(exclude_test + ".py")
 225
 226     if not test_list:
 227         print("No valid test scripts specified. Check that your test is in one "
 228               "of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests")
 229         sys.exit(0)
 230
 231     if args.help:
 232         # Print help for test_runner.py, then print help of the first script (with args removed) and exit.
 233         parser.print_help()
 234         subprocess.check_call([(config["environment"]["SRCDIR"] + '/test/functional/' + test_list[0].split()[0])] + ['-h'])
 235         sys.exit(0)
 236
 237     check_script_list(config["environment"]["SRCDIR"])
 238
 239     if not args.keepcache:
 240         shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True)
 241
 242     run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], args.jobs, args.coverage, passon_args)
 243
 244 def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=False, args=[]):
 245     # Warn if bitcoind is already running (unix only)
 246     try:
 247         if subprocess.check_output(["pidof", "bitcoind"]) is not None:
 248             print("%sWARNING!%s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0]))
 249     except (OSError, subprocess.SubprocessError):
 250         pass
 251
 252     # Warn if there is a cache directory
 253     cache_dir = "%s/test/cache" % build_dir
 254     if os.path.isdir(cache_dir):
 255         print("%sWARNING!%s There is a cache directory here: %s. If tests fail unexpectedly, try deleting the cache directory." % (BOLD[1], BOLD[0], cache_dir))
 256
 257     #Set env vars
 258     if "BITCOIND" not in os.environ:
 259         os.environ["BITCOIND"] = build_dir + '/src/bitcoind' + exeext
 260
 261     tests_dir = src_dir + '/test/functional/'
 262
 263     flags = ["--srcdir={}/src".format(build_dir)] + args
 264     flags.append("--cachedir=%s" % cache_dir)
 265
 266     if enable_coverage:
 267         coverage = RPCCoverage()
 268         flags.append(coverage.flag)
 269         logging.debug("Initializing coverage directory at %s" % coverage.dir)
 270     else:
 271         coverage = None
 272
 273     if len(test_list) > 1 and jobs > 1:
 274         # Populate cache
 275         subprocess.check_output([tests_dir + 'create_cache.py'] + flags)
 276
 277     #Run Tests
 278     job_queue = TestHandler(jobs, tests_dir, test_list, flags)
 279     time0 = time.time()
 280     test_results = []
 281
 282     max_len_name = len(max(test_list, key=len))
 283
 284     for _ in range(len(test_list)):
 285         test_result, stdout, stderr = job_queue.get_next()
 286         test_results.append(test_result)
 287
 288         if test_result.status == "Passed":
 289             logging.debug("\n%s%s%s passed, Duration: %s s" % (BOLD[1], test_result.name, BOLD[0], test_result.time))
 290         elif test_result.status == "Skipped":
 291             logging.debug("\n%s%s%s skipped" % (BOLD[1], test_result.name, BOLD[0]))
 292         else:
 293             print("\n%s%s%s failed, Duration: %s s\n" % (BOLD[1], test_result.name, BOLD[0], test_result.time))
 294             print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n')
 295             print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n')
 296
 297     print_results(test_results, max_len_name, (int(time.time() - time0)))
 298
 299     if coverage:
 300         coverage.report_rpc_coverage()
 301
 302         logging.debug("Cleaning up coverage data")
 303         coverage.cleanup()
 304
 305     all_passed = all(map(lambda test_result: test_result.was_successful, test_results))
 306
 307     sys.exit(not all_passed)
 308
 309 def print_results(test_results, max_len_name, runtime):
 310     results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS   ", "DURATION") + BOLD[0]
 311
 312     test_results.sort(key=lambda result: result.name.lower())
 313     all_passed = True
 314     time_sum = 0
 315
 316     for test_result in test_results:
 317         all_passed = all_passed and test_result.was_successful
 318         time_sum += test_result.time
 319         test_result.padding = max_len_name
 320         results += str(test_result)
 321
 322     status = TICK + "Passed" if all_passed else CROSS + "Failed"
 323     results += BOLD[1] + "\n%s | %s | %s s (accumulated) \n" % ("ALL".ljust(max_len_name), status.ljust(9), time_sum) + BOLD[0]
 324     results += "Runtime: %s s\n" % (runtime)
 325     print(results)
 326
 327 class TestHandler:
 328     """
 329     Trigger the testscrips passed in via the list.
 330     """
 331
 332     def __init__(self, num_tests_parallel, tests_dir, test_list=None, flags=None):
 333         assert(num_tests_parallel >= 1)
 334         self.num_jobs = num_tests_parallel
 335         self.tests_dir = tests_dir
 336         self.test_list = test_list
 337         self.flags = flags
 338         self.num_running = 0
 339         # In case there is a graveyard of zombie bitcoinds, we can apply a
 340         # pseudorandom offset to hopefully jump over them.
 341         # (625 is PORT_RANGE/MAX_NODES)
 342         self.portseed_offset = int(time.time() * 1000) % 625
 343         self.jobs = []
 344
 345     def get_next(self):
 346         while self.num_running < self.num_jobs and self.test_list:
 347             # Add tests
 348             self.num_running += 1
 349             t = self.test_list.pop(0)
 350             port_seed = ["--portseed={}".format(len(self.test_list) + self.portseed_offset)]
 351             log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16)
 352             log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16)
 353             test_argv = t.split()
 354             self.jobs.append((t,
 355                               time.time(),
 356                               subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + port_seed,
 357                                                universal_newlines=True,
 358                                                stdout=log_stdout,
 359                                                stderr=log_stderr),
 360                               log_stdout,
 361                               log_stderr))
 362         if not self.jobs:
 363             raise IndexError('pop from empty list')
 364         while True:
 365             # Return first proc that finishes
 366             time.sleep(.5)
 367             for j in self.jobs:
 368                 (name, time0, proc, log_out, log_err) = j
 369                 if proc.poll() is not None:
 370                     log_out.seek(0), log_err.seek(0)
 371                     [stdout, stderr] = [l.read().decode('utf-8') for l in (log_out, log_err)]
 372                     log_out.close(), log_err.close()
 373                     if proc.returncode == TEST_EXIT_PASSED and stderr == "":
 374                         status = "Passed"
 375                     elif proc.returncode == TEST_EXIT_SKIPPED:
 376                         status = "Skipped"
 377                     else:
 378                         status = "Failed"
 379                     self.num_running -= 1
 380                     self.jobs.remove(j)
 381
 382                     return TestResult(name, status, int(time.time() - time0)), stdout, stderr
 383             print('.', end='', flush=True)
 384
 385 class TestResult():
 386     def __init__(self, name, status, time):
 387         self.name = name
 388         self.status = status
 389         self.time = time
 390         self.padding = 0
 391
 392     def __repr__(self):
 393         if self.status == "Passed":
 394             color = BLUE
 395             glyph = TICK
 396         elif self.status == "Failed":
 397             color = RED
 398             glyph = CROSS
 399         elif self.status == "Skipped":
 400             color = GREY
 401             glyph = CIRCLE
 402
 403         return color[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0]
 404
 405     @property
 406     def was_successful(self):
 407         return self.status != "Failed"
 408
 409
 410 def check_script_list(src_dir):
 411     """Check scripts directory.
 412
 413     Check that there are no scripts in the functional tests directory which are
 414     not being run by pull-tester.py."""
 415     script_dir = src_dir + '/test/functional/'
 416     python_files = set([t for t in os.listdir(script_dir) if t[-3:] == ".py"])
 417     missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS)))
 418     if len(missed_tests) != 0:
 419         print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests)))
 420         if os.getenv('TRAVIS') == 'true':
 421             # On travis this warning is an error to prevent merging incomplete commits into master
 422             sys.exit(1)
 423
 424 class RPCCoverage(object):
 425     """
 426     Coverage reporting utilities for test_runner.
 427
 428     Coverage calculation works by having each test script subprocess write
 429     coverage files into a particular directory. These files contain the RPC
 430     commands invoked during testing, as well as a complete listing of RPC
 431     commands per `bitcoin-cli help` (`rpc_interface.txt`).
 432
 433     After all tests complete, the commands run are combined and diff'd against
 434     the complete list to calculate uncovered RPC commands.
 435
 436     See also: test/functional/test_framework/coverage.py
 437
 438     """
 439     def __init__(self):
 440         self.dir = tempfile.mkdtemp(prefix="coverage")
 441         self.flag = '--coveragedir=%s' % self.dir
 442
 443     def report_rpc_coverage(self):
 444         """
 445         Print out RPC commands that were unexercised by tests.
 446
 447         """
 448         uncovered = self._get_uncovered_rpc_commands()
 449
 450         if uncovered:
 451             print("Uncovered RPC commands:")
 452             print("".join(("  - %s\n" % i) for i in sorted(uncovered)))
 453         else:
 454             print("All RPC commands covered.")
 455
 456     def cleanup(self):
 457         return shutil.rmtree(self.dir)
 458
 459     def _get_uncovered_rpc_commands(self):
 460         """
 461         Return a set of currently untested RPC commands.
 462
 463         """
 464         # This is shared from `test/functional/test-framework/coverage.py`
 465         reference_filename = 'rpc_interface.txt'
 466         coverage_file_prefix = 'coverage.'
 467
 468         coverage_ref_filename = os.path.join(self.dir, reference_filename)
 469         coverage_filenames = set()
 470         all_cmds = set()
 471         covered_cmds = set()
 472
 473         if not os.path.isfile(coverage_ref_filename):
 474             raise RuntimeError("No coverage reference found")
 475
 476         with open(coverage_ref_filename, 'r') as f:
 477             all_cmds.update([i.strip() for i in f.readlines()])
 478
 479         for root, dirs, files in os.walk(self.dir):
 480             for filename in files:
 481                 if filename.startswith(coverage_file_prefix):
 482                     coverage_filenames.add(os.path.join(root, filename))
 483
 484         for filename in coverage_filenames:
 485             with open(filename, 'r') as f:
 486                 covered_cmds.update([i.strip() for i in f.readlines()])
 487
 488         return all_cmds - covered_cmds
 489
 490
 491 if __name__ == '__main__':
 492     main()