test/functional/test_runner.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2014-2016 The Bitcoin Core developers
   3 # Distributed under the MIT software license, see the accompanying
   4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5 """Run regression test suite.
   6
   7 This module calls down into individual test cases via subprocess. It will
   8 forward all unrecognized arguments onto the individual test scripts.
   9
  10 Functional tests are disabled on Windows by default. Use --force to run them anyway.
  11
  12 For a description of arguments recognized by test scripts, see
  13 `test/functional/test_framework/test_framework.py:BitcoinTestFramework.main`.
  14
  15 """
  16
  17 import argparse
  18 import configparser
  19 import os
  20 import time
  21 import shutil
  22 import sys
  23 import subprocess
  24 import tempfile
  25 import re
  26
  27 TEST_EXIT_PASSED = 0
  28 TEST_EXIT_SKIPPED = 77
  29
  30 BASE_SCRIPTS= [
  31     # Scripts that are run by the travis build process.
  32     # Longest test should go first, to favor running tests in parallel
  33     'wallet-hd.py',
  34     'walletbackup.py',
  35     # vv Tests less than 5m vv
  36     'p2p-fullblocktest.py',
  37     'fundrawtransaction.py',
  38     'p2p-compactblocks.py',
  39     'segwit.py',
  40     # vv Tests less than 2m vv
  41     'wallet.py',
  42     'wallet-accounts.py',
  43     'p2p-segwit.py',
  44     'wallet-dump.py',
  45     'listtransactions.py',
  46     # vv Tests less than 60s vv
  47     'sendheaders.py',
  48     'zapwallettxes.py',
  49     'importmulti.py',
  50     'mempool_limit.py',
  51     'merkle_blocks.py',
  52     'receivedby.py',
  53     'abandonconflict.py',
  54     'bip68-112-113-p2p.py',
  55     'rawtransactions.py',
  56     'reindex.py',
  57     # vv Tests less than 30s vv
  58     'mempool_resurrect_test.py',
  59     'txn_doublespend.py --mineblock',
  60     'txn_clone.py',
  61     'getchaintips.py',
  62     'rest.py',
  63     'mempool_spendcoinbase.py',
  64     'mempool_reorg.py',
  65     'httpbasics.py',
  66     'multi_rpc.py',
  67     'proxy_test.py',
  68     'signrawtransactions.py',
  69     'nodehandling.py',
  70     'decodescript.py',
  71     'blockchain.py',
  72     'disablewallet.py',
  73     'net.py',
  74     'keypool.py',
  75     'p2p-mempool.py',
  76     'prioritise_transaction.py',
  77     'invalidblockrequest.py',
  78     'invalidtxrequest.py',
  79     'p2p-versionbits-warning.py',
  80     'preciousblock.py',
  81     'importprunedfunds.py',
  82     'signmessages.py',
  83     'nulldummy.py',
  84     'import-rescan.py',
  85     'bumpfee.py',
  86     'rpcnamedargs.py',
  87     'listsinceblock.py',
  88     'p2p-leaktests.py',
  89 ]
  90
  91 ZMQ_SCRIPTS = [
  92     # ZMQ test can only be run if bitcoin was built with zmq-enabled.
  93     # call test_runner.py with -nozmq to explicitly exclude these tests.
  94     "zmq_test.py"]
  95
  96 EXTENDED_SCRIPTS = [
  97     # These tests are not run by the travis build process.
  98     # Longest test should go first, to favor running tests in parallel
  99     'pruning.py',
 100     # vv Tests less than 20m vv
 101     'smartfees.py',
 102     # vv Tests less than 5m vv
 103     'maxuploadtarget.py',
 104     'mempool_packages.py',
 105     # vv Tests less than 2m vv
 106     'bip68-sequence.py',
 107     'getblocktemplate_longpoll.py',
 108     'p2p-timeouts.py',
 109     # vv Tests less than 60s vv
 110     'bip9-softforks.py',
 111     'p2p-feefilter.py',
 112     'rpcbind_test.py',
 113     # vv Tests less than 30s vv
 114     'bip65-cltv.py',
 115     'bip65-cltv-p2p.py',
 116     'bipdersig-p2p.py',
 117     'bipdersig.py',
 118     'getblocktemplate_proposals.py',
 119     'txn_doublespend.py',
 120     'txn_clone.py --mineblock',
 121     'forknotify.py',
 122     'invalidateblock.py',
 123     'maxblocksinflight.py',
 124     'p2p-acceptblock.py',
 125     'replace-by-fee.py',
 126 ]
 127
 128 ALL_SCRIPTS = BASE_SCRIPTS + ZMQ_SCRIPTS + EXTENDED_SCRIPTS
 129
 130 def main():
 131     # Parse arguments and pass through unrecognised args
 132     parser = argparse.ArgumentParser(add_help=False,
 133                                      usage='%(prog)s [test_runner.py options] [script options] [scripts]',
 134                                      description=__doc__,
 135                                      epilog='''
 136     Help text and arguments for individual test script:''',
 137                                      formatter_class=argparse.RawTextHelpFormatter)
 138     parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface')
 139     parser.add_argument('--exclude', '-x', help='specify a comma-seperated-list of scripts to exclude. Do not include the .py extension in the name.')
 140     parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests')
 141     parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).')
 142     parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit')
 143     parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.')
 144     parser.add_argument('--nozmq', action='store_true', help='do not run the zmq tests')
 145     args, unknown_args = parser.parse_known_args()
 146
 147     # Create a set to store arguments and create the passon string
 148     tests = set(arg for arg in unknown_args if arg[:2] != "--")
 149     passon_args = [arg for arg in unknown_args if arg[:2] == "--"]
 150
 151     # Read config generated by configure.
 152     config = configparser.ConfigParser()
 153     config.read_file(open(os.path.dirname(__file__) + "/config.ini"))
 154
 155     enable_wallet = config["components"].getboolean("ENABLE_WALLET")
 156     enable_utils = config["components"].getboolean("ENABLE_UTILS")
 157     enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND")
 158     enable_zmq = config["components"].getboolean("ENABLE_ZMQ") and not args.nozmq
 159
 160     if config["environment"]["EXEEXT"] == ".exe" and not args.force:
 161         # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
 162         # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
 163         print("Tests currently disabled on Windows by default. Use --force option to enable")
 164         sys.exit(0)
 165
 166     if not (enable_wallet and enable_utils and enable_bitcoind):
 167         print("No functional tests to run. Wallet, utils, and bitcoind must all be enabled")
 168         print("Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make")
 169         sys.exit(0)
 170
 171     # python3-zmq may not be installed. Handle this gracefully and with some helpful info
 172     if enable_zmq:
 173         try:
 174             import zmq
 175         except ImportError:
 176             print("ERROR: \"import zmq\" failed. Use -nozmq to run without the ZMQ tests."
 177                   "To run zmq tests, see dependency info in /test/README.md.")
 178             raise
 179
 180     # Build list of tests
 181     if tests:
 182         # Individual tests have been specified. Run specified tests that exist
 183         # in the ALL_SCRIPTS list. Accept the name with or without .py extension.
 184         test_list = [t for t in ALL_SCRIPTS if
 185                 (t in tests or re.sub(".py$", "", t) in tests)]
 186     else:
 187         # No individual tests have been specified. Run base tests, and
 188         # optionally ZMQ tests and extended tests.
 189         test_list = BASE_SCRIPTS
 190         if enable_zmq:
 191             test_list += ZMQ_SCRIPTS
 192         if args.extended:
 193             test_list += EXTENDED_SCRIPTS
 194             # TODO: BASE_SCRIPTS and EXTENDED_SCRIPTS are sorted by runtime
 195             # (for parallel running efficiency). This combined list will is no
 196             # longer sorted.
 197
 198     # Remove the test cases that the user has explicitly asked to exclude.
 199     if args.exclude:
 200         for exclude_test in args.exclude.split(','):
 201             if exclude_test + ".py" in test_list:
 202                 test_list.remove(exclude_test + ".py")
 203
 204     if not test_list:
 205         print("No valid test scripts specified. Check that your test is in one "
 206               "of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests")
 207         sys.exit(0)
 208
 209     if args.help:
 210         # Print help for test_runner.py, then print help of the first script and exit.
 211         parser.print_help()
 212         subprocess.check_call((config["environment"]["SRCDIR"] + '/test/functional/' + test_list[0]).split() + ['-h'])
 213         sys.exit(0)
 214
 215     run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], args.jobs, args.coverage, passon_args)
 216
 217 def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=False, args=[]):
 218     BOLD = ("","")
 219     if os.name == 'posix':
 220         # primitive formatting on supported
 221         # terminal via ANSI escape sequences:
 222         BOLD = ('\033[0m', '\033[1m')
 223
 224     #Set env vars
 225     if "BITCOIND" not in os.environ:
 226         os.environ["BITCOIND"] = build_dir + '/src/bitcoind' + exeext
 227
 228     tests_dir = src_dir + '/test/functional/'
 229
 230     flags = ["--srcdir={}/src".format(build_dir)] + args
 231     flags.append("--cachedir=%s/test/cache" % build_dir)
 232
 233     if enable_coverage:
 234         coverage = RPCCoverage()
 235         flags.append(coverage.flag)
 236         print("Initializing coverage directory at %s\n" % coverage.dir)
 237     else:
 238         coverage = None
 239
 240     if len(test_list) > 1 and jobs > 1:
 241         # Populate cache
 242         subprocess.check_output([tests_dir + 'create_cache.py'] + flags)
 243
 244     #Run Tests
 245     all_passed = True
 246     time_sum = 0
 247     time0 = time.time()
 248
 249     job_queue = TestHandler(jobs, tests_dir, test_list, flags)
 250
 251     max_len_name = len(max(test_list, key=len))
 252     results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0]
 253     for _ in range(len(test_list)):
 254         (name, stdout, stderr, status, duration) = job_queue.get_next()
 255         all_passed = all_passed and status != "Failed"
 256         time_sum += duration
 257
 258         print('\n' + BOLD[1] + name + BOLD[0] + ":")
 259         print('' if status == "Passed" else stdout + '\n', end='')
 260         print('' if stderr == '' else 'stderr:\n' + stderr + '\n', end='')
 261         print("Status: %s%s%s, Duration: %s s\n" % (BOLD[1], status, BOLD[0], duration))
 262
 263         results += "%s | %s | %s s\n" % (name.ljust(max_len_name), status.ljust(7), duration)
 264
 265     results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(7), time_sum) + BOLD[0]
 266     print(results)
 267     print("\nRuntime: %s s" % (int(time.time() - time0)))
 268
 269     if coverage:
 270         coverage.report_rpc_coverage()
 271
 272         print("Cleaning up coverage data")
 273         coverage.cleanup()
 274
 275     sys.exit(not all_passed)
 276
 277 class TestHandler:
 278     """
 279     Trigger the testscrips passed in via the list.
 280     """
 281
 282     def __init__(self, num_tests_parallel, tests_dir, test_list=None, flags=None):
 283         assert(num_tests_parallel >= 1)
 284         self.num_jobs = num_tests_parallel
 285         self.tests_dir = tests_dir
 286         self.test_list = test_list
 287         self.flags = flags
 288         self.num_running = 0
 289         # In case there is a graveyard of zombie bitcoinds, we can apply a
 290         # pseudorandom offset to hopefully jump over them.
 291         # (625 is PORT_RANGE/MAX_NODES)
 292         self.portseed_offset = int(time.time() * 1000) % 625
 293         self.jobs = []
 294
 295     def get_next(self):
 296         while self.num_running < self.num_jobs and self.test_list:
 297             # Add tests
 298             self.num_running += 1
 299             t = self.test_list.pop(0)
 300             port_seed = ["--portseed={}".format(len(self.test_list) + self.portseed_offset)]
 301             log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16)
 302             log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16)
 303             self.jobs.append((t,
 304                               time.time(),
 305                               subprocess.Popen((self.tests_dir + t).split() + self.flags + port_seed,
 306                                                universal_newlines=True,
 307                                                stdout=log_stdout,
 308                                                stderr=log_stderr),
 309                               log_stdout,
 310                               log_stderr))
 311         if not self.jobs:
 312             raise IndexError('pop from empty list')
 313         while True:
 314             # Return first proc that finishes
 315             time.sleep(.5)
 316             for j in self.jobs:
 317                 (name, time0, proc, log_out, log_err) = j
 318                 if proc.poll() is not None:
 319                     log_out.seek(0), log_err.seek(0)
 320                     [stdout, stderr] = [l.read().decode('utf-8') for l in (log_out, log_err)]
 321                     log_out.close(), log_err.close()
 322                     if proc.returncode == TEST_EXIT_PASSED and stderr == "":
 323                         status = "Passed"
 324                     elif proc.returncode == TEST_EXIT_SKIPPED:
 325                         status = "Skipped"
 326                     else:
 327                         status = "Failed"
 328                     self.num_running -= 1
 329                     self.jobs.remove(j)
 330                     return name, stdout, stderr, status, int(time.time() - time0)
 331             print('.', end='', flush=True)
 332
 333
 334 class RPCCoverage(object):
 335     """
 336     Coverage reporting utilities for test_runner.
 337
 338     Coverage calculation works by having each test script subprocess write
 339     coverage files into a particular directory. These files contain the RPC
 340     commands invoked during testing, as well as a complete listing of RPC
 341     commands per `bitcoin-cli help` (`rpc_interface.txt`).
 342
 343     After all tests complete, the commands run are combined and diff'd against
 344     the complete list to calculate uncovered RPC commands.
 345
 346     See also: test/functional/test_framework/coverage.py
 347
 348     """
 349     def __init__(self):
 350         self.dir = tempfile.mkdtemp(prefix="coverage")
 351         self.flag = '--coveragedir=%s' % self.dir
 352
 353     def report_rpc_coverage(self):
 354         """
 355         Print out RPC commands that were unexercised by tests.
 356
 357         """
 358         uncovered = self._get_uncovered_rpc_commands()
 359
 360         if uncovered:
 361             print("Uncovered RPC commands:")
 362             print("".join(("  - %s\n" % i) for i in sorted(uncovered)))
 363         else:
 364             print("All RPC commands covered.")
 365
 366     def cleanup(self):
 367         return shutil.rmtree(self.dir)
 368
 369     def _get_uncovered_rpc_commands(self):
 370         """
 371         Return a set of currently untested RPC commands.
 372
 373         """
 374         # This is shared from `test/functional/test-framework/coverage.py`
 375         reference_filename = 'rpc_interface.txt'
 376         coverage_file_prefix = 'coverage.'
 377
 378         coverage_ref_filename = os.path.join(self.dir, reference_filename)
 379         coverage_filenames = set()
 380         all_cmds = set()
 381         covered_cmds = set()
 382
 383         if not os.path.isfile(coverage_ref_filename):
 384             raise RuntimeError("No coverage reference found")
 385
 386         with open(coverage_ref_filename, 'r') as f:
 387             all_cmds.update([i.strip() for i in f.readlines()])
 388
 389         for root, dirs, files in os.walk(self.dir):
 390             for filename in files:
 391                 if filename.startswith(coverage_file_prefix):
 392                     coverage_filenames.add(os.path.join(root, filename))
 393
 394         for filename in coverage_filenames:
 395             with open(filename, 'r') as f:
 396                 covered_cmds.update([i.strip() for i in f.readlines()])
 397
 398         return all_cmds - covered_cmds
 399
 400
 401 if __name__ == '__main__':
 402     main()