test/util/bitcoin-util-test.py

   1 #!/usr/bin/env python3
   2 # Copyright 2014 BitPay Inc.
   3 # Copyright 2016-2017 The Bitcoin Core developers
   4 # Distributed under the MIT software license, see the accompanying
   5 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   6 """Test framework for bitcoin utils.
   7
   8 Runs automatically during `make check`.
   9
  10 Can also be run manually."""
  11
  12 import argparse
  13 import binascii
  14 import configparser
  15 import difflib
  16 import json
  17 import logging
  18 import os
  19 import pprint
  20 import subprocess
  21 import sys
  22
  23 def main():
  24     config = configparser.ConfigParser()
  25     config.read_file(open(os.path.dirname(__file__) + "/../config.ini"))
  26
  27     parser = argparse.ArgumentParser(description=__doc__)
  28     parser.add_argument('-v', '--verbose', action='store_true')
  29     args = parser.parse_args()
  30     verbose = args.verbose
  31
  32     if verbose:
  33         level = logging.DEBUG
  34     else:
  35         level = logging.ERROR
  36     formatter = '%(asctime)s - %(levelname)s - %(message)s'
  37     # Add the format/level to the logger
  38     logging.basicConfig(format=formatter, level=level)
  39
  40     bctester(config["environment"]["SRCDIR"] + "/test/util/data", "bitcoin-util-test.json", config["environment"])
  41
  42 def bctester(testDir, input_basename, buildenv):
  43     """ Loads and parses the input file, runs all tests and reports results"""
  44     input_filename = testDir + "/" + input_basename
  45     raw_data = open(input_filename).read()
  46     input_data = json.loads(raw_data)
  47
  48     failed_testcases = []
  49
  50     for testObj in input_data:
  51         try:
  52             bctest(testDir, testObj, buildenv)
  53             logging.info("PASSED: " + testObj["description"])
  54         except:
  55             logging.info("FAILED: " + testObj["description"])
  56             failed_testcases.append(testObj["description"])
  57
  58     if failed_testcases:
  59         error_message = "FAILED_TESTCASES:\n"
  60         error_message += pprint.pformat(failed_testcases, width=400)
  61         logging.error(error_message)
  62         sys.exit(1)
  63     else:
  64         sys.exit(0)
  65
  66 def bctest(testDir, testObj, buildenv):
  67     """Runs a single test, comparing output and RC to expected output and RC.
  68
  69     Raises an error if input can't be read, executable fails, or output/RC
  70     are not as expected. Error is caught by bctester() and reported.
  71     """
  72     # Get the exec names and arguments
  73     execprog = buildenv["BUILDDIR"] + "/src/" + testObj['exec'] + buildenv["EXEEXT"]
  74     execargs = testObj['args']
  75     execrun = [execprog] + execargs
  76
  77     # Read the input data (if there is any)
  78     stdinCfg = None
  79     inputData = None
  80     if "input" in testObj:
  81         filename = testDir + "/" + testObj['input']
  82         inputData = open(filename).read()
  83         stdinCfg = subprocess.PIPE
  84
  85     # Read the expected output data (if there is any)
  86     outputFn = None
  87     outputData = None
  88     if "output_cmp" in testObj:
  89         outputFn = testObj['output_cmp']
  90         outputType = os.path.splitext(outputFn)[1][1:]  # output type from file extension (determines how to compare)
  91         try:
  92             outputData = open(testDir + "/" + outputFn).read()
  93         except:
  94             logging.error("Output file " + outputFn + " can not be opened")
  95             raise
  96         if not outputData:
  97             logging.error("Output data missing for " + outputFn)
  98             raise Exception
  99
 100     # Run the test
 101     proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
 102     try:
 103         outs = proc.communicate(input=inputData)
 104     except OSError:
 105         logging.error("OSError, Failed to execute " + execprog)
 106         raise
 107
 108     if outputData:
 109         data_mismatch, formatting_mismatch = False, False
 110         # Parse command output and expected output
 111         try:
 112             a_parsed = parse_output(outs[0], outputType)
 113         except Exception as e:
 114             logging.error('Error parsing command output as %s: %s' % (outputType, e))
 115             raise
 116         try:
 117             b_parsed = parse_output(outputData, outputType)
 118         except Exception as e:
 119             logging.error('Error parsing expected output %s as %s: %s' % (outputFn, outputType, e))
 120             raise
 121         # Compare data
 122         if a_parsed != b_parsed:
 123             logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")")
 124             data_mismatch = True
 125         # Compare formatting
 126         if outs[0] != outputData:
 127             error_message = "Output formatting mismatch for " + outputFn + ":\n"
 128             error_message += "".join(difflib.context_diff(outputData.splitlines(True),
 129                                                           outs[0].splitlines(True),
 130                                                           fromfile=outputFn,
 131                                                           tofile="returned"))
 132             logging.error(error_message)
 133             formatting_mismatch = True
 134
 135         assert not data_mismatch and not formatting_mismatch
 136
 137     # Compare the return code to the expected return code
 138     wantRC = 0
 139     if "return_code" in testObj:
 140         wantRC = testObj['return_code']
 141     if proc.returncode != wantRC:
 142         logging.error("Return code mismatch for " + outputFn)
 143         raise Exception
 144
 145     if "error_txt" in testObj:
 146         want_error = testObj["error_txt"]
 147         # Compare error text
 148         # TODO: ideally, we'd compare the strings exactly and also assert
 149         # That stderr is empty if no errors are expected. However, bitcoin-tx
 150         # emits DISPLAY errors when running as a windows application on
 151         # linux through wine. Just assert that the expected error text appears
 152         # somewhere in stderr.
 153         if want_error not in outs[1]:
 154             logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip())
 155             raise Exception
 156
 157 def parse_output(a, fmt):
 158     """Parse the output according to specified format.
 159
 160     Raise an error if the output can't be parsed."""
 161     if fmt == 'json':  # json: compare parsed data
 162         return json.loads(a)
 163     elif fmt == 'hex':  # hex: parse and compare binary data
 164         return binascii.a2b_hex(a.strip())
 165     else:
 166         raise NotImplementedError("Don't know how to compare %s" % fmt)
 167
 168 if __name__ == '__main__':
 169     main()