src/test/bctest.py

   1 # Copyright 2014 BitPay Inc.
   2 # Copyright 2016 The Bitcoin Core developers
   3 # Distributed under the MIT software license, see the accompanying
   4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5 from __future__ import division,print_function,unicode_literals
   6 import subprocess
   7 import os
   8 import json
   9 import sys
  10 import binascii
  11 import difflib
  12 import logging
  13 import pprint
  14
  15 def parse_output(a, fmt):
  16     """Parse the output according to specified format.
  17
  18     Raise an error if the output can't be parsed."""
  19     if fmt == 'json': # json: compare parsed data
  20         return json.loads(a)
  21     elif fmt == 'hex': # hex: parse and compare binary data
  22         return binascii.a2b_hex(a.strip())
  23     else:
  24         raise NotImplementedError("Don't know how to compare %s" % fmt)
  25
  26 def bctest(testDir, testObj, exeext):
  27     """Runs a single test, comparing output and RC to expected output and RC.
  28
  29     Raises an error if input can't be read, executable fails, or output/RC
  30     are not as expected. Error is caught by bctester() and reported.
  31     """
  32     # Get the exec names and arguments
  33     execprog = testObj['exec'] + exeext
  34     execargs = testObj['args']
  35     execrun = [execprog] + execargs
  36
  37     # Read the input data (if there is any)
  38     stdinCfg = None
  39     inputData = None
  40     if "input" in testObj:
  41         filename = testDir + "/" + testObj['input']
  42         inputData = open(filename).read()
  43         stdinCfg = subprocess.PIPE
  44
  45     # Read the expected output data (if there is any)
  46     outputFn = None
  47     outputData = None
  48     if "output_cmp" in testObj:
  49         outputFn = testObj['output_cmp']
  50         outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare)
  51         try:
  52             outputData = open(testDir + "/" + outputFn).read()
  53         except:
  54             logging.error("Output file " + outputFn + " can not be opened")
  55             raise
  56         if not outputData:
  57             logging.error("Output data missing for " + outputFn)
  58             raise Exception
  59
  60     # Run the test
  61     proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True)
  62     try:
  63         outs = proc.communicate(input=inputData)
  64     except OSError:
  65         logging.error("OSError, Failed to execute " + execprog)
  66         raise
  67
  68     if outputData:
  69         data_mismatch, formatting_mismatch = False, False
  70         # Parse command output and expected output
  71         try:
  72             a_parsed = parse_output(outs[0], outputType)
  73         except Exception as e:
  74             logging.error('Error parsing command output as %s: %s' % (outputType,e))
  75             raise
  76         try:
  77             b_parsed = parse_output(outputData, outputType)
  78         except Exception as e:
  79             logging.error('Error parsing expected output %s as %s: %s' % (outputFn,outputType,e))
  80             raise
  81         # Compare data
  82         if a_parsed != b_parsed:
  83             logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")")
  84             data_mismatch = True
  85         # Compare formatting
  86         if outs[0] != outputData:
  87             error_message = "Output formatting mismatch for " + outputFn + ":\n"
  88             error_message += "".join(difflib.context_diff(outputData.splitlines(True),
  89                                                           outs[0].splitlines(True),
  90                                                           fromfile=outputFn,
  91                                                           tofile="returned"))
  92             logging.error(error_message)
  93             formatting_mismatch = True
  94
  95         assert not data_mismatch and not formatting_mismatch
  96
  97     # Compare the return code to the expected return code
  98     wantRC = 0
  99     if "return_code" in testObj:
 100         wantRC = testObj['return_code']
 101     if proc.returncode != wantRC:
 102         logging.error("Return code mismatch for " + outputFn)
 103         raise Exception
 104
 105 def bctester(testDir, input_basename, buildenv):
 106     """ Loads and parses the input file, runs all tests and reports results"""
 107     input_filename = testDir + "/" + input_basename
 108     raw_data = open(input_filename).read()
 109     input_data = json.loads(raw_data)
 110
 111     failed_testcases = []
 112
 113     for testObj in input_data:
 114         try:
 115             bctest(testDir, testObj, buildenv.exeext)
 116             logging.info("PASSED: " + testObj["description"])
 117         except:
 118             logging.info("FAILED: " + testObj["description"])
 119             failed_testcases.append(testObj["description"])
 120
 121     if failed_testcases:
 122         error_message = "FAILED_TESTCASES:\n"
 123         error_message += pprint.pformat(failed_testcases, width=400)
 124         logging.error(error_message)
 125         sys.exit(1)
 126     else:
 127         sys.exit(0)