src/test/bctest.py

   1 # Copyright 2014 BitPay Inc.
   2 # Copyright 2016 The Bitcoin Core developers
   3 # Distributed under the MIT software license, see the accompanying
   4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5 from __future__ import division,print_function,unicode_literals
   6 import subprocess
   7 import os
   8 import json
   9 import sys
  10 import binascii
  11 import difflib
  12 import logging
  13
  14 def parse_output(a, fmt):
  15     """Parse the output according to specified format.
  16
  17     Raise an error if the output can't be parsed."""
  18     if fmt == 'json': # json: compare parsed data
  19         return json.loads(a)
  20     elif fmt == 'hex': # hex: parse and compare binary data
  21         return binascii.a2b_hex(a.strip())
  22     else:
  23         raise NotImplementedError("Don't know how to compare %s" % fmt)
  24
  25 def bctest(testDir, testObj, exeext):
  26     """Runs a single test, comparing output and RC to expected output and RC.
  27
  28     Raises an error if input can't be read, executable fails, or output/RC
  29     are not as expected. Error is caught by bctester() and reported.
  30     """
  31     # Get the exec names and arguments
  32     execprog = testObj['exec'] + exeext
  33     execargs = testObj['args']
  34     execrun = [execprog] + execargs
  35
  36     # Read the input data (if there is any)
  37     stdinCfg = None
  38     inputData = None
  39     if "input" in testObj:
  40         filename = testDir + "/" + testObj['input']
  41         inputData = open(filename).read()
  42         stdinCfg = subprocess.PIPE
  43
  44     # Read the expected output data (if there is any)
  45     outputFn = None
  46     outputData = None
  47     if "output_cmp" in testObj:
  48         outputFn = testObj['output_cmp']
  49         outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare)
  50         try:
  51             outputData = open(testDir + "/" + outputFn).read()
  52         except:
  53             logging.error("Output file " + outputFn + " can not be opened")
  54             raise
  55         if not outputData:
  56             logging.error("Output data missing for " + outputFn)
  57             raise Exception
  58
  59     # Run the test
  60     proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True)
  61     try:
  62         outs = proc.communicate(input=inputData)
  63     except OSError:
  64         logging.error("OSError, Failed to execute " + execprog)
  65         raise
  66
  67     if outputData:
  68         # Parse command output and expected output
  69         try:
  70             a_parsed = parse_output(outs[0], outputType)
  71         except Exception as e:
  72             logging.error('Error parsing command output as %s: %s' % (outputType,e))
  73             raise
  74         try:
  75             b_parsed = parse_output(outputData, outputType)
  76         except Exception as e:
  77             logging.error('Error parsing expected output %s as %s: %s' % (outputFn,outputType,e))
  78             raise
  79         # Compare data
  80         if a_parsed != b_parsed:
  81             logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")")
  82             raise Exception
  83         # Compare formatting
  84         if outs[0] != outputData:
  85             error_message = "Output formatting mismatch for " + outputFn + ":\n"
  86             error_message += "".join(difflib.context_diff(outputData.splitlines(True),
  87                                                           outs[0].splitlines(True),
  88                                                           fromfile=outputFn,
  89                                                           tofile="returned"))
  90             logging.error(error_message)
  91             raise Exception
  92
  93     # Compare the return code to the expected return code
  94     wantRC = 0
  95     if "return_code" in testObj:
  96         wantRC = testObj['return_code']
  97     if proc.returncode != wantRC:
  98         logging.error("Return code mismatch for " + outputFn)
  99         raise Exception
 100
 101 def bctester(testDir, input_basename, buildenv):
 102     """ Loads and parses the input file, runs all tests and reports results"""
 103     input_filename = testDir + "/" + input_basename
 104     raw_data = open(input_filename).read()
 105     input_data = json.loads(raw_data)
 106
 107     failed_testcases = []
 108
 109     for testObj in input_data:
 110         try:
 111             bctest(testDir, testObj, buildenv.exeext)
 112             logging.info("PASSED: " + testObj["description"])
 113         except:
 114             logging.info("FAILED: " + testObj["description"])
 115             failed_testcases.append(testObj["description"])
 116
 117     if failed_testcases:
 118         logging.error("FAILED TESTCASES: [" + ", ".join(failed_testcases) + "]")
 119         sys.exit(1)
 120     else:
 121         sys.exit(0)