1 # Copyright 2014 BitPay Inc.
2 # Copyright 2016 The Bitcoin Core developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 from __future__
import division
,print_function
,unicode_literals
14 def parse_output(a
, fmt
):
15 """Parse the output according to specified format.
17 Raise an error if the output can't be parsed."""
18 if fmt
== 'json': # json: compare parsed data
20 elif fmt
== 'hex': # hex: parse and compare binary data
21 return binascii
.a2b_hex(a
.strip())
23 raise NotImplementedError("Don't know how to compare %s" % fmt
)
25 def bctest(testDir
, testObj
, exeext
):
26 """Runs a single test, comparing output and RC to expected output and RC.
28 Raises an error if input can't be read, executable fails, or output/RC
29 are not as expected. Error is caught by bctester() and reported.
31 # Get the exec names and arguments
32 execprog
= testObj
['exec'] + exeext
33 execargs
= testObj
['args']
34 execrun
= [execprog
] + execargs
36 # Read the input data (if there is any)
39 if "input" in testObj
:
40 filename
= testDir
+ "/" + testObj
['input']
41 inputData
= open(filename
).read()
42 stdinCfg
= subprocess
.PIPE
44 # Read the expected output data (if there is any)
47 if "output_cmp" in testObj
:
48 outputFn
= testObj
['output_cmp']
49 outputType
= os
.path
.splitext(outputFn
)[1][1:] # output type from file extension (determines how to compare)
51 outputData
= open(testDir
+ "/" + outputFn
).read()
53 logging
.error("Output file " + outputFn
+ " can not be opened")
56 logging
.error("Output data missing for " + outputFn
)
60 proc
= subprocess
.Popen(execrun
, stdin
=stdinCfg
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
,universal_newlines
=True)
62 outs
= proc
.communicate(input=inputData
)
64 logging
.error("OSError, Failed to execute " + execprog
)
68 # Parse command output and expected output
70 a_parsed
= parse_output(outs
[0], outputType
)
71 except Exception as e
:
72 logging
.error('Error parsing command output as %s: %s' % (outputType
,e
))
75 b_parsed
= parse_output(outputData
, outputType
)
76 except Exception as e
:
77 logging
.error('Error parsing expected output %s as %s: %s' % (outputFn
,outputType
,e
))
80 if a_parsed
!= b_parsed
:
81 logging
.error("Output data mismatch for " + outputFn
+ " (format " + outputType
+ ")")
84 if outs
[0] != outputData
:
85 error_message
= "Output formatting mismatch for " + outputFn
+ ":\n"
86 error_message
+= "".join(difflib
.context_diff(outputData
.splitlines(True),
87 outs
[0].splitlines(True),
90 logging
.error(error_message
)
93 # Compare the return code to the expected return code
95 if "return_code" in testObj
:
96 wantRC
= testObj
['return_code']
97 if proc
.returncode
!= wantRC
:
98 logging
.error("Return code mismatch for " + outputFn
)
101 def bctester(testDir
, input_basename
, buildenv
):
102 """ Loads and parses the input file, runs all tests and reports results"""
103 input_filename
= testDir
+ "/" + input_basename
104 raw_data
= open(input_filename
).read()
105 input_data
= json
.loads(raw_data
)
107 failed_testcases
= []
109 for testObj
in input_data
:
111 bctest(testDir
, testObj
, buildenv
.exeext
)
112 logging
.info("PASSED: " + testObj
["description"])
114 logging
.info("FAILED: " + testObj
["description"])
115 failed_testcases
.append(testObj
["description"])
118 logging
.error("FAILED TESTCASES: [" + ", ".join(failed_testcases
) + "]")