test/functional/test_framework/mininode.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2010 ArtForz -- public domain half-a-node
   3 # Copyright (c) 2012 Jeff Garzik
   4 # Copyright (c) 2010-2016 The Bitcoin Core developers
   5 # Distributed under the MIT software license, see the accompanying
   6 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   7 """Bitcoin P2P network half-a-node.
   8
   9 This python code was modified from ArtForz' public domain  half-a-node, as
  10 found in the mini-node branch of http://github.com/jgarzik/pynode.
  11
  12 NodeConn: an object which manages p2p connectivity to a bitcoin node
  13 NodeConnCB: a base class that describes the interface for receiving
  14             callbacks with network messages from a NodeConn
  15 CBlock, CTransaction, CBlockHeader, CTxIn, CTxOut, etc....:
  16     data structures that should map to corresponding structures in
  17     bitcoin/primitives
  18 msg_block, msg_tx, msg_headers, etc.:
  19     data structures that represent network messages
  20 ser_*, deser_*: functions that handle serialization/deserialization
  21 """
  22
  23 import asyncore
  24 from codecs import encode
  25 from collections import defaultdict
  26 import copy
  27 import hashlib
  28 from io import BytesIO
  29 import logging
  30 import random
  31 import socket
  32 import struct
  33 import sys
  34 import time
  35 from threading import RLock, Thread
  36
  37 from test_framework.siphash import siphash256
  38 from test_framework.util import hex_str_to_bytes, bytes_to_hex_str, wait_until
  39
  40 MIN_VERSION_SUPPORTED = 60001
  41 MY_VERSION = 70014  # past bip-31 for ping/pong
  42 MY_SUBVERSION = b"/python-mininode-tester:0.0.3/"
  43 MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37)
  44
  45 MAX_INV_SZ = 50000
  46 MAX_BLOCK_BASE_SIZE = 1000000
  47
  48 COIN = 100000000 # 1 btc in satoshis
  49
  50 NODE_NETWORK = (1 << 0)
  51 # NODE_GETUTXO = (1 << 1)
  52 # NODE_BLOOM = (1 << 2)
  53 NODE_WITNESS = (1 << 3)
  54 NODE_UNSUPPORTED_SERVICE_BIT_5 = (1 << 5)
  55 NODE_UNSUPPORTED_SERVICE_BIT_7 = (1 << 7)
  56
  57 logger = logging.getLogger("TestFramework.mininode")
  58
  59 # Keep our own socket map for asyncore, so that we can track disconnects
  60 # ourselves (to workaround an issue with closing an asyncore socket when
  61 # using select)
  62 mininode_socket_map = dict()
  63
  64 # One lock for synchronizing all data access between the networking thread (see
  65 # NetworkThread below) and the thread running the test logic.  For simplicity,
  66 # NodeConn acquires this lock whenever delivering a message to a NodeConnCB,
  67 # and whenever adding anything to the send buffer (in send_message()).  This
  68 # lock should be acquired in the thread running the test logic to synchronize
  69 # access to any data shared with the NodeConnCB or NodeConn.
  70 mininode_lock = RLock()
  71
  72 # Serialization/deserialization tools
  73 def sha256(s):
  74     return hashlib.new('sha256', s).digest()
  75
  76 def ripemd160(s):
  77     return hashlib.new('ripemd160', s).digest()
  78
  79 def hash256(s):
  80     return sha256(sha256(s))
  81
  82 def ser_compact_size(l):
  83     r = b""
  84     if l < 253:
  85         r = struct.pack("B", l)
  86     elif l < 0x10000:
  87         r = struct.pack("<BH", 253, l)
  88     elif l < 0x100000000:
  89         r = struct.pack("<BI", 254, l)
  90     else:
  91         r = struct.pack("<BQ", 255, l)
  92     return r
  93
  94 def deser_compact_size(f):
  95     nit = struct.unpack("<B", f.read(1))[0]
  96     if nit == 253:
  97         nit = struct.unpack("<H", f.read(2))[0]
  98     elif nit == 254:
  99         nit = struct.unpack("<I", f.read(4))[0]
 100     elif nit == 255:
 101         nit = struct.unpack("<Q", f.read(8))[0]
 102     return nit
 103
 104 def deser_string(f):
 105     nit = deser_compact_size(f)
 106     return f.read(nit)
 107
 108 def ser_string(s):
 109     return ser_compact_size(len(s)) + s
 110
 111 def deser_uint256(f):
 112     r = 0
 113     for i in range(8):
 114         t = struct.unpack("<I", f.read(4))[0]
 115         r += t << (i * 32)
 116     return r
 117
 118
 119 def ser_uint256(u):
 120     rs = b""
 121     for i in range(8):
 122         rs += struct.pack("<I", u & 0xFFFFFFFF)
 123         u >>= 32
 124     return rs
 125
 126
 127 def uint256_from_str(s):
 128     r = 0
 129     t = struct.unpack("<IIIIIIII", s[:32])
 130     for i in range(8):
 131         r += t[i] << (i * 32)
 132     return r
 133
 134
 135 def uint256_from_compact(c):
 136     nbytes = (c >> 24) & 0xFF
 137     v = (c & 0xFFFFFF) << (8 * (nbytes - 3))
 138     return v
 139
 140
 141 def deser_vector(f, c):
 142     nit = deser_compact_size(f)
 143     r = []
 144     for i in range(nit):
 145         t = c()
 146         t.deserialize(f)
 147         r.append(t)
 148     return r
 149
 150
 151 # ser_function_name: Allow for an alternate serialization function on the
 152 # entries in the vector (we use this for serializing the vector of transactions
 153 # for a witness block).
 154 def ser_vector(l, ser_function_name=None):
 155     r = ser_compact_size(len(l))
 156     for i in l:
 157         if ser_function_name:
 158             r += getattr(i, ser_function_name)()
 159         else:
 160             r += i.serialize()
 161     return r
 162
 163
 164 def deser_uint256_vector(f):
 165     nit = deser_compact_size(f)
 166     r = []
 167     for i in range(nit):
 168         t = deser_uint256(f)
 169         r.append(t)
 170     return r
 171
 172
 173 def ser_uint256_vector(l):
 174     r = ser_compact_size(len(l))
 175     for i in l:
 176         r += ser_uint256(i)
 177     return r
 178
 179
 180 def deser_string_vector(f):
 181     nit = deser_compact_size(f)
 182     r = []
 183     for i in range(nit):
 184         t = deser_string(f)
 185         r.append(t)
 186     return r
 187
 188
 189 def ser_string_vector(l):
 190     r = ser_compact_size(len(l))
 191     for sv in l:
 192         r += ser_string(sv)
 193     return r
 194
 195
 196 def deser_int_vector(f):
 197     nit = deser_compact_size(f)
 198     r = []
 199     for i in range(nit):
 200         t = struct.unpack("<i", f.read(4))[0]
 201         r.append(t)
 202     return r
 203
 204
 205 def ser_int_vector(l):
 206     r = ser_compact_size(len(l))
 207     for i in l:
 208         r += struct.pack("<i", i)
 209     return r
 210
 211 # Deserialize from a hex string representation (eg from RPC)
 212 def FromHex(obj, hex_string):
 213     obj.deserialize(BytesIO(hex_str_to_bytes(hex_string)))
 214     return obj
 215
 216 # Convert a binary-serializable object to hex (eg for submission via RPC)
 217 def ToHex(obj):
 218     return bytes_to_hex_str(obj.serialize())
 219
 220 # Objects that map to bitcoind objects, which can be serialized/deserialized
 221
 222 class CAddress():
 223     def __init__(self):
 224         self.nServices = 1
 225         self.pchReserved = b"\x00" * 10 + b"\xff" * 2
 226         self.ip = "0.0.0.0"
 227         self.port = 0
 228
 229     def deserialize(self, f):
 230         self.nServices = struct.unpack("<Q", f.read(8))[0]
 231         self.pchReserved = f.read(12)
 232         self.ip = socket.inet_ntoa(f.read(4))
 233         self.port = struct.unpack(">H", f.read(2))[0]
 234
 235     def serialize(self):
 236         r = b""
 237         r += struct.pack("<Q", self.nServices)
 238         r += self.pchReserved
 239         r += socket.inet_aton(self.ip)
 240         r += struct.pack(">H", self.port)
 241         return r
 242
 243     def __repr__(self):
 244         return "CAddress(nServices=%i ip=%s port=%i)" % (self.nServices,
 245                                                          self.ip, self.port)
 246
 247 MSG_WITNESS_FLAG = 1<<30
 248
 249 class CInv():
 250     typemap = {
 251         0: "Error",
 252         1: "TX",
 253         2: "Block",
 254         1|MSG_WITNESS_FLAG: "WitnessTx",
 255         2|MSG_WITNESS_FLAG : "WitnessBlock",
 256         4: "CompactBlock"
 257     }
 258
 259     def __init__(self, t=0, h=0):
 260         self.type = t
 261         self.hash = h
 262
 263     def deserialize(self, f):
 264         self.type = struct.unpack("<i", f.read(4))[0]
 265         self.hash = deser_uint256(f)
 266
 267     def serialize(self):
 268         r = b""
 269         r += struct.pack("<i", self.type)
 270         r += ser_uint256(self.hash)
 271         return r
 272
 273     def __repr__(self):
 274         return "CInv(type=%s hash=%064x)" \
 275             % (self.typemap[self.type], self.hash)
 276
 277
 278 class CBlockLocator():
 279     def __init__(self):
 280         self.nVersion = MY_VERSION
 281         self.vHave = []
 282
 283     def deserialize(self, f):
 284         self.nVersion = struct.unpack("<i", f.read(4))[0]
 285         self.vHave = deser_uint256_vector(f)
 286
 287     def serialize(self):
 288         r = b""
 289         r += struct.pack("<i", self.nVersion)
 290         r += ser_uint256_vector(self.vHave)
 291         return r
 292
 293     def __repr__(self):
 294         return "CBlockLocator(nVersion=%i vHave=%s)" \
 295             % (self.nVersion, repr(self.vHave))
 296
 297
 298 class COutPoint():
 299     def __init__(self, hash=0, n=0):
 300         self.hash = hash
 301         self.n = n
 302
 303     def deserialize(self, f):
 304         self.hash = deser_uint256(f)
 305         self.n = struct.unpack("<I", f.read(4))[0]
 306
 307     def serialize(self):
 308         r = b""
 309         r += ser_uint256(self.hash)
 310         r += struct.pack("<I", self.n)
 311         return r
 312
 313     def __repr__(self):
 314         return "COutPoint(hash=%064x n=%i)" % (self.hash, self.n)
 315
 316
 317 class CTxIn():
 318     def __init__(self, outpoint=None, scriptSig=b"", nSequence=0):
 319         if outpoint is None:
 320             self.prevout = COutPoint()
 321         else:
 322             self.prevout = outpoint
 323         self.scriptSig = scriptSig
 324         self.nSequence = nSequence
 325
 326     def deserialize(self, f):
 327         self.prevout = COutPoint()
 328         self.prevout.deserialize(f)
 329         self.scriptSig = deser_string(f)
 330         self.nSequence = struct.unpack("<I", f.read(4))[0]
 331
 332     def serialize(self):
 333         r = b""
 334         r += self.prevout.serialize()
 335         r += ser_string(self.scriptSig)
 336         r += struct.pack("<I", self.nSequence)
 337         return r
 338
 339     def __repr__(self):
 340         return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \
 341             % (repr(self.prevout), bytes_to_hex_str(self.scriptSig),
 342                self.nSequence)
 343
 344
 345 class CTxOut():
 346     def __init__(self, nValue=0, scriptPubKey=b""):
 347         self.nValue = nValue
 348         self.scriptPubKey = scriptPubKey
 349
 350     def deserialize(self, f):
 351         self.nValue = struct.unpack("<q", f.read(8))[0]
 352         self.scriptPubKey = deser_string(f)
 353
 354     def serialize(self):
 355         r = b""
 356         r += struct.pack("<q", self.nValue)
 357         r += ser_string(self.scriptPubKey)
 358         return r
 359
 360     def __repr__(self):
 361         return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \
 362             % (self.nValue // COIN, self.nValue % COIN,
 363                bytes_to_hex_str(self.scriptPubKey))
 364
 365
 366 class CScriptWitness():
 367     def __init__(self):
 368         # stack is a vector of strings
 369         self.stack = []
 370
 371     def __repr__(self):
 372         return "CScriptWitness(%s)" % \
 373                (",".join([bytes_to_hex_str(x) for x in self.stack]))
 374
 375     def is_null(self):
 376         if self.stack:
 377             return False
 378         return True
 379
 380
 381 class CTxInWitness():
 382     def __init__(self):
 383         self.scriptWitness = CScriptWitness()
 384
 385     def deserialize(self, f):
 386         self.scriptWitness.stack = deser_string_vector(f)
 387
 388     def serialize(self):
 389         return ser_string_vector(self.scriptWitness.stack)
 390
 391     def __repr__(self):
 392         return repr(self.scriptWitness)
 393
 394     def is_null(self):
 395         return self.scriptWitness.is_null()
 396
 397
 398 class CTxWitness():
 399     def __init__(self):
 400         self.vtxinwit = []
 401
 402     def deserialize(self, f):
 403         for i in range(len(self.vtxinwit)):
 404             self.vtxinwit[i].deserialize(f)
 405
 406     def serialize(self):
 407         r = b""
 408         # This is different than the usual vector serialization --
 409         # we omit the length of the vector, which is required to be
 410         # the same length as the transaction's vin vector.
 411         for x in self.vtxinwit:
 412             r += x.serialize()
 413         return r
 414
 415     def __repr__(self):
 416         return "CTxWitness(%s)" % \
 417                (';'.join([repr(x) for x in self.vtxinwit]))
 418
 419     def is_null(self):
 420         for x in self.vtxinwit:
 421             if not x.is_null():
 422                 return False
 423         return True
 424
 425
 426 class CTransaction():
 427     def __init__(self, tx=None):
 428         if tx is None:
 429             self.nVersion = 1
 430             self.vin = []
 431             self.vout = []
 432             self.wit = CTxWitness()
 433             self.nLockTime = 0
 434             self.sha256 = None
 435             self.hash = None
 436         else:
 437             self.nVersion = tx.nVersion
 438             self.vin = copy.deepcopy(tx.vin)
 439             self.vout = copy.deepcopy(tx.vout)
 440             self.nLockTime = tx.nLockTime
 441             self.sha256 = tx.sha256
 442             self.hash = tx.hash
 443             self.wit = copy.deepcopy(tx.wit)
 444
 445     def deserialize(self, f):
 446         self.nVersion = struct.unpack("<i", f.read(4))[0]
 447         self.vin = deser_vector(f, CTxIn)
 448         flags = 0
 449         if len(self.vin) == 0:
 450             flags = struct.unpack("<B", f.read(1))[0]
 451             # Not sure why flags can't be zero, but this
 452             # matches the implementation in bitcoind
 453             if (flags != 0):
 454                 self.vin = deser_vector(f, CTxIn)
 455                 self.vout = deser_vector(f, CTxOut)
 456         else:
 457             self.vout = deser_vector(f, CTxOut)
 458         if flags != 0:
 459             self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))]
 460             self.wit.deserialize(f)
 461         self.nLockTime = struct.unpack("<I", f.read(4))[0]
 462         self.sha256 = None
 463         self.hash = None
 464
 465     def serialize_without_witness(self):
 466         r = b""
 467         r += struct.pack("<i", self.nVersion)
 468         r += ser_vector(self.vin)
 469         r += ser_vector(self.vout)
 470         r += struct.pack("<I", self.nLockTime)
 471         return r
 472
 473     # Only serialize with witness when explicitly called for
 474     def serialize_with_witness(self):
 475         flags = 0
 476         if not self.wit.is_null():
 477             flags |= 1
 478         r = b""
 479         r += struct.pack("<i", self.nVersion)
 480         if flags:
 481             dummy = []
 482             r += ser_vector(dummy)
 483             r += struct.pack("<B", flags)
 484         r += ser_vector(self.vin)
 485         r += ser_vector(self.vout)
 486         if flags & 1:
 487             if (len(self.wit.vtxinwit) != len(self.vin)):
 488                 # vtxinwit must have the same length as vin
 489                 self.wit.vtxinwit = self.wit.vtxinwit[:len(self.vin)]
 490                 for i in range(len(self.wit.vtxinwit), len(self.vin)):
 491                     self.wit.vtxinwit.append(CTxInWitness())
 492             r += self.wit.serialize()
 493         r += struct.pack("<I", self.nLockTime)
 494         return r
 495
 496     # Regular serialization is without witness -- must explicitly
 497     # call serialize_with_witness to include witness data.
 498     def serialize(self):
 499         return self.serialize_without_witness()
 500
 501     # Recalculate the txid (transaction hash without witness)
 502     def rehash(self):
 503         self.sha256 = None
 504         self.calc_sha256()
 505
 506     # We will only cache the serialization without witness in
 507     # self.sha256 and self.hash -- those are expected to be the txid.
 508     def calc_sha256(self, with_witness=False):
 509         if with_witness:
 510             # Don't cache the result, just return it
 511             return uint256_from_str(hash256(self.serialize_with_witness()))
 512
 513         if self.sha256 is None:
 514             self.sha256 = uint256_from_str(hash256(self.serialize_without_witness()))
 515         self.hash = encode(hash256(self.serialize())[::-1], 'hex_codec').decode('ascii')
 516
 517     def is_valid(self):
 518         self.calc_sha256()
 519         for tout in self.vout:
 520             if tout.nValue < 0 or tout.nValue > 21000000 * COIN:
 521                 return False
 522         return True
 523
 524     def __repr__(self):
 525         return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \
 526             % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime)
 527
 528
 529 class CBlockHeader():
 530     def __init__(self, header=None):
 531         if header is None:
 532             self.set_null()
 533         else:
 534             self.nVersion = header.nVersion
 535             self.hashPrevBlock = header.hashPrevBlock
 536             self.hashMerkleRoot = header.hashMerkleRoot
 537             self.nTime = header.nTime
 538             self.nBits = header.nBits
 539             self.nNonce = header.nNonce
 540             self.sha256 = header.sha256
 541             self.hash = header.hash
 542             self.calc_sha256()
 543
 544     def set_null(self):
 545         self.nVersion = 1
 546         self.hashPrevBlock = 0
 547         self.hashMerkleRoot = 0
 548         self.nTime = 0
 549         self.nBits = 0
 550         self.nNonce = 0
 551         self.sha256 = None
 552         self.hash = None
 553
 554     def deserialize(self, f):
 555         self.nVersion = struct.unpack("<i", f.read(4))[0]
 556         self.hashPrevBlock = deser_uint256(f)
 557         self.hashMerkleRoot = deser_uint256(f)
 558         self.nTime = struct.unpack("<I", f.read(4))[0]
 559         self.nBits = struct.unpack("<I", f.read(4))[0]
 560         self.nNonce = struct.unpack("<I", f.read(4))[0]
 561         self.sha256 = None
 562         self.hash = None
 563
 564     def serialize(self):
 565         r = b""
 566         r += struct.pack("<i", self.nVersion)
 567         r += ser_uint256(self.hashPrevBlock)
 568         r += ser_uint256(self.hashMerkleRoot)
 569         r += struct.pack("<I", self.nTime)
 570         r += struct.pack("<I", self.nBits)
 571         r += struct.pack("<I", self.nNonce)
 572         return r
 573
 574     def calc_sha256(self):
 575         if self.sha256 is None:
 576             r = b""
 577             r += struct.pack("<i", self.nVersion)
 578             r += ser_uint256(self.hashPrevBlock)
 579             r += ser_uint256(self.hashMerkleRoot)
 580             r += struct.pack("<I", self.nTime)
 581             r += struct.pack("<I", self.nBits)
 582             r += struct.pack("<I", self.nNonce)
 583             self.sha256 = uint256_from_str(hash256(r))
 584             self.hash = encode(hash256(r)[::-1], 'hex_codec').decode('ascii')
 585
 586     def rehash(self):
 587         self.sha256 = None
 588         self.calc_sha256()
 589         return self.sha256
 590
 591     def __repr__(self):
 592         return "CBlockHeader(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x)" \
 593             % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
 594                time.ctime(self.nTime), self.nBits, self.nNonce)
 595
 596
 597 class CBlock(CBlockHeader):
 598     def __init__(self, header=None):
 599         super(CBlock, self).__init__(header)
 600         self.vtx = []
 601
 602     def deserialize(self, f):
 603         super(CBlock, self).deserialize(f)
 604         self.vtx = deser_vector(f, CTransaction)
 605
 606     def serialize(self, with_witness=False):
 607         r = b""
 608         r += super(CBlock, self).serialize()
 609         if with_witness:
 610             r += ser_vector(self.vtx, "serialize_with_witness")
 611         else:
 612             r += ser_vector(self.vtx)
 613         return r
 614
 615     # Calculate the merkle root given a vector of transaction hashes
 616     @classmethod
 617     def get_merkle_root(cls, hashes):
 618         while len(hashes) > 1:
 619             newhashes = []
 620             for i in range(0, len(hashes), 2):
 621                 i2 = min(i+1, len(hashes)-1)
 622                 newhashes.append(hash256(hashes[i] + hashes[i2]))
 623             hashes = newhashes
 624         return uint256_from_str(hashes[0])
 625
 626     def calc_merkle_root(self):
 627         hashes = []
 628         for tx in self.vtx:
 629             tx.calc_sha256()
 630             hashes.append(ser_uint256(tx.sha256))
 631         return self.get_merkle_root(hashes)
 632
 633     def calc_witness_merkle_root(self):
 634         # For witness root purposes, the hash of the
 635         # coinbase, with witness, is defined to be 0...0
 636         hashes = [ser_uint256(0)]
 637
 638         for tx in self.vtx[1:]:
 639             # Calculate the hashes with witness data
 640             hashes.append(ser_uint256(tx.calc_sha256(True)))
 641
 642         return self.get_merkle_root(hashes)
 643
 644     def is_valid(self):
 645         self.calc_sha256()
 646         target = uint256_from_compact(self.nBits)
 647         if self.sha256 > target:
 648             return False
 649         for tx in self.vtx:
 650             if not tx.is_valid():
 651                 return False
 652         if self.calc_merkle_root() != self.hashMerkleRoot:
 653             return False
 654         return True
 655
 656     def solve(self):
 657         self.rehash()
 658         target = uint256_from_compact(self.nBits)
 659         while self.sha256 > target:
 660             self.nNonce += 1
 661             self.rehash()
 662
 663     def __repr__(self):
 664         return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x vtx=%s)" \
 665             % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
 666                time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx))
 667
 668
 669 class PrefilledTransaction():
 670     def __init__(self, index=0, tx = None):
 671         self.index = index
 672         self.tx = tx
 673
 674     def deserialize(self, f):
 675         self.index = deser_compact_size(f)
 676         self.tx = CTransaction()
 677         self.tx.deserialize(f)
 678
 679     def serialize(self, with_witness=False):
 680         r = b""
 681         r += ser_compact_size(self.index)
 682         if with_witness:
 683             r += self.tx.serialize_with_witness()
 684         else:
 685             r += self.tx.serialize_without_witness()
 686         return r
 687
 688     def serialize_with_witness(self):
 689         return self.serialize(with_witness=True)
 690
 691     def __repr__(self):
 692         return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx))
 693
 694 # This is what we send on the wire, in a cmpctblock message.
 695 class P2PHeaderAndShortIDs():
 696     def __init__(self):
 697         self.header = CBlockHeader()
 698         self.nonce = 0
 699         self.shortids_length = 0
 700         self.shortids = []
 701         self.prefilled_txn_length = 0
 702         self.prefilled_txn = []
 703
 704     def deserialize(self, f):
 705         self.header.deserialize(f)
 706         self.nonce = struct.unpack("<Q", f.read(8))[0]
 707         self.shortids_length = deser_compact_size(f)
 708         for i in range(self.shortids_length):
 709             # shortids are defined to be 6 bytes in the spec, so append
 710             # two zero bytes and read it in as an 8-byte number
 711             self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0])
 712         self.prefilled_txn = deser_vector(f, PrefilledTransaction)
 713         self.prefilled_txn_length = len(self.prefilled_txn)
 714
 715     # When using version 2 compact blocks, we must serialize with_witness.
 716     def serialize(self, with_witness=False):
 717         r = b""
 718         r += self.header.serialize()
 719         r += struct.pack("<Q", self.nonce)
 720         r += ser_compact_size(self.shortids_length)
 721         for x in self.shortids:
 722             # We only want the first 6 bytes
 723             r += struct.pack("<Q", x)[0:6]
 724         if with_witness:
 725             r += ser_vector(self.prefilled_txn, "serialize_with_witness")
 726         else:
 727             r += ser_vector(self.prefilled_txn)
 728         return r
 729
 730     def __repr__(self):
 731         return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn))
 732
 733 # P2P version of the above that will use witness serialization (for compact
 734 # block version 2)
 735 class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs):
 736     def serialize(self):
 737         return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True)
 738
 739 # Calculate the BIP 152-compact blocks shortid for a given transaction hash
 740 def calculate_shortid(k0, k1, tx_hash):
 741     expected_shortid = siphash256(k0, k1, tx_hash)
 742     expected_shortid &= 0x0000ffffffffffff
 743     return expected_shortid
 744
 745 # This version gets rid of the array lengths, and reinterprets the differential
 746 # encoding into indices that can be used for lookup.
 747 class HeaderAndShortIDs():
 748     def __init__(self, p2pheaders_and_shortids = None):
 749         self.header = CBlockHeader()
 750         self.nonce = 0
 751         self.shortids = []
 752         self.prefilled_txn = []
 753         self.use_witness = False
 754
 755         if p2pheaders_and_shortids != None:
 756             self.header = p2pheaders_and_shortids.header
 757             self.nonce = p2pheaders_and_shortids.nonce
 758             self.shortids = p2pheaders_and_shortids.shortids
 759             last_index = -1
 760             for x in p2pheaders_and_shortids.prefilled_txn:
 761                 self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx))
 762                 last_index = self.prefilled_txn[-1].index
 763
 764     def to_p2p(self):
 765         if self.use_witness:
 766             ret = P2PHeaderAndShortWitnessIDs()
 767         else:
 768             ret = P2PHeaderAndShortIDs()
 769         ret.header = self.header
 770         ret.nonce = self.nonce
 771         ret.shortids_length = len(self.shortids)
 772         ret.shortids = self.shortids
 773         ret.prefilled_txn_length = len(self.prefilled_txn)
 774         ret.prefilled_txn = []
 775         last_index = -1
 776         for x in self.prefilled_txn:
 777             ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx))
 778             last_index = x.index
 779         return ret
 780
 781     def get_siphash_keys(self):
 782         header_nonce = self.header.serialize()
 783         header_nonce += struct.pack("<Q", self.nonce)
 784         hash_header_nonce_as_str = sha256(header_nonce)
 785         key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0]
 786         key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0]
 787         return [ key0, key1 ]
 788
 789     # Version 2 compact blocks use wtxid in shortids (rather than txid)
 790     def initialize_from_block(self, block, nonce=0, prefill_list = [0], use_witness = False):
 791         self.header = CBlockHeader(block)
 792         self.nonce = nonce
 793         self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ]
 794         self.shortids = []
 795         self.use_witness = use_witness
 796         [k0, k1] = self.get_siphash_keys()
 797         for i in range(len(block.vtx)):
 798             if i not in prefill_list:
 799                 tx_hash = block.vtx[i].sha256
 800                 if use_witness:
 801                     tx_hash = block.vtx[i].calc_sha256(with_witness=True)
 802                 self.shortids.append(calculate_shortid(k0, k1, tx_hash))
 803
 804     def __repr__(self):
 805         return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn))
 806
 807
 808 class BlockTransactionsRequest():
 809
 810     def __init__(self, blockhash=0, indexes = None):
 811         self.blockhash = blockhash
 812         self.indexes = indexes if indexes != None else []
 813
 814     def deserialize(self, f):
 815         self.blockhash = deser_uint256(f)
 816         indexes_length = deser_compact_size(f)
 817         for i in range(indexes_length):
 818             self.indexes.append(deser_compact_size(f))
 819
 820     def serialize(self):
 821         r = b""
 822         r += ser_uint256(self.blockhash)
 823         r += ser_compact_size(len(self.indexes))
 824         for x in self.indexes:
 825             r += ser_compact_size(x)
 826         return r
 827
 828     # helper to set the differentially encoded indexes from absolute ones
 829     def from_absolute(self, absolute_indexes):
 830         self.indexes = []
 831         last_index = -1
 832         for x in absolute_indexes:
 833             self.indexes.append(x-last_index-1)
 834             last_index = x
 835
 836     def to_absolute(self):
 837         absolute_indexes = []
 838         last_index = -1
 839         for x in self.indexes:
 840             absolute_indexes.append(x+last_index+1)
 841             last_index = absolute_indexes[-1]
 842         return absolute_indexes
 843
 844     def __repr__(self):
 845         return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes))
 846
 847
 848 class BlockTransactions():
 849
 850     def __init__(self, blockhash=0, transactions = None):
 851         self.blockhash = blockhash
 852         self.transactions = transactions if transactions != None else []
 853
 854     def deserialize(self, f):
 855         self.blockhash = deser_uint256(f)
 856         self.transactions = deser_vector(f, CTransaction)
 857
 858     def serialize(self, with_witness=False):
 859         r = b""
 860         r += ser_uint256(self.blockhash)
 861         if with_witness:
 862             r += ser_vector(self.transactions, "serialize_with_witness")
 863         else:
 864             r += ser_vector(self.transactions)
 865         return r
 866
 867     def __repr__(self):
 868         return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions))
 869
 870
 871 # Objects that correspond to messages on the wire
 872 class msg_version():
 873     command = b"version"
 874
 875     def __init__(self):
 876         self.nVersion = MY_VERSION
 877         self.nServices = NODE_NETWORK | NODE_WITNESS
 878         self.nTime = int(time.time())
 879         self.addrTo = CAddress()
 880         self.addrFrom = CAddress()
 881         self.nNonce = random.getrandbits(64)
 882         self.strSubVer = MY_SUBVERSION
 883         self.nStartingHeight = -1
 884         self.nRelay = MY_RELAY
 885
 886     def deserialize(self, f):
 887         self.nVersion = struct.unpack("<i", f.read(4))[0]
 888         if self.nVersion == 10300:
 889             self.nVersion = 300
 890         self.nServices = struct.unpack("<Q", f.read(8))[0]
 891         self.nTime = struct.unpack("<q", f.read(8))[0]
 892         self.addrTo = CAddress()
 893         self.addrTo.deserialize(f)
 894
 895         if self.nVersion >= 106:
 896             self.addrFrom = CAddress()
 897             self.addrFrom.deserialize(f)
 898             self.nNonce = struct.unpack("<Q", f.read(8))[0]
 899             self.strSubVer = deser_string(f)
 900         else:
 901             self.addrFrom = None
 902             self.nNonce = None
 903             self.strSubVer = None
 904             self.nStartingHeight = None
 905
 906         if self.nVersion >= 209:
 907             self.nStartingHeight = struct.unpack("<i", f.read(4))[0]
 908         else:
 909             self.nStartingHeight = None
 910
 911         if self.nVersion >= 70001:
 912             # Relay field is optional for version 70001 onwards
 913             try:
 914                 self.nRelay = struct.unpack("<b", f.read(1))[0]
 915             except:
 916                 self.nRelay = 0
 917         else:
 918             self.nRelay = 0
 919
 920     def serialize(self):
 921         r = b""
 922         r += struct.pack("<i", self.nVersion)
 923         r += struct.pack("<Q", self.nServices)
 924         r += struct.pack("<q", self.nTime)
 925         r += self.addrTo.serialize()
 926         r += self.addrFrom.serialize()
 927         r += struct.pack("<Q", self.nNonce)
 928         r += ser_string(self.strSubVer)
 929         r += struct.pack("<i", self.nStartingHeight)
 930         r += struct.pack("<b", self.nRelay)
 931         return r
 932
 933     def __repr__(self):
 934         return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i nRelay=%i)' \
 935             % (self.nVersion, self.nServices, time.ctime(self.nTime),
 936                repr(self.addrTo), repr(self.addrFrom), self.nNonce,
 937                self.strSubVer, self.nStartingHeight, self.nRelay)
 938
 939
 940 class msg_verack():
 941     command = b"verack"
 942
 943     def __init__(self):
 944         pass
 945
 946     def deserialize(self, f):
 947         pass
 948
 949     def serialize(self):
 950         return b""
 951
 952     def __repr__(self):
 953         return "msg_verack()"
 954
 955
 956 class msg_addr():
 957     command = b"addr"
 958
 959     def __init__(self):
 960         self.addrs = []
 961
 962     def deserialize(self, f):
 963         self.addrs = deser_vector(f, CAddress)
 964
 965     def serialize(self):
 966         return ser_vector(self.addrs)
 967
 968     def __repr__(self):
 969         return "msg_addr(addrs=%s)" % (repr(self.addrs))
 970
 971
 972 class msg_inv():
 973     command = b"inv"
 974
 975     def __init__(self, inv=None):
 976         if inv is None:
 977             self.inv = []
 978         else:
 979             self.inv = inv
 980
 981     def deserialize(self, f):
 982         self.inv = deser_vector(f, CInv)
 983
 984     def serialize(self):
 985         return ser_vector(self.inv)
 986
 987     def __repr__(self):
 988         return "msg_inv(inv=%s)" % (repr(self.inv))
 989
 990
 991 class msg_getdata():
 992     command = b"getdata"
 993
 994     def __init__(self, inv=None):
 995         self.inv = inv if inv != None else []
 996
 997     def deserialize(self, f):
 998         self.inv = deser_vector(f, CInv)
 999
1000     def serialize(self):
1001         return ser_vector(self.inv)
1002
1003     def __repr__(self):
1004         return "msg_getdata(inv=%s)" % (repr(self.inv))
1005
1006
1007 class msg_getblocks():
1008     command = b"getblocks"
1009
1010     def __init__(self):
1011         self.locator = CBlockLocator()
1012         self.hashstop = 0
1013
1014     def deserialize(self, f):
1015         self.locator = CBlockLocator()
1016         self.locator.deserialize(f)
1017         self.hashstop = deser_uint256(f)
1018
1019     def serialize(self):
1020         r = b""
1021         r += self.locator.serialize()
1022         r += ser_uint256(self.hashstop)
1023         return r
1024
1025     def __repr__(self):
1026         return "msg_getblocks(locator=%s hashstop=%064x)" \
1027             % (repr(self.locator), self.hashstop)
1028
1029
1030 class msg_tx():
1031     command = b"tx"
1032
1033     def __init__(self, tx=CTransaction()):
1034         self.tx = tx
1035
1036     def deserialize(self, f):
1037         self.tx.deserialize(f)
1038
1039     def serialize(self):
1040         return self.tx.serialize_without_witness()
1041
1042     def __repr__(self):
1043         return "msg_tx(tx=%s)" % (repr(self.tx))
1044
1045 class msg_witness_tx(msg_tx):
1046
1047     def serialize(self):
1048         return self.tx.serialize_with_witness()
1049
1050
1051 class msg_block():
1052     command = b"block"
1053
1054     def __init__(self, block=None):
1055         if block is None:
1056             self.block = CBlock()
1057         else:
1058             self.block = block
1059
1060     def deserialize(self, f):
1061         self.block.deserialize(f)
1062
1063     def serialize(self):
1064         return self.block.serialize()
1065
1066     def __repr__(self):
1067         return "msg_block(block=%s)" % (repr(self.block))
1068
1069 # for cases where a user needs tighter control over what is sent over the wire
1070 # note that the user must supply the name of the command, and the data
1071 class msg_generic():
1072     def __init__(self, command, data=None):
1073         self.command = command
1074         self.data = data
1075
1076     def serialize(self):
1077         return self.data
1078
1079     def __repr__(self):
1080         return "msg_generic()"
1081
1082 class msg_witness_block(msg_block):
1083
1084     def serialize(self):
1085         r = self.block.serialize(with_witness=True)
1086         return r
1087
1088 class msg_getaddr():
1089     command = b"getaddr"
1090
1091     def __init__(self):
1092         pass
1093
1094     def deserialize(self, f):
1095         pass
1096
1097     def serialize(self):
1098         return b""
1099
1100     def __repr__(self):
1101         return "msg_getaddr()"
1102
1103
1104 class msg_ping():
1105     command = b"ping"
1106
1107     def __init__(self, nonce=0):
1108         self.nonce = nonce
1109
1110     def deserialize(self, f):
1111         self.nonce = struct.unpack("<Q", f.read(8))[0]
1112
1113     def serialize(self):
1114         r = b""
1115         r += struct.pack("<Q", self.nonce)
1116         return r
1117
1118     def __repr__(self):
1119         return "msg_ping(nonce=%08x)" % self.nonce
1120
1121
1122 class msg_pong():
1123     command = b"pong"
1124
1125     def __init__(self, nonce=0):
1126         self.nonce = nonce
1127
1128     def deserialize(self, f):
1129         self.nonce = struct.unpack("<Q", f.read(8))[0]
1130
1131     def serialize(self):
1132         r = b""
1133         r += struct.pack("<Q", self.nonce)
1134         return r
1135
1136     def __repr__(self):
1137         return "msg_pong(nonce=%08x)" % self.nonce
1138
1139
1140 class msg_mempool():
1141     command = b"mempool"
1142
1143     def __init__(self):
1144         pass
1145
1146     def deserialize(self, f):
1147         pass
1148
1149     def serialize(self):
1150         return b""
1151
1152     def __repr__(self):
1153         return "msg_mempool()"
1154
1155 class msg_sendheaders():
1156     command = b"sendheaders"
1157
1158     def __init__(self):
1159         pass
1160
1161     def deserialize(self, f):
1162         pass
1163
1164     def serialize(self):
1165         return b""
1166
1167     def __repr__(self):
1168         return "msg_sendheaders()"
1169
1170
1171 # getheaders message has
1172 # number of entries
1173 # vector of hashes
1174 # hash_stop (hash of last desired block header, 0 to get as many as possible)
1175 class msg_getheaders():
1176     command = b"getheaders"
1177
1178     def __init__(self):
1179         self.locator = CBlockLocator()
1180         self.hashstop = 0
1181
1182     def deserialize(self, f):
1183         self.locator = CBlockLocator()
1184         self.locator.deserialize(f)
1185         self.hashstop = deser_uint256(f)
1186
1187     def serialize(self):
1188         r = b""
1189         r += self.locator.serialize()
1190         r += ser_uint256(self.hashstop)
1191         return r
1192
1193     def __repr__(self):
1194         return "msg_getheaders(locator=%s, stop=%064x)" \
1195             % (repr(self.locator), self.hashstop)
1196
1197
1198 # headers message has
1199 # <count> <vector of block headers>
1200 class msg_headers():
1201     command = b"headers"
1202
1203     def __init__(self, headers=None):
1204         self.headers = headers if headers is not None else []
1205
1206     def deserialize(self, f):
1207         # comment in bitcoind indicates these should be deserialized as blocks
1208         blocks = deser_vector(f, CBlock)
1209         for x in blocks:
1210             self.headers.append(CBlockHeader(x))
1211
1212     def serialize(self):
1213         blocks = [CBlock(x) for x in self.headers]
1214         return ser_vector(blocks)
1215
1216     def __repr__(self):
1217         return "msg_headers(headers=%s)" % repr(self.headers)
1218
1219
1220 class msg_reject():
1221     command = b"reject"
1222     REJECT_MALFORMED = 1
1223
1224     def __init__(self):
1225         self.message = b""
1226         self.code = 0
1227         self.reason = b""
1228         self.data = 0
1229
1230     def deserialize(self, f):
1231         self.message = deser_string(f)
1232         self.code = struct.unpack("<B", f.read(1))[0]
1233         self.reason = deser_string(f)
1234         if (self.code != self.REJECT_MALFORMED and
1235                 (self.message == b"block" or self.message == b"tx")):
1236             self.data = deser_uint256(f)
1237
1238     def serialize(self):
1239         r = ser_string(self.message)
1240         r += struct.pack("<B", self.code)
1241         r += ser_string(self.reason)
1242         if (self.code != self.REJECT_MALFORMED and
1243                 (self.message == b"block" or self.message == b"tx")):
1244             r += ser_uint256(self.data)
1245         return r
1246
1247     def __repr__(self):
1248         return "msg_reject: %s %d %s [%064x]" \
1249             % (self.message, self.code, self.reason, self.data)
1250
1251 class msg_feefilter():
1252     command = b"feefilter"
1253
1254     def __init__(self, feerate=0):
1255         self.feerate = feerate
1256
1257     def deserialize(self, f):
1258         self.feerate = struct.unpack("<Q", f.read(8))[0]
1259
1260     def serialize(self):
1261         r = b""
1262         r += struct.pack("<Q", self.feerate)
1263         return r
1264
1265     def __repr__(self):
1266         return "msg_feefilter(feerate=%08x)" % self.feerate
1267
1268 class msg_sendcmpct():
1269     command = b"sendcmpct"
1270
1271     def __init__(self):
1272         self.announce = False
1273         self.version = 1
1274
1275     def deserialize(self, f):
1276         self.announce = struct.unpack("<?", f.read(1))[0]
1277         self.version = struct.unpack("<Q", f.read(8))[0]
1278
1279     def serialize(self):
1280         r = b""
1281         r += struct.pack("<?", self.announce)
1282         r += struct.pack("<Q", self.version)
1283         return r
1284
1285     def __repr__(self):
1286         return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version)
1287
1288 class msg_cmpctblock():
1289     command = b"cmpctblock"
1290
1291     def __init__(self, header_and_shortids = None):
1292         self.header_and_shortids = header_and_shortids
1293
1294     def deserialize(self, f):
1295         self.header_and_shortids = P2PHeaderAndShortIDs()
1296         self.header_and_shortids.deserialize(f)
1297
1298     def serialize(self):
1299         r = b""
1300         r += self.header_and_shortids.serialize()
1301         return r
1302
1303     def __repr__(self):
1304         return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids)
1305
1306 class msg_getblocktxn():
1307     command = b"getblocktxn"
1308
1309     def __init__(self):
1310         self.block_txn_request = None
1311
1312     def deserialize(self, f):
1313         self.block_txn_request = BlockTransactionsRequest()
1314         self.block_txn_request.deserialize(f)
1315
1316     def serialize(self):
1317         r = b""
1318         r += self.block_txn_request.serialize()
1319         return r
1320
1321     def __repr__(self):
1322         return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request))
1323
1324 class msg_blocktxn():
1325     command = b"blocktxn"
1326
1327     def __init__(self):
1328         self.block_transactions = BlockTransactions()
1329
1330     def deserialize(self, f):
1331         self.block_transactions.deserialize(f)
1332
1333     def serialize(self):
1334         r = b""
1335         r += self.block_transactions.serialize()
1336         return r
1337
1338     def __repr__(self):
1339         return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions))
1340
1341 class msg_witness_blocktxn(msg_blocktxn):
1342     def serialize(self):
1343         r = b""
1344         r += self.block_transactions.serialize(with_witness=True)
1345         return r
1346
1347 class NodeConnCB():
1348     """Callback and helper functions for P2P connection to a bitcoind node.
1349
1350     Individual testcases should subclass this and override the on_* methods
1351     if they want to alter message handling behaviour."""
1352     def __init__(self):
1353         # Track whether we have a P2P connection open to the node
1354         self.connected = False
1355         self.connection = None
1356
1357         # Track number of messages of each type received and the most recent
1358         # message of each type
1359         self.message_count = defaultdict(int)
1360         self.last_message = {}
1361
1362         # A count of the number of ping messages we've sent to the node
1363         self.ping_counter = 1
1364
1365     # Message receiving methods
1366
1367     def deliver(self, conn, message):
1368         """Receive message and dispatch message to appropriate callback.
1369
1370         We keep a count of how many of each message type has been received
1371         and the most recent message of each type."""
1372         with mininode_lock:
1373             try:
1374                 command = message.command.decode('ascii')
1375                 self.message_count[command] += 1
1376                 self.last_message[command] = message
1377                 getattr(self, 'on_' + command)(conn, message)
1378             except:
1379                 print("ERROR delivering %s (%s)" % (repr(message),
1380                                                     sys.exc_info()[0]))
1381                 raise
1382
1383     # Callback methods. Can be overridden by subclasses in individual test
1384     # cases to provide custom message handling behaviour.
1385
1386     def on_open(self, conn):
1387         self.connected = True
1388
1389     def on_close(self, conn):
1390         self.connected = False
1391         self.connection = None
1392
1393     def on_addr(self, conn, message): pass
1394     def on_block(self, conn, message): pass
1395     def on_blocktxn(self, conn, message): pass
1396     def on_cmpctblock(self, conn, message): pass
1397     def on_feefilter(self, conn, message): pass
1398     def on_getaddr(self, conn, message): pass
1399     def on_getblocks(self, conn, message): pass
1400     def on_getblocktxn(self, conn, message): pass
1401     def on_getdata(self, conn, message): pass
1402     def on_getheaders(self, conn, message): pass
1403     def on_headers(self, conn, message): pass
1404     def on_mempool(self, conn): pass
1405     def on_pong(self, conn, message): pass
1406     def on_reject(self, conn, message): pass
1407     def on_sendcmpct(self, conn, message): pass
1408     def on_sendheaders(self, conn, message): pass
1409     def on_tx(self, conn, message): pass
1410
1411     def on_inv(self, conn, message):
1412         want = msg_getdata()
1413         for i in message.inv:
1414             if i.type != 0:
1415                 want.inv.append(i)
1416         if len(want.inv):
1417             conn.send_message(want)
1418
1419     def on_ping(self, conn, message):
1420         conn.send_message(msg_pong(message.nonce))
1421
1422     def on_verack(self, conn, message):
1423         self.verack_received = True
1424
1425     def on_version(self, conn, message):
1426         assert message.nVersion >= MIN_VERSION_SUPPORTED, "Version {} received. Test framework only supports versions greater than {}".format(message.nVersion, MIN_VERSION_SUPPORTED)
1427         conn.send_message(msg_verack())
1428         conn.nServices = message.nServices
1429
1430     # Connection helper methods
1431
1432     def add_connection(self, conn):
1433         self.connection = conn
1434
1435     def wait_for_disconnect(self, timeout=60):
1436         test_function = lambda: not self.connected
1437         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1438
1439     # Message receiving helper methods
1440
1441     def wait_for_block(self, blockhash, timeout=60):
1442         test_function = lambda: self.last_message.get("block") and self.last_message["block"].block.rehash() == blockhash
1443         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1444
1445     def wait_for_getdata(self, timeout=60):
1446         test_function = lambda: self.last_message.get("getdata")
1447         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1448
1449     def wait_for_getheaders(self, timeout=60):
1450         test_function = lambda: self.last_message.get("getheaders")
1451         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1452
1453     def wait_for_inv(self, expected_inv, timeout=60):
1454         """Waits for an INV message and checks that the first inv object in the message was as expected."""
1455         if len(expected_inv) > 1:
1456             raise NotImplementedError("wait_for_inv() will only verify the first inv object")
1457         test_function = lambda: self.last_message.get("inv") and \
1458                                 self.last_message["inv"].inv[0].type == expected_inv[0].type and \
1459                                 self.last_message["inv"].inv[0].hash == expected_inv[0].hash
1460         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1461
1462     def wait_for_verack(self, timeout=60):
1463         test_function = lambda: self.message_count["verack"]
1464         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1465
1466     # Message sending helper functions
1467
1468     def send_message(self, message):
1469         if self.connection:
1470             self.connection.send_message(message)
1471         else:
1472             logger.error("Cannot send message. No connection to node!")
1473
1474     def send_and_ping(self, message):
1475         self.send_message(message)
1476         self.sync_with_ping()
1477
1478     # Sync up with the node
1479     def sync_with_ping(self, timeout=60):
1480         self.send_message(msg_ping(nonce=self.ping_counter))
1481         test_function = lambda: self.last_message.get("pong") and self.last_message["pong"].nonce == self.ping_counter
1482         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1483         self.ping_counter += 1
1484
1485 class NodeConn(asyncore.dispatcher):
1486     """The actual NodeConn class
1487
1488     This class provides an interface for a p2p connection to a specified node."""
1489     messagemap = {
1490         b"version": msg_version,
1491         b"verack": msg_verack,
1492         b"addr": msg_addr,
1493         b"inv": msg_inv,
1494         b"getdata": msg_getdata,
1495         b"getblocks": msg_getblocks,
1496         b"tx": msg_tx,
1497         b"block": msg_block,
1498         b"getaddr": msg_getaddr,
1499         b"ping": msg_ping,
1500         b"pong": msg_pong,
1501         b"headers": msg_headers,
1502         b"getheaders": msg_getheaders,
1503         b"reject": msg_reject,
1504         b"mempool": msg_mempool,
1505         b"feefilter": msg_feefilter,
1506         b"sendheaders": msg_sendheaders,
1507         b"sendcmpct": msg_sendcmpct,
1508         b"cmpctblock": msg_cmpctblock,
1509         b"getblocktxn": msg_getblocktxn,
1510         b"blocktxn": msg_blocktxn
1511     }
1512     MAGIC_BYTES = {
1513         "mainnet": b"\xf9\xbe\xb4\xd9",   # mainnet
1514         "testnet3": b"\x0b\x11\x09\x07",  # testnet3
1515         "regtest": b"\xfa\xbf\xb5\xda",   # regtest
1516     }
1517
1518     def __init__(self, dstaddr, dstport, callback, net="regtest", services=NODE_NETWORK|NODE_WITNESS, send_version=True):
1519         asyncore.dispatcher.__init__(self, map=mininode_socket_map)
1520         self.dstaddr = dstaddr
1521         self.dstport = dstport
1522         self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
1523         self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
1524         self.sendbuf = b""
1525         self.recvbuf = b""
1526         self.last_sent = 0
1527         self.state = "connecting"
1528         self.network = net
1529         self.cb = callback
1530         self.disconnect = False
1531         self.nServices = 0
1532
1533         if send_version:
1534             # stuff version msg into sendbuf
1535             vt = msg_version()
1536             vt.nServices = services
1537             vt.addrTo.ip = self.dstaddr
1538             vt.addrTo.port = self.dstport
1539             vt.addrFrom.ip = "0.0.0.0"
1540             vt.addrFrom.port = 0
1541             self.send_message(vt, True)
1542
1543         logger.info('Connecting to Bitcoin Node: %s:%d' % (self.dstaddr, self.dstport))
1544
1545         try:
1546             self.connect((dstaddr, dstport))
1547         except:
1548             self.handle_close()
1549
1550     def handle_connect(self):
1551         if self.state != "connected":
1552             logger.debug("Connected & Listening: %s:%d" % (self.dstaddr, self.dstport))
1553             self.state = "connected"
1554             self.cb.on_open(self)
1555
1556     def handle_close(self):
1557         logger.debug("Closing connection to: %s:%d" % (self.dstaddr, self.dstport))
1558         self.state = "closed"
1559         self.recvbuf = b""
1560         self.sendbuf = b""
1561         try:
1562             self.close()
1563         except:
1564             pass
1565         self.cb.on_close(self)
1566
1567     def handle_read(self):
1568         t = self.recv(8192)
1569         if len(t) > 0:
1570             self.recvbuf += t
1571             self.got_data()
1572
1573     def readable(self):
1574         return True
1575
1576     def writable(self):
1577         with mininode_lock:
1578             pre_connection = self.state == "connecting"
1579             length = len(self.sendbuf)
1580         return (length > 0 or pre_connection)
1581
1582     def handle_write(self):
1583         with mininode_lock:
1584             # asyncore does not expose socket connection, only the first read/write
1585             # event, thus we must check connection manually here to know when we
1586             # actually connect
1587             if self.state == "connecting":
1588                 self.handle_connect()
1589             if not self.writable():
1590                 return
1591
1592             try:
1593                 sent = self.send(self.sendbuf)
1594             except:
1595                 self.handle_close()
1596                 return
1597             self.sendbuf = self.sendbuf[sent:]
1598
1599     def got_data(self):
1600         try:
1601             while True:
1602                 if len(self.recvbuf) < 4:
1603                     return
1604                 if self.recvbuf[:4] != self.MAGIC_BYTES[self.network]:
1605                     raise ValueError("got garbage %s" % repr(self.recvbuf))
1606                 if len(self.recvbuf) < 4 + 12 + 4 + 4:
1607                     return
1608                 command = self.recvbuf[4:4+12].split(b"\x00", 1)[0]
1609                 msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0]
1610                 checksum = self.recvbuf[4+12+4:4+12+4+4]
1611                 if len(self.recvbuf) < 4 + 12 + 4 + 4 + msglen:
1612                     return
1613                 msg = self.recvbuf[4+12+4+4:4+12+4+4+msglen]
1614                 th = sha256(msg)
1615                 h = sha256(th)
1616                 if checksum != h[:4]:
1617                     raise ValueError("got bad checksum " + repr(self.recvbuf))
1618                 self.recvbuf = self.recvbuf[4+12+4+4+msglen:]
1619                 if command not in self.messagemap:
1620                     raise ValueError("Received unknown command from %s:%d: '%s' %s" % (self.dstaddr, self.dstport, command, repr(msg)))
1621                 f = BytesIO(msg)
1622                 t = self.messagemap[command]()
1623                 t.deserialize(f)
1624                 self.got_message(t)
1625         except Exception as e:
1626             logger.exception('Error reading message:', repr(e))
1627             raise
1628
1629     def send_message(self, message, pushbuf=False):
1630         if self.state != "connected" and not pushbuf:
1631             raise IOError('Not connected, no pushbuf')
1632         self._log_message("send", message)
1633         command = message.command
1634         data = message.serialize()
1635         tmsg = self.MAGIC_BYTES[self.network]
1636         tmsg += command
1637         tmsg += b"\x00" * (12 - len(command))
1638         tmsg += struct.pack("<I", len(data))
1639         th = sha256(data)
1640         h = sha256(th)
1641         tmsg += h[:4]
1642         tmsg += data
1643         with mininode_lock:
1644             if (len(self.sendbuf) == 0 and not pushbuf):
1645                 try:
1646                     sent = self.send(tmsg)
1647                     self.sendbuf = tmsg[sent:]
1648                 except BlockingIOError:
1649                     self.sendbuf = tmsg
1650             else:
1651                 self.sendbuf += tmsg
1652             self.last_sent = time.time()
1653
1654     def got_message(self, message):
1655         if self.last_sent + 30 * 60 < time.time():
1656             self.send_message(self.messagemap[b'ping']())
1657         self._log_message("receive", message)
1658         self.cb.deliver(self, message)
1659
1660     def _log_message(self, direction, msg):
1661         if direction == "send":
1662             log_message = "Send message to "
1663         elif direction == "receive":
1664             log_message = "Received message from "
1665         log_message += "%s:%d: %s" % (self.dstaddr, self.dstport, repr(msg)[:500])
1666         if len(log_message) > 500:
1667             log_message += "... (msg truncated)"
1668         logger.debug(log_message)
1669
1670     def disconnect_node(self):
1671         self.disconnect = True
1672
1673
1674 class NetworkThread(Thread):
1675     def run(self):
1676         while mininode_socket_map:
1677             # We check for whether to disconnect outside of the asyncore
1678             # loop to workaround the behavior of asyncore when using
1679             # select
1680             disconnected = []
1681             for fd, obj in mininode_socket_map.items():
1682                 if obj.disconnect:
1683                     disconnected.append(obj)
1684             [ obj.handle_close() for obj in disconnected ]
1685             asyncore.loop(0.1, use_poll=True, map=mininode_socket_map, count=1)
1686         logger.debug("Network thread closing")