test/functional/test_framework/mininode.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2010 ArtForz -- public domain half-a-node
   3 # Copyright (c) 2012 Jeff Garzik
   4 # Copyright (c) 2010-2016 The Bitcoin Core developers
   5 # Distributed under the MIT software license, see the accompanying
   6 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   7 """Bitcoin P2P network half-a-node.
   8
   9 This python code was modified from ArtForz' public domain  half-a-node, as
  10 found in the mini-node branch of http://github.com/jgarzik/pynode.
  11
  12 NodeConn: an object which manages p2p connectivity to a bitcoin node
  13 NodeConnCB: a base class that describes the interface for receiving
  14             callbacks with network messages from a NodeConn
  15 CBlock, CTransaction, CBlockHeader, CTxIn, CTxOut, etc....:
  16     data structures that should map to corresponding structures in
  17     bitcoin/primitives
  18 msg_block, msg_tx, msg_headers, etc.:
  19     data structures that represent network messages
  20 ser_*, deser_*: functions that handle serialization/deserialization
  21 """
  22
  23 import asyncore
  24 from codecs import encode
  25 from collections import defaultdict
  26 import copy
  27 import hashlib
  28 from io import BytesIO
  29 import logging
  30 import random
  31 import socket
  32 import struct
  33 import sys
  34 import time
  35 from threading import RLock, Thread
  36
  37 from test_framework.siphash import siphash256
  38 from test_framework.util import hex_str_to_bytes, bytes_to_hex_str, wait_until
  39
  40 MY_VERSION = 70014  # past bip-31 for ping/pong
  41 MY_SUBVERSION = b"/python-mininode-tester:0.0.3/"
  42 MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37)
  43
  44 MAX_INV_SZ = 50000
  45 MAX_BLOCK_BASE_SIZE = 1000000
  46
  47 COIN = 100000000 # 1 btc in satoshis
  48
  49 NODE_NETWORK = (1 << 0)
  50 # NODE_GETUTXO = (1 << 1)
  51 # NODE_BLOOM = (1 << 2)
  52 NODE_WITNESS = (1 << 3)
  53 NODE_UNSUPPORTED_SERVICE_BIT_5 = (1 << 5)
  54 NODE_UNSUPPORTED_SERVICE_BIT_7 = (1 << 7)
  55
  56 logger = logging.getLogger("TestFramework.mininode")
  57
  58 # Keep our own socket map for asyncore, so that we can track disconnects
  59 # ourselves (to workaround an issue with closing an asyncore socket when
  60 # using select)
  61 mininode_socket_map = dict()
  62
  63 # One lock for synchronizing all data access between the networking thread (see
  64 # NetworkThread below) and the thread running the test logic.  For simplicity,
  65 # NodeConn acquires this lock whenever delivering a message to a NodeConnCB,
  66 # and whenever adding anything to the send buffer (in send_message()).  This
  67 # lock should be acquired in the thread running the test logic to synchronize
  68 # access to any data shared with the NodeConnCB or NodeConn.
  69 mininode_lock = RLock()
  70
  71 # Serialization/deserialization tools
  72 def sha256(s):
  73     return hashlib.new('sha256', s).digest()
  74
  75 def ripemd160(s):
  76     return hashlib.new('ripemd160', s).digest()
  77
  78 def hash256(s):
  79     return sha256(sha256(s))
  80
  81 def ser_compact_size(l):
  82     r = b""
  83     if l < 253:
  84         r = struct.pack("B", l)
  85     elif l < 0x10000:
  86         r = struct.pack("<BH", 253, l)
  87     elif l < 0x100000000:
  88         r = struct.pack("<BI", 254, l)
  89     else:
  90         r = struct.pack("<BQ", 255, l)
  91     return r
  92
  93 def deser_compact_size(f):
  94     nit = struct.unpack("<B", f.read(1))[0]
  95     if nit == 253:
  96         nit = struct.unpack("<H", f.read(2))[0]
  97     elif nit == 254:
  98         nit = struct.unpack("<I", f.read(4))[0]
  99     elif nit == 255:
 100         nit = struct.unpack("<Q", f.read(8))[0]
 101     return nit
 102
 103 def deser_string(f):
 104     nit = deser_compact_size(f)
 105     return f.read(nit)
 106
 107 def ser_string(s):
 108     return ser_compact_size(len(s)) + s
 109
 110 def deser_uint256(f):
 111     r = 0
 112     for i in range(8):
 113         t = struct.unpack("<I", f.read(4))[0]
 114         r += t << (i * 32)
 115     return r
 116
 117
 118 def ser_uint256(u):
 119     rs = b""
 120     for i in range(8):
 121         rs += struct.pack("<I", u & 0xFFFFFFFF)
 122         u >>= 32
 123     return rs
 124
 125
 126 def uint256_from_str(s):
 127     r = 0
 128     t = struct.unpack("<IIIIIIII", s[:32])
 129     for i in range(8):
 130         r += t[i] << (i * 32)
 131     return r
 132
 133
 134 def uint256_from_compact(c):
 135     nbytes = (c >> 24) & 0xFF
 136     v = (c & 0xFFFFFF) << (8 * (nbytes - 3))
 137     return v
 138
 139
 140 def deser_vector(f, c):
 141     nit = deser_compact_size(f)
 142     r = []
 143     for i in range(nit):
 144         t = c()
 145         t.deserialize(f)
 146         r.append(t)
 147     return r
 148
 149
 150 # ser_function_name: Allow for an alternate serialization function on the
 151 # entries in the vector (we use this for serializing the vector of transactions
 152 # for a witness block).
 153 def ser_vector(l, ser_function_name=None):
 154     r = ser_compact_size(len(l))
 155     for i in l:
 156         if ser_function_name:
 157             r += getattr(i, ser_function_name)()
 158         else:
 159             r += i.serialize()
 160     return r
 161
 162
 163 def deser_uint256_vector(f):
 164     nit = deser_compact_size(f)
 165     r = []
 166     for i in range(nit):
 167         t = deser_uint256(f)
 168         r.append(t)
 169     return r
 170
 171
 172 def ser_uint256_vector(l):
 173     r = ser_compact_size(len(l))
 174     for i in l:
 175         r += ser_uint256(i)
 176     return r
 177
 178
 179 def deser_string_vector(f):
 180     nit = deser_compact_size(f)
 181     r = []
 182     for i in range(nit):
 183         t = deser_string(f)
 184         r.append(t)
 185     return r
 186
 187
 188 def ser_string_vector(l):
 189     r = ser_compact_size(len(l))
 190     for sv in l:
 191         r += ser_string(sv)
 192     return r
 193
 194
 195 def deser_int_vector(f):
 196     nit = deser_compact_size(f)
 197     r = []
 198     for i in range(nit):
 199         t = struct.unpack("<i", f.read(4))[0]
 200         r.append(t)
 201     return r
 202
 203
 204 def ser_int_vector(l):
 205     r = ser_compact_size(len(l))
 206     for i in l:
 207         r += struct.pack("<i", i)
 208     return r
 209
 210 # Deserialize from a hex string representation (eg from RPC)
 211 def FromHex(obj, hex_string):
 212     obj.deserialize(BytesIO(hex_str_to_bytes(hex_string)))
 213     return obj
 214
 215 # Convert a binary-serializable object to hex (eg for submission via RPC)
 216 def ToHex(obj):
 217     return bytes_to_hex_str(obj.serialize())
 218
 219 # Objects that map to bitcoind objects, which can be serialized/deserialized
 220
 221 class CAddress():
 222     def __init__(self):
 223         self.nServices = 1
 224         self.pchReserved = b"\x00" * 10 + b"\xff" * 2
 225         self.ip = "0.0.0.0"
 226         self.port = 0
 227
 228     def deserialize(self, f):
 229         self.nServices = struct.unpack("<Q", f.read(8))[0]
 230         self.pchReserved = f.read(12)
 231         self.ip = socket.inet_ntoa(f.read(4))
 232         self.port = struct.unpack(">H", f.read(2))[0]
 233
 234     def serialize(self):
 235         r = b""
 236         r += struct.pack("<Q", self.nServices)
 237         r += self.pchReserved
 238         r += socket.inet_aton(self.ip)
 239         r += struct.pack(">H", self.port)
 240         return r
 241
 242     def __repr__(self):
 243         return "CAddress(nServices=%i ip=%s port=%i)" % (self.nServices,
 244                                                          self.ip, self.port)
 245
 246 MSG_WITNESS_FLAG = 1<<30
 247
 248 class CInv():
 249     typemap = {
 250         0: "Error",
 251         1: "TX",
 252         2: "Block",
 253         1|MSG_WITNESS_FLAG: "WitnessTx",
 254         2|MSG_WITNESS_FLAG : "WitnessBlock",
 255         4: "CompactBlock"
 256     }
 257
 258     def __init__(self, t=0, h=0):
 259         self.type = t
 260         self.hash = h
 261
 262     def deserialize(self, f):
 263         self.type = struct.unpack("<i", f.read(4))[0]
 264         self.hash = deser_uint256(f)
 265
 266     def serialize(self):
 267         r = b""
 268         r += struct.pack("<i", self.type)
 269         r += ser_uint256(self.hash)
 270         return r
 271
 272     def __repr__(self):
 273         return "CInv(type=%s hash=%064x)" \
 274             % (self.typemap[self.type], self.hash)
 275
 276
 277 class CBlockLocator():
 278     def __init__(self):
 279         self.nVersion = MY_VERSION
 280         self.vHave = []
 281
 282     def deserialize(self, f):
 283         self.nVersion = struct.unpack("<i", f.read(4))[0]
 284         self.vHave = deser_uint256_vector(f)
 285
 286     def serialize(self):
 287         r = b""
 288         r += struct.pack("<i", self.nVersion)
 289         r += ser_uint256_vector(self.vHave)
 290         return r
 291
 292     def __repr__(self):
 293         return "CBlockLocator(nVersion=%i vHave=%s)" \
 294             % (self.nVersion, repr(self.vHave))
 295
 296
 297 class COutPoint():
 298     def __init__(self, hash=0, n=0):
 299         self.hash = hash
 300         self.n = n
 301
 302     def deserialize(self, f):
 303         self.hash = deser_uint256(f)
 304         self.n = struct.unpack("<I", f.read(4))[0]
 305
 306     def serialize(self):
 307         r = b""
 308         r += ser_uint256(self.hash)
 309         r += struct.pack("<I", self.n)
 310         return r
 311
 312     def __repr__(self):
 313         return "COutPoint(hash=%064x n=%i)" % (self.hash, self.n)
 314
 315
 316 class CTxIn():
 317     def __init__(self, outpoint=None, scriptSig=b"", nSequence=0):
 318         if outpoint is None:
 319             self.prevout = COutPoint()
 320         else:
 321             self.prevout = outpoint
 322         self.scriptSig = scriptSig
 323         self.nSequence = nSequence
 324
 325     def deserialize(self, f):
 326         self.prevout = COutPoint()
 327         self.prevout.deserialize(f)
 328         self.scriptSig = deser_string(f)
 329         self.nSequence = struct.unpack("<I", f.read(4))[0]
 330
 331     def serialize(self):
 332         r = b""
 333         r += self.prevout.serialize()
 334         r += ser_string(self.scriptSig)
 335         r += struct.pack("<I", self.nSequence)
 336         return r
 337
 338     def __repr__(self):
 339         return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \
 340             % (repr(self.prevout), bytes_to_hex_str(self.scriptSig),
 341                self.nSequence)
 342
 343
 344 class CTxOut():
 345     def __init__(self, nValue=0, scriptPubKey=b""):
 346         self.nValue = nValue
 347         self.scriptPubKey = scriptPubKey
 348
 349     def deserialize(self, f):
 350         self.nValue = struct.unpack("<q", f.read(8))[0]
 351         self.scriptPubKey = deser_string(f)
 352
 353     def serialize(self):
 354         r = b""
 355         r += struct.pack("<q", self.nValue)
 356         r += ser_string(self.scriptPubKey)
 357         return r
 358
 359     def __repr__(self):
 360         return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \
 361             % (self.nValue // COIN, self.nValue % COIN,
 362                bytes_to_hex_str(self.scriptPubKey))
 363
 364
 365 class CScriptWitness():
 366     def __init__(self):
 367         # stack is a vector of strings
 368         self.stack = []
 369
 370     def __repr__(self):
 371         return "CScriptWitness(%s)" % \
 372                (",".join([bytes_to_hex_str(x) for x in self.stack]))
 373
 374     def is_null(self):
 375         if self.stack:
 376             return False
 377         return True
 378
 379
 380 class CTxInWitness():
 381     def __init__(self):
 382         self.scriptWitness = CScriptWitness()
 383
 384     def deserialize(self, f):
 385         self.scriptWitness.stack = deser_string_vector(f)
 386
 387     def serialize(self):
 388         return ser_string_vector(self.scriptWitness.stack)
 389
 390     def __repr__(self):
 391         return repr(self.scriptWitness)
 392
 393     def is_null(self):
 394         return self.scriptWitness.is_null()
 395
 396
 397 class CTxWitness():
 398     def __init__(self):
 399         self.vtxinwit = []
 400
 401     def deserialize(self, f):
 402         for i in range(len(self.vtxinwit)):
 403             self.vtxinwit[i].deserialize(f)
 404
 405     def serialize(self):
 406         r = b""
 407         # This is different than the usual vector serialization --
 408         # we omit the length of the vector, which is required to be
 409         # the same length as the transaction's vin vector.
 410         for x in self.vtxinwit:
 411             r += x.serialize()
 412         return r
 413
 414     def __repr__(self):
 415         return "CTxWitness(%s)" % \
 416                (';'.join([repr(x) for x in self.vtxinwit]))
 417
 418     def is_null(self):
 419         for x in self.vtxinwit:
 420             if not x.is_null():
 421                 return False
 422         return True
 423
 424
 425 class CTransaction():
 426     def __init__(self, tx=None):
 427         if tx is None:
 428             self.nVersion = 1
 429             self.vin = []
 430             self.vout = []
 431             self.wit = CTxWitness()
 432             self.nLockTime = 0
 433             self.sha256 = None
 434             self.hash = None
 435         else:
 436             self.nVersion = tx.nVersion
 437             self.vin = copy.deepcopy(tx.vin)
 438             self.vout = copy.deepcopy(tx.vout)
 439             self.nLockTime = tx.nLockTime
 440             self.sha256 = tx.sha256
 441             self.hash = tx.hash
 442             self.wit = copy.deepcopy(tx.wit)
 443
 444     def deserialize(self, f):
 445         self.nVersion = struct.unpack("<i", f.read(4))[0]
 446         self.vin = deser_vector(f, CTxIn)
 447         flags = 0
 448         if len(self.vin) == 0:
 449             flags = struct.unpack("<B", f.read(1))[0]
 450             # Not sure why flags can't be zero, but this
 451             # matches the implementation in bitcoind
 452             if (flags != 0):
 453                 self.vin = deser_vector(f, CTxIn)
 454                 self.vout = deser_vector(f, CTxOut)
 455         else:
 456             self.vout = deser_vector(f, CTxOut)
 457         if flags != 0:
 458             self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))]
 459             self.wit.deserialize(f)
 460         self.nLockTime = struct.unpack("<I", f.read(4))[0]
 461         self.sha256 = None
 462         self.hash = None
 463
 464     def serialize_without_witness(self):
 465         r = b""
 466         r += struct.pack("<i", self.nVersion)
 467         r += ser_vector(self.vin)
 468         r += ser_vector(self.vout)
 469         r += struct.pack("<I", self.nLockTime)
 470         return r
 471
 472     # Only serialize with witness when explicitly called for
 473     def serialize_with_witness(self):
 474         flags = 0
 475         if not self.wit.is_null():
 476             flags |= 1
 477         r = b""
 478         r += struct.pack("<i", self.nVersion)
 479         if flags:
 480             dummy = []
 481             r += ser_vector(dummy)
 482             r += struct.pack("<B", flags)
 483         r += ser_vector(self.vin)
 484         r += ser_vector(self.vout)
 485         if flags & 1:
 486             if (len(self.wit.vtxinwit) != len(self.vin)):
 487                 # vtxinwit must have the same length as vin
 488                 self.wit.vtxinwit = self.wit.vtxinwit[:len(self.vin)]
 489                 for i in range(len(self.wit.vtxinwit), len(self.vin)):
 490                     self.wit.vtxinwit.append(CTxInWitness())
 491             r += self.wit.serialize()
 492         r += struct.pack("<I", self.nLockTime)
 493         return r
 494
 495     # Regular serialization is without witness -- must explicitly
 496     # call serialize_with_witness to include witness data.
 497     def serialize(self):
 498         return self.serialize_without_witness()
 499
 500     # Recalculate the txid (transaction hash without witness)
 501     def rehash(self):
 502         self.sha256 = None
 503         self.calc_sha256()
 504
 505     # We will only cache the serialization without witness in
 506     # self.sha256 and self.hash -- those are expected to be the txid.
 507     def calc_sha256(self, with_witness=False):
 508         if with_witness:
 509             # Don't cache the result, just return it
 510             return uint256_from_str(hash256(self.serialize_with_witness()))
 511
 512         if self.sha256 is None:
 513             self.sha256 = uint256_from_str(hash256(self.serialize_without_witness()))
 514         self.hash = encode(hash256(self.serialize())[::-1], 'hex_codec').decode('ascii')
 515
 516     def is_valid(self):
 517         self.calc_sha256()
 518         for tout in self.vout:
 519             if tout.nValue < 0 or tout.nValue > 21000000 * COIN:
 520                 return False
 521         return True
 522
 523     def __repr__(self):
 524         return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \
 525             % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime)
 526
 527
 528 class CBlockHeader():
 529     def __init__(self, header=None):
 530         if header is None:
 531             self.set_null()
 532         else:
 533             self.nVersion = header.nVersion
 534             self.hashPrevBlock = header.hashPrevBlock
 535             self.hashMerkleRoot = header.hashMerkleRoot
 536             self.nTime = header.nTime
 537             self.nBits = header.nBits
 538             self.nNonce = header.nNonce
 539             self.sha256 = header.sha256
 540             self.hash = header.hash
 541             self.calc_sha256()
 542
 543     def set_null(self):
 544         self.nVersion = 1
 545         self.hashPrevBlock = 0
 546         self.hashMerkleRoot = 0
 547         self.nTime = 0
 548         self.nBits = 0
 549         self.nNonce = 0
 550         self.sha256 = None
 551         self.hash = None
 552
 553     def deserialize(self, f):
 554         self.nVersion = struct.unpack("<i", f.read(4))[0]
 555         self.hashPrevBlock = deser_uint256(f)
 556         self.hashMerkleRoot = deser_uint256(f)
 557         self.nTime = struct.unpack("<I", f.read(4))[0]
 558         self.nBits = struct.unpack("<I", f.read(4))[0]
 559         self.nNonce = struct.unpack("<I", f.read(4))[0]
 560         self.sha256 = None
 561         self.hash = None
 562
 563     def serialize(self):
 564         r = b""
 565         r += struct.pack("<i", self.nVersion)
 566         r += ser_uint256(self.hashPrevBlock)
 567         r += ser_uint256(self.hashMerkleRoot)
 568         r += struct.pack("<I", self.nTime)
 569         r += struct.pack("<I", self.nBits)
 570         r += struct.pack("<I", self.nNonce)
 571         return r
 572
 573     def calc_sha256(self):
 574         if self.sha256 is None:
 575             r = b""
 576             r += struct.pack("<i", self.nVersion)
 577             r += ser_uint256(self.hashPrevBlock)
 578             r += ser_uint256(self.hashMerkleRoot)
 579             r += struct.pack("<I", self.nTime)
 580             r += struct.pack("<I", self.nBits)
 581             r += struct.pack("<I", self.nNonce)
 582             self.sha256 = uint256_from_str(hash256(r))
 583             self.hash = encode(hash256(r)[::-1], 'hex_codec').decode('ascii')
 584
 585     def rehash(self):
 586         self.sha256 = None
 587         self.calc_sha256()
 588         return self.sha256
 589
 590     def __repr__(self):
 591         return "CBlockHeader(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x)" \
 592             % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
 593                time.ctime(self.nTime), self.nBits, self.nNonce)
 594
 595
 596 class CBlock(CBlockHeader):
 597     def __init__(self, header=None):
 598         super(CBlock, self).__init__(header)
 599         self.vtx = []
 600
 601     def deserialize(self, f):
 602         super(CBlock, self).deserialize(f)
 603         self.vtx = deser_vector(f, CTransaction)
 604
 605     def serialize(self, with_witness=False):
 606         r = b""
 607         r += super(CBlock, self).serialize()
 608         if with_witness:
 609             r += ser_vector(self.vtx, "serialize_with_witness")
 610         else:
 611             r += ser_vector(self.vtx)
 612         return r
 613
 614     # Calculate the merkle root given a vector of transaction hashes
 615     @classmethod
 616     def get_merkle_root(cls, hashes):
 617         while len(hashes) > 1:
 618             newhashes = []
 619             for i in range(0, len(hashes), 2):
 620                 i2 = min(i+1, len(hashes)-1)
 621                 newhashes.append(hash256(hashes[i] + hashes[i2]))
 622             hashes = newhashes
 623         return uint256_from_str(hashes[0])
 624
 625     def calc_merkle_root(self):
 626         hashes = []
 627         for tx in self.vtx:
 628             tx.calc_sha256()
 629             hashes.append(ser_uint256(tx.sha256))
 630         return self.get_merkle_root(hashes)
 631
 632     def calc_witness_merkle_root(self):
 633         # For witness root purposes, the hash of the
 634         # coinbase, with witness, is defined to be 0...0
 635         hashes = [ser_uint256(0)]
 636
 637         for tx in self.vtx[1:]:
 638             # Calculate the hashes with witness data
 639             hashes.append(ser_uint256(tx.calc_sha256(True)))
 640
 641         return self.get_merkle_root(hashes)
 642
 643     def is_valid(self):
 644         self.calc_sha256()
 645         target = uint256_from_compact(self.nBits)
 646         if self.sha256 > target:
 647             return False
 648         for tx in self.vtx:
 649             if not tx.is_valid():
 650                 return False
 651         if self.calc_merkle_root() != self.hashMerkleRoot:
 652             return False
 653         return True
 654
 655     def solve(self):
 656         self.rehash()
 657         target = uint256_from_compact(self.nBits)
 658         while self.sha256 > target:
 659             self.nNonce += 1
 660             self.rehash()
 661
 662     def __repr__(self):
 663         return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x vtx=%s)" \
 664             % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
 665                time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx))
 666
 667
 668 class PrefilledTransaction():
 669     def __init__(self, index=0, tx = None):
 670         self.index = index
 671         self.tx = tx
 672
 673     def deserialize(self, f):
 674         self.index = deser_compact_size(f)
 675         self.tx = CTransaction()
 676         self.tx.deserialize(f)
 677
 678     def serialize(self, with_witness=False):
 679         r = b""
 680         r += ser_compact_size(self.index)
 681         if with_witness:
 682             r += self.tx.serialize_with_witness()
 683         else:
 684             r += self.tx.serialize_without_witness()
 685         return r
 686
 687     def serialize_with_witness(self):
 688         return self.serialize(with_witness=True)
 689
 690     def __repr__(self):
 691         return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx))
 692
 693 # This is what we send on the wire, in a cmpctblock message.
 694 class P2PHeaderAndShortIDs():
 695     def __init__(self):
 696         self.header = CBlockHeader()
 697         self.nonce = 0
 698         self.shortids_length = 0
 699         self.shortids = []
 700         self.prefilled_txn_length = 0
 701         self.prefilled_txn = []
 702
 703     def deserialize(self, f):
 704         self.header.deserialize(f)
 705         self.nonce = struct.unpack("<Q", f.read(8))[0]
 706         self.shortids_length = deser_compact_size(f)
 707         for i in range(self.shortids_length):
 708             # shortids are defined to be 6 bytes in the spec, so append
 709             # two zero bytes and read it in as an 8-byte number
 710             self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0])
 711         self.prefilled_txn = deser_vector(f, PrefilledTransaction)
 712         self.prefilled_txn_length = len(self.prefilled_txn)
 713
 714     # When using version 2 compact blocks, we must serialize with_witness.
 715     def serialize(self, with_witness=False):
 716         r = b""
 717         r += self.header.serialize()
 718         r += struct.pack("<Q", self.nonce)
 719         r += ser_compact_size(self.shortids_length)
 720         for x in self.shortids:
 721             # We only want the first 6 bytes
 722             r += struct.pack("<Q", x)[0:6]
 723         if with_witness:
 724             r += ser_vector(self.prefilled_txn, "serialize_with_witness")
 725         else:
 726             r += ser_vector(self.prefilled_txn)
 727         return r
 728
 729     def __repr__(self):
 730         return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn))
 731
 732 # P2P version of the above that will use witness serialization (for compact
 733 # block version 2)
 734 class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs):
 735     def serialize(self):
 736         return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True)
 737
 738 # Calculate the BIP 152-compact blocks shortid for a given transaction hash
 739 def calculate_shortid(k0, k1, tx_hash):
 740     expected_shortid = siphash256(k0, k1, tx_hash)
 741     expected_shortid &= 0x0000ffffffffffff
 742     return expected_shortid
 743
 744 # This version gets rid of the array lengths, and reinterprets the differential
 745 # encoding into indices that can be used for lookup.
 746 class HeaderAndShortIDs():
 747     def __init__(self, p2pheaders_and_shortids = None):
 748         self.header = CBlockHeader()
 749         self.nonce = 0
 750         self.shortids = []
 751         self.prefilled_txn = []
 752         self.use_witness = False
 753
 754         if p2pheaders_and_shortids != None:
 755             self.header = p2pheaders_and_shortids.header
 756             self.nonce = p2pheaders_and_shortids.nonce
 757             self.shortids = p2pheaders_and_shortids.shortids
 758             last_index = -1
 759             for x in p2pheaders_and_shortids.prefilled_txn:
 760                 self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx))
 761                 last_index = self.prefilled_txn[-1].index
 762
 763     def to_p2p(self):
 764         if self.use_witness:
 765             ret = P2PHeaderAndShortWitnessIDs()
 766         else:
 767             ret = P2PHeaderAndShortIDs()
 768         ret.header = self.header
 769         ret.nonce = self.nonce
 770         ret.shortids_length = len(self.shortids)
 771         ret.shortids = self.shortids
 772         ret.prefilled_txn_length = len(self.prefilled_txn)
 773         ret.prefilled_txn = []
 774         last_index = -1
 775         for x in self.prefilled_txn:
 776             ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx))
 777             last_index = x.index
 778         return ret
 779
 780     def get_siphash_keys(self):
 781         header_nonce = self.header.serialize()
 782         header_nonce += struct.pack("<Q", self.nonce)
 783         hash_header_nonce_as_str = sha256(header_nonce)
 784         key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0]
 785         key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0]
 786         return [ key0, key1 ]
 787
 788     # Version 2 compact blocks use wtxid in shortids (rather than txid)
 789     def initialize_from_block(self, block, nonce=0, prefill_list = [0], use_witness = False):
 790         self.header = CBlockHeader(block)
 791         self.nonce = nonce
 792         self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ]
 793         self.shortids = []
 794         self.use_witness = use_witness
 795         [k0, k1] = self.get_siphash_keys()
 796         for i in range(len(block.vtx)):
 797             if i not in prefill_list:
 798                 tx_hash = block.vtx[i].sha256
 799                 if use_witness:
 800                     tx_hash = block.vtx[i].calc_sha256(with_witness=True)
 801                 self.shortids.append(calculate_shortid(k0, k1, tx_hash))
 802
 803     def __repr__(self):
 804         return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn))
 805
 806
 807 class BlockTransactionsRequest():
 808
 809     def __init__(self, blockhash=0, indexes = None):
 810         self.blockhash = blockhash
 811         self.indexes = indexes if indexes != None else []
 812
 813     def deserialize(self, f):
 814         self.blockhash = deser_uint256(f)
 815         indexes_length = deser_compact_size(f)
 816         for i in range(indexes_length):
 817             self.indexes.append(deser_compact_size(f))
 818
 819     def serialize(self):
 820         r = b""
 821         r += ser_uint256(self.blockhash)
 822         r += ser_compact_size(len(self.indexes))
 823         for x in self.indexes:
 824             r += ser_compact_size(x)
 825         return r
 826
 827     # helper to set the differentially encoded indexes from absolute ones
 828     def from_absolute(self, absolute_indexes):
 829         self.indexes = []
 830         last_index = -1
 831         for x in absolute_indexes:
 832             self.indexes.append(x-last_index-1)
 833             last_index = x
 834
 835     def to_absolute(self):
 836         absolute_indexes = []
 837         last_index = -1
 838         for x in self.indexes:
 839             absolute_indexes.append(x+last_index+1)
 840             last_index = absolute_indexes[-1]
 841         return absolute_indexes
 842
 843     def __repr__(self):
 844         return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes))
 845
 846
 847 class BlockTransactions():
 848
 849     def __init__(self, blockhash=0, transactions = None):
 850         self.blockhash = blockhash
 851         self.transactions = transactions if transactions != None else []
 852
 853     def deserialize(self, f):
 854         self.blockhash = deser_uint256(f)
 855         self.transactions = deser_vector(f, CTransaction)
 856
 857     def serialize(self, with_witness=False):
 858         r = b""
 859         r += ser_uint256(self.blockhash)
 860         if with_witness:
 861             r += ser_vector(self.transactions, "serialize_with_witness")
 862         else:
 863             r += ser_vector(self.transactions)
 864         return r
 865
 866     def __repr__(self):
 867         return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions))
 868
 869
 870 # Objects that correspond to messages on the wire
 871 class msg_version():
 872     command = b"version"
 873
 874     def __init__(self):
 875         self.nVersion = MY_VERSION
 876         self.nServices = NODE_NETWORK | NODE_WITNESS
 877         self.nTime = int(time.time())
 878         self.addrTo = CAddress()
 879         self.addrFrom = CAddress()
 880         self.nNonce = random.getrandbits(64)
 881         self.strSubVer = MY_SUBVERSION
 882         self.nStartingHeight = -1
 883         self.nRelay = MY_RELAY
 884
 885     def deserialize(self, f):
 886         self.nVersion = struct.unpack("<i", f.read(4))[0]
 887         if self.nVersion == 10300:
 888             self.nVersion = 300
 889         self.nServices = struct.unpack("<Q", f.read(8))[0]
 890         self.nTime = struct.unpack("<q", f.read(8))[0]
 891         self.addrTo = CAddress()
 892         self.addrTo.deserialize(f)
 893
 894         if self.nVersion >= 106:
 895             self.addrFrom = CAddress()
 896             self.addrFrom.deserialize(f)
 897             self.nNonce = struct.unpack("<Q", f.read(8))[0]
 898             self.strSubVer = deser_string(f)
 899         else:
 900             self.addrFrom = None
 901             self.nNonce = None
 902             self.strSubVer = None
 903             self.nStartingHeight = None
 904
 905         if self.nVersion >= 209:
 906             self.nStartingHeight = struct.unpack("<i", f.read(4))[0]
 907         else:
 908             self.nStartingHeight = None
 909
 910         if self.nVersion >= 70001:
 911             # Relay field is optional for version 70001 onwards
 912             try:
 913                 self.nRelay = struct.unpack("<b", f.read(1))[0]
 914             except:
 915                 self.nRelay = 0
 916         else:
 917             self.nRelay = 0
 918
 919     def serialize(self):
 920         r = b""
 921         r += struct.pack("<i", self.nVersion)
 922         r += struct.pack("<Q", self.nServices)
 923         r += struct.pack("<q", self.nTime)
 924         r += self.addrTo.serialize()
 925         r += self.addrFrom.serialize()
 926         r += struct.pack("<Q", self.nNonce)
 927         r += ser_string(self.strSubVer)
 928         r += struct.pack("<i", self.nStartingHeight)
 929         r += struct.pack("<b", self.nRelay)
 930         return r
 931
 932     def __repr__(self):
 933         return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i nRelay=%i)' \
 934             % (self.nVersion, self.nServices, time.ctime(self.nTime),
 935                repr(self.addrTo), repr(self.addrFrom), self.nNonce,
 936                self.strSubVer, self.nStartingHeight, self.nRelay)
 937
 938
 939 class msg_verack():
 940     command = b"verack"
 941
 942     def __init__(self):
 943         pass
 944
 945     def deserialize(self, f):
 946         pass
 947
 948     def serialize(self):
 949         return b""
 950
 951     def __repr__(self):
 952         return "msg_verack()"
 953
 954
 955 class msg_addr():
 956     command = b"addr"
 957
 958     def __init__(self):
 959         self.addrs = []
 960
 961     def deserialize(self, f):
 962         self.addrs = deser_vector(f, CAddress)
 963
 964     def serialize(self):
 965         return ser_vector(self.addrs)
 966
 967     def __repr__(self):
 968         return "msg_addr(addrs=%s)" % (repr(self.addrs))
 969
 970
 971 class msg_inv():
 972     command = b"inv"
 973
 974     def __init__(self, inv=None):
 975         if inv is None:
 976             self.inv = []
 977         else:
 978             self.inv = inv
 979
 980     def deserialize(self, f):
 981         self.inv = deser_vector(f, CInv)
 982
 983     def serialize(self):
 984         return ser_vector(self.inv)
 985
 986     def __repr__(self):
 987         return "msg_inv(inv=%s)" % (repr(self.inv))
 988
 989
 990 class msg_getdata():
 991     command = b"getdata"
 992
 993     def __init__(self, inv=None):
 994         self.inv = inv if inv != None else []
 995
 996     def deserialize(self, f):
 997         self.inv = deser_vector(f, CInv)
 998
 999     def serialize(self):
1000         return ser_vector(self.inv)
1001
1002     def __repr__(self):
1003         return "msg_getdata(inv=%s)" % (repr(self.inv))
1004
1005
1006 class msg_getblocks():
1007     command = b"getblocks"
1008
1009     def __init__(self):
1010         self.locator = CBlockLocator()
1011         self.hashstop = 0
1012
1013     def deserialize(self, f):
1014         self.locator = CBlockLocator()
1015         self.locator.deserialize(f)
1016         self.hashstop = deser_uint256(f)
1017
1018     def serialize(self):
1019         r = b""
1020         r += self.locator.serialize()
1021         r += ser_uint256(self.hashstop)
1022         return r
1023
1024     def __repr__(self):
1025         return "msg_getblocks(locator=%s hashstop=%064x)" \
1026             % (repr(self.locator), self.hashstop)
1027
1028
1029 class msg_tx():
1030     command = b"tx"
1031
1032     def __init__(self, tx=CTransaction()):
1033         self.tx = tx
1034
1035     def deserialize(self, f):
1036         self.tx.deserialize(f)
1037
1038     def serialize(self):
1039         return self.tx.serialize_without_witness()
1040
1041     def __repr__(self):
1042         return "msg_tx(tx=%s)" % (repr(self.tx))
1043
1044 class msg_witness_tx(msg_tx):
1045
1046     def serialize(self):
1047         return self.tx.serialize_with_witness()
1048
1049
1050 class msg_block():
1051     command = b"block"
1052
1053     def __init__(self, block=None):
1054         if block is None:
1055             self.block = CBlock()
1056         else:
1057             self.block = block
1058
1059     def deserialize(self, f):
1060         self.block.deserialize(f)
1061
1062     def serialize(self):
1063         return self.block.serialize()
1064
1065     def __repr__(self):
1066         return "msg_block(block=%s)" % (repr(self.block))
1067
1068 # for cases where a user needs tighter control over what is sent over the wire
1069 # note that the user must supply the name of the command, and the data
1070 class msg_generic():
1071     def __init__(self, command, data=None):
1072         self.command = command
1073         self.data = data
1074
1075     def serialize(self):
1076         return self.data
1077
1078     def __repr__(self):
1079         return "msg_generic()"
1080
1081 class msg_witness_block(msg_block):
1082
1083     def serialize(self):
1084         r = self.block.serialize(with_witness=True)
1085         return r
1086
1087 class msg_getaddr():
1088     command = b"getaddr"
1089
1090     def __init__(self):
1091         pass
1092
1093     def deserialize(self, f):
1094         pass
1095
1096     def serialize(self):
1097         return b""
1098
1099     def __repr__(self):
1100         return "msg_getaddr()"
1101
1102
1103 class msg_ping():
1104     command = b"ping"
1105
1106     def __init__(self, nonce=0):
1107         self.nonce = nonce
1108
1109     def deserialize(self, f):
1110         self.nonce = struct.unpack("<Q", f.read(8))[0]
1111
1112     def serialize(self):
1113         r = b""
1114         r += struct.pack("<Q", self.nonce)
1115         return r
1116
1117     def __repr__(self):
1118         return "msg_ping(nonce=%08x)" % self.nonce
1119
1120
1121 class msg_pong():
1122     command = b"pong"
1123
1124     def __init__(self, nonce=0):
1125         self.nonce = nonce
1126
1127     def deserialize(self, f):
1128         self.nonce = struct.unpack("<Q", f.read(8))[0]
1129
1130     def serialize(self):
1131         r = b""
1132         r += struct.pack("<Q", self.nonce)
1133         return r
1134
1135     def __repr__(self):
1136         return "msg_pong(nonce=%08x)" % self.nonce
1137
1138
1139 class msg_mempool():
1140     command = b"mempool"
1141
1142     def __init__(self):
1143         pass
1144
1145     def deserialize(self, f):
1146         pass
1147
1148     def serialize(self):
1149         return b""
1150
1151     def __repr__(self):
1152         return "msg_mempool()"
1153
1154 class msg_sendheaders():
1155     command = b"sendheaders"
1156
1157     def __init__(self):
1158         pass
1159
1160     def deserialize(self, f):
1161         pass
1162
1163     def serialize(self):
1164         return b""
1165
1166     def __repr__(self):
1167         return "msg_sendheaders()"
1168
1169
1170 # getheaders message has
1171 # number of entries
1172 # vector of hashes
1173 # hash_stop (hash of last desired block header, 0 to get as many as possible)
1174 class msg_getheaders():
1175     command = b"getheaders"
1176
1177     def __init__(self):
1178         self.locator = CBlockLocator()
1179         self.hashstop = 0
1180
1181     def deserialize(self, f):
1182         self.locator = CBlockLocator()
1183         self.locator.deserialize(f)
1184         self.hashstop = deser_uint256(f)
1185
1186     def serialize(self):
1187         r = b""
1188         r += self.locator.serialize()
1189         r += ser_uint256(self.hashstop)
1190         return r
1191
1192     def __repr__(self):
1193         return "msg_getheaders(locator=%s, stop=%064x)" \
1194             % (repr(self.locator), self.hashstop)
1195
1196
1197 # headers message has
1198 # <count> <vector of block headers>
1199 class msg_headers():
1200     command = b"headers"
1201
1202     def __init__(self, headers=None):
1203         self.headers = headers if headers is not None else []
1204
1205     def deserialize(self, f):
1206         # comment in bitcoind indicates these should be deserialized as blocks
1207         blocks = deser_vector(f, CBlock)
1208         for x in blocks:
1209             self.headers.append(CBlockHeader(x))
1210
1211     def serialize(self):
1212         blocks = [CBlock(x) for x in self.headers]
1213         return ser_vector(blocks)
1214
1215     def __repr__(self):
1216         return "msg_headers(headers=%s)" % repr(self.headers)
1217
1218
1219 class msg_reject():
1220     command = b"reject"
1221     REJECT_MALFORMED = 1
1222
1223     def __init__(self):
1224         self.message = b""
1225         self.code = 0
1226         self.reason = b""
1227         self.data = 0
1228
1229     def deserialize(self, f):
1230         self.message = deser_string(f)
1231         self.code = struct.unpack("<B", f.read(1))[0]
1232         self.reason = deser_string(f)
1233         if (self.code != self.REJECT_MALFORMED and
1234                 (self.message == b"block" or self.message == b"tx")):
1235             self.data = deser_uint256(f)
1236
1237     def serialize(self):
1238         r = ser_string(self.message)
1239         r += struct.pack("<B", self.code)
1240         r += ser_string(self.reason)
1241         if (self.code != self.REJECT_MALFORMED and
1242                 (self.message == b"block" or self.message == b"tx")):
1243             r += ser_uint256(self.data)
1244         return r
1245
1246     def __repr__(self):
1247         return "msg_reject: %s %d %s [%064x]" \
1248             % (self.message, self.code, self.reason, self.data)
1249
1250 class msg_feefilter():
1251     command = b"feefilter"
1252
1253     def __init__(self, feerate=0):
1254         self.feerate = feerate
1255
1256     def deserialize(self, f):
1257         self.feerate = struct.unpack("<Q", f.read(8))[0]
1258
1259     def serialize(self):
1260         r = b""
1261         r += struct.pack("<Q", self.feerate)
1262         return r
1263
1264     def __repr__(self):
1265         return "msg_feefilter(feerate=%08x)" % self.feerate
1266
1267 class msg_sendcmpct():
1268     command = b"sendcmpct"
1269
1270     def __init__(self):
1271         self.announce = False
1272         self.version = 1
1273
1274     def deserialize(self, f):
1275         self.announce = struct.unpack("<?", f.read(1))[0]
1276         self.version = struct.unpack("<Q", f.read(8))[0]
1277
1278     def serialize(self):
1279         r = b""
1280         r += struct.pack("<?", self.announce)
1281         r += struct.pack("<Q", self.version)
1282         return r
1283
1284     def __repr__(self):
1285         return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version)
1286
1287 class msg_cmpctblock():
1288     command = b"cmpctblock"
1289
1290     def __init__(self, header_and_shortids = None):
1291         self.header_and_shortids = header_and_shortids
1292
1293     def deserialize(self, f):
1294         self.header_and_shortids = P2PHeaderAndShortIDs()
1295         self.header_and_shortids.deserialize(f)
1296
1297     def serialize(self):
1298         r = b""
1299         r += self.header_and_shortids.serialize()
1300         return r
1301
1302     def __repr__(self):
1303         return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids)
1304
1305 class msg_getblocktxn():
1306     command = b"getblocktxn"
1307
1308     def __init__(self):
1309         self.block_txn_request = None
1310
1311     def deserialize(self, f):
1312         self.block_txn_request = BlockTransactionsRequest()
1313         self.block_txn_request.deserialize(f)
1314
1315     def serialize(self):
1316         r = b""
1317         r += self.block_txn_request.serialize()
1318         return r
1319
1320     def __repr__(self):
1321         return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request))
1322
1323 class msg_blocktxn():
1324     command = b"blocktxn"
1325
1326     def __init__(self):
1327         self.block_transactions = BlockTransactions()
1328
1329     def deserialize(self, f):
1330         self.block_transactions.deserialize(f)
1331
1332     def serialize(self):
1333         r = b""
1334         r += self.block_transactions.serialize()
1335         return r
1336
1337     def __repr__(self):
1338         return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions))
1339
1340 class msg_witness_blocktxn(msg_blocktxn):
1341     def serialize(self):
1342         r = b""
1343         r += self.block_transactions.serialize(with_witness=True)
1344         return r
1345
1346 class NodeConnCB():
1347     """Callback and helper functions for P2P connection to a bitcoind node.
1348
1349     Individual testcases should subclass this and override the on_* methods
1350     if they want to alter message handling behaviour."""
1351     def __init__(self):
1352         # Track whether we have a P2P connection open to the node
1353         self.connected = False
1354         self.connection = None
1355
1356         # Track number of messages of each type received and the most recent
1357         # message of each type
1358         self.message_count = defaultdict(int)
1359         self.last_message = {}
1360
1361         # A count of the number of ping messages we've sent to the node
1362         self.ping_counter = 1
1363
1364     # Message receiving methods
1365
1366     def deliver(self, conn, message):
1367         """Receive message and dispatch message to appropriate callback.
1368
1369         We keep a count of how many of each message type has been received
1370         and the most recent message of each type."""
1371         with mininode_lock:
1372             try:
1373                 command = message.command.decode('ascii')
1374                 self.message_count[command] += 1
1375                 self.last_message[command] = message
1376                 getattr(self, 'on_' + command)(conn, message)
1377             except:
1378                 print("ERROR delivering %s (%s)" % (repr(message),
1379                                                     sys.exc_info()[0]))
1380                 raise
1381
1382     # Callback methods. Can be overridden by subclasses in individual test
1383     # cases to provide custom message handling behaviour.
1384
1385     def on_open(self, conn):
1386         self.connected = True
1387
1388     def on_close(self, conn):
1389         self.connected = False
1390         self.connection = None
1391
1392     def on_addr(self, conn, message): pass
1393     def on_block(self, conn, message): pass
1394     def on_blocktxn(self, conn, message): pass
1395     def on_cmpctblock(self, conn, message): pass
1396     def on_feefilter(self, conn, message): pass
1397     def on_getaddr(self, conn, message): pass
1398     def on_getblocks(self, conn, message): pass
1399     def on_getblocktxn(self, conn, message): pass
1400     def on_getdata(self, conn, message): pass
1401     def on_getheaders(self, conn, message): pass
1402     def on_headers(self, conn, message): pass
1403     def on_mempool(self, conn): pass
1404     def on_pong(self, conn, message): pass
1405     def on_reject(self, conn, message): pass
1406     def on_sendcmpct(self, conn, message): pass
1407     def on_sendheaders(self, conn, message): pass
1408     def on_tx(self, conn, message): pass
1409
1410     def on_inv(self, conn, message):
1411         want = msg_getdata()
1412         for i in message.inv:
1413             if i.type != 0:
1414                 want.inv.append(i)
1415         if len(want.inv):
1416             conn.send_message(want)
1417
1418     def on_ping(self, conn, message):
1419         conn.send_message(msg_pong(message.nonce))
1420
1421     def on_verack(self, conn, message):
1422         conn.ver_recv = conn.ver_send
1423         self.verack_received = True
1424
1425     def on_version(self, conn, message):
1426         if message.nVersion >= 209:
1427             conn.send_message(msg_verack())
1428         conn.ver_send = min(MY_VERSION, message.nVersion)
1429         if message.nVersion < 209:
1430             conn.ver_recv = conn.ver_send
1431         conn.nServices = message.nServices
1432
1433     # Connection helper methods
1434
1435     def add_connection(self, conn):
1436         self.connection = conn
1437
1438     def wait_for_disconnect(self, timeout=60):
1439         test_function = lambda: not self.connected
1440         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1441
1442     # Message receiving helper methods
1443
1444     def wait_for_block(self, blockhash, timeout=60):
1445         test_function = lambda: self.last_message.get("block") and self.last_message["block"].block.rehash() == blockhash
1446         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1447
1448     def wait_for_getdata(self, timeout=60):
1449         test_function = lambda: self.last_message.get("getdata")
1450         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1451
1452     def wait_for_getheaders(self, timeout=60):
1453         test_function = lambda: self.last_message.get("getheaders")
1454         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1455
1456     def wait_for_inv(self, expected_inv, timeout=60):
1457         """Waits for an INV message and checks that the first inv object in the message was as expected."""
1458         if len(expected_inv) > 1:
1459             raise NotImplementedError("wait_for_inv() will only verify the first inv object")
1460         test_function = lambda: self.last_message.get("inv") and \
1461                                 self.last_message["inv"].inv[0].type == expected_inv[0].type and \
1462                                 self.last_message["inv"].inv[0].hash == expected_inv[0].hash
1463         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1464
1465     def wait_for_verack(self, timeout=60):
1466         test_function = lambda: self.message_count["verack"]
1467         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1468
1469     # Message sending helper functions
1470
1471     def send_message(self, message):
1472         if self.connection:
1473             self.connection.send_message(message)
1474         else:
1475             logger.error("Cannot send message. No connection to node!")
1476
1477     def send_and_ping(self, message):
1478         self.send_message(message)
1479         self.sync_with_ping()
1480
1481     # Sync up with the node
1482     def sync_with_ping(self, timeout=60):
1483         self.send_message(msg_ping(nonce=self.ping_counter))
1484         test_function = lambda: self.last_message.get("pong") and self.last_message["pong"].nonce == self.ping_counter
1485         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1486         self.ping_counter += 1
1487
1488 class NodeConn(asyncore.dispatcher):
1489     """The actual NodeConn class
1490
1491     This class provides an interface for a p2p connection to a specified node."""
1492     messagemap = {
1493         b"version": msg_version,
1494         b"verack": msg_verack,
1495         b"addr": msg_addr,
1496         b"inv": msg_inv,
1497         b"getdata": msg_getdata,
1498         b"getblocks": msg_getblocks,
1499         b"tx": msg_tx,
1500         b"block": msg_block,
1501         b"getaddr": msg_getaddr,
1502         b"ping": msg_ping,
1503         b"pong": msg_pong,
1504         b"headers": msg_headers,
1505         b"getheaders": msg_getheaders,
1506         b"reject": msg_reject,
1507         b"mempool": msg_mempool,
1508         b"feefilter": msg_feefilter,
1509         b"sendheaders": msg_sendheaders,
1510         b"sendcmpct": msg_sendcmpct,
1511         b"cmpctblock": msg_cmpctblock,
1512         b"getblocktxn": msg_getblocktxn,
1513         b"blocktxn": msg_blocktxn
1514     }
1515     MAGIC_BYTES = {
1516         "mainnet": b"\xf9\xbe\xb4\xd9",   # mainnet
1517         "testnet3": b"\x0b\x11\x09\x07",  # testnet3
1518         "regtest": b"\xfa\xbf\xb5\xda",   # regtest
1519     }
1520
1521     def __init__(self, dstaddr, dstport, rpc, callback, net="regtest", services=NODE_NETWORK|NODE_WITNESS, send_version=True):
1522         asyncore.dispatcher.__init__(self, map=mininode_socket_map)
1523         self.dstaddr = dstaddr
1524         self.dstport = dstport
1525         self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
1526         self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
1527         self.sendbuf = b""
1528         self.recvbuf = b""
1529         self.ver_send = 209
1530         self.ver_recv = 209
1531         self.last_sent = 0
1532         self.state = "connecting"
1533         self.network = net
1534         self.cb = callback
1535         self.disconnect = False
1536         self.nServices = 0
1537
1538         if send_version:
1539             # stuff version msg into sendbuf
1540             vt = msg_version()
1541             vt.nServices = services
1542             vt.addrTo.ip = self.dstaddr
1543             vt.addrTo.port = self.dstport
1544             vt.addrFrom.ip = "0.0.0.0"
1545             vt.addrFrom.port = 0
1546             self.send_message(vt, True)
1547
1548         logger.info('Connecting to Bitcoin Node: %s:%d' % (self.dstaddr, self.dstport))
1549
1550         try:
1551             self.connect((dstaddr, dstport))
1552         except:
1553             self.handle_close()
1554         self.rpc = rpc
1555
1556     def handle_connect(self):
1557         if self.state != "connected":
1558             logger.debug("Connected & Listening: %s:%d" % (self.dstaddr, self.dstport))
1559             self.state = "connected"
1560             self.cb.on_open(self)
1561
1562     def handle_close(self):
1563         logger.debug("Closing connection to: %s:%d" % (self.dstaddr, self.dstport))
1564         self.state = "closed"
1565         self.recvbuf = b""
1566         self.sendbuf = b""
1567         try:
1568             self.close()
1569         except:
1570             pass
1571         self.cb.on_close(self)
1572
1573     def handle_read(self):
1574         t = self.recv(8192)
1575         if len(t) > 0:
1576             self.recvbuf += t
1577             self.got_data()
1578
1579     def readable(self):
1580         return True
1581
1582     def writable(self):
1583         with mininode_lock:
1584             pre_connection = self.state == "connecting"
1585             length = len(self.sendbuf)
1586         return (length > 0 or pre_connection)
1587
1588     def handle_write(self):
1589         with mininode_lock:
1590             # asyncore does not expose socket connection, only the first read/write
1591             # event, thus we must check connection manually here to know when we
1592             # actually connect
1593             if self.state == "connecting":
1594                 self.handle_connect()
1595             if not self.writable():
1596                 return
1597
1598             try:
1599                 sent = self.send(self.sendbuf)
1600             except:
1601                 self.handle_close()
1602                 return
1603             self.sendbuf = self.sendbuf[sent:]
1604
1605     def got_data(self):
1606         try:
1607             while True:
1608                 if len(self.recvbuf) < 4:
1609                     return
1610                 if self.recvbuf[:4] != self.MAGIC_BYTES[self.network]:
1611                     raise ValueError("got garbage %s" % repr(self.recvbuf))
1612                 if self.ver_recv < 209:
1613                     if len(self.recvbuf) < 4 + 12 + 4:
1614                         return
1615                     command = self.recvbuf[4:4+12].split(b"\x00", 1)[0]
1616                     msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0]
1617                     checksum = None
1618                     if len(self.recvbuf) < 4 + 12 + 4 + msglen:
1619                         return
1620                     msg = self.recvbuf[4+12+4:4+12+4+msglen]
1621                     self.recvbuf = self.recvbuf[4+12+4+msglen:]
1622                 else:
1623                     if len(self.recvbuf) < 4 + 12 + 4 + 4:
1624                         return
1625                     command = self.recvbuf[4:4+12].split(b"\x00", 1)[0]
1626                     msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0]
1627                     checksum = self.recvbuf[4+12+4:4+12+4+4]
1628                     if len(self.recvbuf) < 4 + 12 + 4 + 4 + msglen:
1629                         return
1630                     msg = self.recvbuf[4+12+4+4:4+12+4+4+msglen]
1631                     th = sha256(msg)
1632                     h = sha256(th)
1633                     if checksum != h[:4]:
1634                         raise ValueError("got bad checksum " + repr(self.recvbuf))
1635                     self.recvbuf = self.recvbuf[4+12+4+4+msglen:]
1636                 if command in self.messagemap:
1637                     f = BytesIO(msg)
1638                     t = self.messagemap[command]()
1639                     t.deserialize(f)
1640                     self.got_message(t)
1641                 else:
1642                     logger.warning("Received unknown command from %s:%d: '%s' %s" % (self.dstaddr, self.dstport, command, repr(msg)))
1643                     raise ValueError("Unknown command: '%s'" % (command))
1644         except Exception as e:
1645             logger.exception('got_data:', repr(e))
1646             raise
1647
1648     def send_message(self, message, pushbuf=False):
1649         if self.state != "connected" and not pushbuf:
1650             raise IOError('Not connected, no pushbuf')
1651         self._log_message("send", message)
1652         command = message.command
1653         data = message.serialize()
1654         tmsg = self.MAGIC_BYTES[self.network]
1655         tmsg += command
1656         tmsg += b"\x00" * (12 - len(command))
1657         tmsg += struct.pack("<I", len(data))
1658         if self.ver_send >= 209:
1659             th = sha256(data)
1660             h = sha256(th)
1661             tmsg += h[:4]
1662         tmsg += data
1663         with mininode_lock:
1664             if (len(self.sendbuf) == 0 and not pushbuf):
1665                 try:
1666                     sent = self.send(tmsg)
1667                     self.sendbuf = tmsg[sent:]
1668                 except BlockingIOError:
1669                     self.sendbuf = tmsg
1670             else:
1671                 self.sendbuf += tmsg
1672             self.last_sent = time.time()
1673
1674     def got_message(self, message):
1675         if self.last_sent + 30 * 60 < time.time():
1676             self.send_message(self.messagemap[b'ping']())
1677         self._log_message("receive", message)
1678         self.cb.deliver(self, message)
1679
1680     def _log_message(self, direction, msg):
1681         if direction == "send":
1682             log_message = "Send message to "
1683         elif direction == "receive":
1684             log_message = "Received message from "
1685         log_message += "%s:%d: %s" % (self.dstaddr, self.dstport, repr(msg)[:500])
1686         if len(log_message) > 500:
1687             log_message += "... (msg truncated)"
1688         logger.debug(log_message)
1689
1690     def disconnect_node(self):
1691         self.disconnect = True
1692
1693
1694 class NetworkThread(Thread):
1695     def run(self):
1696         while mininode_socket_map:
1697             # We check for whether to disconnect outside of the asyncore
1698             # loop to workaround the behavior of asyncore when using
1699             # select
1700             disconnected = []
1701             for fd, obj in mininode_socket_map.items():
1702                 if obj.disconnect:
1703                     disconnected.append(obj)
1704             [ obj.handle_close() for obj in disconnected ]
1705             asyncore.loop(0.1, use_poll=True, map=mininode_socket_map, count=1)
1706         logger.debug("Network thread closing")