test/functional/test_framework/mininode.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2010 ArtForz -- public domain half-a-node
   3 # Copyright (c) 2012 Jeff Garzik
   4 # Copyright (c) 2010-2016 The Bitcoin Core developers
   5 # Distributed under the MIT software license, see the accompanying
   6 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   7 """Bitcoin P2P network half-a-node.
   8
   9 This python code was modified from ArtForz' public domain  half-a-node, as
  10 found in the mini-node branch of http://github.com/jgarzik/pynode.
  11
  12 NodeConn: an object which manages p2p connectivity to a bitcoin node
  13 NodeConnCB: a base class that describes the interface for receiving
  14             callbacks with network messages from a NodeConn
  15 CBlock, CTransaction, CBlockHeader, CTxIn, CTxOut, etc....:
  16     data structures that should map to corresponding structures in
  17     bitcoin/primitives
  18 msg_block, msg_tx, msg_headers, etc.:
  19     data structures that represent network messages
  20 ser_*, deser_*: functions that handle serialization/deserialization
  21 """
  22
  23 import struct
  24 import socket
  25 import asyncore
  26 import time
  27 import sys
  28 import random
  29 from .util import hex_str_to_bytes, bytes_to_hex_str
  30 from io import BytesIO
  31 from codecs import encode
  32 import hashlib
  33 from threading import RLock
  34 from threading import Thread
  35 import logging
  36 import copy
  37 from test_framework.siphash import siphash256
  38
  39 BIP0031_VERSION = 60000
  40 MY_VERSION = 70014  # past bip-31 for ping/pong
  41 MY_SUBVERSION = b"/python-mininode-tester:0.0.3/"
  42 MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37)
  43
  44 MAX_INV_SZ = 50000
  45 MAX_BLOCK_BASE_SIZE = 1000000
  46
  47 COIN = 100000000 # 1 btc in satoshis
  48
  49 NODE_NETWORK = (1 << 0)
  50 NODE_GETUTXO = (1 << 1)
  51 NODE_BLOOM = (1 << 2)
  52 NODE_WITNESS = (1 << 3)
  53
  54 logger = logging.getLogger("TestFramework.mininode")
  55
  56 # Keep our own socket map for asyncore, so that we can track disconnects
  57 # ourselves (to workaround an issue with closing an asyncore socket when
  58 # using select)
  59 mininode_socket_map = dict()
  60
  61 # One lock for synchronizing all data access between the networking thread (see
  62 # NetworkThread below) and the thread running the test logic.  For simplicity,
  63 # NodeConn acquires this lock whenever delivering a message to to a NodeConnCB,
  64 # and whenever adding anything to the send buffer (in send_message()).  This
  65 # lock should be acquired in the thread running the test logic to synchronize
  66 # access to any data shared with the NodeConnCB or NodeConn.
  67 mininode_lock = RLock()
  68
  69 # Serialization/deserialization tools
  70 def sha256(s):
  71     return hashlib.new('sha256', s).digest()
  72
  73 def ripemd160(s):
  74     return hashlib.new('ripemd160', s).digest()
  75
  76 def hash256(s):
  77     return sha256(sha256(s))
  78
  79 def ser_compact_size(l):
  80     r = b""
  81     if l < 253:
  82         r = struct.pack("B", l)
  83     elif l < 0x10000:
  84         r = struct.pack("<BH", 253, l)
  85     elif l < 0x100000000:
  86         r = struct.pack("<BI", 254, l)
  87     else:
  88         r = struct.pack("<BQ", 255, l)
  89     return r
  90
  91 def deser_compact_size(f):
  92     nit = struct.unpack("<B", f.read(1))[0]
  93     if nit == 253:
  94         nit = struct.unpack("<H", f.read(2))[0]
  95     elif nit == 254:
  96         nit = struct.unpack("<I", f.read(4))[0]
  97     elif nit == 255:
  98         nit = struct.unpack("<Q", f.read(8))[0]
  99     return nit
 100
 101 def deser_string(f):
 102     nit = deser_compact_size(f)
 103     return f.read(nit)
 104
 105 def ser_string(s):
 106     return ser_compact_size(len(s)) + s
 107
 108 def deser_uint256(f):
 109     r = 0
 110     for i in range(8):
 111         t = struct.unpack("<I", f.read(4))[0]
 112         r += t << (i * 32)
 113     return r
 114
 115
 116 def ser_uint256(u):
 117     rs = b""
 118     for i in range(8):
 119         rs += struct.pack("<I", u & 0xFFFFFFFF)
 120         u >>= 32
 121     return rs
 122
 123
 124 def uint256_from_str(s):
 125     r = 0
 126     t = struct.unpack("<IIIIIIII", s[:32])
 127     for i in range(8):
 128         r += t[i] << (i * 32)
 129     return r
 130
 131
 132 def uint256_from_compact(c):
 133     nbytes = (c >> 24) & 0xFF
 134     v = (c & 0xFFFFFF) << (8 * (nbytes - 3))
 135     return v
 136
 137
 138 def deser_vector(f, c):
 139     nit = deser_compact_size(f)
 140     r = []
 141     for i in range(nit):
 142         t = c()
 143         t.deserialize(f)
 144         r.append(t)
 145     return r
 146
 147
 148 # ser_function_name: Allow for an alternate serialization function on the
 149 # entries in the vector (we use this for serializing the vector of transactions
 150 # for a witness block).
 151 def ser_vector(l, ser_function_name=None):
 152     r = ser_compact_size(len(l))
 153     for i in l:
 154         if ser_function_name:
 155             r += getattr(i, ser_function_name)()
 156         else:
 157             r += i.serialize()
 158     return r
 159
 160
 161 def deser_uint256_vector(f):
 162     nit = deser_compact_size(f)
 163     r = []
 164     for i in range(nit):
 165         t = deser_uint256(f)
 166         r.append(t)
 167     return r
 168
 169
 170 def ser_uint256_vector(l):
 171     r = ser_compact_size(len(l))
 172     for i in l:
 173         r += ser_uint256(i)
 174     return r
 175
 176
 177 def deser_string_vector(f):
 178     nit = deser_compact_size(f)
 179     r = []
 180     for i in range(nit):
 181         t = deser_string(f)
 182         r.append(t)
 183     return r
 184
 185
 186 def ser_string_vector(l):
 187     r = ser_compact_size(len(l))
 188     for sv in l:
 189         r += ser_string(sv)
 190     return r
 191
 192
 193 def deser_int_vector(f):
 194     nit = deser_compact_size(f)
 195     r = []
 196     for i in range(nit):
 197         t = struct.unpack("<i", f.read(4))[0]
 198         r.append(t)
 199     return r
 200
 201
 202 def ser_int_vector(l):
 203     r = ser_compact_size(len(l))
 204     for i in l:
 205         r += struct.pack("<i", i)
 206     return r
 207
 208 # Deserialize from a hex string representation (eg from RPC)
 209 def FromHex(obj, hex_string):
 210     obj.deserialize(BytesIO(hex_str_to_bytes(hex_string)))
 211     return obj
 212
 213 # Convert a binary-serializable object to hex (eg for submission via RPC)
 214 def ToHex(obj):
 215     return bytes_to_hex_str(obj.serialize())
 216
 217 # Objects that map to bitcoind objects, which can be serialized/deserialized
 218
 219 class CAddress(object):
 220     def __init__(self):
 221         self.nServices = 1
 222         self.pchReserved = b"\x00" * 10 + b"\xff" * 2
 223         self.ip = "0.0.0.0"
 224         self.port = 0
 225
 226     def deserialize(self, f):
 227         self.nServices = struct.unpack("<Q", f.read(8))[0]
 228         self.pchReserved = f.read(12)
 229         self.ip = socket.inet_ntoa(f.read(4))
 230         self.port = struct.unpack(">H", f.read(2))[0]
 231
 232     def serialize(self):
 233         r = b""
 234         r += struct.pack("<Q", self.nServices)
 235         r += self.pchReserved
 236         r += socket.inet_aton(self.ip)
 237         r += struct.pack(">H", self.port)
 238         return r
 239
 240     def __repr__(self):
 241         return "CAddress(nServices=%i ip=%s port=%i)" % (self.nServices,
 242                                                          self.ip, self.port)
 243
 244 MSG_WITNESS_FLAG = 1<<30
 245
 246 class CInv(object):
 247     typemap = {
 248         0: "Error",
 249         1: "TX",
 250         2: "Block",
 251         1|MSG_WITNESS_FLAG: "WitnessTx",
 252         2|MSG_WITNESS_FLAG : "WitnessBlock",
 253         4: "CompactBlock"
 254     }
 255
 256     def __init__(self, t=0, h=0):
 257         self.type = t
 258         self.hash = h
 259
 260     def deserialize(self, f):
 261         self.type = struct.unpack("<i", f.read(4))[0]
 262         self.hash = deser_uint256(f)
 263
 264     def serialize(self):
 265         r = b""
 266         r += struct.pack("<i", self.type)
 267         r += ser_uint256(self.hash)
 268         return r
 269
 270     def __repr__(self):
 271         return "CInv(type=%s hash=%064x)" \
 272             % (self.typemap[self.type], self.hash)
 273
 274
 275 class CBlockLocator(object):
 276     def __init__(self):
 277         self.nVersion = MY_VERSION
 278         self.vHave = []
 279
 280     def deserialize(self, f):
 281         self.nVersion = struct.unpack("<i", f.read(4))[0]
 282         self.vHave = deser_uint256_vector(f)
 283
 284     def serialize(self):
 285         r = b""
 286         r += struct.pack("<i", self.nVersion)
 287         r += ser_uint256_vector(self.vHave)
 288         return r
 289
 290     def __repr__(self):
 291         return "CBlockLocator(nVersion=%i vHave=%s)" \
 292             % (self.nVersion, repr(self.vHave))
 293
 294
 295 class COutPoint(object):
 296     def __init__(self, hash=0, n=0):
 297         self.hash = hash
 298         self.n = n
 299
 300     def deserialize(self, f):
 301         self.hash = deser_uint256(f)
 302         self.n = struct.unpack("<I", f.read(4))[0]
 303
 304     def serialize(self):
 305         r = b""
 306         r += ser_uint256(self.hash)
 307         r += struct.pack("<I", self.n)
 308         return r
 309
 310     def __repr__(self):
 311         return "COutPoint(hash=%064x n=%i)" % (self.hash, self.n)
 312
 313
 314 class CTxIn(object):
 315     def __init__(self, outpoint=None, scriptSig=b"", nSequence=0):
 316         if outpoint is None:
 317             self.prevout = COutPoint()
 318         else:
 319             self.prevout = outpoint
 320         self.scriptSig = scriptSig
 321         self.nSequence = nSequence
 322
 323     def deserialize(self, f):
 324         self.prevout = COutPoint()
 325         self.prevout.deserialize(f)
 326         self.scriptSig = deser_string(f)
 327         self.nSequence = struct.unpack("<I", f.read(4))[0]
 328
 329     def serialize(self):
 330         r = b""
 331         r += self.prevout.serialize()
 332         r += ser_string(self.scriptSig)
 333         r += struct.pack("<I", self.nSequence)
 334         return r
 335
 336     def __repr__(self):
 337         return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \
 338             % (repr(self.prevout), bytes_to_hex_str(self.scriptSig),
 339                self.nSequence)
 340
 341
 342 class CTxOut(object):
 343     def __init__(self, nValue=0, scriptPubKey=b""):
 344         self.nValue = nValue
 345         self.scriptPubKey = scriptPubKey
 346
 347     def deserialize(self, f):
 348         self.nValue = struct.unpack("<q", f.read(8))[0]
 349         self.scriptPubKey = deser_string(f)
 350
 351     def serialize(self):
 352         r = b""
 353         r += struct.pack("<q", self.nValue)
 354         r += ser_string(self.scriptPubKey)
 355         return r
 356
 357     def __repr__(self):
 358         return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \
 359             % (self.nValue // COIN, self.nValue % COIN,
 360                bytes_to_hex_str(self.scriptPubKey))
 361
 362
 363 class CScriptWitness(object):
 364     def __init__(self):
 365         # stack is a vector of strings
 366         self.stack = []
 367
 368     def __repr__(self):
 369         return "CScriptWitness(%s)" % \
 370                (",".join([bytes_to_hex_str(x) for x in self.stack]))
 371
 372     def is_null(self):
 373         if self.stack:
 374             return False
 375         return True
 376
 377
 378 class CTxInWitness(object):
 379     def __init__(self):
 380         self.scriptWitness = CScriptWitness()
 381
 382     def deserialize(self, f):
 383         self.scriptWitness.stack = deser_string_vector(f)
 384
 385     def serialize(self):
 386         return ser_string_vector(self.scriptWitness.stack)
 387
 388     def __repr__(self):
 389         return repr(self.scriptWitness)
 390
 391     def is_null(self):
 392         return self.scriptWitness.is_null()
 393
 394
 395 class CTxWitness(object):
 396     def __init__(self):
 397         self.vtxinwit = []
 398
 399     def deserialize(self, f):
 400         for i in range(len(self.vtxinwit)):
 401             self.vtxinwit[i].deserialize(f)
 402
 403     def serialize(self):
 404         r = b""
 405         # This is different than the usual vector serialization --
 406         # we omit the length of the vector, which is required to be
 407         # the same length as the transaction's vin vector.
 408         for x in self.vtxinwit:
 409             r += x.serialize()
 410         return r
 411
 412     def __repr__(self):
 413         return "CTxWitness(%s)" % \
 414                (';'.join([repr(x) for x in self.vtxinwit]))
 415
 416     def is_null(self):
 417         for x in self.vtxinwit:
 418             if not x.is_null():
 419                 return False
 420         return True
 421
 422
 423 class CTransaction(object):
 424     def __init__(self, tx=None):
 425         if tx is None:
 426             self.nVersion = 1
 427             self.vin = []
 428             self.vout = []
 429             self.wit = CTxWitness()
 430             self.nLockTime = 0
 431             self.sha256 = None
 432             self.hash = None
 433         else:
 434             self.nVersion = tx.nVersion
 435             self.vin = copy.deepcopy(tx.vin)
 436             self.vout = copy.deepcopy(tx.vout)
 437             self.nLockTime = tx.nLockTime
 438             self.sha256 = tx.sha256
 439             self.hash = tx.hash
 440             self.wit = copy.deepcopy(tx.wit)
 441
 442     def deserialize(self, f):
 443         self.nVersion = struct.unpack("<i", f.read(4))[0]
 444         self.vin = deser_vector(f, CTxIn)
 445         flags = 0
 446         if len(self.vin) == 0:
 447             flags = struct.unpack("<B", f.read(1))[0]
 448             # Not sure why flags can't be zero, but this
 449             # matches the implementation in bitcoind
 450             if (flags != 0):
 451                 self.vin = deser_vector(f, CTxIn)
 452                 self.vout = deser_vector(f, CTxOut)
 453         else:
 454             self.vout = deser_vector(f, CTxOut)
 455         if flags != 0:
 456             self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))]
 457             self.wit.deserialize(f)
 458         self.nLockTime = struct.unpack("<I", f.read(4))[0]
 459         self.sha256 = None
 460         self.hash = None
 461
 462     def serialize_without_witness(self):
 463         r = b""
 464         r += struct.pack("<i", self.nVersion)
 465         r += ser_vector(self.vin)
 466         r += ser_vector(self.vout)
 467         r += struct.pack("<I", self.nLockTime)
 468         return r
 469
 470     # Only serialize with witness when explicitly called for
 471     def serialize_with_witness(self):
 472         flags = 0
 473         if not self.wit.is_null():
 474             flags |= 1
 475         r = b""
 476         r += struct.pack("<i", self.nVersion)
 477         if flags:
 478             dummy = []
 479             r += ser_vector(dummy)
 480             r += struct.pack("<B", flags)
 481         r += ser_vector(self.vin)
 482         r += ser_vector(self.vout)
 483         if flags & 1:
 484             if (len(self.wit.vtxinwit) != len(self.vin)):
 485                 # vtxinwit must have the same length as vin
 486                 self.wit.vtxinwit = self.wit.vtxinwit[:len(self.vin)]
 487                 for i in range(len(self.wit.vtxinwit), len(self.vin)):
 488                     self.wit.vtxinwit.append(CTxInWitness())
 489             r += self.wit.serialize()
 490         r += struct.pack("<I", self.nLockTime)
 491         return r
 492
 493     # Regular serialization is without witness -- must explicitly
 494     # call serialize_with_witness to include witness data.
 495     def serialize(self):
 496         return self.serialize_without_witness()
 497
 498     # Recalculate the txid (transaction hash without witness)
 499     def rehash(self):
 500         self.sha256 = None
 501         self.calc_sha256()
 502
 503     # We will only cache the serialization without witness in
 504     # self.sha256 and self.hash -- those are expected to be the txid.
 505     def calc_sha256(self, with_witness=False):
 506         if with_witness:
 507             # Don't cache the result, just return it
 508             return uint256_from_str(hash256(self.serialize_with_witness()))
 509
 510         if self.sha256 is None:
 511             self.sha256 = uint256_from_str(hash256(self.serialize_without_witness()))
 512         self.hash = encode(hash256(self.serialize())[::-1], 'hex_codec').decode('ascii')
 513
 514     def is_valid(self):
 515         self.calc_sha256()
 516         for tout in self.vout:
 517             if tout.nValue < 0 or tout.nValue > 21000000 * COIN:
 518                 return False
 519         return True
 520
 521     def __repr__(self):
 522         return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \
 523             % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime)
 524
 525
 526 class CBlockHeader(object):
 527     def __init__(self, header=None):
 528         if header is None:
 529             self.set_null()
 530         else:
 531             self.nVersion = header.nVersion
 532             self.hashPrevBlock = header.hashPrevBlock
 533             self.hashMerkleRoot = header.hashMerkleRoot
 534             self.nTime = header.nTime
 535             self.nBits = header.nBits
 536             self.nNonce = header.nNonce
 537             self.sha256 = header.sha256
 538             self.hash = header.hash
 539             self.calc_sha256()
 540
 541     def set_null(self):
 542         self.nVersion = 1
 543         self.hashPrevBlock = 0
 544         self.hashMerkleRoot = 0
 545         self.nTime = 0
 546         self.nBits = 0
 547         self.nNonce = 0
 548         self.sha256 = None
 549         self.hash = None
 550
 551     def deserialize(self, f):
 552         self.nVersion = struct.unpack("<i", f.read(4))[0]
 553         self.hashPrevBlock = deser_uint256(f)
 554         self.hashMerkleRoot = deser_uint256(f)
 555         self.nTime = struct.unpack("<I", f.read(4))[0]
 556         self.nBits = struct.unpack("<I", f.read(4))[0]
 557         self.nNonce = struct.unpack("<I", f.read(4))[0]
 558         self.sha256 = None
 559         self.hash = None
 560
 561     def serialize(self):
 562         r = b""
 563         r += struct.pack("<i", self.nVersion)
 564         r += ser_uint256(self.hashPrevBlock)
 565         r += ser_uint256(self.hashMerkleRoot)
 566         r += struct.pack("<I", self.nTime)
 567         r += struct.pack("<I", self.nBits)
 568         r += struct.pack("<I", self.nNonce)
 569         return r
 570
 571     def calc_sha256(self):
 572         if self.sha256 is None:
 573             r = b""
 574             r += struct.pack("<i", self.nVersion)
 575             r += ser_uint256(self.hashPrevBlock)
 576             r += ser_uint256(self.hashMerkleRoot)
 577             r += struct.pack("<I", self.nTime)
 578             r += struct.pack("<I", self.nBits)
 579             r += struct.pack("<I", self.nNonce)
 580             self.sha256 = uint256_from_str(hash256(r))
 581             self.hash = encode(hash256(r)[::-1], 'hex_codec').decode('ascii')
 582
 583     def rehash(self):
 584         self.sha256 = None
 585         self.calc_sha256()
 586         return self.sha256
 587
 588     def __repr__(self):
 589         return "CBlockHeader(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x)" \
 590             % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
 591                time.ctime(self.nTime), self.nBits, self.nNonce)
 592
 593
 594 class CBlock(CBlockHeader):
 595     def __init__(self, header=None):
 596         super(CBlock, self).__init__(header)
 597         self.vtx = []
 598
 599     def deserialize(self, f):
 600         super(CBlock, self).deserialize(f)
 601         self.vtx = deser_vector(f, CTransaction)
 602
 603     def serialize(self, with_witness=False):
 604         r = b""
 605         r += super(CBlock, self).serialize()
 606         if with_witness:
 607             r += ser_vector(self.vtx, "serialize_with_witness")
 608         else:
 609             r += ser_vector(self.vtx)
 610         return r
 611
 612     # Calculate the merkle root given a vector of transaction hashes
 613     def get_merkle_root(self, hashes):
 614         while len(hashes) > 1:
 615             newhashes = []
 616             for i in range(0, len(hashes), 2):
 617                 i2 = min(i+1, len(hashes)-1)
 618                 newhashes.append(hash256(hashes[i] + hashes[i2]))
 619             hashes = newhashes
 620         return uint256_from_str(hashes[0])
 621
 622     def calc_merkle_root(self):
 623         hashes = []
 624         for tx in self.vtx:
 625             tx.calc_sha256()
 626             hashes.append(ser_uint256(tx.sha256))
 627         return self.get_merkle_root(hashes)
 628
 629     def calc_witness_merkle_root(self):
 630         # For witness root purposes, the hash of the
 631         # coinbase, with witness, is defined to be 0...0
 632         hashes = [ser_uint256(0)]
 633
 634         for tx in self.vtx[1:]:
 635             # Calculate the hashes with witness data
 636             hashes.append(ser_uint256(tx.calc_sha256(True)))
 637
 638         return self.get_merkle_root(hashes)
 639
 640     def is_valid(self):
 641         self.calc_sha256()
 642         target = uint256_from_compact(self.nBits)
 643         if self.sha256 > target:
 644             return False
 645         for tx in self.vtx:
 646             if not tx.is_valid():
 647                 return False
 648         if self.calc_merkle_root() != self.hashMerkleRoot:
 649             return False
 650         return True
 651
 652     def solve(self):
 653         self.rehash()
 654         target = uint256_from_compact(self.nBits)
 655         while self.sha256 > target:
 656             self.nNonce += 1
 657             self.rehash()
 658
 659     def __repr__(self):
 660         return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x vtx=%s)" \
 661             % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
 662                time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx))
 663
 664
 665 class CUnsignedAlert(object):
 666     def __init__(self):
 667         self.nVersion = 1
 668         self.nRelayUntil = 0
 669         self.nExpiration = 0
 670         self.nID = 0
 671         self.nCancel = 0
 672         self.setCancel = []
 673         self.nMinVer = 0
 674         self.nMaxVer = 0
 675         self.setSubVer = []
 676         self.nPriority = 0
 677         self.strComment = b""
 678         self.strStatusBar = b""
 679         self.strReserved = b""
 680
 681     def deserialize(self, f):
 682         self.nVersion = struct.unpack("<i", f.read(4))[0]
 683         self.nRelayUntil = struct.unpack("<q", f.read(8))[0]
 684         self.nExpiration = struct.unpack("<q", f.read(8))[0]
 685         self.nID = struct.unpack("<i", f.read(4))[0]
 686         self.nCancel = struct.unpack("<i", f.read(4))[0]
 687         self.setCancel = deser_int_vector(f)
 688         self.nMinVer = struct.unpack("<i", f.read(4))[0]
 689         self.nMaxVer = struct.unpack("<i", f.read(4))[0]
 690         self.setSubVer = deser_string_vector(f)
 691         self.nPriority = struct.unpack("<i", f.read(4))[0]
 692         self.strComment = deser_string(f)
 693         self.strStatusBar = deser_string(f)
 694         self.strReserved = deser_string(f)
 695
 696     def serialize(self):
 697         r = b""
 698         r += struct.pack("<i", self.nVersion)
 699         r += struct.pack("<q", self.nRelayUntil)
 700         r += struct.pack("<q", self.nExpiration)
 701         r += struct.pack("<i", self.nID)
 702         r += struct.pack("<i", self.nCancel)
 703         r += ser_int_vector(self.setCancel)
 704         r += struct.pack("<i", self.nMinVer)
 705         r += struct.pack("<i", self.nMaxVer)
 706         r += ser_string_vector(self.setSubVer)
 707         r += struct.pack("<i", self.nPriority)
 708         r += ser_string(self.strComment)
 709         r += ser_string(self.strStatusBar)
 710         r += ser_string(self.strReserved)
 711         return r
 712
 713     def __repr__(self):
 714         return "CUnsignedAlert(nVersion %d, nRelayUntil %d, nExpiration %d, nID %d, nCancel %d, nMinVer %d, nMaxVer %d, nPriority %d, strComment %s, strStatusBar %s, strReserved %s)" \
 715             % (self.nVersion, self.nRelayUntil, self.nExpiration, self.nID,
 716                self.nCancel, self.nMinVer, self.nMaxVer, self.nPriority,
 717                self.strComment, self.strStatusBar, self.strReserved)
 718
 719
 720 class CAlert(object):
 721     def __init__(self):
 722         self.vchMsg = b""
 723         self.vchSig = b""
 724
 725     def deserialize(self, f):
 726         self.vchMsg = deser_string(f)
 727         self.vchSig = deser_string(f)
 728
 729     def serialize(self):
 730         r = b""
 731         r += ser_string(self.vchMsg)
 732         r += ser_string(self.vchSig)
 733         return r
 734
 735     def __repr__(self):
 736         return "CAlert(vchMsg.sz %d, vchSig.sz %d)" \
 737             % (len(self.vchMsg), len(self.vchSig))
 738
 739
 740 class PrefilledTransaction(object):
 741     def __init__(self, index=0, tx = None):
 742         self.index = index
 743         self.tx = tx
 744
 745     def deserialize(self, f):
 746         self.index = deser_compact_size(f)
 747         self.tx = CTransaction()
 748         self.tx.deserialize(f)
 749
 750     def serialize(self, with_witness=False):
 751         r = b""
 752         r += ser_compact_size(self.index)
 753         if with_witness:
 754             r += self.tx.serialize_with_witness()
 755         else:
 756             r += self.tx.serialize_without_witness()
 757         return r
 758
 759     def serialize_with_witness(self):
 760         return self.serialize(with_witness=True)
 761
 762     def __repr__(self):
 763         return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx))
 764
 765 # This is what we send on the wire, in a cmpctblock message.
 766 class P2PHeaderAndShortIDs(object):
 767     def __init__(self):
 768         self.header = CBlockHeader()
 769         self.nonce = 0
 770         self.shortids_length = 0
 771         self.shortids = []
 772         self.prefilled_txn_length = 0
 773         self.prefilled_txn = []
 774
 775     def deserialize(self, f):
 776         self.header.deserialize(f)
 777         self.nonce = struct.unpack("<Q", f.read(8))[0]
 778         self.shortids_length = deser_compact_size(f)
 779         for i in range(self.shortids_length):
 780             # shortids are defined to be 6 bytes in the spec, so append
 781             # two zero bytes and read it in as an 8-byte number
 782             self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0])
 783         self.prefilled_txn = deser_vector(f, PrefilledTransaction)
 784         self.prefilled_txn_length = len(self.prefilled_txn)
 785
 786     # When using version 2 compact blocks, we must serialize with_witness.
 787     def serialize(self, with_witness=False):
 788         r = b""
 789         r += self.header.serialize()
 790         r += struct.pack("<Q", self.nonce)
 791         r += ser_compact_size(self.shortids_length)
 792         for x in self.shortids:
 793             # We only want the first 6 bytes
 794             r += struct.pack("<Q", x)[0:6]
 795         if with_witness:
 796             r += ser_vector(self.prefilled_txn, "serialize_with_witness")
 797         else:
 798             r += ser_vector(self.prefilled_txn)
 799         return r
 800
 801     def __repr__(self):
 802         return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn))
 803
 804 # P2P version of the above that will use witness serialization (for compact
 805 # block version 2)
 806 class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs):
 807     def serialize(self):
 808         return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True)
 809
 810 # Calculate the BIP 152-compact blocks shortid for a given transaction hash
 811 def calculate_shortid(k0, k1, tx_hash):
 812     expected_shortid = siphash256(k0, k1, tx_hash)
 813     expected_shortid &= 0x0000ffffffffffff
 814     return expected_shortid
 815
 816 # This version gets rid of the array lengths, and reinterprets the differential
 817 # encoding into indices that can be used for lookup.
 818 class HeaderAndShortIDs(object):
 819     def __init__(self, p2pheaders_and_shortids = None):
 820         self.header = CBlockHeader()
 821         self.nonce = 0
 822         self.shortids = []
 823         self.prefilled_txn = []
 824         self.use_witness = False
 825
 826         if p2pheaders_and_shortids != None:
 827             self.header = p2pheaders_and_shortids.header
 828             self.nonce = p2pheaders_and_shortids.nonce
 829             self.shortids = p2pheaders_and_shortids.shortids
 830             last_index = -1
 831             for x in p2pheaders_and_shortids.prefilled_txn:
 832                 self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx))
 833                 last_index = self.prefilled_txn[-1].index
 834
 835     def to_p2p(self):
 836         if self.use_witness:
 837             ret = P2PHeaderAndShortWitnessIDs()
 838         else:
 839             ret = P2PHeaderAndShortIDs()
 840         ret.header = self.header
 841         ret.nonce = self.nonce
 842         ret.shortids_length = len(self.shortids)
 843         ret.shortids = self.shortids
 844         ret.prefilled_txn_length = len(self.prefilled_txn)
 845         ret.prefilled_txn = []
 846         last_index = -1
 847         for x in self.prefilled_txn:
 848             ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx))
 849             last_index = x.index
 850         return ret
 851
 852     def get_siphash_keys(self):
 853         header_nonce = self.header.serialize()
 854         header_nonce += struct.pack("<Q", self.nonce)
 855         hash_header_nonce_as_str = sha256(header_nonce)
 856         key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0]
 857         key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0]
 858         return [ key0, key1 ]
 859
 860     # Version 2 compact blocks use wtxid in shortids (rather than txid)
 861     def initialize_from_block(self, block, nonce=0, prefill_list = [0], use_witness = False):
 862         self.header = CBlockHeader(block)
 863         self.nonce = nonce
 864         self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ]
 865         self.shortids = []
 866         self.use_witness = use_witness
 867         [k0, k1] = self.get_siphash_keys()
 868         for i in range(len(block.vtx)):
 869             if i not in prefill_list:
 870                 tx_hash = block.vtx[i].sha256
 871                 if use_witness:
 872                     tx_hash = block.vtx[i].calc_sha256(with_witness=True)
 873                 self.shortids.append(calculate_shortid(k0, k1, tx_hash))
 874
 875     def __repr__(self):
 876         return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn))
 877
 878
 879 class BlockTransactionsRequest(object):
 880
 881     def __init__(self, blockhash=0, indexes = None):
 882         self.blockhash = blockhash
 883         self.indexes = indexes if indexes != None else []
 884
 885     def deserialize(self, f):
 886         self.blockhash = deser_uint256(f)
 887         indexes_length = deser_compact_size(f)
 888         for i in range(indexes_length):
 889             self.indexes.append(deser_compact_size(f))
 890
 891     def serialize(self):
 892         r = b""
 893         r += ser_uint256(self.blockhash)
 894         r += ser_compact_size(len(self.indexes))
 895         for x in self.indexes:
 896             r += ser_compact_size(x)
 897         return r
 898
 899     # helper to set the differentially encoded indexes from absolute ones
 900     def from_absolute(self, absolute_indexes):
 901         self.indexes = []
 902         last_index = -1
 903         for x in absolute_indexes:
 904             self.indexes.append(x-last_index-1)
 905             last_index = x
 906
 907     def to_absolute(self):
 908         absolute_indexes = []
 909         last_index = -1
 910         for x in self.indexes:
 911             absolute_indexes.append(x+last_index+1)
 912             last_index = absolute_indexes[-1]
 913         return absolute_indexes
 914
 915     def __repr__(self):
 916         return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes))
 917
 918
 919 class BlockTransactions(object):
 920
 921     def __init__(self, blockhash=0, transactions = None):
 922         self.blockhash = blockhash
 923         self.transactions = transactions if transactions != None else []
 924
 925     def deserialize(self, f):
 926         self.blockhash = deser_uint256(f)
 927         self.transactions = deser_vector(f, CTransaction)
 928
 929     def serialize(self, with_witness=False):
 930         r = b""
 931         r += ser_uint256(self.blockhash)
 932         if with_witness:
 933             r += ser_vector(self.transactions, "serialize_with_witness")
 934         else:
 935             r += ser_vector(self.transactions)
 936         return r
 937
 938     def __repr__(self):
 939         return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions))
 940
 941
 942 # Objects that correspond to messages on the wire
 943 class msg_version(object):
 944     command = b"version"
 945
 946     def __init__(self):
 947         self.nVersion = MY_VERSION
 948         self.nServices = 1
 949         self.nTime = int(time.time())
 950         self.addrTo = CAddress()
 951         self.addrFrom = CAddress()
 952         self.nNonce = random.getrandbits(64)
 953         self.strSubVer = MY_SUBVERSION
 954         self.nStartingHeight = -1
 955         self.nRelay = MY_RELAY
 956
 957     def deserialize(self, f):
 958         self.nVersion = struct.unpack("<i", f.read(4))[0]
 959         if self.nVersion == 10300:
 960             self.nVersion = 300
 961         self.nServices = struct.unpack("<Q", f.read(8))[0]
 962         self.nTime = struct.unpack("<q", f.read(8))[0]
 963         self.addrTo = CAddress()
 964         self.addrTo.deserialize(f)
 965
 966         if self.nVersion >= 106:
 967             self.addrFrom = CAddress()
 968             self.addrFrom.deserialize(f)
 969             self.nNonce = struct.unpack("<Q", f.read(8))[0]
 970             self.strSubVer = deser_string(f)
 971         else:
 972             self.addrFrom = None
 973             self.nNonce = None
 974             self.strSubVer = None
 975             self.nStartingHeight = None
 976
 977         if self.nVersion >= 209:
 978             self.nStartingHeight = struct.unpack("<i", f.read(4))[0]
 979         else:
 980             self.nStartingHeight = None
 981
 982         if self.nVersion >= 70001:
 983             # Relay field is optional for version 70001 onwards
 984             try:
 985                 self.nRelay = struct.unpack("<b", f.read(1))[0]
 986             except:
 987                 self.nRelay = 0
 988         else:
 989             self.nRelay = 0
 990
 991     def serialize(self):
 992         r = b""
 993         r += struct.pack("<i", self.nVersion)
 994         r += struct.pack("<Q", self.nServices)
 995         r += struct.pack("<q", self.nTime)
 996         r += self.addrTo.serialize()
 997         r += self.addrFrom.serialize()
 998         r += struct.pack("<Q", self.nNonce)
 999         r += ser_string(self.strSubVer)
1000         r += struct.pack("<i", self.nStartingHeight)
1001         r += struct.pack("<b", self.nRelay)
1002         return r
1003
1004     def __repr__(self):
1005         return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i nRelay=%i)' \
1006             % (self.nVersion, self.nServices, time.ctime(self.nTime),
1007                repr(self.addrTo), repr(self.addrFrom), self.nNonce,
1008                self.strSubVer, self.nStartingHeight, self.nRelay)
1009
1010
1011 class msg_verack(object):
1012     command = b"verack"
1013
1014     def __init__(self):
1015         pass
1016
1017     def deserialize(self, f):
1018         pass
1019
1020     def serialize(self):
1021         return b""
1022
1023     def __repr__(self):
1024         return "msg_verack()"
1025
1026
1027 class msg_addr(object):
1028     command = b"addr"
1029
1030     def __init__(self):
1031         self.addrs = []
1032
1033     def deserialize(self, f):
1034         self.addrs = deser_vector(f, CAddress)
1035
1036     def serialize(self):
1037         return ser_vector(self.addrs)
1038
1039     def __repr__(self):
1040         return "msg_addr(addrs=%s)" % (repr(self.addrs))
1041
1042
1043 class msg_alert(object):
1044     command = b"alert"
1045
1046     def __init__(self):
1047         self.alert = CAlert()
1048
1049     def deserialize(self, f):
1050         self.alert = CAlert()
1051         self.alert.deserialize(f)
1052
1053     def serialize(self):
1054         r = b""
1055         r += self.alert.serialize()
1056         return r
1057
1058     def __repr__(self):
1059         return "msg_alert(alert=%s)" % (repr(self.alert), )
1060
1061
1062 class msg_inv(object):
1063     command = b"inv"
1064
1065     def __init__(self, inv=None):
1066         if inv is None:
1067             self.inv = []
1068         else:
1069             self.inv = inv
1070
1071     def deserialize(self, f):
1072         self.inv = deser_vector(f, CInv)
1073
1074     def serialize(self):
1075         return ser_vector(self.inv)
1076
1077     def __repr__(self):
1078         return "msg_inv(inv=%s)" % (repr(self.inv))
1079
1080
1081 class msg_getdata(object):
1082     command = b"getdata"
1083
1084     def __init__(self, inv=None):
1085         self.inv = inv if inv != None else []
1086
1087     def deserialize(self, f):
1088         self.inv = deser_vector(f, CInv)
1089
1090     def serialize(self):
1091         return ser_vector(self.inv)
1092
1093     def __repr__(self):
1094         return "msg_getdata(inv=%s)" % (repr(self.inv))
1095
1096
1097 class msg_getblocks(object):
1098     command = b"getblocks"
1099
1100     def __init__(self):
1101         self.locator = CBlockLocator()
1102         self.hashstop = 0
1103
1104     def deserialize(self, f):
1105         self.locator = CBlockLocator()
1106         self.locator.deserialize(f)
1107         self.hashstop = deser_uint256(f)
1108
1109     def serialize(self):
1110         r = b""
1111         r += self.locator.serialize()
1112         r += ser_uint256(self.hashstop)
1113         return r
1114
1115     def __repr__(self):
1116         return "msg_getblocks(locator=%s hashstop=%064x)" \
1117             % (repr(self.locator), self.hashstop)
1118
1119
1120 class msg_tx(object):
1121     command = b"tx"
1122
1123     def __init__(self, tx=CTransaction()):
1124         self.tx = tx
1125
1126     def deserialize(self, f):
1127         self.tx.deserialize(f)
1128
1129     def serialize(self):
1130         return self.tx.serialize_without_witness()
1131
1132     def __repr__(self):
1133         return "msg_tx(tx=%s)" % (repr(self.tx))
1134
1135 class msg_witness_tx(msg_tx):
1136
1137     def serialize(self):
1138         return self.tx.serialize_with_witness()
1139
1140
1141 class msg_block(object):
1142     command = b"block"
1143
1144     def __init__(self, block=None):
1145         if block is None:
1146             self.block = CBlock()
1147         else:
1148             self.block = block
1149
1150     def deserialize(self, f):
1151         self.block.deserialize(f)
1152
1153     def serialize(self):
1154         return self.block.serialize()
1155
1156     def __repr__(self):
1157         return "msg_block(block=%s)" % (repr(self.block))
1158
1159 # for cases where a user needs tighter control over what is sent over the wire
1160 # note that the user must supply the name of the command, and the data
1161 class msg_generic(object):
1162     def __init__(self, command, data=None):
1163         self.command = command
1164         self.data = data
1165
1166     def serialize(self):
1167         return self.data
1168
1169     def __repr__(self):
1170         return "msg_generic()"
1171
1172 class msg_witness_block(msg_block):
1173
1174     def serialize(self):
1175         r = self.block.serialize(with_witness=True)
1176         return r
1177
1178 class msg_getaddr(object):
1179     command = b"getaddr"
1180
1181     def __init__(self):
1182         pass
1183
1184     def deserialize(self, f):
1185         pass
1186
1187     def serialize(self):
1188         return b""
1189
1190     def __repr__(self):
1191         return "msg_getaddr()"
1192
1193
1194 class msg_ping_prebip31(object):
1195     command = b"ping"
1196
1197     def __init__(self):
1198         pass
1199
1200     def deserialize(self, f):
1201         pass
1202
1203     def serialize(self):
1204         return b""
1205
1206     def __repr__(self):
1207         return "msg_ping() (pre-bip31)"
1208
1209
1210 class msg_ping(object):
1211     command = b"ping"
1212
1213     def __init__(self, nonce=0):
1214         self.nonce = nonce
1215
1216     def deserialize(self, f):
1217         self.nonce = struct.unpack("<Q", f.read(8))[0]
1218
1219     def serialize(self):
1220         r = b""
1221         r += struct.pack("<Q", self.nonce)
1222         return r
1223
1224     def __repr__(self):
1225         return "msg_ping(nonce=%08x)" % self.nonce
1226
1227
1228 class msg_pong(object):
1229     command = b"pong"
1230
1231     def __init__(self, nonce=0):
1232         self.nonce = nonce
1233
1234     def deserialize(self, f):
1235         self.nonce = struct.unpack("<Q", f.read(8))[0]
1236
1237     def serialize(self):
1238         r = b""
1239         r += struct.pack("<Q", self.nonce)
1240         return r
1241
1242     def __repr__(self):
1243         return "msg_pong(nonce=%08x)" % self.nonce
1244
1245
1246 class msg_mempool(object):
1247     command = b"mempool"
1248
1249     def __init__(self):
1250         pass
1251
1252     def deserialize(self, f):
1253         pass
1254
1255     def serialize(self):
1256         return b""
1257
1258     def __repr__(self):
1259         return "msg_mempool()"
1260
1261 class msg_sendheaders(object):
1262     command = b"sendheaders"
1263
1264     def __init__(self):
1265         pass
1266
1267     def deserialize(self, f):
1268         pass
1269
1270     def serialize(self):
1271         return b""
1272
1273     def __repr__(self):
1274         return "msg_sendheaders()"
1275
1276
1277 # getheaders message has
1278 # number of entries
1279 # vector of hashes
1280 # hash_stop (hash of last desired block header, 0 to get as many as possible)
1281 class msg_getheaders(object):
1282     command = b"getheaders"
1283
1284     def __init__(self):
1285         self.locator = CBlockLocator()
1286         self.hashstop = 0
1287
1288     def deserialize(self, f):
1289         self.locator = CBlockLocator()
1290         self.locator.deserialize(f)
1291         self.hashstop = deser_uint256(f)
1292
1293     def serialize(self):
1294         r = b""
1295         r += self.locator.serialize()
1296         r += ser_uint256(self.hashstop)
1297         return r
1298
1299     def __repr__(self):
1300         return "msg_getheaders(locator=%s, stop=%064x)" \
1301             % (repr(self.locator), self.hashstop)
1302
1303
1304 # headers message has
1305 # <count> <vector of block headers>
1306 class msg_headers(object):
1307     command = b"headers"
1308
1309     def __init__(self):
1310         self.headers = []
1311
1312     def deserialize(self, f):
1313         # comment in bitcoind indicates these should be deserialized as blocks
1314         blocks = deser_vector(f, CBlock)
1315         for x in blocks:
1316             self.headers.append(CBlockHeader(x))
1317
1318     def serialize(self):
1319         blocks = [CBlock(x) for x in self.headers]
1320         return ser_vector(blocks)
1321
1322     def __repr__(self):
1323         return "msg_headers(headers=%s)" % repr(self.headers)
1324
1325
1326 class msg_reject(object):
1327     command = b"reject"
1328     REJECT_MALFORMED = 1
1329
1330     def __init__(self):
1331         self.message = b""
1332         self.code = 0
1333         self.reason = b""
1334         self.data = 0
1335
1336     def deserialize(self, f):
1337         self.message = deser_string(f)
1338         self.code = struct.unpack("<B", f.read(1))[0]
1339         self.reason = deser_string(f)
1340         if (self.code != self.REJECT_MALFORMED and
1341                 (self.message == b"block" or self.message == b"tx")):
1342             self.data = deser_uint256(f)
1343
1344     def serialize(self):
1345         r = ser_string(self.message)
1346         r += struct.pack("<B", self.code)
1347         r += ser_string(self.reason)
1348         if (self.code != self.REJECT_MALFORMED and
1349                 (self.message == b"block" or self.message == b"tx")):
1350             r += ser_uint256(self.data)
1351         return r
1352
1353     def __repr__(self):
1354         return "msg_reject: %s %d %s [%064x]" \
1355             % (self.message, self.code, self.reason, self.data)
1356
1357 # Helper function
1358 def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf')):
1359     attempt = 0
1360     elapsed = 0
1361
1362     while attempt < attempts and elapsed < timeout:
1363         with mininode_lock:
1364             if predicate():
1365                 return True
1366         attempt += 1
1367         elapsed += 0.05
1368         time.sleep(0.05)
1369
1370     return False
1371
1372 class msg_feefilter(object):
1373     command = b"feefilter"
1374
1375     def __init__(self, feerate=0):
1376         self.feerate = feerate
1377
1378     def deserialize(self, f):
1379         self.feerate = struct.unpack("<Q", f.read(8))[0]
1380
1381     def serialize(self):
1382         r = b""
1383         r += struct.pack("<Q", self.feerate)
1384         return r
1385
1386     def __repr__(self):
1387         return "msg_feefilter(feerate=%08x)" % self.feerate
1388
1389 class msg_sendcmpct(object):
1390     command = b"sendcmpct"
1391
1392     def __init__(self):
1393         self.announce = False
1394         self.version = 1
1395
1396     def deserialize(self, f):
1397         self.announce = struct.unpack("<?", f.read(1))[0]
1398         self.version = struct.unpack("<Q", f.read(8))[0]
1399
1400     def serialize(self):
1401         r = b""
1402         r += struct.pack("<?", self.announce)
1403         r += struct.pack("<Q", self.version)
1404         return r
1405
1406     def __repr__(self):
1407         return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version)
1408
1409 class msg_cmpctblock(object):
1410     command = b"cmpctblock"
1411
1412     def __init__(self, header_and_shortids = None):
1413         self.header_and_shortids = header_and_shortids
1414
1415     def deserialize(self, f):
1416         self.header_and_shortids = P2PHeaderAndShortIDs()
1417         self.header_and_shortids.deserialize(f)
1418
1419     def serialize(self):
1420         r = b""
1421         r += self.header_and_shortids.serialize()
1422         return r
1423
1424     def __repr__(self):
1425         return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids)
1426
1427 class msg_getblocktxn(object):
1428     command = b"getblocktxn"
1429
1430     def __init__(self):
1431         self.block_txn_request = None
1432
1433     def deserialize(self, f):
1434         self.block_txn_request = BlockTransactionsRequest()
1435         self.block_txn_request.deserialize(f)
1436
1437     def serialize(self):
1438         r = b""
1439         r += self.block_txn_request.serialize()
1440         return r
1441
1442     def __repr__(self):
1443         return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request))
1444
1445 class msg_blocktxn(object):
1446     command = b"blocktxn"
1447
1448     def __init__(self):
1449         self.block_transactions = BlockTransactions()
1450
1451     def deserialize(self, f):
1452         self.block_transactions.deserialize(f)
1453
1454     def serialize(self):
1455         r = b""
1456         r += self.block_transactions.serialize()
1457         return r
1458
1459     def __repr__(self):
1460         return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions))
1461
1462 class msg_witness_blocktxn(msg_blocktxn):
1463     def serialize(self):
1464         r = b""
1465         r += self.block_transactions.serialize(with_witness=True)
1466         return r
1467
1468 # This is what a callback should look like for NodeConn
1469 # Reimplement the on_* functions to provide handling for events
1470 class NodeConnCB(object):
1471     def __init__(self):
1472         self.verack_received = False
1473         # deliver_sleep_time is helpful for debugging race conditions in p2p
1474         # tests; it causes message delivery to sleep for the specified time
1475         # before acquiring the global lock and delivering the next message.
1476         self.deliver_sleep_time = None
1477         # Remember the services our peer has advertised
1478         self.peer_services = None
1479         self.connection = None
1480         self.ping_counter = 1
1481         self.last_pong = msg_pong()
1482
1483     def deliver(self, conn, message):
1484         deliver_sleep = self.get_deliver_sleep_time()
1485         if deliver_sleep is not None:
1486             time.sleep(deliver_sleep)
1487         with mininode_lock:
1488             try:
1489                 getattr(self, 'on_' + message.command.decode('ascii'))(conn, message)
1490             except:
1491                 logger.exception("ERROR delivering %s" % repr(message))
1492
1493     def set_deliver_sleep_time(self, value):
1494         with mininode_lock:
1495             self.deliver_sleep_time = value
1496
1497     def get_deliver_sleep_time(self):
1498         with mininode_lock:
1499             return self.deliver_sleep_time
1500
1501     # Callbacks which can be overridden by subclasses
1502     #################################################
1503
1504     def on_addr(self, conn, message): pass
1505     def on_alert(self, conn, message): pass
1506     def on_block(self, conn, message): pass
1507     def on_blocktxn(self, conn, message): pass
1508     def on_close(self, conn): pass
1509     def on_cmpctblock(self, conn, message): pass
1510     def on_feefilter(self, conn, message): pass
1511     def on_getaddr(self, conn, message): pass
1512     def on_getblocks(self, conn, message): pass
1513     def on_getblocktxn(self, conn, message): pass
1514     def on_getdata(self, conn, message): pass
1515     def on_getheaders(self, conn, message): pass
1516     def on_headers(self, conn, message): pass
1517     def on_mempool(self, conn): pass
1518     def on_open(self, conn): pass
1519     def on_reject(self, conn, message): pass
1520     def on_sendcmpct(self, conn, message): pass
1521     def on_sendheaders(self, conn, message): pass
1522     def on_tx(self, conn, message): pass
1523
1524     def on_inv(self, conn, message):
1525         want = msg_getdata()
1526         for i in message.inv:
1527             if i.type != 0:
1528                 want.inv.append(i)
1529         if len(want.inv):
1530             conn.send_message(want)
1531
1532     def on_ping(self, conn, message):
1533         if conn.ver_send > BIP0031_VERSION:
1534             conn.send_message(msg_pong(message.nonce))
1535
1536     def on_pong(self, conn, message):
1537         self.last_pong = message
1538
1539     def on_verack(self, conn, message):
1540         conn.ver_recv = conn.ver_send
1541         self.verack_received = True
1542
1543     def on_version(self, conn, message):
1544         if message.nVersion >= 209:
1545             conn.send_message(msg_verack())
1546         conn.ver_send = min(MY_VERSION, message.nVersion)
1547         if message.nVersion < 209:
1548             conn.ver_recv = conn.ver_send
1549         conn.nServices = message.nServices
1550
1551     # Helper functions
1552     ##################
1553
1554     def add_connection(self, conn):
1555         self.connection = conn
1556
1557     # Wrapper for the NodeConn's send_message function
1558     def send_message(self, message):
1559         self.connection.send_message(message)
1560
1561     def send_and_ping(self, message):
1562         self.send_message(message)
1563         self.sync_with_ping()
1564
1565     # Sync up with the node
1566     def sync_with_ping(self, timeout=60):
1567         def received_pong():
1568             return (self.last_pong.nonce == self.ping_counter)
1569         self.send_message(msg_ping(nonce=self.ping_counter))
1570         success = wait_until(received_pong, timeout=timeout)
1571         if not success:
1572             logger.error("sync_with_ping failed!")
1573             raise AssertionError("sync_with_ping failed!")
1574         self.ping_counter += 1
1575
1576         return success
1577
1578     # Spin until verack message is received from the node.
1579     # Tests may want to use this as a signal that the test can begin.
1580     # This can be called from the testing thread, so it needs to acquire the
1581     # global lock.
1582     def wait_for_verack(self):
1583         while True:
1584             with mininode_lock:
1585                 if self.verack_received:
1586                     return
1587             time.sleep(0.05)
1588
1589 # The actual NodeConn class
1590 # This class provides an interface for a p2p connection to a specified node
1591 class NodeConn(asyncore.dispatcher):
1592     messagemap = {
1593         b"version": msg_version,
1594         b"verack": msg_verack,
1595         b"addr": msg_addr,
1596         b"alert": msg_alert,
1597         b"inv": msg_inv,
1598         b"getdata": msg_getdata,
1599         b"getblocks": msg_getblocks,
1600         b"tx": msg_tx,
1601         b"block": msg_block,
1602         b"getaddr": msg_getaddr,
1603         b"ping": msg_ping,
1604         b"pong": msg_pong,
1605         b"headers": msg_headers,
1606         b"getheaders": msg_getheaders,
1607         b"reject": msg_reject,
1608         b"mempool": msg_mempool,
1609         b"feefilter": msg_feefilter,
1610         b"sendheaders": msg_sendheaders,
1611         b"sendcmpct": msg_sendcmpct,
1612         b"cmpctblock": msg_cmpctblock,
1613         b"getblocktxn": msg_getblocktxn,
1614         b"blocktxn": msg_blocktxn
1615     }
1616     MAGIC_BYTES = {
1617         "mainnet": b"\xf9\xbe\xb4\xd9",   # mainnet
1618         "testnet3": b"\x0b\x11\x09\x07",  # testnet3
1619         "regtest": b"\xfa\xbf\xb5\xda",   # regtest
1620     }
1621
1622     def __init__(self, dstaddr, dstport, rpc, callback, net="regtest", services=NODE_NETWORK, send_version=True):
1623         asyncore.dispatcher.__init__(self, map=mininode_socket_map)
1624         self.dstaddr = dstaddr
1625         self.dstport = dstport
1626         self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
1627         self.sendbuf = b""
1628         self.recvbuf = b""
1629         self.ver_send = 209
1630         self.ver_recv = 209
1631         self.last_sent = 0
1632         self.state = "connecting"
1633         self.network = net
1634         self.cb = callback
1635         self.disconnect = False
1636         self.nServices = 0
1637
1638         if send_version:
1639             # stuff version msg into sendbuf
1640             vt = msg_version()
1641             vt.nServices = services
1642             vt.addrTo.ip = self.dstaddr
1643             vt.addrTo.port = self.dstport
1644             vt.addrFrom.ip = "0.0.0.0"
1645             vt.addrFrom.port = 0
1646             self.send_message(vt, True)
1647
1648         logger.info('Connecting to Bitcoin Node: %s:%d' % (self.dstaddr, self.dstport))
1649
1650         try:
1651             self.connect((dstaddr, dstport))
1652         except:
1653             self.handle_close()
1654         self.rpc = rpc
1655
1656     def handle_connect(self):
1657         if self.state != "connected":
1658             logger.debug("Connected & Listening: %s:%d" % (self.dstaddr, self.dstport))
1659             self.state = "connected"
1660             self.cb.on_open(self)
1661
1662     def handle_close(self):
1663         logger.debug("Closing connection to: %s:%d" % (self.dstaddr, self.dstport))
1664         self.state = "closed"
1665         self.recvbuf = b""
1666         self.sendbuf = b""
1667         try:
1668             self.close()
1669         except:
1670             pass
1671         self.cb.on_close(self)
1672
1673     def handle_read(self):
1674         try:
1675             t = self.recv(8192)
1676             if len(t) > 0:
1677                 self.recvbuf += t
1678                 self.got_data()
1679         except:
1680             pass
1681
1682     def readable(self):
1683         return True
1684
1685     def writable(self):
1686         with mininode_lock:
1687             pre_connection = self.state == "connecting"
1688             length = len(self.sendbuf)
1689         return (length > 0 or pre_connection)
1690
1691     def handle_write(self):
1692         with mininode_lock:
1693             # asyncore does not expose socket connection, only the first read/write
1694             # event, thus we must check connection manually here to know when we
1695             # actually connect
1696             if self.state == "connecting":
1697                 self.handle_connect()
1698             if not self.writable():
1699                 return
1700
1701             try:
1702                 sent = self.send(self.sendbuf)
1703             except:
1704                 self.handle_close()
1705                 return
1706             self.sendbuf = self.sendbuf[sent:]
1707
1708     def got_data(self):
1709         try:
1710             while True:
1711                 if len(self.recvbuf) < 4:
1712                     return
1713                 if self.recvbuf[:4] != self.MAGIC_BYTES[self.network]:
1714                     raise ValueError("got garbage %s" % repr(self.recvbuf))
1715                 if self.ver_recv < 209:
1716                     if len(self.recvbuf) < 4 + 12 + 4:
1717                         return
1718                     command = self.recvbuf[4:4+12].split(b"\x00", 1)[0]
1719                     msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0]
1720                     checksum = None
1721                     if len(self.recvbuf) < 4 + 12 + 4 + msglen:
1722                         return
1723                     msg = self.recvbuf[4+12+4:4+12+4+msglen]
1724                     self.recvbuf = self.recvbuf[4+12+4+msglen:]
1725                 else:
1726                     if len(self.recvbuf) < 4 + 12 + 4 + 4:
1727                         return
1728                     command = self.recvbuf[4:4+12].split(b"\x00", 1)[0]
1729                     msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0]
1730                     checksum = self.recvbuf[4+12+4:4+12+4+4]
1731                     if len(self.recvbuf) < 4 + 12 + 4 + 4 + msglen:
1732                         return
1733                     msg = self.recvbuf[4+12+4+4:4+12+4+4+msglen]
1734                     th = sha256(msg)
1735                     h = sha256(th)
1736                     if checksum != h[:4]:
1737                         raise ValueError("got bad checksum " + repr(self.recvbuf))
1738                     self.recvbuf = self.recvbuf[4+12+4+4+msglen:]
1739                 if command in self.messagemap:
1740                     f = BytesIO(msg)
1741                     t = self.messagemap[command]()
1742                     t.deserialize(f)
1743                     self.got_message(t)
1744                 else:
1745                     logger.warning("Received unknown command from %s:%d: '%s' %s" % (self.dstaddr, self.dstport, command, repr(msg)))
1746         except Exception as e:
1747             logger.exception('got_data:', repr(e))
1748
1749     def send_message(self, message, pushbuf=False):
1750         if self.state != "connected" and not pushbuf:
1751             raise IOError('Not connected, no pushbuf')
1752         logger.debug("Send message to %s:%d: %s" % (self.dstaddr, self.dstport, repr(message)))
1753         command = message.command
1754         data = message.serialize()
1755         tmsg = self.MAGIC_BYTES[self.network]
1756         tmsg += command
1757         tmsg += b"\x00" * (12 - len(command))
1758         tmsg += struct.pack("<I", len(data))
1759         if self.ver_send >= 209:
1760             th = sha256(data)
1761             h = sha256(th)
1762             tmsg += h[:4]
1763         tmsg += data
1764         with mininode_lock:
1765             self.sendbuf += tmsg
1766             self.last_sent = time.time()
1767
1768     def got_message(self, message):
1769         if message.command == b"version":
1770             if message.nVersion <= BIP0031_VERSION:
1771                 self.messagemap[b'ping'] = msg_ping_prebip31
1772         if self.last_sent + 30 * 60 < time.time():
1773             self.send_message(self.messagemap[b'ping']())
1774         logger.debug("Received message from %s:%d: %s" % (self.dstaddr, self.dstport, repr(message)))
1775         self.cb.deliver(self, message)
1776
1777     def disconnect_node(self):
1778         self.disconnect = True
1779
1780
1781 class NetworkThread(Thread):
1782     def run(self):
1783         while mininode_socket_map:
1784             # We check for whether to disconnect outside of the asyncore
1785             # loop to workaround the behavior of asyncore when using
1786             # select
1787             disconnected = []
1788             for fd, obj in mininode_socket_map.items():
1789                 if obj.disconnect:
1790                     disconnected.append(obj)
1791             [ obj.handle_close() for obj in disconnected ]
1792             asyncore.loop(0.1, use_poll=True, map=mininode_socket_map, count=1)
1793
1794
1795 # An exception we can raise if we detect a potential disconnect
1796 # (p2p or rpc) before the test is complete
1797 class EarlyDisconnectError(Exception):
1798     def __init__(self, value):
1799         self.value = value
1800
1801     def __str__(self):
1802         return repr(self.value)