test/functional/test_framework/mininode.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2010 ArtForz -- public domain half-a-node
   3 # Copyright (c) 2012 Jeff Garzik
   4 # Copyright (c) 2010-2016 The Bitcoin Core developers
   5 # Distributed under the MIT software license, see the accompanying
   6 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   7 """Bitcoin P2P network half-a-node.
   8
   9 This python code was modified from ArtForz' public domain  half-a-node, as
  10 found in the mini-node branch of http://github.com/jgarzik/pynode.
  11
  12 NodeConn: an object which manages p2p connectivity to a bitcoin node
  13 NodeConnCB: a base class that describes the interface for receiving
  14             callbacks with network messages from a NodeConn
  15 CBlock, CTransaction, CBlockHeader, CTxIn, CTxOut, etc....:
  16     data structures that should map to corresponding structures in
  17     bitcoin/primitives
  18 msg_block, msg_tx, msg_headers, etc.:
  19     data structures that represent network messages
  20 ser_*, deser_*: functions that handle serialization/deserialization
  21 """
  22
  23 import asyncore
  24 from codecs import encode
  25 from collections import defaultdict
  26 import copy
  27 import hashlib
  28 from io import BytesIO
  29 import logging
  30 import random
  31 import socket
  32 import struct
  33 import sys
  34 import time
  35 from threading import RLock, Thread
  36
  37 from test_framework.siphash import siphash256
  38 from test_framework.util import hex_str_to_bytes, bytes_to_hex_str, wait_until
  39
  40 BIP0031_VERSION = 60000
  41 MY_VERSION = 70014  # past bip-31 for ping/pong
  42 MY_SUBVERSION = b"/python-mininode-tester:0.0.3/"
  43 MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37)
  44
  45 MAX_INV_SZ = 50000
  46 MAX_BLOCK_BASE_SIZE = 1000000
  47
  48 COIN = 100000000 # 1 btc in satoshis
  49
  50 NODE_NETWORK = (1 << 0)
  51 # NODE_GETUTXO = (1 << 1)
  52 # NODE_BLOOM = (1 << 2)
  53 NODE_WITNESS = (1 << 3)
  54 NODE_UNSUPPORTED_SERVICE_BIT_5 = (1 << 5)
  55 NODE_UNSUPPORTED_SERVICE_BIT_7 = (1 << 7)
  56
  57 logger = logging.getLogger("TestFramework.mininode")
  58
  59 # Keep our own socket map for asyncore, so that we can track disconnects
  60 # ourselves (to workaround an issue with closing an asyncore socket when
  61 # using select)
  62 mininode_socket_map = dict()
  63
  64 # One lock for synchronizing all data access between the networking thread (see
  65 # NetworkThread below) and the thread running the test logic.  For simplicity,
  66 # NodeConn acquires this lock whenever delivering a message to a NodeConnCB,
  67 # and whenever adding anything to the send buffer (in send_message()).  This
  68 # lock should be acquired in the thread running the test logic to synchronize
  69 # access to any data shared with the NodeConnCB or NodeConn.
  70 mininode_lock = RLock()
  71
  72 # Serialization/deserialization tools
  73 def sha256(s):
  74     return hashlib.new('sha256', s).digest()
  75
  76 def ripemd160(s):
  77     return hashlib.new('ripemd160', s).digest()
  78
  79 def hash256(s):
  80     return sha256(sha256(s))
  81
  82 def ser_compact_size(l):
  83     r = b""
  84     if l < 253:
  85         r = struct.pack("B", l)
  86     elif l < 0x10000:
  87         r = struct.pack("<BH", 253, l)
  88     elif l < 0x100000000:
  89         r = struct.pack("<BI", 254, l)
  90     else:
  91         r = struct.pack("<BQ", 255, l)
  92     return r
  93
  94 def deser_compact_size(f):
  95     nit = struct.unpack("<B", f.read(1))[0]
  96     if nit == 253:
  97         nit = struct.unpack("<H", f.read(2))[0]
  98     elif nit == 254:
  99         nit = struct.unpack("<I", f.read(4))[0]
 100     elif nit == 255:
 101         nit = struct.unpack("<Q", f.read(8))[0]
 102     return nit
 103
 104 def deser_string(f):
 105     nit = deser_compact_size(f)
 106     return f.read(nit)
 107
 108 def ser_string(s):
 109     return ser_compact_size(len(s)) + s
 110
 111 def deser_uint256(f):
 112     r = 0
 113     for i in range(8):
 114         t = struct.unpack("<I", f.read(4))[0]
 115         r += t << (i * 32)
 116     return r
 117
 118
 119 def ser_uint256(u):
 120     rs = b""
 121     for i in range(8):
 122         rs += struct.pack("<I", u & 0xFFFFFFFF)
 123         u >>= 32
 124     return rs
 125
 126
 127 def uint256_from_str(s):
 128     r = 0
 129     t = struct.unpack("<IIIIIIII", s[:32])
 130     for i in range(8):
 131         r += t[i] << (i * 32)
 132     return r
 133
 134
 135 def uint256_from_compact(c):
 136     nbytes = (c >> 24) & 0xFF
 137     v = (c & 0xFFFFFF) << (8 * (nbytes - 3))
 138     return v
 139
 140
 141 def deser_vector(f, c):
 142     nit = deser_compact_size(f)
 143     r = []
 144     for i in range(nit):
 145         t = c()
 146         t.deserialize(f)
 147         r.append(t)
 148     return r
 149
 150
 151 # ser_function_name: Allow for an alternate serialization function on the
 152 # entries in the vector (we use this for serializing the vector of transactions
 153 # for a witness block).
 154 def ser_vector(l, ser_function_name=None):
 155     r = ser_compact_size(len(l))
 156     for i in l:
 157         if ser_function_name:
 158             r += getattr(i, ser_function_name)()
 159         else:
 160             r += i.serialize()
 161     return r
 162
 163
 164 def deser_uint256_vector(f):
 165     nit = deser_compact_size(f)
 166     r = []
 167     for i in range(nit):
 168         t = deser_uint256(f)
 169         r.append(t)
 170     return r
 171
 172
 173 def ser_uint256_vector(l):
 174     r = ser_compact_size(len(l))
 175     for i in l:
 176         r += ser_uint256(i)
 177     return r
 178
 179
 180 def deser_string_vector(f):
 181     nit = deser_compact_size(f)
 182     r = []
 183     for i in range(nit):
 184         t = deser_string(f)
 185         r.append(t)
 186     return r
 187
 188
 189 def ser_string_vector(l):
 190     r = ser_compact_size(len(l))
 191     for sv in l:
 192         r += ser_string(sv)
 193     return r
 194
 195
 196 def deser_int_vector(f):
 197     nit = deser_compact_size(f)
 198     r = []
 199     for i in range(nit):
 200         t = struct.unpack("<i", f.read(4))[0]
 201         r.append(t)
 202     return r
 203
 204
 205 def ser_int_vector(l):
 206     r = ser_compact_size(len(l))
 207     for i in l:
 208         r += struct.pack("<i", i)
 209     return r
 210
 211 # Deserialize from a hex string representation (eg from RPC)
 212 def FromHex(obj, hex_string):
 213     obj.deserialize(BytesIO(hex_str_to_bytes(hex_string)))
 214     return obj
 215
 216 # Convert a binary-serializable object to hex (eg for submission via RPC)
 217 def ToHex(obj):
 218     return bytes_to_hex_str(obj.serialize())
 219
 220 # Objects that map to bitcoind objects, which can be serialized/deserialized
 221
 222 class CAddress():
 223     def __init__(self):
 224         self.nServices = 1
 225         self.pchReserved = b"\x00" * 10 + b"\xff" * 2
 226         self.ip = "0.0.0.0"
 227         self.port = 0
 228
 229     def deserialize(self, f):
 230         self.nServices = struct.unpack("<Q", f.read(8))[0]
 231         self.pchReserved = f.read(12)
 232         self.ip = socket.inet_ntoa(f.read(4))
 233         self.port = struct.unpack(">H", f.read(2))[0]
 234
 235     def serialize(self):
 236         r = b""
 237         r += struct.pack("<Q", self.nServices)
 238         r += self.pchReserved
 239         r += socket.inet_aton(self.ip)
 240         r += struct.pack(">H", self.port)
 241         return r
 242
 243     def __repr__(self):
 244         return "CAddress(nServices=%i ip=%s port=%i)" % (self.nServices,
 245                                                          self.ip, self.port)
 246
 247 MSG_WITNESS_FLAG = 1<<30
 248
 249 class CInv():
 250     typemap = {
 251         0: "Error",
 252         1: "TX",
 253         2: "Block",
 254         1|MSG_WITNESS_FLAG: "WitnessTx",
 255         2|MSG_WITNESS_FLAG : "WitnessBlock",
 256         4: "CompactBlock"
 257     }
 258
 259     def __init__(self, t=0, h=0):
 260         self.type = t
 261         self.hash = h
 262
 263     def deserialize(self, f):
 264         self.type = struct.unpack("<i", f.read(4))[0]
 265         self.hash = deser_uint256(f)
 266
 267     def serialize(self):
 268         r = b""
 269         r += struct.pack("<i", self.type)
 270         r += ser_uint256(self.hash)
 271         return r
 272
 273     def __repr__(self):
 274         return "CInv(type=%s hash=%064x)" \
 275             % (self.typemap[self.type], self.hash)
 276
 277
 278 class CBlockLocator():
 279     def __init__(self):
 280         self.nVersion = MY_VERSION
 281         self.vHave = []
 282
 283     def deserialize(self, f):
 284         self.nVersion = struct.unpack("<i", f.read(4))[0]
 285         self.vHave = deser_uint256_vector(f)
 286
 287     def serialize(self):
 288         r = b""
 289         r += struct.pack("<i", self.nVersion)
 290         r += ser_uint256_vector(self.vHave)
 291         return r
 292
 293     def __repr__(self):
 294         return "CBlockLocator(nVersion=%i vHave=%s)" \
 295             % (self.nVersion, repr(self.vHave))
 296
 297
 298 class COutPoint():
 299     def __init__(self, hash=0, n=0):
 300         self.hash = hash
 301         self.n = n
 302
 303     def deserialize(self, f):
 304         self.hash = deser_uint256(f)
 305         self.n = struct.unpack("<I", f.read(4))[0]
 306
 307     def serialize(self):
 308         r = b""
 309         r += ser_uint256(self.hash)
 310         r += struct.pack("<I", self.n)
 311         return r
 312
 313     def __repr__(self):
 314         return "COutPoint(hash=%064x n=%i)" % (self.hash, self.n)
 315
 316
 317 class CTxIn():
 318     def __init__(self, outpoint=None, scriptSig=b"", nSequence=0):
 319         if outpoint is None:
 320             self.prevout = COutPoint()
 321         else:
 322             self.prevout = outpoint
 323         self.scriptSig = scriptSig
 324         self.nSequence = nSequence
 325
 326     def deserialize(self, f):
 327         self.prevout = COutPoint()
 328         self.prevout.deserialize(f)
 329         self.scriptSig = deser_string(f)
 330         self.nSequence = struct.unpack("<I", f.read(4))[0]
 331
 332     def serialize(self):
 333         r = b""
 334         r += self.prevout.serialize()
 335         r += ser_string(self.scriptSig)
 336         r += struct.pack("<I", self.nSequence)
 337         return r
 338
 339     def __repr__(self):
 340         return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \
 341             % (repr(self.prevout), bytes_to_hex_str(self.scriptSig),
 342                self.nSequence)
 343
 344
 345 class CTxOut():
 346     def __init__(self, nValue=0, scriptPubKey=b""):
 347         self.nValue = nValue
 348         self.scriptPubKey = scriptPubKey
 349
 350     def deserialize(self, f):
 351         self.nValue = struct.unpack("<q", f.read(8))[0]
 352         self.scriptPubKey = deser_string(f)
 353
 354     def serialize(self):
 355         r = b""
 356         r += struct.pack("<q", self.nValue)
 357         r += ser_string(self.scriptPubKey)
 358         return r
 359
 360     def __repr__(self):
 361         return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \
 362             % (self.nValue // COIN, self.nValue % COIN,
 363                bytes_to_hex_str(self.scriptPubKey))
 364
 365
 366 class CScriptWitness():
 367     def __init__(self):
 368         # stack is a vector of strings
 369         self.stack = []
 370
 371     def __repr__(self):
 372         return "CScriptWitness(%s)" % \
 373                (",".join([bytes_to_hex_str(x) for x in self.stack]))
 374
 375     def is_null(self):
 376         if self.stack:
 377             return False
 378         return True
 379
 380
 381 class CTxInWitness():
 382     def __init__(self):
 383         self.scriptWitness = CScriptWitness()
 384
 385     def deserialize(self, f):
 386         self.scriptWitness.stack = deser_string_vector(f)
 387
 388     def serialize(self):
 389         return ser_string_vector(self.scriptWitness.stack)
 390
 391     def __repr__(self):
 392         return repr(self.scriptWitness)
 393
 394     def is_null(self):
 395         return self.scriptWitness.is_null()
 396
 397
 398 class CTxWitness():
 399     def __init__(self):
 400         self.vtxinwit = []
 401
 402     def deserialize(self, f):
 403         for i in range(len(self.vtxinwit)):
 404             self.vtxinwit[i].deserialize(f)
 405
 406     def serialize(self):
 407         r = b""
 408         # This is different than the usual vector serialization --
 409         # we omit the length of the vector, which is required to be
 410         # the same length as the transaction's vin vector.
 411         for x in self.vtxinwit:
 412             r += x.serialize()
 413         return r
 414
 415     def __repr__(self):
 416         return "CTxWitness(%s)" % \
 417                (';'.join([repr(x) for x in self.vtxinwit]))
 418
 419     def is_null(self):
 420         for x in self.vtxinwit:
 421             if not x.is_null():
 422                 return False
 423         return True
 424
 425
 426 class CTransaction():
 427     def __init__(self, tx=None):
 428         if tx is None:
 429             self.nVersion = 1
 430             self.vin = []
 431             self.vout = []
 432             self.wit = CTxWitness()
 433             self.nLockTime = 0
 434             self.sha256 = None
 435             self.hash = None
 436         else:
 437             self.nVersion = tx.nVersion
 438             self.vin = copy.deepcopy(tx.vin)
 439             self.vout = copy.deepcopy(tx.vout)
 440             self.nLockTime = tx.nLockTime
 441             self.sha256 = tx.sha256
 442             self.hash = tx.hash
 443             self.wit = copy.deepcopy(tx.wit)
 444
 445     def deserialize(self, f):
 446         self.nVersion = struct.unpack("<i", f.read(4))[0]
 447         self.vin = deser_vector(f, CTxIn)
 448         flags = 0
 449         if len(self.vin) == 0:
 450             flags = struct.unpack("<B", f.read(1))[0]
 451             # Not sure why flags can't be zero, but this
 452             # matches the implementation in bitcoind
 453             if (flags != 0):
 454                 self.vin = deser_vector(f, CTxIn)
 455                 self.vout = deser_vector(f, CTxOut)
 456         else:
 457             self.vout = deser_vector(f, CTxOut)
 458         if flags != 0:
 459             self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))]
 460             self.wit.deserialize(f)
 461         self.nLockTime = struct.unpack("<I", f.read(4))[0]
 462         self.sha256 = None
 463         self.hash = None
 464
 465     def serialize_without_witness(self):
 466         r = b""
 467         r += struct.pack("<i", self.nVersion)
 468         r += ser_vector(self.vin)
 469         r += ser_vector(self.vout)
 470         r += struct.pack("<I", self.nLockTime)
 471         return r
 472
 473     # Only serialize with witness when explicitly called for
 474     def serialize_with_witness(self):
 475         flags = 0
 476         if not self.wit.is_null():
 477             flags |= 1
 478         r = b""
 479         r += struct.pack("<i", self.nVersion)
 480         if flags:
 481             dummy = []
 482             r += ser_vector(dummy)
 483             r += struct.pack("<B", flags)
 484         r += ser_vector(self.vin)
 485         r += ser_vector(self.vout)
 486         if flags & 1:
 487             if (len(self.wit.vtxinwit) != len(self.vin)):
 488                 # vtxinwit must have the same length as vin
 489                 self.wit.vtxinwit = self.wit.vtxinwit[:len(self.vin)]
 490                 for i in range(len(self.wit.vtxinwit), len(self.vin)):
 491                     self.wit.vtxinwit.append(CTxInWitness())
 492             r += self.wit.serialize()
 493         r += struct.pack("<I", self.nLockTime)
 494         return r
 495
 496     # Regular serialization is without witness -- must explicitly
 497     # call serialize_with_witness to include witness data.
 498     def serialize(self):
 499         return self.serialize_without_witness()
 500
 501     # Recalculate the txid (transaction hash without witness)
 502     def rehash(self):
 503         self.sha256 = None
 504         self.calc_sha256()
 505
 506     # We will only cache the serialization without witness in
 507     # self.sha256 and self.hash -- those are expected to be the txid.
 508     def calc_sha256(self, with_witness=False):
 509         if with_witness:
 510             # Don't cache the result, just return it
 511             return uint256_from_str(hash256(self.serialize_with_witness()))
 512
 513         if self.sha256 is None:
 514             self.sha256 = uint256_from_str(hash256(self.serialize_without_witness()))
 515         self.hash = encode(hash256(self.serialize())[::-1], 'hex_codec').decode('ascii')
 516
 517     def is_valid(self):
 518         self.calc_sha256()
 519         for tout in self.vout:
 520             if tout.nValue < 0 or tout.nValue > 21000000 * COIN:
 521                 return False
 522         return True
 523
 524     def __repr__(self):
 525         return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \
 526             % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime)
 527
 528
 529 class CBlockHeader():
 530     def __init__(self, header=None):
 531         if header is None:
 532             self.set_null()
 533         else:
 534             self.nVersion = header.nVersion
 535             self.hashPrevBlock = header.hashPrevBlock
 536             self.hashMerkleRoot = header.hashMerkleRoot
 537             self.nTime = header.nTime
 538             self.nBits = header.nBits
 539             self.nNonce = header.nNonce
 540             self.sha256 = header.sha256
 541             self.hash = header.hash
 542             self.calc_sha256()
 543
 544     def set_null(self):
 545         self.nVersion = 1
 546         self.hashPrevBlock = 0
 547         self.hashMerkleRoot = 0
 548         self.nTime = 0
 549         self.nBits = 0
 550         self.nNonce = 0
 551         self.sha256 = None
 552         self.hash = None
 553
 554     def deserialize(self, f):
 555         self.nVersion = struct.unpack("<i", f.read(4))[0]
 556         self.hashPrevBlock = deser_uint256(f)
 557         self.hashMerkleRoot = deser_uint256(f)
 558         self.nTime = struct.unpack("<I", f.read(4))[0]
 559         self.nBits = struct.unpack("<I", f.read(4))[0]
 560         self.nNonce = struct.unpack("<I", f.read(4))[0]
 561         self.sha256 = None
 562         self.hash = None
 563
 564     def serialize(self):
 565         r = b""
 566         r += struct.pack("<i", self.nVersion)
 567         r += ser_uint256(self.hashPrevBlock)
 568         r += ser_uint256(self.hashMerkleRoot)
 569         r += struct.pack("<I", self.nTime)
 570         r += struct.pack("<I", self.nBits)
 571         r += struct.pack("<I", self.nNonce)
 572         return r
 573
 574     def calc_sha256(self):
 575         if self.sha256 is None:
 576             r = b""
 577             r += struct.pack("<i", self.nVersion)
 578             r += ser_uint256(self.hashPrevBlock)
 579             r += ser_uint256(self.hashMerkleRoot)
 580             r += struct.pack("<I", self.nTime)
 581             r += struct.pack("<I", self.nBits)
 582             r += struct.pack("<I", self.nNonce)
 583             self.sha256 = uint256_from_str(hash256(r))
 584             self.hash = encode(hash256(r)[::-1], 'hex_codec').decode('ascii')
 585
 586     def rehash(self):
 587         self.sha256 = None
 588         self.calc_sha256()
 589         return self.sha256
 590
 591     def __repr__(self):
 592         return "CBlockHeader(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x)" \
 593             % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
 594                time.ctime(self.nTime), self.nBits, self.nNonce)
 595
 596
 597 class CBlock(CBlockHeader):
 598     def __init__(self, header=None):
 599         super(CBlock, self).__init__(header)
 600         self.vtx = []
 601
 602     def deserialize(self, f):
 603         super(CBlock, self).deserialize(f)
 604         self.vtx = deser_vector(f, CTransaction)
 605
 606     def serialize(self, with_witness=False):
 607         r = b""
 608         r += super(CBlock, self).serialize()
 609         if with_witness:
 610             r += ser_vector(self.vtx, "serialize_with_witness")
 611         else:
 612             r += ser_vector(self.vtx)
 613         return r
 614
 615     # Calculate the merkle root given a vector of transaction hashes
 616     @classmethod
 617     def get_merkle_root(cls, hashes):
 618         while len(hashes) > 1:
 619             newhashes = []
 620             for i in range(0, len(hashes), 2):
 621                 i2 = min(i+1, len(hashes)-1)
 622                 newhashes.append(hash256(hashes[i] + hashes[i2]))
 623             hashes = newhashes
 624         return uint256_from_str(hashes[0])
 625
 626     def calc_merkle_root(self):
 627         hashes = []
 628         for tx in self.vtx:
 629             tx.calc_sha256()
 630             hashes.append(ser_uint256(tx.sha256))
 631         return self.get_merkle_root(hashes)
 632
 633     def calc_witness_merkle_root(self):
 634         # For witness root purposes, the hash of the
 635         # coinbase, with witness, is defined to be 0...0
 636         hashes = [ser_uint256(0)]
 637
 638         for tx in self.vtx[1:]:
 639             # Calculate the hashes with witness data
 640             hashes.append(ser_uint256(tx.calc_sha256(True)))
 641
 642         return self.get_merkle_root(hashes)
 643
 644     def is_valid(self):
 645         self.calc_sha256()
 646         target = uint256_from_compact(self.nBits)
 647         if self.sha256 > target:
 648             return False
 649         for tx in self.vtx:
 650             if not tx.is_valid():
 651                 return False
 652         if self.calc_merkle_root() != self.hashMerkleRoot:
 653             return False
 654         return True
 655
 656     def solve(self):
 657         self.rehash()
 658         target = uint256_from_compact(self.nBits)
 659         while self.sha256 > target:
 660             self.nNonce += 1
 661             self.rehash()
 662
 663     def __repr__(self):
 664         return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x vtx=%s)" \
 665             % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
 666                time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx))
 667
 668
 669 class CUnsignedAlert():
 670     def __init__(self):
 671         self.nVersion = 1
 672         self.nRelayUntil = 0
 673         self.nExpiration = 0
 674         self.nID = 0
 675         self.nCancel = 0
 676         self.setCancel = []
 677         self.nMinVer = 0
 678         self.nMaxVer = 0
 679         self.setSubVer = []
 680         self.nPriority = 0
 681         self.strComment = b""
 682         self.strStatusBar = b""
 683         self.strReserved = b""
 684
 685     def deserialize(self, f):
 686         self.nVersion = struct.unpack("<i", f.read(4))[0]
 687         self.nRelayUntil = struct.unpack("<q", f.read(8))[0]
 688         self.nExpiration = struct.unpack("<q", f.read(8))[0]
 689         self.nID = struct.unpack("<i", f.read(4))[0]
 690         self.nCancel = struct.unpack("<i", f.read(4))[0]
 691         self.setCancel = deser_int_vector(f)
 692         self.nMinVer = struct.unpack("<i", f.read(4))[0]
 693         self.nMaxVer = struct.unpack("<i", f.read(4))[0]
 694         self.setSubVer = deser_string_vector(f)
 695         self.nPriority = struct.unpack("<i", f.read(4))[0]
 696         self.strComment = deser_string(f)
 697         self.strStatusBar = deser_string(f)
 698         self.strReserved = deser_string(f)
 699
 700     def serialize(self):
 701         r = b""
 702         r += struct.pack("<i", self.nVersion)
 703         r += struct.pack("<q", self.nRelayUntil)
 704         r += struct.pack("<q", self.nExpiration)
 705         r += struct.pack("<i", self.nID)
 706         r += struct.pack("<i", self.nCancel)
 707         r += ser_int_vector(self.setCancel)
 708         r += struct.pack("<i", self.nMinVer)
 709         r += struct.pack("<i", self.nMaxVer)
 710         r += ser_string_vector(self.setSubVer)
 711         r += struct.pack("<i", self.nPriority)
 712         r += ser_string(self.strComment)
 713         r += ser_string(self.strStatusBar)
 714         r += ser_string(self.strReserved)
 715         return r
 716
 717     def __repr__(self):
 718         return "CUnsignedAlert(nVersion %d, nRelayUntil %d, nExpiration %d, nID %d, nCancel %d, nMinVer %d, nMaxVer %d, nPriority %d, strComment %s, strStatusBar %s, strReserved %s)" \
 719             % (self.nVersion, self.nRelayUntil, self.nExpiration, self.nID,
 720                self.nCancel, self.nMinVer, self.nMaxVer, self.nPriority,
 721                self.strComment, self.strStatusBar, self.strReserved)
 722
 723
 724 class CAlert():
 725     def __init__(self):
 726         self.vchMsg = b""
 727         self.vchSig = b""
 728
 729     def deserialize(self, f):
 730         self.vchMsg = deser_string(f)
 731         self.vchSig = deser_string(f)
 732
 733     def serialize(self):
 734         r = b""
 735         r += ser_string(self.vchMsg)
 736         r += ser_string(self.vchSig)
 737         return r
 738
 739     def __repr__(self):
 740         return "CAlert(vchMsg.sz %d, vchSig.sz %d)" \
 741             % (len(self.vchMsg), len(self.vchSig))
 742
 743
 744 class PrefilledTransaction():
 745     def __init__(self, index=0, tx = None):
 746         self.index = index
 747         self.tx = tx
 748
 749     def deserialize(self, f):
 750         self.index = deser_compact_size(f)
 751         self.tx = CTransaction()
 752         self.tx.deserialize(f)
 753
 754     def serialize(self, with_witness=False):
 755         r = b""
 756         r += ser_compact_size(self.index)
 757         if with_witness:
 758             r += self.tx.serialize_with_witness()
 759         else:
 760             r += self.tx.serialize_without_witness()
 761         return r
 762
 763     def serialize_with_witness(self):
 764         return self.serialize(with_witness=True)
 765
 766     def __repr__(self):
 767         return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx))
 768
 769 # This is what we send on the wire, in a cmpctblock message.
 770 class P2PHeaderAndShortIDs():
 771     def __init__(self):
 772         self.header = CBlockHeader()
 773         self.nonce = 0
 774         self.shortids_length = 0
 775         self.shortids = []
 776         self.prefilled_txn_length = 0
 777         self.prefilled_txn = []
 778
 779     def deserialize(self, f):
 780         self.header.deserialize(f)
 781         self.nonce = struct.unpack("<Q", f.read(8))[0]
 782         self.shortids_length = deser_compact_size(f)
 783         for i in range(self.shortids_length):
 784             # shortids are defined to be 6 bytes in the spec, so append
 785             # two zero bytes and read it in as an 8-byte number
 786             self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0])
 787         self.prefilled_txn = deser_vector(f, PrefilledTransaction)
 788         self.prefilled_txn_length = len(self.prefilled_txn)
 789
 790     # When using version 2 compact blocks, we must serialize with_witness.
 791     def serialize(self, with_witness=False):
 792         r = b""
 793         r += self.header.serialize()
 794         r += struct.pack("<Q", self.nonce)
 795         r += ser_compact_size(self.shortids_length)
 796         for x in self.shortids:
 797             # We only want the first 6 bytes
 798             r += struct.pack("<Q", x)[0:6]
 799         if with_witness:
 800             r += ser_vector(self.prefilled_txn, "serialize_with_witness")
 801         else:
 802             r += ser_vector(self.prefilled_txn)
 803         return r
 804
 805     def __repr__(self):
 806         return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn))
 807
 808 # P2P version of the above that will use witness serialization (for compact
 809 # block version 2)
 810 class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs):
 811     def serialize(self):
 812         return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True)
 813
 814 # Calculate the BIP 152-compact blocks shortid for a given transaction hash
 815 def calculate_shortid(k0, k1, tx_hash):
 816     expected_shortid = siphash256(k0, k1, tx_hash)
 817     expected_shortid &= 0x0000ffffffffffff
 818     return expected_shortid
 819
 820 # This version gets rid of the array lengths, and reinterprets the differential
 821 # encoding into indices that can be used for lookup.
 822 class HeaderAndShortIDs():
 823     def __init__(self, p2pheaders_and_shortids = None):
 824         self.header = CBlockHeader()
 825         self.nonce = 0
 826         self.shortids = []
 827         self.prefilled_txn = []
 828         self.use_witness = False
 829
 830         if p2pheaders_and_shortids != None:
 831             self.header = p2pheaders_and_shortids.header
 832             self.nonce = p2pheaders_and_shortids.nonce
 833             self.shortids = p2pheaders_and_shortids.shortids
 834             last_index = -1
 835             for x in p2pheaders_and_shortids.prefilled_txn:
 836                 self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx))
 837                 last_index = self.prefilled_txn[-1].index
 838
 839     def to_p2p(self):
 840         if self.use_witness:
 841             ret = P2PHeaderAndShortWitnessIDs()
 842         else:
 843             ret = P2PHeaderAndShortIDs()
 844         ret.header = self.header
 845         ret.nonce = self.nonce
 846         ret.shortids_length = len(self.shortids)
 847         ret.shortids = self.shortids
 848         ret.prefilled_txn_length = len(self.prefilled_txn)
 849         ret.prefilled_txn = []
 850         last_index = -1
 851         for x in self.prefilled_txn:
 852             ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx))
 853             last_index = x.index
 854         return ret
 855
 856     def get_siphash_keys(self):
 857         header_nonce = self.header.serialize()
 858         header_nonce += struct.pack("<Q", self.nonce)
 859         hash_header_nonce_as_str = sha256(header_nonce)
 860         key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0]
 861         key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0]
 862         return [ key0, key1 ]
 863
 864     # Version 2 compact blocks use wtxid in shortids (rather than txid)
 865     def initialize_from_block(self, block, nonce=0, prefill_list = [0], use_witness = False):
 866         self.header = CBlockHeader(block)
 867         self.nonce = nonce
 868         self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ]
 869         self.shortids = []
 870         self.use_witness = use_witness
 871         [k0, k1] = self.get_siphash_keys()
 872         for i in range(len(block.vtx)):
 873             if i not in prefill_list:
 874                 tx_hash = block.vtx[i].sha256
 875                 if use_witness:
 876                     tx_hash = block.vtx[i].calc_sha256(with_witness=True)
 877                 self.shortids.append(calculate_shortid(k0, k1, tx_hash))
 878
 879     def __repr__(self):
 880         return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn))
 881
 882
 883 class BlockTransactionsRequest():
 884
 885     def __init__(self, blockhash=0, indexes = None):
 886         self.blockhash = blockhash
 887         self.indexes = indexes if indexes != None else []
 888
 889     def deserialize(self, f):
 890         self.blockhash = deser_uint256(f)
 891         indexes_length = deser_compact_size(f)
 892         for i in range(indexes_length):
 893             self.indexes.append(deser_compact_size(f))
 894
 895     def serialize(self):
 896         r = b""
 897         r += ser_uint256(self.blockhash)
 898         r += ser_compact_size(len(self.indexes))
 899         for x in self.indexes:
 900             r += ser_compact_size(x)
 901         return r
 902
 903     # helper to set the differentially encoded indexes from absolute ones
 904     def from_absolute(self, absolute_indexes):
 905         self.indexes = []
 906         last_index = -1
 907         for x in absolute_indexes:
 908             self.indexes.append(x-last_index-1)
 909             last_index = x
 910
 911     def to_absolute(self):
 912         absolute_indexes = []
 913         last_index = -1
 914         for x in self.indexes:
 915             absolute_indexes.append(x+last_index+1)
 916             last_index = absolute_indexes[-1]
 917         return absolute_indexes
 918
 919     def __repr__(self):
 920         return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes))
 921
 922
 923 class BlockTransactions():
 924
 925     def __init__(self, blockhash=0, transactions = None):
 926         self.blockhash = blockhash
 927         self.transactions = transactions if transactions != None else []
 928
 929     def deserialize(self, f):
 930         self.blockhash = deser_uint256(f)
 931         self.transactions = deser_vector(f, CTransaction)
 932
 933     def serialize(self, with_witness=False):
 934         r = b""
 935         r += ser_uint256(self.blockhash)
 936         if with_witness:
 937             r += ser_vector(self.transactions, "serialize_with_witness")
 938         else:
 939             r += ser_vector(self.transactions)
 940         return r
 941
 942     def __repr__(self):
 943         return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions))
 944
 945
 946 # Objects that correspond to messages on the wire
 947 class msg_version():
 948     command = b"version"
 949
 950     def __init__(self):
 951         self.nVersion = MY_VERSION
 952         self.nServices = NODE_NETWORK | NODE_WITNESS
 953         self.nTime = int(time.time())
 954         self.addrTo = CAddress()
 955         self.addrFrom = CAddress()
 956         self.nNonce = random.getrandbits(64)
 957         self.strSubVer = MY_SUBVERSION
 958         self.nStartingHeight = -1
 959         self.nRelay = MY_RELAY
 960
 961     def deserialize(self, f):
 962         self.nVersion = struct.unpack("<i", f.read(4))[0]
 963         if self.nVersion == 10300:
 964             self.nVersion = 300
 965         self.nServices = struct.unpack("<Q", f.read(8))[0]
 966         self.nTime = struct.unpack("<q", f.read(8))[0]
 967         self.addrTo = CAddress()
 968         self.addrTo.deserialize(f)
 969
 970         if self.nVersion >= 106:
 971             self.addrFrom = CAddress()
 972             self.addrFrom.deserialize(f)
 973             self.nNonce = struct.unpack("<Q", f.read(8))[0]
 974             self.strSubVer = deser_string(f)
 975         else:
 976             self.addrFrom = None
 977             self.nNonce = None
 978             self.strSubVer = None
 979             self.nStartingHeight = None
 980
 981         if self.nVersion >= 209:
 982             self.nStartingHeight = struct.unpack("<i", f.read(4))[0]
 983         else:
 984             self.nStartingHeight = None
 985
 986         if self.nVersion >= 70001:
 987             # Relay field is optional for version 70001 onwards
 988             try:
 989                 self.nRelay = struct.unpack("<b", f.read(1))[0]
 990             except:
 991                 self.nRelay = 0
 992         else:
 993             self.nRelay = 0
 994
 995     def serialize(self):
 996         r = b""
 997         r += struct.pack("<i", self.nVersion)
 998         r += struct.pack("<Q", self.nServices)
 999         r += struct.pack("<q", self.nTime)
1000         r += self.addrTo.serialize()
1001         r += self.addrFrom.serialize()
1002         r += struct.pack("<Q", self.nNonce)
1003         r += ser_string(self.strSubVer)
1004         r += struct.pack("<i", self.nStartingHeight)
1005         r += struct.pack("<b", self.nRelay)
1006         return r
1007
1008     def __repr__(self):
1009         return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i nRelay=%i)' \
1010             % (self.nVersion, self.nServices, time.ctime(self.nTime),
1011                repr(self.addrTo), repr(self.addrFrom), self.nNonce,
1012                self.strSubVer, self.nStartingHeight, self.nRelay)
1013
1014
1015 class msg_verack():
1016     command = b"verack"
1017
1018     def __init__(self):
1019         pass
1020
1021     def deserialize(self, f):
1022         pass
1023
1024     def serialize(self):
1025         return b""
1026
1027     def __repr__(self):
1028         return "msg_verack()"
1029
1030
1031 class msg_addr():
1032     command = b"addr"
1033
1034     def __init__(self):
1035         self.addrs = []
1036
1037     def deserialize(self, f):
1038         self.addrs = deser_vector(f, CAddress)
1039
1040     def serialize(self):
1041         return ser_vector(self.addrs)
1042
1043     def __repr__(self):
1044         return "msg_addr(addrs=%s)" % (repr(self.addrs))
1045
1046
1047 class msg_alert():
1048     command = b"alert"
1049
1050     def __init__(self):
1051         self.alert = CAlert()
1052
1053     def deserialize(self, f):
1054         self.alert = CAlert()
1055         self.alert.deserialize(f)
1056
1057     def serialize(self):
1058         r = b""
1059         r += self.alert.serialize()
1060         return r
1061
1062     def __repr__(self):
1063         return "msg_alert(alert=%s)" % (repr(self.alert), )
1064
1065
1066 class msg_inv():
1067     command = b"inv"
1068
1069     def __init__(self, inv=None):
1070         if inv is None:
1071             self.inv = []
1072         else:
1073             self.inv = inv
1074
1075     def deserialize(self, f):
1076         self.inv = deser_vector(f, CInv)
1077
1078     def serialize(self):
1079         return ser_vector(self.inv)
1080
1081     def __repr__(self):
1082         return "msg_inv(inv=%s)" % (repr(self.inv))
1083
1084
1085 class msg_getdata():
1086     command = b"getdata"
1087
1088     def __init__(self, inv=None):
1089         self.inv = inv if inv != None else []
1090
1091     def deserialize(self, f):
1092         self.inv = deser_vector(f, CInv)
1093
1094     def serialize(self):
1095         return ser_vector(self.inv)
1096
1097     def __repr__(self):
1098         return "msg_getdata(inv=%s)" % (repr(self.inv))
1099
1100
1101 class msg_getblocks():
1102     command = b"getblocks"
1103
1104     def __init__(self):
1105         self.locator = CBlockLocator()
1106         self.hashstop = 0
1107
1108     def deserialize(self, f):
1109         self.locator = CBlockLocator()
1110         self.locator.deserialize(f)
1111         self.hashstop = deser_uint256(f)
1112
1113     def serialize(self):
1114         r = b""
1115         r += self.locator.serialize()
1116         r += ser_uint256(self.hashstop)
1117         return r
1118
1119     def __repr__(self):
1120         return "msg_getblocks(locator=%s hashstop=%064x)" \
1121             % (repr(self.locator), self.hashstop)
1122
1123
1124 class msg_tx():
1125     command = b"tx"
1126
1127     def __init__(self, tx=CTransaction()):
1128         self.tx = tx
1129
1130     def deserialize(self, f):
1131         self.tx.deserialize(f)
1132
1133     def serialize(self):
1134         return self.tx.serialize_without_witness()
1135
1136     def __repr__(self):
1137         return "msg_tx(tx=%s)" % (repr(self.tx))
1138
1139 class msg_witness_tx(msg_tx):
1140
1141     def serialize(self):
1142         return self.tx.serialize_with_witness()
1143
1144
1145 class msg_block():
1146     command = b"block"
1147
1148     def __init__(self, block=None):
1149         if block is None:
1150             self.block = CBlock()
1151         else:
1152             self.block = block
1153
1154     def deserialize(self, f):
1155         self.block.deserialize(f)
1156
1157     def serialize(self):
1158         return self.block.serialize()
1159
1160     def __repr__(self):
1161         return "msg_block(block=%s)" % (repr(self.block))
1162
1163 # for cases where a user needs tighter control over what is sent over the wire
1164 # note that the user must supply the name of the command, and the data
1165 class msg_generic():
1166     def __init__(self, command, data=None):
1167         self.command = command
1168         self.data = data
1169
1170     def serialize(self):
1171         return self.data
1172
1173     def __repr__(self):
1174         return "msg_generic()"
1175
1176 class msg_witness_block(msg_block):
1177
1178     def serialize(self):
1179         r = self.block.serialize(with_witness=True)
1180         return r
1181
1182 class msg_getaddr():
1183     command = b"getaddr"
1184
1185     def __init__(self):
1186         pass
1187
1188     def deserialize(self, f):
1189         pass
1190
1191     def serialize(self):
1192         return b""
1193
1194     def __repr__(self):
1195         return "msg_getaddr()"
1196
1197
1198 class msg_ping_prebip31():
1199     command = b"ping"
1200
1201     def __init__(self):
1202         pass
1203
1204     def deserialize(self, f):
1205         pass
1206
1207     def serialize(self):
1208         return b""
1209
1210     def __repr__(self):
1211         return "msg_ping() (pre-bip31)"
1212
1213
1214 class msg_ping():
1215     command = b"ping"
1216
1217     def __init__(self, nonce=0):
1218         self.nonce = nonce
1219
1220     def deserialize(self, f):
1221         self.nonce = struct.unpack("<Q", f.read(8))[0]
1222
1223     def serialize(self):
1224         r = b""
1225         r += struct.pack("<Q", self.nonce)
1226         return r
1227
1228     def __repr__(self):
1229         return "msg_ping(nonce=%08x)" % self.nonce
1230
1231
1232 class msg_pong():
1233     command = b"pong"
1234
1235     def __init__(self, nonce=0):
1236         self.nonce = nonce
1237
1238     def deserialize(self, f):
1239         self.nonce = struct.unpack("<Q", f.read(8))[0]
1240
1241     def serialize(self):
1242         r = b""
1243         r += struct.pack("<Q", self.nonce)
1244         return r
1245
1246     def __repr__(self):
1247         return "msg_pong(nonce=%08x)" % self.nonce
1248
1249
1250 class msg_mempool():
1251     command = b"mempool"
1252
1253     def __init__(self):
1254         pass
1255
1256     def deserialize(self, f):
1257         pass
1258
1259     def serialize(self):
1260         return b""
1261
1262     def __repr__(self):
1263         return "msg_mempool()"
1264
1265 class msg_sendheaders():
1266     command = b"sendheaders"
1267
1268     def __init__(self):
1269         pass
1270
1271     def deserialize(self, f):
1272         pass
1273
1274     def serialize(self):
1275         return b""
1276
1277     def __repr__(self):
1278         return "msg_sendheaders()"
1279
1280
1281 # getheaders message has
1282 # number of entries
1283 # vector of hashes
1284 # hash_stop (hash of last desired block header, 0 to get as many as possible)
1285 class msg_getheaders():
1286     command = b"getheaders"
1287
1288     def __init__(self):
1289         self.locator = CBlockLocator()
1290         self.hashstop = 0
1291
1292     def deserialize(self, f):
1293         self.locator = CBlockLocator()
1294         self.locator.deserialize(f)
1295         self.hashstop = deser_uint256(f)
1296
1297     def serialize(self):
1298         r = b""
1299         r += self.locator.serialize()
1300         r += ser_uint256(self.hashstop)
1301         return r
1302
1303     def __repr__(self):
1304         return "msg_getheaders(locator=%s, stop=%064x)" \
1305             % (repr(self.locator), self.hashstop)
1306
1307
1308 # headers message has
1309 # <count> <vector of block headers>
1310 class msg_headers():
1311     command = b"headers"
1312
1313     def __init__(self, headers=None):
1314         self.headers = headers if headers is not None else []
1315
1316     def deserialize(self, f):
1317         # comment in bitcoind indicates these should be deserialized as blocks
1318         blocks = deser_vector(f, CBlock)
1319         for x in blocks:
1320             self.headers.append(CBlockHeader(x))
1321
1322     def serialize(self):
1323         blocks = [CBlock(x) for x in self.headers]
1324         return ser_vector(blocks)
1325
1326     def __repr__(self):
1327         return "msg_headers(headers=%s)" % repr(self.headers)
1328
1329
1330 class msg_reject():
1331     command = b"reject"
1332     REJECT_MALFORMED = 1
1333
1334     def __init__(self):
1335         self.message = b""
1336         self.code = 0
1337         self.reason = b""
1338         self.data = 0
1339
1340     def deserialize(self, f):
1341         self.message = deser_string(f)
1342         self.code = struct.unpack("<B", f.read(1))[0]
1343         self.reason = deser_string(f)
1344         if (self.code != self.REJECT_MALFORMED and
1345                 (self.message == b"block" or self.message == b"tx")):
1346             self.data = deser_uint256(f)
1347
1348     def serialize(self):
1349         r = ser_string(self.message)
1350         r += struct.pack("<B", self.code)
1351         r += ser_string(self.reason)
1352         if (self.code != self.REJECT_MALFORMED and
1353                 (self.message == b"block" or self.message == b"tx")):
1354             r += ser_uint256(self.data)
1355         return r
1356
1357     def __repr__(self):
1358         return "msg_reject: %s %d %s [%064x]" \
1359             % (self.message, self.code, self.reason, self.data)
1360
1361 class msg_feefilter():
1362     command = b"feefilter"
1363
1364     def __init__(self, feerate=0):
1365         self.feerate = feerate
1366
1367     def deserialize(self, f):
1368         self.feerate = struct.unpack("<Q", f.read(8))[0]
1369
1370     def serialize(self):
1371         r = b""
1372         r += struct.pack("<Q", self.feerate)
1373         return r
1374
1375     def __repr__(self):
1376         return "msg_feefilter(feerate=%08x)" % self.feerate
1377
1378 class msg_sendcmpct():
1379     command = b"sendcmpct"
1380
1381     def __init__(self):
1382         self.announce = False
1383         self.version = 1
1384
1385     def deserialize(self, f):
1386         self.announce = struct.unpack("<?", f.read(1))[0]
1387         self.version = struct.unpack("<Q", f.read(8))[0]
1388
1389     def serialize(self):
1390         r = b""
1391         r += struct.pack("<?", self.announce)
1392         r += struct.pack("<Q", self.version)
1393         return r
1394
1395     def __repr__(self):
1396         return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version)
1397
1398 class msg_cmpctblock():
1399     command = b"cmpctblock"
1400
1401     def __init__(self, header_and_shortids = None):
1402         self.header_and_shortids = header_and_shortids
1403
1404     def deserialize(self, f):
1405         self.header_and_shortids = P2PHeaderAndShortIDs()
1406         self.header_and_shortids.deserialize(f)
1407
1408     def serialize(self):
1409         r = b""
1410         r += self.header_and_shortids.serialize()
1411         return r
1412
1413     def __repr__(self):
1414         return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids)
1415
1416 class msg_getblocktxn():
1417     command = b"getblocktxn"
1418
1419     def __init__(self):
1420         self.block_txn_request = None
1421
1422     def deserialize(self, f):
1423         self.block_txn_request = BlockTransactionsRequest()
1424         self.block_txn_request.deserialize(f)
1425
1426     def serialize(self):
1427         r = b""
1428         r += self.block_txn_request.serialize()
1429         return r
1430
1431     def __repr__(self):
1432         return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request))
1433
1434 class msg_blocktxn():
1435     command = b"blocktxn"
1436
1437     def __init__(self):
1438         self.block_transactions = BlockTransactions()
1439
1440     def deserialize(self, f):
1441         self.block_transactions.deserialize(f)
1442
1443     def serialize(self):
1444         r = b""
1445         r += self.block_transactions.serialize()
1446         return r
1447
1448     def __repr__(self):
1449         return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions))
1450
1451 class msg_witness_blocktxn(msg_blocktxn):
1452     def serialize(self):
1453         r = b""
1454         r += self.block_transactions.serialize(with_witness=True)
1455         return r
1456
1457 class NodeConnCB():
1458     """Callback and helper functions for P2P connection to a bitcoind node.
1459
1460     Individual testcases should subclass this and override the on_* methods
1461     if they want to alter message handling behaviour.
1462     """
1463
1464     def __init__(self):
1465         # Track whether we have a P2P connection open to the node
1466         self.connected = False
1467         self.connection = None
1468
1469         # Track number of messages of each type received and the most recent
1470         # message of each type
1471         self.message_count = defaultdict(int)
1472         self.last_message = {}
1473
1474         # A count of the number of ping messages we've sent to the node
1475         self.ping_counter = 1
1476
1477         # deliver_sleep_time is helpful for debugging race conditions in p2p
1478         # tests; it causes message delivery to sleep for the specified time
1479         # before acquiring the global lock and delivering the next message.
1480         self.deliver_sleep_time = None
1481
1482     # Message receiving methods
1483
1484     def deliver(self, conn, message):
1485         """Receive message and dispatch message to appropriate callback.
1486
1487         We keep a count of how many of each message type has been received
1488         and the most recent message of each type.
1489
1490         Optionally waits for deliver_sleep_time before dispatching message.
1491         """
1492
1493         deliver_sleep = self.get_deliver_sleep_time()
1494         if deliver_sleep is not None:
1495             time.sleep(deliver_sleep)
1496         with mininode_lock:
1497             try:
1498                 command = message.command.decode('ascii')
1499                 self.message_count[command] += 1
1500                 self.last_message[command] = message
1501                 getattr(self, 'on_' + command)(conn, message)
1502             except:
1503                 print("ERROR delivering %s (%s)" % (repr(message),
1504                                                     sys.exc_info()[0]))
1505                 raise
1506
1507     def get_deliver_sleep_time(self):
1508         with mininode_lock:
1509             return self.deliver_sleep_time
1510
1511     # Callback methods. Can be overridden by subclasses in individual test
1512     # cases to provide custom message handling behaviour.
1513
1514     def on_open(self, conn):
1515         self.connected = True
1516
1517     def on_close(self, conn):
1518         self.connected = False
1519         self.connection = None
1520
1521     def on_addr(self, conn, message): pass
1522     def on_alert(self, conn, message): pass
1523     def on_block(self, conn, message): pass
1524     def on_blocktxn(self, conn, message): pass
1525     def on_cmpctblock(self, conn, message): pass
1526     def on_feefilter(self, conn, message): pass
1527     def on_getaddr(self, conn, message): pass
1528     def on_getblocks(self, conn, message): pass
1529     def on_getblocktxn(self, conn, message): pass
1530     def on_getdata(self, conn, message): pass
1531     def on_getheaders(self, conn, message): pass
1532     def on_headers(self, conn, message): pass
1533     def on_mempool(self, conn): pass
1534     def on_pong(self, conn, message): pass
1535     def on_reject(self, conn, message): pass
1536     def on_sendcmpct(self, conn, message): pass
1537     def on_sendheaders(self, conn, message): pass
1538     def on_tx(self, conn, message): pass
1539
1540     def on_inv(self, conn, message):
1541         want = msg_getdata()
1542         for i in message.inv:
1543             if i.type != 0:
1544                 want.inv.append(i)
1545         if len(want.inv):
1546             conn.send_message(want)
1547
1548     def on_ping(self, conn, message):
1549         if conn.ver_send > BIP0031_VERSION:
1550             conn.send_message(msg_pong(message.nonce))
1551
1552     def on_verack(self, conn, message):
1553         conn.ver_recv = conn.ver_send
1554         self.verack_received = True
1555
1556     def on_version(self, conn, message):
1557         if message.nVersion >= 209:
1558             conn.send_message(msg_verack())
1559         conn.ver_send = min(MY_VERSION, message.nVersion)
1560         if message.nVersion < 209:
1561             conn.ver_recv = conn.ver_send
1562         conn.nServices = message.nServices
1563
1564     # Connection helper methods
1565
1566     def add_connection(self, conn):
1567         self.connection = conn
1568
1569     def wait_for_disconnect(self, timeout=60):
1570         test_function = lambda: not self.connected
1571         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1572
1573     # Message receiving helper methods
1574
1575     def wait_for_block(self, blockhash, timeout=60):
1576         test_function = lambda: self.last_message.get("block") and self.last_message["block"].block.rehash() == blockhash
1577         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1578
1579     def wait_for_getdata(self, timeout=60):
1580         test_function = lambda: self.last_message.get("getdata")
1581         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1582
1583     def wait_for_getheaders(self, timeout=60):
1584         test_function = lambda: self.last_message.get("getheaders")
1585         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1586
1587     def wait_for_inv(self, expected_inv, timeout=60):
1588         """Waits for an INV message and checks that the first inv object in the message was as expected."""
1589         if len(expected_inv) > 1:
1590             raise NotImplementedError("wait_for_inv() will only verify the first inv object")
1591         test_function = lambda: self.last_message.get("inv") and \
1592                                 self.last_message["inv"].inv[0].type == expected_inv[0].type and \
1593                                 self.last_message["inv"].inv[0].hash == expected_inv[0].hash
1594         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1595
1596     def wait_for_verack(self, timeout=60):
1597         test_function = lambda: self.message_count["verack"]
1598         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1599
1600     # Message sending helper functions
1601
1602     def send_message(self, message):
1603         if self.connection:
1604             self.connection.send_message(message)
1605         else:
1606             logger.error("Cannot send message. No connection to node!")
1607
1608     def send_and_ping(self, message):
1609         self.send_message(message)
1610         self.sync_with_ping()
1611
1612     # Sync up with the node
1613     def sync_with_ping(self, timeout=60):
1614         self.send_message(msg_ping(nonce=self.ping_counter))
1615         test_function = lambda: self.last_message.get("pong") and self.last_message["pong"].nonce == self.ping_counter
1616         wait_until(test_function, timeout=timeout, lock=mininode_lock)
1617         self.ping_counter += 1
1618
1619 # The actual NodeConn class
1620 # This class provides an interface for a p2p connection to a specified node
1621 class NodeConn(asyncore.dispatcher):
1622     messagemap = {
1623         b"version": msg_version,
1624         b"verack": msg_verack,
1625         b"addr": msg_addr,
1626         b"alert": msg_alert,
1627         b"inv": msg_inv,
1628         b"getdata": msg_getdata,
1629         b"getblocks": msg_getblocks,
1630         b"tx": msg_tx,
1631         b"block": msg_block,
1632         b"getaddr": msg_getaddr,
1633         b"ping": msg_ping,
1634         b"pong": msg_pong,
1635         b"headers": msg_headers,
1636         b"getheaders": msg_getheaders,
1637         b"reject": msg_reject,
1638         b"mempool": msg_mempool,
1639         b"feefilter": msg_feefilter,
1640         b"sendheaders": msg_sendheaders,
1641         b"sendcmpct": msg_sendcmpct,
1642         b"cmpctblock": msg_cmpctblock,
1643         b"getblocktxn": msg_getblocktxn,
1644         b"blocktxn": msg_blocktxn
1645     }
1646     MAGIC_BYTES = {
1647         "mainnet": b"\xf9\xbe\xb4\xd9",   # mainnet
1648         "testnet3": b"\x0b\x11\x09\x07",  # testnet3
1649         "regtest": b"\xfa\xbf\xb5\xda",   # regtest
1650     }
1651
1652     def __init__(self, dstaddr, dstport, rpc, callback, net="regtest", services=NODE_NETWORK|NODE_WITNESS, send_version=True):
1653         asyncore.dispatcher.__init__(self, map=mininode_socket_map)
1654         self.dstaddr = dstaddr
1655         self.dstport = dstport
1656         self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
1657         self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
1658         self.sendbuf = b""
1659         self.recvbuf = b""
1660         self.ver_send = 209
1661         self.ver_recv = 209
1662         self.last_sent = 0
1663         self.state = "connecting"
1664         self.network = net
1665         self.cb = callback
1666         self.disconnect = False
1667         self.nServices = 0
1668
1669         if send_version:
1670             # stuff version msg into sendbuf
1671             vt = msg_version()
1672             vt.nServices = services
1673             vt.addrTo.ip = self.dstaddr
1674             vt.addrTo.port = self.dstport
1675             vt.addrFrom.ip = "0.0.0.0"
1676             vt.addrFrom.port = 0
1677             self.send_message(vt, True)
1678
1679         logger.info('Connecting to Bitcoin Node: %s:%d' % (self.dstaddr, self.dstport))
1680
1681         try:
1682             self.connect((dstaddr, dstport))
1683         except:
1684             self.handle_close()
1685         self.rpc = rpc
1686
1687     def handle_connect(self):
1688         if self.state != "connected":
1689             logger.debug("Connected & Listening: %s:%d" % (self.dstaddr, self.dstport))
1690             self.state = "connected"
1691             self.cb.on_open(self)
1692
1693     def handle_close(self):
1694         logger.debug("Closing connection to: %s:%d" % (self.dstaddr, self.dstport))
1695         self.state = "closed"
1696         self.recvbuf = b""
1697         self.sendbuf = b""
1698         try:
1699             self.close()
1700         except:
1701             pass
1702         self.cb.on_close(self)
1703
1704     def handle_read(self):
1705         t = self.recv(8192)
1706         if len(t) > 0:
1707             self.recvbuf += t
1708             self.got_data()
1709
1710     def readable(self):
1711         return True
1712
1713     def writable(self):
1714         with mininode_lock:
1715             pre_connection = self.state == "connecting"
1716             length = len(self.sendbuf)
1717         return (length > 0 or pre_connection)
1718
1719     def handle_write(self):
1720         with mininode_lock:
1721             # asyncore does not expose socket connection, only the first read/write
1722             # event, thus we must check connection manually here to know when we
1723             # actually connect
1724             if self.state == "connecting":
1725                 self.handle_connect()
1726             if not self.writable():
1727                 return
1728
1729             try:
1730                 sent = self.send(self.sendbuf)
1731             except:
1732                 self.handle_close()
1733                 return
1734             self.sendbuf = self.sendbuf[sent:]
1735
1736     def got_data(self):
1737         try:
1738             while True:
1739                 if len(self.recvbuf) < 4:
1740                     return
1741                 if self.recvbuf[:4] != self.MAGIC_BYTES[self.network]:
1742                     raise ValueError("got garbage %s" % repr(self.recvbuf))
1743                 if self.ver_recv < 209:
1744                     if len(self.recvbuf) < 4 + 12 + 4:
1745                         return
1746                     command = self.recvbuf[4:4+12].split(b"\x00", 1)[0]
1747                     msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0]
1748                     checksum = None
1749                     if len(self.recvbuf) < 4 + 12 + 4 + msglen:
1750                         return
1751                     msg = self.recvbuf[4+12+4:4+12+4+msglen]
1752                     self.recvbuf = self.recvbuf[4+12+4+msglen:]
1753                 else:
1754                     if len(self.recvbuf) < 4 + 12 + 4 + 4:
1755                         return
1756                     command = self.recvbuf[4:4+12].split(b"\x00", 1)[0]
1757                     msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0]
1758                     checksum = self.recvbuf[4+12+4:4+12+4+4]
1759                     if len(self.recvbuf) < 4 + 12 + 4 + 4 + msglen:
1760                         return
1761                     msg = self.recvbuf[4+12+4+4:4+12+4+4+msglen]
1762                     th = sha256(msg)
1763                     h = sha256(th)
1764                     if checksum != h[:4]:
1765                         raise ValueError("got bad checksum " + repr(self.recvbuf))
1766                     self.recvbuf = self.recvbuf[4+12+4+4+msglen:]
1767                 if command in self.messagemap:
1768                     f = BytesIO(msg)
1769                     t = self.messagemap[command]()
1770                     t.deserialize(f)
1771                     self.got_message(t)
1772                 else:
1773                     logger.warning("Received unknown command from %s:%d: '%s' %s" % (self.dstaddr, self.dstport, command, repr(msg)))
1774                     raise ValueError("Unknown command: '%s'" % (command))
1775         except Exception as e:
1776             logger.exception('got_data:', repr(e))
1777             raise
1778
1779     def send_message(self, message, pushbuf=False):
1780         if self.state != "connected" and not pushbuf:
1781             raise IOError('Not connected, no pushbuf')
1782         self._log_message("send", message)
1783         command = message.command
1784         data = message.serialize()
1785         tmsg = self.MAGIC_BYTES[self.network]
1786         tmsg += command
1787         tmsg += b"\x00" * (12 - len(command))
1788         tmsg += struct.pack("<I", len(data))
1789         if self.ver_send >= 209:
1790             th = sha256(data)
1791             h = sha256(th)
1792             tmsg += h[:4]
1793         tmsg += data
1794         with mininode_lock:
1795             if (len(self.sendbuf) == 0 and not pushbuf):
1796                 try:
1797                     sent = self.send(tmsg)
1798                     self.sendbuf = tmsg[sent:]
1799                 except BlockingIOError:
1800                     self.sendbuf = tmsg
1801             else:
1802                 self.sendbuf += tmsg
1803             self.last_sent = time.time()
1804
1805     def got_message(self, message):
1806         if message.command == b"version":
1807             if message.nVersion <= BIP0031_VERSION:
1808                 self.messagemap[b'ping'] = msg_ping_prebip31
1809         if self.last_sent + 30 * 60 < time.time():
1810             self.send_message(self.messagemap[b'ping']())
1811         self._log_message("receive", message)
1812         self.cb.deliver(self, message)
1813
1814     def _log_message(self, direction, msg):
1815         if direction == "send":
1816             log_message = "Send message to "
1817         elif direction == "receive":
1818             log_message = "Received message from "
1819         log_message += "%s:%d: %s" % (self.dstaddr, self.dstport, repr(msg)[:500])
1820         if len(log_message) > 500:
1821             log_message += "... (msg truncated)"
1822         logger.debug(log_message)
1823
1824     def disconnect_node(self):
1825         self.disconnect = True
1826
1827
1828 class NetworkThread(Thread):
1829     def run(self):
1830         while mininode_socket_map:
1831             # We check for whether to disconnect outside of the asyncore
1832             # loop to workaround the behavior of asyncore when using
1833             # select
1834             disconnected = []
1835             for fd, obj in mininode_socket_map.items():
1836                 if obj.disconnect:
1837                     disconnected.append(obj)
1838             [ obj.handle_close() for obj in disconnected ]
1839             asyncore.loop(0.1, use_poll=True, map=mininode_socket_map, count=1)
1840         logger.debug("Network thread closing")
1841
1842
1843 # An exception we can raise if we detect a potential disconnect
1844 # (p2p or rpc) before the test is complete
1845 class EarlyDisconnectError(Exception):
1846     def __init__(self, value):
1847         self.value = value
1848
1849     def __str__(self):
1850         return repr(self.value)