test/functional/test_framework/messages.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2010 ArtForz -- public domain half-a-node
   3 # Copyright (c) 2012 Jeff Garzik
   4 # Copyright (c) 2010-2017 The Bitcoin Core developers
   5 # Distributed under the MIT software license, see the accompanying
   6 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   7 """Bitcoin test framework primitive and message strcutures
   8
   9 CBlock, CTransaction, CBlockHeader, CTxIn, CTxOut, etc....:
  10     data structures that should map to corresponding structures in
  11     bitcoin/primitives
  12
  13 msg_block, msg_tx, msg_headers, etc.:
  14     data structures that represent network messages
  15
  16 ser_*, deser_*: functions that handle serialization/deserialization."""
  17 from codecs import encode
  18 import copy
  19 import hashlib
  20 from io import BytesIO
  21 import random
  22 import socket
  23 import struct
  24 import time
  25
  26 from test_framework.siphash import siphash256
  27 from test_framework.util import hex_str_to_bytes, bytes_to_hex_str, wait_until
  28
  29 MIN_VERSION_SUPPORTED = 60001
  30 MY_VERSION = 70014  # past bip-31 for ping/pong
  31 MY_SUBVERSION = b"/python-mininode-tester:0.0.3/"
  32 MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37)
  33
  34 MAX_INV_SZ = 50000
  35 MAX_BLOCK_BASE_SIZE = 1000000
  36
  37 COIN = 100000000 # 1 btc in satoshis
  38
  39 NODE_NETWORK = (1 << 0)
  40 # NODE_GETUTXO = (1 << 1)
  41 # NODE_BLOOM = (1 << 2)
  42 NODE_WITNESS = (1 << 3)
  43 NODE_UNSUPPORTED_SERVICE_BIT_5 = (1 << 5)
  44 NODE_UNSUPPORTED_SERVICE_BIT_7 = (1 << 7)
  45
  46 # Serialization/deserialization tools
  47 def sha256(s):
  48     return hashlib.new('sha256', s).digest()
  49
  50 def ripemd160(s):
  51     return hashlib.new('ripemd160', s).digest()
  52
  53 def hash256(s):
  54     return sha256(sha256(s))
  55
  56 def ser_compact_size(l):
  57     r = b""
  58     if l < 253:
  59         r = struct.pack("B", l)
  60     elif l < 0x10000:
  61         r = struct.pack("<BH", 253, l)
  62     elif l < 0x100000000:
  63         r = struct.pack("<BI", 254, l)
  64     else:
  65         r = struct.pack("<BQ", 255, l)
  66     return r
  67
  68 def deser_compact_size(f):
  69     nit = struct.unpack("<B", f.read(1))[0]
  70     if nit == 253:
  71         nit = struct.unpack("<H", f.read(2))[0]
  72     elif nit == 254:
  73         nit = struct.unpack("<I", f.read(4))[0]
  74     elif nit == 255:
  75         nit = struct.unpack("<Q", f.read(8))[0]
  76     return nit
  77
  78 def deser_string(f):
  79     nit = deser_compact_size(f)
  80     return f.read(nit)
  81
  82 def ser_string(s):
  83     return ser_compact_size(len(s)) + s
  84
  85 def deser_uint256(f):
  86     r = 0
  87     for i in range(8):
  88         t = struct.unpack("<I", f.read(4))[0]
  89         r += t << (i * 32)
  90     return r
  91
  92
  93 def ser_uint256(u):
  94     rs = b""
  95     for i in range(8):
  96         rs += struct.pack("<I", u & 0xFFFFFFFF)
  97         u >>= 32
  98     return rs
  99
 100
 101 def uint256_from_str(s):
 102     r = 0
 103     t = struct.unpack("<IIIIIIII", s[:32])
 104     for i in range(8):
 105         r += t[i] << (i * 32)
 106     return r
 107
 108
 109 def uint256_from_compact(c):
 110     nbytes = (c >> 24) & 0xFF
 111     v = (c & 0xFFFFFF) << (8 * (nbytes - 3))
 112     return v
 113
 114
 115 def deser_vector(f, c):
 116     nit = deser_compact_size(f)
 117     r = []
 118     for i in range(nit):
 119         t = c()
 120         t.deserialize(f)
 121         r.append(t)
 122     return r
 123
 124
 125 # ser_function_name: Allow for an alternate serialization function on the
 126 # entries in the vector (we use this for serializing the vector of transactions
 127 # for a witness block).
 128 def ser_vector(l, ser_function_name=None):
 129     r = ser_compact_size(len(l))
 130     for i in l:
 131         if ser_function_name:
 132             r += getattr(i, ser_function_name)()
 133         else:
 134             r += i.serialize()
 135     return r
 136
 137
 138 def deser_uint256_vector(f):
 139     nit = deser_compact_size(f)
 140     r = []
 141     for i in range(nit):
 142         t = deser_uint256(f)
 143         r.append(t)
 144     return r
 145
 146
 147 def ser_uint256_vector(l):
 148     r = ser_compact_size(len(l))
 149     for i in l:
 150         r += ser_uint256(i)
 151     return r
 152
 153
 154 def deser_string_vector(f):
 155     nit = deser_compact_size(f)
 156     r = []
 157     for i in range(nit):
 158         t = deser_string(f)
 159         r.append(t)
 160     return r
 161
 162
 163 def ser_string_vector(l):
 164     r = ser_compact_size(len(l))
 165     for sv in l:
 166         r += ser_string(sv)
 167     return r
 168
 169
 170 # Deserialize from a hex string representation (eg from RPC)
 171 def FromHex(obj, hex_string):
 172     obj.deserialize(BytesIO(hex_str_to_bytes(hex_string)))
 173     return obj
 174
 175 # Convert a binary-serializable object to hex (eg for submission via RPC)
 176 def ToHex(obj):
 177     return bytes_to_hex_str(obj.serialize())
 178
 179 # Objects that map to bitcoind objects, which can be serialized/deserialized
 180
 181 class CAddress():
 182     def __init__(self):
 183         self.nServices = 1
 184         self.pchReserved = b"\x00" * 10 + b"\xff" * 2
 185         self.ip = "0.0.0.0"
 186         self.port = 0
 187
 188     def deserialize(self, f):
 189         self.nServices = struct.unpack("<Q", f.read(8))[0]
 190         self.pchReserved = f.read(12)
 191         self.ip = socket.inet_ntoa(f.read(4))
 192         self.port = struct.unpack(">H", f.read(2))[0]
 193
 194     def serialize(self):
 195         r = b""
 196         r += struct.pack("<Q", self.nServices)
 197         r += self.pchReserved
 198         r += socket.inet_aton(self.ip)
 199         r += struct.pack(">H", self.port)
 200         return r
 201
 202     def __repr__(self):
 203         return "CAddress(nServices=%i ip=%s port=%i)" % (self.nServices,
 204                                                          self.ip, self.port)
 205
 206 MSG_WITNESS_FLAG = 1<<30
 207
 208 class CInv():
 209     typemap = {
 210         0: "Error",
 211         1: "TX",
 212         2: "Block",
 213         1|MSG_WITNESS_FLAG: "WitnessTx",
 214         2|MSG_WITNESS_FLAG : "WitnessBlock",
 215         4: "CompactBlock"
 216     }
 217
 218     def __init__(self, t=0, h=0):
 219         self.type = t
 220         self.hash = h
 221
 222     def deserialize(self, f):
 223         self.type = struct.unpack("<i", f.read(4))[0]
 224         self.hash = deser_uint256(f)
 225
 226     def serialize(self):
 227         r = b""
 228         r += struct.pack("<i", self.type)
 229         r += ser_uint256(self.hash)
 230         return r
 231
 232     def __repr__(self):
 233         return "CInv(type=%s hash=%064x)" \
 234             % (self.typemap[self.type], self.hash)
 235
 236
 237 class CBlockLocator():
 238     def __init__(self):
 239         self.nVersion = MY_VERSION
 240         self.vHave = []
 241
 242     def deserialize(self, f):
 243         self.nVersion = struct.unpack("<i", f.read(4))[0]
 244         self.vHave = deser_uint256_vector(f)
 245
 246     def serialize(self):
 247         r = b""
 248         r += struct.pack("<i", self.nVersion)
 249         r += ser_uint256_vector(self.vHave)
 250         return r
 251
 252     def __repr__(self):
 253         return "CBlockLocator(nVersion=%i vHave=%s)" \
 254             % (self.nVersion, repr(self.vHave))
 255
 256
 257 class COutPoint():
 258     def __init__(self, hash=0, n=0):
 259         self.hash = hash
 260         self.n = n
 261
 262     def deserialize(self, f):
 263         self.hash = deser_uint256(f)
 264         self.n = struct.unpack("<I", f.read(4))[0]
 265
 266     def serialize(self):
 267         r = b""
 268         r += ser_uint256(self.hash)
 269         r += struct.pack("<I", self.n)
 270         return r
 271
 272     def __repr__(self):
 273         return "COutPoint(hash=%064x n=%i)" % (self.hash, self.n)
 274
 275
 276 class CTxIn():
 277     def __init__(self, outpoint=None, scriptSig=b"", nSequence=0):
 278         if outpoint is None:
 279             self.prevout = COutPoint()
 280         else:
 281             self.prevout = outpoint
 282         self.scriptSig = scriptSig
 283         self.nSequence = nSequence
 284
 285     def deserialize(self, f):
 286         self.prevout = COutPoint()
 287         self.prevout.deserialize(f)
 288         self.scriptSig = deser_string(f)
 289         self.nSequence = struct.unpack("<I", f.read(4))[0]
 290
 291     def serialize(self):
 292         r = b""
 293         r += self.prevout.serialize()
 294         r += ser_string(self.scriptSig)
 295         r += struct.pack("<I", self.nSequence)
 296         return r
 297
 298     def __repr__(self):
 299         return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \
 300             % (repr(self.prevout), bytes_to_hex_str(self.scriptSig),
 301                self.nSequence)
 302
 303
 304 class CTxOut():
 305     def __init__(self, nValue=0, scriptPubKey=b""):
 306         self.nValue = nValue
 307         self.scriptPubKey = scriptPubKey
 308
 309     def deserialize(self, f):
 310         self.nValue = struct.unpack("<q", f.read(8))[0]
 311         self.scriptPubKey = deser_string(f)
 312
 313     def serialize(self):
 314         r = b""
 315         r += struct.pack("<q", self.nValue)
 316         r += ser_string(self.scriptPubKey)
 317         return r
 318
 319     def __repr__(self):
 320         return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \
 321             % (self.nValue // COIN, self.nValue % COIN,
 322                bytes_to_hex_str(self.scriptPubKey))
 323
 324
 325 class CScriptWitness():
 326     def __init__(self):
 327         # stack is a vector of strings
 328         self.stack = []
 329
 330     def __repr__(self):
 331         return "CScriptWitness(%s)" % \
 332                (",".join([bytes_to_hex_str(x) for x in self.stack]))
 333
 334     def is_null(self):
 335         if self.stack:
 336             return False
 337         return True
 338
 339
 340 class CTxInWitness():
 341     def __init__(self):
 342         self.scriptWitness = CScriptWitness()
 343
 344     def deserialize(self, f):
 345         self.scriptWitness.stack = deser_string_vector(f)
 346
 347     def serialize(self):
 348         return ser_string_vector(self.scriptWitness.stack)
 349
 350     def __repr__(self):
 351         return repr(self.scriptWitness)
 352
 353     def is_null(self):
 354         return self.scriptWitness.is_null()
 355
 356
 357 class CTxWitness():
 358     def __init__(self):
 359         self.vtxinwit = []
 360
 361     def deserialize(self, f):
 362         for i in range(len(self.vtxinwit)):
 363             self.vtxinwit[i].deserialize(f)
 364
 365     def serialize(self):
 366         r = b""
 367         # This is different than the usual vector serialization --
 368         # we omit the length of the vector, which is required to be
 369         # the same length as the transaction's vin vector.
 370         for x in self.vtxinwit:
 371             r += x.serialize()
 372         return r
 373
 374     def __repr__(self):
 375         return "CTxWitness(%s)" % \
 376                (';'.join([repr(x) for x in self.vtxinwit]))
 377
 378     def is_null(self):
 379         for x in self.vtxinwit:
 380             if not x.is_null():
 381                 return False
 382         return True
 383
 384
 385 class CTransaction():
 386     def __init__(self, tx=None):
 387         if tx is None:
 388             self.nVersion = 1
 389             self.vin = []
 390             self.vout = []
 391             self.wit = CTxWitness()
 392             self.nLockTime = 0
 393             self.sha256 = None
 394             self.hash = None
 395         else:
 396             self.nVersion = tx.nVersion
 397             self.vin = copy.deepcopy(tx.vin)
 398             self.vout = copy.deepcopy(tx.vout)
 399             self.nLockTime = tx.nLockTime
 400             self.sha256 = tx.sha256
 401             self.hash = tx.hash
 402             self.wit = copy.deepcopy(tx.wit)
 403
 404     def deserialize(self, f):
 405         self.nVersion = struct.unpack("<i", f.read(4))[0]
 406         self.vin = deser_vector(f, CTxIn)
 407         flags = 0
 408         if len(self.vin) == 0:
 409             flags = struct.unpack("<B", f.read(1))[0]
 410             # Not sure why flags can't be zero, but this
 411             # matches the implementation in bitcoind
 412             if (flags != 0):
 413                 self.vin = deser_vector(f, CTxIn)
 414                 self.vout = deser_vector(f, CTxOut)
 415         else:
 416             self.vout = deser_vector(f, CTxOut)
 417         if flags != 0:
 418             self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))]
 419             self.wit.deserialize(f)
 420         self.nLockTime = struct.unpack("<I", f.read(4))[0]
 421         self.sha256 = None
 422         self.hash = None
 423
 424     def serialize_without_witness(self):
 425         r = b""
 426         r += struct.pack("<i", self.nVersion)
 427         r += ser_vector(self.vin)
 428         r += ser_vector(self.vout)
 429         r += struct.pack("<I", self.nLockTime)
 430         return r
 431
 432     # Only serialize with witness when explicitly called for
 433     def serialize_with_witness(self):
 434         flags = 0
 435         if not self.wit.is_null():
 436             flags |= 1
 437         r = b""
 438         r += struct.pack("<i", self.nVersion)
 439         if flags:
 440             dummy = []
 441             r += ser_vector(dummy)
 442             r += struct.pack("<B", flags)
 443         r += ser_vector(self.vin)
 444         r += ser_vector(self.vout)
 445         if flags & 1:
 446             if (len(self.wit.vtxinwit) != len(self.vin)):
 447                 # vtxinwit must have the same length as vin
 448                 self.wit.vtxinwit = self.wit.vtxinwit[:len(self.vin)]
 449                 for i in range(len(self.wit.vtxinwit), len(self.vin)):
 450                     self.wit.vtxinwit.append(CTxInWitness())
 451             r += self.wit.serialize()
 452         r += struct.pack("<I", self.nLockTime)
 453         return r
 454
 455     # Regular serialization is without witness -- must explicitly
 456     # call serialize_with_witness to include witness data.
 457     def serialize(self):
 458         return self.serialize_without_witness()
 459
 460     # Recalculate the txid (transaction hash without witness)
 461     def rehash(self):
 462         self.sha256 = None
 463         self.calc_sha256()
 464
 465     # We will only cache the serialization without witness in
 466     # self.sha256 and self.hash -- those are expected to be the txid.
 467     def calc_sha256(self, with_witness=False):
 468         if with_witness:
 469             # Don't cache the result, just return it
 470             return uint256_from_str(hash256(self.serialize_with_witness()))
 471
 472         if self.sha256 is None:
 473             self.sha256 = uint256_from_str(hash256(self.serialize_without_witness()))
 474         self.hash = encode(hash256(self.serialize())[::-1], 'hex_codec').decode('ascii')
 475
 476     def is_valid(self):
 477         self.calc_sha256()
 478         for tout in self.vout:
 479             if tout.nValue < 0 or tout.nValue > 21000000 * COIN:
 480                 return False
 481         return True
 482
 483     def __repr__(self):
 484         return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \
 485             % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime)
 486
 487
 488 class CBlockHeader():
 489     def __init__(self, header=None):
 490         if header is None:
 491             self.set_null()
 492         else:
 493             self.nVersion = header.nVersion
 494             self.hashPrevBlock = header.hashPrevBlock
 495             self.hashMerkleRoot = header.hashMerkleRoot
 496             self.nTime = header.nTime
 497             self.nBits = header.nBits
 498             self.nNonce = header.nNonce
 499             self.sha256 = header.sha256
 500             self.hash = header.hash
 501             self.calc_sha256()
 502
 503     def set_null(self):
 504         self.nVersion = 1
 505         self.hashPrevBlock = 0
 506         self.hashMerkleRoot = 0
 507         self.nTime = 0
 508         self.nBits = 0
 509         self.nNonce = 0
 510         self.sha256 = None
 511         self.hash = None
 512
 513     def deserialize(self, f):
 514         self.nVersion = struct.unpack("<i", f.read(4))[0]
 515         self.hashPrevBlock = deser_uint256(f)
 516         self.hashMerkleRoot = deser_uint256(f)
 517         self.nTime = struct.unpack("<I", f.read(4))[0]
 518         self.nBits = struct.unpack("<I", f.read(4))[0]
 519         self.nNonce = struct.unpack("<I", f.read(4))[0]
 520         self.sha256 = None
 521         self.hash = None
 522
 523     def serialize(self):
 524         r = b""
 525         r += struct.pack("<i", self.nVersion)
 526         r += ser_uint256(self.hashPrevBlock)
 527         r += ser_uint256(self.hashMerkleRoot)
 528         r += struct.pack("<I", self.nTime)
 529         r += struct.pack("<I", self.nBits)
 530         r += struct.pack("<I", self.nNonce)
 531         return r
 532
 533     def calc_sha256(self):
 534         if self.sha256 is None:
 535             r = b""
 536             r += struct.pack("<i", self.nVersion)
 537             r += ser_uint256(self.hashPrevBlock)
 538             r += ser_uint256(self.hashMerkleRoot)
 539             r += struct.pack("<I", self.nTime)
 540             r += struct.pack("<I", self.nBits)
 541             r += struct.pack("<I", self.nNonce)
 542             self.sha256 = uint256_from_str(hash256(r))
 543             self.hash = encode(hash256(r)[::-1], 'hex_codec').decode('ascii')
 544
 545     def rehash(self):
 546         self.sha256 = None
 547         self.calc_sha256()
 548         return self.sha256
 549
 550     def __repr__(self):
 551         return "CBlockHeader(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x)" \
 552             % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
 553                time.ctime(self.nTime), self.nBits, self.nNonce)
 554
 555
 556 class CBlock(CBlockHeader):
 557     def __init__(self, header=None):
 558         super(CBlock, self).__init__(header)
 559         self.vtx = []
 560
 561     def deserialize(self, f):
 562         super(CBlock, self).deserialize(f)
 563         self.vtx = deser_vector(f, CTransaction)
 564
 565     def serialize(self, with_witness=False):
 566         r = b""
 567         r += super(CBlock, self).serialize()
 568         if with_witness:
 569             r += ser_vector(self.vtx, "serialize_with_witness")
 570         else:
 571             r += ser_vector(self.vtx)
 572         return r
 573
 574     # Calculate the merkle root given a vector of transaction hashes
 575     @classmethod
 576     def get_merkle_root(cls, hashes):
 577         while len(hashes) > 1:
 578             newhashes = []
 579             for i in range(0, len(hashes), 2):
 580                 i2 = min(i+1, len(hashes)-1)
 581                 newhashes.append(hash256(hashes[i] + hashes[i2]))
 582             hashes = newhashes
 583         return uint256_from_str(hashes[0])
 584
 585     def calc_merkle_root(self):
 586         hashes = []
 587         for tx in self.vtx:
 588             tx.calc_sha256()
 589             hashes.append(ser_uint256(tx.sha256))
 590         return self.get_merkle_root(hashes)
 591
 592     def calc_witness_merkle_root(self):
 593         # For witness root purposes, the hash of the
 594         # coinbase, with witness, is defined to be 0...0
 595         hashes = [ser_uint256(0)]
 596
 597         for tx in self.vtx[1:]:
 598             # Calculate the hashes with witness data
 599             hashes.append(ser_uint256(tx.calc_sha256(True)))
 600
 601         return self.get_merkle_root(hashes)
 602
 603     def is_valid(self):
 604         self.calc_sha256()
 605         target = uint256_from_compact(self.nBits)
 606         if self.sha256 > target:
 607             return False
 608         for tx in self.vtx:
 609             if not tx.is_valid():
 610                 return False
 611         if self.calc_merkle_root() != self.hashMerkleRoot:
 612             return False
 613         return True
 614
 615     def solve(self):
 616         self.rehash()
 617         target = uint256_from_compact(self.nBits)
 618         while self.sha256 > target:
 619             self.nNonce += 1
 620             self.rehash()
 621
 622     def __repr__(self):
 623         return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x vtx=%s)" \
 624             % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
 625                time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx))
 626
 627
 628 class PrefilledTransaction():
 629     def __init__(self, index=0, tx = None):
 630         self.index = index
 631         self.tx = tx
 632
 633     def deserialize(self, f):
 634         self.index = deser_compact_size(f)
 635         self.tx = CTransaction()
 636         self.tx.deserialize(f)
 637
 638     def serialize(self, with_witness=False):
 639         r = b""
 640         r += ser_compact_size(self.index)
 641         if with_witness:
 642             r += self.tx.serialize_with_witness()
 643         else:
 644             r += self.tx.serialize_without_witness()
 645         return r
 646
 647     def serialize_with_witness(self):
 648         return self.serialize(with_witness=True)
 649
 650     def __repr__(self):
 651         return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx))
 652
 653 # This is what we send on the wire, in a cmpctblock message.
 654 class P2PHeaderAndShortIDs():
 655     def __init__(self):
 656         self.header = CBlockHeader()
 657         self.nonce = 0
 658         self.shortids_length = 0
 659         self.shortids = []
 660         self.prefilled_txn_length = 0
 661         self.prefilled_txn = []
 662
 663     def deserialize(self, f):
 664         self.header.deserialize(f)
 665         self.nonce = struct.unpack("<Q", f.read(8))[0]
 666         self.shortids_length = deser_compact_size(f)
 667         for i in range(self.shortids_length):
 668             # shortids are defined to be 6 bytes in the spec, so append
 669             # two zero bytes and read it in as an 8-byte number
 670             self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0])
 671         self.prefilled_txn = deser_vector(f, PrefilledTransaction)
 672         self.prefilled_txn_length = len(self.prefilled_txn)
 673
 674     # When using version 2 compact blocks, we must serialize with_witness.
 675     def serialize(self, with_witness=False):
 676         r = b""
 677         r += self.header.serialize()
 678         r += struct.pack("<Q", self.nonce)
 679         r += ser_compact_size(self.shortids_length)
 680         for x in self.shortids:
 681             # We only want the first 6 bytes
 682             r += struct.pack("<Q", x)[0:6]
 683         if with_witness:
 684             r += ser_vector(self.prefilled_txn, "serialize_with_witness")
 685         else:
 686             r += ser_vector(self.prefilled_txn)
 687         return r
 688
 689     def __repr__(self):
 690         return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn))
 691
 692 # P2P version of the above that will use witness serialization (for compact
 693 # block version 2)
 694 class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs):
 695     def serialize(self):
 696         return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True)
 697
 698 # Calculate the BIP 152-compact blocks shortid for a given transaction hash
 699 def calculate_shortid(k0, k1, tx_hash):
 700     expected_shortid = siphash256(k0, k1, tx_hash)
 701     expected_shortid &= 0x0000ffffffffffff
 702     return expected_shortid
 703
 704 # This version gets rid of the array lengths, and reinterprets the differential
 705 # encoding into indices that can be used for lookup.
 706 class HeaderAndShortIDs():
 707     def __init__(self, p2pheaders_and_shortids = None):
 708         self.header = CBlockHeader()
 709         self.nonce = 0
 710         self.shortids = []
 711         self.prefilled_txn = []
 712         self.use_witness = False
 713
 714         if p2pheaders_and_shortids != None:
 715             self.header = p2pheaders_and_shortids.header
 716             self.nonce = p2pheaders_and_shortids.nonce
 717             self.shortids = p2pheaders_and_shortids.shortids
 718             last_index = -1
 719             for x in p2pheaders_and_shortids.prefilled_txn:
 720                 self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx))
 721                 last_index = self.prefilled_txn[-1].index
 722
 723     def to_p2p(self):
 724         if self.use_witness:
 725             ret = P2PHeaderAndShortWitnessIDs()
 726         else:
 727             ret = P2PHeaderAndShortIDs()
 728         ret.header = self.header
 729         ret.nonce = self.nonce
 730         ret.shortids_length = len(self.shortids)
 731         ret.shortids = self.shortids
 732         ret.prefilled_txn_length = len(self.prefilled_txn)
 733         ret.prefilled_txn = []
 734         last_index = -1
 735         for x in self.prefilled_txn:
 736             ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx))
 737             last_index = x.index
 738         return ret
 739
 740     def get_siphash_keys(self):
 741         header_nonce = self.header.serialize()
 742         header_nonce += struct.pack("<Q", self.nonce)
 743         hash_header_nonce_as_str = sha256(header_nonce)
 744         key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0]
 745         key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0]
 746         return [ key0, key1 ]
 747
 748     # Version 2 compact blocks use wtxid in shortids (rather than txid)
 749     def initialize_from_block(self, block, nonce=0, prefill_list = [0], use_witness = False):
 750         self.header = CBlockHeader(block)
 751         self.nonce = nonce
 752         self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ]
 753         self.shortids = []
 754         self.use_witness = use_witness
 755         [k0, k1] = self.get_siphash_keys()
 756         for i in range(len(block.vtx)):
 757             if i not in prefill_list:
 758                 tx_hash = block.vtx[i].sha256
 759                 if use_witness:
 760                     tx_hash = block.vtx[i].calc_sha256(with_witness=True)
 761                 self.shortids.append(calculate_shortid(k0, k1, tx_hash))
 762
 763     def __repr__(self):
 764         return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn))
 765
 766
 767 class BlockTransactionsRequest():
 768
 769     def __init__(self, blockhash=0, indexes = None):
 770         self.blockhash = blockhash
 771         self.indexes = indexes if indexes != None else []
 772
 773     def deserialize(self, f):
 774         self.blockhash = deser_uint256(f)
 775         indexes_length = deser_compact_size(f)
 776         for i in range(indexes_length):
 777             self.indexes.append(deser_compact_size(f))
 778
 779     def serialize(self):
 780         r = b""
 781         r += ser_uint256(self.blockhash)
 782         r += ser_compact_size(len(self.indexes))
 783         for x in self.indexes:
 784             r += ser_compact_size(x)
 785         return r
 786
 787     # helper to set the differentially encoded indexes from absolute ones
 788     def from_absolute(self, absolute_indexes):
 789         self.indexes = []
 790         last_index = -1
 791         for x in absolute_indexes:
 792             self.indexes.append(x-last_index-1)
 793             last_index = x
 794
 795     def to_absolute(self):
 796         absolute_indexes = []
 797         last_index = -1
 798         for x in self.indexes:
 799             absolute_indexes.append(x+last_index+1)
 800             last_index = absolute_indexes[-1]
 801         return absolute_indexes
 802
 803     def __repr__(self):
 804         return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes))
 805
 806
 807 class BlockTransactions():
 808
 809     def __init__(self, blockhash=0, transactions = None):
 810         self.blockhash = blockhash
 811         self.transactions = transactions if transactions != None else []
 812
 813     def deserialize(self, f):
 814         self.blockhash = deser_uint256(f)
 815         self.transactions = deser_vector(f, CTransaction)
 816
 817     def serialize(self, with_witness=False):
 818         r = b""
 819         r += ser_uint256(self.blockhash)
 820         if with_witness:
 821             r += ser_vector(self.transactions, "serialize_with_witness")
 822         else:
 823             r += ser_vector(self.transactions)
 824         return r
 825
 826     def __repr__(self):
 827         return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions))
 828
 829
 830 # Objects that correspond to messages on the wire
 831 class msg_version():
 832     command = b"version"
 833
 834     def __init__(self):
 835         self.nVersion = MY_VERSION
 836         self.nServices = NODE_NETWORK | NODE_WITNESS
 837         self.nTime = int(time.time())
 838         self.addrTo = CAddress()
 839         self.addrFrom = CAddress()
 840         self.nNonce = random.getrandbits(64)
 841         self.strSubVer = MY_SUBVERSION
 842         self.nStartingHeight = -1
 843         self.nRelay = MY_RELAY
 844
 845     def deserialize(self, f):
 846         self.nVersion = struct.unpack("<i", f.read(4))[0]
 847         if self.nVersion == 10300:
 848             self.nVersion = 300
 849         self.nServices = struct.unpack("<Q", f.read(8))[0]
 850         self.nTime = struct.unpack("<q", f.read(8))[0]
 851         self.addrTo = CAddress()
 852         self.addrTo.deserialize(f)
 853
 854         if self.nVersion >= 106:
 855             self.addrFrom = CAddress()
 856             self.addrFrom.deserialize(f)
 857             self.nNonce = struct.unpack("<Q", f.read(8))[0]
 858             self.strSubVer = deser_string(f)
 859         else:
 860             self.addrFrom = None
 861             self.nNonce = None
 862             self.strSubVer = None
 863             self.nStartingHeight = None
 864
 865         if self.nVersion >= 209:
 866             self.nStartingHeight = struct.unpack("<i", f.read(4))[0]
 867         else:
 868             self.nStartingHeight = None
 869
 870         if self.nVersion >= 70001:
 871             # Relay field is optional for version 70001 onwards
 872             try:
 873                 self.nRelay = struct.unpack("<b", f.read(1))[0]
 874             except:
 875                 self.nRelay = 0
 876         else:
 877             self.nRelay = 0
 878
 879     def serialize(self):
 880         r = b""
 881         r += struct.pack("<i", self.nVersion)
 882         r += struct.pack("<Q", self.nServices)
 883         r += struct.pack("<q", self.nTime)
 884         r += self.addrTo.serialize()
 885         r += self.addrFrom.serialize()
 886         r += struct.pack("<Q", self.nNonce)
 887         r += ser_string(self.strSubVer)
 888         r += struct.pack("<i", self.nStartingHeight)
 889         r += struct.pack("<b", self.nRelay)
 890         return r
 891
 892     def __repr__(self):
 893         return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i nRelay=%i)' \
 894             % (self.nVersion, self.nServices, time.ctime(self.nTime),
 895                repr(self.addrTo), repr(self.addrFrom), self.nNonce,
 896                self.strSubVer, self.nStartingHeight, self.nRelay)
 897
 898
 899 class msg_verack():
 900     command = b"verack"
 901
 902     def __init__(self):
 903         pass
 904
 905     def deserialize(self, f):
 906         pass
 907
 908     def serialize(self):
 909         return b""
 910
 911     def __repr__(self):
 912         return "msg_verack()"
 913
 914
 915 class msg_addr():
 916     command = b"addr"
 917
 918     def __init__(self):
 919         self.addrs = []
 920
 921     def deserialize(self, f):
 922         self.addrs = deser_vector(f, CAddress)
 923
 924     def serialize(self):
 925         return ser_vector(self.addrs)
 926
 927     def __repr__(self):
 928         return "msg_addr(addrs=%s)" % (repr(self.addrs))
 929
 930
 931 class msg_inv():
 932     command = b"inv"
 933
 934     def __init__(self, inv=None):
 935         if inv is None:
 936             self.inv = []
 937         else:
 938             self.inv = inv
 939
 940     def deserialize(self, f):
 941         self.inv = deser_vector(f, CInv)
 942
 943     def serialize(self):
 944         return ser_vector(self.inv)
 945
 946     def __repr__(self):
 947         return "msg_inv(inv=%s)" % (repr(self.inv))
 948
 949
 950 class msg_getdata():
 951     command = b"getdata"
 952
 953     def __init__(self, inv=None):
 954         self.inv = inv if inv != None else []
 955
 956     def deserialize(self, f):
 957         self.inv = deser_vector(f, CInv)
 958
 959     def serialize(self):
 960         return ser_vector(self.inv)
 961
 962     def __repr__(self):
 963         return "msg_getdata(inv=%s)" % (repr(self.inv))
 964
 965
 966 class msg_getblocks():
 967     command = b"getblocks"
 968
 969     def __init__(self):
 970         self.locator = CBlockLocator()
 971         self.hashstop = 0
 972
 973     def deserialize(self, f):
 974         self.locator = CBlockLocator()
 975         self.locator.deserialize(f)
 976         self.hashstop = deser_uint256(f)
 977
 978     def serialize(self):
 979         r = b""
 980         r += self.locator.serialize()
 981         r += ser_uint256(self.hashstop)
 982         return r
 983
 984     def __repr__(self):
 985         return "msg_getblocks(locator=%s hashstop=%064x)" \
 986             % (repr(self.locator), self.hashstop)
 987
 988
 989 class msg_tx():
 990     command = b"tx"
 991
 992     def __init__(self, tx=CTransaction()):
 993         self.tx = tx
 994
 995     def deserialize(self, f):
 996         self.tx.deserialize(f)
 997
 998     def serialize(self):
 999         return self.tx.serialize_without_witness()
1000
1001     def __repr__(self):
1002         return "msg_tx(tx=%s)" % (repr(self.tx))
1003
1004 class msg_witness_tx(msg_tx):
1005
1006     def serialize(self):
1007         return self.tx.serialize_with_witness()
1008
1009
1010 class msg_block():
1011     command = b"block"
1012
1013     def __init__(self, block=None):
1014         if block is None:
1015             self.block = CBlock()
1016         else:
1017             self.block = block
1018
1019     def deserialize(self, f):
1020         self.block.deserialize(f)
1021
1022     def serialize(self):
1023         return self.block.serialize()
1024
1025     def __repr__(self):
1026         return "msg_block(block=%s)" % (repr(self.block))
1027
1028 # for cases where a user needs tighter control over what is sent over the wire
1029 # note that the user must supply the name of the command, and the data
1030 class msg_generic():
1031     def __init__(self, command, data=None):
1032         self.command = command
1033         self.data = data
1034
1035     def serialize(self):
1036         return self.data
1037
1038     def __repr__(self):
1039         return "msg_generic()"
1040
1041 class msg_witness_block(msg_block):
1042
1043     def serialize(self):
1044         r = self.block.serialize(with_witness=True)
1045         return r
1046
1047 class msg_getaddr():
1048     command = b"getaddr"
1049
1050     def __init__(self):
1051         pass
1052
1053     def deserialize(self, f):
1054         pass
1055
1056     def serialize(self):
1057         return b""
1058
1059     def __repr__(self):
1060         return "msg_getaddr()"
1061
1062
1063 class msg_ping():
1064     command = b"ping"
1065
1066     def __init__(self, nonce=0):
1067         self.nonce = nonce
1068
1069     def deserialize(self, f):
1070         self.nonce = struct.unpack("<Q", f.read(8))[0]
1071
1072     def serialize(self):
1073         r = b""
1074         r += struct.pack("<Q", self.nonce)
1075         return r
1076
1077     def __repr__(self):
1078         return "msg_ping(nonce=%08x)" % self.nonce
1079
1080
1081 class msg_pong():
1082     command = b"pong"
1083
1084     def __init__(self, nonce=0):
1085         self.nonce = nonce
1086
1087     def deserialize(self, f):
1088         self.nonce = struct.unpack("<Q", f.read(8))[0]
1089
1090     def serialize(self):
1091         r = b""
1092         r += struct.pack("<Q", self.nonce)
1093         return r
1094
1095     def __repr__(self):
1096         return "msg_pong(nonce=%08x)" % self.nonce
1097
1098
1099 class msg_mempool():
1100     command = b"mempool"
1101
1102     def __init__(self):
1103         pass
1104
1105     def deserialize(self, f):
1106         pass
1107
1108     def serialize(self):
1109         return b""
1110
1111     def __repr__(self):
1112         return "msg_mempool()"
1113
1114 class msg_sendheaders():
1115     command = b"sendheaders"
1116
1117     def __init__(self):
1118         pass
1119
1120     def deserialize(self, f):
1121         pass
1122
1123     def serialize(self):
1124         return b""
1125
1126     def __repr__(self):
1127         return "msg_sendheaders()"
1128
1129
1130 # getheaders message has
1131 # number of entries
1132 # vector of hashes
1133 # hash_stop (hash of last desired block header, 0 to get as many as possible)
1134 class msg_getheaders():
1135     command = b"getheaders"
1136
1137     def __init__(self):
1138         self.locator = CBlockLocator()
1139         self.hashstop = 0
1140
1141     def deserialize(self, f):
1142         self.locator = CBlockLocator()
1143         self.locator.deserialize(f)
1144         self.hashstop = deser_uint256(f)
1145
1146     def serialize(self):
1147         r = b""
1148         r += self.locator.serialize()
1149         r += ser_uint256(self.hashstop)
1150         return r
1151
1152     def __repr__(self):
1153         return "msg_getheaders(locator=%s, stop=%064x)" \
1154             % (repr(self.locator), self.hashstop)
1155
1156
1157 # headers message has
1158 # <count> <vector of block headers>
1159 class msg_headers():
1160     command = b"headers"
1161
1162     def __init__(self, headers=None):
1163         self.headers = headers if headers is not None else []
1164
1165     def deserialize(self, f):
1166         # comment in bitcoind indicates these should be deserialized as blocks
1167         blocks = deser_vector(f, CBlock)
1168         for x in blocks:
1169             self.headers.append(CBlockHeader(x))
1170
1171     def serialize(self):
1172         blocks = [CBlock(x) for x in self.headers]
1173         return ser_vector(blocks)
1174
1175     def __repr__(self):
1176         return "msg_headers(headers=%s)" % repr(self.headers)
1177
1178
1179 class msg_reject():
1180     command = b"reject"
1181     REJECT_MALFORMED = 1
1182
1183     def __init__(self):
1184         self.message = b""
1185         self.code = 0
1186         self.reason = b""
1187         self.data = 0
1188
1189     def deserialize(self, f):
1190         self.message = deser_string(f)
1191         self.code = struct.unpack("<B", f.read(1))[0]
1192         self.reason = deser_string(f)
1193         if (self.code != self.REJECT_MALFORMED and
1194                 (self.message == b"block" or self.message == b"tx")):
1195             self.data = deser_uint256(f)
1196
1197     def serialize(self):
1198         r = ser_string(self.message)
1199         r += struct.pack("<B", self.code)
1200         r += ser_string(self.reason)
1201         if (self.code != self.REJECT_MALFORMED and
1202                 (self.message == b"block" or self.message == b"tx")):
1203             r += ser_uint256(self.data)
1204         return r
1205
1206     def __repr__(self):
1207         return "msg_reject: %s %d %s [%064x]" \
1208             % (self.message, self.code, self.reason, self.data)
1209
1210 class msg_feefilter():
1211     command = b"feefilter"
1212
1213     def __init__(self, feerate=0):
1214         self.feerate = feerate
1215
1216     def deserialize(self, f):
1217         self.feerate = struct.unpack("<Q", f.read(8))[0]
1218
1219     def serialize(self):
1220         r = b""
1221         r += struct.pack("<Q", self.feerate)
1222         return r
1223
1224     def __repr__(self):
1225         return "msg_feefilter(feerate=%08x)" % self.feerate
1226
1227 class msg_sendcmpct():
1228     command = b"sendcmpct"
1229
1230     def __init__(self):
1231         self.announce = False
1232         self.version = 1
1233
1234     def deserialize(self, f):
1235         self.announce = struct.unpack("<?", f.read(1))[0]
1236         self.version = struct.unpack("<Q", f.read(8))[0]
1237
1238     def serialize(self):
1239         r = b""
1240         r += struct.pack("<?", self.announce)
1241         r += struct.pack("<Q", self.version)
1242         return r
1243
1244     def __repr__(self):
1245         return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version)
1246
1247 class msg_cmpctblock():
1248     command = b"cmpctblock"
1249
1250     def __init__(self, header_and_shortids = None):
1251         self.header_and_shortids = header_and_shortids
1252
1253     def deserialize(self, f):
1254         self.header_and_shortids = P2PHeaderAndShortIDs()
1255         self.header_and_shortids.deserialize(f)
1256
1257     def serialize(self):
1258         r = b""
1259         r += self.header_and_shortids.serialize()
1260         return r
1261
1262     def __repr__(self):
1263         return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids)
1264
1265 class msg_getblocktxn():
1266     command = b"getblocktxn"
1267
1268     def __init__(self):
1269         self.block_txn_request = None
1270
1271     def deserialize(self, f):
1272         self.block_txn_request = BlockTransactionsRequest()
1273         self.block_txn_request.deserialize(f)
1274
1275     def serialize(self):
1276         r = b""
1277         r += self.block_txn_request.serialize()
1278         return r
1279
1280     def __repr__(self):
1281         return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request))
1282
1283 class msg_blocktxn():
1284     command = b"blocktxn"
1285
1286     def __init__(self):
1287         self.block_transactions = BlockTransactions()
1288
1289     def deserialize(self, f):
1290         self.block_transactions.deserialize(f)
1291
1292     def serialize(self):
1293         r = b""
1294         r += self.block_transactions.serialize()
1295         return r
1296
1297     def __repr__(self):
1298         return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions))
1299
1300 class msg_witness_blocktxn(msg_blocktxn):
1301     def serialize(self):
1302         r = b""
1303         r += self.block_transactions.serialize(with_witness=True)
1304         return r