test/functional/test_framework/messages.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2010 ArtForz -- public domain half-a-node
   3 # Copyright (c) 2012 Jeff Garzik
   4 # Copyright (c) 2010-2017 The Bitcoin Core developers
   5 # Distributed under the MIT software license, see the accompanying
   6 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   7 """Bitcoin test framework primitive and message strcutures
   8
   9 CBlock, CTransaction, CBlockHeader, CTxIn, CTxOut, etc....:
  10     data structures that should map to corresponding structures in
  11     bitcoin/primitives
  12
  13 msg_block, msg_tx, msg_headers, etc.:
  14     data structures that represent network messages
  15
  16 ser_*, deser_*: functions that handle serialization/deserialization."""
  17 from codecs import encode
  18 import copy
  19 import hashlib
  20 from io import BytesIO
  21 import random
  22 import socket
  23 import struct
  24 import time
  25
  26 from test_framework.siphash import siphash256
  27 from test_framework.util import hex_str_to_bytes, bytes_to_hex_str
  28
  29 MIN_VERSION_SUPPORTED = 60001
  30 MY_VERSION = 70014  # past bip-31 for ping/pong
  31 MY_SUBVERSION = b"/python-mininode-tester:0.0.3/"
  32 MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37)
  33
  34 MAX_INV_SZ = 50000
  35 MAX_BLOCK_BASE_SIZE = 1000000
  36
  37 COIN = 100000000 # 1 btc in satoshis
  38
  39 NODE_NETWORK = (1 << 0)
  40 # NODE_GETUTXO = (1 << 1)
  41 NODE_BLOOM = (1 << 2)
  42 NODE_WITNESS = (1 << 3)
  43 NODE_UNSUPPORTED_SERVICE_BIT_5 = (1 << 5)
  44 NODE_UNSUPPORTED_SERVICE_BIT_7 = (1 << 7)
  45 NODE_NETWORK_LIMITED = (1 << 10)
  46
  47 # Serialization/deserialization tools
  48 def sha256(s):
  49     return hashlib.new('sha256', s).digest()
  50
  51 def ripemd160(s):
  52     return hashlib.new('ripemd160', s).digest()
  53
  54 def hash256(s):
  55     return sha256(sha256(s))
  56
  57 def ser_compact_size(l):
  58     r = b""
  59     if l < 253:
  60         r = struct.pack("B", l)
  61     elif l < 0x10000:
  62         r = struct.pack("<BH", 253, l)
  63     elif l < 0x100000000:
  64         r = struct.pack("<BI", 254, l)
  65     else:
  66         r = struct.pack("<BQ", 255, l)
  67     return r
  68
  69 def deser_compact_size(f):
  70     nit = struct.unpack("<B", f.read(1))[0]
  71     if nit == 253:
  72         nit = struct.unpack("<H", f.read(2))[0]
  73     elif nit == 254:
  74         nit = struct.unpack("<I", f.read(4))[0]
  75     elif nit == 255:
  76         nit = struct.unpack("<Q", f.read(8))[0]
  77     return nit
  78
  79 def deser_string(f):
  80     nit = deser_compact_size(f)
  81     return f.read(nit)
  82
  83 def ser_string(s):
  84     return ser_compact_size(len(s)) + s
  85
  86 def deser_uint256(f):
  87     r = 0
  88     for i in range(8):
  89         t = struct.unpack("<I", f.read(4))[0]
  90         r += t << (i * 32)
  91     return r
  92
  93
  94 def ser_uint256(u):
  95     rs = b""
  96     for i in range(8):
  97         rs += struct.pack("<I", u & 0xFFFFFFFF)
  98         u >>= 32
  99     return rs
 100
 101
 102 def uint256_from_str(s):
 103     r = 0
 104     t = struct.unpack("<IIIIIIII", s[:32])
 105     for i in range(8):
 106         r += t[i] << (i * 32)
 107     return r
 108
 109
 110 def uint256_from_compact(c):
 111     nbytes = (c >> 24) & 0xFF
 112     v = (c & 0xFFFFFF) << (8 * (nbytes - 3))
 113     return v
 114
 115
 116 def deser_vector(f, c):
 117     nit = deser_compact_size(f)
 118     r = []
 119     for i in range(nit):
 120         t = c()
 121         t.deserialize(f)
 122         r.append(t)
 123     return r
 124
 125
 126 # ser_function_name: Allow for an alternate serialization function on the
 127 # entries in the vector (we use this for serializing the vector of transactions
 128 # for a witness block).
 129 def ser_vector(l, ser_function_name=None):
 130     r = ser_compact_size(len(l))
 131     for i in l:
 132         if ser_function_name:
 133             r += getattr(i, ser_function_name)()
 134         else:
 135             r += i.serialize()
 136     return r
 137
 138
 139 def deser_uint256_vector(f):
 140     nit = deser_compact_size(f)
 141     r = []
 142     for i in range(nit):
 143         t = deser_uint256(f)
 144         r.append(t)
 145     return r
 146
 147
 148 def ser_uint256_vector(l):
 149     r = ser_compact_size(len(l))
 150     for i in l:
 151         r += ser_uint256(i)
 152     return r
 153
 154
 155 def deser_string_vector(f):
 156     nit = deser_compact_size(f)
 157     r = []
 158     for i in range(nit):
 159         t = deser_string(f)
 160         r.append(t)
 161     return r
 162
 163
 164 def ser_string_vector(l):
 165     r = ser_compact_size(len(l))
 166     for sv in l:
 167         r += ser_string(sv)
 168     return r
 169
 170
 171 # Deserialize from a hex string representation (eg from RPC)
 172 def FromHex(obj, hex_string):
 173     obj.deserialize(BytesIO(hex_str_to_bytes(hex_string)))
 174     return obj
 175
 176 # Convert a binary-serializable object to hex (eg for submission via RPC)
 177 def ToHex(obj):
 178     return bytes_to_hex_str(obj.serialize())
 179
 180 # Objects that map to bitcoind objects, which can be serialized/deserialized
 181
 182 class CAddress():
 183     def __init__(self):
 184         self.nServices = 1
 185         self.pchReserved = b"\x00" * 10 + b"\xff" * 2
 186         self.ip = "0.0.0.0"
 187         self.port = 0
 188
 189     def deserialize(self, f):
 190         self.nServices = struct.unpack("<Q", f.read(8))[0]
 191         self.pchReserved = f.read(12)
 192         self.ip = socket.inet_ntoa(f.read(4))
 193         self.port = struct.unpack(">H", f.read(2))[0]
 194
 195     def serialize(self):
 196         r = b""
 197         r += struct.pack("<Q", self.nServices)
 198         r += self.pchReserved
 199         r += socket.inet_aton(self.ip)
 200         r += struct.pack(">H", self.port)
 201         return r
 202
 203     def __repr__(self):
 204         return "CAddress(nServices=%i ip=%s port=%i)" % (self.nServices,
 205                                                          self.ip, self.port)
 206
 207 MSG_WITNESS_FLAG = 1<<30
 208
 209 class CInv():
 210     typemap = {
 211         0: "Error",
 212         1: "TX",
 213         2: "Block",
 214         1|MSG_WITNESS_FLAG: "WitnessTx",
 215         2|MSG_WITNESS_FLAG : "WitnessBlock",
 216         4: "CompactBlock"
 217     }
 218
 219     def __init__(self, t=0, h=0):
 220         self.type = t
 221         self.hash = h
 222
 223     def deserialize(self, f):
 224         self.type = struct.unpack("<i", f.read(4))[0]
 225         self.hash = deser_uint256(f)
 226
 227     def serialize(self):
 228         r = b""
 229         r += struct.pack("<i", self.type)
 230         r += ser_uint256(self.hash)
 231         return r
 232
 233     def __repr__(self):
 234         return "CInv(type=%s hash=%064x)" \
 235             % (self.typemap[self.type], self.hash)
 236
 237
 238 class CBlockLocator():
 239     def __init__(self):
 240         self.nVersion = MY_VERSION
 241         self.vHave = []
 242
 243     def deserialize(self, f):
 244         self.nVersion = struct.unpack("<i", f.read(4))[0]
 245         self.vHave = deser_uint256_vector(f)
 246
 247     def serialize(self):
 248         r = b""
 249         r += struct.pack("<i", self.nVersion)
 250         r += ser_uint256_vector(self.vHave)
 251         return r
 252
 253     def __repr__(self):
 254         return "CBlockLocator(nVersion=%i vHave=%s)" \
 255             % (self.nVersion, repr(self.vHave))
 256
 257
 258 class COutPoint():
 259     def __init__(self, hash=0, n=0):
 260         self.hash = hash
 261         self.n = n
 262
 263     def deserialize(self, f):
 264         self.hash = deser_uint256(f)
 265         self.n = struct.unpack("<I", f.read(4))[0]
 266
 267     def serialize(self):
 268         r = b""
 269         r += ser_uint256(self.hash)
 270         r += struct.pack("<I", self.n)
 271         return r
 272
 273     def __repr__(self):
 274         return "COutPoint(hash=%064x n=%i)" % (self.hash, self.n)
 275
 276
 277 class CTxIn():
 278     def __init__(self, outpoint=None, scriptSig=b"", nSequence=0):
 279         if outpoint is None:
 280             self.prevout = COutPoint()
 281         else:
 282             self.prevout = outpoint
 283         self.scriptSig = scriptSig
 284         self.nSequence = nSequence
 285
 286     def deserialize(self, f):
 287         self.prevout = COutPoint()
 288         self.prevout.deserialize(f)
 289         self.scriptSig = deser_string(f)
 290         self.nSequence = struct.unpack("<I", f.read(4))[0]
 291
 292     def serialize(self):
 293         r = b""
 294         r += self.prevout.serialize()
 295         r += ser_string(self.scriptSig)
 296         r += struct.pack("<I", self.nSequence)
 297         return r
 298
 299     def __repr__(self):
 300         return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \
 301             % (repr(self.prevout), bytes_to_hex_str(self.scriptSig),
 302                self.nSequence)
 303
 304
 305 class CTxOut():
 306     def __init__(self, nValue=0, scriptPubKey=b""):
 307         self.nValue = nValue
 308         self.scriptPubKey = scriptPubKey
 309
 310     def deserialize(self, f):
 311         self.nValue = struct.unpack("<q", f.read(8))[0]
 312         self.scriptPubKey = deser_string(f)
 313
 314     def serialize(self):
 315         r = b""
 316         r += struct.pack("<q", self.nValue)
 317         r += ser_string(self.scriptPubKey)
 318         return r
 319
 320     def __repr__(self):
 321         return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \
 322             % (self.nValue // COIN, self.nValue % COIN,
 323                bytes_to_hex_str(self.scriptPubKey))
 324
 325
 326 class CScriptWitness():
 327     def __init__(self):
 328         # stack is a vector of strings
 329         self.stack = []
 330
 331     def __repr__(self):
 332         return "CScriptWitness(%s)" % \
 333                (",".join([bytes_to_hex_str(x) for x in self.stack]))
 334
 335     def is_null(self):
 336         if self.stack:
 337             return False
 338         return True
 339
 340
 341 class CTxInWitness():
 342     def __init__(self):
 343         self.scriptWitness = CScriptWitness()
 344
 345     def deserialize(self, f):
 346         self.scriptWitness.stack = deser_string_vector(f)
 347
 348     def serialize(self):
 349         return ser_string_vector(self.scriptWitness.stack)
 350
 351     def __repr__(self):
 352         return repr(self.scriptWitness)
 353
 354     def is_null(self):
 355         return self.scriptWitness.is_null()
 356
 357
 358 class CTxWitness():
 359     def __init__(self):
 360         self.vtxinwit = []
 361
 362     def deserialize(self, f):
 363         for i in range(len(self.vtxinwit)):
 364             self.vtxinwit[i].deserialize(f)
 365
 366     def serialize(self):
 367         r = b""
 368         # This is different than the usual vector serialization --
 369         # we omit the length of the vector, which is required to be
 370         # the same length as the transaction's vin vector.
 371         for x in self.vtxinwit:
 372             r += x.serialize()
 373         return r
 374
 375     def __repr__(self):
 376         return "CTxWitness(%s)" % \
 377                (';'.join([repr(x) for x in self.vtxinwit]))
 378
 379     def is_null(self):
 380         for x in self.vtxinwit:
 381             if not x.is_null():
 382                 return False
 383         return True
 384
 385
 386 class CTransaction():
 387     def __init__(self, tx=None):
 388         if tx is None:
 389             self.nVersion = 1
 390             self.vin = []
 391             self.vout = []
 392             self.wit = CTxWitness()
 393             self.nLockTime = 0
 394             self.sha256 = None
 395             self.hash = None
 396         else:
 397             self.nVersion = tx.nVersion
 398             self.vin = copy.deepcopy(tx.vin)
 399             self.vout = copy.deepcopy(tx.vout)
 400             self.nLockTime = tx.nLockTime
 401             self.sha256 = tx.sha256
 402             self.hash = tx.hash
 403             self.wit = copy.deepcopy(tx.wit)
 404
 405     def deserialize(self, f):
 406         self.nVersion = struct.unpack("<i", f.read(4))[0]
 407         self.vin = deser_vector(f, CTxIn)
 408         flags = 0
 409         if len(self.vin) == 0:
 410             flags = struct.unpack("<B", f.read(1))[0]
 411             # Not sure why flags can't be zero, but this
 412             # matches the implementation in bitcoind
 413             if (flags != 0):
 414                 self.vin = deser_vector(f, CTxIn)
 415                 self.vout = deser_vector(f, CTxOut)
 416         else:
 417             self.vout = deser_vector(f, CTxOut)
 418         if flags != 0:
 419             self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))]
 420             self.wit.deserialize(f)
 421         self.nLockTime = struct.unpack("<I", f.read(4))[0]
 422         self.sha256 = None
 423         self.hash = None
 424
 425     def serialize_without_witness(self):
 426         r = b""
 427         r += struct.pack("<i", self.nVersion)
 428         r += ser_vector(self.vin)
 429         r += ser_vector(self.vout)
 430         r += struct.pack("<I", self.nLockTime)
 431         return r
 432
 433     # Only serialize with witness when explicitly called for
 434     def serialize_with_witness(self):
 435         flags = 0
 436         if not self.wit.is_null():
 437             flags |= 1
 438         r = b""
 439         r += struct.pack("<i", self.nVersion)
 440         if flags:
 441             dummy = []
 442             r += ser_vector(dummy)
 443             r += struct.pack("<B", flags)
 444         r += ser_vector(self.vin)
 445         r += ser_vector(self.vout)
 446         if flags & 1:
 447             if (len(self.wit.vtxinwit) != len(self.vin)):
 448                 # vtxinwit must have the same length as vin
 449                 self.wit.vtxinwit = self.wit.vtxinwit[:len(self.vin)]
 450                 for i in range(len(self.wit.vtxinwit), len(self.vin)):
 451                     self.wit.vtxinwit.append(CTxInWitness())
 452             r += self.wit.serialize()
 453         r += struct.pack("<I", self.nLockTime)
 454         return r
 455
 456     # Regular serialization is without witness -- must explicitly
 457     # call serialize_with_witness to include witness data.
 458     def serialize(self):
 459         return self.serialize_without_witness()
 460
 461     # Recalculate the txid (transaction hash without witness)
 462     def rehash(self):
 463         self.sha256 = None
 464         self.calc_sha256()
 465
 466     # We will only cache the serialization without witness in
 467     # self.sha256 and self.hash -- those are expected to be the txid.
 468     def calc_sha256(self, with_witness=False):
 469         if with_witness:
 470             # Don't cache the result, just return it
 471             return uint256_from_str(hash256(self.serialize_with_witness()))
 472
 473         if self.sha256 is None:
 474             self.sha256 = uint256_from_str(hash256(self.serialize_without_witness()))
 475         self.hash = encode(hash256(self.serialize())[::-1], 'hex_codec').decode('ascii')
 476
 477     def is_valid(self):
 478         self.calc_sha256()
 479         for tout in self.vout:
 480             if tout.nValue < 0 or tout.nValue > 21000000 * COIN:
 481                 return False
 482         return True
 483
 484     def __repr__(self):
 485         return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \
 486             % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime)
 487
 488
 489 class CBlockHeader():
 490     def __init__(self, header=None):
 491         if header is None:
 492             self.set_null()
 493         else:
 494             self.nVersion = header.nVersion
 495             self.hashPrevBlock = header.hashPrevBlock
 496             self.hashMerkleRoot = header.hashMerkleRoot
 497             self.nTime = header.nTime
 498             self.nBits = header.nBits
 499             self.nNonce = header.nNonce
 500             self.sha256 = header.sha256
 501             self.hash = header.hash
 502             self.calc_sha256()
 503
 504     def set_null(self):
 505         self.nVersion = 1
 506         self.hashPrevBlock = 0
 507         self.hashMerkleRoot = 0
 508         self.nTime = 0
 509         self.nBits = 0
 510         self.nNonce = 0
 511         self.sha256 = None
 512         self.hash = None
 513
 514     def deserialize(self, f):
 515         self.nVersion = struct.unpack("<i", f.read(4))[0]
 516         self.hashPrevBlock = deser_uint256(f)
 517         self.hashMerkleRoot = deser_uint256(f)
 518         self.nTime = struct.unpack("<I", f.read(4))[0]
 519         self.nBits = struct.unpack("<I", f.read(4))[0]
 520         self.nNonce = struct.unpack("<I", f.read(4))[0]
 521         self.sha256 = None
 522         self.hash = None
 523
 524     def serialize(self):
 525         r = b""
 526         r += struct.pack("<i", self.nVersion)
 527         r += ser_uint256(self.hashPrevBlock)
 528         r += ser_uint256(self.hashMerkleRoot)
 529         r += struct.pack("<I", self.nTime)
 530         r += struct.pack("<I", self.nBits)
 531         r += struct.pack("<I", self.nNonce)
 532         return r
 533
 534     def calc_sha256(self):
 535         if self.sha256 is None:
 536             r = b""
 537             r += struct.pack("<i", self.nVersion)
 538             r += ser_uint256(self.hashPrevBlock)
 539             r += ser_uint256(self.hashMerkleRoot)
 540             r += struct.pack("<I", self.nTime)
 541             r += struct.pack("<I", self.nBits)
 542             r += struct.pack("<I", self.nNonce)
 543             self.sha256 = uint256_from_str(hash256(r))
 544             self.hash = encode(hash256(r)[::-1], 'hex_codec').decode('ascii')
 545
 546     def rehash(self):
 547         self.sha256 = None
 548         self.calc_sha256()
 549         return self.sha256
 550
 551     def __repr__(self):
 552         return "CBlockHeader(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x)" \
 553             % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
 554                time.ctime(self.nTime), self.nBits, self.nNonce)
 555
 556
 557 class CBlock(CBlockHeader):
 558     def __init__(self, header=None):
 559         super(CBlock, self).__init__(header)
 560         self.vtx = []
 561
 562     def deserialize(self, f):
 563         super(CBlock, self).deserialize(f)
 564         self.vtx = deser_vector(f, CTransaction)
 565
 566     def serialize(self, with_witness=False):
 567         r = b""
 568         r += super(CBlock, self).serialize()
 569         if with_witness:
 570             r += ser_vector(self.vtx, "serialize_with_witness")
 571         else:
 572             r += ser_vector(self.vtx)
 573         return r
 574
 575     # Calculate the merkle root given a vector of transaction hashes
 576     @classmethod
 577     def get_merkle_root(cls, hashes):
 578         while len(hashes) > 1:
 579             newhashes = []
 580             for i in range(0, len(hashes), 2):
 581                 i2 = min(i+1, len(hashes)-1)
 582                 newhashes.append(hash256(hashes[i] + hashes[i2]))
 583             hashes = newhashes
 584         return uint256_from_str(hashes[0])
 585
 586     def calc_merkle_root(self):
 587         hashes = []
 588         for tx in self.vtx:
 589             tx.calc_sha256()
 590             hashes.append(ser_uint256(tx.sha256))
 591         return self.get_merkle_root(hashes)
 592
 593     def calc_witness_merkle_root(self):
 594         # For witness root purposes, the hash of the
 595         # coinbase, with witness, is defined to be 0...0
 596         hashes = [ser_uint256(0)]
 597
 598         for tx in self.vtx[1:]:
 599             # Calculate the hashes with witness data
 600             hashes.append(ser_uint256(tx.calc_sha256(True)))
 601
 602         return self.get_merkle_root(hashes)
 603
 604     def is_valid(self):
 605         self.calc_sha256()
 606         target = uint256_from_compact(self.nBits)
 607         if self.sha256 > target:
 608             return False
 609         for tx in self.vtx:
 610             if not tx.is_valid():
 611                 return False
 612         if self.calc_merkle_root() != self.hashMerkleRoot:
 613             return False
 614         return True
 615
 616     def solve(self):
 617         self.rehash()
 618         target = uint256_from_compact(self.nBits)
 619         while self.sha256 > target:
 620             self.nNonce += 1
 621             self.rehash()
 622
 623     def __repr__(self):
 624         return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x vtx=%s)" \
 625             % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
 626                time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx))
 627
 628
 629 class PrefilledTransaction():
 630     def __init__(self, index=0, tx = None):
 631         self.index = index
 632         self.tx = tx
 633
 634     def deserialize(self, f):
 635         self.index = deser_compact_size(f)
 636         self.tx = CTransaction()
 637         self.tx.deserialize(f)
 638
 639     def serialize(self, with_witness=False):
 640         r = b""
 641         r += ser_compact_size(self.index)
 642         if with_witness:
 643             r += self.tx.serialize_with_witness()
 644         else:
 645             r += self.tx.serialize_without_witness()
 646         return r
 647
 648     def serialize_with_witness(self):
 649         return self.serialize(with_witness=True)
 650
 651     def __repr__(self):
 652         return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx))
 653
 654 # This is what we send on the wire, in a cmpctblock message.
 655 class P2PHeaderAndShortIDs():
 656     def __init__(self):
 657         self.header = CBlockHeader()
 658         self.nonce = 0
 659         self.shortids_length = 0
 660         self.shortids = []
 661         self.prefilled_txn_length = 0
 662         self.prefilled_txn = []
 663
 664     def deserialize(self, f):
 665         self.header.deserialize(f)
 666         self.nonce = struct.unpack("<Q", f.read(8))[0]
 667         self.shortids_length = deser_compact_size(f)
 668         for i in range(self.shortids_length):
 669             # shortids are defined to be 6 bytes in the spec, so append
 670             # two zero bytes and read it in as an 8-byte number
 671             self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0])
 672         self.prefilled_txn = deser_vector(f, PrefilledTransaction)
 673         self.prefilled_txn_length = len(self.prefilled_txn)
 674
 675     # When using version 2 compact blocks, we must serialize with_witness.
 676     def serialize(self, with_witness=False):
 677         r = b""
 678         r += self.header.serialize()
 679         r += struct.pack("<Q", self.nonce)
 680         r += ser_compact_size(self.shortids_length)
 681         for x in self.shortids:
 682             # We only want the first 6 bytes
 683             r += struct.pack("<Q", x)[0:6]
 684         if with_witness:
 685             r += ser_vector(self.prefilled_txn, "serialize_with_witness")
 686         else:
 687             r += ser_vector(self.prefilled_txn)
 688         return r
 689
 690     def __repr__(self):
 691         return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn))
 692
 693 # P2P version of the above that will use witness serialization (for compact
 694 # block version 2)
 695 class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs):
 696     def serialize(self):
 697         return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True)
 698
 699 # Calculate the BIP 152-compact blocks shortid for a given transaction hash
 700 def calculate_shortid(k0, k1, tx_hash):
 701     expected_shortid = siphash256(k0, k1, tx_hash)
 702     expected_shortid &= 0x0000ffffffffffff
 703     return expected_shortid
 704
 705 # This version gets rid of the array lengths, and reinterprets the differential
 706 # encoding into indices that can be used for lookup.
 707 class HeaderAndShortIDs():
 708     def __init__(self, p2pheaders_and_shortids = None):
 709         self.header = CBlockHeader()
 710         self.nonce = 0
 711         self.shortids = []
 712         self.prefilled_txn = []
 713         self.use_witness = False
 714
 715         if p2pheaders_and_shortids != None:
 716             self.header = p2pheaders_and_shortids.header
 717             self.nonce = p2pheaders_and_shortids.nonce
 718             self.shortids = p2pheaders_and_shortids.shortids
 719             last_index = -1
 720             for x in p2pheaders_and_shortids.prefilled_txn:
 721                 self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx))
 722                 last_index = self.prefilled_txn[-1].index
 723
 724     def to_p2p(self):
 725         if self.use_witness:
 726             ret = P2PHeaderAndShortWitnessIDs()
 727         else:
 728             ret = P2PHeaderAndShortIDs()
 729         ret.header = self.header
 730         ret.nonce = self.nonce
 731         ret.shortids_length = len(self.shortids)
 732         ret.shortids = self.shortids
 733         ret.prefilled_txn_length = len(self.prefilled_txn)
 734         ret.prefilled_txn = []
 735         last_index = -1
 736         for x in self.prefilled_txn:
 737             ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx))
 738             last_index = x.index
 739         return ret
 740
 741     def get_siphash_keys(self):
 742         header_nonce = self.header.serialize()
 743         header_nonce += struct.pack("<Q", self.nonce)
 744         hash_header_nonce_as_str = sha256(header_nonce)
 745         key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0]
 746         key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0]
 747         return [ key0, key1 ]
 748
 749     # Version 2 compact blocks use wtxid in shortids (rather than txid)
 750     def initialize_from_block(self, block, nonce=0, prefill_list = [0], use_witness = False):
 751         self.header = CBlockHeader(block)
 752         self.nonce = nonce
 753         self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ]
 754         self.shortids = []
 755         self.use_witness = use_witness
 756         [k0, k1] = self.get_siphash_keys()
 757         for i in range(len(block.vtx)):
 758             if i not in prefill_list:
 759                 tx_hash = block.vtx[i].sha256
 760                 if use_witness:
 761                     tx_hash = block.vtx[i].calc_sha256(with_witness=True)
 762                 self.shortids.append(calculate_shortid(k0, k1, tx_hash))
 763
 764     def __repr__(self):
 765         return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn))
 766
 767
 768 class BlockTransactionsRequest():
 769
 770     def __init__(self, blockhash=0, indexes = None):
 771         self.blockhash = blockhash
 772         self.indexes = indexes if indexes != None else []
 773
 774     def deserialize(self, f):
 775         self.blockhash = deser_uint256(f)
 776         indexes_length = deser_compact_size(f)
 777         for i in range(indexes_length):
 778             self.indexes.append(deser_compact_size(f))
 779
 780     def serialize(self):
 781         r = b""
 782         r += ser_uint256(self.blockhash)
 783         r += ser_compact_size(len(self.indexes))
 784         for x in self.indexes:
 785             r += ser_compact_size(x)
 786         return r
 787
 788     # helper to set the differentially encoded indexes from absolute ones
 789     def from_absolute(self, absolute_indexes):
 790         self.indexes = []
 791         last_index = -1
 792         for x in absolute_indexes:
 793             self.indexes.append(x-last_index-1)
 794             last_index = x
 795
 796     def to_absolute(self):
 797         absolute_indexes = []
 798         last_index = -1
 799         for x in self.indexes:
 800             absolute_indexes.append(x+last_index+1)
 801             last_index = absolute_indexes[-1]
 802         return absolute_indexes
 803
 804     def __repr__(self):
 805         return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes))
 806
 807
 808 class BlockTransactions():
 809
 810     def __init__(self, blockhash=0, transactions = None):
 811         self.blockhash = blockhash
 812         self.transactions = transactions if transactions != None else []
 813
 814     def deserialize(self, f):
 815         self.blockhash = deser_uint256(f)
 816         self.transactions = deser_vector(f, CTransaction)
 817
 818     def serialize(self, with_witness=False):
 819         r = b""
 820         r += ser_uint256(self.blockhash)
 821         if with_witness:
 822             r += ser_vector(self.transactions, "serialize_with_witness")
 823         else:
 824             r += ser_vector(self.transactions)
 825         return r
 826
 827     def __repr__(self):
 828         return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions))
 829
 830
 831 # Objects that correspond to messages on the wire
 832 class msg_version():
 833     command = b"version"
 834
 835     def __init__(self):
 836         self.nVersion = MY_VERSION
 837         self.nServices = NODE_NETWORK | NODE_WITNESS
 838         self.nTime = int(time.time())
 839         self.addrTo = CAddress()
 840         self.addrFrom = CAddress()
 841         self.nNonce = random.getrandbits(64)
 842         self.strSubVer = MY_SUBVERSION
 843         self.nStartingHeight = -1
 844         self.nRelay = MY_RELAY
 845
 846     def deserialize(self, f):
 847         self.nVersion = struct.unpack("<i", f.read(4))[0]
 848         if self.nVersion == 10300:
 849             self.nVersion = 300
 850         self.nServices = struct.unpack("<Q", f.read(8))[0]
 851         self.nTime = struct.unpack("<q", f.read(8))[0]
 852         self.addrTo = CAddress()
 853         self.addrTo.deserialize(f)
 854
 855         if self.nVersion >= 106:
 856             self.addrFrom = CAddress()
 857             self.addrFrom.deserialize(f)
 858             self.nNonce = struct.unpack("<Q", f.read(8))[0]
 859             self.strSubVer = deser_string(f)
 860         else:
 861             self.addrFrom = None
 862             self.nNonce = None
 863             self.strSubVer = None
 864             self.nStartingHeight = None
 865
 866         if self.nVersion >= 209:
 867             self.nStartingHeight = struct.unpack("<i", f.read(4))[0]
 868         else:
 869             self.nStartingHeight = None
 870
 871         if self.nVersion >= 70001:
 872             # Relay field is optional for version 70001 onwards
 873             try:
 874                 self.nRelay = struct.unpack("<b", f.read(1))[0]
 875             except:
 876                 self.nRelay = 0
 877         else:
 878             self.nRelay = 0
 879
 880     def serialize(self):
 881         r = b""
 882         r += struct.pack("<i", self.nVersion)
 883         r += struct.pack("<Q", self.nServices)
 884         r += struct.pack("<q", self.nTime)
 885         r += self.addrTo.serialize()
 886         r += self.addrFrom.serialize()
 887         r += struct.pack("<Q", self.nNonce)
 888         r += ser_string(self.strSubVer)
 889         r += struct.pack("<i", self.nStartingHeight)
 890         r += struct.pack("<b", self.nRelay)
 891         return r
 892
 893     def __repr__(self):
 894         return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i nRelay=%i)' \
 895             % (self.nVersion, self.nServices, time.ctime(self.nTime),
 896                repr(self.addrTo), repr(self.addrFrom), self.nNonce,
 897                self.strSubVer, self.nStartingHeight, self.nRelay)
 898
 899
 900 class msg_verack():
 901     command = b"verack"
 902
 903     def __init__(self):
 904         pass
 905
 906     def deserialize(self, f):
 907         pass
 908
 909     def serialize(self):
 910         return b""
 911
 912     def __repr__(self):
 913         return "msg_verack()"
 914
 915
 916 class msg_addr():
 917     command = b"addr"
 918
 919     def __init__(self):
 920         self.addrs = []
 921
 922     def deserialize(self, f):
 923         self.addrs = deser_vector(f, CAddress)
 924
 925     def serialize(self):
 926         return ser_vector(self.addrs)
 927
 928     def __repr__(self):
 929         return "msg_addr(addrs=%s)" % (repr(self.addrs))
 930
 931
 932 class msg_inv():
 933     command = b"inv"
 934
 935     def __init__(self, inv=None):
 936         if inv is None:
 937             self.inv = []
 938         else:
 939             self.inv = inv
 940
 941     def deserialize(self, f):
 942         self.inv = deser_vector(f, CInv)
 943
 944     def serialize(self):
 945         return ser_vector(self.inv)
 946
 947     def __repr__(self):
 948         return "msg_inv(inv=%s)" % (repr(self.inv))
 949
 950
 951 class msg_getdata():
 952     command = b"getdata"
 953
 954     def __init__(self, inv=None):
 955         self.inv = inv if inv != None else []
 956
 957     def deserialize(self, f):
 958         self.inv = deser_vector(f, CInv)
 959
 960     def serialize(self):
 961         return ser_vector(self.inv)
 962
 963     def __repr__(self):
 964         return "msg_getdata(inv=%s)" % (repr(self.inv))
 965
 966
 967 class msg_getblocks():
 968     command = b"getblocks"
 969
 970     def __init__(self):
 971         self.locator = CBlockLocator()
 972         self.hashstop = 0
 973
 974     def deserialize(self, f):
 975         self.locator = CBlockLocator()
 976         self.locator.deserialize(f)
 977         self.hashstop = deser_uint256(f)
 978
 979     def serialize(self):
 980         r = b""
 981         r += self.locator.serialize()
 982         r += ser_uint256(self.hashstop)
 983         return r
 984
 985     def __repr__(self):
 986         return "msg_getblocks(locator=%s hashstop=%064x)" \
 987             % (repr(self.locator), self.hashstop)
 988
 989
 990 class msg_tx():
 991     command = b"tx"
 992
 993     def __init__(self, tx=CTransaction()):
 994         self.tx = tx
 995
 996     def deserialize(self, f):
 997         self.tx.deserialize(f)
 998
 999     def serialize(self):
1000         return self.tx.serialize_without_witness()
1001
1002     def __repr__(self):
1003         return "msg_tx(tx=%s)" % (repr(self.tx))
1004
1005 class msg_witness_tx(msg_tx):
1006
1007     def serialize(self):
1008         return self.tx.serialize_with_witness()
1009
1010
1011 class msg_block():
1012     command = b"block"
1013
1014     def __init__(self, block=None):
1015         if block is None:
1016             self.block = CBlock()
1017         else:
1018             self.block = block
1019
1020     def deserialize(self, f):
1021         self.block.deserialize(f)
1022
1023     def serialize(self):
1024         return self.block.serialize()
1025
1026     def __repr__(self):
1027         return "msg_block(block=%s)" % (repr(self.block))
1028
1029 # for cases where a user needs tighter control over what is sent over the wire
1030 # note that the user must supply the name of the command, and the data
1031 class msg_generic():
1032     def __init__(self, command, data=None):
1033         self.command = command
1034         self.data = data
1035
1036     def serialize(self):
1037         return self.data
1038
1039     def __repr__(self):
1040         return "msg_generic()"
1041
1042 class msg_witness_block(msg_block):
1043
1044     def serialize(self):
1045         r = self.block.serialize(with_witness=True)
1046         return r
1047
1048 class msg_getaddr():
1049     command = b"getaddr"
1050
1051     def __init__(self):
1052         pass
1053
1054     def deserialize(self, f):
1055         pass
1056
1057     def serialize(self):
1058         return b""
1059
1060     def __repr__(self):
1061         return "msg_getaddr()"
1062
1063
1064 class msg_ping():
1065     command = b"ping"
1066
1067     def __init__(self, nonce=0):
1068         self.nonce = nonce
1069
1070     def deserialize(self, f):
1071         self.nonce = struct.unpack("<Q", f.read(8))[0]
1072
1073     def serialize(self):
1074         r = b""
1075         r += struct.pack("<Q", self.nonce)
1076         return r
1077
1078     def __repr__(self):
1079         return "msg_ping(nonce=%08x)" % self.nonce
1080
1081
1082 class msg_pong():
1083     command = b"pong"
1084
1085     def __init__(self, nonce=0):
1086         self.nonce = nonce
1087
1088     def deserialize(self, f):
1089         self.nonce = struct.unpack("<Q", f.read(8))[0]
1090
1091     def serialize(self):
1092         r = b""
1093         r += struct.pack("<Q", self.nonce)
1094         return r
1095
1096     def __repr__(self):
1097         return "msg_pong(nonce=%08x)" % self.nonce
1098
1099
1100 class msg_mempool():
1101     command = b"mempool"
1102
1103     def __init__(self):
1104         pass
1105
1106     def deserialize(self, f):
1107         pass
1108
1109     def serialize(self):
1110         return b""
1111
1112     def __repr__(self):
1113         return "msg_mempool()"
1114
1115 class msg_sendheaders():
1116     command = b"sendheaders"
1117
1118     def __init__(self):
1119         pass
1120
1121     def deserialize(self, f):
1122         pass
1123
1124     def serialize(self):
1125         return b""
1126
1127     def __repr__(self):
1128         return "msg_sendheaders()"
1129
1130
1131 # getheaders message has
1132 # number of entries
1133 # vector of hashes
1134 # hash_stop (hash of last desired block header, 0 to get as many as possible)
1135 class msg_getheaders():
1136     command = b"getheaders"
1137
1138     def __init__(self):
1139         self.locator = CBlockLocator()
1140         self.hashstop = 0
1141
1142     def deserialize(self, f):
1143         self.locator = CBlockLocator()
1144         self.locator.deserialize(f)
1145         self.hashstop = deser_uint256(f)
1146
1147     def serialize(self):
1148         r = b""
1149         r += self.locator.serialize()
1150         r += ser_uint256(self.hashstop)
1151         return r
1152
1153     def __repr__(self):
1154         return "msg_getheaders(locator=%s, stop=%064x)" \
1155             % (repr(self.locator), self.hashstop)
1156
1157
1158 # headers message has
1159 # <count> <vector of block headers>
1160 class msg_headers():
1161     command = b"headers"
1162
1163     def __init__(self, headers=None):
1164         self.headers = headers if headers is not None else []
1165
1166     def deserialize(self, f):
1167         # comment in bitcoind indicates these should be deserialized as blocks
1168         blocks = deser_vector(f, CBlock)
1169         for x in blocks:
1170             self.headers.append(CBlockHeader(x))
1171
1172     def serialize(self):
1173         blocks = [CBlock(x) for x in self.headers]
1174         return ser_vector(blocks)
1175
1176     def __repr__(self):
1177         return "msg_headers(headers=%s)" % repr(self.headers)
1178
1179
1180 class msg_reject():
1181     command = b"reject"
1182     REJECT_MALFORMED = 1
1183
1184     def __init__(self):
1185         self.message = b""
1186         self.code = 0
1187         self.reason = b""
1188         self.data = 0
1189
1190     def deserialize(self, f):
1191         self.message = deser_string(f)
1192         self.code = struct.unpack("<B", f.read(1))[0]
1193         self.reason = deser_string(f)
1194         if (self.code != self.REJECT_MALFORMED and
1195                 (self.message == b"block" or self.message == b"tx")):
1196             self.data = deser_uint256(f)
1197
1198     def serialize(self):
1199         r = ser_string(self.message)
1200         r += struct.pack("<B", self.code)
1201         r += ser_string(self.reason)
1202         if (self.code != self.REJECT_MALFORMED and
1203                 (self.message == b"block" or self.message == b"tx")):
1204             r += ser_uint256(self.data)
1205         return r
1206
1207     def __repr__(self):
1208         return "msg_reject: %s %d %s [%064x]" \
1209             % (self.message, self.code, self.reason, self.data)
1210
1211 class msg_feefilter():
1212     command = b"feefilter"
1213
1214     def __init__(self, feerate=0):
1215         self.feerate = feerate
1216
1217     def deserialize(self, f):
1218         self.feerate = struct.unpack("<Q", f.read(8))[0]
1219
1220     def serialize(self):
1221         r = b""
1222         r += struct.pack("<Q", self.feerate)
1223         return r
1224
1225     def __repr__(self):
1226         return "msg_feefilter(feerate=%08x)" % self.feerate
1227
1228 class msg_sendcmpct():
1229     command = b"sendcmpct"
1230
1231     def __init__(self):
1232         self.announce = False
1233         self.version = 1
1234
1235     def deserialize(self, f):
1236         self.announce = struct.unpack("<?", f.read(1))[0]
1237         self.version = struct.unpack("<Q", f.read(8))[0]
1238
1239     def serialize(self):
1240         r = b""
1241         r += struct.pack("<?", self.announce)
1242         r += struct.pack("<Q", self.version)
1243         return r
1244
1245     def __repr__(self):
1246         return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version)
1247
1248 class msg_cmpctblock():
1249     command = b"cmpctblock"
1250
1251     def __init__(self, header_and_shortids = None):
1252         self.header_and_shortids = header_and_shortids
1253
1254     def deserialize(self, f):
1255         self.header_and_shortids = P2PHeaderAndShortIDs()
1256         self.header_and_shortids.deserialize(f)
1257
1258     def serialize(self):
1259         r = b""
1260         r += self.header_and_shortids.serialize()
1261         return r
1262
1263     def __repr__(self):
1264         return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids)
1265
1266 class msg_getblocktxn():
1267     command = b"getblocktxn"
1268
1269     def __init__(self):
1270         self.block_txn_request = None
1271
1272     def deserialize(self, f):
1273         self.block_txn_request = BlockTransactionsRequest()
1274         self.block_txn_request.deserialize(f)
1275
1276     def serialize(self):
1277         r = b""
1278         r += self.block_txn_request.serialize()
1279         return r
1280
1281     def __repr__(self):
1282         return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request))
1283
1284 class msg_blocktxn():
1285     command = b"blocktxn"
1286
1287     def __init__(self):
1288         self.block_transactions = BlockTransactions()
1289
1290     def deserialize(self, f):
1291         self.block_transactions.deserialize(f)
1292
1293     def serialize(self):
1294         r = b""
1295         r += self.block_transactions.serialize()
1296         return r
1297
1298     def __repr__(self):
1299         return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions))
1300
1301 class msg_witness_blocktxn(msg_blocktxn):
1302     def serialize(self):
1303         r = b""
1304         r += self.block_transactions.serialize(with_witness=True)
1305         return r