qa/rpc-tests/p2p-compactblocks.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2016 The Bitcoin Core developers
   3 # Distributed under the MIT software license, see the accompanying
   4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5
   6 from test_framework.mininode import *
   7 from test_framework.test_framework import BitcoinTestFramework
   8 from test_framework.util import *
   9 from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment
  10 from test_framework.siphash import siphash256
  11 from test_framework.script import CScript, OP_TRUE
  12
  13 '''
  14 CompactBlocksTest -- test compact blocks (BIP 152)
  15
  16 Version 1 compact blocks are pre-segwit (txids)
  17 Version 2 compact blocks are post-segwit (wtxids)
  18 '''
  19
  20 # TestNode: A peer we use to send messages to bitcoind, and store responses.
  21 class TestNode(SingleNodeConnCB):
  22     def __init__(self):
  23         SingleNodeConnCB.__init__(self)
  24         self.last_sendcmpct = []
  25         self.last_headers = None
  26         self.last_inv = None
  27         self.last_cmpctblock = None
  28         self.block_announced = False
  29         self.last_getdata = None
  30         self.last_getheaders = None
  31         self.last_getblocktxn = None
  32         self.last_block = None
  33         self.last_blocktxn = None
  34         # Store the hashes of blocks we've seen announced.
  35         # This is for synchronizing the p2p message traffic,
  36         # so we can eg wait until a particular block is announced.
  37         self.set_announced_blockhashes = set()
  38
  39     def on_sendcmpct(self, conn, message):
  40         self.last_sendcmpct.append(message)
  41
  42     def on_block(self, conn, message):
  43         self.last_block = message
  44
  45     def on_cmpctblock(self, conn, message):
  46         self.last_cmpctblock = message
  47         self.block_announced = True
  48         self.last_cmpctblock.header_and_shortids.header.calc_sha256()
  49         self.set_announced_blockhashes.add(self.last_cmpctblock.header_and_shortids.header.sha256)
  50
  51     def on_headers(self, conn, message):
  52         self.last_headers = message
  53         self.block_announced = True
  54         for x in self.last_headers.headers:
  55             x.calc_sha256()
  56             self.set_announced_blockhashes.add(x.sha256)
  57
  58     def on_inv(self, conn, message):
  59         self.last_inv = message
  60         for x in self.last_inv.inv:
  61             if x.type == 2:
  62                 self.block_announced = True
  63                 self.set_announced_blockhashes.add(x.hash)
  64
  65     def on_getdata(self, conn, message):
  66         self.last_getdata = message
  67
  68     def on_getheaders(self, conn, message):
  69         self.last_getheaders = message
  70
  71     def on_getblocktxn(self, conn, message):
  72         self.last_getblocktxn = message
  73
  74     def on_blocktxn(self, conn, message):
  75         self.last_blocktxn = message
  76
  77     # Requires caller to hold mininode_lock
  78     def received_block_announcement(self):
  79         return self.block_announced
  80
  81     def clear_block_announcement(self):
  82         with mininode_lock:
  83             self.block_announced = False
  84             self.last_inv = None
  85             self.last_headers = None
  86             self.last_cmpctblock = None
  87
  88     def get_headers(self, locator, hashstop):
  89         msg = msg_getheaders()
  90         msg.locator.vHave = locator
  91         msg.hashstop = hashstop
  92         self.connection.send_message(msg)
  93
  94     def send_header_for_blocks(self, new_blocks):
  95         headers_message = msg_headers()
  96         headers_message.headers = [CBlockHeader(b) for b in new_blocks]
  97         self.send_message(headers_message)
  98
  99     def request_headers_and_sync(self, locator, hashstop=0):
 100         self.clear_block_announcement()
 101         self.get_headers(locator, hashstop)
 102         assert(wait_until(self.received_block_announcement, timeout=30))
 103         assert(self.received_block_announcement())
 104         self.clear_block_announcement()
 105
 106     # Block until a block announcement for a particular block hash is
 107     # received.
 108     def wait_for_block_announcement(self, block_hash, timeout=30):
 109         def received_hash():
 110             return (block_hash in self.set_announced_blockhashes)
 111         return wait_until(received_hash, timeout=timeout)
 112
 113 class CompactBlocksTest(BitcoinTestFramework):
 114     def __init__(self):
 115         super().__init__()
 116         self.setup_clean_chain = True
 117         # Node0 = pre-segwit, node1 = segwit-aware
 118         self.num_nodes = 2
 119         self.utxos = []
 120
 121     def setup_network(self):
 122         self.nodes = []
 123
 124         # Start up node0 to be a version 1, pre-segwit node.
 125         self.nodes = start_nodes(self.num_nodes, self.options.tmpdir,
 126                 [["-debug", "-logtimemicros=1", "-bip9params=segwit:0:0"],
 127                  ["-debug", "-logtimemicros", "-txindex"]])
 128         connect_nodes(self.nodes[0], 1)
 129
 130     def build_block_on_tip(self, node, segwit=False):
 131         height = node.getblockcount()
 132         tip = node.getbestblockhash()
 133         mtp = node.getblockheader(tip)['mediantime']
 134         block = create_block(int(tip, 16), create_coinbase(height + 1), mtp + 1)
 135         block.nVersion = 4
 136         if segwit:
 137             add_witness_commitment(block)
 138         block.solve()
 139         return block
 140
 141     # Create 10 more anyone-can-spend utxo's for testing.
 142     def make_utxos(self):
 143         # Doesn't matter which node we use, just use node0.
 144         block = self.build_block_on_tip(self.nodes[0])
 145         self.test_node.send_and_ping(msg_block(block))
 146         assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256)
 147         self.nodes[0].generate(100)
 148
 149         total_value = block.vtx[0].vout[0].nValue
 150         out_value = total_value // 10
 151         tx = CTransaction()
 152         tx.vin.append(CTxIn(COutPoint(block.vtx[0].sha256, 0), b''))
 153         for i in range(10):
 154             tx.vout.append(CTxOut(out_value, CScript([OP_TRUE])))
 155         tx.rehash()
 156
 157         block2 = self.build_block_on_tip(self.nodes[0])
 158         block2.vtx.append(tx)
 159         block2.hashMerkleRoot = block2.calc_merkle_root()
 160         block2.solve()
 161         self.test_node.send_and_ping(msg_block(block2))
 162         assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256)
 163         self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)])
 164         return
 165
 166     # Test "sendcmpct" (between peers preferring the same version):
 167     # - No compact block announcements unless sendcmpct is sent.
 168     # - If sendcmpct is sent with version > preferred_version, the message is ignored.
 169     # - If sendcmpct is sent with boolean 0, then block announcements are not
 170     #   made with compact blocks.
 171     # - If sendcmpct is then sent with boolean 1, then new block announcements
 172     #   are made with compact blocks.
 173     # If old_node is passed in, request compact blocks with version=preferred-1
 174     # and verify that it receives block announcements via compact block.
 175     def test_sendcmpct(self, node, test_node, preferred_version, old_node=None):
 176         # Make sure we get a SENDCMPCT message from our peer
 177         def received_sendcmpct():
 178             return (len(test_node.last_sendcmpct) > 0)
 179         got_message = wait_until(received_sendcmpct, timeout=30)
 180         assert(received_sendcmpct())
 181         assert(got_message)
 182         with mininode_lock:
 183             # Check that the first version received is the preferred one
 184             assert_equal(test_node.last_sendcmpct[0].version, preferred_version)
 185             # And that we receive versions down to 1.
 186             assert_equal(test_node.last_sendcmpct[-1].version, 1)
 187             test_node.last_sendcmpct = []
 188
 189         tip = int(node.getbestblockhash(), 16)
 190
 191         def check_announcement_of_new_block(node, peer, predicate):
 192             peer.clear_block_announcement()
 193             block_hash = int(node.generate(1)[0], 16)
 194             peer.wait_for_block_announcement(block_hash, timeout=30)
 195             assert(peer.block_announced)
 196             assert(got_message)
 197
 198             with mininode_lock:
 199                 assert predicate(peer), (
 200                     "block_hash={!r}, cmpctblock={!r}, inv={!r}".format(
 201                         block_hash, peer.last_cmpctblock, peer.last_inv))
 202
 203         # We shouldn't get any block announcements via cmpctblock yet.
 204         check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None)
 205
 206         # Try one more time, this time after requesting headers.
 207         test_node.request_headers_and_sync(locator=[tip])
 208         check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None and p.last_inv is not None)
 209
 210         # Test a few ways of using sendcmpct that should NOT
 211         # result in compact block announcements.
 212         # Before each test, sync the headers chain.
 213         test_node.request_headers_and_sync(locator=[tip])
 214
 215         # Now try a SENDCMPCT message with too-high version
 216         sendcmpct = msg_sendcmpct()
 217         sendcmpct.version = preferred_version+1
 218         sendcmpct.announce = True
 219         test_node.send_and_ping(sendcmpct)
 220         check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None)
 221
 222         # Headers sync before next test.
 223         test_node.request_headers_and_sync(locator=[tip])
 224
 225         # Now try a SENDCMPCT message with valid version, but announce=False
 226         sendcmpct.version = preferred_version
 227         sendcmpct.announce = False
 228         test_node.send_and_ping(sendcmpct)
 229         check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None)
 230
 231         # Headers sync before next test.
 232         test_node.request_headers_and_sync(locator=[tip])
 233
 234         # Finally, try a SENDCMPCT message with announce=True
 235         sendcmpct.version = preferred_version
 236         sendcmpct.announce = True
 237         test_node.send_and_ping(sendcmpct)
 238         check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None)
 239
 240         # Try one more time (no headers sync should be needed!)
 241         check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None)
 242
 243         # Try one more time, after turning on sendheaders
 244         test_node.send_and_ping(msg_sendheaders())
 245         check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None)
 246
 247         # Try one more time, after sending a version-1, announce=false message.
 248         sendcmpct.version = preferred_version-1
 249         sendcmpct.announce = False
 250         test_node.send_and_ping(sendcmpct)
 251         check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None)
 252
 253         # Now turn off announcements
 254         sendcmpct.version = preferred_version
 255         sendcmpct.announce = False
 256         test_node.send_and_ping(sendcmpct)
 257         check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None and p.last_headers is not None)
 258
 259         if old_node is not None:
 260             # Verify that a peer using an older protocol version can receive
 261             # announcements from this node.
 262             sendcmpct.version = preferred_version-1
 263             sendcmpct.announce = True
 264             old_node.send_and_ping(sendcmpct)
 265             # Header sync
 266             old_node.request_headers_and_sync(locator=[tip])
 267             check_announcement_of_new_block(node, old_node, lambda p: p.last_cmpctblock is not None)
 268
 269     # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last.
 270     def test_invalid_cmpctblock_message(self):
 271         self.nodes[0].generate(101)
 272         block = self.build_block_on_tip(self.nodes[0])
 273
 274         cmpct_block = P2PHeaderAndShortIDs()
 275         cmpct_block.header = CBlockHeader(block)
 276         cmpct_block.prefilled_txn_length = 1
 277         # This index will be too high
 278         prefilled_txn = PrefilledTransaction(1, block.vtx[0])
 279         cmpct_block.prefilled_txn = [prefilled_txn]
 280         self.test_node.send_and_ping(msg_cmpctblock(cmpct_block))
 281         assert(int(self.nodes[0].getbestblockhash(), 16) == block.hashPrevBlock)
 282
 283     # Compare the generated shortids to what we expect based on BIP 152, given
 284     # bitcoind's choice of nonce.
 285     def test_compactblock_construction(self, node, test_node, version, use_witness_address):
 286         # Generate a bunch of transactions.
 287         node.generate(101)
 288         num_transactions = 25
 289         address = node.getnewaddress()
 290         if use_witness_address:
 291             # Want at least one segwit spend, so move all funds to
 292             # a witness address.
 293             address = node.addwitnessaddress(address)
 294             value_to_send = node.getbalance()
 295             node.sendtoaddress(address, satoshi_round(value_to_send-Decimal(0.1)))
 296             node.generate(1)
 297
 298         segwit_tx_generated = False
 299         for i in range(num_transactions):
 300             txid = node.sendtoaddress(address, 0.1)
 301             hex_tx = node.gettransaction(txid)["hex"]
 302             tx = FromHex(CTransaction(), hex_tx)
 303             if not tx.wit.is_null():
 304                 segwit_tx_generated = True
 305
 306         if use_witness_address:
 307             assert(segwit_tx_generated) # check that our test is not broken
 308
 309         # Wait until we've seen the block announcement for the resulting tip
 310         tip = int(node.getbestblockhash(), 16)
 311         assert(test_node.wait_for_block_announcement(tip))
 312
 313         # Now mine a block, and look at the resulting compact block.
 314         test_node.clear_block_announcement()
 315         block_hash = int(node.generate(1)[0], 16)
 316
 317         # Store the raw block in our internal format.
 318         block = FromHex(CBlock(), node.getblock("%02x" % block_hash, False))
 319         [tx.calc_sha256() for tx in block.vtx]
 320         block.rehash()
 321
 322         # Don't care which type of announcement came back for this test; just
 323         # request the compact block if we didn't get one yet.
 324         wait_until(test_node.received_block_announcement, timeout=30)
 325         assert(test_node.received_block_announcement())
 326
 327         with mininode_lock:
 328             if test_node.last_cmpctblock is None:
 329                 test_node.clear_block_announcement()
 330                 inv = CInv(4, block_hash)  # 4 == "CompactBlock"
 331                 test_node.send_message(msg_getdata([inv]))
 332
 333         wait_until(test_node.received_block_announcement, timeout=30)
 334         assert(test_node.received_block_announcement())
 335
 336         # Now we should have the compactblock
 337         header_and_shortids = None
 338         with mininode_lock:
 339             assert(test_node.last_cmpctblock is not None)
 340             # Convert the on-the-wire representation to absolute indexes
 341             header_and_shortids = HeaderAndShortIDs(test_node.last_cmpctblock.header_and_shortids)
 342
 343         # Check that we got the right block!
 344         header_and_shortids.header.calc_sha256()
 345         assert_equal(header_and_shortids.header.sha256, block_hash)
 346
 347         # Make sure the prefilled_txn appears to have included the coinbase
 348         assert(len(header_and_shortids.prefilled_txn) >= 1)
 349         assert_equal(header_and_shortids.prefilled_txn[0].index, 0)
 350
 351         # Check that all prefilled_txn entries match what's in the block.
 352         for entry in header_and_shortids.prefilled_txn:
 353             entry.tx.calc_sha256()
 354             # This checks the non-witness parts of the tx agree
 355             assert_equal(entry.tx.sha256, block.vtx[entry.index].sha256)
 356
 357             # And this checks the witness
 358             wtxid = entry.tx.calc_sha256(True)
 359             if version == 2:
 360                 assert_equal(wtxid, block.vtx[entry.index].calc_sha256(True))
 361             else:
 362                 # Shouldn't have received a witness
 363                 assert(entry.tx.wit.is_null())
 364
 365         # Check that the cmpctblock message announced all the transactions.
 366         assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx))
 367
 368         # And now check that all the shortids are as expected as well.
 369         # Determine the siphash keys to use.
 370         [k0, k1] = header_and_shortids.get_siphash_keys()
 371
 372         index = 0
 373         while index < len(block.vtx):
 374             if (len(header_and_shortids.prefilled_txn) > 0 and
 375                     header_and_shortids.prefilled_txn[0].index == index):
 376                 # Already checked prefilled transactions above
 377                 header_and_shortids.prefilled_txn.pop(0)
 378             else:
 379                 tx_hash = block.vtx[index].sha256
 380                 if version == 2:
 381                     tx_hash = block.vtx[index].calc_sha256(True)
 382                 shortid = calculate_shortid(k0, k1, tx_hash)
 383                 assert_equal(shortid, header_and_shortids.shortids[0])
 384                 header_and_shortids.shortids.pop(0)
 385             index += 1
 386
 387     # Test that bitcoind requests compact blocks when we announce new blocks
 388     # via header or inv, and that responding to getblocktxn causes the block
 389     # to be successfully reconstructed.
 390     # Post-segwit: upgraded nodes would only make this request of cb-version-2,
 391     # NODE_WITNESS peers.  Unupgraded nodes would still make this request of
 392     # any cb-version-1-supporting peer.
 393     def test_compactblock_requests(self, node, test_node, version, segwit):
 394         # Try announcing a block with an inv or header, expect a compactblock
 395         # request
 396         for announce in ["inv", "header"]:
 397             block = self.build_block_on_tip(node, segwit=segwit)
 398             with mininode_lock:
 399                 test_node.last_getdata = None
 400
 401             if announce == "inv":
 402                 test_node.send_message(msg_inv([CInv(2, block.sha256)]))
 403                 success = wait_until(lambda: test_node.last_getheaders is not None, timeout=30)
 404                 assert(success)
 405                 test_node.send_header_for_blocks([block])
 406             else:
 407                 test_node.send_header_for_blocks([block])
 408             success = wait_until(lambda: test_node.last_getdata is not None, timeout=30)
 409             assert(success)
 410             assert_equal(len(test_node.last_getdata.inv), 1)
 411             assert_equal(test_node.last_getdata.inv[0].type, 4)
 412             assert_equal(test_node.last_getdata.inv[0].hash, block.sha256)
 413
 414             # Send back a compactblock message that omits the coinbase
 415             comp_block = HeaderAndShortIDs()
 416             comp_block.header = CBlockHeader(block)
 417             comp_block.nonce = 0
 418             [k0, k1] = comp_block.get_siphash_keys()
 419             coinbase_hash = block.vtx[0].sha256
 420             if version == 2:
 421                 coinbase_hash = block.vtx[0].calc_sha256(True)
 422             comp_block.shortids = [
 423                     calculate_shortid(k0, k1, coinbase_hash) ]
 424             test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
 425             assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock)
 426             # Expect a getblocktxn message.
 427             with mininode_lock:
 428                 assert(test_node.last_getblocktxn is not None)
 429                 absolute_indexes = test_node.last_getblocktxn.block_txn_request.to_absolute()
 430             assert_equal(absolute_indexes, [0])  # should be a coinbase request
 431
 432             # Send the coinbase, and verify that the tip advances.
 433             if version == 2:
 434                 msg = msg_witness_blocktxn()
 435             else:
 436                 msg = msg_blocktxn()
 437             msg.block_transactions.blockhash = block.sha256
 438             msg.block_transactions.transactions = [block.vtx[0]]
 439             test_node.send_and_ping(msg)
 440             assert_equal(int(node.getbestblockhash(), 16), block.sha256)
 441
 442     # Create a chain of transactions from given utxo, and add to a new block.
 443     def build_block_with_transactions(self, node, utxo, num_transactions):
 444         block = self.build_block_on_tip(node)
 445
 446         for i in range(num_transactions):
 447             tx = CTransaction()
 448             tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b''))
 449             tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE])))
 450             tx.rehash()
 451             utxo = [tx.sha256, 0, tx.vout[0].nValue]
 452             block.vtx.append(tx)
 453
 454         block.hashMerkleRoot = block.calc_merkle_root()
 455         block.solve()
 456         return block
 457
 458     # Test that we only receive getblocktxn requests for transactions that the
 459     # node needs, and that responding to them causes the block to be
 460     # reconstructed.
 461     def test_getblocktxn_requests(self, node, test_node, version):
 462         with_witness = (version==2)
 463
 464         def test_getblocktxn_response(compact_block, peer, expected_result):
 465             msg = msg_cmpctblock(compact_block.to_p2p())
 466             peer.send_and_ping(msg)
 467             with mininode_lock:
 468                 assert(peer.last_getblocktxn is not None)
 469                 absolute_indexes = peer.last_getblocktxn.block_txn_request.to_absolute()
 470             assert_equal(absolute_indexes, expected_result)
 471
 472         def test_tip_after_message(node, peer, msg, tip):
 473             peer.send_and_ping(msg)
 474             assert_equal(int(node.getbestblockhash(), 16), tip)
 475
 476         # First try announcing compactblocks that won't reconstruct, and verify
 477         # that we receive getblocktxn messages back.
 478         utxo = self.utxos.pop(0)
 479
 480         block = self.build_block_with_transactions(node, utxo, 5)
 481         self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
 482         comp_block = HeaderAndShortIDs()
 483         comp_block.initialize_from_block(block, use_witness=with_witness)
 484
 485         test_getblocktxn_response(comp_block, test_node, [1, 2, 3, 4, 5])
 486
 487         msg_bt = msg_blocktxn()
 488         if with_witness:
 489             msg_bt = msg_witness_blocktxn() # serialize with witnesses
 490         msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[1:])
 491         test_tip_after_message(node, test_node, msg_bt, block.sha256)
 492
 493         utxo = self.utxos.pop(0)
 494         block = self.build_block_with_transactions(node, utxo, 5)
 495         self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
 496
 497         # Now try interspersing the prefilled transactions
 498         comp_block.initialize_from_block(block, prefill_list=[0, 1, 5], use_witness=with_witness)
 499         test_getblocktxn_response(comp_block, test_node, [2, 3, 4])
 500         msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5])
 501         test_tip_after_message(node, test_node, msg_bt, block.sha256)
 502
 503         # Now try giving one transaction ahead of time.
 504         utxo = self.utxos.pop(0)
 505         block = self.build_block_with_transactions(node, utxo, 5)
 506         self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
 507         test_node.send_and_ping(msg_tx(block.vtx[1]))
 508         assert(block.vtx[1].hash in node.getrawmempool())
 509
 510         # Prefill 4 out of the 6 transactions, and verify that only the one
 511         # that was not in the mempool is requested.
 512         comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4], use_witness=with_witness)
 513         test_getblocktxn_response(comp_block, test_node, [5])
 514
 515         msg_bt.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]])
 516         test_tip_after_message(node, test_node, msg_bt, block.sha256)
 517
 518         # Now provide all transactions to the node before the block is
 519         # announced and verify reconstruction happens immediately.
 520         utxo = self.utxos.pop(0)
 521         block = self.build_block_with_transactions(node, utxo, 10)
 522         self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
 523         for tx in block.vtx[1:]:
 524             test_node.send_message(msg_tx(tx))
 525         test_node.sync_with_ping()
 526         # Make sure all transactions were accepted.
 527         mempool = node.getrawmempool()
 528         for tx in block.vtx[1:]:
 529             assert(tx.hash in mempool)
 530
 531         # Clear out last request.
 532         with mininode_lock:
 533             test_node.last_getblocktxn = None
 534
 535         # Send compact block
 536         comp_block.initialize_from_block(block, prefill_list=[0], use_witness=with_witness)
 537         test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256)
 538         with mininode_lock:
 539             # Shouldn't have gotten a request for any transaction
 540             assert(test_node.last_getblocktxn is None)
 541
 542     # Incorrectly responding to a getblocktxn shouldn't cause the block to be
 543     # permanently failed.
 544     def test_incorrect_blocktxn_response(self, node, test_node, version):
 545         if (len(self.utxos) == 0):
 546             self.make_utxos()
 547         utxo = self.utxos.pop(0)
 548
 549         block = self.build_block_with_transactions(node, utxo, 10)
 550         self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
 551         # Relay the first 5 transactions from the block in advance
 552         for tx in block.vtx[1:6]:
 553             test_node.send_message(msg_tx(tx))
 554         test_node.sync_with_ping()
 555         # Make sure all transactions were accepted.
 556         mempool = node.getrawmempool()
 557         for tx in block.vtx[1:6]:
 558             assert(tx.hash in mempool)
 559
 560         # Send compact block
 561         comp_block = HeaderAndShortIDs()
 562         comp_block.initialize_from_block(block, prefill_list=[0], use_witness=(version == 2))
 563         test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
 564         absolute_indexes = []
 565         with mininode_lock:
 566             assert(test_node.last_getblocktxn is not None)
 567             absolute_indexes = test_node.last_getblocktxn.block_txn_request.to_absolute()
 568         assert_equal(absolute_indexes, [6, 7, 8, 9, 10])
 569
 570         # Now give an incorrect response.
 571         # Note that it's possible for bitcoind to be smart enough to know we're
 572         # lying, since it could check to see if the shortid matches what we're
 573         # sending, and eg disconnect us for misbehavior.  If that behavior
 574         # change were made, we could just modify this test by having a
 575         # different peer provide the block further down, so that we're still
 576         # verifying that the block isn't marked bad permanently. This is good
 577         # enough for now.
 578         msg = msg_blocktxn()
 579         if version==2:
 580             msg = msg_witness_blocktxn()
 581         msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:])
 582         test_node.send_and_ping(msg)
 583
 584         # Tip should not have updated
 585         assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock)
 586
 587         # We should receive a getdata request
 588         success = wait_until(lambda: test_node.last_getdata is not None, timeout=10)
 589         assert(success)
 590         assert_equal(len(test_node.last_getdata.inv), 1)
 591         assert(test_node.last_getdata.inv[0].type == 2 or test_node.last_getdata.inv[0].type == 2|MSG_WITNESS_FLAG)
 592         assert_equal(test_node.last_getdata.inv[0].hash, block.sha256)
 593
 594         # Deliver the block
 595         if version==2:
 596             test_node.send_and_ping(msg_witness_block(block))
 597         else:
 598             test_node.send_and_ping(msg_block(block))
 599         assert_equal(int(node.getbestblockhash(), 16), block.sha256)
 600
 601     def test_getblocktxn_handler(self, node, test_node, version):
 602         # bitcoind will not send blocktxn responses for blocks whose height is
 603         # more than 10 blocks deep.
 604         MAX_GETBLOCKTXN_DEPTH = 10
 605         chain_height = node.getblockcount()
 606         current_height = chain_height
 607         while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH):
 608             block_hash = node.getblockhash(current_height)
 609             block = FromHex(CBlock(), node.getblock(block_hash, False))
 610
 611             msg = msg_getblocktxn()
 612             msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [])
 613             num_to_request = random.randint(1, len(block.vtx))
 614             msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request)))
 615             test_node.send_message(msg)
 616             success = wait_until(lambda: test_node.last_blocktxn is not None, timeout=10)
 617             assert(success)
 618
 619             [tx.calc_sha256() for tx in block.vtx]
 620             with mininode_lock:
 621                 assert_equal(test_node.last_blocktxn.block_transactions.blockhash, int(block_hash, 16))
 622                 all_indices = msg.block_txn_request.to_absolute()
 623                 for index in all_indices:
 624                     tx = test_node.last_blocktxn.block_transactions.transactions.pop(0)
 625                     tx.calc_sha256()
 626                     assert_equal(tx.sha256, block.vtx[index].sha256)
 627                     if version == 1:
 628                         # Witnesses should have been stripped
 629                         assert(tx.wit.is_null())
 630                     else:
 631                         # Check that the witness matches
 632                         assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True))
 633                 test_node.last_blocktxn = None
 634             current_height -= 1
 635
 636         # Next request should send a full block response, as we're past the
 637         # allowed depth for a blocktxn response.
 638         block_hash = node.getblockhash(current_height)
 639         msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0])
 640         with mininode_lock:
 641             test_node.last_block = None
 642             test_node.last_blocktxn = None
 643         test_node.send_and_ping(msg)
 644         with mininode_lock:
 645             test_node.last_block.block.calc_sha256()
 646             assert_equal(test_node.last_block.block.sha256, int(block_hash, 16))
 647             assert_equal(test_node.last_blocktxn, None)
 648
 649     def test_compactblocks_not_at_tip(self, node, test_node):
 650         # Test that requesting old compactblocks doesn't work.
 651         MAX_CMPCTBLOCK_DEPTH = 5
 652         new_blocks = []
 653         for i in range(MAX_CMPCTBLOCK_DEPTH + 1):
 654             test_node.clear_block_announcement()
 655             new_blocks.append(node.generate(1)[0])
 656             wait_until(test_node.received_block_announcement, timeout=30)
 657
 658         test_node.clear_block_announcement()
 659         test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))]))
 660         success = wait_until(lambda: test_node.last_cmpctblock is not None, timeout=30)
 661         assert(success)
 662
 663         test_node.clear_block_announcement()
 664         node.generate(1)
 665         wait_until(test_node.received_block_announcement, timeout=30)
 666         test_node.clear_block_announcement()
 667         with mininode_lock:
 668             test_node.last_block = None
 669         test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))]))
 670         success = wait_until(lambda: test_node.last_block is not None, timeout=30)
 671         assert(success)
 672         with mininode_lock:
 673             test_node.last_block.block.calc_sha256()
 674             assert_equal(test_node.last_block.block.sha256, int(new_blocks[0], 16))
 675
 676         # Generate an old compactblock, and verify that it's not accepted.
 677         cur_height = node.getblockcount()
 678         hashPrevBlock = int(node.getblockhash(cur_height-5), 16)
 679         block = self.build_block_on_tip(node)
 680         block.hashPrevBlock = hashPrevBlock
 681         block.solve()
 682
 683         comp_block = HeaderAndShortIDs()
 684         comp_block.initialize_from_block(block)
 685         test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
 686
 687         tips = node.getchaintips()
 688         found = False
 689         for x in tips:
 690             if x["hash"] == block.hash:
 691                 assert_equal(x["status"], "headers-only")
 692                 found = True
 693                 break
 694         assert(found)
 695
 696         # Requesting this block via getblocktxn should silently fail
 697         # (to avoid fingerprinting attacks).
 698         msg = msg_getblocktxn()
 699         msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0])
 700         with mininode_lock:
 701             test_node.last_blocktxn = None
 702         test_node.send_and_ping(msg)
 703         with mininode_lock:
 704             assert(test_node.last_blocktxn is None)
 705
 706     def activate_segwit(self, node):
 707         node.generate(144*3)
 708         assert_equal(get_bip9_status(node, "segwit")["status"], 'active')
 709
 710     def test_end_to_end_block_relay(self, node, listeners):
 711         utxo = self.utxos.pop(0)
 712
 713         block = self.build_block_with_transactions(node, utxo, 10)
 714
 715         [l.clear_block_announcement() for l in listeners]
 716
 717         # ToHex() won't serialize with witness, but this block has no witnesses
 718         # anyway. TODO: repeat this test with witness tx's to a segwit node.
 719         node.submitblock(ToHex(block))
 720
 721         for l in listeners:
 722             wait_until(lambda: l.received_block_announcement(), timeout=30)
 723         with mininode_lock:
 724             for l in listeners:
 725                 assert(l.last_cmpctblock is not None)
 726                 l.last_cmpctblock.header_and_shortids.header.calc_sha256()
 727                 assert_equal(l.last_cmpctblock.header_and_shortids.header.sha256, block.sha256)
 728
 729     # Test that we don't get disconnected if we relay a compact block with valid header,
 730     # but invalid transactions.
 731     def test_invalid_tx_in_compactblock(self, node, test_node, use_segwit):
 732         assert(len(self.utxos))
 733         utxo = self.utxos[0]
 734
 735         block = self.build_block_with_transactions(node, utxo, 5)
 736         del block.vtx[3]
 737         block.hashMerkleRoot = block.calc_merkle_root()
 738         if use_segwit:
 739             # If we're testing with segwit, also drop the coinbase witness,
 740             # but include the witness commitment.
 741             add_witness_commitment(block)
 742             block.vtx[0].wit.vtxinwit = []
 743         block.solve()
 744
 745         # Now send the compact block with all transactions prefilled, and
 746         # verify that we don't get disconnected.
 747         comp_block = HeaderAndShortIDs()
 748         comp_block.initialize_from_block(block, prefill_list=[0, 1, 2, 3, 4], use_witness=use_segwit)
 749         msg = msg_cmpctblock(comp_block.to_p2p())
 750         test_node.send_and_ping(msg)
 751
 752         # Check that the tip didn't advance
 753         assert(int(node.getbestblockhash(), 16) is not block.sha256)
 754         test_node.sync_with_ping()
 755
 756     # Helper for enabling cb announcements
 757     # Send the sendcmpct request and sync headers
 758     def request_cb_announcements(self, peer, node, version):
 759         tip = node.getbestblockhash()
 760         peer.get_headers(locator=[int(tip, 16)], hashstop=0)
 761
 762         msg = msg_sendcmpct()
 763         msg.version = version
 764         msg.announce = True
 765         peer.send_and_ping(msg)
 766
 767     def test_compactblock_reconstruction_multiple_peers(self, node, stalling_peer, delivery_peer):
 768         assert(len(self.utxos))
 769
 770         def announce_cmpct_block(node, peer):
 771             utxo = self.utxos.pop(0)
 772             block = self.build_block_with_transactions(node, utxo, 5)
 773
 774             cmpct_block = HeaderAndShortIDs()
 775             cmpct_block.initialize_from_block(block)
 776             msg = msg_cmpctblock(cmpct_block.to_p2p())
 777             peer.send_and_ping(msg)
 778             with mininode_lock:
 779                 assert(peer.last_getblocktxn is not None)
 780             return block, cmpct_block
 781
 782         block, cmpct_block = announce_cmpct_block(node, stalling_peer)
 783
 784         for tx in block.vtx[1:]:
 785             delivery_peer.send_message(msg_tx(tx))
 786         delivery_peer.sync_with_ping()
 787         mempool = node.getrawmempool()
 788         for tx in block.vtx[1:]:
 789             assert(tx.hash in mempool)
 790
 791         delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p()))
 792         assert_equal(int(node.getbestblockhash(), 16), block.sha256)
 793
 794         self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
 795
 796         # Now test that delivering an invalid compact block won't break relay
 797
 798         block, cmpct_block = announce_cmpct_block(node, stalling_peer)
 799         for tx in block.vtx[1:]:
 800             delivery_peer.send_message(msg_tx(tx))
 801         delivery_peer.sync_with_ping()
 802
 803         cmpct_block.prefilled_txn[0].tx.wit.vtxinwit = [ CTxInWitness() ]
 804         cmpct_block.prefilled_txn[0].tx.wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(0)]
 805
 806         cmpct_block.use_witness = True
 807         delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p()))
 808         assert(int(node.getbestblockhash(), 16) != block.sha256)
 809
 810         msg = msg_blocktxn()
 811         msg.block_transactions.blockhash = block.sha256
 812         msg.block_transactions.transactions = block.vtx[1:]
 813         stalling_peer.send_and_ping(msg)
 814         assert_equal(int(node.getbestblockhash(), 16), block.sha256)
 815
 816     def run_test(self):
 817         # Setup the p2p connections and start up the network thread.
 818         self.test_node = TestNode()
 819         self.segwit_node = TestNode()
 820         self.old_node = TestNode()  # version 1 peer <--> segwit node
 821
 822         connections = []
 823         connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node))
 824         connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1],
 825                     self.segwit_node, services=NODE_NETWORK|NODE_WITNESS))
 826         connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1],
 827                     self.old_node, services=NODE_NETWORK))
 828         self.test_node.add_connection(connections[0])
 829         self.segwit_node.add_connection(connections[1])
 830         self.old_node.add_connection(connections[2])
 831
 832         NetworkThread().start()  # Start up network handling in another thread
 833
 834         # Test logic begins here
 835         self.test_node.wait_for_verack()
 836
 837         # We will need UTXOs to construct transactions in later tests.
 838         self.make_utxos()
 839
 840         print("Running tests, pre-segwit activation:")
 841
 842         print("\tTesting SENDCMPCT p2p message... ")
 843         self.test_sendcmpct(self.nodes[0], self.test_node, 1)
 844         sync_blocks(self.nodes)
 845         self.test_sendcmpct(self.nodes[1], self.segwit_node, 2, old_node=self.old_node)
 846         sync_blocks(self.nodes)
 847
 848         print("\tTesting compactblock construction...")
 849         self.test_compactblock_construction(self.nodes[0], self.test_node, 1, False)
 850         sync_blocks(self.nodes)
 851         self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, False)
 852         sync_blocks(self.nodes)
 853
 854         print("\tTesting compactblock requests... ")
 855         self.test_compactblock_requests(self.nodes[0], self.test_node, 1, False)
 856         sync_blocks(self.nodes)
 857         self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, False)
 858         sync_blocks(self.nodes)
 859
 860         print("\tTesting getblocktxn requests...")
 861         self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1)
 862         sync_blocks(self.nodes)
 863         self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2)
 864         sync_blocks(self.nodes)
 865
 866         print("\tTesting getblocktxn handler...")
 867         self.test_getblocktxn_handler(self.nodes[0], self.test_node, 1)
 868         sync_blocks(self.nodes)
 869         self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2)
 870         self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1)
 871         sync_blocks(self.nodes)
 872
 873         print("\tTesting compactblock requests/announcements not at chain tip...")
 874         self.test_compactblocks_not_at_tip(self.nodes[0], self.test_node)
 875         sync_blocks(self.nodes)
 876         self.test_compactblocks_not_at_tip(self.nodes[1], self.segwit_node)
 877         self.test_compactblocks_not_at_tip(self.nodes[1], self.old_node)
 878         sync_blocks(self.nodes)
 879
 880         print("\tTesting handling of incorrect blocktxn responses...")
 881         self.test_incorrect_blocktxn_response(self.nodes[0], self.test_node, 1)
 882         sync_blocks(self.nodes)
 883         self.test_incorrect_blocktxn_response(self.nodes[1], self.segwit_node, 2)
 884         sync_blocks(self.nodes)
 885
 886         # End-to-end block relay tests
 887         print("\tTesting end-to-end block relay...")
 888         self.request_cb_announcements(self.test_node, self.nodes[0], 1)
 889         self.request_cb_announcements(self.old_node, self.nodes[1], 1)
 890         self.request_cb_announcements(self.segwit_node, self.nodes[1], 2)
 891         self.test_end_to_end_block_relay(self.nodes[0], [self.segwit_node, self.test_node, self.old_node])
 892         self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node])
 893
 894         print("\tTesting handling of invalid compact blocks...")
 895         self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False)
 896         self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, False)
 897         self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, False)
 898
 899         print("\tTesting reconstructing compact blocks from all peers...")
 900         self.test_compactblock_reconstruction_multiple_peers(self.nodes[1], self.segwit_node, self.old_node)
 901         sync_blocks(self.nodes)
 902
 903         # Advance to segwit activation
 904         print ("\nAdvancing to segwit activation\n")
 905         self.activate_segwit(self.nodes[1])
 906         print ("Running tests, post-segwit activation...")
 907
 908         print("\tTesting compactblock construction...")
 909         self.test_compactblock_construction(self.nodes[1], self.old_node, 1, True)
 910         self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, True)
 911         sync_blocks(self.nodes)
 912
 913         print("\tTesting compactblock requests (unupgraded node)... ")
 914         self.test_compactblock_requests(self.nodes[0], self.test_node, 1, True)
 915
 916         print("\tTesting getblocktxn requests (unupgraded node)...")
 917         self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1)
 918
 919         # Need to manually sync node0 and node1, because post-segwit activation,
 920         # node1 will not download blocks from node0.
 921         print("\tSyncing nodes...")
 922         assert(self.nodes[0].getbestblockhash() != self.nodes[1].getbestblockhash())
 923         while (self.nodes[0].getblockcount() > self.nodes[1].getblockcount()):
 924             block_hash = self.nodes[0].getblockhash(self.nodes[1].getblockcount()+1)
 925             self.nodes[1].submitblock(self.nodes[0].getblock(block_hash, False))
 926         assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash())
 927
 928         print("\tTesting compactblock requests (segwit node)... ")
 929         self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, True)
 930
 931         print("\tTesting getblocktxn requests (segwit node)...")
 932         self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2)
 933         sync_blocks(self.nodes)
 934
 935         print("\tTesting getblocktxn handler (segwit node should return witnesses)...")
 936         self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2)
 937         self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1)
 938
 939         # Test that if we submitblock to node1, we'll get a compact block
 940         # announcement to all peers.
 941         # (Post-segwit activation, blocks won't propagate from node0 to node1
 942         # automatically, so don't bother testing a block announced to node0.)
 943         print("\tTesting end-to-end block relay...")
 944         self.request_cb_announcements(self.test_node, self.nodes[0], 1)
 945         self.request_cb_announcements(self.old_node, self.nodes[1], 1)
 946         self.request_cb_announcements(self.segwit_node, self.nodes[1], 2)
 947         self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node])
 948
 949         print("\tTesting handling of invalid compact blocks...")
 950         self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False)
 951         self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, True)
 952         self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, True)
 953
 954         print("\tTesting invalid index in cmpctblock message...")
 955         self.test_invalid_cmpctblock_message()
 956
 957
 958 if __name__ == '__main__':
 959     CompactBlocksTest().main()