test/functional/p2p-compactblocks.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2016 The Bitcoin Core developers
   3 # Distributed under the MIT software license, see the accompanying
   4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5 """Test compact blocks (BIP 152).
   6
   7 Version 1 compact blocks are pre-segwit (txids)
   8 Version 2 compact blocks are post-segwit (wtxids)
   9 """
  10
  11 from test_framework.mininode import *
  12 from test_framework.test_framework import BitcoinTestFramework
  13 from test_framework.util import *
  14 from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment
  15 from test_framework.script import CScript, OP_TRUE
  16
  17 # TestNode: A peer we use to send messages to bitcoind, and store responses.
  18 class TestNode(NodeConnCB):
  19     def __init__(self):
  20         super().__init__()
  21         self.last_sendcmpct = []
  22         self.block_announced = False
  23         # Store the hashes of blocks we've seen announced.
  24         # This is for synchronizing the p2p message traffic,
  25         # so we can eg wait until a particular block is announced.
  26         self.announced_blockhashes = set()
  27
  28     def on_sendcmpct(self, conn, message):
  29         self.last_sendcmpct.append(message)
  30
  31     def on_cmpctblock(self, conn, message):
  32         self.block_announced = True
  33         self.last_message["cmpctblock"].header_and_shortids.header.calc_sha256()
  34         self.announced_blockhashes.add(self.last_message["cmpctblock"].header_and_shortids.header.sha256)
  35
  36     def on_headers(self, conn, message):
  37         self.block_announced = True
  38         for x in self.last_message["headers"].headers:
  39             x.calc_sha256()
  40             self.announced_blockhashes.add(x.sha256)
  41
  42     def on_inv(self, conn, message):
  43         for x in self.last_message["inv"].inv:
  44             if x.type == 2:
  45                 self.block_announced = True
  46                 self.announced_blockhashes.add(x.hash)
  47
  48     # Requires caller to hold mininode_lock
  49     def received_block_announcement(self):
  50         return self.block_announced
  51
  52     def clear_block_announcement(self):
  53         with mininode_lock:
  54             self.block_announced = False
  55             self.last_message.pop("inv", None)
  56             self.last_message.pop("headers", None)
  57             self.last_message.pop("cmpctblock", None)
  58
  59     def get_headers(self, locator, hashstop):
  60         msg = msg_getheaders()
  61         msg.locator.vHave = locator
  62         msg.hashstop = hashstop
  63         self.connection.send_message(msg)
  64
  65     def send_header_for_blocks(self, new_blocks):
  66         headers_message = msg_headers()
  67         headers_message.headers = [CBlockHeader(b) for b in new_blocks]
  68         self.send_message(headers_message)
  69
  70     def request_headers_and_sync(self, locator, hashstop=0):
  71         self.clear_block_announcement()
  72         self.get_headers(locator, hashstop)
  73         wait_until(self.received_block_announcement, timeout=30, lock=mininode_lock)
  74         self.clear_block_announcement()
  75
  76     # Block until a block announcement for a particular block hash is
  77     # received.
  78     def wait_for_block_announcement(self, block_hash, timeout=30):
  79         def received_hash():
  80             return (block_hash in self.announced_blockhashes)
  81         wait_until(received_hash, timeout=timeout, lock=mininode_lock)
  82
  83     def send_await_disconnect(self, message, timeout=30):
  84         """Sends a message to the node and wait for disconnect.
  85
  86         This is used when we want to send a message into the node that we expect
  87         will get us disconnected, eg an invalid block."""
  88         self.send_message(message)
  89         wait_until(lambda: not self.connected, timeout=timeout, lock=mininode_lock)
  90
  91 class CompactBlocksTest(BitcoinTestFramework):
  92     def __init__(self):
  93         super().__init__()
  94         self.setup_clean_chain = True
  95         # Node0 = pre-segwit, node1 = segwit-aware
  96         self.num_nodes = 2
  97         self.extra_args = [["-vbparams=segwit:0:0"], ["-txindex"]]
  98         self.utxos = []
  99
 100     def build_block_on_tip(self, node, segwit=False):
 101         height = node.getblockcount()
 102         tip = node.getbestblockhash()
 103         mtp = node.getblockheader(tip)['mediantime']
 104         block = create_block(int(tip, 16), create_coinbase(height + 1), mtp + 1)
 105         block.nVersion = 4
 106         if segwit:
 107             add_witness_commitment(block)
 108         block.solve()
 109         return block
 110
 111     # Create 10 more anyone-can-spend utxo's for testing.
 112     def make_utxos(self):
 113         # Doesn't matter which node we use, just use node0.
 114         block = self.build_block_on_tip(self.nodes[0])
 115         self.test_node.send_and_ping(msg_block(block))
 116         assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256)
 117         self.nodes[0].generate(100)
 118
 119         total_value = block.vtx[0].vout[0].nValue
 120         out_value = total_value // 10
 121         tx = CTransaction()
 122         tx.vin.append(CTxIn(COutPoint(block.vtx[0].sha256, 0), b''))
 123         for i in range(10):
 124             tx.vout.append(CTxOut(out_value, CScript([OP_TRUE])))
 125         tx.rehash()
 126
 127         block2 = self.build_block_on_tip(self.nodes[0])
 128         block2.vtx.append(tx)
 129         block2.hashMerkleRoot = block2.calc_merkle_root()
 130         block2.solve()
 131         self.test_node.send_and_ping(msg_block(block2))
 132         assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256)
 133         self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)])
 134         return
 135
 136     # Test "sendcmpct" (between peers preferring the same version):
 137     # - No compact block announcements unless sendcmpct is sent.
 138     # - If sendcmpct is sent with version > preferred_version, the message is ignored.
 139     # - If sendcmpct is sent with boolean 0, then block announcements are not
 140     #   made with compact blocks.
 141     # - If sendcmpct is then sent with boolean 1, then new block announcements
 142     #   are made with compact blocks.
 143     # If old_node is passed in, request compact blocks with version=preferred-1
 144     # and verify that it receives block announcements via compact block.
 145     def test_sendcmpct(self, node, test_node, preferred_version, old_node=None):
 146         # Make sure we get a SENDCMPCT message from our peer
 147         def received_sendcmpct():
 148             return (len(test_node.last_sendcmpct) > 0)
 149         wait_until(received_sendcmpct, timeout=30, lock=mininode_lock)
 150         with mininode_lock:
 151             # Check that the first version received is the preferred one
 152             assert_equal(test_node.last_sendcmpct[0].version, preferred_version)
 153             # And that we receive versions down to 1.
 154             assert_equal(test_node.last_sendcmpct[-1].version, 1)
 155             test_node.last_sendcmpct = []
 156
 157         tip = int(node.getbestblockhash(), 16)
 158
 159         def check_announcement_of_new_block(node, peer, predicate):
 160             peer.clear_block_announcement()
 161             block_hash = int(node.generate(1)[0], 16)
 162             peer.wait_for_block_announcement(block_hash, timeout=30)
 163             assert(peer.block_announced)
 164
 165             with mininode_lock:
 166                 assert predicate(peer), (
 167                     "block_hash={!r}, cmpctblock={!r}, inv={!r}".format(
 168                         block_hash, peer.last_message.get("cmpctblock", None), peer.last_message.get("inv", None)))
 169
 170         # We shouldn't get any block announcements via cmpctblock yet.
 171         check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message)
 172
 173         # Try one more time, this time after requesting headers.
 174         test_node.request_headers_and_sync(locator=[tip])
 175         check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "inv" in p.last_message)
 176
 177         # Test a few ways of using sendcmpct that should NOT
 178         # result in compact block announcements.
 179         # Before each test, sync the headers chain.
 180         test_node.request_headers_and_sync(locator=[tip])
 181
 182         # Now try a SENDCMPCT message with too-high version
 183         sendcmpct = msg_sendcmpct()
 184         sendcmpct.version = preferred_version+1
 185         sendcmpct.announce = True
 186         test_node.send_and_ping(sendcmpct)
 187         check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message)
 188
 189         # Headers sync before next test.
 190         test_node.request_headers_and_sync(locator=[tip])
 191
 192         # Now try a SENDCMPCT message with valid version, but announce=False
 193         sendcmpct.version = preferred_version
 194         sendcmpct.announce = False
 195         test_node.send_and_ping(sendcmpct)
 196         check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message)
 197
 198         # Headers sync before next test.
 199         test_node.request_headers_and_sync(locator=[tip])
 200
 201         # Finally, try a SENDCMPCT message with announce=True
 202         sendcmpct.version = preferred_version
 203         sendcmpct.announce = True
 204         test_node.send_and_ping(sendcmpct)
 205         check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message)
 206
 207         # Try one more time (no headers sync should be needed!)
 208         check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message)
 209
 210         # Try one more time, after turning on sendheaders
 211         test_node.send_and_ping(msg_sendheaders())
 212         check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message)
 213
 214         # Try one more time, after sending a version-1, announce=false message.
 215         sendcmpct.version = preferred_version-1
 216         sendcmpct.announce = False
 217         test_node.send_and_ping(sendcmpct)
 218         check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message)
 219
 220         # Now turn off announcements
 221         sendcmpct.version = preferred_version
 222         sendcmpct.announce = False
 223         test_node.send_and_ping(sendcmpct)
 224         check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "headers" in p.last_message)
 225
 226         if old_node is not None:
 227             # Verify that a peer using an older protocol version can receive
 228             # announcements from this node.
 229             sendcmpct.version = preferred_version-1
 230             sendcmpct.announce = True
 231             old_node.send_and_ping(sendcmpct)
 232             # Header sync
 233             old_node.request_headers_and_sync(locator=[tip])
 234             check_announcement_of_new_block(node, old_node, lambda p: "cmpctblock" in p.last_message)
 235
 236     # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last.
 237     def test_invalid_cmpctblock_message(self):
 238         self.nodes[0].generate(101)
 239         block = self.build_block_on_tip(self.nodes[0])
 240
 241         cmpct_block = P2PHeaderAndShortIDs()
 242         cmpct_block.header = CBlockHeader(block)
 243         cmpct_block.prefilled_txn_length = 1
 244         # This index will be too high
 245         prefilled_txn = PrefilledTransaction(1, block.vtx[0])
 246         cmpct_block.prefilled_txn = [prefilled_txn]
 247         self.test_node.send_await_disconnect(msg_cmpctblock(cmpct_block))
 248         assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock)
 249
 250     # Compare the generated shortids to what we expect based on BIP 152, given
 251     # bitcoind's choice of nonce.
 252     def test_compactblock_construction(self, node, test_node, version, use_witness_address):
 253         # Generate a bunch of transactions.
 254         node.generate(101)
 255         num_transactions = 25
 256         address = node.getnewaddress()
 257         if use_witness_address:
 258             # Want at least one segwit spend, so move all funds to
 259             # a witness address.
 260             address = node.addwitnessaddress(address)
 261             value_to_send = node.getbalance()
 262             node.sendtoaddress(address, satoshi_round(value_to_send-Decimal(0.1)))
 263             node.generate(1)
 264
 265         segwit_tx_generated = False
 266         for i in range(num_transactions):
 267             txid = node.sendtoaddress(address, 0.1)
 268             hex_tx = node.gettransaction(txid)["hex"]
 269             tx = FromHex(CTransaction(), hex_tx)
 270             if not tx.wit.is_null():
 271                 segwit_tx_generated = True
 272
 273         if use_witness_address:
 274             assert(segwit_tx_generated) # check that our test is not broken
 275
 276         # Wait until we've seen the block announcement for the resulting tip
 277         tip = int(node.getbestblockhash(), 16)
 278         test_node.wait_for_block_announcement(tip)
 279
 280         # Make sure we will receive a fast-announce compact block
 281         self.request_cb_announcements(test_node, node, version)
 282
 283         # Now mine a block, and look at the resulting compact block.
 284         test_node.clear_block_announcement()
 285         block_hash = int(node.generate(1)[0], 16)
 286
 287         # Store the raw block in our internal format.
 288         block = FromHex(CBlock(), node.getblock("%02x" % block_hash, False))
 289         for tx in block.vtx:
 290             tx.calc_sha256()
 291         block.rehash()
 292
 293         # Wait until the block was announced (via compact blocks)
 294         wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock)
 295
 296         # Now fetch and check the compact block
 297         header_and_shortids = None
 298         with mininode_lock:
 299             assert("cmpctblock" in test_node.last_message)
 300             # Convert the on-the-wire representation to absolute indexes
 301             header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids)
 302         self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block)
 303
 304         # Now fetch the compact block using a normal non-announce getdata
 305         with mininode_lock:
 306             test_node.clear_block_announcement()
 307             inv = CInv(4, block_hash)  # 4 == "CompactBlock"
 308             test_node.send_message(msg_getdata([inv]))
 309
 310         wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock)
 311
 312         # Now fetch and check the compact block
 313         header_and_shortids = None
 314         with mininode_lock:
 315             assert("cmpctblock" in test_node.last_message)
 316             # Convert the on-the-wire representation to absolute indexes
 317             header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids)
 318         self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block)
 319
 320     def check_compactblock_construction_from_block(self, version, header_and_shortids, block_hash, block):
 321         # Check that we got the right block!
 322         header_and_shortids.header.calc_sha256()
 323         assert_equal(header_and_shortids.header.sha256, block_hash)
 324
 325         # Make sure the prefilled_txn appears to have included the coinbase
 326         assert(len(header_and_shortids.prefilled_txn) >= 1)
 327         assert_equal(header_and_shortids.prefilled_txn[0].index, 0)
 328
 329         # Check that all prefilled_txn entries match what's in the block.
 330         for entry in header_and_shortids.prefilled_txn:
 331             entry.tx.calc_sha256()
 332             # This checks the non-witness parts of the tx agree
 333             assert_equal(entry.tx.sha256, block.vtx[entry.index].sha256)
 334
 335             # And this checks the witness
 336             wtxid = entry.tx.calc_sha256(True)
 337             if version == 2:
 338                 assert_equal(wtxid, block.vtx[entry.index].calc_sha256(True))
 339             else:
 340                 # Shouldn't have received a witness
 341                 assert(entry.tx.wit.is_null())
 342
 343         # Check that the cmpctblock message announced all the transactions.
 344         assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx))
 345
 346         # And now check that all the shortids are as expected as well.
 347         # Determine the siphash keys to use.
 348         [k0, k1] = header_and_shortids.get_siphash_keys()
 349
 350         index = 0
 351         while index < len(block.vtx):
 352             if (len(header_and_shortids.prefilled_txn) > 0 and
 353                     header_and_shortids.prefilled_txn[0].index == index):
 354                 # Already checked prefilled transactions above
 355                 header_and_shortids.prefilled_txn.pop(0)
 356             else:
 357                 tx_hash = block.vtx[index].sha256
 358                 if version == 2:
 359                     tx_hash = block.vtx[index].calc_sha256(True)
 360                 shortid = calculate_shortid(k0, k1, tx_hash)
 361                 assert_equal(shortid, header_and_shortids.shortids[0])
 362                 header_and_shortids.shortids.pop(0)
 363             index += 1
 364
 365     # Test that bitcoind requests compact blocks when we announce new blocks
 366     # via header or inv, and that responding to getblocktxn causes the block
 367     # to be successfully reconstructed.
 368     # Post-segwit: upgraded nodes would only make this request of cb-version-2,
 369     # NODE_WITNESS peers.  Unupgraded nodes would still make this request of
 370     # any cb-version-1-supporting peer.
 371     def test_compactblock_requests(self, node, test_node, version, segwit):
 372         # Try announcing a block with an inv or header, expect a compactblock
 373         # request
 374         for announce in ["inv", "header"]:
 375             block = self.build_block_on_tip(node, segwit=segwit)
 376             with mininode_lock:
 377                 test_node.last_message.pop("getdata", None)
 378
 379             if announce == "inv":
 380                 test_node.send_message(msg_inv([CInv(2, block.sha256)]))
 381                 wait_until(lambda: "getheaders" in test_node.last_message, timeout=30, lock=mininode_lock)
 382                 test_node.send_header_for_blocks([block])
 383             else:
 384                 test_node.send_header_for_blocks([block])
 385             wait_until(lambda: "getdata" in test_node.last_message, timeout=30, lock=mininode_lock)
 386             assert_equal(len(test_node.last_message["getdata"].inv), 1)
 387             assert_equal(test_node.last_message["getdata"].inv[0].type, 4)
 388             assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256)
 389
 390             # Send back a compactblock message that omits the coinbase
 391             comp_block = HeaderAndShortIDs()
 392             comp_block.header = CBlockHeader(block)
 393             comp_block.nonce = 0
 394             [k0, k1] = comp_block.get_siphash_keys()
 395             coinbase_hash = block.vtx[0].sha256
 396             if version == 2:
 397                 coinbase_hash = block.vtx[0].calc_sha256(True)
 398             comp_block.shortids = [
 399                     calculate_shortid(k0, k1, coinbase_hash) ]
 400             test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
 401             assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock)
 402             # Expect a getblocktxn message.
 403             with mininode_lock:
 404                 assert("getblocktxn" in test_node.last_message)
 405                 absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute()
 406             assert_equal(absolute_indexes, [0])  # should be a coinbase request
 407
 408             # Send the coinbase, and verify that the tip advances.
 409             if version == 2:
 410                 msg = msg_witness_blocktxn()
 411             else:
 412                 msg = msg_blocktxn()
 413             msg.block_transactions.blockhash = block.sha256
 414             msg.block_transactions.transactions = [block.vtx[0]]
 415             test_node.send_and_ping(msg)
 416             assert_equal(int(node.getbestblockhash(), 16), block.sha256)
 417
 418     # Create a chain of transactions from given utxo, and add to a new block.
 419     def build_block_with_transactions(self, node, utxo, num_transactions):
 420         block = self.build_block_on_tip(node)
 421
 422         for i in range(num_transactions):
 423             tx = CTransaction()
 424             tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b''))
 425             tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE])))
 426             tx.rehash()
 427             utxo = [tx.sha256, 0, tx.vout[0].nValue]
 428             block.vtx.append(tx)
 429
 430         block.hashMerkleRoot = block.calc_merkle_root()
 431         block.solve()
 432         return block
 433
 434     # Test that we only receive getblocktxn requests for transactions that the
 435     # node needs, and that responding to them causes the block to be
 436     # reconstructed.
 437     def test_getblocktxn_requests(self, node, test_node, version):
 438         with_witness = (version==2)
 439
 440         def test_getblocktxn_response(compact_block, peer, expected_result):
 441             msg = msg_cmpctblock(compact_block.to_p2p())
 442             peer.send_and_ping(msg)
 443             with mininode_lock:
 444                 assert("getblocktxn" in peer.last_message)
 445                 absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute()
 446             assert_equal(absolute_indexes, expected_result)
 447
 448         def test_tip_after_message(node, peer, msg, tip):
 449             peer.send_and_ping(msg)
 450             assert_equal(int(node.getbestblockhash(), 16), tip)
 451
 452         # First try announcing compactblocks that won't reconstruct, and verify
 453         # that we receive getblocktxn messages back.
 454         utxo = self.utxos.pop(0)
 455
 456         block = self.build_block_with_transactions(node, utxo, 5)
 457         self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
 458         comp_block = HeaderAndShortIDs()
 459         comp_block.initialize_from_block(block, use_witness=with_witness)
 460
 461         test_getblocktxn_response(comp_block, test_node, [1, 2, 3, 4, 5])
 462
 463         msg_bt = msg_blocktxn()
 464         if with_witness:
 465             msg_bt = msg_witness_blocktxn() # serialize with witnesses
 466         msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[1:])
 467         test_tip_after_message(node, test_node, msg_bt, block.sha256)
 468
 469         utxo = self.utxos.pop(0)
 470         block = self.build_block_with_transactions(node, utxo, 5)
 471         self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
 472
 473         # Now try interspersing the prefilled transactions
 474         comp_block.initialize_from_block(block, prefill_list=[0, 1, 5], use_witness=with_witness)
 475         test_getblocktxn_response(comp_block, test_node, [2, 3, 4])
 476         msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5])
 477         test_tip_after_message(node, test_node, msg_bt, block.sha256)
 478
 479         # Now try giving one transaction ahead of time.
 480         utxo = self.utxos.pop(0)
 481         block = self.build_block_with_transactions(node, utxo, 5)
 482         self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
 483         test_node.send_and_ping(msg_tx(block.vtx[1]))
 484         assert(block.vtx[1].hash in node.getrawmempool())
 485
 486         # Prefill 4 out of the 6 transactions, and verify that only the one
 487         # that was not in the mempool is requested.
 488         comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4], use_witness=with_witness)
 489         test_getblocktxn_response(comp_block, test_node, [5])
 490
 491         msg_bt.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]])
 492         test_tip_after_message(node, test_node, msg_bt, block.sha256)
 493
 494         # Now provide all transactions to the node before the block is
 495         # announced and verify reconstruction happens immediately.
 496         utxo = self.utxos.pop(0)
 497         block = self.build_block_with_transactions(node, utxo, 10)
 498         self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
 499         for tx in block.vtx[1:]:
 500             test_node.send_message(msg_tx(tx))
 501         test_node.sync_with_ping()
 502         # Make sure all transactions were accepted.
 503         mempool = node.getrawmempool()
 504         for tx in block.vtx[1:]:
 505             assert(tx.hash in mempool)
 506
 507         # Clear out last request.
 508         with mininode_lock:
 509             test_node.last_message.pop("getblocktxn", None)
 510
 511         # Send compact block
 512         comp_block.initialize_from_block(block, prefill_list=[0], use_witness=with_witness)
 513         test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256)
 514         with mininode_lock:
 515             # Shouldn't have gotten a request for any transaction
 516             assert("getblocktxn" not in test_node.last_message)
 517
 518     # Incorrectly responding to a getblocktxn shouldn't cause the block to be
 519     # permanently failed.
 520     def test_incorrect_blocktxn_response(self, node, test_node, version):
 521         if (len(self.utxos) == 0):
 522             self.make_utxos()
 523         utxo = self.utxos.pop(0)
 524
 525         block = self.build_block_with_transactions(node, utxo, 10)
 526         self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
 527         # Relay the first 5 transactions from the block in advance
 528         for tx in block.vtx[1:6]:
 529             test_node.send_message(msg_tx(tx))
 530         test_node.sync_with_ping()
 531         # Make sure all transactions were accepted.
 532         mempool = node.getrawmempool()
 533         for tx in block.vtx[1:6]:
 534             assert(tx.hash in mempool)
 535
 536         # Send compact block
 537         comp_block = HeaderAndShortIDs()
 538         comp_block.initialize_from_block(block, prefill_list=[0], use_witness=(version == 2))
 539         test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
 540         absolute_indexes = []
 541         with mininode_lock:
 542             assert("getblocktxn" in test_node.last_message)
 543             absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute()
 544         assert_equal(absolute_indexes, [6, 7, 8, 9, 10])
 545
 546         # Now give an incorrect response.
 547         # Note that it's possible for bitcoind to be smart enough to know we're
 548         # lying, since it could check to see if the shortid matches what we're
 549         # sending, and eg disconnect us for misbehavior.  If that behavior
 550         # change were made, we could just modify this test by having a
 551         # different peer provide the block further down, so that we're still
 552         # verifying that the block isn't marked bad permanently. This is good
 553         # enough for now.
 554         msg = msg_blocktxn()
 555         if version==2:
 556             msg = msg_witness_blocktxn()
 557         msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:])
 558         test_node.send_and_ping(msg)
 559
 560         # Tip should not have updated
 561         assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock)
 562
 563         # We should receive a getdata request
 564         wait_until(lambda: "getdata" in test_node.last_message, timeout=10, lock=mininode_lock)
 565         assert_equal(len(test_node.last_message["getdata"].inv), 1)
 566         assert(test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2|MSG_WITNESS_FLAG)
 567         assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256)
 568
 569         # Deliver the block
 570         if version==2:
 571             test_node.send_and_ping(msg_witness_block(block))
 572         else:
 573             test_node.send_and_ping(msg_block(block))
 574         assert_equal(int(node.getbestblockhash(), 16), block.sha256)
 575
 576     def test_getblocktxn_handler(self, node, test_node, version):
 577         # bitcoind will not send blocktxn responses for blocks whose height is
 578         # more than 10 blocks deep.
 579         MAX_GETBLOCKTXN_DEPTH = 10
 580         chain_height = node.getblockcount()
 581         current_height = chain_height
 582         while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH):
 583             block_hash = node.getblockhash(current_height)
 584             block = FromHex(CBlock(), node.getblock(block_hash, False))
 585
 586             msg = msg_getblocktxn()
 587             msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [])
 588             num_to_request = random.randint(1, len(block.vtx))
 589             msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request)))
 590             test_node.send_message(msg)
 591             wait_until(lambda: "blocktxn" in test_node.last_message, timeout=10, lock=mininode_lock)
 592
 593             [tx.calc_sha256() for tx in block.vtx]
 594             with mininode_lock:
 595                 assert_equal(test_node.last_message["blocktxn"].block_transactions.blockhash, int(block_hash, 16))
 596                 all_indices = msg.block_txn_request.to_absolute()
 597                 for index in all_indices:
 598                     tx = test_node.last_message["blocktxn"].block_transactions.transactions.pop(0)
 599                     tx.calc_sha256()
 600                     assert_equal(tx.sha256, block.vtx[index].sha256)
 601                     if version == 1:
 602                         # Witnesses should have been stripped
 603                         assert(tx.wit.is_null())
 604                     else:
 605                         # Check that the witness matches
 606                         assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True))
 607                 test_node.last_message.pop("blocktxn", None)
 608             current_height -= 1
 609
 610         # Next request should send a full block response, as we're past the
 611         # allowed depth for a blocktxn response.
 612         block_hash = node.getblockhash(current_height)
 613         msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0])
 614         with mininode_lock:
 615             test_node.last_message.pop("block", None)
 616             test_node.last_message.pop("blocktxn", None)
 617         test_node.send_and_ping(msg)
 618         with mininode_lock:
 619             test_node.last_message["block"].block.calc_sha256()
 620             assert_equal(test_node.last_message["block"].block.sha256, int(block_hash, 16))
 621             assert "blocktxn" not in test_node.last_message
 622
 623     def test_compactblocks_not_at_tip(self, node, test_node):
 624         # Test that requesting old compactblocks doesn't work.
 625         MAX_CMPCTBLOCK_DEPTH = 5
 626         new_blocks = []
 627         for i in range(MAX_CMPCTBLOCK_DEPTH + 1):
 628             test_node.clear_block_announcement()
 629             new_blocks.append(node.generate(1)[0])
 630             wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock)
 631
 632         test_node.clear_block_announcement()
 633         test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))]))
 634         wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30, lock=mininode_lock)
 635
 636         test_node.clear_block_announcement()
 637         node.generate(1)
 638         wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock)
 639         test_node.clear_block_announcement()
 640         with mininode_lock:
 641             test_node.last_message.pop("block", None)
 642         test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))]))
 643         wait_until(lambda: "block" in test_node.last_message, timeout=30, lock=mininode_lock)
 644         with mininode_lock:
 645             test_node.last_message["block"].block.calc_sha256()
 646             assert_equal(test_node.last_message["block"].block.sha256, int(new_blocks[0], 16))
 647
 648         # Generate an old compactblock, and verify that it's not accepted.
 649         cur_height = node.getblockcount()
 650         hashPrevBlock = int(node.getblockhash(cur_height-5), 16)
 651         block = self.build_block_on_tip(node)
 652         block.hashPrevBlock = hashPrevBlock
 653         block.solve()
 654
 655         comp_block = HeaderAndShortIDs()
 656         comp_block.initialize_from_block(block)
 657         test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
 658
 659         tips = node.getchaintips()
 660         found = False
 661         for x in tips:
 662             if x["hash"] == block.hash:
 663                 assert_equal(x["status"], "headers-only")
 664                 found = True
 665                 break
 666         assert(found)
 667
 668         # Requesting this block via getblocktxn should silently fail
 669         # (to avoid fingerprinting attacks).
 670         msg = msg_getblocktxn()
 671         msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0])
 672         with mininode_lock:
 673             test_node.last_message.pop("blocktxn", None)
 674         test_node.send_and_ping(msg)
 675         with mininode_lock:
 676             assert "blocktxn" not in test_node.last_message
 677
 678     def activate_segwit(self, node):
 679         node.generate(144*3)
 680         assert_equal(get_bip9_status(node, "segwit")["status"], 'active')
 681
 682     def test_end_to_end_block_relay(self, node, listeners):
 683         utxo = self.utxos.pop(0)
 684
 685         block = self.build_block_with_transactions(node, utxo, 10)
 686
 687         [l.clear_block_announcement() for l in listeners]
 688
 689         # ToHex() won't serialize with witness, but this block has no witnesses
 690         # anyway. TODO: repeat this test with witness tx's to a segwit node.
 691         node.submitblock(ToHex(block))
 692
 693         for l in listeners:
 694             wait_until(lambda: l.received_block_announcement(), timeout=30, lock=mininode_lock)
 695         with mininode_lock:
 696             for l in listeners:
 697                 assert "cmpctblock" in l.last_message
 698                 l.last_message["cmpctblock"].header_and_shortids.header.calc_sha256()
 699                 assert_equal(l.last_message["cmpctblock"].header_and_shortids.header.sha256, block.sha256)
 700
 701     # Test that we don't get disconnected if we relay a compact block with valid header,
 702     # but invalid transactions.
 703     def test_invalid_tx_in_compactblock(self, node, test_node, use_segwit):
 704         assert(len(self.utxos))
 705         utxo = self.utxos[0]
 706
 707         block = self.build_block_with_transactions(node, utxo, 5)
 708         del block.vtx[3]
 709         block.hashMerkleRoot = block.calc_merkle_root()
 710         if use_segwit:
 711             # If we're testing with segwit, also drop the coinbase witness,
 712             # but include the witness commitment.
 713             add_witness_commitment(block)
 714             block.vtx[0].wit.vtxinwit = []
 715         block.solve()
 716
 717         # Now send the compact block with all transactions prefilled, and
 718         # verify that we don't get disconnected.
 719         comp_block = HeaderAndShortIDs()
 720         comp_block.initialize_from_block(block, prefill_list=[0, 1, 2, 3, 4], use_witness=use_segwit)
 721         msg = msg_cmpctblock(comp_block.to_p2p())
 722         test_node.send_and_ping(msg)
 723
 724         # Check that the tip didn't advance
 725         assert(int(node.getbestblockhash(), 16) is not block.sha256)
 726         test_node.sync_with_ping()
 727
 728     # Helper for enabling cb announcements
 729     # Send the sendcmpct request and sync headers
 730     def request_cb_announcements(self, peer, node, version):
 731         tip = node.getbestblockhash()
 732         peer.get_headers(locator=[int(tip, 16)], hashstop=0)
 733
 734         msg = msg_sendcmpct()
 735         msg.version = version
 736         msg.announce = True
 737         peer.send_and_ping(msg)
 738
 739     def test_compactblock_reconstruction_multiple_peers(self, node, stalling_peer, delivery_peer):
 740         assert(len(self.utxos))
 741
 742         def announce_cmpct_block(node, peer):
 743             utxo = self.utxos.pop(0)
 744             block = self.build_block_with_transactions(node, utxo, 5)
 745
 746             cmpct_block = HeaderAndShortIDs()
 747             cmpct_block.initialize_from_block(block)
 748             msg = msg_cmpctblock(cmpct_block.to_p2p())
 749             peer.send_and_ping(msg)
 750             with mininode_lock:
 751                 assert "getblocktxn" in peer.last_message
 752             return block, cmpct_block
 753
 754         block, cmpct_block = announce_cmpct_block(node, stalling_peer)
 755
 756         for tx in block.vtx[1:]:
 757             delivery_peer.send_message(msg_tx(tx))
 758         delivery_peer.sync_with_ping()
 759         mempool = node.getrawmempool()
 760         for tx in block.vtx[1:]:
 761             assert(tx.hash in mempool)
 762
 763         delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p()))
 764         assert_equal(int(node.getbestblockhash(), 16), block.sha256)
 765
 766         self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
 767
 768         # Now test that delivering an invalid compact block won't break relay
 769
 770         block, cmpct_block = announce_cmpct_block(node, stalling_peer)
 771         for tx in block.vtx[1:]:
 772             delivery_peer.send_message(msg_tx(tx))
 773         delivery_peer.sync_with_ping()
 774
 775         cmpct_block.prefilled_txn[0].tx.wit.vtxinwit = [ CTxInWitness() ]
 776         cmpct_block.prefilled_txn[0].tx.wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(0)]
 777
 778         cmpct_block.use_witness = True
 779         delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p()))
 780         assert(int(node.getbestblockhash(), 16) != block.sha256)
 781
 782         msg = msg_blocktxn()
 783         msg.block_transactions.blockhash = block.sha256
 784         msg.block_transactions.transactions = block.vtx[1:]
 785         stalling_peer.send_and_ping(msg)
 786         assert_equal(int(node.getbestblockhash(), 16), block.sha256)
 787
 788     def run_test(self):
 789         # Setup the p2p connections and start up the network thread.
 790         self.test_node = TestNode()
 791         self.segwit_node = TestNode()
 792         self.old_node = TestNode()  # version 1 peer <--> segwit node
 793
 794         connections = []
 795         connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node))
 796         connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1],
 797                     self.segwit_node, services=NODE_NETWORK|NODE_WITNESS))
 798         connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1],
 799                     self.old_node, services=NODE_NETWORK))
 800         self.test_node.add_connection(connections[0])
 801         self.segwit_node.add_connection(connections[1])
 802         self.old_node.add_connection(connections[2])
 803
 804         NetworkThread().start()  # Start up network handling in another thread
 805
 806         # Test logic begins here
 807         self.test_node.wait_for_verack()
 808
 809         # We will need UTXOs to construct transactions in later tests.
 810         self.make_utxos()
 811
 812         self.log.info("Running tests, pre-segwit activation:")
 813
 814         self.log.info("Testing SENDCMPCT p2p message... ")
 815         self.test_sendcmpct(self.nodes[0], self.test_node, 1)
 816         sync_blocks(self.nodes)
 817         self.test_sendcmpct(self.nodes[1], self.segwit_node, 2, old_node=self.old_node)
 818         sync_blocks(self.nodes)
 819
 820         self.log.info("Testing compactblock construction...")
 821         self.test_compactblock_construction(self.nodes[0], self.test_node, 1, False)
 822         sync_blocks(self.nodes)
 823         self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, False)
 824         sync_blocks(self.nodes)
 825
 826         self.log.info("Testing compactblock requests... ")
 827         self.test_compactblock_requests(self.nodes[0], self.test_node, 1, False)
 828         sync_blocks(self.nodes)
 829         self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, False)
 830         sync_blocks(self.nodes)
 831
 832         self.log.info("Testing getblocktxn requests...")
 833         self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1)
 834         sync_blocks(self.nodes)
 835         self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2)
 836         sync_blocks(self.nodes)
 837
 838         self.log.info("Testing getblocktxn handler...")
 839         self.test_getblocktxn_handler(self.nodes[0], self.test_node, 1)
 840         sync_blocks(self.nodes)
 841         self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2)
 842         self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1)
 843         sync_blocks(self.nodes)
 844
 845         self.log.info("Testing compactblock requests/announcements not at chain tip...")
 846         self.test_compactblocks_not_at_tip(self.nodes[0], self.test_node)
 847         sync_blocks(self.nodes)
 848         self.test_compactblocks_not_at_tip(self.nodes[1], self.segwit_node)
 849         self.test_compactblocks_not_at_tip(self.nodes[1], self.old_node)
 850         sync_blocks(self.nodes)
 851
 852         self.log.info("Testing handling of incorrect blocktxn responses...")
 853         self.test_incorrect_blocktxn_response(self.nodes[0], self.test_node, 1)
 854         sync_blocks(self.nodes)
 855         self.test_incorrect_blocktxn_response(self.nodes[1], self.segwit_node, 2)
 856         sync_blocks(self.nodes)
 857
 858         # End-to-end block relay tests
 859         self.log.info("Testing end-to-end block relay...")
 860         self.request_cb_announcements(self.test_node, self.nodes[0], 1)
 861         self.request_cb_announcements(self.old_node, self.nodes[1], 1)
 862         self.request_cb_announcements(self.segwit_node, self.nodes[1], 2)
 863         self.test_end_to_end_block_relay(self.nodes[0], [self.segwit_node, self.test_node, self.old_node])
 864         self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node])
 865
 866         self.log.info("Testing handling of invalid compact blocks...")
 867         self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False)
 868         self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, False)
 869         self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, False)
 870
 871         self.log.info("Testing reconstructing compact blocks from all peers...")
 872         self.test_compactblock_reconstruction_multiple_peers(self.nodes[1], self.segwit_node, self.old_node)
 873         sync_blocks(self.nodes)
 874
 875         # Advance to segwit activation
 876         self.log.info("Advancing to segwit activation")
 877         self.activate_segwit(self.nodes[1])
 878         self.log.info("Running tests, post-segwit activation...")
 879
 880         self.log.info("Testing compactblock construction...")
 881         self.test_compactblock_construction(self.nodes[1], self.old_node, 1, True)
 882         self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, True)
 883         sync_blocks(self.nodes)
 884
 885         self.log.info("Testing compactblock requests (unupgraded node)... ")
 886         self.test_compactblock_requests(self.nodes[0], self.test_node, 1, True)
 887
 888         self.log.info("Testing getblocktxn requests (unupgraded node)...")
 889         self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1)
 890
 891         # Need to manually sync node0 and node1, because post-segwit activation,
 892         # node1 will not download blocks from node0.
 893         self.log.info("Syncing nodes...")
 894         assert(self.nodes[0].getbestblockhash() != self.nodes[1].getbestblockhash())
 895         while (self.nodes[0].getblockcount() > self.nodes[1].getblockcount()):
 896             block_hash = self.nodes[0].getblockhash(self.nodes[1].getblockcount()+1)
 897             self.nodes[1].submitblock(self.nodes[0].getblock(block_hash, False))
 898         assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash())
 899
 900         self.log.info("Testing compactblock requests (segwit node)... ")
 901         self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, True)
 902
 903         self.log.info("Testing getblocktxn requests (segwit node)...")
 904         self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2)
 905         sync_blocks(self.nodes)
 906
 907         self.log.info("Testing getblocktxn handler (segwit node should return witnesses)...")
 908         self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2)
 909         self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1)
 910
 911         # Test that if we submitblock to node1, we'll get a compact block
 912         # announcement to all peers.
 913         # (Post-segwit activation, blocks won't propagate from node0 to node1
 914         # automatically, so don't bother testing a block announced to node0.)
 915         self.log.info("Testing end-to-end block relay...")
 916         self.request_cb_announcements(self.test_node, self.nodes[0], 1)
 917         self.request_cb_announcements(self.old_node, self.nodes[1], 1)
 918         self.request_cb_announcements(self.segwit_node, self.nodes[1], 2)
 919         self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node])
 920
 921         self.log.info("Testing handling of invalid compact blocks...")
 922         self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False)
 923         self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, True)
 924         self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, True)
 925
 926         self.log.info("Testing invalid index in cmpctblock message...")
 927         self.test_invalid_cmpctblock_message()
 928
 929
 930 if __name__ == '__main__':
 931     CompactBlocksTest().main()