test/functional/p2p-compactblocks.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2016 The Bitcoin Core developers
   3 # Distributed under the MIT software license, see the accompanying
   4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5 """Test compact blocks (BIP 152).
   6
   7 Version 1 compact blocks are pre-segwit (txids)
   8 Version 2 compact blocks are post-segwit (wtxids)
   9 """
  10
  11 from test_framework.mininode import *
  12 from test_framework.test_framework import BitcoinTestFramework
  13 from test_framework.util import *
  14 from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment
  15 from test_framework.script import CScript, OP_TRUE
  16
  17 # TestNode: A peer we use to send messages to bitcoind, and store responses.
  18 class TestNode(NodeConnCB):
  19     def __init__(self):
  20         super().__init__()
  21         self.last_sendcmpct = []
  22         self.block_announced = False
  23         # Store the hashes of blocks we've seen announced.
  24         # This is for synchronizing the p2p message traffic,
  25         # so we can eg wait until a particular block is announced.
  26         self.announced_blockhashes = set()
  27
  28     def on_sendcmpct(self, conn, message):
  29         self.last_sendcmpct.append(message)
  30
  31     def on_cmpctblock(self, conn, message):
  32         self.block_announced = True
  33         self.last_message["cmpctblock"].header_and_shortids.header.calc_sha256()
  34         self.announced_blockhashes.add(self.last_message["cmpctblock"].header_and_shortids.header.sha256)
  35
  36     def on_headers(self, conn, message):
  37         self.block_announced = True
  38         for x in self.last_message["headers"].headers:
  39             x.calc_sha256()
  40             self.announced_blockhashes.add(x.sha256)
  41
  42     def on_inv(self, conn, message):
  43         for x in self.last_message["inv"].inv:
  44             if x.type == 2:
  45                 self.block_announced = True
  46                 self.announced_blockhashes.add(x.hash)
  47
  48     # Requires caller to hold mininode_lock
  49     def received_block_announcement(self):
  50         return self.block_announced
  51
  52     def clear_block_announcement(self):
  53         with mininode_lock:
  54             self.block_announced = False
  55             self.last_message.pop("inv", None)
  56             self.last_message.pop("headers", None)
  57             self.last_message.pop("cmpctblock", None)
  58
  59     def get_headers(self, locator, hashstop):
  60         msg = msg_getheaders()
  61         msg.locator.vHave = locator
  62         msg.hashstop = hashstop
  63         self.connection.send_message(msg)
  64
  65     def send_header_for_blocks(self, new_blocks):
  66         headers_message = msg_headers()
  67         headers_message.headers = [CBlockHeader(b) for b in new_blocks]
  68         self.send_message(headers_message)
  69
  70     def request_headers_and_sync(self, locator, hashstop=0):
  71         self.clear_block_announcement()
  72         self.get_headers(locator, hashstop)
  73         assert wait_until(self.received_block_announcement, timeout=30)
  74         self.clear_block_announcement()
  75
  76     # Block until a block announcement for a particular block hash is
  77     # received.
  78     def wait_for_block_announcement(self, block_hash, timeout=30):
  79         def received_hash():
  80             return (block_hash in self.announced_blockhashes)
  81         return wait_until(received_hash, timeout=timeout)
  82
  83     def send_await_disconnect(self, message, timeout=30):
  84         """Sends a message to the node and wait for disconnect.
  85
  86         This is used when we want to send a message into the node that we expect
  87         will get us disconnected, eg an invalid block."""
  88         self.send_message(message)
  89         success = wait_until(lambda: not self.connected, timeout=timeout)
  90         if not success:
  91             logger.error("send_await_disconnect failed!")
  92             raise AssertionError("send_await_disconnect failed!")
  93         return success
  94
  95 class CompactBlocksTest(BitcoinTestFramework):
  96     def __init__(self):
  97         super().__init__()
  98         self.setup_clean_chain = True
  99         # Node0 = pre-segwit, node1 = segwit-aware
 100         self.num_nodes = 2
 101         self.extra_args = [["-vbparams=segwit:0:0"], ["-txindex"]]
 102         self.utxos = []
 103
 104     def build_block_on_tip(self, node, segwit=False):
 105         height = node.getblockcount()
 106         tip = node.getbestblockhash()
 107         mtp = node.getblockheader(tip)['mediantime']
 108         block = create_block(int(tip, 16), create_coinbase(height + 1), mtp + 1)
 109         block.nVersion = 4
 110         if segwit:
 111             add_witness_commitment(block)
 112         block.solve()
 113         return block
 114
 115     # Create 10 more anyone-can-spend utxo's for testing.
 116     def make_utxos(self):
 117         # Doesn't matter which node we use, just use node0.
 118         block = self.build_block_on_tip(self.nodes[0])
 119         self.test_node.send_and_ping(msg_block(block))
 120         assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256)
 121         self.nodes[0].generate(100)
 122
 123         total_value = block.vtx[0].vout[0].nValue
 124         out_value = total_value // 10
 125         tx = CTransaction()
 126         tx.vin.append(CTxIn(COutPoint(block.vtx[0].sha256, 0), b''))
 127         for i in range(10):
 128             tx.vout.append(CTxOut(out_value, CScript([OP_TRUE])))
 129         tx.rehash()
 130
 131         block2 = self.build_block_on_tip(self.nodes[0])
 132         block2.vtx.append(tx)
 133         block2.hashMerkleRoot = block2.calc_merkle_root()
 134         block2.solve()
 135         self.test_node.send_and_ping(msg_block(block2))
 136         assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256)
 137         self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)])
 138         return
 139
 140     # Test "sendcmpct" (between peers preferring the same version):
 141     # - No compact block announcements unless sendcmpct is sent.
 142     # - If sendcmpct is sent with version > preferred_version, the message is ignored.
 143     # - If sendcmpct is sent with boolean 0, then block announcements are not
 144     #   made with compact blocks.
 145     # - If sendcmpct is then sent with boolean 1, then new block announcements
 146     #   are made with compact blocks.
 147     # If old_node is passed in, request compact blocks with version=preferred-1
 148     # and verify that it receives block announcements via compact block.
 149     def test_sendcmpct(self, node, test_node, preferred_version, old_node=None):
 150         # Make sure we get a SENDCMPCT message from our peer
 151         def received_sendcmpct():
 152             return (len(test_node.last_sendcmpct) > 0)
 153         got_message = wait_until(received_sendcmpct, timeout=30)
 154         assert(received_sendcmpct())
 155         assert(got_message)
 156         with mininode_lock:
 157             # Check that the first version received is the preferred one
 158             assert_equal(test_node.last_sendcmpct[0].version, preferred_version)
 159             # And that we receive versions down to 1.
 160             assert_equal(test_node.last_sendcmpct[-1].version, 1)
 161             test_node.last_sendcmpct = []
 162
 163         tip = int(node.getbestblockhash(), 16)
 164
 165         def check_announcement_of_new_block(node, peer, predicate):
 166             peer.clear_block_announcement()
 167             block_hash = int(node.generate(1)[0], 16)
 168             peer.wait_for_block_announcement(block_hash, timeout=30)
 169             assert(peer.block_announced)
 170             assert(got_message)
 171
 172             with mininode_lock:
 173                 assert predicate(peer), (
 174                     "block_hash={!r}, cmpctblock={!r}, inv={!r}".format(
 175                         block_hash, peer.last_message.get("cmpctblock", None), peer.last_message.get("inv", None)))
 176
 177         # We shouldn't get any block announcements via cmpctblock yet.
 178         check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message)
 179
 180         # Try one more time, this time after requesting headers.
 181         test_node.request_headers_and_sync(locator=[tip])
 182         check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "inv" in p.last_message)
 183
 184         # Test a few ways of using sendcmpct that should NOT
 185         # result in compact block announcements.
 186         # Before each test, sync the headers chain.
 187         test_node.request_headers_and_sync(locator=[tip])
 188
 189         # Now try a SENDCMPCT message with too-high version
 190         sendcmpct = msg_sendcmpct()
 191         sendcmpct.version = preferred_version+1
 192         sendcmpct.announce = True
 193         test_node.send_and_ping(sendcmpct)
 194         check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message)
 195
 196         # Headers sync before next test.
 197         test_node.request_headers_and_sync(locator=[tip])
 198
 199         # Now try a SENDCMPCT message with valid version, but announce=False
 200         sendcmpct.version = preferred_version
 201         sendcmpct.announce = False
 202         test_node.send_and_ping(sendcmpct)
 203         check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message)
 204
 205         # Headers sync before next test.
 206         test_node.request_headers_and_sync(locator=[tip])
 207
 208         # Finally, try a SENDCMPCT message with announce=True
 209         sendcmpct.version = preferred_version
 210         sendcmpct.announce = True
 211         test_node.send_and_ping(sendcmpct)
 212         check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message)
 213
 214         # Try one more time (no headers sync should be needed!)
 215         check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message)
 216
 217         # Try one more time, after turning on sendheaders
 218         test_node.send_and_ping(msg_sendheaders())
 219         check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message)
 220
 221         # Try one more time, after sending a version-1, announce=false message.
 222         sendcmpct.version = preferred_version-1
 223         sendcmpct.announce = False
 224         test_node.send_and_ping(sendcmpct)
 225         check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message)
 226
 227         # Now turn off announcements
 228         sendcmpct.version = preferred_version
 229         sendcmpct.announce = False
 230         test_node.send_and_ping(sendcmpct)
 231         check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "headers" in p.last_message)
 232
 233         if old_node is not None:
 234             # Verify that a peer using an older protocol version can receive
 235             # announcements from this node.
 236             sendcmpct.version = preferred_version-1
 237             sendcmpct.announce = True
 238             old_node.send_and_ping(sendcmpct)
 239             # Header sync
 240             old_node.request_headers_and_sync(locator=[tip])
 241             check_announcement_of_new_block(node, old_node, lambda p: "cmpctblock" in p.last_message)
 242
 243     # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last.
 244     def test_invalid_cmpctblock_message(self):
 245         self.nodes[0].generate(101)
 246         block = self.build_block_on_tip(self.nodes[0])
 247
 248         cmpct_block = P2PHeaderAndShortIDs()
 249         cmpct_block.header = CBlockHeader(block)
 250         cmpct_block.prefilled_txn_length = 1
 251         # This index will be too high
 252         prefilled_txn = PrefilledTransaction(1, block.vtx[0])
 253         cmpct_block.prefilled_txn = [prefilled_txn]
 254         self.test_node.send_await_disconnect(msg_cmpctblock(cmpct_block))
 255         assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock)
 256
 257     # Compare the generated shortids to what we expect based on BIP 152, given
 258     # bitcoind's choice of nonce.
 259     def test_compactblock_construction(self, node, test_node, version, use_witness_address):
 260         # Generate a bunch of transactions.
 261         node.generate(101)
 262         num_transactions = 25
 263         address = node.getnewaddress()
 264         if use_witness_address:
 265             # Want at least one segwit spend, so move all funds to
 266             # a witness address.
 267             address = node.addwitnessaddress(address)
 268             value_to_send = node.getbalance()
 269             node.sendtoaddress(address, satoshi_round(value_to_send-Decimal(0.1)))
 270             node.generate(1)
 271
 272         segwit_tx_generated = False
 273         for i in range(num_transactions):
 274             txid = node.sendtoaddress(address, 0.1)
 275             hex_tx = node.gettransaction(txid)["hex"]
 276             tx = FromHex(CTransaction(), hex_tx)
 277             if not tx.wit.is_null():
 278                 segwit_tx_generated = True
 279
 280         if use_witness_address:
 281             assert(segwit_tx_generated) # check that our test is not broken
 282
 283         # Wait until we've seen the block announcement for the resulting tip
 284         tip = int(node.getbestblockhash(), 16)
 285         assert(test_node.wait_for_block_announcement(tip))
 286
 287         # Make sure we will receive a fast-announce compact block
 288         self.request_cb_announcements(test_node, node, version)
 289
 290         # Now mine a block, and look at the resulting compact block.
 291         test_node.clear_block_announcement()
 292         block_hash = int(node.generate(1)[0], 16)
 293
 294         # Store the raw block in our internal format.
 295         block = FromHex(CBlock(), node.getblock("%02x" % block_hash, False))
 296         [tx.calc_sha256() for tx in block.vtx]
 297         block.rehash()
 298
 299         # Wait until the block was announced (via compact blocks)
 300         wait_until(test_node.received_block_announcement, timeout=30)
 301         assert(test_node.received_block_announcement())
 302
 303         # Now fetch and check the compact block
 304         header_and_shortids = None
 305         with mininode_lock:
 306             assert("cmpctblock" in test_node.last_message)
 307             # Convert the on-the-wire representation to absolute indexes
 308             header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids)
 309         self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block)
 310
 311         # Now fetch the compact block using a normal non-announce getdata
 312         with mininode_lock:
 313             test_node.clear_block_announcement()
 314             inv = CInv(4, block_hash)  # 4 == "CompactBlock"
 315             test_node.send_message(msg_getdata([inv]))
 316
 317         wait_until(test_node.received_block_announcement, timeout=30)
 318         assert(test_node.received_block_announcement())
 319
 320         # Now fetch and check the compact block
 321         header_and_shortids = None
 322         with mininode_lock:
 323             assert("cmpctblock" in test_node.last_message)
 324             # Convert the on-the-wire representation to absolute indexes
 325             header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids)
 326         self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block)
 327
 328     def check_compactblock_construction_from_block(self, version, header_and_shortids, block_hash, block):
 329         # Check that we got the right block!
 330         header_and_shortids.header.calc_sha256()
 331         assert_equal(header_and_shortids.header.sha256, block_hash)
 332
 333         # Make sure the prefilled_txn appears to have included the coinbase
 334         assert(len(header_and_shortids.prefilled_txn) >= 1)
 335         assert_equal(header_and_shortids.prefilled_txn[0].index, 0)
 336
 337         # Check that all prefilled_txn entries match what's in the block.
 338         for entry in header_and_shortids.prefilled_txn:
 339             entry.tx.calc_sha256()
 340             # This checks the non-witness parts of the tx agree
 341             assert_equal(entry.tx.sha256, block.vtx[entry.index].sha256)
 342
 343             # And this checks the witness
 344             wtxid = entry.tx.calc_sha256(True)
 345             if version == 2:
 346                 assert_equal(wtxid, block.vtx[entry.index].calc_sha256(True))
 347             else:
 348                 # Shouldn't have received a witness
 349                 assert(entry.tx.wit.is_null())
 350
 351         # Check that the cmpctblock message announced all the transactions.
 352         assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx))
 353
 354         # And now check that all the shortids are as expected as well.
 355         # Determine the siphash keys to use.
 356         [k0, k1] = header_and_shortids.get_siphash_keys()
 357
 358         index = 0
 359         while index < len(block.vtx):
 360             if (len(header_and_shortids.prefilled_txn) > 0 and
 361                     header_and_shortids.prefilled_txn[0].index == index):
 362                 # Already checked prefilled transactions above
 363                 header_and_shortids.prefilled_txn.pop(0)
 364             else:
 365                 tx_hash = block.vtx[index].sha256
 366                 if version == 2:
 367                     tx_hash = block.vtx[index].calc_sha256(True)
 368                 shortid = calculate_shortid(k0, k1, tx_hash)
 369                 assert_equal(shortid, header_and_shortids.shortids[0])
 370                 header_and_shortids.shortids.pop(0)
 371             index += 1
 372
 373     # Test that bitcoind requests compact blocks when we announce new blocks
 374     # via header or inv, and that responding to getblocktxn causes the block
 375     # to be successfully reconstructed.
 376     # Post-segwit: upgraded nodes would only make this request of cb-version-2,
 377     # NODE_WITNESS peers.  Unupgraded nodes would still make this request of
 378     # any cb-version-1-supporting peer.
 379     def test_compactblock_requests(self, node, test_node, version, segwit):
 380         # Try announcing a block with an inv or header, expect a compactblock
 381         # request
 382         for announce in ["inv", "header"]:
 383             block = self.build_block_on_tip(node, segwit=segwit)
 384             with mininode_lock:
 385                 test_node.last_message.pop("getdata", None)
 386
 387             if announce == "inv":
 388                 test_node.send_message(msg_inv([CInv(2, block.sha256)]))
 389                 success = wait_until(lambda: "getheaders" in test_node.last_message, timeout=30)
 390                 assert(success)
 391                 test_node.send_header_for_blocks([block])
 392             else:
 393                 test_node.send_header_for_blocks([block])
 394             success = wait_until(lambda: "getdata" in test_node.last_message, timeout=30)
 395             assert(success)
 396             assert_equal(len(test_node.last_message["getdata"].inv), 1)
 397             assert_equal(test_node.last_message["getdata"].inv[0].type, 4)
 398             assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256)
 399
 400             # Send back a compactblock message that omits the coinbase
 401             comp_block = HeaderAndShortIDs()
 402             comp_block.header = CBlockHeader(block)
 403             comp_block.nonce = 0
 404             [k0, k1] = comp_block.get_siphash_keys()
 405             coinbase_hash = block.vtx[0].sha256
 406             if version == 2:
 407                 coinbase_hash = block.vtx[0].calc_sha256(True)
 408             comp_block.shortids = [
 409                     calculate_shortid(k0, k1, coinbase_hash) ]
 410             test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
 411             assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock)
 412             # Expect a getblocktxn message.
 413             with mininode_lock:
 414                 assert("getblocktxn" in test_node.last_message)
 415                 absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute()
 416             assert_equal(absolute_indexes, [0])  # should be a coinbase request
 417
 418             # Send the coinbase, and verify that the tip advances.
 419             if version == 2:
 420                 msg = msg_witness_blocktxn()
 421             else:
 422                 msg = msg_blocktxn()
 423             msg.block_transactions.blockhash = block.sha256
 424             msg.block_transactions.transactions = [block.vtx[0]]
 425             test_node.send_and_ping(msg)
 426             assert_equal(int(node.getbestblockhash(), 16), block.sha256)
 427
 428     # Create a chain of transactions from given utxo, and add to a new block.
 429     def build_block_with_transactions(self, node, utxo, num_transactions):
 430         block = self.build_block_on_tip(node)
 431
 432         for i in range(num_transactions):
 433             tx = CTransaction()
 434             tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b''))
 435             tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE])))
 436             tx.rehash()
 437             utxo = [tx.sha256, 0, tx.vout[0].nValue]
 438             block.vtx.append(tx)
 439
 440         block.hashMerkleRoot = block.calc_merkle_root()
 441         block.solve()
 442         return block
 443
 444     # Test that we only receive getblocktxn requests for transactions that the
 445     # node needs, and that responding to them causes the block to be
 446     # reconstructed.
 447     def test_getblocktxn_requests(self, node, test_node, version):
 448         with_witness = (version==2)
 449
 450         def test_getblocktxn_response(compact_block, peer, expected_result):
 451             msg = msg_cmpctblock(compact_block.to_p2p())
 452             peer.send_and_ping(msg)
 453             with mininode_lock:
 454                 assert("getblocktxn" in peer.last_message)
 455                 absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute()
 456             assert_equal(absolute_indexes, expected_result)
 457
 458         def test_tip_after_message(node, peer, msg, tip):
 459             peer.send_and_ping(msg)
 460             assert_equal(int(node.getbestblockhash(), 16), tip)
 461
 462         # First try announcing compactblocks that won't reconstruct, and verify
 463         # that we receive getblocktxn messages back.
 464         utxo = self.utxos.pop(0)
 465
 466         block = self.build_block_with_transactions(node, utxo, 5)
 467         self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
 468         comp_block = HeaderAndShortIDs()
 469         comp_block.initialize_from_block(block, use_witness=with_witness)
 470
 471         test_getblocktxn_response(comp_block, test_node, [1, 2, 3, 4, 5])
 472
 473         msg_bt = msg_blocktxn()
 474         if with_witness:
 475             msg_bt = msg_witness_blocktxn() # serialize with witnesses
 476         msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[1:])
 477         test_tip_after_message(node, test_node, msg_bt, block.sha256)
 478
 479         utxo = self.utxos.pop(0)
 480         block = self.build_block_with_transactions(node, utxo, 5)
 481         self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
 482
 483         # Now try interspersing the prefilled transactions
 484         comp_block.initialize_from_block(block, prefill_list=[0, 1, 5], use_witness=with_witness)
 485         test_getblocktxn_response(comp_block, test_node, [2, 3, 4])
 486         msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5])
 487         test_tip_after_message(node, test_node, msg_bt, block.sha256)
 488
 489         # Now try giving one transaction ahead of time.
 490         utxo = self.utxos.pop(0)
 491         block = self.build_block_with_transactions(node, utxo, 5)
 492         self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
 493         test_node.send_and_ping(msg_tx(block.vtx[1]))
 494         assert(block.vtx[1].hash in node.getrawmempool())
 495
 496         # Prefill 4 out of the 6 transactions, and verify that only the one
 497         # that was not in the mempool is requested.
 498         comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4], use_witness=with_witness)
 499         test_getblocktxn_response(comp_block, test_node, [5])
 500
 501         msg_bt.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]])
 502         test_tip_after_message(node, test_node, msg_bt, block.sha256)
 503
 504         # Now provide all transactions to the node before the block is
 505         # announced and verify reconstruction happens immediately.
 506         utxo = self.utxos.pop(0)
 507         block = self.build_block_with_transactions(node, utxo, 10)
 508         self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
 509         for tx in block.vtx[1:]:
 510             test_node.send_message(msg_tx(tx))
 511         test_node.sync_with_ping()
 512         # Make sure all transactions were accepted.
 513         mempool = node.getrawmempool()
 514         for tx in block.vtx[1:]:
 515             assert(tx.hash in mempool)
 516
 517         # Clear out last request.
 518         with mininode_lock:
 519             test_node.last_message.pop("getblocktxn", None)
 520
 521         # Send compact block
 522         comp_block.initialize_from_block(block, prefill_list=[0], use_witness=with_witness)
 523         test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256)
 524         with mininode_lock:
 525             # Shouldn't have gotten a request for any transaction
 526             assert("getblocktxn" not in test_node.last_message)
 527
 528     # Incorrectly responding to a getblocktxn shouldn't cause the block to be
 529     # permanently failed.
 530     def test_incorrect_blocktxn_response(self, node, test_node, version):
 531         if (len(self.utxos) == 0):
 532             self.make_utxos()
 533         utxo = self.utxos.pop(0)
 534
 535         block = self.build_block_with_transactions(node, utxo, 10)
 536         self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
 537         # Relay the first 5 transactions from the block in advance
 538         for tx in block.vtx[1:6]:
 539             test_node.send_message(msg_tx(tx))
 540         test_node.sync_with_ping()
 541         # Make sure all transactions were accepted.
 542         mempool = node.getrawmempool()
 543         for tx in block.vtx[1:6]:
 544             assert(tx.hash in mempool)
 545
 546         # Send compact block
 547         comp_block = HeaderAndShortIDs()
 548         comp_block.initialize_from_block(block, prefill_list=[0], use_witness=(version == 2))
 549         test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
 550         absolute_indexes = []
 551         with mininode_lock:
 552             assert("getblocktxn" in test_node.last_message)
 553             absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute()
 554         assert_equal(absolute_indexes, [6, 7, 8, 9, 10])
 555
 556         # Now give an incorrect response.
 557         # Note that it's possible for bitcoind to be smart enough to know we're
 558         # lying, since it could check to see if the shortid matches what we're
 559         # sending, and eg disconnect us for misbehavior.  If that behavior
 560         # change were made, we could just modify this test by having a
 561         # different peer provide the block further down, so that we're still
 562         # verifying that the block isn't marked bad permanently. This is good
 563         # enough for now.
 564         msg = msg_blocktxn()
 565         if version==2:
 566             msg = msg_witness_blocktxn()
 567         msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:])
 568         test_node.send_and_ping(msg)
 569
 570         # Tip should not have updated
 571         assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock)
 572
 573         # We should receive a getdata request
 574         success = wait_until(lambda: "getdata" in test_node.last_message, timeout=10)
 575         assert(success)
 576         assert_equal(len(test_node.last_message["getdata"].inv), 1)
 577         assert(test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2|MSG_WITNESS_FLAG)
 578         assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256)
 579
 580         # Deliver the block
 581         if version==2:
 582             test_node.send_and_ping(msg_witness_block(block))
 583         else:
 584             test_node.send_and_ping(msg_block(block))
 585         assert_equal(int(node.getbestblockhash(), 16), block.sha256)
 586
 587     def test_getblocktxn_handler(self, node, test_node, version):
 588         # bitcoind will not send blocktxn responses for blocks whose height is
 589         # more than 10 blocks deep.
 590         MAX_GETBLOCKTXN_DEPTH = 10
 591         chain_height = node.getblockcount()
 592         current_height = chain_height
 593         while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH):
 594             block_hash = node.getblockhash(current_height)
 595             block = FromHex(CBlock(), node.getblock(block_hash, False))
 596
 597             msg = msg_getblocktxn()
 598             msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [])
 599             num_to_request = random.randint(1, len(block.vtx))
 600             msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request)))
 601             test_node.send_message(msg)
 602             success = wait_until(lambda: "blocktxn" in test_node.last_message, timeout=10)
 603             assert(success)
 604
 605             [tx.calc_sha256() for tx in block.vtx]
 606             with mininode_lock:
 607                 assert_equal(test_node.last_message["blocktxn"].block_transactions.blockhash, int(block_hash, 16))
 608                 all_indices = msg.block_txn_request.to_absolute()
 609                 for index in all_indices:
 610                     tx = test_node.last_message["blocktxn"].block_transactions.transactions.pop(0)
 611                     tx.calc_sha256()
 612                     assert_equal(tx.sha256, block.vtx[index].sha256)
 613                     if version == 1:
 614                         # Witnesses should have been stripped
 615                         assert(tx.wit.is_null())
 616                     else:
 617                         # Check that the witness matches
 618                         assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True))
 619                 test_node.last_message.pop("blocktxn", None)
 620             current_height -= 1
 621
 622         # Next request should send a full block response, as we're past the
 623         # allowed depth for a blocktxn response.
 624         block_hash = node.getblockhash(current_height)
 625         msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0])
 626         with mininode_lock:
 627             test_node.last_message.pop("block", None)
 628             test_node.last_message.pop("blocktxn", None)
 629         test_node.send_and_ping(msg)
 630         with mininode_lock:
 631             test_node.last_message["block"].block.calc_sha256()
 632             assert_equal(test_node.last_message["block"].block.sha256, int(block_hash, 16))
 633             assert "blocktxn" not in test_node.last_message
 634
 635     def test_compactblocks_not_at_tip(self, node, test_node):
 636         # Test that requesting old compactblocks doesn't work.
 637         MAX_CMPCTBLOCK_DEPTH = 5
 638         new_blocks = []
 639         for i in range(MAX_CMPCTBLOCK_DEPTH + 1):
 640             test_node.clear_block_announcement()
 641             new_blocks.append(node.generate(1)[0])
 642             wait_until(test_node.received_block_announcement, timeout=30)
 643
 644         test_node.clear_block_announcement()
 645         test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))]))
 646         success = wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30)
 647         assert(success)
 648
 649         test_node.clear_block_announcement()
 650         node.generate(1)
 651         wait_until(test_node.received_block_announcement, timeout=30)
 652         test_node.clear_block_announcement()
 653         with mininode_lock:
 654             test_node.last_message.pop("block", None)
 655         test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))]))
 656         success = wait_until(lambda: "block" in test_node.last_message, timeout=30)
 657         assert(success)
 658         with mininode_lock:
 659             test_node.last_message["block"].block.calc_sha256()
 660             assert_equal(test_node.last_message["block"].block.sha256, int(new_blocks[0], 16))
 661
 662         # Generate an old compactblock, and verify that it's not accepted.
 663         cur_height = node.getblockcount()
 664         hashPrevBlock = int(node.getblockhash(cur_height-5), 16)
 665         block = self.build_block_on_tip(node)
 666         block.hashPrevBlock = hashPrevBlock
 667         block.solve()
 668
 669         comp_block = HeaderAndShortIDs()
 670         comp_block.initialize_from_block(block)
 671         test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
 672
 673         tips = node.getchaintips()
 674         found = False
 675         for x in tips:
 676             if x["hash"] == block.hash:
 677                 assert_equal(x["status"], "headers-only")
 678                 found = True
 679                 break
 680         assert(found)
 681
 682         # Requesting this block via getblocktxn should silently fail
 683         # (to avoid fingerprinting attacks).
 684         msg = msg_getblocktxn()
 685         msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0])
 686         with mininode_lock:
 687             test_node.last_message.pop("blocktxn", None)
 688         test_node.send_and_ping(msg)
 689         with mininode_lock:
 690             assert "blocktxn" not in test_node.last_message
 691
 692     def activate_segwit(self, node):
 693         node.generate(144*3)
 694         assert_equal(get_bip9_status(node, "segwit")["status"], 'active')
 695
 696     def test_end_to_end_block_relay(self, node, listeners):
 697         utxo = self.utxos.pop(0)
 698
 699         block = self.build_block_with_transactions(node, utxo, 10)
 700
 701         [l.clear_block_announcement() for l in listeners]
 702
 703         # ToHex() won't serialize with witness, but this block has no witnesses
 704         # anyway. TODO: repeat this test with witness tx's to a segwit node.
 705         node.submitblock(ToHex(block))
 706
 707         for l in listeners:
 708             wait_until(lambda: l.received_block_announcement(), timeout=30)
 709         with mininode_lock:
 710             for l in listeners:
 711                 assert "cmpctblock" in l.last_message
 712                 l.last_message["cmpctblock"].header_and_shortids.header.calc_sha256()
 713                 assert_equal(l.last_message["cmpctblock"].header_and_shortids.header.sha256, block.sha256)
 714
 715     # Test that we don't get disconnected if we relay a compact block with valid header,
 716     # but invalid transactions.
 717     def test_invalid_tx_in_compactblock(self, node, test_node, use_segwit):
 718         assert(len(self.utxos))
 719         utxo = self.utxos[0]
 720
 721         block = self.build_block_with_transactions(node, utxo, 5)
 722         del block.vtx[3]
 723         block.hashMerkleRoot = block.calc_merkle_root()
 724         if use_segwit:
 725             # If we're testing with segwit, also drop the coinbase witness,
 726             # but include the witness commitment.
 727             add_witness_commitment(block)
 728             block.vtx[0].wit.vtxinwit = []
 729         block.solve()
 730
 731         # Now send the compact block with all transactions prefilled, and
 732         # verify that we don't get disconnected.
 733         comp_block = HeaderAndShortIDs()
 734         comp_block.initialize_from_block(block, prefill_list=[0, 1, 2, 3, 4], use_witness=use_segwit)
 735         msg = msg_cmpctblock(comp_block.to_p2p())
 736         test_node.send_and_ping(msg)
 737
 738         # Check that the tip didn't advance
 739         assert(int(node.getbestblockhash(), 16) is not block.sha256)
 740         test_node.sync_with_ping()
 741
 742     # Helper for enabling cb announcements
 743     # Send the sendcmpct request and sync headers
 744     def request_cb_announcements(self, peer, node, version):
 745         tip = node.getbestblockhash()
 746         peer.get_headers(locator=[int(tip, 16)], hashstop=0)
 747
 748         msg = msg_sendcmpct()
 749         msg.version = version
 750         msg.announce = True
 751         peer.send_and_ping(msg)
 752
 753     def test_compactblock_reconstruction_multiple_peers(self, node, stalling_peer, delivery_peer):
 754         assert(len(self.utxos))
 755
 756         def announce_cmpct_block(node, peer):
 757             utxo = self.utxos.pop(0)
 758             block = self.build_block_with_transactions(node, utxo, 5)
 759
 760             cmpct_block = HeaderAndShortIDs()
 761             cmpct_block.initialize_from_block(block)
 762             msg = msg_cmpctblock(cmpct_block.to_p2p())
 763             peer.send_and_ping(msg)
 764             with mininode_lock:
 765                 assert "getblocktxn" in peer.last_message
 766             return block, cmpct_block
 767
 768         block, cmpct_block = announce_cmpct_block(node, stalling_peer)
 769
 770         for tx in block.vtx[1:]:
 771             delivery_peer.send_message(msg_tx(tx))
 772         delivery_peer.sync_with_ping()
 773         mempool = node.getrawmempool()
 774         for tx in block.vtx[1:]:
 775             assert(tx.hash in mempool)
 776
 777         delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p()))
 778         assert_equal(int(node.getbestblockhash(), 16), block.sha256)
 779
 780         self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
 781
 782         # Now test that delivering an invalid compact block won't break relay
 783
 784         block, cmpct_block = announce_cmpct_block(node, stalling_peer)
 785         for tx in block.vtx[1:]:
 786             delivery_peer.send_message(msg_tx(tx))
 787         delivery_peer.sync_with_ping()
 788
 789         cmpct_block.prefilled_txn[0].tx.wit.vtxinwit = [ CTxInWitness() ]
 790         cmpct_block.prefilled_txn[0].tx.wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(0)]
 791
 792         cmpct_block.use_witness = True
 793         delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p()))
 794         assert(int(node.getbestblockhash(), 16) != block.sha256)
 795
 796         msg = msg_blocktxn()
 797         msg.block_transactions.blockhash = block.sha256
 798         msg.block_transactions.transactions = block.vtx[1:]
 799         stalling_peer.send_and_ping(msg)
 800         assert_equal(int(node.getbestblockhash(), 16), block.sha256)
 801
 802     def run_test(self):
 803         # Setup the p2p connections and start up the network thread.
 804         self.test_node = TestNode()
 805         self.segwit_node = TestNode()
 806         self.old_node = TestNode()  # version 1 peer <--> segwit node
 807
 808         connections = []
 809         connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node))
 810         connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1],
 811                     self.segwit_node, services=NODE_NETWORK|NODE_WITNESS))
 812         connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1],
 813                     self.old_node, services=NODE_NETWORK))
 814         self.test_node.add_connection(connections[0])
 815         self.segwit_node.add_connection(connections[1])
 816         self.old_node.add_connection(connections[2])
 817
 818         NetworkThread().start()  # Start up network handling in another thread
 819
 820         # Test logic begins here
 821         self.test_node.wait_for_verack()
 822
 823         # We will need UTXOs to construct transactions in later tests.
 824         self.make_utxos()
 825
 826         self.log.info("Running tests, pre-segwit activation:")
 827
 828         self.log.info("Testing SENDCMPCT p2p message... ")
 829         self.test_sendcmpct(self.nodes[0], self.test_node, 1)
 830         sync_blocks(self.nodes)
 831         self.test_sendcmpct(self.nodes[1], self.segwit_node, 2, old_node=self.old_node)
 832         sync_blocks(self.nodes)
 833
 834         self.log.info("Testing compactblock construction...")
 835         self.test_compactblock_construction(self.nodes[0], self.test_node, 1, False)
 836         sync_blocks(self.nodes)
 837         self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, False)
 838         sync_blocks(self.nodes)
 839
 840         self.log.info("Testing compactblock requests... ")
 841         self.test_compactblock_requests(self.nodes[0], self.test_node, 1, False)
 842         sync_blocks(self.nodes)
 843         self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, False)
 844         sync_blocks(self.nodes)
 845
 846         self.log.info("Testing getblocktxn requests...")
 847         self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1)
 848         sync_blocks(self.nodes)
 849         self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2)
 850         sync_blocks(self.nodes)
 851
 852         self.log.info("Testing getblocktxn handler...")
 853         self.test_getblocktxn_handler(self.nodes[0], self.test_node, 1)
 854         sync_blocks(self.nodes)
 855         self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2)
 856         self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1)
 857         sync_blocks(self.nodes)
 858
 859         self.log.info("Testing compactblock requests/announcements not at chain tip...")
 860         self.test_compactblocks_not_at_tip(self.nodes[0], self.test_node)
 861         sync_blocks(self.nodes)
 862         self.test_compactblocks_not_at_tip(self.nodes[1], self.segwit_node)
 863         self.test_compactblocks_not_at_tip(self.nodes[1], self.old_node)
 864         sync_blocks(self.nodes)
 865
 866         self.log.info("Testing handling of incorrect blocktxn responses...")
 867         self.test_incorrect_blocktxn_response(self.nodes[0], self.test_node, 1)
 868         sync_blocks(self.nodes)
 869         self.test_incorrect_blocktxn_response(self.nodes[1], self.segwit_node, 2)
 870         sync_blocks(self.nodes)
 871
 872         # End-to-end block relay tests
 873         self.log.info("Testing end-to-end block relay...")
 874         self.request_cb_announcements(self.test_node, self.nodes[0], 1)
 875         self.request_cb_announcements(self.old_node, self.nodes[1], 1)
 876         self.request_cb_announcements(self.segwit_node, self.nodes[1], 2)
 877         self.test_end_to_end_block_relay(self.nodes[0], [self.segwit_node, self.test_node, self.old_node])
 878         self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node])
 879
 880         self.log.info("Testing handling of invalid compact blocks...")
 881         self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False)
 882         self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, False)
 883         self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, False)
 884
 885         self.log.info("Testing reconstructing compact blocks from all peers...")
 886         self.test_compactblock_reconstruction_multiple_peers(self.nodes[1], self.segwit_node, self.old_node)
 887         sync_blocks(self.nodes)
 888
 889         # Advance to segwit activation
 890         self.log.info("Advancing to segwit activation")
 891         self.activate_segwit(self.nodes[1])
 892         self.log.info("Running tests, post-segwit activation...")
 893
 894         self.log.info("Testing compactblock construction...")
 895         self.test_compactblock_construction(self.nodes[1], self.old_node, 1, True)
 896         self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, True)
 897         sync_blocks(self.nodes)
 898
 899         self.log.info("Testing compactblock requests (unupgraded node)... ")
 900         self.test_compactblock_requests(self.nodes[0], self.test_node, 1, True)
 901
 902         self.log.info("Testing getblocktxn requests (unupgraded node)...")
 903         self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1)
 904
 905         # Need to manually sync node0 and node1, because post-segwit activation,
 906         # node1 will not download blocks from node0.
 907         self.log.info("Syncing nodes...")
 908         assert(self.nodes[0].getbestblockhash() != self.nodes[1].getbestblockhash())
 909         while (self.nodes[0].getblockcount() > self.nodes[1].getblockcount()):
 910             block_hash = self.nodes[0].getblockhash(self.nodes[1].getblockcount()+1)
 911             self.nodes[1].submitblock(self.nodes[0].getblock(block_hash, False))
 912         assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash())
 913
 914         self.log.info("Testing compactblock requests (segwit node)... ")
 915         self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, True)
 916
 917         self.log.info("Testing getblocktxn requests (segwit node)...")
 918         self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2)
 919         sync_blocks(self.nodes)
 920
 921         self.log.info("Testing getblocktxn handler (segwit node should return witnesses)...")
 922         self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2)
 923         self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1)
 924
 925         # Test that if we submitblock to node1, we'll get a compact block
 926         # announcement to all peers.
 927         # (Post-segwit activation, blocks won't propagate from node0 to node1
 928         # automatically, so don't bother testing a block announced to node0.)
 929         self.log.info("Testing end-to-end block relay...")
 930         self.request_cb_announcements(self.test_node, self.nodes[0], 1)
 931         self.request_cb_announcements(self.old_node, self.nodes[1], 1)
 932         self.request_cb_announcements(self.segwit_node, self.nodes[1], 2)
 933         self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node])
 934
 935         self.log.info("Testing handling of invalid compact blocks...")
 936         self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False)
 937         self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, True)
 938         self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, True)
 939
 940         self.log.info("Testing invalid index in cmpctblock message...")
 941         self.test_invalid_cmpctblock_message()
 942
 943
 944 if __name__ == '__main__':
 945     CompactBlocksTest().main()